###############################################################
###############################################################
#########
######### different_goldstandards.R
######### author: Samir Rachid Zaim
######### date: 10/18/2018
#########
######### input: cell line MCF7 data
######### output:
###############################################################
###############################################################
rm(list=ls())

for(fdr.cutoff in c(0.01,0.05, 0.1, 0.2)){
  
  
  FDR.cutoff <- fdr.cutoff 
  
  res_total <- data.frame(Precision = numeric(), 
                          Recall = numeric(),
                          DEGs_Predicted = numeric(),
                          True_TotalDEGs = numeric())
  
  get_results <- function(preds, label, fdr.cutoff=0.05, goldstandard='edgeR'){
    preds.true <- (preds < fdr.cutoff)
    
    tp <- sum(preds.true & label)
    fp <- sum(preds.true & !label)
    fn <- sum(!preds.true & label)
    
    prec = tp / (tp + fp)
    rec  = tp / (tp + fn)
    
    res <- data.frame(Precision = prec,
                      Recall = rec,
                      DEGs_Predicted = sum(preds.true),
                      True_TotalDEGs = sum(label),
                      FalsePositives = fp,
                      TruePositives = tp,
                      FalseNegatives = fn,
                      GoldStandard = goldstandard
    )
    return(res)
  }
  
  require(data.table)
  require(precrec)
  require(ggplot2)
  source('~/Dropbox/Lab-Tools-Files/R-Lab-Tools/RNA-Seq Tools/sourceDir.R')
  sourceDir('~/Dropbox/Lab-Tools-Files/R-Lab-Tools/RNA-Seq Tools/iDEG')
  sourceDir('~/Dropbox/Lab-Tools-Files/R-Lab-Tools/RNA-Seq Tools')
  
  p1 = fread("~/Dropbox/iDEG-2018/clinical validation/data/sample-562-574.csv")
  p2 = fread("~/Dropbox/iDEG-2018/clinical validation/data/sample-563-577.csv")
  p3 = fread("~/Dropbox/iDEG-2018/clinical validation/data/sample-564-572.csv")
  p4 = fread("~/Dropbox/iDEG-2018/clinical validation/data/sample-565-576.csv")
  p5 = fread("~/Dropbox/iDEG-2018/clinical validation/data/sample-566-570.csv")
  p6 = fread("~/Dropbox/iDEG-2018/clinical validation/data/sample-568-575.csv")
  p7 = fread("~/Dropbox/iDEG-2018/clinical validation/data/sample-569-571.csv")
  
  #### generate cohort matrix
  count.mat = cbind(p1,p2[,2:3,with=F],p3[,2:3,with=F],p4[,2:3,with=F],
                    p5[,2:3,with=F],p6[,2:3,with=F],p7[,2:3,with=F])
  pairs <- c('562-574','563-577','564-572','565-576','566-570','568-575','569-571')
  
  #### randomly sample 4 isogenic pairs to use as the gold-standard
  #### and use the remaining 3 for training
  set.seed(1111)
  gs.pairs <- sample(pairs, 4)
  tst.pairs<-  pairs[ !  pairs %in% gs.pairs]
  
  ## find samples using string matching
  idx.gs <- grep( paste(unlist( strsplit(gs.pairs, '-')),collapse='|') , colnames(count.mat))
  gs.mat <- as.data.frame(count.mat[, idx.gs, with=F]); row.names(gs.mat) <- p1$V1
  ts.mat <- as.data.frame(count.mat[, -c(1,idx.gs), with=F]); row.names(ts.mat) <- p1$V1
  
  #### determine goldstandard technique
  techniques = c('NOISeq','edgeR','DESeq','DESeq2','DEGseq','iDEG')
  gs.technique = 'edgeR'
  
  evaluators <- techniques[!techniques %in% gs.technique]
  
  
  ###############################################################
  ###############################################################
  ####
  #### edgeR goldstandard
  
  
  #### using individual samples
  #### to form n-of-1 predictions
  
  ## edgeR as Gold standard
  DEG_gs.edgeR <- DEG_edgeR(countTable = gs.mat,
                            conditions = rep(c('normal','tumor'),4),
                            replicateType = 'genetically identical model organisms',
                            disper = .1)
  
  y_true <- p.adjust(DEG_gs.edgeR[[1]]$PValue, method='BY') < .005
  
  for(i in 1:3){
    
    
    ## NOISeq
    NOISeq.preds <- DEG_noiseq(ts.mat[, c(i,i+1)], conditions =c('normal','tumor'))
    NOISeq.preds <- NOISeq.preds$prob
    NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
    
    ## DESeq
    conditions = c('normal','tumor')
    countTable = ts.mat[,c(i,i+1)]; colnames(countTable) <-  conditions
    DESeq.preds  <- DEG_DESeq(countTable, conditions = conditions )
    DESeq.preds  <-DESeq.preds$padj
    DESeq.preds[which(is.na(DESeq.preds))] <- 1
    
    
    ## DESeq2
    DESeq2.preds <- DEG_DESeq2(countTable, conditions= conditions)
    DESeq2.preds <- DESeq2.preds$padj
    DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
    
    ## DEGseq
    DEGseq.preds <- DEG_DEGseq(ts.mat[, c(i,i+1)], FDR_threshold= .1)
    DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
    DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
    DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
    
    ## iDEG
    baseline = ts.mat[,i]; names(baseline) <- row.names(ts.mat)
    case     = ts.mat[,i+1]; names(case)     <- row.names(ts.mat)
    
    iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
    iDEG.preds <- iDEG.preds[[1]]$local_fdr
    
    ## mixEnrich
    mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
    mixEnrich.preds  <- mixEnrich.preds$posterior_prob_RP
    
    ensemble =  rowMeans(cbind(1-DEGseq.preds, NOISeq.preds,mixEnrich.preds))
    
    # res= data.frame(iDEG=1-iDEG.preds,
    #                 DEGseq=1-DEGseq.preds,
    #                 NOISeq=NOISeq.preds,
    #                 edgeR_gs= y_true)
    
    
    res = data.frame(rbind(do.call(rbind, 
                                   lapply(list(DEGseq.preds, 1-NOISeq.preds,DESeq.preds, DESeq2.preds, 1-mixEnrich.preds, 1-ensemble),
                                          function(x) get_results(x, y_true, fdr.cutoff = fdr.cutoff, goldstandard = 'edgeR'))),
                           get_results(iDEG.preds, label=y_true, fdr.cutoff = fdr.cutoff)))
    res$method = c('DEGseq','NOISeq','DESeq','DESeq2','mixEnrich','Ensemble', 'iDEG')
    
    res_total = rbind(res_total, res)
    
  }
  
  
  ####
  ####
  ###############################################################
  ###############################################################
  
  
  
  ###############################################################
  ###############################################################
  ####
  #### DESeq  goldstandard
  
  ## DESeq as Gold Standard
  gs.cohort.conditions =   c(rep(c('normal','tumor'),4))
  
  DESeq.preds  <- DEG_DESeq.cb(gs.mat, conditions =gs.cohort.conditions )
  DESeq.preds <- DESeq.preds$padj
  DESeq.preds[which(is.na(DESeq.preds))] <- 1
  y_true <- DESeq.preds < .001
  
  for(i in 1:3){
    
    ## NOISeq
    NOISeq.preds <- DEG_noiseq(ts.mat[, c(i,i+1)], conditions =c('normal','tumor'))
    NOISeq.preds <- NOISeq.preds$prob
    NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
    
    ## edgeR
    edgeR.preds <- DEG_edgeR(countTable = ts.mat[,c(i, i+1)],
                             conditions = c('normal','tumor'),
                             replicateType = 'genetically identical model organisms',
                             disper = .1)
    
    edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
    
    ##
    conditions = c('normal','tumor')
    countTable = ts.mat[,c(i,i+1)]; colnames(countTable) <-  conditions
    
    ## DESeq2
    DESeq2.preds <- DEG_DESeq2(countTable, conditions= conditions)
    DESeq2.preds <- DESeq2.preds$padj
    DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
    
    ## DEGseq
    DEGseq.preds <- DEG_DEGseq(ts.mat[, c(i,i+1)], FDR_threshold= .1)
    DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
    DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
    DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
    
    
    ## iDEG
    baseline = ts.mat[,i]; names(baseline) <- row.names(ts.mat)
    case     = ts.mat[,i+1]; names(case)     <- row.names(ts.mat)
    
    iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
    iDEG.preds <- iDEG.preds[[1]]$local_fdr
    
    ## mixEnrich
    mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
    mixEnrich.preds  <- mixEnrich.preds$posterior_prob_RP
    
    ensemble =  rowMeans(cbind(1-DEGseq.preds, NOISeq.preds,mixEnrich.preds))
    
    res = data.frame(rbind(do.call(rbind, 
                                   lapply(list(DEGseq.preds, 1-NOISeq.preds,edgeR.preds, DESeq2.preds, 1-mixEnrich.preds, 1-ensemble),
                                          function(x) get_results(x, y_true, fdr.cutoff = fdr.cutoff, goldstandard = 'DESeq'))),
                           get_results(iDEG.preds, label=y_true, fdr.cutoff = fdr.cutoff)))
    res$method = c('DEGseq','NOISeq','edgeR','DESeq2','mixEnrich','Ensemble', 'iDEG')
    
    
    res_total = rbind(res_total, res)
    
    
  }
  
  
  ####
  ####
  ###############################################################
  ###############################################################
  
  ###############################################################
  ###############################################################
  ####
  #### DESeq2  goldstandard
  
  ## DESeq2 as Gold Standard
  gs.cohort.conditions =   c(rep(c('normal','tumor'),4))
  
  DESeq2.preds  <- DEG_DESeq2.cb(gs.mat, conditions =gs.cohort.conditions )
  DESeq2.preds <- DESeq2.preds$padj
  DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
  
  y_true <- DESeq2.preds < .0005
  
  for(i in 1:3){
    
    
    ## NOISeq
    NOISeq.preds <- DEG_noiseq(ts.mat[, c(i,i+1)], conditions =c('normal','tumor'))
    NOISeq.preds <- NOISeq.preds$prob
    NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
    
    ## edgeR
    edgeR.preds <- DEG_edgeR(countTable = ts.mat[,c(i, i+1)],
                             conditions = c('normal','tumor'),
                             replicateType = 'genetically identical model organisms',
                             disper = .1)
    
    edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
    
    ##
    conditions = c('normal','tumor')
    countTable = ts.mat[,c(i,i+1)]; colnames(countTable) <-  conditions
    
    ## DESeq2
    DESeq.preds <- DEG_DESeq(countTable, conditions= conditions)
    DESeq.preds <- DESeq.preds$padj
    DESeq.preds[which(is.na(DESeq.preds))] <- 1
    
    ## DEGseq
    DEGseq.preds <- DEG_DEGseq(ts.mat[, c(i,i+1)], FDR_threshold= .1)
    DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
    DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
    DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
    
    
    ## iDEG
    baseline = ts.mat[,i]; names(baseline) <- row.names(ts.mat)
    case     = ts.mat[,i+1]; names(case)     <- row.names(ts.mat)
    
    iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
    iDEG.preds <- iDEG.preds[[1]]$local_fdr
    
    ## mixEnrich
    mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
    mixEnrich.preds  <- mixEnrich.preds$posterior_prob_RP
    
    ensemble =  rowMeans(cbind(1-DEGseq.preds, NOISeq.preds,mixEnrich.preds))
    
    #
    
    #     res= data.frame(iDEG=1-iDEG.preds,
    #                     DEGseq=1-DEGseq.preds,
    #                     NOISeq=NOISeq.preds,
    #                     edgeR_gs= y_true)
    
    res = data.frame(rbind(do.call(rbind, 
                                   lapply(list(DEGseq.preds, 1-NOISeq.preds,edgeR.preds, DESeq.preds, 1-mixEnrich.preds, 1-ensemble),
                                          function(x) get_results(x, y_true, fdr.cutoff = fdr.cutoff, goldstandard = 'DESeq2'))),
                           get_results(iDEG.preds, label=y_true, fdr.cutoff = fdr.cutoff)))
    res$method = c('DEGseq','NOISeq','edgeR','DESeq','mixEnrich','Ensemble','iDEG')
    
    res_total = rbind(res_total, res)
    
  }
  
  ####
  ####
  ###############################################################
  ###############################################################
  
  ###############################################################
  ###############################################################
  ####
  #### DEGseq  goldstandard
  ####
  gs.cohort.conditions =   c(rep(c('normal','tumor'),4))
  
  ## DEGseq
  DEGseq.preds <- DEG_DEGseq.cb(gs.mat[,c(1,3,5,7)], gs.mat[,c(2,4,6,8)], FDR_threshold= .1)
  DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
  DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
  DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
  y_true <- DEGseq.preds < 3.56227e-12 #quantile(DEGseq.preds, .15) = 3.56227e-12
  
  for(i in 1:3){
    
    ## NOISeq
    NOISeq.preds <- DEG_noiseq(ts.mat[, c(i,i+1)], conditions =c('normal','tumor'))
    NOISeq.preds <- NOISeq.preds$prob
    NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
    
    ## edgeR
    edgeR.preds <- DEG_edgeR(countTable = ts.mat[,c(i, i+1)],
                             conditions = c('normal','tumor'),
                             replicateType = 'genetically identical model organisms',
                             disper = .1)
    
    edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
    
    ##
    conditions = c('normal','tumor')
    countTable = ts.mat[,c(i,i+1)]; colnames(countTable) <-  conditions
    
    ## DESeq2
    DESeq2.preds <- DEG_DESeq2(countTable, conditions= conditions)
    DESeq2.preds <- DESeq2.preds$padj
    DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
    
    ## DESeq
    DESeq.preds <- DEG_DESeq(countTable, conditions= conditions)
    DESeq.preds <- DESeq.preds$padj
    DESeq.preds[which(is.na(DESeq.preds))] <- 1
    
    
    ## DEGseq
    DEGseq.preds <- DEG_DEGseq(ts.mat[, c(i,i+1)], FDR_threshold= .1)
    DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
    DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
    DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
    
    
    ## iDEG
    baseline = ts.mat[,i]; names(baseline) <- row.names(ts.mat)
    case     = ts.mat[,i+1]; names(case)     <- row.names(ts.mat)
    
    iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
    iDEG.preds <- iDEG.preds[[1]]$local_fdr
    
    ## mixEnrich
    mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
    mixEnrich.preds  <- mixEnrich.preds$posterior_prob_RP
    
    ensemble =  rowMeans(cbind(1-edgeR.preds, NOISeq.preds,mixEnrich.preds))
    
    
    # res= data.frame(iDEG=1-iDEG.preds,
    #                 DEGseq=1-DEGseq.preds,
    #                 NOISeq=NOISeq.preds,
    #                 edgeR_gs= y_true)
    
    
    res = data.frame(rbind(do.call(rbind, 
                                   lapply(list(DESeq.preds, 1-NOISeq.preds,edgeR.preds, DESeq2.preds, 1-mixEnrich.preds, 1-ensemble),
                                          function(x) get_results(x, y_true, fdr.cutoff = fdr.cutoff, goldstandard = 'DEGseq'))),
                           get_results(iDEG.preds, label=y_true, fdr.cutoff = fdr.cutoff)))
    res$method = c('DESeq','NOISeq','edgeR','DESeq2','mixEnrich','Ensemble','iDEG')
    
    
    res_total = rbind(res_total, res)
  }
  
  ####
  ####
  ###############################################################
  ###############################################################
  
  ###############################################################
  ###############################################################
  ####
  #### NOISeq  goldstandard
  
  countTable = ts.mat
  ## DESeq as Gold Standard
  gs.cohort.conditions =   c(rep(c('normal','tumor'),4))
  
  ## NOISeq
  NOISeq.cb.preds <- DEG_noiseq.cb(gs.mat, conditions =gs.cohort.conditions)
  NOISeq.preds <- NOISeq.cb.preds$prob
  NOISeq.preds <-  p.adjust(1-NOISeq.preds, 'BY') #(transform to p-values)
  NOISeq.preds[which(is.na(NOISeq.preds))] <- 1 
  y_true <- NOISeq.preds < 0.07828687 # 0.07828687 - quantile(NOISeq.preds, 15)
  
  for(i in 1:3){
    
    ## edgeR
    edgeR.preds <- DEG_edgeR(countTable = ts.mat[,c(i, i+1)],
                             conditions = c('normal','tumor'),
                             replicateType = 'genetically identical model organisms',
                             disper = .1)
    
    edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
    
    ##
    conditions = c('normal','tumor')
    countTable = ts.mat[,c(i,i+1)]; colnames(countTable) <-  conditions
    
    ## DESeq2
    DESeq2.preds <- DEG_DESeq2(countTable, conditions= conditions)
    DESeq2.preds <- DESeq2.preds$padj
    DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
    
    ## DESeq
    DESeq.preds <- DEG_DESeq(countTable, conditions= conditions)
    DESeq.preds <- DESeq.preds$padj
    DESeq.preds[which(is.na(DESeq.preds))] <- 1
    
    
    ## DEGseq
    DEGseq.preds <- DEG_DEGseq(ts.mat[, c(i,i+1)], FDR_threshold= .1)
    DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
    DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
    DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
    
    ## iDEG
    baseline = ts.mat[,i]; names(baseline) <- row.names(ts.mat)
    case     = ts.mat[,i+1]; names(case)     <- row.names(ts.mat)
    
    iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
    iDEG.preds <- iDEG.preds[[1]]$local_fdr
    
    ## mixEnrich
    mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
    mixEnrich.preds  <- mixEnrich.preds$posterior_prob_RP
    
    ensemble =  rowMeans(cbind(1-DEGseq.preds, 1-edgeR.preds,mixEnrich.preds))
      
    res = data.frame(rbind(do.call(rbind, 
                                 lapply(list(DEGseq.preds,DESeq.preds,edgeR.preds, DESeq2.preds, 1-mixEnrich.preds, 1-ensemble),
                                        function(x) get_results(x, y_true, fdr.cutoff = fdr.cutoff, goldstandard = 'NOISeq'))),
                         get_results(iDEG.preds, label=y_true, fdr.cutoff = fdr.cutoff)))
    res$method  = c('DEGseq','DESeq','edgeR','DESeq2','mixEnrich','Ensemble','iDEG')
  
  
    res_total = rbind(res_total, res)
  
  }
  
  write.csv(res_total, paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/cell-line data/full_tabular_results', fdr.cutoff,'.csv', sep=''), row.names = F)
  
  # require(ggplot2)
  res_total = data.table(res_total)
  # res_total$method = gsub('iDEG[1-3]+', 'iDEG', res_total$method)
  # res_total$method = gsub('DESeq2[1-9]+', 'DESeq_2', res_total$method)
  # 
  means <- res_total[, list(Recall =mean(Recall, na.rm=T), Precision =mean(Precision, na.rm=T)), by=method]
  write.csv(means, paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/cell-line data/summary_tabular_results', fdr.cutoff,'.csv', sep=''), row.names = F)
}