###############################################################
###############################################################
#########
######### different_goldstandards_yeast.R
######### author: Samir Rachid Zaim
######### date: 10/18/2018
#########
######### input: cell line MCF7 data
######### output:
###############################################################
###############################################################
rm(list=ls())
fdr.cutoff <- FDR.cutoff <- .2

res_total <- data.frame(Precision = numeric(), 
                        Recall = numeric(),
                        DEGs_Predicted = numeric(),
                        True_TotalDEGs = numeric())

get_results <- function(preds, label, fdr.cutoff=0.05, goldstandard='edgeR'){
  preds.true <- (preds < fdr.cutoff)
  
  tp <- sum(preds.true & label)
  fp <- sum(preds.true & !label)
  fn <- sum(!preds.true & label)
  
  prec = tp / (tp + fp)
  rec  = tp / (tp + fn)
  
  res <- data.frame(Precision = prec,
                    Recall = rec,
                    DEGs_Predicted = sum(preds.true),
                    True_TotalDEGs = sum(label),
                    FalsePositives = fp,
                    TruePositives = tp,
                    FalseNegatives = fn,
                    GoldStandard = goldstandard
  )
  return(res)
}


require(data.table)
require(precrec)
require(ggplot2)
source('~/Dropbox/Lab-Tools-Files/R-Lab-Tools/RNA-Seq Tools/sourceDir.R')
sourceDir('~/Dropbox/Lab-Tools-Files/R-Lab-Tools/RNA-Seq Tools/iDEG')
sourceDir('~/Dropbox/Lab-Tools-Files/R-Lab-Tools/RNA-Seq Tools')

yeast.data <- fread('~/Dropbox/TBC-2018-single-subject-methods-accuracies/data/full_combined_dataset.csv', data.table = F)
rownames(yeast.data) <- yeast.data$GeneSymbol
yeast.data <- yeast.data[, !names(yeast.data) %in% 'GeneSymbol']

#### generate cohort matrix
mutant_rep_names <- colnames(yeast.data)[grep('Snf2',colnames(yeast.data))]
wt_rep_names <- colnames(yeast.data)[grep('WT',colnames(yeast.data))]

#### randomly sample 4 isogenic pairs to use as the gold-standard
#### and use the remaining 3 for training
set.seed(1111)
gs.pairs <- c(sample(mutant_rep_names, 30), sample(wt_rep_names, 30))
tst.pairs<-  colnames(yeast.data)[ !  colnames(yeast.data) %in% gs.pairs]

## find samples using string matching
gs.mat <- yeast.data[, names(yeast.data) %in% gs.pairs]
ts.mat <- yeast.data[, !names(yeast.data) %in% gs.pairs]


###############################################################
###############################################################
####
#### edgeR goldstandard


#### using individual samples
#### to form n-of-1 predictions

## edgeR as Gold standard
DEG_gs.edgeR <- DEG_edgeR(countTable = gs.mat,
                          conditions =  c(rep('normal',30), rep('tumor',30)),
                          replicateType = 'genetically identical model organisms',
                          disper = .1)

y_true <- p.adjust(DEG_gs.edgeR[[1]]$PValue, method='BY') < FDR.cutoff

for(i in 1:12){
  cat(paste('\n\nrunning analysis for sample',i,'\n\n'))
  
  ## NOISeq
  NOISeq.preds <- DEG_noiseq(ts.mat[, c(i,i+12)], conditions =c('normal','tumor'))
  NOISeq.preds <- NOISeq.preds$prob
  NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
  
  ## DESeq
  conditions = c('normal','tumor')
  countTable = ts.mat[,c(i,i+12)]; colnames(countTable) <-  conditions
  DESeq.preds  <- DEG_DESeq(countTable, conditions = conditions )
  DESeq.preds  <-DESeq.preds$padj
  DESeq.preds[which(is.na(DESeq.preds))] <- 1
  
  
  ## DESeq2
  DESeq2.preds <- DEG_DESeq2(countTable, conditions= conditions)
  DESeq2.preds <- DESeq2.preds$padj
  DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
  
  ## DEGseq
  DEGseq.preds <- DEG_DEGseq(ts.mat[, c(i,i+12)], FDR_threshold= .1)
  DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
  DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
  DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
  
  ## iDEG
  baseline = ts.mat[,i]; names(baseline) <- row.names(ts.mat)
  case     = ts.mat[,i+12]; names(case)     <- row.names(ts.mat)
  
  iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
  iDEG.preds <- iDEG.preds[[1]]$local_fdr
  
  ## mixEnrich
  mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
  mixEnrich.preds  <- mixEnrich.preds$posterior_prob_RP
  
  ensemble =  rowMeans(cbind(1-DEGseq.preds, NOISeq.preds,mixEnrich.preds))
  
  
  # res= data.frame(iDEG=1-iDEG.preds,
  #                 DEGseq=1-DEGseq.preds,
  #                 NOISeq=NOISeq.preds,
  #                 edgeR_gs= y_true)
  
  
  sscurves = evalmod(scores = list(1-iDEG.preds, 1-DEGseq.preds, NOISeq.preds,
                                   1-DESeq.preds, 1-DESeq2.preds, mixEnrich.preds, rowMeans(cbind(1-DEGseq.preds, NOISeq.preds,
                                                                                                  mixEnrich.preds))),
                     labels= y_true,
                     modnames=c('iDEG','DEGseq','NOISeq','DESeq','DESeq2', 'Mixture Model','Ensemble'))
  
  
  res = data.frame(rbind(do.call(rbind, 
                                 lapply(list(DEGseq.preds, 1-NOISeq.preds,DESeq.preds, DESeq2.preds, 1-mixEnrich.preds, 1-ensemble),
                                        function(x) get_results(x, y_true, fdr.cutoff = fdr.cutoff, goldstandard = 'edgeR'))),
                         get_results(iDEG.preds, label=y_true, fdr.cutoff = fdr.cutoff)))
  res$method = c('DEGseq','NOISeq','DESeq','DESeq2','mixEnrich', 'Ensemble','iDEG')
  
  res_total = rbind(res_total, res)
  
  # 
  # pdf(paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast/edgeR_gs.sample',i,'_roc.pdf',sep=''),
  #     width = 11)
  # autoplot(sscurves, 'ROC')
  # dev.off()
  # 
}
# 
# #### Using all 3 samples
# #### to form cohort predictions
# 
# 
# ## NOISeq
# cohort.conditions <- c(rep(c('normal','tumor'),3))
# 
# countTable = ts.mat
# NOISeq.cb.preds <- DEG_noiseq.cb(ts.mat, conditions =cohort.conditions)
# NOISeq.preds <- NOISeq.cb.preds$prob
# NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
# 
# ## DESeq
# DESeq.preds  <- DEG_DESeq.cb(ts.mat, conditions = cohort.conditions )
# DESeq.preds  <- DESeq.preds$padj
# DESeq.preds[which(is.na(DESeq.preds))] <- 1
# 
# 
# ## DESeq2
# DESeq2.preds <- DEG_DESeq2.cb(ts.mat, conditions= cohort.conditions)
# DESeq2.preds <- DESeq2.preds$padj
# DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
# 
# ## DEGseq
# DEGseq.preds <- DEG_DEGseq(ts.mat, FDR_threshold= .1)
# DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
# DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
# DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
# 
# ## iDEG
# baseline = ts.mat[,1]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,2]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds1  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds1 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,3]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,4]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds2  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds2 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,5]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,6]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds3  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds3 <- iDEG.preds[[1]]$local_fdr
# 
# pred.mat <- data.table(cbind(iDEG.preds1, iDEG.preds2, iDEG.preds3))
# iDEG.preds <- apply(pred.mat, 1 , mean)
# 
# pred.mat <- data.table(cbind(mixEnrich.preds1, mixEnrich.preds2, mixEnrich.preds3))
# mixEnrich.preds  <-  apply(pred.mat, 1 , mean)
# 
# # 
# # res= data.frame(iDEG=1-iDEG.preds,
# #                 DEGseq=1-DEGseq.preds,
# #                 NOISeq=NOISeq.preds,
# #                 edgeR_gs= y_true,
# #                 DEseq = DESeq.preds,
# #                 DESeq2= DESeq2.preds)
# 
# 
# sscurves = evalmod(scores = list(1-iDEG.preds, 1-DEGseq.preds, NOISeq.preds,
#                                  1-DESeq.preds, 1-DESeq2.preds, mixEnrich.preds, rowMeans(cbind(1-DEGseq.preds, NOISeq.preds,
#                                                                                                 mixEnrich.preds))),
#                    labels= y_true,
#                    modnames=c('iDEG','DEGseq','NOISeq','DESeq','DESeq2', 'Mixture Model','Ensemble'))
# 
# pdf('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast//edgeR_gs.full.pdf',
#     width = 11)
# autoplot(sscurves)
# #title('Gold Standard: EdgeR\nFull Sample')
# dev.off()

####
####
###############################################################
###############################################################



###############################################################
###############################################################
####
#### DESeq  goldstandard

## DESeq as Gold Standard
gs.cohort.conditions =   c(rep('normal',30), rep('tumor',30))

DESeq.preds  <- DEG_DESeq.cb(gs.mat, conditions =gs.cohort.conditions )
DESeq.preds <- DESeq.preds$padj
DESeq.preds[which(is.na(DESeq.preds))] <- 1
y_true <- DESeq.preds < FDR.cutoff

for(i in 1:12){
  
  ## NOISeq
  NOISeq.preds <- DEG_noiseq(ts.mat[, c(i,i+12)], conditions =c('normal','tumor'))
  NOISeq.preds <- NOISeq.preds$prob
  NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
  
  ## edgeR
  edgeR.preds <- DEG_edgeR(countTable = ts.mat[,c(i, i+12)],
                           conditions = c('normal','tumor'),
                           replicateType = 'genetically identical model organisms',
                           disper = .1)
  
  edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
  
  ##
  conditions = c('normal','tumor')
  countTable = ts.mat[,c(i,i+12)]; colnames(countTable) <-  conditions
  
  ## DESeq2
  DESeq2.preds <- DEG_DESeq2(countTable, conditions= conditions)
  DESeq2.preds <- DESeq2.preds$padj
  DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
  
  ## DEGseq
  DEGseq.preds <- DEG_DEGseq(ts.mat[, c(i,i+12)], FDR_threshold= .1)
  DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
  DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
  DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
  
  
  ## iDEG
  baseline = ts.mat[,i]; names(baseline) <- row.names(ts.mat)
  case     = ts.mat[,i+12]; names(case)     <- row.names(ts.mat)
  
  iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
  iDEG.preds <- iDEG.preds[[1]]$local_fdr
  
  ## mixEnrich
  mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
  mixEnrich.preds  <- mixEnrich.preds$posterior_prob_RP
  
  ensemble =  rowMeans(cbind(1-DEGseq.preds, NOISeq.preds,mixEnrich.preds))
  
  #     res= data.frame(iDEG=1-iDEG.preds,
  #                     DEGseq=1-DEGseq.preds,
  #                     NOISeq=NOISeq.preds,
  #                     edgeR_gs= y_true)
  
  
  sscurves = evalmod(scores = list(1-iDEG.preds, 1-DEGseq.preds, NOISeq.preds,
                                   1-edgeR.preds, 1-DESeq2.preds, mixEnrich.preds, 
                                   rowMeans(cbind( NOISeq.preds,
                                                   1-edgeR.preds, mixEnrich.preds))),
                     labels= y_true,
                     modnames=c('iDEG','DEGseq','NOISeq','edgeR','DESeq2', 'Mixture Model','Ensemble'))
  
  res = data.frame(rbind(do.call(rbind, 
                                 lapply(list(DEGseq.preds, 1-NOISeq.preds,edgeR.preds, DESeq2.preds, 1-mixEnrich.preds, 1-ensemble),
                                        function(x) get_results(x, y_true, fdr.cutoff = fdr.cutoff, goldstandard = 'DESeq'))),
                         get_results(iDEG.preds, label=y_true, fdr.cutoff = fdr.cutoff)))
  res$method = c('DEGseq','NOISeq','edgeR','DESeq2','mixEnrich','Ensemble','iDEG')
  
  
  res_total = rbind(res_total, res)
  
  # pdf(paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast/deseq_gs.sample',i,'_roc.pdf',sep=''),
  #     width = 11)
  # autoplot(sscurves, 'ROC')
  # #title(paste('Gold Standard: EdgeR\nSample ',i,sep=''))
  # dev.off()
  
}

# #### Using all 3 samples
# #### to form cohort predictions
# cohort.conditions <- c(rep(c('normal','tumor'),3))
# 
# edgeR.preds <- DEG_edgeR(countTable = ts.mat,
#                          conditions = cohort.conditions,
#                          replicateType = 'genetically identical model organisms',
#                          disper = .1)
# 
# edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
# 
# ## NOISeq
# countTable = ts.mat
# NOISeq.cb.preds <- DEG_noiseq.cb(ts.mat, conditions =cohort.conditions)
# NOISeq.preds <- NOISeq.cb.preds$prob
# NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
# 
# ## DESeq2
# DESeq2.preds <- DEG_DESeq2.cb(ts.mat, conditions= cohort.conditions)
# DESeq2.preds <- DESeq2.preds$padj
# DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
# 
# ## DEGseq
# DEGseq.preds <- DEG_DEGseq(ts.mat, FDR_threshold= .1)
# DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
# DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
# DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
# 
# 
# ## iDEG
# baseline = ts.mat[,1]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,2]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds1  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds1 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,3]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,4]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds2  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds2 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,5]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,6]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds3  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds3 <- iDEG.preds[[1]]$local_fdr
# 
# pred.mat <- data.table(cbind(iDEG.preds1, iDEG.preds2, iDEG.preds3))
# iDEG.preds <- apply(pred.mat, 1 , mean)
# 
# pred.mat <- data.table(cbind(mixEnrich.preds1, mixEnrich.preds2, mixEnrich.preds3))
# mixEnrich.preds  <-  apply(pred.mat, 1 , mean)
# 
# ## remove NA's
# idx =which( !is.na(iDEG.preds) & !is.na(DEGseq.preds) & !is.na(NOISeq.preds) &
#               !is.na(edgeR.preds)& !is.na(DESeq2.preds) & !is.na(mixEnrich.preds) & !is.na(y_true))
# # res= data.frame(iDEG=1-iDEG.preds,
# #                 DEGseq=1-DEGseq.preds,
# #                 NOISeq=NOISeq.preds,
# #                 edgeR_gs= y_true,
# #                 DEseq = DESeq.preds,
# #                 DESeq2= DESeq2.preds)
# 
# 
# 
# sscurves = evalmod(scores = list(1-iDEG.preds, 1-DEGseq.preds, NOISeq.preds,
#                                  1-edgeR.preds, 1-DESeq2.preds, mixEnrich.preds, 
#                                  rowMeans(cbind( NOISeq.preds,
#                                                  1-edgeR.preds, mixEnrich.preds))),
#                    labels= y_true,
#                    modnames=c('iDEG','DEGseq','NOISeq','edgeR','DESeq2', 'Mixture Model','Ensemble'))
# 
# pdf('~/Dropbox/iDEG-2018/clinical validation/results/improved reference standard/deseq_gs.full.pdf',
#     width = 11)
# autoplot(sscurves)
# #title('Gold Standard: DESeq\nFull Sample')
# dev.off()

####
####
###############################################################
###############################################################

###############################################################
###############################################################
####
#### DESeq2  goldstandard

## DESeq2 as Gold Standard
gs.cohort.conditions =   c(rep('normal',30), rep('tumor',30))

DESeq2.preds  <- DEG_DESeq2.cb(gs.mat, conditions =gs.cohort.conditions )
DESeq2.preds <- DESeq2.preds$padj
DESeq2.preds[which(is.na(DESeq2.preds))] <- 1

y_true <- DESeq2.preds < FDR.cutoff

for(i in 1:12){
  
  ## NOISeq
  NOISeq.preds <- DEG_noiseq(ts.mat[, c(i,i+12)], conditions =c('normal','tumor'))
  NOISeq.preds <- NOISeq.preds$prob
  NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
  
  ## edgeR
  edgeR.preds <- DEG_edgeR(countTable = ts.mat[,c(i, i+12)],
                           conditions = c('normal','tumor'),
                           replicateType = 'genetically identical model organisms',
                           disper = .1)
  
  edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
  
  ##
  conditions = c('normal','tumor')
  countTable = ts.mat[,c(i,i+12)]; colnames(countTable) <-  conditions
  
  ## DESeq2
  DESeq.preds <- DEG_DESeq(countTable, conditions= conditions)
  DESeq.preds <- DESeq.preds$padj
  DESeq.preds[which(is.na(DESeq.preds))] <- 1
  
  ## DEGseq
  DEGseq.preds <- DEG_DEGseq(ts.mat[, c(i,i+12)], FDR_threshold= .1)
  DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
  DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
  DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
  
  
  ## iDEG
  baseline = ts.mat[,i]; names(baseline) <- row.names(ts.mat)
  case     = ts.mat[,i+12]; names(case)     <- row.names(ts.mat)
  
  iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
  iDEG.preds <- iDEG.preds[[1]]$local_fdr
  
  ## mixEnrich
  mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
  mixEnrich.preds  <- mixEnrich.preds$posterior_prob_RP
  
  ensemble =  rowMeans(cbind(1-DEGseq.preds, NOISeq.preds,mixEnrich.preds))
  
  #
  
  #     res= data.frame(iDEG=1-iDEG.preds,
  #                     DEGseq=1-DEGseq.preds,
  #                     NOISeq=NOISeq.preds,
  #                     edgeR_gs= y_true)
  
  
  sscurves = evalmod(scores = list(1-iDEG.preds, 1-DEGseq.preds, NOISeq.preds,
                                   1-edgeR.preds, 1-DESeq.preds, mixEnrich.preds,
                                   rowMeans(cbind( NOISeq.preds,
                                                   1-edgeR.preds, mixEnrich.preds))),
                     labels= y_true,
                     modnames=c('iDEG','DEGseq','NOISeq','edgeR','DESeq', 'Mixture Models','Ensemble'))
  
  
  res = data.frame(rbind(do.call(rbind, 
                                 lapply(list(DEGseq.preds, 1-NOISeq.preds,edgeR.preds, DESeq.preds, 1-mixEnrich.preds, 1-ensemble),
                                        function(x) get_results(x, y_true, fdr.cutoff = fdr.cutoff, goldstandard = 'DESeq2'))),
                         get_results(iDEG.preds, label=y_true, fdr.cutoff = fdr.cutoff)))
  res$method = c('DEGseq','NOISeq','edgeR','DESeq','mixEnrich', 'Ensemble','iDEG')
  
  res_total = rbind(res_total, res)
  
  # pdf(paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast/deseq2_gs.sample',i,'_roc.pdf',sep=''),
  #     width = 11)
  # autoplot(sscurves, 'ROC')
  # #title(paste('Gold Standard: EdgeR\nSample ',i,sep=''))
  # dev.off()
  
}

# #### Using all 3 samples
# #### to form cohort predictions
# cohort.conditions <- c(rep(c('normal','tumor'),3))
# 
# edgeR.preds <- DEG_edgeR(countTable = ts.mat,
#                          conditions = cohort.conditions,
#                          replicateType = 'genetically identical model organisms',
#                          disper = .1)
# 
# edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
# 
# ## NOISeq
# countTable = ts.mat
# NOISeq.cb.preds <- DEG_noiseq.cb(ts.mat, conditions =cohort.conditions)
# NOISeq.preds <- NOISeq.cb.preds$prob
# NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
# 
# ## DESeq2
# DESeq.preds <- DEG_DESeq.cb(ts.mat, conditions= cohort.conditions)
# DESeq.preds <- DESeq.preds$padj
# DESeq.preds[which(is.na(DESeq.preds))] <- 1
# 
# ## DEGseq
# DEGseq.preds <- DEG_DEGseq(ts.mat, FDR_threshold= .1)
# DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
# DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
# DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
# 
# 
# ## iDEG
# baseline = ts.mat[,1]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,2]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds1  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds1 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,3]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,4]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds2  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds2 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,5]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,6]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds3  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds3 <- iDEG.preds[[1]]$local_fdr
# 
# pred.mat <- data.table(cbind(iDEG.preds1, iDEG.preds2, iDEG.preds3))
# iDEG.preds <- apply(pred.mat, 1 , mean)
# 
# pred.mat <- data.table(cbind(mixEnrich.preds1, mixEnrich.preds2, mixEnrich.preds3))
# mixEnrich.preds  <-  apply(pred.mat, 1 , mean)
# 
# ## remove NA's
# idx =which( !is.na(iDEG.preds) & !is.na(DEGseq.preds) & !is.na(NOISeq.preds) &
#               !is.na(edgeR.preds)& !is.na(DESeq.preds) & !is.na(mixEnrich.preds) & !is.na(y_true))
# 
# # res= data.frame(iDEG=1-iDEG.preds,
# #                 DEGseq=1-DEGseq.preds,
# #                 NOISeq=NOISeq.preds,
# #                 edgeR_gs= y_true,
# #                 DEseq = DESeq.preds,
# #                 DESeq2= DESeq2.preds)
# 
# 
# sscurves = evalmod(scores = list(1-iDEG.preds, 1-DEGseq.preds, NOISeq.preds,
#                                  1-edgeR.preds, 1-DESeq.preds, mixEnrich.preds,
#                                  rowMeans(cbind(NOISeq.preds,
#                                                 1-edgeR.preds, mixEnrich.preds))),
#                    labels= y_true,
#                    modnames=c('iDEG','DEGseq','NOISeq','edgeR','DESeq', 'Mixture Models','Ensemble'))
# 
# pdf('~/Dropbox/iDEG-2018/clinical validation/results/improved reference standard/deseq2_gs.full.pdf',
#     width = 11)
# autoplot(sscurves)
# #title('Gold Standard: DESeq2\nFull Sample')
# dev.off()

####
####
###############################################################
###############################################################

###############################################################
###############################################################
####
#### DEGseq  goldstandard
## DESeq as Gold Standard
gs.cohort.conditions =   c(rep('normal',30), rep('tumor',30))

## DEGseq
DEGseq.preds <- DEG_DEGseq.cb(gs.mat[,1:30],gs.mat[,31:60], FDR_threshold= .1)
DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
y_true <- DEGseq.preds < FDR.cutoff #quantile(DEGseq.preds, .15) = 3.56227e-12

for(i in 1:12){
  
  ## NOISeq
  NOISeq.preds <- DEG_noiseq(ts.mat[, c(i,i+12)], conditions =c('normal','tumor'))
  NOISeq.preds <- NOISeq.preds$prob
  NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
  
  ## edgeR
  edgeR.preds <- DEG_edgeR(countTable = ts.mat[,c(i, i+12)],
                           conditions = c('normal','tumor'),
                           replicateType = 'genetically identical model organisms',
                           disper = .1)
  
  edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
  
  ##
  conditions = c('normal','tumor')
  countTable = ts.mat[,c(i,i+12)]; colnames(countTable) <-  conditions
  
  ## DESeq2
  DESeq2.preds <- DEG_DESeq2(countTable, conditions= conditions)
  DESeq2.preds <- DESeq2.preds$padj
  DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
  
  ## DESeq
  DESeq.preds <- DEG_DESeq(countTable, conditions= conditions)
  DESeq.preds <- DESeq.preds$padj
  DESeq.preds[which(is.na(DESeq.preds))] <- 1
  
  
  ## DEGseq
  DEGseq.preds <- DEG_DEGseq(ts.mat[, c(i,i+12)], FDR_threshold= .1)
  DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
  DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
  DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
  
  
  ## iDEG
  baseline = ts.mat[,i]; names(baseline) <- row.names(ts.mat)
  case     = ts.mat[,i+12]; names(case)     <- row.names(ts.mat)
  
  iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
  iDEG.preds <- iDEG.preds[[1]]$local_fdr
  
  ## mixEnrich
  mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
  mixEnrich.preds  <- mixEnrich.preds$posterior_prob_RP
  
  ## remove NA's
  idx =which( !is.na(iDEG.preds) & !is.na(DESeq.preds) & !is.na(NOISeq.preds) &
                !is.na(edgeR.preds)& !is.na(DESeq2.preds) & !is.na(mixEnrich.preds) & !is.na(y_true))
  
  # res= data.frame(iDEG=1-iDEG.preds,
  #                 DEGseq=1-DEGseq.preds,
  #                 NOISeq=NOISeq.preds,
  #                 edgeR_gs= y_true)
  
  ensemble =  rowMeans(cbind(1-edgeR.preds, NOISeq.preds,mixEnrich.preds))
  
  
  sscurves = evalmod(scores = list(1-iDEG.preds, 1-DESeq.preds, NOISeq.preds,
                                   1-edgeR.preds, 1-DESeq2.preds, mixEnrich.preds,
                                   rowMeans(cbind( NOISeq.preds,
                                                   1-edgeR.preds, mixEnrich.preds))),
                     labels= y_true,
                     modnames=c('iDEG','DESeq','NOISeq','edgeR','DESeq2', 'Mixture Models','Ensemble'))
  
  
  res = data.frame(rbind(do.call(rbind, 
                                 lapply(list(DESeq.preds, 1-NOISeq.preds,edgeR.preds, DESeq2.preds, 1-mixEnrich.preds, 1-ensemble),
                                        function(x) get_results(x, y_true, fdr.cutoff = fdr.cutoff, goldstandard = 'DEGseq'))),
                         get_results(iDEG.preds, label=y_true, fdr.cutoff = fdr.cutoff)))
  res$method = c('DESeq','NOISeq','edgeR','DESeq2','mixEnrich','Ensemble','iDEG')
  
  
  res_total = rbind(res_total, res)
  
  # pdf(paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast/degseq_gs.sample',i,'_roc.pdf',sep=''),
  #     width = 11)
  # autoplot(sscurves, 'ROC')
  # #title(paste('Gold Standard: EdgeR\nSample ',i,sep=''))
  # dev.off()
}

# #### Using all 3 samples
# #### to form cohort predictions
# cohort.conditions <- c(rep(c('normal','tumor'),3))
# 
# edgeR.preds <- DEG_edgeR(countTable = ts.mat,
#                          conditions = cohort.conditions,
#                          replicateType = 'genetically identical model organisms',
#                          disper = .1)
# 
# edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
# 
# ## NOISeq
# countTable = ts.mat
# NOISeq.cb.preds <- DEG_noiseq.cb(ts.mat, conditions =cohort.conditions)
# NOISeq.preds <- NOISeq.cb.preds$prob
# NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
# 
# ## DESeq2
# DESeq2.preds <- DEG_DESeq2.cb(ts.mat, conditions= cohort.conditions)
# DESeq2.preds <- DESeq2.preds$padj
# DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
# 
# ## DESeq
# DESeq.preds <- DEG_DESeq.cb(ts.mat, conditions= cohort.conditions)
# DESeq.preds <- DESeq.preds$padj
# DESeq.preds[which(is.na(DESeq.preds))] <- 1
# 
# ## DEGseq
# DEGseq.preds <- DEG_DEGseq(ts.mat, FDR_threshold= .1)
# DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
# DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
# DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
# 
# 
# ## iDEG
# baseline = ts.mat[,1]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,2]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds1  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds1 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,3]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,4]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds2  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds2 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,5]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,6]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds3  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds3 <- iDEG.preds[[1]]$local_fdr
# 
# pred.mat <- data.table(cbind(iDEG.preds1, iDEG.preds2, iDEG.preds3))
# iDEG.preds <- apply(pred.mat, 1 , mean)
# 
# pred.mat <- data.table(cbind(mixEnrich.preds1, mixEnrich.preds2, mixEnrich.preds3))
# mixEnrich.preds  <-  apply(pred.mat, 1 , mean)
# 
# ## remove NA's
# idx =which( !is.na(iDEG.preds) & !is.na(DESeq.preds) & !is.na(DEGseq.preds) &
#               !is.na(edgeR.preds)& !is.na(DESeq2.preds) & !is.na(mixEnrich.preds) & !is.na(y_true))
# 
# 
# # res= data.frame(iDEG=1-iDEG.preds,
# #                 DEGseq=1-DEGseq.preds,
# #                 NOISeq=NOISeq.preds,
# #                 edgeR_gs= y_true,
# #                 DEseq = DESeq.preds,
# #                 DESeq2= DESeq2.preds)
# 
# 
# sscurves = evalmod(scores = list(1-iDEG.preds, 1-DESeq.preds, NOISeq.preds,
#                                  1-edgeR.preds, 1-DESeq2.preds, mixEnrich.preds,
#                                  rowMeans(cbind( NOISeq.preds,
#                                                  1-edgeR.preds, mixEnrich.preds))),
#                    labels= y_true,
#                    modnames=c('iDEG','DESeq','NOISeq','edgeR','DESeq2', 'Mixture Models','Ensemble'))
# 
# 
# pdf('~/Dropbox/iDEG-2018/clinical validation/results/improved reference standard/degseq_gs.full.pdf',
#     width = 11)
# autoplot(sscurves)
# #title('Gold Standard: DEGseq\nFull Sample')
# dev.off()

####
####
###############################################################
###############################################################

###############################################################
###############################################################
####
#### NOISeq  goldstandard

countTable = ts.mat
## DESeq as Gold Standard
gs.cohort.conditions =   c(rep('normal',30), rep('tumor',30))

## NOISeq
NOISeq.cb.preds <- DEG_noiseq.cb(gs.mat, conditions =gs.cohort.conditions)
NOISeq.preds <- NOISeq.cb.preds$prob
NOISeq.preds <-  p.adjust(1-NOISeq.preds, 'BY') #(transform to p-values)
NOISeq.preds[which(is.na(NOISeq.preds))] <- 1 
y_true <- NOISeq.preds < FDR.cutoff # 0.07828687 - quantile(NOISeq.preds, 15)

for(i in 1:12){
  
  ## edgeR
  edgeR.preds <- DEG_edgeR(countTable = ts.mat[,c(i, i+12)],
                           conditions = c('normal','tumor'),
                           replicateType = 'genetically identical model organisms',
                           disper = .1)
  
  edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
  
  ##
  conditions = c('normal','tumor')
  countTable = ts.mat[,c(i,i+12)]; colnames(countTable) <-  conditions
  
  ## DESeq2
  DESeq2.preds <- DEG_DESeq2(countTable, conditions= conditions)
  DESeq2.preds <- DESeq2.preds$padj
  DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
  
  ## DESeq
  DESeq.preds <- DEG_DESeq(countTable, conditions= conditions)
  DESeq.preds <- DESeq.preds$padj
  DESeq.preds[which(is.na(DESeq.preds))] <- 1
  
  
  ## DEGseq
  DEGseq.preds <- DEG_DEGseq(ts.mat[, c(i,i+12)], FDR_threshold= .1)
  DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
  DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
  DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
  
  ## iDEG
  baseline = ts.mat[,i]; names(baseline) <- row.names(ts.mat)
  case     = ts.mat[,i+12]; names(case)     <- row.names(ts.mat)
  
  iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
  iDEG.preds <- iDEG.preds[[1]]$local_fdr
  
  ## mixEnrich
  mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
  mixEnrich.preds  <- mixEnrich.preds$posterior_prob_RP
  
  ensemble =  rowMeans(cbind(1-DEGseq.preds, 1-edgeR.preds,mixEnrich.preds))
  
  # res= data.frame(iDEG=1-iDEG.preds,
  #                 DEGseq=1-DEGseq.preds,
  #                 NOISeq=NOISeq.preds,
  #                 edgeR_gs= y_true)
  # 
  
  sscurves = evalmod(scores = list(1-iDEG.preds, 1-DESeq.preds, 1-DEGseq.preds,
                                   1-edgeR.preds, 1-DESeq2.preds, mixEnrich.preds,
                                   rowMeans(cbind(1-DEGseq.preds,
                                                  1-edgeR.preds, mixEnrich.preds))),
                     labels= y_true,
                     modnames=c('iDEG','DESeq','DEGseq','edgeR','DESeq2', 'Mixture Models','Ensemble'))
  
  
  res = data.frame(rbind(do.call(rbind, 
                                 lapply(list(DEGseq.preds,DESeq.preds,edgeR.preds, DESeq2.preds, 1-mixEnrich.preds, 1-ensemble),
                                        function(x) get_results(x, y_true, fdr.cutoff = fdr.cutoff, goldstandard = 'NOISeq'))),
                         get_results(iDEG.preds, label=y_true, fdr.cutoff = fdr.cutoff)))
  res$method  = c('DEGseq','DESeq','edgeR','DESeq2','mixEnrich','Ensemble','iDEG')
  
  
  res_total = rbind(res_total, res)
  
}

write.csv(res_total, paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast/full_tabular_results', fdr.cutoff,'.csv', sep=''), row.names = F)

# require(ggplot2)
res_total = data.table(res_total)
# res_total$method = gsub('iDEG[1-3]+', 'iDEG', res_total$method)
# res_total$method = gsub('DESeq2[1-9]+', 'DESeq_2', res_total$method)
# 
means <- res_total[, list(Recall =mean(Recall, na.rm=T), Precision =mean(Precision, na.rm=T), DEGs_Predicted=round(mean(DEGs_Predicted, na.rm=T)),
                          FalsePositives=round(mean(FalsePositives,na.rm=T)), FalseNegatives=round(mean(FalsePositives))), by=method]
write.csv(means, paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast/summary_tabular_results', fdr.cutoff,'.csv', sep=''), row.names = F)
