###############################################################
###############################################################
#########
######### different_goldstandards_yeast.R
######### author: Samir Rachid Zaim
######### date: 10/18/2018
#########
######### input: cell line MCF7 data
######### output:
###############################################################
###############################################################
rm(list=ls())
FDR.cutoff=.05

require(data.table)
require(precrec)
require(ggplot2)
source('~/Dropbox/Lab-Tools-Files/R-Lab-Tools/RNA-Seq Tools/sourceDir.R')
sourceDir('~/Dropbox/Lab-Tools-Files/R-Lab-Tools/RNA-Seq Tools/iDEG')
sourceDir('~/Dropbox/Lab-Tools-Files/R-Lab-Tools/RNA-Seq Tools')

yeast.data <- fread('~/Dropbox/TBC-2018-single-subject-methods-accuracies/data/full_combined_dataset.csv', data.table = F)
rownames(yeast.data) <- yeast.data$GeneSymbol
yeast.data <- yeast.data[, !names(yeast.data) %in% 'GeneSymbol']

#### generate cohort matrix
mutant_rep_names <- colnames(yeast.data)[grep('Snf2',colnames(yeast.data))]
wt_rep_names <- colnames(yeast.data)[grep('WT',colnames(yeast.data))]

#### randomly sample 4 isogenic pairs to use as the gold-standard
#### and use the remaining 3 for training
set.seed(1111)
gs.pairs <- c(sample(mutant_rep_names, 30), sample(wt_rep_names, 30))
tst.pairs<-  colnames(yeast.data)[ !  colnames(yeast.data) %in% gs.pairs]

## find samples using string matching
gs.mat <- yeast.data[, names(yeast.data) %in% gs.pairs]
ts.mat <- yeast.data[, !names(yeast.data) %in% gs.pairs]


###############################################################
###############################################################
####
#### edgeR goldstandard

auc.mat = data.frame(modnames = character(), 
                     dsids=numeric(),
                     curvetypes=character(),
                     aucs=numeric())

#### using individual samples
#### to form n-of-1 predictions

## edgeR as Gold standard
DEG_gs.edgeR <- DEG_edgeR(countTable = gs.mat,
                          conditions =  c(rep('normal',30), rep('tumor',30)),
                          replicateType = 'genetically identical model organisms',
                          disper = .1)

y_true <- p.adjust(DEG_gs.edgeR[[1]]$PValue, method='BY') < FDR.cutoff

for(i in 1:12){
  cat(paste('\n\nrunning analysis for sample',i,'\n\n'))
  
  ## NOISeq
  NOISeq.preds <- DEG_noiseq(ts.mat[, c(i,i+12)], conditions =c('normal','tumor'))
  NOISeq.preds <- NOISeq.preds$prob
  NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
  
  ## DESeq
  conditions = c('normal','tumor')
  countTable = ts.mat[,c(i,i+12)]; colnames(countTable) <-  conditions
  DESeq.preds  <- DEG_DESeq(countTable, conditions = conditions )
  DESeq.preds  <-DESeq.preds$padj
  DESeq.preds[which(is.na(DESeq.preds))] <- 1
  
  
  ## DESeq2
  DESeq2.preds <- DEG_DESeq2(countTable, conditions= conditions)
  DESeq2.preds <- DESeq2.preds$padj
  DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
  
  ## DEGseq
  DEGseq.preds <- DEG_DEGseq(ts.mat[, c(i,i+12)], FDR_threshold= .1)
  DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
  DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
  DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
  
  ## iDEG
  baseline = ts.mat[,i]; names(baseline) <- row.names(ts.mat)
  case     = ts.mat[,i+12]; names(case)     <- row.names(ts.mat)
  
  iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
  iDEG.preds <- iDEG.preds[[1]]$local_fdr
  
  ## mixEnrich
  mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
  mixEnrich.preds  <- mixEnrich.preds$posterior_prob_RP
  
  # res= data.frame(iDEG=1-iDEG.preds,
  #                 DEGseq=1-DEGseq.preds,
  #                 NOISeq=NOISeq.preds,
  #                 edgeR_gs= y_true)
  
  
  sscurves = evalmod(scores = list(1-iDEG.preds, 1-DEGseq.preds, NOISeq.preds,
                                   1-DESeq.preds, 1-DESeq2.preds, mixEnrich.preds, rowMeans(cbind(1-DEGseq.preds, NOISeq.preds,
                                                                                                  mixEnrich.preds))),
                     labels= y_true,
                     modnames=c('iDEG','DEGseq','NOISeq','DESeq','DESeq2', 'Mixture Model','Ensemble'))
  
  
  
  pdf(paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast/edgeR_gs.sample',i,'_prc.pdf',sep=''),
      onefile = F)
  autoplot(sscurves)
  dev.off()
  
  auc.mat = rbind(auc.mat, auc(sscurves))
  
  # 
  # pdf(paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast/edgeR_gs.sample',i,'_roc.pdf',sep=''),
  #     width = 11)
  # autoplot(sscurves, 'ROC')
  # dev.off()
  # 
}
# 
# #### Using all 3 samples
# #### to form cohort predictions
# 
# 
# ## NOISeq
# cohort.conditions <- c(rep(c('normal','tumor'),3))
# 
# countTable = ts.mat
# NOISeq.cb.preds <- DEG_noiseq.cb(ts.mat, conditions =cohort.conditions)
# NOISeq.preds <- NOISeq.cb.preds$prob
# NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
# 
# ## DESeq
# DESeq.preds  <- DEG_DESeq.cb(ts.mat, conditions = cohort.conditions )
# DESeq.preds  <- DESeq.preds$padj
# DESeq.preds[which(is.na(DESeq.preds))] <- 1
# 
# 
# ## DESeq2
# DESeq2.preds <- DEG_DESeq2.cb(ts.mat, conditions= cohort.conditions)
# DESeq2.preds <- DESeq2.preds$padj
# DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
# 
# ## DEGseq
# DEGseq.preds <- DEG_DEGseq(ts.mat, FDR_threshold= .1)
# DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
# DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
# DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
# 
# ## iDEG
# baseline = ts.mat[,1]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,2]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds1  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds1 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,3]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,4]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds2  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds2 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,5]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,6]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds3  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds3 <- iDEG.preds[[1]]$local_fdr
# 
# pred.mat <- data.table(cbind(iDEG.preds1, iDEG.preds2, iDEG.preds3))
# iDEG.preds <- apply(pred.mat, 1 , mean)
# 
# pred.mat <- data.table(cbind(mixEnrich.preds1, mixEnrich.preds2, mixEnrich.preds3))
# mixEnrich.preds  <-  apply(pred.mat, 1 , mean)
# 
# # 
# # res= data.frame(iDEG=1-iDEG.preds,
# #                 DEGseq=1-DEGseq.preds,
# #                 NOISeq=NOISeq.preds,
# #                 edgeR_gs= y_true,
# #                 DEseq = DESeq.preds,
# #                 DESeq2= DESeq2.preds)
# 
# 
# sscurves = evalmod(scores = list(1-iDEG.preds, 1-DEGseq.preds, NOISeq.preds,
#                                  1-DESeq.preds, 1-DESeq2.preds, mixEnrich.preds, rowMeans(cbind(1-DEGseq.preds, NOISeq.preds,
#                                                                                                 mixEnrich.preds))),
#                    labels= y_true,
#                    modnames=c('iDEG','DEGseq','NOISeq','DESeq','DESeq2', 'Mixture Model','Ensemble'))
# 
# pdf('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast//edgeR_gs.full.pdf',
#     width = 11)
# autoplot(sscurves)
# #title('Gold Standard: EdgeR\nFull Sample')
# dev.off()

####
####
###############################################################
###############################################################



###############################################################
###############################################################
####
#### DESeq  goldstandard

## DESeq as Gold Standard
gs.cohort.conditions =   c(rep('normal',30), rep('tumor',30))

DESeq.preds  <- DEG_DESeq.cb(gs.mat, conditions =gs.cohort.conditions )
DESeq.preds <- DESeq.preds$padj
DESeq.preds[which(is.na(DESeq.preds))] <- 1
y_true <- DESeq.preds < FDR.cutoff

for(i in 1:12){
  
  ## NOISeq
  NOISeq.preds <- DEG_noiseq(ts.mat[, c(i,i+12)], conditions =c('normal','tumor'))
  NOISeq.preds <- NOISeq.preds$prob
  NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
  
  ## edgeR
  edgeR.preds <- DEG_edgeR(countTable = ts.mat[,c(i, i+12)],
                           conditions = c('normal','tumor'),
                           replicateType = 'genetically identical model organisms',
                           disper = .1)
  
  edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
  
  ##
  conditions = c('normal','tumor')
  countTable = ts.mat[,c(i,i+12)]; colnames(countTable) <-  conditions
  
  ## DESeq2
  DESeq2.preds <- DEG_DESeq2(countTable, conditions= conditions)
  DESeq2.preds <- DESeq2.preds$padj
  DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
  
  ## DEGseq
  DEGseq.preds <- DEG_DEGseq(ts.mat[, c(i,i+12)], FDR_threshold= .1)
  DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
  DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
  DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
  
  
  ## iDEG
  baseline = ts.mat[,i]; names(baseline) <- row.names(ts.mat)
  case     = ts.mat[,i+12]; names(case)     <- row.names(ts.mat)
  
  iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
  iDEG.preds <- iDEG.preds[[1]]$local_fdr
  
  ## mixEnrich
  mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
  mixEnrich.preds  <- mixEnrich.preds$posterior_prob_RP
  
  #     res= data.frame(iDEG=1-iDEG.preds,
  #                     DEGseq=1-DEGseq.preds,
  #                     NOISeq=NOISeq.preds,
  #                     edgeR_gs= y_true)
  
  
  sscurves = evalmod(scores = list(1-iDEG.preds, 1-DEGseq.preds, NOISeq.preds,
                                   1-edgeR.preds, 1-DESeq2.preds, mixEnrich.preds, 
                                   rowMeans(cbind( NOISeq.preds,
                                                   1-edgeR.preds, mixEnrich.preds))),
                     labels= y_true,
                     modnames=c('iDEG','DEGseq','NOISeq','edgeR','DESeq2', 'Mixture Model','Ensemble'))
  
  
  pdf(paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast/deseq_gs.sample',i,'_prc.pdf',sep=''),
      onefile = F)
  autoplot(sscurves)
  #title(paste('Gold Standard: EdgeR\nSample ',i,sep=''))
  dev.off()
  
  # pdf(paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast/deseq_gs.sample',i,'_roc.pdf',sep=''),
  #     width = 11)
  # autoplot(sscurves, 'ROC')
  # #title(paste('Gold Standard: EdgeR\nSample ',i,sep=''))
  # dev.off()
  auc.mat = rbind(auc.mat, auc(sscurves))
  
  
}

# #### Using all 3 samples
# #### to form cohort predictions
# cohort.conditions <- c(rep(c('normal','tumor'),3))
# 
# edgeR.preds <- DEG_edgeR(countTable = ts.mat,
#                          conditions = cohort.conditions,
#                          replicateType = 'genetically identical model organisms',
#                          disper = .1)
# 
# edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
# 
# ## NOISeq
# countTable = ts.mat
# NOISeq.cb.preds <- DEG_noiseq.cb(ts.mat, conditions =cohort.conditions)
# NOISeq.preds <- NOISeq.cb.preds$prob
# NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
# 
# ## DESeq2
# DESeq2.preds <- DEG_DESeq2.cb(ts.mat, conditions= cohort.conditions)
# DESeq2.preds <- DESeq2.preds$padj
# DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
# 
# ## DEGseq
# DEGseq.preds <- DEG_DEGseq(ts.mat, FDR_threshold= .1)
# DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
# DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
# DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
# 
# 
# ## iDEG
# baseline = ts.mat[,1]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,2]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds1  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds1 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,3]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,4]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds2  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds2 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,5]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,6]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds3  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds3 <- iDEG.preds[[1]]$local_fdr
# 
# pred.mat <- data.table(cbind(iDEG.preds1, iDEG.preds2, iDEG.preds3))
# iDEG.preds <- apply(pred.mat, 1 , mean)
# 
# pred.mat <- data.table(cbind(mixEnrich.preds1, mixEnrich.preds2, mixEnrich.preds3))
# mixEnrich.preds  <-  apply(pred.mat, 1 , mean)
# 
# ## remove NA's
# idx =which( !is.na(iDEG.preds) & !is.na(DEGseq.preds) & !is.na(NOISeq.preds) &
#               !is.na(edgeR.preds)& !is.na(DESeq2.preds) & !is.na(mixEnrich.preds) & !is.na(y_true))
# # res= data.frame(iDEG=1-iDEG.preds,
# #                 DEGseq=1-DEGseq.preds,
# #                 NOISeq=NOISeq.preds,
# #                 edgeR_gs= y_true,
# #                 DEseq = DESeq.preds,
# #                 DESeq2= DESeq2.preds)
# 
# 
# 
# sscurves = evalmod(scores = list(1-iDEG.preds, 1-DEGseq.preds, NOISeq.preds,
#                                  1-edgeR.preds, 1-DESeq2.preds, mixEnrich.preds, 
#                                  rowMeans(cbind( NOISeq.preds,
#                                                  1-edgeR.preds, mixEnrich.preds))),
#                    labels= y_true,
#                    modnames=c('iDEG','DEGseq','NOISeq','edgeR','DESeq2', 'Mixture Model','Ensemble'))
# 
# pdf('~/Dropbox/iDEG-2018/clinical validation/results/improved reference standard/deseq_gs.full.pdf',
#     width = 11)
# autoplot(sscurves)
# #title('Gold Standard: DESeq\nFull Sample')
# dev.off()

####
####
###############################################################
###############################################################

###############################################################
###############################################################
####
#### DESeq2  goldstandard

## DESeq2 as Gold Standard
gs.cohort.conditions =   c(rep('normal',30), rep('tumor',30))

DESeq2.preds  <- DEG_DESeq2.cb(gs.mat, conditions =gs.cohort.conditions )
DESeq2.preds <- DESeq2.preds$padj
DESeq2.preds[which(is.na(DESeq2.preds))] <- 1

y_true <- DESeq2.preds < FDR.cutoff

for(i in 1:12){
  
  ## NOISeq
  NOISeq.preds <- DEG_noiseq(ts.mat[, c(i,i+12)], conditions =c('normal','tumor'))
  NOISeq.preds <- NOISeq.preds$prob
  NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
  
  ## edgeR
  edgeR.preds <- DEG_edgeR(countTable = ts.mat[,c(i, i+12)],
                           conditions = c('normal','tumor'),
                           replicateType = 'genetically identical model organisms',
                           disper = .1)
  
  edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
  
  ##
  conditions = c('normal','tumor')
  countTable = ts.mat[,c(i,i+12)]; colnames(countTable) <-  conditions
  
  ## DESeq2
  DESeq.preds <- DEG_DESeq(countTable, conditions= conditions)
  DESeq.preds <- DESeq.preds$padj
  DESeq.preds[which(is.na(DESeq.preds))] <- 1
  
  ## DEGseq
  DEGseq.preds <- DEG_DEGseq(ts.mat[, c(i,i+12)], FDR_threshold= .1)
  DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
  DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
  DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
  
  
  ## iDEG
  baseline = ts.mat[,i]; names(baseline) <- row.names(ts.mat)
  case     = ts.mat[,i+12]; names(case)     <- row.names(ts.mat)
  
  iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
  iDEG.preds <- iDEG.preds[[1]]$local_fdr
  
  ## mixEnrich
  mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
  mixEnrich.preds  <- mixEnrich.preds$posterior_prob_RP
  
  #
  
  #     res= data.frame(iDEG=1-iDEG.preds,
  #                     DEGseq=1-DEGseq.preds,
  #                     NOISeq=NOISeq.preds,
  #                     edgeR_gs= y_true)
  
  
  sscurves = evalmod(scores = list(1-iDEG.preds, 1-DEGseq.preds, NOISeq.preds,
                                   1-edgeR.preds, 1-DESeq.preds, mixEnrich.preds,
                                   rowMeans(cbind( NOISeq.preds,
                                                   1-edgeR.preds, mixEnrich.preds))),
                     labels= y_true,
                     modnames=c('iDEG','DEGseq','NOISeq','edgeR','DESeq', 'Mixture Models','Ensemble'))
  
  
  pdf(paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast/deseq2_gs.sample',i,'_prc.pdf',sep=''),
      onefile = F) 
  autoplot(sscurves)
  #title(paste('Gold Standard: EdgeR\nSample ',i,sep=''))
  dev.off()
  
  # pdf(paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast/deseq2_gs.sample',i,'_roc.pdf',sep=''),
  #     width = 11)
  # autoplot(sscurves, 'ROC')
  # #title(paste('Gold Standard: EdgeR\nSample ',i,sep=''))
  # dev.off()
  auc.mat = rbind(auc.mat, auc(sscurves))
  
  
}

# #### Using all 3 samples
# #### to form cohort predictions
# cohort.conditions <- c(rep(c('normal','tumor'),3))
# 
# edgeR.preds <- DEG_edgeR(countTable = ts.mat,
#                          conditions = cohort.conditions,
#                          replicateType = 'genetically identical model organisms',
#                          disper = .1)
# 
# edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
# 
# ## NOISeq
# countTable = ts.mat
# NOISeq.cb.preds <- DEG_noiseq.cb(ts.mat, conditions =cohort.conditions)
# NOISeq.preds <- NOISeq.cb.preds$prob
# NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
# 
# ## DESeq2
# DESeq.preds <- DEG_DESeq.cb(ts.mat, conditions= cohort.conditions)
# DESeq.preds <- DESeq.preds$padj
# DESeq.preds[which(is.na(DESeq.preds))] <- 1
# 
# ## DEGseq
# DEGseq.preds <- DEG_DEGseq(ts.mat, FDR_threshold= .1)
# DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
# DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
# DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
# 
# 
# ## iDEG
# baseline = ts.mat[,1]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,2]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds1  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds1 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,3]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,4]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds2  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds2 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,5]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,6]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds3  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds3 <- iDEG.preds[[1]]$local_fdr
# 
# pred.mat <- data.table(cbind(iDEG.preds1, iDEG.preds2, iDEG.preds3))
# iDEG.preds <- apply(pred.mat, 1 , mean)
# 
# pred.mat <- data.table(cbind(mixEnrich.preds1, mixEnrich.preds2, mixEnrich.preds3))
# mixEnrich.preds  <-  apply(pred.mat, 1 , mean)
# 
# ## remove NA's
# idx =which( !is.na(iDEG.preds) & !is.na(DEGseq.preds) & !is.na(NOISeq.preds) &
#               !is.na(edgeR.preds)& !is.na(DESeq.preds) & !is.na(mixEnrich.preds) & !is.na(y_true))
# 
# # res= data.frame(iDEG=1-iDEG.preds,
# #                 DEGseq=1-DEGseq.preds,
# #                 NOISeq=NOISeq.preds,
# #                 edgeR_gs= y_true,
# #                 DEseq = DESeq.preds,
# #                 DESeq2= DESeq2.preds)
# 
# 
# sscurves = evalmod(scores = list(1-iDEG.preds, 1-DEGseq.preds, NOISeq.preds,
#                                  1-edgeR.preds, 1-DESeq.preds, mixEnrich.preds,
#                                  rowMeans(cbind(NOISeq.preds,
#                                                 1-edgeR.preds, mixEnrich.preds))),
#                    labels= y_true,
#                    modnames=c('iDEG','DEGseq','NOISeq','edgeR','DESeq', 'Mixture Models','Ensemble'))
# 
# pdf('~/Dropbox/iDEG-2018/clinical validation/results/improved reference standard/deseq2_gs.full.pdf',
#     width = 11)
# autoplot(sscurves)
# #title('Gold Standard: DESeq2\nFull Sample')
# dev.off()

####
####
###############################################################
###############################################################

###############################################################
###############################################################
####
#### DEGseq  goldstandard
## DESeq as Gold Standard
gs.cohort.conditions =   c(rep('normal',30), rep('tumor',30))

## DEGseq
DEGseq.preds <- DEG_DEGseq.cb(gs.mat[,1:30],gs.mat[,31:60], FDR_threshold= .1)
DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
y_true <- DEGseq.preds < FDR.cutoff #quantile(DEGseq.preds, .15) = 3.56227e-12

for(i in 1:12){
  
  ## NOISeq
  NOISeq.preds <- DEG_noiseq(ts.mat[, c(i,i+12)], conditions =c('normal','tumor'))
  NOISeq.preds <- NOISeq.preds$prob
  NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
  
  ## edgeR
  edgeR.preds <- DEG_edgeR(countTable = ts.mat[,c(i, i+12)],
                           conditions = c('normal','tumor'),
                           replicateType = 'genetically identical model organisms',
                           disper = .1)
  
  edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
  
  ##
  conditions = c('normal','tumor')
  countTable = ts.mat[,c(i,i+12)]; colnames(countTable) <-  conditions
  
  ## DESeq2
  DESeq2.preds <- DEG_DESeq2(countTable, conditions= conditions)
  DESeq2.preds <- DESeq2.preds$padj
  DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
  
  ## DESeq
  DESeq.preds <- DEG_DESeq(countTable, conditions= conditions)
  DESeq.preds <- DESeq.preds$padj
  DESeq.preds[which(is.na(DESeq.preds))] <- 1
  
  
  ## DEGseq
  DEGseq.preds <- DEG_DEGseq(ts.mat[, c(i,i+12)], FDR_threshold= .1)
  DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
  DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
  DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
  
  
  ## iDEG
  baseline = ts.mat[,i]; names(baseline) <- row.names(ts.mat)
  case     = ts.mat[,i+12]; names(case)     <- row.names(ts.mat)
  
  iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
  iDEG.preds <- iDEG.preds[[1]]$local_fdr
  
  ## mixEnrich
  mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
  mixEnrich.preds  <- mixEnrich.preds$posterior_prob_RP
  
  ## remove NA's
  idx =which( !is.na(iDEG.preds) & !is.na(DESeq.preds) & !is.na(NOISeq.preds) &
                !is.na(edgeR.preds)& !is.na(DESeq2.preds) & !is.na(mixEnrich.preds) & !is.na(y_true))
  
  # res= data.frame(iDEG=1-iDEG.preds,
  #                 DEGseq=1-DEGseq.preds,
  #                 NOISeq=NOISeq.preds,
  #                 edgeR_gs= y_true)
  
  
  sscurves = evalmod(scores = list(1-iDEG.preds, 1-DESeq.preds, NOISeq.preds,
                                   1-edgeR.preds, 1-DESeq2.preds, mixEnrich.preds,
                                   rowMeans(cbind( NOISeq.preds,
                                                   1-edgeR.preds, mixEnrich.preds))),
                     labels= y_true,
                     modnames=c('iDEG','DESeq','NOISeq','edgeR','DESeq2', 'Mixture Models','Ensemble'))
  
  
  pdf(paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast/degseq_gs.sample',i,'_prc.pdf',sep=''),
      onefile = F)
  autoplot(sscurves)
  #title(paste('Gold Standard: EdgeR\nSample ',i,sep=''))
  dev.off()
  
  # pdf(paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast/degseq_gs.sample',i,'_roc.pdf',sep=''),
  #     width = 11)
  # autoplot(sscurves, 'ROC')
  # #title(paste('Gold Standard: EdgeR\nSample ',i,sep=''))
  # dev.off()
  auc.mat = rbind(auc.mat, auc(sscurves))
  
}

# #### Using all 3 samples
# #### to form cohort predictions
# cohort.conditions <- c(rep(c('normal','tumor'),3))
# 
# edgeR.preds <- DEG_edgeR(countTable = ts.mat,
#                          conditions = cohort.conditions,
#                          replicateType = 'genetically identical model organisms',
#                          disper = .1)
# 
# edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
# 
# ## NOISeq
# countTable = ts.mat
# NOISeq.cb.preds <- DEG_noiseq.cb(ts.mat, conditions =cohort.conditions)
# NOISeq.preds <- NOISeq.cb.preds$prob
# NOISeq.preds[which(is.na(NOISeq.preds))] <- 0 
# 
# ## DESeq2
# DESeq2.preds <- DEG_DESeq2.cb(ts.mat, conditions= cohort.conditions)
# DESeq2.preds <- DESeq2.preds$padj
# DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
# 
# ## DESeq
# DESeq.preds <- DEG_DESeq.cb(ts.mat, conditions= cohort.conditions)
# DESeq.preds <- DESeq.preds$padj
# DESeq.preds[which(is.na(DESeq.preds))] <- 1
# 
# ## DEGseq
# DEGseq.preds <- DEG_DEGseq(ts.mat, FDR_threshold= .1)
# DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
# DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
# DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
# 
# 
# ## iDEG
# baseline = ts.mat[,1]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,2]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds1  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds1 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,3]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,4]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds2  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds2 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,5]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,6]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds3  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds3 <- iDEG.preds[[1]]$local_fdr
# 
# pred.mat <- data.table(cbind(iDEG.preds1, iDEG.preds2, iDEG.preds3))
# iDEG.preds <- apply(pred.mat, 1 , mean)
# 
# pred.mat <- data.table(cbind(mixEnrich.preds1, mixEnrich.preds2, mixEnrich.preds3))
# mixEnrich.preds  <-  apply(pred.mat, 1 , mean)
# 
# ## remove NA's
# idx =which( !is.na(iDEG.preds) & !is.na(DESeq.preds) & !is.na(DEGseq.preds) &
#               !is.na(edgeR.preds)& !is.na(DESeq2.preds) & !is.na(mixEnrich.preds) & !is.na(y_true))
# 
# 
# # res= data.frame(iDEG=1-iDEG.preds,
# #                 DEGseq=1-DEGseq.preds,
# #                 NOISeq=NOISeq.preds,
# #                 edgeR_gs= y_true,
# #                 DEseq = DESeq.preds,
# #                 DESeq2= DESeq2.preds)
# 
# 
# sscurves = evalmod(scores = list(1-iDEG.preds, 1-DESeq.preds, NOISeq.preds,
#                                  1-edgeR.preds, 1-DESeq2.preds, mixEnrich.preds,
#                                  rowMeans(cbind( NOISeq.preds,
#                                                  1-edgeR.preds, mixEnrich.preds))),
#                    labels= y_true,
#                    modnames=c('iDEG','DESeq','NOISeq','edgeR','DESeq2', 'Mixture Models','Ensemble'))
# 
# 
# pdf('~/Dropbox/iDEG-2018/clinical validation/results/improved reference standard/degseq_gs.full.pdf',
#     width = 11)
# autoplot(sscurves)
# #title('Gold Standard: DEGseq\nFull Sample')
# dev.off()

####
####
###############################################################
###############################################################

###############################################################
###############################################################
####
#### NOISeq  goldstandard

countTable = ts.mat
## DESeq as Gold Standard
gs.cohort.conditions =   c(rep('normal',30), rep('tumor',30))

## NOISeq
NOISeq.cb.preds <- DEG_noiseq.cb(gs.mat, conditions =gs.cohort.conditions)
NOISeq.preds <- NOISeq.cb.preds$prob
NOISeq.preds <-  p.adjust(1-NOISeq.preds, 'BY') #(transform to p-values)
NOISeq.preds[which(is.na(NOISeq.preds))] <- 1 
y_true <- NOISeq.preds < FDR.cutoff # 0.07828687 - quantile(NOISeq.preds, 15)

for(i in 1:12){
  
  ## edgeR
  edgeR.preds <- DEG_edgeR(countTable = ts.mat[,c(i, i+12)],
                           conditions = c('normal','tumor'),
                           replicateType = 'genetically identical model organisms',
                           disper = .1)
  
  edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
  
  ##
  conditions = c('normal','tumor')
  countTable = ts.mat[,c(i,i+12)]; colnames(countTable) <-  conditions
  
  ## DESeq2
  DESeq2.preds <- DEG_DESeq2(countTable, conditions= conditions)
  DESeq2.preds <- DESeq2.preds$padj
  DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
  
  ## DESeq
  DESeq.preds <- DEG_DESeq(countTable, conditions= conditions)
  DESeq.preds <- DESeq.preds$padj
  DESeq.preds[which(is.na(DESeq.preds))] <- 1
  
  
  ## DEGseq
  DEGseq.preds <- DEG_DEGseq(ts.mat[, c(i,i+12)], FDR_threshold= .1)
  DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
  DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
  DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
  
  ## iDEG
  baseline = ts.mat[,i]; names(baseline) <- row.names(ts.mat)
  case     = ts.mat[,i+12]; names(case)     <- row.names(ts.mat)
  
  iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
  iDEG.preds <- iDEG.preds[[1]]$local_fdr
  
  ## mixEnrich
  mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
  mixEnrich.preds  <- mixEnrich.preds$posterior_prob_RP
  
  # res= data.frame(iDEG=1-iDEG.preds,
  #                 DEGseq=1-DEGseq.preds,
  #                 NOISeq=NOISeq.preds,
  #                 edgeR_gs= y_true)
  # 
  
  sscurves = evalmod(scores = list(1-iDEG.preds, 1-DESeq.preds, 1-DEGseq.preds,
                                   1-edgeR.preds, 1-DESeq2.preds, mixEnrich.preds,
                                   rowMeans(cbind(1-DEGseq.preds,
                                                  1-edgeR.preds, mixEnrich.preds))),
                     labels= y_true,
                     modnames=c('iDEG','DESeq','DEGseq','edgeR','DESeq2', 'Mixture Models','Ensemble'))
  
  
  pdf(paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast/noiseq_gs.sample',i,'_prc.pdf',sep=''),
      onefile = F)
  autoplot(sscurves)
  #title(paste('Gold Standard: EdgeR\nSample ',i,sep=''))
  dev.off()
  
  # pdf(paste('~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast/noiseq_gs.sample',i,'_roc.pdf',sep=''),
  #     width = 11)
  # autoplot(sscurves, 'ROC')
  # #title(paste('Gold Standard: EdgeR\nSample ',i,sep=''))
  # dev.off()
  
  auc.mat = rbind(auc.mat, auc(sscurves))
  
  
}

# #### Using all 3 samples
# #### to form cohort predictions
# cohort.conditions <- c(rep(c('normal','tumor'),3))
# 
# edgeR.preds <- DEG_edgeR(countTable = ts.mat,
#                          conditions = cohort.conditions,
#                          replicateType = 'genetically identical model organisms',
#                          disper = .1)
# 
# edgeR.preds <- p.adjust(edgeR.preds[[1]]$PValue,'BY')
# 
# ## NOISeq
# countTable = ts.mat
# NOISeq.cb.preds <- DEG_noiseq.cb(ts.mat, conditions =cohort.conditions)
# NOISeq.preds <- NOISeq.cb.preds$prob
# 
# ## DESeq2
# DESeq2.preds <- DEG_DESeq2.cb(ts.mat, conditions= cohort.conditions)
# DESeq2.preds <- DESeq2.preds$padj
# DESeq2.preds[which(is.na(DESeq2.preds))] <- 1
# 
# ## DESeq
# DESeq.preds <- DEG_DESeq.cb(ts.mat, conditions= cohort.conditions)
# DESeq.preds <- DESeq.preds$padj
# DESeq.preds[which(is.na(DESeq.preds))] <- 1
# 
# ## DEGseq
# DEGseq.preds <- DEG_DEGseq(ts.mat, FDR_threshold= .1)
# DEGseq.preds <- DEGseq.preds[order(DEGseq.preds$GeneNames),]
# DEGseq.preds <- DEGseq.preds[,'q.value.Benjamini.et.al..1995.']
# DEGseq.preds[which(is.na(DEGseq.preds))] <- 1 
# 
# ## iDEG
# baseline = ts.mat[,1]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,2]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds1  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds1 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,3]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,4]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds2  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds2 <- iDEG.preds[[1]]$local_fdr
# 
# baseline = ts.mat[,5]; names(baseline) <- row.names(ts.mat)
# case     = ts.mat[,6]; names(case)     <- row.names(ts.mat)
# 
# mixEnrich.preds <- compute_nof1_MixEnrich_transcript(baseline, case, row.names(ts.mat) )
# mixEnrich.preds3  <- mixEnrich.preds$posterior_prob_RP
# 
# iDEG.preds  <-iDEG(baseline, case, normalization=F, dataDistribution='NB')
# iDEG.preds3 <- iDEG.preds[[1]]$local_fdr
# 
# pred.mat <- data.table(cbind(iDEG.preds1, iDEG.preds2, iDEG.preds3))
# iDEG.preds <- apply(pred.mat, 1 , mean)
# 
# pred.mat <- data.table(cbind(mixEnrich.preds1, mixEnrich.preds2, mixEnrich.preds3))
# mixEnrich.preds  <-  apply(pred.mat, 1 , mean)
# 
# ## remove NA's
# idx =which( !is.na(iDEG.preds) & !is.na(DESeq.preds) & !is.na(DEGseq.preds) &
#               !is.na(edgeR.preds)& !is.na(DESeq2.preds) & !is.na(mixEnrich.preds) & !is.na(y_true))
# 
# # res= data.frame(iDEG=1-iDEG.preds,
# #                 DEGseq=1-DEGseq.preds,
# #                 NOISeq=NOISeq.preds,
# #                 edgeR_gs= y_true,
# #                 DEseq = DESeq.preds,
# #                 DESeq2= DESeq2.preds)
# 
# 
# sscurves = evalmod(scores = list(1-iDEG.preds, 1-DESeq.preds, 1-DEGseq.preds,
#                                  1-edgeR.preds, 1-DESeq2.preds, mixEnrich.preds,
#                                  rowMeans(cbind( 1-DEGseq.preds,
#                                                  1-edgeR.preds,  mixEnrich.preds))),
#                    labels= y_true,
#                    modnames=c('iDEG','DESeq','DEGseq','edgeR','DESeq2', 'Mixture Models','Ensemble'))
# 
# pdf('~/Dropbox/iDEG-2018/clinical validation/results/improved reference standard/NOISeq_gs.full.pdf',
#     width = 11)
# autoplot(sscurves)
# #title('Gold Standard: NOISeq\nFull Sample')
# dev.off()
# 
# ####
# ####
# ###############################################################
# ###############################################################
# 
# 
# auc.mat = rbind(auc.mat, auc(sscurves))

write.csv(auc.mat, '~/Dropbox/TBC-2018-single-subject-methods-accuracies/Results/yeast/aucs.csv')
