---
title: "MetaSNV v2 Gene Content Report"
date: "`r Sys.Date()`"
output:
  html_document:
    code_folding: hide
    number_sections: yes
    smart: no
    toc: yes
    df_print: paged
    toc_float: no
params:
  corrPcutOff_default:
    value: 0.8
  corrScutOff_default:
    value: 0.6
  speciesID:
    value: null
  statCutoff:
    value: 0.05
  rStatCutoff:
    value: 0.5
  rStatCutoff_strong:
    value: 0.7
  minObs:
    value: 10
  subpopOutDir:
    value: ''
  bamSuffix:
    value: '.bam'
  geneFamilyType:
    value: 'Gene'
---

```{r setup, include=FALSE}

knitr::opts_chunk$set(
  echo=TRUE,
  comment=NA,
  warning=TRUE,
  error=FALSE, # keep running even if error encountered
  message=FALSE,
  rows.print=25)

library(ggplot2,warn.conflicts = FALSE)
library(dplyr,warn.conflicts = FALSE)
library(tidyr,warn.conflicts = FALSE)
library(readr,warn.conflicts = FALSE)
library(ggrepel,warn.conflicts = FALSE)
library(DT,warn.conflicts = FALSE)
#library(lemon,warn.conflicts = FALSE)

theme_set(theme_bw())
#knit_print.data.frame <- lemon_print

speciesID <- params$speciesID 
subpopOutDir <- params$subpopOutDir 
corrPcutOff_default <- params$corrPcutOff_default #<- 0.8
corrScutOff_default <- params$corrScutOff_default #<- 0.6
statCutoff <- params$statCutoff #<- 0.05
rStatCutoff <- params$rStatCutoff # 0.5
rStatCutoff_strong <- params$rStatCutoff_strong # 0.7
minObs <- params$minObs
bamSuffix <- params$bamSuffix
geneFamilyType <- params$geneFamilyType
```

<font size="14">**Species: "`r as.character(speciesID)`"**</font>

# Settings

```{r}

do.call(rbind,params) %>% data.frame(stringsAsFactors = F) %>% DT::datatable()
```

**Taxonomic classification**

```{r}
taxaDf <- getSpeciesTaxonomy(speciesID)
if(is.null(taxaDf)){
  print(paste("Taxonomy unknown for ",speciesID))
}else{
  taxaDf
}
```


# Associations between subspecies and gene families

Based on correlations between the abundances of the within-species clusters and the abundances of gene families

```{r}
pathCorrP <- paste0(subpopOutDir,"/",speciesID,"_corr",geneFamilyType,"-pearson.tsv")
pathCorrS <- paste0(subpopOutDir,"/",speciesID,"_corr",geneFamilyType,"-spearman.tsv")

pathClusterGenesDf <- paste0(subpopOutDir,"/",speciesID,"_corr",geneFamilyType,"-clusterSpecificGenes.tsv")
pathSpeciesGenesDf <- paste0(subpopOutDir,"/",speciesID,"_corr",geneFamilyType,"-speciesSpecificGenes.tsv")
pathClusterGenesAbundancesDf <-  paste0(subpopOutDir,"/",speciesID,"_corr",geneFamilyType,"-clusterSpecificGeneAbundances.tsv")

if(!file.exists(pathCorrP)){
  print(paste0("Aborting due to missing file: ",pathCorrP))
  knitr::knit_exit()
}
if(!file.exists(pathCorrS)){
  print(paste0("Aborting due to missing file: ",pathCorrS))
  knitr::knit_exit()
}
if(!file.exists(pathClusterGenesDf)){
  print(paste0("Aborting due to missing file: ",pathClusterGenesDf))
  knitr::knit_exit()
}
if(!file.exists(pathSpeciesGenesDf)){
  print(paste0("Aborting due to missing file: ",pathSpeciesGenesDf))
  knitr::knit_exit()
}
```

Using files:

```{r}
data.frame(files=c(pathCorrP,pathCorrS,pathClusterGenesDf,pathSpeciesGenesDf))
```


```{r}
corrP <- read_tsv(pathCorrP,col_names = T, col_types = cols(
  geneFamily = col_character(),
  cluster = col_integer(),
  statistic = col_double(),
  #parameter = col_logical(),
  p.value = col_double(),
  estimate = col_double(),
  null.value = col_integer(),
  alternative = col_character(),
  method = col_character(),
  #data.name = col_logical(),
  conf.int = col_logical(),
  conf.int.low = col_double(),
  conf.int.high = col_double(),
  nObs = col_double(),
  q.valueBH = col_double()
)
)
corrS <- read_tsv(pathCorrS,col_names = T, col_types = cols(
  geneFamily = col_character(),
  cluster = col_integer(),
  statistic = col_double(),
  #parameter = col_logical(),
  p.value = col_double(),
  estimate = col_double(),
  null.value = col_integer(),
  alternative = col_character(),
  method = col_character(),
  #data.name = col_logical(),
  nObs = col_double(),
  q.valueBH = col_double()
))

clusterGenesDf <- read_tsv(pathClusterGenesDf, 
                           col_types = cols(
  geneFamily = col_character(),
  cluster = col_character(),
  geneIsCorrelated = col_logical(),
  geneIsNotCorrelated = col_logical()
)) %>% 
  mutate(cluster = sub(cluster, pattern = "Cluster ",replacement = ""))

speciesGenesDf <- read_tsv(pathSpeciesGenesDf,
                           col_types = cols(
  geneFamily = col_character(),
  cluster = col_character(),
  geneIsCorrelated = col_logical(),
  geneIsNotCorrelated = col_logical()
)) %>% 
  mutate(cluster = sub(cluster, pattern = "Cluster ",replacement = ""))

if(!"corrP" %in% ls()){
  print(paste0("Aborting due to error in loading file: ",pathCorrP))
  knitr::knit_exit()
}
if(!"corrS" %in% ls()){
  print(paste0("Aborting due to error in loading file: ",pathCorrS))
  knitr::knit_exit()
}
if(!"clusterGenesDf" %in% ls()){
  print(paste0("Aborting due to error in loading file: ",pathClusterGenesDf))
  knitr::knit_exit()
}
if(!"speciesGenesDf" %in% ls()){
  print(paste0("Aborting due to error in loading file: ",pathSpeciesGenesDf))
  knitr::knit_exit()
}
if( max(corrS$nObs, corrP$nObs) < minObs ){
  print(paste0("Number of observations in correlations (",max(corrS$nObs,max(corrP$nObs)), ") is too small to be meaningful. Aborting report."))
  knitr::knit_exit()
}

nGenes <- length(unique(clusterGenesDf$geneFamily))
nGenesSpecies <- length(unique(speciesGenesDf$geneFamily))
```


Number of genes tested with Pearson correlations: `r nrow(corrP)`

Number of significant Pearson correlations (q value < `r statCutoff`):  `r sum(corrP$q.valueBH < statCutoff)`


Number of genes tested with Spearman correlations: `r nrow(corrS)`

Number of significant Spearman correlations (q value < `r statCutoff`): `r sum(corrS$q.valueBH < statCutoff)`

Number of genes specific to one or more clusters: `r nGenes`

Number of genes correlated with overall species abundance: `r nGenes`

```{r}
speciesGenes <- speciesGenesDf %>% filter(geneIsCorrelated) %>% pull(geneFamily) %>% unique()

corrS$conf.int <- NA
corrS$conf.int.low <- NA
corrS$conf.int.high <- NA
corr <- rbind(corrP,corrS) %>% 
  mutate(statSigLgl = q.valueBH < statCutoff,
         statisticallySignificant = if_else(q.valueBH < statCutoff,true = paste0("q < ",statCutoff),false = paste0("q >= ",statCutoff)),
         #cluster = paste0("Subspecies ",cluster),
         cluster=as.character(cluster)
         ) %>% 
  rename(correlationR = estimate) %>% 
  left_join(clusterGenesDf,by = c("geneFamily","cluster")) %>% 
  mutate(geneIsCorrelatedLgl = geneIsCorrelated,
         geneIsNotCorrelatedLgl = geneIsNotCorrelated,
          geneIsCorrelated = case_when( geneIsCorrelated & !geneIsNotCorrelated ~ "Correlated",
                                       !geneIsCorrelated & geneIsNotCorrelated ~ "Not correlated",
                                       TRUE ~ "Unclear")) %>% 
  select(-geneIsNotCorrelated) %>% 
  mutate(geneCorrelatedWithSpecies = geneFamily %in% speciesGenes,
         clusterStr = case_when(cluster == "-1" ~ "Species",
                                T ~ paste0("Cluster ",cluster)))

nClus <- n_distinct(corr$cluster)
```

# Comparison of Pearson and Spearman correlation results

```{r test for both cols}
haveBothSP <- all( c("pearson","spearman") %in% unique(as.character(corr$method)) ) 
```

```{r, eval=!haveBothSP}
print("Results not found for both methods.")
```

Top right box highlights correlations with high values in both methods

```{r}
corrW <- corr %>% 
  select(geneFamily, cluster,clusterStr, method, correlationR,
         statisticallySignificant,statSigLgl,
         geneIsCorrelated,geneIsCorrelatedLgl,geneCorrelatedWithSpecies) %>% 
  spread(key = method,value = correlationR,fill = NA)
x <- sum(is.na(corrW$pearson) | is.na(corrW$spearman))
y <- nrow(corrW)
```


Only plotting correlations that are statistically significant (q < 0.05).
Not plotting `r x` out of `r y` data points due to value not being present for both Pearson and Spearman correlations. 

Correlation thresholds illustrated with dotted lines.


```{r, eval=haveBothSP}
corrW %>% 
  filter(!is.na(corrW$pearson) & !is.na(corrW$spearman)) %>% 
  filter(statSigLgl) %>% 
  #mutate(color = case_when( geneIsCorrelated )) %>% 
  ggplot(aes(x=pearson,y=spearman))+
  # geom_polygon(data=data.frame(),
  #              aes(x=c(corrPcutOff_default,corrPcutOff_default,1,1),
  #                  y=c(corrScutOff_default,1,1,corrScutOff_default)),fill="yellow",alpha=0.25)+
  # #geom_hline(yintercept = rStatCutoff)+geom_vline(xintercept = rStatCutoff)+
  geom_hline(yintercept = corrScutOff_default,color="grey70",linetype="dotted")+
  geom_vline(xintercept = corrPcutOff_default,color="grey70",linetype="dotted")+
  geom_point(aes(color=clusterStr,shape=geneIsCorrelated),alpha=0.3)+
  coord_equal()+ylim(c(NA,1))+xlim(c(NA,1))+
  scale_color_manual("Correlation\nmeasured\nwith",values=viridisLite::viridis(n=nClus+1))+
  scale_shape_manual("Gene specifically \ncorrelated with \ncluster",values = c(19,1,4))+
  guides(fill=NULL)+
  ylab("Spearman correlation (R)\nbetween gene (family) and cluster")+
  xlab("Pearson correlation (R)\nbetween gene (family) and cluster")
  
```


# Focus on cluster-specific gene correlations


Here we consider a gene correlated with a cluster if it has: 

- minimum of 10 observations (data points)
- Pearson correlation R statistic > 0.8 (computed on log10‐transformed relative abundances)
- Spearman correlation R statistic > 0.6 
- q value < 0.05 for both Spearman and Pearson

Additionally, genes must be: 
- correlated with >= 1 subspecies
- correlation R statistic for at least one subspecies must be < 0.2 for both Spearman and Pearson
- either strongly correlated or strongly not correlated for every subspecies

```{r}
if(nrow(clusterGenesDf) == 0){
  cat("No genes were specifially associated with any of the clusters.")
  knitr::knit_exit()
}
```


```{r}
corrSubspeciesSpecific <- corr %>% 
  filter(geneIsCorrelatedLgl | geneIsNotCorrelatedLgl)

sigGeneFams <-  corrSubspeciesSpecific %>% 
  arrange(-correlationR) %>% 
  pull(geneFamily) %>% unique()
```

Found `r length(sigGeneFams)` correlated gene families.

```{r}
corr %>% 
  filter(geneIsCorrelated == "Correlated") %>% 
  group_by(cluster)  %>% 
  count(geneIsCorrelated,name = "Number of genes correlated")%>% 
  select(-geneIsCorrelated) %>% 
  rename(Subspecies = cluster)
```

## Table of cluster-specific genes

Printing up to first 1000 genes of file:

```{r}
print(pathClusterGenesDf)
```

```{r}
df <- clusterGenesDf %>% 
  mutate(cluster = paste("Cluster ",cluster)) %>% 
  mutate(geneIsCorrelated = case_when( geneIsCorrelated & !geneIsNotCorrelated ~ "Correlated",
                                       !geneIsCorrelated & geneIsNotCorrelated ~ "-",
                                       TRUE ~ "Unclear")) %>% 
  select(-geneIsNotCorrelated) 
df %>% 
  spread(key = cluster,value = geneIsCorrelated) %>% 
  head(1000)%>% 
  DT::datatable(filter="top")
```

Distribution of subspecies-specific correlation strengths

```{r}
corrSubspeciesSpecific %>%
  filter(geneIsCorrelatedLgl) %>% 
  mutate(cluster = paste("Cluster ",cluster)) %>% 
  ggplot(aes(x=correlationR))+
    facet_grid(method~cluster,switch = "y") +
  geom_histogram(binwidth = 0.025)+
  #xlim(c( (rStatCutoff-0.1),1.05))+
  ylab("Number of gene families")+
  xlab("Strength of correlation (R)")
```


## Example plots of strong subspecies-specific correlations

```{r}
if(!file.exists(pathClusterGenesAbundancesDf)){
  print(paste0("Aborting due to missing file: ",pathClusterGenesAbundancesDf))
  knitr::knit_exit()
}
#Loading the gene and cluster abundances to plot the specifc genes
abundancesToPlot <- read_tsv(pathClusterGenesAbundancesDf,
                             col_types = cols(
                               geneFamily = col_character(),
                               sampleName = col_character(),
                               geneFamAbund = col_double(),
                               cluster = col_double(),
                               clustAbund = col_double()
                             )) 

if(nrow(abundancesToPlot) == 0){
  print(paste0("Aborting due to empty file: ",pathClusterGenesAbundancesDf))
  knitr::knit_exit()
}

maxToPlot <- 10
tooManyToPlot <- length(sigGeneFams) > maxToPlot

# keep top best and bottom worst genes for illustration  
sigGeneFamsTop <- head(sigGeneFams,n = ceiling(maxToPlot/2))
sigGeneFamsBottom <- tail(sigGeneFams,n = floor(maxToPlot/2))
sigGeneFams_maxToPlot <- c(sigGeneFamsTop,sigGeneFamsBottom) %>% 
  unique() # take unique in case length < maxToPlot

x<-abundancesToPlot$geneFamAbund
pseudocount <- min(x[x>0],na.rm = T)/1000

plotFamCorr <- function(geneFam) {
  print(paste("Gene:",geneFam))
  corr %>% 
    filter(cluster != "-1") %>% 
    filter(geneFamily == geneFam)%>% 
    select(geneFamily, cluster, method, correlationR, q.valueBH, statSigLgl) %>%
    arrange(cluster) %>% rename(Subspecies = cluster) %>%  
    as.data.frame() %>% 
    print()
# 
# corrsToPlot %>% filter(geneFamily == geneFam,.preserve = T) %>% head() %>% print()
# corrsToPlot %>% filter(geneFamily == geneFam,.preserve = T) %>% tail() %>% print()

  
  geneFam <- sigGeneFamsTop[1]
  p <- abundancesToPlot %>% 
    filter(geneFamily == geneFam,.preserve = T) %>%
    # remove samples that don't have any of the clusters
    group_by(sampleName) %>% 
#    filter(any(clustAbund != 0)) %>% 
    mutate(geneFamAbund = geneFamAbund + pseudocount) %>% # for log10
    mutate(cluster = paste0("Subspecies ",cluster)) %>% 
    ggplot(aes(x= geneFamAbund, y=clustAbund)) + 
    geom_point()+
    facet_grid(.~cluster)+
    ggtitle(paste0("Species: ",speciesID,", linear scale\nGene family: ",geneFam))+
    xlab("Abundance of gene family")+
    ylab(paste0("Abundance of subspecies \n(plus pseudocount",format(pseudocount,digits = 2),")"))+
    coord_cartesian(ylim = c(pseudocount,max(1,max(abundancesToPlot$clustAbund,na.rm = T),na.rm = T)))
    #ylim(0,max(1,max(abundancesToPlot$clustAbund,na.rm = T),na.rm = T))
  print(p)

  print(p+scale_x_log10()+scale_y_log10()+
          ggtitle(paste0("Species: ",speciesID,", log10 scale\nGene family: ",geneFam)))

}
```

Plotting at most `r maxToPlot` examples

```{r,eval=!tooManyToPlot}
if( !tooManyToPlot){
print("Plotting all valid correlations")
  tmp <- sapply(sigGeneFams, plotFamCorr)
  rm(tmp)
}
```

```{r,eval=tooManyToPlot, results='asis'}
if(tooManyToPlot){
  cat("### Best correlations")
  print("")
  cat("")
  print(paste0("Too many gene families to plot all of them, only plotting best ",floor(maxToPlot/2),
               " and 'worst' ",ceiling(maxToPlot/2), " (all above quality cutoffs)"))
}
```

```{r,eval=tooManyToPlot}
if(tooManyToPlot){
  tmp <- sapply(sigGeneFamsTop, plotFamCorr)
  rm(tmp)  
}
```


```{r,eval=tooManyToPlot, results='asis'}
if(tooManyToPlot){
  cat("### 'Worst' of the best correlations")
}
```

```{r,eval=tooManyToPlot}
if(tooManyToPlot){
  tmp <- sapply(sigGeneFamsBottom, plotFamCorr)
  rm(tmp)  
}
```

# Session Info

```{r}
sessionInfo()
```

