
writeGenotypeFreqs <- function(clustDf,snvFreqs,species,outputDir,uniqSubpopSnvFreqThreshold){

  clustDf$sampleName <- row.names(clustDf)
  commonSamples <- intersect(colnames(snvFreqs), rownames(clustDf))
  clustDf_sub <- clustDf[commonSamples,]
  snvFreqs_sub <- snvFreqs[,commonSamples]

  clustAssignments <- clustDf_sub$clust
  clustIDs <- unique(clustAssignments)
  nClusters <- length(clustIDs)
  if ( nClusters <= 1) {
    write(x = 'Single cluster',file = paste(outputDir,species,'_hap_out.txt',sep=''),append = F )
  }else{

    #Make -1 into NA. Makes everything a lot faster as NA is natively taken care of.
    # -1 is generated by metaSNV
    snvFreqs_sub <- makeNA(snvFreqs_sub)

    # start logging
    write(x = '',file = paste(outputDir,species,'_hap_out.txt',sep=''),append = F )

    # preserving for posterity, may have been used in Costea subspecies paper
    # if (species=='411489' | species=='470145') {
    #   res <- computeUniquePos(d=90, data_sub,cdf_sub,species,outputDir)
    # } else if (species=='411470') {
    #   res <- computeUniquePos(d=80, data_sub,cdf_sub,species,outputDir)
    # } else if (species=='556259' | species=='657317') {
    #   res <- computeUniquePos(d=60, data_sub,cdf_sub,species,outputDir)
    # } else {
    #   res <- computeUniquePos(d=70, data_sub,cdf_sub,species,outputDir)
    # }
    # d = 80% is reported in costea2017 paper
    # d = cutoff mean SNV frequency difference between samples within cluster vs all other samples
    res <- computeUniquePosPerCluster(uniqThreshold=uniqSubpopSnvFreqThreshold*100, snvFreqs_sub,clustDf_sub,species,outputDir)
    #res <- computeDifferentialSNVs(snvFreqs_sub,clustDf_sub,species,outputDir)
    if(is.null(unlist(res))){
      write(x = paste("No genotyping positions for ",species),
            file = paste(outputDir,species,'_hap_out.txt',sep=''),append = T )
      return()
    }

    # res is a list with two elements, both are table with two columns:
    # (1) the frequency of a cluster's genotype in the sample and
    # (2) the cluster number of the genotype
    # res[[1]] has the freq of the cluster based on mean abundance of SNVs
    # res[[2]] has the freq of the cluster based on median abundance of SNVs
    freq_data_mean <- res[[1]]
    freq_data_median <- res[[2]]
    clusterID_1 <- clustIDs[1]
    coll <- freq_data_median[freq_data_median[,2]==clusterID_1,1] # get freq column for cluster1
    coll <- sapply(coll,FUN = as.numeric)
    for (i in 2:nClusters ) {
      clusterID_i <- clustIDs[i]
      colln <- freq_data_median[freq_data_median[,2]==clusterID_i,1]
      if(length(colln) == 0){next()} # if there are not distinctive SNVs for this cluster, skip it
      colln <- sapply(colln, as.numeric)
      coll <- cbind(coll,colln) # add column for every other cluster
    }
    if(ncol(coll) < nClusters ){
      write(x = paste("At least one cluster is missing genotyping positions for ",species,
                      ". Aborting, but this could be fixed."),
            file = paste(outputDir,species,'_hap_out.txt',sep=''),append = T )
      return()
    }
    # coll is now a df with 1 column per cluster and 1 row per sample
    # each column value is the frequency of the cluster based on the freq of that cluster's unique SNVs
    # if the unique snvs were not or could not be well chosen, then the sample may look like it belongs to > 1 cluster
    # or it could look like it belongs to zero clusters

    # NA for classification, due to low coverage of gSNVs
    samplesWithoutClassification <- rownames(coll[is.na(rowSums(coll)),])
    
    # number of samples that don't nicely fit into one of the defined clusters
    badSamples <- names(which(!is.na(rowSums(coll)) & (rowSums(coll) > 120 | rowSums(coll) < 80)))
    nBadSamples <- length(badSamples)
    if (nBadSamples > 0.15*nrow(coll)) { # more than 10% bad samples
      warning(paste0('Genotype-specific SNV frequency cutoff is bad for species ',species,
                     ". See details in file ",paste0(outputDir,species,'_hap_out.txt')))
      write(x = 'Cutoff is bad',file = paste(outputDir,species,'_hap_out.txt',sep=''),append = T )
      write(x = paste('In ',nBadSamples,' out of ',nrow(coll),' samples, ',
                      'the summed abundance of all clusters per sample is >120% or < 80%,',
                      ' based on the frequencies of the genotyping SNVs.'),
            file = paste(outputDir,species,'_hap_out.txt',sep=''),append = T )

      write(x = "Samples with incoherent cluster abundance measured based on genotyping SNVs: ",
            file = paste(outputDir,species,'_hap_out.txt',sep=''),append = T )
      write(x = badSamples,
            file = paste(outputDir,species,'_hap_out.txt',sep=''),append = T )

      return()
    }

    # coherent, non NA cluster assignments
    goodSamples <- rownames(coll)[!rownames(coll) %in% c(badSamples,samplesWithoutClassification)]
    
    # correct assignments
    nCorrAssign <- sum(clustDf_sub[goodSamples,"clust"] == apply(coll[goodSamples,],1,which.max))
    
    write(x = paste0('Genotyping-based assignment of discovery samples to clusters was correct for ',nCorrAssign,' samples. ',
                    'Determined any cluster assignment from genotyping SNVs for ',
                    length(goodSamples),' out of ',nrow(coll),' samples. Of those samples without assignments: ',
                    length(badSamples) ,'  had incoherect cluster assignments and ',
                    length(samplesWithoutClassification) ,' lacked coverage of genotyping SNVs.'),
          file = paste(outputDir,species,'_hap_out.txt',sep=''),append = T )

    write.table(freq_data_mean,paste(outputDir,species,'_hap_freq_mean.tab',sep=''),sep='\t',quote=F)
    write.table(freq_data_median,paste(outputDir,species,'_hap_freq_median.tab',sep=''),sep='\t',quote=F)

  }
  return()
}


computeUniquePos = function(uniqThreshold=80, snvFreqs_sub, clustDf_sub, species, outputDir) {
  freq_data_mean <- NULL
  freq_data_median <- NULL
  pos_df <- NULL

  if(max(clustDf_sub$clust) < 1){
    return(NULL)
  }
  snvFreqs_sub_orig <- snvFreqs_sub
  for (i in unique(clustDf_sub$clust)) {
    clusterSamples <- clustDf_sub[clustDf_sub$clust==i,"sampleName"]
    nonClusterSamples <- clustDf_sub[clustDf_sub$clust!=i,"sampleName"]
    snvFreqs_sub <- snvFreqs_sub_orig
    
    # remove SNVs that have NA in >20% of cluster samples
    snvFreqs_sub_inClus <- snvFreqs_sub[,clusterSamples]
    propNA_inCluster <- rowSums(is.na(snvFreqs_sub_inClus))/ncol(snvFreqs_sub_inClus)
    snvFreqs_sub <- snvFreqs_sub[propNA_inCluster<0.2,]

    # in non-cluster samples, no coverage (NA) is same as not present (freq=0)
    df2 <- snvFreqs_sub[,nonClusterSamples]
    df2[is.na(df2)] <- 0
    snvFreqs_sub[,nonClusterSamples] <- df2

    # if you have a lot of non-cluster samples, the mean can hide many with e.g. freq=100
    meanAbundsInCluster <- rowMeans(snvFreqs_sub[,clusterSamples],na.rm = T)
    df2 <- snvFreqs_sub[,nonClusterSamples]
    # how many non cluster samples have the same-ish freq as within the cluster
    propMatch <- rowSums(df2<=(meanAbundsInCluster+5) & df2>=(meanAbundsInCluster-5)) / ncol(df2)
    # remove SNVs where >=20% of non-cluster samples have a cluster-like abundance
    df2 <- df2[propMatch<0.2,]
    snvsOk <- rownames(df2)
    snvFreqs_sub <- snvFreqs_sub[snvsOk,] 


    # get difference in mean frequency of SNV in samples within cluster vs not in cluster
    fDist <- (abs( rowMeans(snvFreqs_sub[,clusterSamples],na.rm = T) -
                     rowMeans(snvFreqs_sub[,nonClusterSamples],na.rm = T) ) )

    #x<-summary(fDist)
    fDist[is.na(fDist)] <- 0
    fDist[is.nan(fDist)] <- 0
    # get those SNPs where its mean frequency within the cluster is higher||lower than
    # mean frequency outside the cluster by at least 'd' percentage points
    oList <- which(fDist >= uniqThreshold)

    if (length(oList) == 0) {
      status <- paste('No unique genotyping positions for species',species,"cluster",i,
                    "(species has",length(unique(clustDf_sub$clust)),"total clusters)")
      warning(status)
      write(status, paste(outputDir,species,'_hap_out.txt',sep=''),append = T)
      #write(paste(paste(names(x),x,sep="="),collapse=","), paste(outputDir,species,'_hap_out.txt',sep=''),append = T)
      #write(paste(fDist,collapse = ", "), paste(outputDir,species,'_hap_out.txt',sep=''),append = T)
      next()
    }

    fDist_data <- snvFreqs_sub[oList,]

    # for each SNV, get whether most cluster samples have the major allele (1) or not (0)
    # the major allele has frequency >=50
    spec_snp <- which(majorAllel(fDist_data[,clusterSamples]) == 0)
    # for those samples that have the minor allele, change the frequency from e.g. 3 -> 97
    # don't alter frequencies for major allele or those with value == -1
    if (length(spec_snp) > 0) {
      fDist_data[spec_snp,] <- 100-fDist_data[spec_snp,]
    }
    hap_positions <- data.frame(posId = rownames(fDist_data), flip = FALSE)
    hap_positions$flip[spec_snp] <- TRUE
    write.table(hap_positions,paste(outputDir,species,'_',i,'_hap_positions.tab',sep=''),sep='\t',quote=F)

    freq_data_mean <- rbind(freq_data_mean,cbind(apply(fDist_data,2,mean,na.rm=T),i))
    freq_data_median <- rbind(freq_data_median,cbind(apply(fDist_data,2,median,na.rm=T),i))
  }

  return(list(freq_data_mean,freq_data_median))

}



computeUniquePosPerCluster = function(uniqThreshold=80, snvFreqs_sub, clustDf_sub, species, outputDir) {
  freq_data_mean <- NULL
  freq_data_median <- NULL
  pos_df <- NULL
  
  if(max(clustDf_sub$clust) < 1){
    return(NULL)
  }
  snvFreqs_sub_orig <- snvFreqs_sub
  for (i in unique(clustDf_sub$clust)) {
    clusterSamples <- clustDf_sub[clustDf_sub$clust==i,"sampleName"]
    nonClusterSamples <- clustDf_sub[clustDf_sub$clust!=i,"sampleName"]
    snvFreqs_sub <- snvFreqs_sub_orig
    
    # remove SNVs that have NA in >20% of cluster samples
    snvFreqs_sub_inClus <- snvFreqs_sub[,clusterSamples]
    propNA_inCluster <- rowSums(is.na(snvFreqs_sub_inClus))/ncol(snvFreqs_sub_inClus)
    snvFreqs_sub <- snvFreqs_sub[propNA_inCluster<0.2,]

    # NA means vertical coverage of this position was < 5 in this sample
    # this could be due to the position not existing in this subspecies, or the species being at low abundance in the sample
    # if position is NA in most (>80%) non-cluster samples, then unlikely due to abundance
    # in that case, in non-cluster samples, no coverage (NA) is same as opposite allele, here coded as -101 so that the difference to 0 and to 100 are both > 100
    # snvsToReplace <- rowSums(is.na(snvFreqs_sub[,nonClusterSamples]))/ncol(snvFreqs_sub[,nonClusterSamples]) > 0.8
    # if(sum(snvsToReplace)>0){
    #   df2 <- snvFreqs_sub[snvsToReplace,nonClusterSamples]
    #   df2[is.na(df2)] <- -101
    #   snvFreqs_sub[snvsToReplace,nonClusterSamples] <- df2
    # }
    # remove SNVs that have NA in >20% of non-cluster samples
    snvFreqs_sub_nonClus <- snvFreqs_sub[,nonClusterSamples]
    propNA_nonCluster <- rowSums(is.na(snvFreqs_sub_nonClus))/ncol(snvFreqs_sub_nonClus)
    snvFreqs_sub <- snvFreqs_sub[propNA_nonCluster<0.2,]


    gSNVCandidates <- list()
    # check the abundance difference for each SNV for each cluster pair
    for (j in unique(clustDf_sub$clust)){
      if(i!=j){
        nonClusterSamples_j <- clustDf_sub[clustDf_sub$clust==j,"sampleName"]
        fDist <- (abs( rowMeans(snvFreqs_sub[,clusterSamples],na.rm = T) -
                         rowMeans(snvFreqs_sub[,nonClusterSamples_j],na.rm = T) ) )  
        fDist[is.na(fDist)] <- 0
        fDist[is.nan(fDist)] <- 0
        # get those SNPs where its mean frequency within the cluster is higher||lower than
        # mean frequency outside the cluster by at least 'd' percentage points
        gSNVCandidates[[as.character(j)]] <- names(fDist[fDist>uniqThreshold])
      }
    }    
    # get the SNVs that are distinct across all cluster pairs
    oList <- Reduce(intersect, gSNVCandidates)
    
    if (length(oList) == 0) {
      status <- paste('No unique genotyping positions for species',species,"cluster",i,
                      "(species has",length(unique(clustDf_sub$clust)),"total clusters)")
      warning(status)
      write(status, paste(outputDir,species,'_hap_out.txt',sep=''),append = T)
      #write(paste(paste(names(x),x,sep="="),collapse=","), paste(outputDir,species,'_hap_out.txt',sep=''),append = T)
      #write(paste(fDist,collapse = ", "), paste(outputDir,species,'_hap_out.txt',sep=''),append = T)
      next()
    }
    
    fDist_data <- snvFreqs_sub[oList,]
    
    # for each SNV, get whether most cluster samples have the major allele (1) or not (0)
    # the major allele has frequency >=50
    spec_snp <- which(majorAllel(fDist_data[,clusterSamples]) == 0)
    # for those samples that have the minor allele, change the frequency from e.g. 3 -> 97
    # don't alter frequencies for major allele or those with value == -1
    if (length(spec_snp) > 0) {
      fDist_data[spec_snp,] <- 100-fDist_data[spec_snp,]
    }
    hap_positions <- data.frame(posId = rownames(fDist_data), flip = FALSE)
    hap_positions$flip[spec_snp] <- TRUE
    write.table(hap_positions,paste(outputDir,species,'_',i,'_hap_positions.tab',sep=''),sep='\t',quote=F)
    
    freq_data_mean <- rbind(freq_data_mean,cbind(apply(fDist_data,2,mean,na.rm=T),i))
    freq_data_median <- rbind(freq_data_median,cbind(apply(fDist_data,2,median,na.rm=T),i))
  }
  
  return(list(freq_data_mean,freq_data_median))
  
}


computeDifferentialSNVs = function(snvFreqs, clustDf, species, outputDir, qValCut=0.05) {

  if( length(unique(clustDf$clust)) == 1){
    return(NULL)
  }

  # add cluster annotation to snv freq data
  snvFreqsDf <- snvFreqs %>%
    mutate(snv = row.names(.)) %>%
    gather(key = "sampleName",value = "freq",-snv) %>%
    filter(!is.na(freq)) %>%
    left_join(clustDf,by="sampleName") %>% # get the cluster each sample was assigned to (some samples may not have assignment)
    replace_na(list(clust="notAssigned")) %>%
    mutate(clust = factor(clust))

  # first use multi-group statistical test to pick those snv positions that do not have the same distribution/frequency across all clusters
  # (could be that two groups have same distribution and two groups have different)
  sigSNVs <- snvFreqsDf %>%
    group_by(snv, clust) %>%
    summarise(freq = list(freq)) %>%
    group_by(snv) %>%
    summarise(pValue = kruskal.test(freq)$p.value,
              #statistic = kruskal.test(freq)$statistic,
              qValue = p.adjust(pValue,method = "BH")) %>%
    filter(qValue < qValCut)

  # only keep SNVs where average value is distinct in one cluster than in any other cluster (without pooling data from other clusters)
  getClustAssocSNVs <- function(clustA, clustB){

    if(clustA == clustB){return(NULL)}
    snvFreqsDf %>%
      filter(snv %in% sigSNVs$snv) %>%
      filter(clust %in% c(clustA,clustB)) %>%
      mutate(clustMembership = if_else(clust == clustA,true = "inClust",false="notInClust")) %>%
      group_by(snv, clustMembership) %>%
      summarise(freq = list(freq)) %>%
      spread(clustMembership,freq) %>%
      group_by(snv) %>%
      mutate(
        clust = clustA,
        vsClust = clustB,
        meanFreqInClus = mean(unlist(inClust)),
        meanFreqNotInClus = mean(unlist(notInClust)),
        medianFreqInClus = median(unlist(inClust)),
        medianFreqNotInClus = median(unlist(notInClust)),
        pValue = wilcox.test(unlist(inClust), unlist(notInClust))$p.value,
        #tValue = wilcox.test(unlist(inClust), unlist(notInClust))$statistic,
        qValue = p.adjust(pValue,method = "BH")) %>%
      mutate(higherAbundInCluster = meanFreqInClus > meanFreqNotInClus) %>%
      filter( #(meanFreqInClus > meanFreqNotInClus | medianFreqInClus > medianFreqNotInClus) , # keep these but use 'flip' business below
        qValue < qValCut) %>%
      select(-inClust,-notInClust)
    #replace_na(replace = list(freq=0)) %>%
  }
  allClusters <- as.character(unique(snvFreqsDf$clust))
  comparisons <- tidyr::crossing(allClusters,allClusters) %>%
    rename(clustA = allClusters, clustB = allClusters1) %>%
    filter(clustA != clustB)
  sigSNVsPerCluster <- purrr::pmap_df(comparisons,.f = getClustAssocSNVs)

  # this can still result in non distinct SNVs, such as here:
  # snv                             clust vsClust meanFreqInClus meanFreqNotInClus medianFreqInClus medianFreqNotInClus            pValue            qValue
  # <chr>                           <chr> <chr>            <dbl>             <dbl>            <dbl>               <dbl>             <dbl>             <dbl>
  # 1 referenceGenome:-:1516328:A>G:. 1     2                 26.2             0.595             25.0                   0 0.000000000000483 0.000000000000483
  # 2 referenceGenome:-:1516328:A>G:. 3     2                 23.9             0.595             21.1                   0 0.00000000000222  0.00000000000222
  # 3 referenceGenome:-:2450610:G>A:. 1     2                 24.7             0.233             22.2                   0 0.000000000000307 0.000000000000307
  # 4 referenceGenome:-:2450610:G>A:. 3     2                 22.9             0.233             23.6                   0 0.00000000000928  0.00000000000928

  # remove snv positions where SNV was not distinctive vs all other clusters
  # and all in the same direction (clustA was higher or lower than all other clusters)
  sigSNVsPerCluster <- sigSNVsPerCluster %>%
    group_by(snv,clust, higherAbundInCluster) %>% filter(n() == length(allClusters)-1 ) %>%
    arrange(snv,clust, vsClust) %>%
    ungroup()

  clustersWithoutSNVs <- sigSNVsPerCluster %>% group_by(clust) %>% filter(n()==0) %>% pull(clust)

  if (length(clustersWithoutSNVs) == 0) {
    status <- paste('Species',species,': No distinctive genotyping positions for cluster(s)',
                    paste(clustersWithoutSNVs,collapse=","),
                    "(species has",length(allClusters),"total clusters)")
    warning(status)
    write(status, paste(outputDir,species,'_hap_out.txt',sep=''),append = T)
  }

  sigSNVsPerCluster

  oList <- sigSNVsPerCluster %>% pull(snv) %>% sort() %>% unique()
  fDist_data <- snvFreqs[oList,]

  # keeping this from before -- not sure I understand it
  freq_data_mean <- NULL
  freq_data_median <- NULL
  for(i in allClusters){
    clusterSamples <- clustDf[clustDf$clust==i,"sampleName"]

    # for each SNV/sample pair, get whether the sample has the major allele (1) or not (0)
    # the major allele has frequency >=50
    spec_snp <- which(majorAllel(snvFreqs[oList,clusterSamples]) == 0)

    # for those samples that have the minor allele, change the frequency from e.g. 3 -> 97
    # don't alter frequencies for major allele or those with value == -1
    if (length(spec_snp) > 0) {
      fDist_data[spec_snp,] <- 100-fDist_data[spec_snp,]
    }
    hap_positions <- data.frame(posId = rownames(fDist_data), flip = FALSE)
    hap_positions$flip[spec_snp] <- TRUE
    write.table(hap_positions,paste(outputDir,species,'_',i,'_hap_positions.tab',sep=''),sep='\t',quote=F)

    freq_data_mean <- rbind.data.frame(freq_data_mean,cbind(apply(fDist_data,2,mean,na.rm=T),i))
    freq_data_median <- rbind.data.frame(freq_data_median,cbind(apply(fDist_data,2,median,na.rm=T),i))

  }

  return(list(freq_data_mean,freq_data_median))

}

computeDifferentialSNVsRF <- function(snvFreqs, clustDf, species, outputDir, qValCut=0.05){

  allClusters <- as.character(unique(clustDf$clust))

  if( length(allClusters) == 1){
    return(NULL)
  }

  # add cluster annotation to snv freq data
  snvFreqsDf <- snvFreqs %>%
    mutate(snv = row.names(.)) %>%
    gather(key = "sampleName",value = "freq",-snv) %>%
    filter(!is.na(freq)) %>%
    left_join(clustDf,by="sampleName") %>% # get the cluster each sample was assigned to (some samples may not have assignment)
    replace_na(list(clust="notAssigned")) %>%
    mutate(clust = factor(clust))

  # first use multi-group statistical test to pick those snv positions that do not have the same distribution/frequency across all clusters
  # (could be that two groups have same distribution and two groups have different)
  sigSNVs <- snvFreqsDf %>%
    group_by(snv, clust) %>%
    summarise(freq = list(freq)) %>%
    group_by(snv) %>%
    summarise(pValue = kruskal.test(freq)$p.value,
              #statistic = kruskal.test(freq)$statistic,
              qValue = p.adjust(pValue,method = "BH")) %>%
    filter(qValue < qValCut) %>%
    pull(snv)

  head(sigSNVs)
  snvFreqsDf %>% filter(snv %in% sigSNVs) %>% head()
  rfDf <- snvFreqsDf %>%
    filter(snv %in% sigSNVs) %>%
    select(snv, freq, clust, sampleName) %>%
    spread(key = snv, value = freq) %>%
    select(-sampleName) %>%
    mutate(clust = factor(clust))

  rfDf[is.na(rfDf)] <- 0 # FIX THIS
  #formula = clust ~ ., # formula interface doesn't work with special characters in col names

# commenting out for now since not using to avoid ranger dependency
    # rfMod <- ranger::ranger(importance = "impurity",
  #                         data = rfDf[,1:50],
  #                         write.forest = T,
  #                         dependent.variable.name = "clust")

  # the problem is if a new sample doesn't have info for all the SNV locations....
  predict(rfMod,rfDf[,1:49])
}
