#!/usr/bin/env python3

import pysam
import glob
import os
import sys
import re
import pandas as pd
if sys.version_info < (3, 2):
    import subprocess32 as sp
else:
    import subprocess as sp

# Each term enclosed by parenthesis can be used extracted later from a re.match using an expresion and a string.
# (...){0,1} means that it is either 0 or 1 times there, and can also be extracted.
# re.match(expression, string).group(<number>) can be used to extract a matching part of a string.
# I don't have an example string here, but I'm exracting: 
# 1: Accession of Acceptor
# 2: Accession of Donor
# 3: Cut position (where the insert takes place). The first position in the sequence that stems from the insert.
# 4: First base that stems from donor
# 5: Length of the insertion (currently not used)
exp = re.compile('>From_(.*)_to_(.*) , First inserted base in acceptor: ([0-9]*), First extracted base in donor: ([0-9]*), Length of extract: ([0-9]*)')
negexp = re.compile('.*_([0-9]*_){0,1}([0-9]*)_.*_ins__mod.candidates')
# Get aligned/overall bases from blast output. Address via .group(1) and .group(2), respectively.
# \(...\) means actual parenthesis in the string.
fra = re.compile('.*?Identities = ([0-9]+)/([0-9]+) \([0-9]+%\), Gaps = [0-9]+/[0-9]+ \([0-9]+%\)', re.DOTALL)

# hgt_eval_ALL_0_pair_9_ALL_0_526469930_Burkholderia_pseudomallei_MSHR305_chromosome_2_ins_97797_mod_218777860_Desulfatibacillum_alkenivorans_AK-01_chromosome_from-to_6254811-6282811_mod_556574580_556571789.tsv
# 1: task number
# 2: pair number
# 3: original giAcc
# 4: acc insertion position
# 5: original giDon
# 6: don start pos
# 7: don end pos
# 8: giAcc passed to Daisy
# 9: giDon passed to Daisy
tsv_exp = re.compile('hgt_eval_.*_([0-9]*){0,1}_pair_([0-9]*){0,1}_.*_[0-9]*_([0-9]*)_ins_([0-9]*)_mod_([0-9]*)_from-to_([0-9]*)-([0-9]*)_mod_([0-9]*)_([0-9]*).tsv')
# hgt_eval_mgps_0_pair_0_mgps_0_470173039_ins_2368707_mod_478446832_from-to_410555-438555_mod_58036263_521187702.tsv
# tsv_exp = re.compile('hgt_eval_.*_([0-9]*){0,1}_pair_([0-9]*){0,1}_.*_[0-9]*_([0-9]*)_.*_ins_([0-9]*)_mod_([0-9]*)_.*_from-to_([0-9]*)-([0-9]*)_mod_([0-9]*)_([0-9]*).tsv')
# hgt_eval_NEG_0_pair_0_NEG_0_387502364_Propionibacterium_acnes_6609_chromosome_ins__mod_387502364_50841496.tsv
# 1: task number
# 2: pair number
# 3: original giAcc
# 4: giAcc passed to Daisy
# 5: giDon passed to Daisy
tsv_negexp = re.compile('hgt_eval_.*_([0-9]*){0,1}_pair_([0-9]*){0,1}_.*_[0-9]*_([0-9]*)_.*_ins__mod_([0-9]*)_([0-9]*).tsv')

def _getLength(fasta):
    with open(fasta, 'r') as f:
        length = 0
        for line in f:
            if line[0] == '>':
                pass
            else:
                length += len(line) - 1
        return int(length)


def blasta(dir_, giAcc, giAccs, ff, file_, refdir, a_snp, a_sir, a_svir, insertPos):
    #dumpdir = '/home/trappek/wiss/NG4_Daten/kathrin/sebio_backup/geneTransf/ducksimulation/OUTPUT/NEGSTATS-10/'
    #dumpdir = './'
    dumpdir = dir_
    rundir = os.path.dirname(sys.argv[0])
    if (rundir != ''):
        rundir += '/'
    else:
        rundir = './'

    # getting variated acceptor
    cmd = ['find', refdir, '-type', 'f', '-regextype', 'posix-extended', '-iregex', '^{}{}.*.fasta.gz$'.format(refdir, giAcc)]
    refacc = str(sp.check_output(cmd), 'utf-8').strip()
    cmd = 'gzip -fdk {}'.format(refacc)
    sp.check_call(cmd, shell=True)
    refacc = refacc.rstrip('.gz')
    null = open(os.devnull, 'w')
    accLength = _getLength(refacc)

    # Extract acceptor parts before and behind insertPos
    acc1 = '{}acc_part1.fa'.format(dumpdir)
    acc2 = '{}acc_part2.fa'.format(dumpdir)
    #cmd = ['{}sak'.format(rundir), refacc, '-i', '{}-{}'.format(0, insertPos - 1), '-o', acc1]
    cmd = ['sak', refacc, '-i', '{}-{}'.format(0, insertPos - 1), '-o', acc1]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)

    #cmd = ['{}sak'.format(rundir), refacc, '-i', '{}-{}'.format(insertPos - 1, accLength), '-o', acc2]
    cmd = ['sak', refacc, '-i', '{}-{}'.format(insertPos - 1, accLength), '-o', acc2]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)

    # Variate acceptor parts
    accVar1 = '{}accVar_part1.fa'.format(dumpdir)
    accVar2 = '{}accVar_part2.fa'.format(dumpdir)
#if args.b_new or not dsys.checkExistence(logger, 'Variating Acceptor', accVar):
    #cmd = ['{}mason_variator'.format(rundir), '-ir', acc1, '-of', accVar1, '-ov', accVar1[:-2] + 'vcf', '--snp-rate', a_snp, '--small-indel-rate', a_sir, '--sv-indel-rate', a_svir]
    cmd = ['mason_variator', '-ir', acc1, '-of', accVar1, '-ov', accVar1[:-2] + 'vcf', '--snp-rate', a_snp, '--small-indel-rate', a_sir, '--sv-indel-rate', a_svir]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    #cmd = ['{}mason_variator'.format(rundir), '-ir', acc2, '-of', accVar2, '-ov', accVar2[:-2] + 'vcf', '--snp-rate', a_snp, '--small-indel-rate', a_sir, '--sv-indel-rate', a_svir]
    cmd = ['mason_variator', '-ir', acc2, '-of', accVar2, '-ov', accVar2[:-2] + 'vcf', '--snp-rate', a_snp, '--small-indel-rate', a_sir, '--sv-indel-rate', a_svir]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)

    # Delete first fasta line ('>') from accVar2
    cmd = ['sed', '-i', '/^>/d', accVar2]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)

   # Concatenate and standardise hgt organism
    cmd = 'cat {} {} > {}accVar_raw.fa'.format(accVar1, accVar2, dumpdir)
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT, shell=True)
    #cmd = ['{}sak'.format(rundir), '{}accVar_raw.fa'.format(dumpdir), '-o', '{}accVar.fa'.format(dumpdir)]
    cmd = ['sak', '{}accVar_raw.fa'.format(dumpdir), '-o', '{}accVar.fa'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)

    # identity via read mapping congruency
    # simulate reads of accVar.fa 100x, 150bp, fragments: 500bp mean, 50bp sd (as in simulation)
    num_reads = int(100 * _getLength('{}accVar.fa'.format(dumpdir)) / (2 * 150))

    #if args.b_new or not dsys.checkExistence(logger, 'Simulating Reads', reads):
    #cmd = ['home/RKID1/trappek/bin/mason_simulator', '-ir', '{}accVar.fa'.format(dumpdir), '-n', str(num_reads), '-o', '{}sim_1.fa'.format(dumpdir), '-or', '{}sim_2.fa'.format(dumpdir),  '--fragment-mean-size', '500', '--fragment-size-std-dev', '50', '--illumina-read-length', '150']
    #cmd = ['{}mason_simulator'.format(rundir), '-ir', '{}accVar.fa'.format(dumpdir), '-n', str(num_reads), '-o', '{}sim_1.fa'.format(dumpdir), '-or', '{}sim_2.fa'.format(dumpdir),  '--fragment-mean-size', '500', '--fragment-size-std-dev', '50', '--illumina-read-length', '150']
    cmd = ['mason_simulator', '-ir', '{}accVar.fa'.format(dumpdir), '-n', str(num_reads), '-o', '{}sim_1.fa'.format(dumpdir), '-or', '{}sim_2.fa'.format(dumpdir),  '--fragment-mean-size', '500', '--fragment-size-std-dev', '50', '--illumina-read-length', '150']
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)

    # map reads to refacc to get original similarity --yara_e=5 --yara_le=150 --yara_ll=150 --yara_t=24
    #cmd = ['/home/RKID1/trappek/bin/yara_indexer', refacc, '-o', '{}refacc_index'.format(dumpdir)]
    #cmd = ['{}yara_indexer'.format(rundir), refacc, '-o', '{}refacc_index'.format(dumpdir)]
    cmd = ['yara_indexer', refacc, '-o', '{}refacc_index'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    #cmd = ['{}yara_mapper'.format(rundir), '{}refacc_index'.format(dumpdir), '{}sim_1.fa'.format(dumpdir), '{}sim_2.fa'.format(dumpdir), '-o', '{}refmap.sam'.format(dumpdir), '-e', '5', '-sa', 'omit', '-t', '10']
    cmd = ['yara_mapper', '{}refacc_index'.format(dumpdir), '{}sim_1.fa'.format(dumpdir), '{}sim_2.fa'.format(dumpdir), '-o', '{}refmap.sam'.format(dumpdir), '-e', '5', '-sa', 'omit', '-t', '10']
    # new yara
    #cmd = ['/home/RKID1/trappek/bin/yara_mapper', 'subject_index', 'sim_1.fa', 'sim_2.fa', '-o', 'submap.sam', '-e', '5', '-ll', '500', '-ld', '50', '-t', '24']
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)

    # read in sam file and count hits
    rf = pysam.Samfile('{}refmap.sam'.format(dumpdir),'r')
    num_matches = float(0.0)
    for read in rf:
        if not read.is_unmapped:
            num_matches += 1.0
    print('refmap original_indent = {}/{}'.format(num_matches, num_reads * 2))
    original_ident = num_matches/float(num_reads * 2)
    ff.write('original ident {} \n'.format(original_ident))
    rf.close()
    cmd = ['rm', '{}refmap.sam'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = 'rm {}refacc_index*'.format(dumpdir)
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT, shell=True)

    # map reads to subject/candidate  --yara_e=5 --yara_le=150 --yara_ll=150 --yara_t=24
    cmd = ['find', refdir, '-type', 'f', '-regextype', 'posix-extended', '-iregex', '^{}{}.*.fasta.gz$'.format(refdir, giAccs)]
    subject = str(sp.check_output(cmd), 'utf-8').strip()
    cmd = 'gzip -fdk {}'.format(subject)
    sp.check_call(cmd, shell=True)
    subject = subject.rstrip('.gz')
    #cmd = ['{}yara_indexer'.format(rundir), subject, '-o', '{}subject_index'.format(dumpdir)]
    cmd = ['yara_indexer', subject, '-o', '{}subject_index'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    #cmd = ['{}yara_mapper'.format(rundir), '{}subject_index'.format(dumpdir), '{}sim_1.fa'.format(dumpdir), '{}sim_2.fa'.format(dumpdir), '-o', '{}submap.sam'.format(dumpdir), '-e', '5', '-sa', 'omit', '-t', '20']
    # new yara
    #cmd = ['{}yara_mapper'.format(rundir), '{}subject_index'.format(dumpdir), '{}sim_1.fa'.format(dumpdir), '{}sim_2.fa'.format(dumpdir), '-o', '{}submap.sam'.format(dumpdir), '-e', '5', '-sa', 'omit', '-t', '10']
    cmd = ['yara_mapper', '{}subject_index'.format(dumpdir), '{}sim_1.fa'.format(dumpdir), '{}sim_2.fa'.format(dumpdir), '-o', '{}submap.sam'.format(dumpdir), '-e', '5', '-sa', 'omit', '-t', '10']
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)

    # read in sam file and count hits
    sf = pysam.Samfile('{}submap.sam'.format(dumpdir),'r')
    num_matches = float(0.0)
    for read in sf:
        if not read.is_unmapped:
            num_matches += 1.0
    print('submap res_indent = {}/{}'.format(num_matches, num_reads * 2))
    res_ident = num_matches/float(num_reads * 2)
    ff.write('res ident {} \n'.format(res_ident))
    sf.close()

    cmd = ['rm', '{}sim_1.fa'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', '{}sim_2.fa'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', '{}submap.sam'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = 'rm {}subject_index*'.format(dumpdir)
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT, shell=True)

    # TODO read/benchmark what identity is still "valid"
    accepted = (False, float(0.8))
    best_res_ident = float(0.0)

    if res_ident > accepted[1]:
        accepted = (True, res_ident)
    elif res_ident > best_res_ident:
        best_res_ident = res_ident
    if accepted[0]:
        ff.write('Acceptor accepted [{0:.7s}% | {1:.7s}%]. {2:.7s}% similarity of reported acceptor to origin in: {3}\n'.format(
            str(original_ident*100), str((original_ident - accepted[1])*100), str(accepted[1]*100), os.path.basename(file_)))
    else:
        ff.write('Acceptor failed [{0:.7s}% | {1:.7s}%]. {2:.7s}% similarity of reported acceptor to origin in: {3}\n'.format(
            str(original_ident*100), str((original_ident - best_res_ident)*100), str(best_res_ident*100), os.path.basename(file_)))

    cmd = ['rm', '{}accVar.fa'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', '{}accVar.fa.fai'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', '{}accVar_raw.fa'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', '{}accVar_part1.fa'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', '-f', '{}accVar_part2.fa'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', '{}accVar_part1.vcf'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', '{}accVar_part2.vcf'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', '{}acc_part1.fa'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', '{}acc_part2.fa'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', '{}acc_part1.fa.fai'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', '{}acc_part2.fa.fai'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', refacc]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', subject]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    return accepted[0]


def blastd(dir_, giDon, giDons, ff, file_, refdir, d_snp, d_sir, d_msis):

    dumpdir = dir_
    #dumpdir = './'
    rundir = os.path.dirname(sys.argv[0])
    if (rundir != ''):
        rundir += '/'

    # getting variated donor extraction
    cmd = ['find', refdir, '-type', 'f', '-regextype', 'posix-extended', '-iregex', '^{}{}.*.fasta.gz$'.format(refdir, giDon)]
    refdon = str(sp.check_output(cmd), 'utf-8').strip()
    cmd = 'gzip -fdk {}'.format(refdon)
    sp.check_call(cmd, shell=True)
    refdon = refdon.rstrip('.gz')
    # cut = first base that stems from insert
    fasta_dir = os.path.join(os.path.dirname(dir_), '../fasta')
    with open(os.path.join(fasta_dir, '{}.fa'.format(os.path.basename(file_).split('.')[0]))) as f:
        header = f.readline().strip()
    cut = int(re.match(exp, header).group(4))
    #end = int(re.match(tsv_exp, str(os.path.basename(file_))).group(7))
    null = open(os.devnull, 'w')
    print (file_, cut)
    #cmd = ['{}sak'.format(rundir), refdon, '-i', '{}-{}'.format(cut - 1,cut + 28000 - 1), '-o', '{}cache.fa'.format(dumpdir)]
    cmd = ['sak', refdon, '-i', '{}-{}'.format(cut - 1,cut + 28000 - 1), '-o', '{}cache.fa'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    #cmd = ['{}mason_variator'.format(rundir), '-ir', '{}cache.fa'.format(dumpdir), '-of', '{}donVar.fa'.format(dumpdir), '-ov', '{}var.vcf'.format(dumpdir), '--snp-rate', d_snp, '--small-indel-rate', d_sir, '--max-small-indel-size', d_msis]
    cmd = ['mason_variator', '-ir', '{}cache.fa'.format(dumpdir), '-of', '{}donVar.fa'.format(dumpdir), '-ov', '{}var.vcf'.format(dumpdir), '--snp-rate', d_snp, '--small-indel-rate', d_sir, '--max-small-indel-size', d_msis]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    # get original similarity
    cmd = ['blastn', '-query', '{}donVar.fa'.format(dumpdir), '-subject', refdon]
    m = re.match(fra, str(sp.check_output(cmd), 'utf-8'))
    try:
        frac1 = float(m.group(1))
        frac2 = float(m.group(2))
        original = frac1/frac2
    except AttributeError:
        original = float(0.0)
    accepted = (False, float(0.8))
    # in case the original donor was not reported, find best matching (highest BLAST score) donor among the reported ones
    bestBlastScore = 0.0
    #for reported in giDons:
    #print('Checking {} | {}'.format(giDon, reported))
    cmd = ['find', refdir, '-type', 'f', '-regextype', 'posix-extended', '-iregex', '^{}{}.*.fasta.gz$'.format(refdir, giDons)] #reported
    subject = str(sp.check_output(cmd), 'utf-8').strip()
    cmd = 'gzip -fdk {}'.format(subject)
    sp.check_call(cmd, shell=True)
    subject = subject.rstrip('.gz')
    #cmd = ['blastn', '-query', 'donVar.fa', '-subject', subject]
    #m = re.match(fra, str(sp.check_output(cmd)))
    #print ('subject ', subject)
    cmd = ['blastn', '-query', '{}donVar.fa'.format(dumpdir), '-subject', subject, '-outfmt', '10 nident length sstart send', '-max_target_seqs', '2']
    blastout = str(sp.check_output(cmd), 'utf-8').split('\n')[0]
    #print ('blastout ', blastout) 
    blast_sdon = int(0)
    blast_edon = int(0)
    don_length_cov = float(0.0)
    res = float(0.0)

    # TODO check alignment length (95%(?) of original fragment length)

    #try:
    if blastout:
        frac1 = float(blastout.split(',')[0])
        frac2 = float(blastout.split(',')[1])
        res = frac1/frac2
        don_length_cov = frac2/28000.0
        ff.write('don blast output ident {} length {} sstart {} send {}'.format(blastout.split(',')[0], blastout.split(',')[1], blastout.split(',')[2], blastout.split(',')[3]))
        ff.write('don length coverage {}'.format(don_length_cov))
    #except AttributeError:
        #res = float(0.0)
    if res > accepted[1] and don_length_cov > 0.94:
        accepted = (True, res)
        blast_sdon = int(blastout.split(',')[2])
        blast_edon = int(blastout.split(',')[3])
    elif res > bestBlastScore and don_length_cov > 0.94:
        bestBlastScore = res
        blast_sdon = int(blastout.split(',')[2])
        blast_edon = int(blastout.split(',')[3])
        
    if accepted[0]:
        ff.write('Donor accepted [{0:.7s}% | {1:.7s}%]. {2:.7s}% similarity of reported donor to origin in: {3}\n'.format(
            str(original*100), str((original-accepted[1])*100), str(accepted[1]*100), os.path.basename(file_)))
    else:
        ff.write('Donor failed [{0:.7s}% | {1:.7s}%]. {2:.7s}% similarity of reported donor to origin in: {3}\n'.format(
            str(original*100), str((original-bestBlastScore)*100), str(bestBlastScore*100), os.path.basename(file_)))
    cmd = ['rm', '{}cache.fa'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', '{}cache.fa.fai'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', '{}donVar.fa'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', '{}var.vcf'.format(dumpdir)]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', refdon]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    cmd = ['rm', subject]
    sp.check_call(cmd, stdout=null, stderr=sp.STDOUT)
    return [accepted[0], blast_sdon, blast_edon]

def blastad(dir_, giAcc, giAccs, giDon, giDons, ff, file_, refdir, a_snp, a_sir, a_svir, d_snp, d_sir, d_msis, insertPos):
    ff.write('Acceptor and donor failed in: {}\n'.format(
        os.path.basename(file_)))
    acc = blasta(dir_, giAcc, giAccs, ff, file_, refdir, a_snp, a_sir, a_svir, insertPos)
    [don, blast_sdon, blast_edon] = blastd(dir_, giDon, giDons, ff, file_, refdir, d_snp, d_sir, d_msis)
    return [acc, don, blast_sdon, blast_edon]


def foundHGT(header, df_hgt, pos_variance = 20, blast_don_start = 0, blast_don_end = 0, blast_acc = False):
    
    global exp
    accPos = int(re.match(exp, header).group(3))
    donPos_start = int(re.match(exp, header).group(4))
    if (blast_don_start > 0):
        donPos_start = int(blast_don_start)
    donPos_end = donPos_start + int(re.match(exp, header).group(5)) - 1
    if (blast_don_end > 0):
        donPos_end = int(blast_don_end)
    found_hgt = False
    #print (hgt_file)
    #print (accPos, donPos_start, donPos_end)
    # TODO differentiate between true hits and blast hits to get the correct ground truth for bp precision calculation
    for line in df_hgt.iterrows():
        line = line[1] # pandas returns tuple, we need the actual values
        #print (fields)
        #print (abs(accPos - int(fields[0])), abs(donPos_start - int(fields[4])), abs(donPos_end - int(fields[5])))
        #0=AS 1=AE 2=MC  3=BS:MC 4=DS 5=DE 6=MC 7=Split 8=PS-S 9=PS-W  10=Phage 11=BS:MC 12=BS:PS-S 13=BS:PS-W
        # thresholds as in Daisy paper: (3)<=5 or >=95, (11,12,13)>=95
        # other thresholds: abs(accPos-(0))<=10, abs(donPos_start-(4))<=10, abs(donPos_end-(5))<=10
        # Skip acceptor if it is a blast acceptor bc it is basically not possible to get the true acceptor region there...
        # (so a blast acceptor is treated like wild card)
        if not blast_acc and abs(accPos - int(line['AS'])) > pos_variance:
            continue
        if int(line['BS:MC']) > 5 and int(line['BS:MC']) < 95:
            continue
        if abs(donPos_start - int(line['DS'])) > pos_variance:
            continue
        if abs(donPos_end - int(line['DE'])) > pos_variance:
            continue
        if int(line['BS:MC.1']) < 95 or int(line['BS:PS-S']) < 95 or int(line['BS:PS-W']) < 95:
            continue
        found_hgt = True
        #print (found_hgt)
        return found_hgt
    return found_hgt


def neg_foundHGT(df_hgt):
    
    # Returns true if any HGT with thresholds applied is reported, i.e. entries with t>95 or t<5, see below and Daisy thresholds
    found_hgt = False
    for line in df_hgt.iterrows():
        line = line[1]
        if int(line['BS:MC']) > 5 and int(line['BS:MC']) < 95:
            continue
        if int(line['BS:MC.1']) < 95 or int(line['BS:PS-S']) < 95 or int(line['BS:PS-W']) < 95:
            continue
        found_hgt = True
        return found_hgt
    return found_hgt


def pos_overallStats(hgt_dir, refdir, stats_file, a_snp, a_sir, a_svir, d_snp, d_sir, d_msis):

    global tsv_exp
    #st_file = os.path.join(dir_, stats_file)
    st_file = stats_file
    #print st_file
    with open(st_file, 'w') as ff:
        fileList = glob.glob(os.path.join(hgt_dir, '*.tsv'))
        #fileList = glob.glob(os.path.join(dir_, '*.candidates'))
        # sort hgt_eval files according to simulation number
        fileList.sort(key=lambda x: (int(os.path.basename(x).split('_')[1].split('.tsv')[0])))
        # Counter for # simulation, donald pairs, donald blast pairs, daisy pairs, daisy blast pairs
        c_sims = 0 # nr (successful) simulations
        c_pairs = 0 # total nr pairs
        c_dnld_tp = 0 # correct pairs identified by Donald
        c_dnld_bltp = 0 # correct blast pairs (both acc+don blast)
        c_dnld_bltp_acc = 0 # correct blast pairs (only acc blast, donOrig)
        c_dnld_bltp_don = 0 # correct blast pairs (only don blast, accOrig)
        c_dnld_fp = 0 # wrong pairs
        c_dsy_tp = 0 # correct pairs identified/supported by Daisy
        c_dsy_bltp = 0 # correct blast pairs (both acc+don blast)
        c_dsy_bltp_acc = 0 # correct blast pairs (only acc blast, donOrig)
        c_dsy_bltp_don = 0 # correct blast pairs (only don blast, accOrig)
        c_dsy_tn = 0 # false pairs not supported by Daisy
        c_dsy_fp = 0 # wrong reported hgts
        c_dsy_blfp = 0 # wrong reported hgts
        c_dsy_fn = 0 # correct pairs not supported by Daisy
        c_dsy_blfn = 0 # blast-correct pairs not supported by Daisy
        missing = list()
        # initial/first simulation number (1)
        expSimNum = 1
        # examine simulation numbers to determine missing/crashed simulations (missing consecutive number)
        for file_ in fileList:
            c_sims += 1
            currSimNum = int(os.path.basename(file_).split('_')[1].split('.tsv')[0])
            if expSimNum != currSimNum:
                if (currSimNum - expSimNum > 1):
                    for x in range(expSimNum + 1, currSimNum):
                        missing.append(x)
                expSimNum = currSimNum
            # extract original giAccOrig, giDonOrig and the reported giAccRep, giDonRep passed on to Daisy from file name
            
            donald_tp = False
            donald_blastpos = False

            with open(os.path.join(hgt_dir, '../fasta', 'simulation_{}.fa'.format(currSimNum))) as f:
                header = f.readline().strip()
            giAccOrig = re.match(exp, header).group(1)
            giDonOrig = re.match(exp, header).group(2)
            accPos = int(re.match(exp, header).group(3))

            pairs = list()
            with open(os.path.join(hgt_dir, '../candidates/', 'simulation_{}_acceptors.txt'.format(currSimNum))) as f, open(os.path.join(hgt_dir, '../candidates/', 'simulation_{}_donors.txt'.format(currSimNum))) as g:
                acceptors = list()
                donors = list()
                for line in f:
                    acceptors.append(line.strip())
                for line in g:
                    donors.append(line.strip())
                from itertools import product
                pairs = list(product(acceptors, donors))
            
            df = pd.read_table(file_, comment = '#') if pairs != [] else None

            blastacc = dict()
            blastdon = dict()

            for pair in pairs:

                c_pairs += 1
                #giAcc = set()
                #giAcc.add(int(re.match(tsv_exp, str(os.path.basename(file_))).group(8)))
                #giDon = set()
                #giDon.add(int(re.match(tsv_exp, str(os.path.basename(file_))).group(9)))
                #giAcc = int(re.match(tsv_exp, str(os.path.basename(file_))).group(8))
                #giDon = int(re.match(tsv_exp, str(os.path.basename(file_))).group(9))
                giAcc = pair[0]
                giDon = pair[1]
                
                acc = giAcc == giAccOrig
                don = giDon == giDonOrig
                #print (giAcc, giAccOrig, acc)
                #print (giDon, giDonOrig, don)
                df_hgt = df.loc[(df['AN'] == giAcc) & (df['DN'] == giDon)]
                if acc and don:
                    donald_tp = True
                    # get Daisy stats
                    dsy_tp = foundHGT(header, df_hgt)
                    c_dsy_tp += int(dsy_tp)
                    c_dsy_fn += 1 - int(dsy_tp)
                elif not acc and not don:
                    # Determine blast ident of reported and original acc and don, accept for Donald if ident > 80%
                    if giAcc not in blastacc:
                        acc = blasta(hgt_dir, giAccOrig, giAcc, ff, str(os.path.basename(file_)), refdir, a_snp, a_sir, a_svir, accPos)
                        blastacc[giAcc] = acc
                    if giDon not in blastdon:
                        [don, blast_sdon, blast_edon] = blastd(hgt_dir, giDonOrig, giDon, ff, str(os.path.basename(file_)), refdir, d_snp, d_sir, d_msis)
                        blastdon[giDon] = [don, blast_sdon, blast_edon]
                    # if Blast ident accepted, Daisy can get a TP hit by confirming gis and positions or get a FN/FP if reporting anything else
                    # call foundHGT with higher position variance for blast hits
                    #TODO integrate blast_sdon, blast_edon in foundHGT
                    if blastacc[giAcc] and blastdon[giDon][0]:
                        donald_blastpos = True
                        #dsy_tp = foundHGT(header, df_hgt, 100, blast_sdon, blast_edon, True) #TODO check allowed breakpoint variance (100?)
                        dsy_tp = foundHGT(header, df_hgt, 100, blastdon[giDon][1], blastdon[giDon][2], True)
                        c_dsy_bltp += int(dsy_tp)
                        #c_dsy_bltp_acc += int(dsy_tp)
                        #c_dsy_bltp_don += int(dsy_tp)
                        c_dsy_tp += int(dsy_tp)
                        c_dsy_fn += 1 - int(dsy_tp)
                        c_dsy_blfn += 1 - int(dsy_tp)
                    else:
                        dsy_fp = neg_foundHGT(df_hgt)
                        c_dsy_fp += int(dsy_fp)
                        c_dsy_blfp += int(dsy_fp)
                        c_dsy_tn += 1 - int(dsy_fp)
                elif not acc:
                    if giAcc not in blastacc:
                        # Determine blast ident of reported and original acc, accept for Donald if ident > 80%
                        acc = blasta(hgt_dir, giAccOrig, giAcc, ff, str(os.path.basename(file_)), refdir, a_snp, a_sir, a_svir, accPos)
                        blastacc[giAcc] = acc
                    # if Blast ident accepted, Daisy can get a TP hit by confirming gis and positions or get a FN/FP if reporting anything else
                    if blastacc[giAcc]:
                        donald_blastpos = True
                        dsy_tp = foundHGT(header, df_hgt, 100, 0, 0, True) #TODO check allowed breakpoint variance (100?)
                        c_dsy_bltp_acc += int(dsy_tp)
                        c_dsy_tp += int(dsy_tp)
                        c_dsy_fn += 1 - int(dsy_tp)
                        c_dsy_blfn += 1 - int(dsy_tp)
                    else:
                        dsy_fp = neg_foundHGT(df_hgt)
                        c_dsy_fp += int(dsy_fp)
                        c_dsy_blfp += int(dsy_fp)
                        c_dsy_tn += 1 - int(dsy_fp)
                elif not don:
                    if giDon not in blastdon:
                        # Determine blast ident of reported and original don, accept for Donald if ident > 80%
                        [don, blast_sdon, blast_edon] = blastd(hgt_dir, giDonOrig, giDon, ff, str(os.path.basename(file_)), refdir, d_snp, d_sir, d_msis)
                        blastdon[giDon] = [don, blast_sdon, blast_edon]
                    # if Blast ident accepted, Daisy can get a TP hit by confirming gis and positions or get a FN/FP if reporting anything else
                    #TODO integrate blast_sdon, blast_edon in foundHGT
                    if blastdon[giDon][0]:
                        donald_blastpos = True
                        dsy_tp = foundHGT(header, df_hgt, 100, blastdon[giDon][1], blastdon[giDon][2]) #TODO check allowed breakpoint variance (100?)
                        c_dsy_bltp_don += int(dsy_tp)
                        c_dsy_tp += int(dsy_tp)
                        c_dsy_fn += 1 - int(dsy_tp)
                        c_dsy_blfn += 1 - int(dsy_tp)
                    else:
                        dsy_fp = neg_foundHGT(df_hgt)
                        c_dsy_fp += int(dsy_fp)
                        c_dsy_blfp += int(dsy_fp)
                        c_dsy_tn += 1 - int(dsy_fp)
                else:
                    pass

            c_dnld_tp += int(donald_tp)
            c_dnld_bltp += int(donald_blastpos and not donald_tp)
            c_dnld_fp += 1 - int(donald_tp or donald_blastpos) if pairs != [] else 0

        print('=== HGT Sim Results ===')
        print('Number of simulations: {}'.format(c_sims))
        print('Number of pairs: {}'.format(c_pairs))
        print('=== Donald ===')
        print('Found {} TP, {} FP and {} FN out of {} simulations.'.format(c_dnld_tp+c_dnld_bltp, c_dnld_fp, c_sims-c_dnld_tp-c_dnld_bltp-c_dnld_fp, c_sims))
        print('From the TPs, there are {} blast-pairs, {} blast-accs, and {} blast-dons.'.format(c_dnld_bltp, c_dnld_bltp_acc, c_dnld_bltp_don))
        print('Resulting in a sensitivity of {0:.4s}'.format( str(
            float(c_dnld_tp+c_dnld_bltp) / float(c_sims))))

        print('=== Daisy ===')
        print('Found {} TP, {} TN, {} FP and {} FN out of {} pairs.'.format(c_dsy_tp, c_dsy_tn, c_dsy_fp, c_dsy_fn, c_pairs))
        print('From the TPs, there are {} blast-pairs, {} blast-accs, and {} blast-dons.'.format(c_dsy_bltp, c_dsy_bltp_acc, c_dsy_bltp_don))
        print('From the FPs, there are {} total blast-pairs.'.format(c_dsy_blfp))
        print('From the FNs, there are {} total blast-pairs.'.format(c_dsy_blfn))
        print('Resulting in a sensitivity of {0:.4s}'.format( str(
            float(c_dsy_tp + c_dsy_tn) / float(c_pairs)) if c_pairs > 0 else '0.0' ))
        print(('Following hgt sim files were not found (ommitting missing'
                ' files at the end): {}').format(missing))

        ff.write('=== HGT Sim Results ===\n')
        ff.write('Number of simulations: {}\n'.format(c_sims))
        ff.write('Number of pairs: {}\n'.format(c_pairs))
        ff.write('=== Donald ===\n')
        ff.write('Found {} TP, {} FP and {} FN out of {} simulations.\n'.format(c_dnld_tp+c_dnld_bltp, c_dnld_fp, c_sims-c_dnld_tp-c_dnld_bltp-c_dnld_fp, c_sims))
        ff.write('From the TPs, there are {} blast-pairs, {} blast-accs, and {} blast-dons.\n'.format(c_dnld_bltp, c_dnld_bltp_acc, c_dnld_bltp_don))
        ff.write('Resulting in a sensitivity of {0:.4s}\n'.format( str(
            float(c_dnld_tp+c_dnld_bltp) / float(c_sims))))

        ff.write('=== Daisy ===\n')
        ff.write('Found {} TP, {} TN, {} FP and {} FN out of {} pairs.\n'.format(c_dsy_tp, c_dsy_tn, c_dsy_fp, c_dsy_fn, c_pairs))
        ff.write('From the TPs, there are {} blast-pairs, {} blast-accs, and {} blast-dons.\n'.format(c_dsy_bltp, c_dsy_bltp_acc, c_dsy_bltp_don))
        ff.write('From the FPs, there are {} total blast-pairs.\n'.format(c_dsy_blfp))
        ff.write('From the FNs, there are {} total blast-pairs.\n'.format(c_dsy_blfn))
        ff.write('Resulting in a sensitivity of {0:.4s}\n'.format( str(
            float(c_dsy_tp + c_dsy_tn) / float(c_pairs)) if c_pairs > 0 else '0.0' ))
        ff.write(('Following hgt sim files were not found (ommitting missing'
                ' files at the end): {}').format(missing))



def neg_overallStats(hgt_dir, refdir, stats_file, a_snp, a_sir, a_svir, d_snp, d_sir, d_msis):
#def neg_overallStats(dir_, refdir, stats_file, a_snp, a_sir, a_svir):

    global tsv_exp
    #st_file = glob.glob(os.path.join(dir_, stats_file))
    st_file = stats_file
    with open(st_file, 'w') as ff:
        fileList = glob.glob(os.path.join(hgt_dir, '../candidates/', '*_candidates.tsv'))
        fileList.sort(key=lambda x: int(os.path.basename(x).split('_')[1]))
        c_sims = 0 # nr (successful) simulations
        missingSim = list()
        expSimNum = int(os.path.basename(fileList[0]).split('_')[1])
        for file_ in fileList:
            currSimNum = int(os.path.basename(file_).split('_')[1])
            if expSimNum != currSimNum:
                for x in range(expSimNum, currSimNum):
                    missingSim.append(x)
                expSimNum = currSimNum
            expSimNum += 1

        fileList = glob.glob(os.path.join(hgt_dir, '*.tsv'))
        fileList.sort(key=lambda x: (int(os.path.basename(x).split('_')[1].split('.tsv')[0])))
        # Counter for # simulation, donald pairs, donald blast pairs, daisy pairs, daisy blast pairs
        c_pairs = 0.0 # total nr pairs
        c_dnld_fp = 0.0 # wrong pairs
        c_dnld_tp = 0.0 # no pairs found
        c_dsy_fp = 0.0 # wrong reported hgts
        missing = list()
        # initial/first simulation number (most likely 1)
        #expSimNum = int(os.path.basename(fileList[0]).split('_')[3])
        expSimNum = 1
        # examine simulation numbers to determine missing/crashed simulations (missing consecutive number)
        for file_ in fileList:
            c_sims += 1
            currSimNum = int(os.path.basename(file_).split('_')[1].split('.tsv')[0])
            if expSimNum != currSimNum:
                if (currSimNum - expSimNum > 1):
                    for x in range(expSimNum + 1, currSimNum):
                        missing.append(x)
                expSimNum = currSimNum

            pairs = list()
            with open(os.path.join(hgt_dir, '../candidates/', 'simulation_{}_acceptors.txt'.format(currSimNum))) as f, open(os.path.join(hgt_dir, '../candidates/', 'simulation_{}_donors.txt'.format(currSimNum))) as g:
                acceptors = list()
                donors = list()
                for line in f:
                    acceptors.append(line.strip())
                for line in g:
                    donors.append(line.strip())
                from itertools import product
                pairs = list(product(acceptors, donors))

            df = pd.read_table(file_, comment = '#') if pairs != [] else None

            b_dnld_tp = True

            for pair in pairs:
                giAcc = pair[0]
                giDon = pair[1]
                if giAcc == giDon:
                    continue
                b_dnld_tp = False
                c_pairs += 1
                c_dnld_fp += 1
                df_hgt = df.loc[(df['AN'] == giAcc) & (df['DN'] == giDon)]
                dsy_fp = neg_foundHGT(df_hgt)
                c_dsy_fp += int(dsy_fp)

            c_dnld_tp += int(b_dnld_tp)

        print('=== HGT Sim Results ===')
        print('Number of simulations: {}'.format(c_sims))
        print('Number of TP (at most one candidates found): {}'.format(int(c_dnld_tp)))
        print('Number of pairs inducing a Daisy call: {}'.format(int(c_dnld_fp)))
        print('Resulting in a Donald specificity of {0:.4s}'.format( str(
            float(c_dnld_tp) / float(c_sims))))

        print('=== Daisy ===')
        print('Found {} FP out of {} pairs.'.format(int(c_dsy_fp), int(c_pairs)))
        print('Resulting in a Daisy specificity of {0:.4s}'.format( str(
            float(c_pairs - c_dsy_fp) / float(c_pairs)) if c_pairs > 0 else '1.0'))
        print(('Following candidate files were not found (ommitting missing'
                ' files at the end): {}').format(missingSim))
        print(('Following hgt sim files were not found (ommitting missing'
                ' files at the end): {}').format(missing))

        ff.write('=== HGT Sim Results ===\n')
        ff.write('Number of simulations: {}\n'.format(c_sims))
        ff.write('Number of TP (at most one candidate found): {}\n'.format(int(c_dnld_tp)))
        ff.write('Number of pairs inducing a Daisy call: {}\n'.format(int(c_dnld_fp)))
        ff.write('Resulting in a Donalds specificity of {0:.4s}\n'.format( str(float(c_dnld_tp) / float(c_sims))))

        ff.write('=== Daisy ===\n')
        ff.write('Found {} FP out of {} pairs.\n'.format(int(c_dsy_fp), int(c_pairs)))
        ff.write('Resulting in a specificity of {0:.4s}\n'.format( str( float(c_pairs - c_dsy_fp) / float(c_pairs)) if c_pairs > 0  else '1.0'))
        ff.write(('Following candidate files were not found (ommitting missing'
                ' files at the end): {}\n').format(missingSim))
        ff.write(('Following hgt sim files were not found (ommitting missing'
                ' files at the end): {}').format(missing))



if __name__ == '__main__':
    import argparse
    
    def _valid_int(i):
        try:
            return str(int(i))
        except:
            raise argparse.ArgumentTypeError('{} is not an int.'.format(i))

    def _valid_float(f):
        try:
            return str(float(f))
        except:
            raise argparse.ArgumentTypeError('{} is not a float.'.format(f))

    parser = argparse.ArgumentParser(
        description='Get statistics from candidates file',add_help=False)
    required = parser.add_argument_group('required arguments')
    optional = parser.add_argument_group('optional arguments')
    required.add_argument('dir', type=str,
                        help='Directory containing hgt_eval files')
    required.add_argument('refdir', type=str,
                        help='Directory containing reference FASTA files')
    optional.add_argument('-h', '--help', action='help', 
                        help='Show this help message and exit')
    optional.add_argument('-rep', nargs='?', type=str, default='stats.txt',
                        help='File to write report')
    optional.add_argument('-a_snp', nargs='?', type=_valid_float, default='0.01',
                        help='Mason variator acceptor SNP rate (Default: %(default)s)')
    optional.add_argument('-a_sir', nargs='?', type=_valid_float, default='0.001',
                        help='Mason variator acceptor small indel rate (Default: %(default)s)')
    optional.add_argument('-a_svir', nargs='?', type=_valid_float, default='0.00001',
                        help='Mason variator acceptor sv indel rate (Default: %(default)s)')
    optional.add_argument('-d_snp', nargs='?', type=_valid_float, default='0.001',
                        help='Mason variator donor SNP rate (Default: %(default)s)')
    optional.add_argument('-d_sir', nargs='?', type=_valid_float, default='0.001',
                        help='Mason variator donor small indel rate (Default: %(default)s)')
    optional.add_argument('-d_msis', nargs='?', type=_valid_int, default='4',
                        help='Mason variator donor max small indel size (Default: %(default)s)')
    optional.add_argument('-neg', action='store_true', default=False, help='Negative simulation evaluation')
    args = parser.parse_args()
    print(args)
    if args.neg:
        neg_overallStats(args.dir, args.refdir, args.rep, args.a_snp, args.a_sir, args.a_svir, args.d_snp, args.d_sir, args.d_msis)
    else:
        pos_overallStats(args.dir, args.refdir, args.rep, args.a_snp, args.a_sir, args.a_svir, args.d_snp, args.d_sir, args.d_msis)
