#!/usr/bin/env python3
# -*- coding: utf-8 -*-
__authors__ = ["Henning Schiebenhoefer", "Juliane Schmachtenberg", "Tobias Marschall", "Marcel Martin",
               "Johannes Köster"]
__copyright__ = "Copyright 2015, Johannes Köster"
__email__ = "koester@jimmy.harvard.edu"
__license__ = "MIT"

import hashlib
import os
import re
import sqlite3

import pandas as pd
import pytest
import shutil
import subprocess
import sys

from os.path import join
from shutil import which
from pathlib import Path


from utils.create_summary import create_summary
from utils.input_output import load_yaml

from utils.vars import PROPHANE_DIR

if not which("snakemake"):
    raise Exception("snakemake not in PATH. For testing, install snakemake with "
                    "'pip install -e .'. You should do this in a separate environment "
                    "(via conda or virtualenv).")


def dpath(path):
    """
    get path to a data file (relative to the directory this
    test lives in)
    """
    return os.path.realpath(join(os.path.dirname(__file__), path))


def md5sum(filename):
    data = open(filename, 'rb').read()
    return hashlib.md5(data).hexdigest()


def is_ci():
    return "CI" in os.environ


ci = pytest.mark.skipif(not is_ci(), reason="not in CI")


def copy(src, dst):
    if os.path.isdir(src):
        shutil.copytree(src, os.path.join(dst, os.path.basename(src)))
    else:
        shutil.copy(src, dst)


def run_prophane(test_resource_dir, job_cfg_file, shouldfail=False, no_tmpdir=False, check_md5=True, cores=3,
                 check_for_expected_results=True, check_for_expected_dbs=False, snakemake_args=None):
    """
    Test the Prophane on 'input' folder in path.
    There must be a subdirectory named
    expected-results in the path.
    """
    if snakemake_args is None:
        snakemake_args = []
    expected_results_dir = join(os.path.dirname(__file__), test_resource_dir, 'expected-results')
    expected_dbs_dir = join(os.path.dirname(__file__), test_resource_dir, 'expected-dbs')
    prophane_executable = join(PROPHANE_DIR, 'prophane.py')
    cores = str(cores)

    if check_for_expected_results:
        assert os.path.exists(expected_results_dir) and os.path.isdir(
            expected_results_dir), '{} does not exist'.format(expected_results_dir)
    if check_for_expected_dbs:
        assert os.path.exists(expected_dbs_dir) and os.path.isdir(
            expected_dbs_dir), '{} does not exist'.format(expected_dbs_dir)

    job_cfg = load_yaml(job_cfg_file)
    jobdir = job_cfg["output_dir"]
    db_dir = job_cfg["db_base_dir"]

    workdir = test_resource_dir if no_tmpdir else jobdir

    # run prophane
    cmd = [prophane_executable, "run",
           job_cfg_file,
           "--directory", workdir,
           "--cores", cores,
           "--stats", "stats.txt",
           *snakemake_args]
    try:
        subprocess.check_call(cmd)
    except subprocess.CalledProcessError as e:
        print(f'Error in snakemake invocation: {e}', file=sys.stderr)
        success = False
    else:
        success = True

    if shouldfail:
        assert not success, "expected error on execution"
    else:
        assert success, "expected successful execution"
        if check_for_expected_results:
            compare_with_expected_results(jobdir, expected_results_dir, check_md5)

        if check_for_expected_dbs:
            compare_with_expected_results(db_dir, expected_dbs_dir, check_md5,
                                          # diamond libs are not reproducible
                                          skip_md5_patterns=[r'.*/diamond_db\.dmnd$'])


def assert_dataframes_equal(df1, df2):
    pd.testing.assert_frame_equal(df1, df2, check_dtype=False)


def assert_summary_content_equal(file1, file2):
    def read_summary(f):
        return pd.read_csv(f, sep="\t")

    dfs_ready_for_comparison = []
    for file in [file1, file2]:
        df = read_summary(file)
        # sort members
        df_groups = df[df['level'] == 'group'].set_index('#pg')
        pg_dict = df_groups['members_identifier'].to_dict()
        # group by protein group members
        groups = df.groupby(lambda x: pg_dict[df.loc[x, '#pg']])
        df_sorted = pd.DataFrame()
        # sort each protein group
        for pg_str, df_pg in groups:
            df_pg = df_pg.sort_values(['level', 'members_identifier'])
            df_sorted = df_sorted.append(df_pg)
        df = df_sorted
        # drop protein group number column, as this is not reproducible at the moment
        df = df.drop(columns="#pg")
        # reindex rows
        df = df.reset_index(drop=True)
        dfs_ready_for_comparison.append(df)
    try:
        assert_dataframes_equal(*dfs_ready_for_comparison)
    except AssertionError as e:
        raise AssertionError(
            f"""
summary.txt files are not equal!
left file:\t{file1}
right file:\t{file2}

File comparison result (first unequal column):
{e}
            """
        )


def test_assert_summary_content_equal():
    # equal summaries of different order
    f1, f2 = [join(os.path.dirname(__file__), "resources/test_is_summary_content_equal/equal", f)
              for f in ["summary1.txt", "summary2.txt"]]
    assert_summary_content_equal(f1, f2)  # , 'summary files [{}, {}] are equal'.format(f1, f2)


def test_assert_summary_content_not_equal():
    # unequal summaries
    f1, f2 = [join(os.path.dirname(__file__), "resources/test_is_summary_content_equal/unequal", f)
              for f in ["summary1.txt", "summary2.txt"]]
    with pytest.raises(AssertionError):
        assert_summary_content_equal(f1, f2)  # , 'summary files [{}, {}] are equal'.format(f1, f2)


def get_files_in_dir(expected_results_dir):
    lst_f = []
    for current_loop_dir, subdirs, list_expected_files in os.walk(expected_results_dir):
        for expected_filename in list_expected_files:
            expected_filepath = join(current_loop_dir, expected_filename)
            lst_f.append(expected_filepath)
    return lst_f


def assert_table_content_is_equal(expected_filepath, produced_filepath, sep="\t", sort_by_first_col=True, *args,
                                  **kwargs):
    dfs = [pd.read_csv(f, sep=sep, *args, **kwargs) for f in [expected_filepath, produced_filepath]]
    if sort_by_first_col:
        dfs = [df.sort_values(df.columns[0]).reset_index(drop=True) for df in dfs]
    try:
        assert_dataframes_equal(*dfs)
    except AssertionError as e:
        raise AssertionError(
            f"""
    gzipped tables are not equal!
    left file:\t{expected_filepath}
    right file:\t{produced_filepath}

    File comparison result (first unequal column):
    {e}
                """
        )


def lca_test_on_summary(test_ressource_dir, tmpdir, lca_method):
    job_dir = os.path.join(test_ressource_dir, 'job_dir', 'summary.txt')
    tax_map_dir = os.path.join(test_ressource_dir, 'job_dir', 'taxdump.map.gz')
    output = os.path.join(tmpdir, f"summary_{lca_method}.txt")
    path_to_script = os.path.abspath('utils/lca_determination/weighted_lca_on_full_summary.py')
    cmd = ["python3", path_to_script, "-r", job_dir, "-o", output, "-m", lca_method, "-x", tax_map_dir]
    return_code = subprocess.call(cmd)
    if return_code != 0:
        raise Exception(f"weighted_lca_on_full_summary failed for command '{' '.join(cmd)}'")


def compare_with_expected_results(result_dir, expected_results_dir, check_md5=True, skip_md5_patterns=None):
    lst_expected_files = get_files_in_dir(expected_results_dir)
    for expected_filepath in lst_expected_files:
        if os.path.basename(expected_filepath) == ".gitignore":
            continue
        rel_file_path = os.path.relpath(expected_filepath, expected_results_dir)
        produced_filepath = os.path.abspath(join(result_dir, rel_file_path))
        assert os.path.exists(
            produced_filepath), 'expected file "{}" not produced'.format(
            rel_file_path)
        if os.path.basename(expected_filepath) == "summary.txt":
            assert_summary_content_equal(expected_filepath, produced_filepath)
        # map files can have different md5sums although their content is equal. In addition, some maps (e.g. ncbi
        # tax2annot) have a random row order.
        elif os.path.basename(expected_filepath) == "protein_groups_db.sql":
            compare_sql_databases(expected_filepath, produced_filepath)
        elif os.path.basename(expected_filepath) in ["id2annot.map.gz", "acc2tax.map.gz", "tax2annot.map.gz"]:
            assert_table_content_is_equal(expected_filepath, produced_filepath, compression="gzip")
        elif check_md5:
            if skip_md5_patterns:
                if any([re.match(pat, rel_file_path) for pat in skip_md5_patterns]):
                    print(f"Skipping md5-check of '{rel_file_path}'")
                    continue
            assert md5sum(produced_filepath) == md5sum(expected_filepath), \
                'wrong result produced for file "{}"'.format(rel_file_path)


def get_values_to_compare(file, columns_to_ignore=None):
    rows = []
    with open(file, 'r') as f:
        for line in f:
            if line.startswith("#"):
                continue
            columns = line.split('\t')
            if columns_to_ignore:
                del columns[columns_to_ignore[0]:columns_to_ignore[1]]
            rows.append(columns)
    return rows


def compare_with_expected_results_ignore_comment_lines_and_qvalues(result_dir, expected_results_dir, files_to_compare):
    lst_expected_files = [expected_results_dir / f for f in files_to_compare]
    lst_result_files = [result_dir / f for f in files_to_compare]
    for expected_filepath, result_filepath in zip(lst_expected_files, lst_result_files):
        if os.path.basename(expected_filepath) == ".gitignore":
            continue
        assert os.path.exists(result_filepath), 'expected file "{}" not produced'.format(
            Path(result_filepath).name)
        columns_to_ignore = [2,4] if not 'hits' in str(result_filepath) else [10,12]
        expected_result = get_values_to_compare(expected_filepath, columns_to_ignore)
        produced_result = get_values_to_compare(result_filepath, columns_to_ignore)
        assert produced_result == expected_result, 'wrong result produced for file "{}"'.format(Path(result_filepath).name)


def compare_sql_table(conn_exp, conn_res):
    tableCompare = "SELECT name FROM sqlite_master WHERE type='table' order by name"
    cursor_exp = conn_exp.cursor()
    result_exp = cursor_exp.execute(tableCompare)
    cursor_res = conn_res.cursor()
    result_res = cursor_res.execute(tableCompare)
    for row_exp in result_exp:
        row_res = result_res.fetchone()
        assert row_res == row_exp


def compare_sql_table_entries(table, conn_exp, conn_res):
    entryCompare = f"SELECT * from {table}"
    cursor_exp = conn_exp.cursor()
    cursor_res = conn_res.cursor()
    result_exp = cursor_exp.execute(entryCompare)
    result_res = cursor_res.execute(entryCompare)
    for row_exp, row_res in zip(result_exp, result_res):
        assert row_exp == row_res


def compare_sql_databases(expected_filepath, produced_filepath):
    conn_exp = sqlite3.connect(expected_filepath)
    conn_res = sqlite3.connect(produced_filepath)
    compare_sql_table(conn_exp, conn_res)
    for table in ['ProteinGroupsQuant', 'ProteinGroupsProteins', 'ProteinGroupsInformation', 'ProteinAccSpectra', 'Samples']:
        compare_sql_table_entries(table, conn_exp, conn_res)


def test_full_analysis_dryrun(test_dbs_preprocessed, prophane_input, tmpdir, conda_dir):
    test_ressource_dir = 'resources/test_full_analysis_and_file_presence'
    config_file, job_dir = prophane_input(test_ressource_dir, str(tmpdir))
    snakemake_args = ["--dryrun", "--conda-prefix", conda_dir]
    run_prophane(test_ressource_dir, job_cfg_file=config_file, check_for_expected_results=False,
                 snakemake_args=snakemake_args)


def test_full_analysis_dryrun_wo_dbs(prophane_input, tmpdir, conda_dir):
    test_ressource_dir = 'resources/test_db_preprocessing'
    db_dir = tmpdir.join("test_dbs")
    db_dir.mkdir()
    config_file, job_dir = prophane_input(test_ressource_dir, str(tmpdir))
    snakemake_args = ["--dryrun", "--conda-prefix", conda_dir]
    run_prophane(test_ressource_dir, job_cfg_file=config_file, check_for_expected_results=False,
                 snakemake_args=snakemake_args)


def test_full_analysis_dryrun_wo_sample_groups(test_dbs_only_dl_files, prophane_input, tmpdir, conda_dir):
    test_resource_dir = 'resources/test_full_analysis_dryrun_wo_sample_groups'
    config_file, job_dir = prophane_input(test_resource_dir, str(tmpdir))
    snakemake_args = ["--conda-prefix", conda_dir, "parse_search_result_into_protein_groups_sql_db"]
    run_prophane(test_resource_dir, job_cfg_file=config_file, check_md5=True, snakemake_args=snakemake_args)


def test_full_analysis_and_file_presence(test_dbs_only_dl_files, prophane_input, tmpdir, conda_dir):
    test_ressource_dir = 'resources/test_full_analysis_and_file_presence'
    config_file, job_dir = prophane_input(test_ressource_dir, str(tmpdir))
    snakemake_args = ["--conda-prefix", conda_dir]
    run_prophane(test_ressource_dir, job_cfg_file=config_file, check_md5=False, snakemake_args=snakemake_args)


def test_mafft_on_single_member_protein_groups(test_dbs_only_dl_files, prophane_input, tmpdir, conda_dir):
    test_ressource_dir = 'resources/test_mafft_on_single_member_protein_groups'
    config_file, job_dir = prophane_input(test_ressource_dir, str(tmpdir))
    snakemake_args = ["--conda-prefix", conda_dir, "report_mafft"]
    run_prophane(test_ressource_dir, job_cfg_file=config_file, check_md5=False, snakemake_args=snakemake_args,
                 check_for_expected_results=False)


def test_lca_on_maps_with_spectra(prophane_test_dbs, prophane_input, tmpdir, conda_dir):
    from utils.lca_determination.determine_lca import create_lca

    task_to_n_lca_levels = {'foam': 6, 'dbcan': 2, 'resfams_full': 5, 'eggnog': 5, 'pfams': 5, 'tigrfams': 4,
                            'custom_map': 3,
                            'ncbi_nr': 7, 'uniprot_complete': 7, 'uniprot_sp': 7, 'uniprot_tr': 7, 'resfams_core': 5}
    test_ressource_dir = Path(dpath('resources/test_lca_on_maps_with_spectra'))
    config_file, job_dir = prophane_input(test_ressource_dir, str(tmpdir))
    job_dir = Path(job_dir)
    expected_results_dir = test_ressource_dir / 'expected-results/'
    pg_db = job_dir / 'pgs/protein_groups_db.sql'
    tax_map = job_dir / 'input/taxdump.map.gz'
    method = 'lca'
    for result_map in (job_dir / "tasks").glob("*.map"):
        lca_out = job_dir / "tasks" / (result_map.stem + '.lca')
        lca_support_out = job_dir / "tasks" / (result_map.stem + '.lcasupport')
        n_lca_levels = task_to_n_lca_levels[str(result_map.stem).split('_on_')[1].split('.')[0]]
        is_taxonomic_annotation = result_map.stem.startswith('tax_')
        create_lca(task_map=result_map, pg_db=pg_db, output_lca=lca_out, output_lca_support=lca_support_out,
                   method=method, is_taxonomic_annotation=is_taxonomic_annotation, n_lca_levels=n_lca_levels,
                   path_to_taxmap=tax_map, threshold=1)
    compare_with_expected_results(job_dir, expected_results_dir)


def test_summary_on_maps_with_spectra(prophane_test_dbs, prophane_input, tmpdir, conda_dir):
    test_ressource_dir = Path(dpath('resources/test_summary_on_maps_with_spectra/'))
    config_file, job_dir = prophane_input(test_ressource_dir, str(tmpdir))
    target = 'summary.txt'
    snakemake_args = ["--conda-prefix", conda_dir, target]
    run_prophane(test_ressource_dir, job_cfg_file=config_file, check_md5=False,
                 snakemake_args=snakemake_args)


def test_summary_on_maps_with_multiple_sample_groups(prophane_test_dbs, prophane_input, tmpdir, conda_dir):
    test_ressource_dir = Path(dpath('resources/test_summary_on_maps_with_multiple_sample_groups/'))
    config_file, job_dir = prophane_input(test_ressource_dir, str(tmpdir))
    target = 'summary.txt'
    snakemake_args = ["--conda-prefix", conda_dir, target]
    run_prophane(test_ressource_dir, job_cfg_file=config_file, check_md5=False, snakemake_args=snakemake_args)


def test_cazy_db_result_mapping(prophane_test_dbs, prophane_input, tmpdir, conda_dir):
    test_ressource_dir = Path(dpath('resources/test_cazy_db_result_mapping/'))
    config_file, job_dir = prophane_input(test_ressource_dir, str(tmpdir))
    targets = ["tasks/fun_annot_by_hmmer-latest_on_dbcan.vv8.task0.map",
               "tasks/fun_annot_by_hmmer-latest_on_dbcan.vv8.task1.map"]
    snakemake_args = ["--conda-prefix", conda_dir] + targets
    run_prophane(test_ressource_dir, job_cfg_file=config_file, check_md5=True, snakemake_args=snakemake_args)


def test_democratic_lca_on_maps_with_spectra(prophane_test_dbs, prophane_input, tmpdir, conda_dir):
    from utils.lca_determination.determine_lca import create_lca

    task_to_n_lca_levels = {'foam': 6, 'dbcan': 2, 'resfams_full': 5, 'eggnog': 5, 'pfams': 5, 'tigrfams': 4,
                            'custom_map': 3,
                            'ncbi_nr': 7, 'uniprot_complete': 7, 'uniprot_sp': 7, 'uniprot_tr': 7, 'resfams_core': 5}
    test_ressource_dir = Path(dpath('resources/test_democratic_lca_on_maps_with_spectra'))
    config_file, job_dir = prophane_input(test_ressource_dir, str(tmpdir))
    job_dir = Path(job_dir)
    expected_results_dir = test_ressource_dir / 'expected-results/'
    pg_db = job_dir / 'pgs/protein_groups_db.sql'
    tax_map = job_dir / 'input/taxdump.map.gz'
    method = 'democratic_lca'
    for result_map in (job_dir / "tasks").glob("*.map"):
        lca_out = job_dir / "tasks" / (result_map.stem + '.lca')
        lca_support_out = job_dir / "tasks" / (result_map.stem + '.lcasupport')
        n_lca_levels = task_to_n_lca_levels[str(result_map.stem).split('_on_')[1].split('.')[0]]
        is_taxonomic_annotation = result_map.stem.startswith('tax_')
        create_lca(task_map=result_map, pg_db=pg_db, output_lca=lca_out, output_lca_support=lca_support_out,
                   method=method, is_taxonomic_annotation=is_taxonomic_annotation, n_lca_levels=n_lca_levels,
                   path_to_taxmap=tax_map, threshold=None)
    compare_with_expected_results(job_dir, expected_results_dir)


def test_seqs_all_faa_from_generic_table(test_dbs_only_dl_files, prophane_input, tmpdir, conda_dir):
    test_ressource_dir = 'resources/test_seqs_all_faa_from_generic_table'
    config_file, job_dir = prophane_input(test_ressource_dir, str(tmpdir))
    snakemake_args = ["--conda-prefix", conda_dir, 'seqs/all.faa']
    run_prophane(test_ressource_dir, job_cfg_file=config_file, check_md5=True, snakemake_args=snakemake_args)


def test_seqs_all_faa_from_fasta_gz(test_dbs_only_dl_files, prophane_input, tmpdir):
    test_ressource_dir = 'resources/test_seqs_all_faa_from_fasta_gz'
    config_file, job_dir = prophane_input(test_ressource_dir, str(tmpdir))
    snakemake_args = ['seqs/all.faa']
    run_prophane(test_ressource_dir, job_cfg_file=config_file, check_md5=True, snakemake_args=snakemake_args)


def test_parse_generic_table(f_proteomic_search_result, tmpdir):
    test_ressource_dir = os.path.abspath('tests/resources/test_parse_generic_table')
    expected_results_dir = os.path.join(test_ressource_dir, 'expected-results')
    decoy_exp = 'DECOY$'
    style = os.path.abspath('styles/generic.yaml')
    input_table, job_dir = f_proteomic_search_result(test_ressource_dir, tmpdir)
    output_db = os.path.join(job_dir, 'protein_groups_db.sql')
    required_sample_descriptors = ["sample B::replicate 1", "sample A::replicate 1"]

    from utils.search_result_parsing import ProteomicSearchResultParser
    protein_parser = ProteomicSearchResultParser(input_table, style, output_db, decoy_exp,
                                                 required_sample_descriptors)
    protein_parser.create_pg_sql_db()
    compare_with_expected_results(job_dir, expected_results_dir)


def test_parse_from_scaffold(tmpdir, f_proteomic_search_result):
    test_ressource_dir = dpath('resources/test_parse_scaffold_table')
    expected_results_dir = os.path.join(test_ressource_dir, 'expected-results')
    decoy_exp = 'DECOY$'
    style = Path(PROPHANE_DIR) / 'styles/scaffold_4_8_x.yaml'
    input_table, job_dir = f_proteomic_search_result(test_ressource_dir, tmpdir)
    output_db = os.path.join(job_dir, 'protein_groups_db.sql')
    required_sample_descriptors = ["sample A::replicate 1"]

    from utils.search_result_parsing import ProteomicSearchResultParser
    protein_parser = ProteomicSearchResultParser(input_table, style, output_db, decoy_exp,
                                                 required_sample_descriptors)
    protein_parser.create_pg_sql_db()
    compare_with_expected_results(job_dir, expected_results_dir)


def test_parse_from_mztab(tmpdir, f_proteomic_search_result):
    test_ressource_dir = dpath('resources/test_parse_mztab')
    expected_results_dir = os.path.join(test_ressource_dir, 'expected-results')
    # decoy_exp = 'REV_$'
    decoy_exp = '^REV_|^CON_'
    style = dpath('../styles/mztab_1_0_0.yaml')
    input_table, job_dir = f_proteomic_search_result(test_ressource_dir, tmpdir)
    output_db = os.path.join(job_dir, 'protein_groups_db.sql')
    required_sample_descriptors = ["ms_run[10]", "ms_run[11]", "ms_run[12]", "ms_run[13]", "ms_run[14]", "ms_run[15]",
                                   "ms_run[16]", "ms_run[17]", "ms_run[18]", "ms_run[19]", "ms_run[1]", "ms_run[20]",
                                   "ms_run[21]", "ms_run[22]", "ms_run[23]", "ms_run[24]", "ms_run[25]", "ms_run[26]",
                                   "ms_run[27]", "ms_run[28]", "ms_run[2]", "ms_run[3]", "ms_run[4]", "ms_run[5]",
                                   "ms_run[6]", "ms_run[7]", "ms_run[8]", "ms_run[9]"]
    from utils.search_result_parsing import ProteomicSearchResultParser
    protein_parser = ProteomicSearchResultParser(input_table, style, output_db, decoy_exp, required_sample_descriptors)
    protein_parser.create_pg_sql_db()
    compare_with_expected_results(job_dir, expected_results_dir)


def test_parse_from_proteome_discoverer(tmpdir, f_proteomic_search_result):
    test_ressource_dir = Path(dpath('resources/test_parse_proteome_discoverer'))
    expected_results_dir = test_ressource_dir / 'expected-results'
    decoy_exp = 'REV_$'
    style = Path(dpath('../styles/proteome_discoverer_2_5.yaml'))
    input_table, job_dir = f_proteomic_search_result(test_ressource_dir, tmpdir)
    output_db = os.path.join(job_dir, 'protein_groups_db.sql')
    required_sample_descriptors = ['F1', 'F2', 'F3', 'F4', 'F5', 'F6', 'F7', 'F8', 'F9', 'F10', 'F11', 'F12', 'F13',
                                   'F14', 'F15', 'F16', 'F17', 'F18', 'F19', 'F20', 'F21', 'F22', 'F23', 'F24']

    from utils.search_result_parsing import ProteomicSearchResultParser
    protein_parser = ProteomicSearchResultParser(input_table, style, output_db, decoy_exp, required_sample_descriptors)
    protein_parser.create_pg_sql_db()
    compare_with_expected_results(job_dir, expected_results_dir)


def test_parse_from_proteome_discoverer_without_master_column(tmpdir, f_proteomic_search_result):
    test_ressource_dir = Path(dpath('resources/test_parse_proteome_discoverer_2'))
    expected_results_dir = test_ressource_dir / 'expected-results'
    decoy_exp = 'REV_$'
    style = Path(dpath('../styles/proteome_discoverer_2_5.yaml'))
    input_table, job_dir = f_proteomic_search_result(test_ressource_dir, tmpdir)
    output_db = os.path.join(job_dir, 'protein_groups_db.sql')
    required_sample_descriptors = ['F1', 'F2', 'F3', 'F4', 'F5', 'F6', 'F7', 'F8', 'F9', 'F10', 'F11', 'F12', 'F13',
                                   'F14', 'F15', 'F16', 'F17', 'F18', 'F19', 'F20', 'F21', 'F22', 'F23', 'F24']

    from utils.search_result_parsing import ProteomicSearchResultParser
    protein_parser = ProteomicSearchResultParser(input_table, style, output_db, decoy_exp, required_sample_descriptors)
    protein_parser.create_pg_sql_db()
    compare_with_expected_results(job_dir, expected_results_dir)


def test_parse_from_mzident1(tmpdir, f_proteomic_search_result):
    """
    :param tmpdir: tmp path
    :param f_proteomic_search_result: path to search results
    """
    test_ressource_dir = dpath('resources/test_mzIdent_parser')
    expected_results_dir = os.path.join(test_ressource_dir, 'expected-results')
    decoy_exp = 'DECOY$'
    style = dpath('../styles/mzident_1_2_0.yaml')
    input_table, job_dir = f_proteomic_search_result(test_ressource_dir, tmpdir)
    output_db = os.path.join(job_dir, 'protein_groups_db.sql')
    required_sample_descriptors = ['SPECTRADATA_1']

    from utils.search_result_parsing import ProteomicSearchResultParser
    protein_parser = ProteomicSearchResultParser(input_table, style, output_db, decoy_exp,
                                                 required_sample_descriptors)
    protein_parser.create_pg_sql_db()
    compare_with_expected_results(job_dir, expected_results_dir)


def test_parse_from_mzident2(tmpdir, f_proteomic_search_result):
    test_ressource_dir = dpath('resources/test_mzIdent_parser_2')
    expected_results_dir = os.path.join(test_ressource_dir, 'expected-results')
    decoy_exp = '^REV_'
    style = dpath('../styles/mzident_1_2_0.yaml')
    input_table, job_dir = f_proteomic_search_result(test_ressource_dir, tmpdir)
    output_db = os.path.join(job_dir, 'protein_groups_db.sql')
    required_sample_descriptors = ['MSB29562CplusBand_10 (F127143)']

    from utils.search_result_parsing import ProteomicSearchResultParser
    protein_parser = ProteomicSearchResultParser(input_table, style, output_db, decoy_exp,
                                                 required_sample_descriptors)
    protein_parser.create_pg_sql_db()
    compare_with_expected_results(job_dir, expected_results_dir)


def test_parse_from_mzident3(tmpdir, f_proteomic_search_result):
    test_ressource_dir = dpath('resources/test_mzIdent_parser_3')
    expected_results_dir = os.path.join(test_ressource_dir, 'expected-results')
    decoy_exp = '_REVERSED'
    style = dpath('../styles/mzident_1_2_0.yaml')
    input_table, job_dir = f_proteomic_search_result(test_ressource_dir, tmpdir)
    output_db = os.path.join(job_dir, 'protein_groups_db.sql')
    required_sample_descriptors = ['qExactive01819.mgf', 'qExactive2.mgf']

    from utils.search_result_parsing import ProteomicSearchResultParser
    protein_parser = ProteomicSearchResultParser(input_table, style, output_db, decoy_exp,
                                                 required_sample_descriptors)
    protein_parser.create_pg_sql_db()
    compare_with_expected_results(job_dir, expected_results_dir)


def test_parse_from_mpa_portable(tmpdir, f_proteomic_search_result):
    test_ressource_dir = 'tests/resources/test_parse_MPA_portable_table'
    required_sample_descriptors = ["sample1"]
    parse_pg_db_from_mpa_table_and_compare_with_expected_db(test_ressource_dir, f_proteomic_search_result, tmpdir,
                                                            required_sample_descriptors)


def test_parse_from_mpa_server(tmpdir, f_proteomic_search_result):
    test_ressource_dir = 'tests/resources/test_parse_MPA_server_table'
    required_sample_descriptors = ["sample1"]
    parse_pg_db_from_mpa_table_and_compare_with_expected_db(test_ressource_dir, f_proteomic_search_result, tmpdir,
                                                            required_sample_descriptors)


def parse_pg_db_from_mpa_table_and_compare_with_expected_db(test_ressource_dir, f_proteomic_search_result, tmpdir,
                                                            required_sample_descriptors):
    test_ressource_dir = os.path.abspath(test_ressource_dir)
    expected_results_dir = os.path.join(test_ressource_dir, 'expected-results')
    decoy_exp = 'DECOY$'
    style = os.path.abspath('styles/mpa_1_8_x.yaml')
    input_table, job_dir = f_proteomic_search_result(test_ressource_dir, tmpdir)
    output_db = os.path.join(job_dir, 'protein_groups_db.sql')
    from utils.search_result_parsing import ProteomicSearchResultParser
    protein_parser = ProteomicSearchResultParser(input_table, style, output_db, decoy_exp,
                                                 required_sample_descriptors)
    protein_parser.create_pg_sql_db()
    compare_with_expected_results(job_dir, expected_results_dir)


def test_parse_from_mpa_server_multisample_table(tmpdir, f_proteomic_search_result):
    test_ressource_dir = dpath('resources/test_parse_MPA_server_multisample_table')
    required_sample_descriptors = ["S13.mgf", "S20.mgf", "S23.mgf", "S6.mgf", "S7.mgf", "S25.mgf", "S27.mgf", "S28.mgf",
                                   "S34.mgf", "S36.mgf", "S44.mgf", "S51.mgf", "S61.mgf", "S65.mgf", "S72.mgf"]
    test_ressource_dir = os.path.abspath(test_ressource_dir)
    expected_results_dir = os.path.join(test_ressource_dir, 'expected-results')
    decoy_exp = 'DECOY$'
    style = dpath('../styles/mpa_server_multisample.yaml')
    input_table, job_dir = f_proteomic_search_result(test_ressource_dir, tmpdir)
    output_db = os.path.join(job_dir, 'protein_groups_db.sql')
    from utils.search_result_parsing import ProteomicSearchResultParser
    parser = ProteomicSearchResultParser(input_table, style, output_db, decoy_exp, required_sample_descriptors)
    parser.create_pg_sql_db()
    compare_with_expected_results(job_dir, expected_results_dir)


def test_full_analysis_and_file_presence_with_map_style_migration(
        prophane_test_dbs_before_map_gz, prophane_input, tmpdir, conda_dir
):
    test_dbs = prophane_test_dbs_before_map_gz
    test_ressource_dir = dpath('resources/test_full_analysis_and_file_presence')
    config_file, job_dir = prophane_input(test_ressource_dir, str(tmpdir), db_dir=test_dbs)
    snakemake_args = ["--conda-prefix", conda_dir]
    run_prophane(test_ressource_dir, job_cfg_file=config_file, check_md5=False, snakemake_args=snakemake_args)


def test_get_sequences_from_fasta():
    fastas = [dpath("resources/test_full_analysis_and_file_presence/job_dir/input/target_db.fasta")]
    accs = {"wasd", "A0A2S1TQ58", "A0A126QUZ5"}
    from utils.fasta_handling import get_sequences_from_fasta
    d_seqs = get_sequences_from_fasta(accs, fastas)
    assert "A0A2S1TQ58" in d_seqs
    assert "A0A126QUZ5" in d_seqs


def test_parsing_of_non_utf_encoded_file(tmpdir, f_proteomic_search_result):
    test_ressource_dir = dpath("resources/test_parse_scaffold_table_ISO-8859-15")
    expected_results_dir = os.path.join(test_ressource_dir, 'expected-results')
    decoy_exp = 'DECOY$'
    style = os.path.join(PROPHANE_DIR, 'styles/scaffold_4_8_x.yaml')
    input_table, job_dir = f_proteomic_search_result(test_ressource_dir, tmpdir)
    output_db = os.path.join(job_dir, 'protein_groups_db.sql')
    required_sample_descriptors = ["sample A::replicate 1"]

    from utils.search_result_parsing import ProteomicSearchResultParser
    protein_parser = ProteomicSearchResultParser(input_table, style, output_db, decoy_exp,
                                                 required_sample_descriptors)
    protein_parser.create_pg_sql_db()
    compare_with_expected_results(job_dir, expected_results_dir)


def test_db_setup(test_dbs_only_dl_files, prophane_input, tmpdir, conda_dir):
    test_ressource_dir = dpath('resources/test_db_preprocessing')
    config_file, _ = prophane_input(test_ressource_dir, str(tmpdir))
    snakemake_args = ["--conda-prefix", conda_dir, "db_setup"]
    run_prophane(test_ressource_dir, job_cfg_file=config_file, check_md5=False, check_for_expected_results=False,
                 check_for_expected_dbs=True, snakemake_args=snakemake_args)


def test_databases_init(prophane_test_dbs, tmpdir):
    from utils.input_output import get_yamls_in_path
    yamls = get_yamls_in_path(prophane_test_dbs)
    from utils.databases.api_class import Databases
    dbs = Databases(yamls)
    assert dbs.get_db_objects()


def test_group_lca_on_summary_file_05001(tmpdir):
    test_ressource_dir = os.path.abspath('tests/resources/test_group_lca_on_summary_05001')
    expected_results_dir = os.path.join(test_ressource_dir, 'expected-results')
    lca_test_on_summary(test_ressource_dir, tmpdir, 'lca')
    compare_with_expected_results(tmpdir, expected_results_dir)


def test_democratic_lca_on_summary_file(tmpdir):
    test_ressource_dir = os.path.abspath('tests/resources/test_democratic_lca_on_summary')
    expected_results_dir = os.path.join(test_ressource_dir, 'expected-results')
    lca_test_on_summary(test_ressource_dir, tmpdir, 'democratic_lca')
    compare_with_expected_results(tmpdir, expected_results_dir)


def test_create_emapper_command(tmpdir, conda_dir):
    from utils.emapper_utils import create_emapper_command
    expected_results_dir = os.path.abspath('tests/resources/test_emapper_command/expected-results')
    job_dir = os.path.join(tmpdir, 'job_dir/')
    os.mkdir(job_dir)
    output_cmd_file = job_dir + '/fun_annot_by_emapper_on_eggnog_task_0_md5sum.result-cmd.txt'
    output_yaml = job_dir + '/fun_annot_by_emapper_on_eggnog_task_0_md5sum.yaml'
    task_dict = {}
    task_dict['params'] = {'m': 'diamond'}
    task_dict['prog'] = 'emapper'
    task_dict['db_type'] = 'eggnog_v5'
    query_file = 'tests/resources/test_emapper_command/job_dir/test_queries.fa'
    prog_db_string = 'tests/test_dbs/eggnog_v5/5.0.2/emapper_dl/data'
    result_file_name = 'fun_annot_by_emapper_on_eggnog_task_0.result'
    task_dict['params']['block_size'] = 2.0

    create_emapper_command(output_cmd_file, output_yaml, task_dict, query_file, prog_db_string, result_file_name, 'path/to/output', 4)
    compare_with_expected_results(job_dir, expected_results_dir)


def test_eggnog_v_5(test_dbs_only_dl_files, prophane_input, tmpdir, conda_dir):
    test_ressource_dir = Path(dpath('resources/test_emapper'))
    expected_results_dir = test_ressource_dir / 'expected-results'
    config_file, job_dir = prophane_input(test_ressource_dir, str(tmpdir))
    job_dir = Path(job_dir)
    target = "tasks/fun_annot_by_emapper_v2_on_eggnog_v5.v5.0.2.task0.result"
    snakemake_args = ["--conda-prefix", conda_dir, target]
    run_prophane(test_ressource_dir, job_cfg_file=config_file, check_md5=False,
                 check_for_expected_results=False, snakemake_args=snakemake_args)
    compare_with_expected_results_ignore_comment_lines_and_qvalues(
        job_dir,
        expected_results_dir,
        files_to_compare=[target + e for e in ["", ".emapper.hits", ".emapper.seed_orthologs"]]
    )


def test_emapper_analysis_and_file_presence(test_dbs_only_dl_files, prophane_input, tmpdir, conda_dir):
    test_ressource_dir = Path(dpath('resources/test_emapper_full_analysis'))
    expected_results_dir = test_ressource_dir / 'expected-results'
    config_file, job_dir = prophane_input(test_ressource_dir, str(tmpdir))
    snakemake_args = ["--conda-prefix", conda_dir, "--reason"]
    run_prophane(test_ressource_dir, job_cfg_file=config_file, check_md5=False, snakemake_args=snakemake_args)
    emapper_results = ["tasks/fun_annot_by_emapper_v2_on_eggnog_v5.v5.0.2.task0.result" + e for
                       e in ["", ".emapper.hits", ".emapper.seed_orthologs", ".emapper.annotations"]]
    # compare emapper results
    compare_with_expected_results_ignore_comment_lines_and_qvalues(
        Path(job_dir),
        expected_results_dir,
        files_to_compare=emapper_results)
    # compare summary and all other files
    compare_with_expected_results(
        job_dir,
        expected_results_dir,
        skip_md5_patterns=['.*\.result.*']
    )


def test_emapper_v2_write_task_map(tmpdir):
    from utils.databases.eggnog.eggnog import EggnogDatabaseV5
    resource_dir = Path(dpath("resources/test_emapper_v2_write_task_map"))
    copy(resource_dir / "job_dir", tmpdir)
    out_file = tmpdir / "job_dir" / "map.map"
    emapper_result = tmpdir / "job_dir" / "emapper_v2.result"
    source_task_string = "fun_from_eggNog_5 [task 0]"
    EggnogDatabaseV5.write_task_map(task_map=out_file,
                                    emapper_result=emapper_result,
                                    source_task_string=source_task_string,
                                    unclassified_annotation_string='unclassified')
    compare_with_expected_results(result_dir=tmpdir / "job_dir",
                                  expected_results_dir=resource_dir / "expected-results")


def test_emapper_v1_write_task_map(test_dbs_only_dl_files, prophane_input, tmpdir, conda_dir):
    test_ressource_dir = Path(dpath('resources/test_emapper_v1_write_task_map'))
    config_file, job_dir = prophane_input(test_ressource_dir, str(tmpdir))
    eggnog_processed_dir = Path(job_dir) / '../test_dbs/eggnog/4.5.1/processed'
    eggnog_processed_dir.mkdir()
    copy(test_ressource_dir/"job_dir/input/id2annot_eggnog_v4.map.gz", eggnog_processed_dir/'id2annot.map.gz')
    target = "tasks/fun_annot_by_emapper_v1_on_eggnog_v4.v4.5.1.task0.map"
    snakemake_args = ["--conda-prefix", conda_dir, target]
    run_prophane(test_ressource_dir, job_cfg_file=config_file, check_md5=True,
                 check_for_expected_results=True, snakemake_args=snakemake_args)
