#!/usr/bin/env bash
#------------------------------------------------------------------------------
# Central script to manage user mappings
# Mapping is the process of adding a new species to an existing clustering
# without changing the nr of clusters.
# Input:
#  * target clade (todo)
#  * new fasta with assigned taxid
#  * target clade
#    1. todo file including new species
#    2. pairwise A_A alignments, A_B BRH's
#    3. target clustering file
#    4. root clade split file
#------------------------------------------------------------------------------
CMD_ODBMAP=${CMD_ODBMAP:=$(basename $0)}
usage() {
    {
        echo
        echo "usage: $CMD_ODBMAP <command> [<command args>]"
        echo
        echo "For quick start help:"
        echo
        echo "> $CMD_ODBMAP help"
        echo
    }  1>&2
}

QUICKSTART_HELP=1
EXTENDED_HELP=0
command_help() {
    if (( QUICKSTART_HELP == 1 )); then
        echo "------------"
        echo " QUICKSTART"
        echo "------------"
        echo
        echo "To map a fasta file to OrthoDB:"
        echo 
        echo "> $CMD_ODBMAP MAP <run label> <fasta file> [<node>]"
        echo
        echo "<run label>   An arbitrary label used with other commands to refer to this mapping."
        echo "<fasta file>  The input fasta file to be mapped."
        echo "<node>        Optional: the target OrthoDB node (as a NCBI taxid)."
        echo "              If not given it's inferred using busco."
        echo
        echo "To get the result:"
        echo
        echo "> $CMD_ODBMAP RESULT <run label>"
        echo
        echo "For further help see:"
        echo 
        echo "> $CMD_ODBMAP help+"
        echo
        return 0
    fi
    echo "-----------------------------------------------------------------------------------------------"
    echo "Command    : HELP"
    echo "Args       : <none>"
    echo "Description: prints this help"
    echo
    echo "Command    : MAP"
    echo "Args       : <run label> [<taxid>:]<input file> [<target OrthoDB node id>]"
    echo "           : <run label>                A unique label for this run, used later as a reference."
    echo "           : <taxid>:<input file>       Input file defining fasta files to be mapped, optionally prefixed by an ID."
    echo "                                        The file may be:"
    echo "                                          1. fasta file"
    echo "                                          2. file containing a list of fasta files"
    echo "                                          3. like 2 but each line also contains a label"
    echo
    echo "                                        If no ID is given, it is set to a hash based on the filename." 
    echo "                                        The ID has no computational significance."
    echo
    echo "           : <target node id>   optional target node id, it can be any of the below"
    echo "                     1. OrthoDB node id (NCBI taxid)"
    echo "                     2. label given by 'local:<label>' -> will use local orthology defined by <label>"
    echo "                     3. not given or '-'               -> OrthoDB node id will be established using BUSCO auto lineage"
    echo "Description: map the given fasta file against the OrthoDB node id"
    echo
    echo "Command    : REPORT"
    echo "Args       : [<run label>]"
    echo "Description: Produces a report on the mapping run. If no run label is given, it will list all labels."
    echo
    echo "Command    : RESULT"
    echo "Args       : [<run label>] [what=<what>]"
    echo "           : <what>     optionally request what to be given as result."
    echo "                        'fas'     fasta input file"
    echo "                        'map'     mapping of internal fasta labels to input labels"
    echo "                        'cls'     final clusters with internal ids"
    echo "                        'odb'     final clusters with orthodb ids"
    echo "                        'hit'     like 'odn' but only with mapped labels"
    echo "                        'ann'     annotation for each mapped label"
    echo "                        'des'     cluster annotations"
    echo
    echo "Description: get result files, 'what' is ignored in busco mode"
    echo
    echo "Command    : BUSCO_RESULT"
    echo "Args       : [<run label>] [what=<nodeid|lineage|result|string|file>]"
    echo "Description: get busco specific results"
    echo
    #echo "Command    : HASRESULT"
    #echo "Args       : [<run label>] [mode=(map|busco)] [what=fasta|map|cluster|all]"
    #echo "Description: returns ec=0 if corresponding result exists, else ec=1"
    #echo
    echo "Command    : STATUS"
    echo "Args       : [<run label>]"
    echo "Description: get status flag of mapping"
    echo
    echo "Command    : PACKAGE"
    echo "Args       : <run label> [check]"
    echo "Options    : check    make sure all files are there"
    echo "Description: package the result in one tar ball"
    echo
    echo "Command    : DELETE"
    echo "Args       : [<run label> go]"
    echo "Options    : go       keyword, delete the mapping, otherwise print what should be done"
    echo "Description: deletes the mapping result"
    echo "             will not delete fasta or pairwise results"
    echo

    
    (( EXTENDED_HELP == 1 )) || return 0
    echo "-----------------------------------------------------------------------------------------------"
    echo "Extended help with advanced functions."
    echo
    echo "A 'subnode' is a CSV list of taxids that defines a sub set of the node to be mapped against."
    echo "-----------------------------------------------------------------------------------------------"
    echo
    echo "Command    : RESETRUN"
    echo "Args       : <run label>"
    echo "Description: resets the run such that a subsequent call to 'run' will create any missing files"
    echo
    echo    
    echo "Command    : DBINFO"
    echo "Args       : none"
    echo "Description: list all downloaded nodes"
    echo
    echo "Command    : DOWNLOAD"
    echo "Args       : <node taxid | ?[<clade>] > [full]"
    echo "Description: download mapping data from OrthoDB for given node id(s)"
    echo "           : <node taxid> is a CSV list of node ids or a query of format ?<clade>"
    echo "           :              If arg is a query and only '?' it will print all nodes."
    echo "           :              If the query has a clade specifier, it will print nodes for that clade."
    echo "           :              Note that only the first three characters are significant."
    echo "           : <full>       add this keyword to download full data (not subsampled)"
    echo
    #------------------------------------------------------------------------
    #----> only internal use
    #echo "Command    : INIT"
    #echo "Args       : none"
    #echo "Description: if not done, initialise mapping environment"
    #echo
    #------------------------------------------------------------------------
    echo "Command    : IMPORT"
    echo "Args       : \"<txid1:file1>\" [\"<txid2:file2>\" \"<txid3:file3>\" ... ]"
    echo "Description: import fasta files"
    echo "             Each file entry is given as quoted pair of a user defined txid and a file name."
    echo "             The file may be:"
    echo "               1. fasta file"
    echo "               2. file containing a list of fasta files"
    echo "               3. like 2 but each line also contains a label"
    echo
    echo "Command    : DBIMPORT"
    echo "Args       : <nodeid> [full]"
    echo "Description: import from DB the required files for mapping against the given node id"
    echo "           : <node id> is a CSV list of node ids"
    echo "           : <full>    add this keyword to download full data (not subsampled)"
    echo
    echo "Command    : LIST"
    echo "Args       : [fasta|taxid|nodes]"
    echo "Description: list data"
    echo
    echo "Command    : INFO"
    echo "Args       : [<what>]"
    echo "           : <what> is an optional string indicating what to print"
    echo "             'apiver'     OrthoDB API version"
    echo "             'apiurl'     OrthoDB API full URL"
    echo "             'work'       mapper work root directory"
    echo "             'project'    mapper run directory"
    echo "             'orthover'   orthologer version"
    echo "Description: print info about setup"    
    echo
    echo "Command    : RUN"
    echo "Args       : <user taxid> [nodeid=<nodeid>] [mode=<busco|map|mapprof>] [subsample=<N>] [score=<des|asc|sde|sas>] [using=<subnode|taxids>] [label=<explicit label>] [check] [go]"
    echo "Options    : check     make sure target of each step is really there"
    echo "           : type      'busco' or 'map' - what to run"
    echo "           : nodeid    NCBI taxid of target mapping node"
    echo "           : subsample (map) select a maximum of <N> genes per cluster based on score and type"
    echo "           : score     (map) together with 'subsample' - defines subsample strategy"
    echo "                       des    - sort by descending score"
    echo "                       asc    - sort by ascending score"
    echo "                       sde    - sort by descending score, select in steps"
    echo "                       sas    - sort by ascending score, select in steps"
    echo "                       sco    - sort genes first by overall species coverage and then by score"
    echo "                       ssc    - like sco, select stepwise"
    echo "           : using     (map) either a subnode taxid which defines the taxids to use"
    echo "           :           or a CSV list of taxids (enforced to be a subset from <nodeid>)"
    echo "           : go        run the job, otherwise print what should be done"
    echo
    echo "Description: Runs the either mapping or busco for given taxid wrt nodeid."
    echo "             If 'mode' == 'map' then:"
    echo "               'subsample': the mapping will be against a subsampled set of all species in the nodeid."
    echo "               'using'    : is requested, then the mapping will be against those taxids."
    echo "               otherwise, all species are used for the mapping (time consuming!)."
    echo
    echo "             If 'mode' == 'busco' then run busco on the given user taxid at the given nodeid (lineage)."
    echo "             If 'mode' == 'mapprof' then use mmseqs profiles for mapping."
    echo "             The profiles can be generated using orthologer.sh with POSTPROC_LABEL='ogmmprof'."
    echo
    echo "Command    : RUNLABEL"
    echo "Args       : <user taxid> [nodeid=<nodeid>] [mode=<busco|map>] [subsample=<N>] [using=<subnode|taxids>] [label=<explicit label>]"
    echo "Options    : same as for RUN, except [go|check]"
    echo "Description: returns the run label"
    echo
    echo "Command    : FINDNODES"
    echo "Args       : <ncbi taxid>"
    echo "Description: prints all downloaded nodes containing the given taxid"
    echo
    echo "Command    : CHECK"
    echo "Args       : none"
    echo "Description: check setup"
    echo
    echo "-----------------------------------------------------------------------------------------------"
    echo " Command used internally"
    echo "-----------------------------------------------------------------------------------------------"
    echo "Command    : AVAILABLE"
    echo "Args       : none"
    echo "Description: checks if a pipeline is running - returns zero if not"
    echo
    echo "Command    : INITRUN"
    echo "Args       : <nodeid> <user taxid> [<subnode>]"
    echo "Description: initialise run - create required todo file; called by RUN"
    echo
}

if [[ $# -eq 0 ]]; then
    usage
    exit 0
fi

#----------------------------------------------------------------------
# get command
__command="$1"
__command=${__command^^}

[[ ${__command} == 'H' || ${__command:0:4} == 'HELP'  ]] && {
    ext="${__command:4}"
    [[ "$ext" == "+"  ]] && QUICKSTART_HELP=0
    [[ "$ext" == "++" ]] && EXTENDED_HELP=1 && QUICKSTART_HELP=0
    command_help
    exit 0
}

#----------------------------------------------------------------------
set -u
CMD_PIPELINE="orthomapper.sh"

# code base
[[ -d "$DIR_ORTHOLOGER" ]] || export DIR_ORTHOLOGER=$(cd $(dirname "${BASH_SOURCE[@]}")/../ && pwd)

#--------------------------------------------------------------------------
# Check prerequisites (bash version)
source $DIR_ORTHOLOGER/bin/check_prerequisites.sh

# Check user id - possibly fail if root
source $DIR_ORTHOLOGER/bin/check_runas_root.sh
#--------------------------------------------------------------------------

#-----------------------------------------------------------------------------------
# pwd -> will be the directory from where this is run
export DIR_PROJECT=${DIR_PROJECT:-$(pwd)}
[[ -d "$DIR_PROJECT"  ]] || { echo "ERROR: mapping root is not readable or does not exist at DIR_PROJECT=$DIR_PROJECT" 1>&2; exit 255; }

#----------------------------------------
# load local orthomapper_conf.sh
if [[ -f "$DIR_PROJECT/orthomapper_conf.sh" ]]; then
    source $DIR_PROJECT/orthomapper_conf.sh
fi

# source the function file - this will also source orthologer common_odb.sh
source "$DIR_ORTHOLOGER/bin/orthomapper_func.sh"

# DB import from OrthoDB
source "$DIR_ORTHOLOGER/bin/orthomapper_dbi.sh"

# get busco related functions
source "$DIR_ORTHOLOGER/bin/orthomapper_busco.sh"

# load orthomapper_levels.sh
source "$DIR_ORTHOLOGER/bin/set_odbmap_nodes.sh"

# user id's; raw and hashed
export MAP_UID_RAW=""
export MAP_UID_HASH=""
export EXTENDED_HELP=0
export ERROR_MSG=""

#-----------------------------------------------------------------------------------
init() {
    echo "init: got $# args"
}

command_config() {
    echo "config"
}

command_init() {
    map_info_msg "start command INIT"
    map_check_pipeline && { map_info_msg  "pipeline looks OK";   return 0; }
    map_info_msg "call main setup script"
    $DIR_ORTHOLOGER/bin/setup.sh || return 255
    map_info_msg "pipeline successfully initiated"
}

command_info() {
    if [[ $# -eq 0 ]]; then
        map_dump_setup
        return 0
    fi
    local what="$1"
    what="${what,,}"
    if   [[ "$what" == "apiver" ]]; then
        echo "$URL_ORTHODB_API_VERSION"
    elif [[ "$what" == "apiurl" ]]; then
        echo "$URL_ORTHODB_API"
    elif [[ "$what" == "work" ]]; then
        echo "$ODBMAPPER_WORK"
    elif [[ "$what" == "project" ]]; then
        echo "$DIR_PROJECT"
    elif [[ "$what" == "orthover" ]]; then
        echo "$OP_VERSION"
    else
        echo "error: invalid argument $what"
        return 255
    fi
    return 0
}

command_list() {
    local listtaxid=0
    local listfasta=1
    local listnodes=1
    if [[ $# -gt 0 ]]; then
        listfasta=0
        listnodes=0
        listtaxid=0
    fi
    while [[ $# -gt 0 ]]; do
        x="$1"; x="${x,,}"; x=${x:0:3}
        [[ $x == 'fas' ]] && listfasta=1
        [[ $x == 'nod' ]] && listnodes=1
        [[ $x == 'tax' ]] && listtaxid=1
        shift
    done

    (( listfasta >0 && listtaxid > 0 )) && listtaxid=0
    
    local taxids=()
    if (( listtaxid > 0 )); then
        echo
        echo '# -> Imported taxids'
        local fname=""
        local label=""
        local fasta=""
        find $DIR_PROJECT/import -name '*.txt' -print0 | while read -d $'\0' fname; do
            label=$(basename "$fname" .txt)
            fasta="$DIR_PROJECT/Rawdata/${label}.fs.maptxt"
            [[ -f $fasta ]] || continue
            printf "%s\n" "$label"
        done | sort
    fi
    
    # list specific stuff
    if (( listfasta > 0 )); then
        # - imported files
        echo
        echo '# -> Imported fasta files'
        printf "# %-14s %16s         %s\n" "Label" "Nr of genes" "File name"
    
        local fname=""
        local label=""
        local size=""
        local fasta=""
        find $DIR_PROJECT/import -name '*.txt' -print0 | while read -d $'\0' fname; do
            label=$(basename "$fname" .txt)
            fasta="$DIR_PROJECT/Rawdata/${label}.fs.maptxt"
            [[ -f $fasta ]] || continue
            size=$(wc -l < "$fasta")
            printf "%-16s %16s         %s\n" "$label" "$size" "$(trim_name $fasta $DIR_PROJECT)"
        done | sort
    fi

    if (( listnodes > 0 )); then
        echo
        echo '# -> Imported nodes'
        printf "# %-18s %12s %12s %12s\n" "Clade" "Nodeid" "Size" "Flag"
        local nodeid=""
        local result=()
        local clade=""
        local ftodo=""
        local flag=""
        find "$DIR_PROJECT/todo" -name '*'."${EXT_TODO}" -print0 | while read -d $'\0' fname; do
            nodeid=$(basename "$fname" .${EXT_TODO})
            [[ "${nodeid:0:4}" == "node" ]] && continue
            [[ "${nodeid:0:1}" == "g"    ]] && continue
            # need to check if it's a subsampled todo!!!!
            result=($(map_find_node $nodeid))
            [[ $? -eq 0 ]] || continue
            clade=${result[0]}
            ftodo="${result[1]}"
            ftraw="${result[2]}"
            size="NA"
            rootvar=MAP_NODE_ROOT_${clade^^}
            flag=""
            [[ "${!rootvar}" == "$nodeid" ]] && flag="root"
            [[ -f "$ftodo" ]] && size=$(wc -l < <(grep -v "^#" "$ftodo"))
            printf "%-20s %12s %12s %12s\n" "$clade" "$(basename $ftraw .todo)" "$size" "$flag"
        done | sort -u
    fi
}

command_import() {
    debug_msg "running command_import"
    [[ $# -gt 0 ]] || { map_error_msg "import \"<taxid>:<fasta file>\""; return 255; }
    map_check_pipeline || { map_error_msg "import: pipeline is not setup, missing $CHECK_MISSING"; return 255; }

    local fall=("$@")    
    local f=""
    local msg=""
    local ec=0
    local nerr=0
    for f in ${fall[@]}; do
	    local tok=($(tokenize ':' "$f"))
	    [[ ${#tok[@]} -eq 2 ]] || { map_warn_msg "not in proper format ( \"<taxid>:<file>\") - skipping <$f>"; continue; }
	    local taxid=${tok[0]}
	    local fdata=${tok[1]}
        ###local str=$(printf "%-16s %-16s" "$MAP_UID_HASH" "$taxid")
	    map_import_fasta "$taxid" "$fdata"
	    ec=$?
	    if [[ $ec -eq 0 ]]; then
	        #ok_status_msg "$str"
            map_info_msg "imported id <$taxid> , file $(trim_name $fdata)"
	    elif [[ $ec -eq 1 ]]; then
            map_warn_msg "$IMPORT_MSG"
	        #xwarn_status_msg "$str" "$IMPORT_MSG"
	    else
            map_error_msg "$IMPORT_MSG"
            nerr=$((nerr+1))
	        #xerror_status_msg "$str" "$IMPORT_MSG"
	    fi
    done
    (( nerr == 0 )) || return 255
    return 0
}

#
# Query downloadable node(s)
# The input is fo fmt: ?<root clade>
# where <root clade> is any of:
# - EUK   eukaryota
# - VER
# - ART
# ...
# Look for all MAP_NODES_XXX in etc/<version>/orthomapper_levels.sh.
# In this function only the 3 first letters are significant.
#
command_download_query() {
    local query="$1"
    query=${query:1:3}
    query=${query^^}
    [[ ${#query} == 3 ]] || query=""
    local found=0
    map_info_msg "will print all downloadable clades and nodes"
    for clade in ${MAP_ROOT_CLADES[@]}; do
        local clup=${clade^^}
        local clshort=${clup:0:3}
        if [[ -n $query ]]; then
            [[ $clshort == $query ]] || continue
        fi
        found=1
        local rootvar="MAP_NODE_ROOT_${clup}"
        local nodesvar="MAP_NODES_${clup}"
        local nodesarr="${nodesvar}[@]"
        for node in ${!nodesarr}; do
            printf "%-24s %12s" "$clup" "$node"
            if [[ "$node" == "${!rootvar}" ]]; then
                echo "    ROOT"
            else
                echo
            fi
        done
    done
    (( found == 1 )) || {
        map_warn_msg "nothing found!"
        if [[ -n $query ]]; then
            map_warn_msg "the query argument maybe wrong: $query"
            map_warn_msg "only the first three characters are significant and should match one of the clades available"
            map_warn_msg "example: ?art will query all Arthropoda nodes"
        fi
    }
}

#
# Download clade.
# If the given clade starts with '?' it will print which clades are available.
#
command_download() {
    verbose_info_msg "running command_download"
    [[ $# -ge 1 ]] || { map_error_msg "download <node taxid> [full]"; return 255; }
    local nodeid="$1"
    if [[ ${nodeid:0:1} == "?" ]]; then
        command_download_query "$nodeid"
        return $?
    fi

    if [[ $# -gt 1 ]]; then
        local opt="$2"
        [[ "${opt,,}" == "full" ]] && DBI_USE_FULL=1
    fi

    # make a list of node ids (replacing commas with space)
    local nodes=( $(echo "$nodeid" | tr ',' ' ') )
    local nerr=0
    for node in ${nodes[@]}; do
        dbi_find_and_install_node "$node" || { map_error_msg "unable to find mapping data for node in OrthoDB"; nerr=255; }
    done
    return $nerr
}

#check_and_init_todo() {
#    local node="$1"
#
#    # get source node
#    map_find_orthodb_todo "$node" "CHECK" || return 255
#    map_info_msg "for node $node , found root $ODB_CLADE and todo files $(basename $ODB_TODO_USE) and $(basename $ODB_TODO_FULL)"
#    PL_TODO="$ODB_TODO_USE"
#    
#    # update todo
#    update_todo 1 || { map_error_msg "failed updating todo file $PL_TODO"; return 1; }
#    return 0
#}

command_dbimport() {
    verbose_info_msg "running command_dbimport"
    [[ $# -ge 1     ]] || { map_error_msg "dbimport <CSV list of node name ids> [full]"; return 255; }
    map_check_pipeline || { map_error_msg "dbimport: pipeline is not setup for user (ec=$?), missing $CHECK_MISSING"; return 255; }
    local nodeid="$1"
    [[ -n "$nodeid" ]] || nodeid="auto"
    if [[ "$nodeid" == "auto" ]]; then
        map_info_msg "dbimport : node id is set to 'auto' - it will be inferred using BUSCO"
        return 0
    fi
    [[ $# -gt 1 ]] && {
        local opt="$2"
        [[ "${opt,,}" == "full" ]] && DBI_USE_FULL=1
    }
    local req_DBI_USE_FULL=$DBI_USE_FULL
    
    # make a list of node ids (replacing commas with space)
    local nodes=( $(echo "$nodeid" | tr ',' ' ') )
    local ec=0
    
    # first make sure ALL of the requested nodes are in OrthoDB
    for node in ${nodes[@]}; do
        # skip auto keyword
        [[ $node == "auto" ]] && continue
        if map_node_is_viridae $node; then
            map_info_msg "given node=$node is part of viridae -> force FULL data set"
            DBI_USE_FULL=1
        else
            DBI_USE_FULL=$req_DBI_USE_FULL
        fi
        map_info_msg "attempt to import DB node $node"
        map_find_orthodb_todo "$node" "CHECK" #>& /dev/null
        ec=$?
        if (( ec != 0 )); then
            if (( ec > 3 )); then
                map_error_msg "given node id ($node) is not supported"
                map_info_msg "it should be installed at $MAP_ORTHODB_DATA"
                return 255
            fi
            if (( ec == 1 )); then
                map_error_msg "full node missing at $ODB_TODO_FULL"
                return 255
            else
                map_error_msg "subsampled node missing at $ODB_TODO_SUBSAMPLED"
                return 255
            fi
        fi
    done

    #--------------------------------------------------
    # do the actual import
    cd $DIR_PROJECT
    map_info_msg "command_dbimport - all requested nodes are available : $nodeid"
    map_info_msg "command_dbimport - moved to project dir at $(pwd)"
    if (( DBI_USE_FULL == 1 )); then
        map_info_msg "will import OrthoDB FULL data set"
    else
        map_info_msg "will import OrthoDB SUBSAMPLED data set"
    fi
    
    # all are good - go ahead and import
    for node in ${nodes[@]}; do
        map_find_orthodb_todo "$node" || return 255
        map_info_msg "for node $node , found root $ODB_CLADE and todo files $(trim_name $ODB_TODO_USE) and $(trim_name $ODB_TODO_FULL)"
        PL_TODO="$ODB_TODO_USE"
        # update todo
        update_todo 1 || { map_error_msg "failed updating todo file $PL_TODO"; return 1; }
        
        ####check_and_init_todo "$node" || return 255
        
        map_info_msg "create project directories based on todo file $(trim_name $PL_TODO)"
        create_project_dirs || { map_error_msg "failed creating project directories"; continue; }
        
        map_info_msg "importing OrthoDB data for todo file $ODB_TODO_USE (clade = $ODB_CLADE)"
        map_import_db "$ODB_CLADE" "$ODB_TODO_USE" "$ODB_TODO_FULL" || return 255
    done
    return 0
}

check_cmd() {
    local f="$1"
    local status="OK"
    local rval=0
    local str=$(printf '%-80s' "$f")
    if [[ -x $f ]]; then
        rval=0
        ok_status_msg "$str"
    else
        xerror_status_msg "$str" "file not found"
        rval=1
    fi
    return $rval
}

precheck_run() {
    local rval=0
    local bad=""
    {
        check_cmd $DIR_ORTHOLOGER/bin/extract_single_inpar.py  || { rval=1; bad="$bad,$DIR_ORTHOLOGER/bin/extract_single_inpar.py"; }
        check_cmd $DIR_BRHCLUS/brhclus                       || { rval=1; bad="$bad,$DIR_BRHCLUS/brhclus"; }
        check_cmd $DIR_BRHCLUS/mergeinpar                    || { rval=1; bad="$bad,$DIR_BRHCLUS/mergeinpar"; }
    } >& /dev/null
    [[ ${bad:0:1} == "," ]] && bad=${bad:1}
    [[ $rval -eq 0 ]] || { map_error_msg "missing executable(s): $bad"; return $rval; }
    return 0
}

#-----------------------------------------------------------------------------------
# Parse RUN arguments
#
# _RUN_LABEL             either user set or constructed run label
# _RUN_MODE              either 'map' or 'busco'
# _RUN_TAXIDS            labels of fasta to be mapped
# _RUN_NODEID            NCBI taxid of the lineage (node)
# _RUN_SUBSET           CSV list of OrthoDB taxids OR the keyword 'auto'
# _RUN_SUBSAMPLE         max nr of genes subsampled per cluster
# _RUN_SCOREDIR          sorting of genes in subsample
# _RUN_GO                if 0 - dry run, 1 - go ahead
# _RUN_CHECK             just check
# 
#-----------------------------------------------------------------------------------
parse_run_args() {
    [[ $# -ge $_REQ_ARGS ]] || {
        map_error_msg "usage: [mode=<map|busco>] [nodeid=<node id>] [subsample=<N> [score=(des|asc)]] [using=<subnode>] [label=<map label>]"
        return 255;
    }
    map_parse_run_args "$@"
    return $?
}

command_initrun() {
    _REQ_ARGS=0
    parse_run_args initrun "$@" || return 255

    map_check_pipeline || { map_error_msg "initrun: pipeline is not setup, missing $CHECK_MISSING"; return 255; }
    if [[ "$_RUN_MODE" == "map" ]]; then
        # initialise todo file
        map_init_todo || return 255
        map_info_msg "created todo file: $MAP_TODO"
    fi
    
    return 0
}

# check that no other pipeline is running
# returns:
#    0 - pipeline is available
#    1 - another process is running
#  255 - user pipeline directory not found
check_if_available() {
    map_check_pipeline || { map_error_msg "pipeline is not setup, missing $CHECK_MISSING"; return 255; }
    
    # cd to project dir
    cd $DIR_PROJECT || { map_error_msg "unable to cd to $DIR_PROJECT"; return 255; }

    # get status
    pipeline -S | grep -eq 'RUNNING'
    [[ $? -eq 0 ]] && {
        # it's running
        return 1
    }
    # it's available!
    return 0
}

command_runlabel() {
    _REQ_ARGS=1
    parse_run_args runlabel "$@" || return 255
    echo $_RUN_LABEL
}

#------------------------------------
# run busco for given user taxid
# busco is run in a directory specific
# for the user taxid
# DIR_PROJECT/busco/<taxid>/
#
# Must call map_busco_setup() before.
# If setup is OK it sets BUSCO_SETUP_OK=1
#------------------------------------
run_busco() {
    [[ -n $_RUN_SINGLE_TAXID ]] || {
        map_error_msg "busco mode only supports a single taxid"
        return 255
    }
    
    # check if enabled and setup is OK
    (( BUSCO_ENABLED == 0 )) && {
        map_warn_msg "called run_busco() when BUSCO is not enabled"
        return 0
    }
    if (( BUSCO_SETUP_OK == 0 )); then
        map_error_msg "BUSCO is not properly setup - see busco_setup()"
        map_error_msg "$BUSCO_SETUP_MSG"
        return 255
    fi
    #------------------------------------------------------------------
    # first check environment
    [[ -d "$DIR_PROJECT" ]] || {
        map_error_msg "run_busco: missing user project directory $DIR_PROJECT"
        return 255
    }
    map_busco_load_levels_map || {
	    map_error_msg "failed loading BUSCO levels map"
	    return 255
    }

    local ec=0
    
    #------------------------------------------------------------------
    if [[ -n $BUSCO_CMD ]]; then
        map_info_msg "run BUSCO using local install ($BUSCO_CMD)"
    else
        map_info_msg "run BUSCO through docker image $BUSCO_DOCKER_IMAGE"
    fi

    # remove any non alphanumeric chars from user taxid
    local taxid=$(map_cleanup_taxid $_RUN_SINGLE_TAXID)

    # create workdir if it does not exist
    [[ -n $BUSCO_WD_ROOT ]] || {
        map_error_msg "BUSCO_WD_ROOT is not set!"
        return 255
    }
    mkdir -p "$BUSCO_WD_ROOT" || {
        map_error_msg "unable to create BUSCO_WD_ROOT = $BUSCO_WD_ROOT"
        return 255
    }
    map_info_msg "BUSCO working dir set to $BUSCO_WD_ROOT"
    
    # attempt to read BUSCO metadata
    # 0  -> data exists
    # 1  -> dir exists but no matching metadata file
    # >1 -> some other error
    map_busco_read_metadata $_RUN_NODEID
    ec=$?
    if [[ $ec -eq 0   ]]; then
        if map_busco_is_filled_metadata; then
            map_info_msg "BUSCO results already exists for taxid $taxid"
            return 0
        fi
        map_info_msg "BUSCO empty metadata file already exists for taxid $taxid"
    elif [[ $ec -ne 1 ]]; then
        map_error_msg "something went wrong when reading BUSCO metadata (ec=$ec)"
        return 255
    fi
    
    # prepare to run
    # The mapping expects the node (lineage) to be given as a NCBI taxid.
    # BUSCO uses name strings to identify lineages.
    # First, need to map given node id to BUSCO lineage name.
    BUSCO_LINEAGE=""
    BUSCO_LINEAGE_TAXID=""
    if [[ -n $_RUN_NODEID ]]; then
        map_busco_taxid_to_busco_lineage "$_RUN_NODEID" || return 255
        map_info_msg "will run BUSCO at lineage $BUSCO_LINEAGE ($_RUN_NODEID)"
    else
        map_info_msg "will run BUSCO using auto lineage"
    fi
    
    # set input fasta file
    local inputFasta="$DIR_PROJECT/Rawdata/${_RUN_SINGLE_TAXID}.fs"
    [[ -f $inputFasta ]] || {
        map_error_msg "run_busco: missing input fasta at $inputFasta"
        return 255
    }

    # save metadata in busco work dir
    # this will potentially save the file with _auto extension (no NODEID)
    BUSCO_INPUT_FASTA=$inputFasta
    BUSCO_INPUT_LINEAGE=$BUSCO_LINEAGE
    [[ -n $BUSCO_INPUT_LINEAGE ]] || BUSCO_INPUT_LINEAGE="auto"

    # run busco
    map_busco_run "$inputFasta" "$BUSCO_LINEAGE" || return 255

    # analyse result, in particular get the selected lineage
    # this will also save BUSCO metadata
    map_busco_extract_results "$_RUN_SINGLE_TAXID" || return 255

    map_info_msg "BUSCO run done"
    return 0
}


# Functions to save and restore TODO file names as they may be reset by a call to 'pipeline'
_SAVE_TODO_=""
_SAVE_TODO1_=""
_SAVE_TODO2_=""
save_todo() {
    _SAVE_TODO_="$PL_TODO"
    _SAVE_TODO1_="$PL_TODO1"
    _SAVE_TODO2_="$PL_TODO2"
}
restore_todo() {
    PL_TODO="$_SAVE_TODO_"
    PL_TODO1="$_SAVE_TODO1_"
    PL_TODO2="$_SAVE_TODO2_"
}

export SET_MAKE_MISSING_ALL=0

# make code for alignment+
export SET_ALIGNMENT_MAKE=3

check_db_files_and_run() {
    SET_ALIGNMENT_MAKE=3
    local ftodo=$1
    local nchks=${2-10}
    # get at most the first 10 labels
    local labels=()
    if (( nchks == 0 )); then
        labels=( $(grep -h -v "^#" $ftodo) )
    else
        labels=( $(grep -h -v "^#" $ftodo | head -n $nchks) )
    fi
    
    local L=
    local nbad=0
    for L in ${labels[@]}; do
        local f=$DIR_PROJECT/Sequences/${L}.
        check_formatdb_files "$f" 1 || {
            nbad=1
            break
        }
    done
    # seems OK - skip
    (( nbad == 0 )) && return 0

    map_info_msg "DB files for the selected aligner ($ALIGNMENT_LABEL) seems to be missing."
    map_info_msg "Will generate the missing files."
    if is_dryrun; then
        map_info_msg "skip - dry run"
    else
        # make sure formatdb only runs on missing files before running
        base_pipeline -t $ftodo -f formatdb-:0,formatdb:3 >& /dev/null || return 255
        base_pipeline -t $ftodo -r formatdb || return 255
        restore_todo
        
        # make sure all the consequenct steps are set
        SET_MAKE_MISSING_ALL=1
        
        # formatdb will be updated -> set 'MAKE ALL' (2) for ALIGNMENT+
        SET_ALIGNMENT_MAKE=2
    fi
    return 0
}

export DO_GENEID_MAPPING=${DO_GENEID_MAPPING:=1}
# load command_run_mmprof
source "$DIR_ORTHOLOGER/bin/orthomapper_mmprof.sh"
command_run() {
    local ec=0
    _REQ_ARGS=0
    parse_run_args run "$@" || return 255

    if (( _RUN_IS_LOCAL )); then
        if [[ "$_RUN_MODE" != "map" ]]; then
            map_error_msg "runmode '$_RUN_MODE' is not currently supported with a local target"
            return 255
        fi
    fi
    if [[ "${_RUN_MODE:0:3}" != "map" ]]; then
        if [[ ${#_RUN_TAXIDS[@]} -gt 1 ]]; then
            map_error_msg "multiple taxids is not supported in this mode = $_RUN_MODE"
            return 255
        fi
    fi
    local ntaxids=${#_RUN_TAXIDS[@]}
    if (( ntaxids == 0 )); then
        [[ -n $_RUN_SINGLE_TAXID ]] && ntaxids=1
    fi
        
    map_info_msg "------------------------------------"
    map_info_msg "Start RUN with parameters:"
    map_info_msg "args             $@"
    map_info_msg "mode             $_RUN_MODE"
    map_info_msg "label            $_RUN_LABEL"
    map_info_msg "N(taxids)        ${#_RUN_TAXIDS[@]}"
    map_info_msg "hashid           $_RUN_HASHID"
    map_info_msg "nodeid           $_RUN_NODEID"
    map_info_msg "nodeid is local  $_RUN_IS_LOCAL"
    map_info_msg "using            $_RUN_SUBSET"
    map_info_msg "subsample        $_RUN_SUBSAMPLE"
    map_info_msg "score            $_RUN_SCOREDIR"
    map_info_msg "------------------------------------"

    #------------------------------------------------------------
    # special new command
    # map using mmseqs profiles
    #------------------------------------------------------------
    if [[ "$_RUN_MODE" == "mapprof" ]]; then
        command_run_mmprof
        return $?
    fi
    
    #------------------------------------------------------------
    # check BUSCO
    #------------------------------------------------------------
    # mode is BUSCO
    if [[ "$_RUN_MODE" == "busco" ]]; then
        map_busco_setup || {
            map_error_msg "requested BUSCO is not setup correctly"
            map_error_msg "$BUSCO_SETUP_MSG"
            return 255
        }
        (( BUSCO_ENABLED == 1 )) || {
            map_error_msg "BUSCO requested but it is not enabled"
            return 255
        }
    fi

    # mode is MAP with empty nodeid
    if [[ "$_RUN_MODE" == "map" ]] && [[ -z "$_RUN_NODEID" ]]; then
        map_busco_setup || {
            map_error_msg "mapping with no apriori node id - required BUSCO is not setup correctly"
            map_error_msg "$BUSCO_SETUP_MSG"
            return 255
        }
        (( BUSCO_ENABLED == 1 )) || {
            map_error_msg "BUSCO required but it is not enabled (no lineage is given)"
            return 255
        }
    fi
    
    #------------------------------------------------------------
    # prepare run
    #------------------------------------------------------------
    # create log file
    local logfile=$(map_runlog_name)

    local T0busco=$(time_nsec)
    DRY_RUN=0
    [[ $_RUN_GO -eq 0 ]] && {
        DRY_RUN=1
        map_info_msg "add 'go' to the command line to actually run"
    }
    [[ $_RUN_CHECK -eq 1 ]] && {
        map_info_msg "will make full check that all partial results exists"
    }

    # makes sure the pipeline env is correct
    map_check_pipeline || {
        map_error_msg "run: pipeline is not setup, missing $CHECK_MISSING"
        return 255
    }

    #------------------------------------------------------------
    # Run possibly only BUSCO
    #------------------------------------------------------------
    if [[ "$_RUN_MODE" == "busco" ]]; then
        map_info_msg "run only BUSCO requested"
        ec=0
        if ! is_dryrun; then
	        map_info_msg "logfile at $logfile"
            run_busco >& $logfile
            ec=$?
            map_info_msg "BUSCO finished in $(delta_time $T0busco) seconds"
            (( ec == 0 )) || map_error_msg "BUSCO finished in error with ec=$ec"
            map_info_msg "see logfile at $logfile"
            map_write_metadata
        fi
        return $ec
    fi
    
    local usedBusco="no"
    
    # check all tools
    verbose_info_msg "precheck..."
    precheck_run || return 255

    #------------------------------------------------------------
    # Run BUSCO auto lineage if no node id is given .
    #------------------------------------------------------------
    if [[ -z $_RUN_NODEID ]]; then
        if (( BUSCO_ENABLED == 0 )); then
            map_error_msg "no nodeid is given - BUSCO is required but BUSCO is disabled (set BUSCO_ENABLED=1)"
            return 255
        fi

        map_info_msg "no node id given - run BUSCO auto lineage"
        if ! is_dryrun; then
            run_busco || {
		        map_error_msg "BUSCO autolineage failed"
		        return 255
	        }
            [[ -n "$BUSCO_RES_COMPLETE" ]] || {
                map_error_msg "BUSCO autolineage passed with zero error code - but BUSCO_RES_COMPLETE is NOT set!"
                return 255
            }
            if (( $(echo "$BUSCO_RES_COMPLETE < $BUSCO_MINSCORE" | bc -l) )); then
		        map_error_msg "No BUSCO good enough lineage found"
                map_error_msg "Score $BUSCO_RES_COMPLETE < $BUSCO_MINSCORE"
		        return 255
	        fi
	    
            # get NCBI taxid of BUSCO lineage
            map_update_run_nodeid "$BUSCO_LINEAGE_TAXID" || return 255
            
            map_info_msg "BUSCO found lineage <$BUSCO_LINEAGE> , taxid <$BUSCO_LINEAGE_TAXID>"
	    
            # download and import OrthoDB data for that lineage
            command_download "$_RUN_NODEID"    || return 255
            command_dbimport "$_RUN_NODEID" go || return 255
            usedBusco="yes"
            
	        # new logfile name as NODEID is now defined
	        logfile=$(map_runlog_name)
            
            map_write_metadata
        fi
    fi
    local dTbusco=$(delta_time $T0busco)

    #------------------------------------------------------------
    # Run sub-sampling of fasta file, if requested.
    # This can be done by
    # 1. subsampling each fasta file
    # 2. subsampling taxid space within node
    # 3. using 1 and 2
    # 4. list of given taxids (case not treated in map_subsample)
    #
    # See subsample_fasta() and script bin/subsample_cluster_genes.py
    # Nr 4 is given by _RUN_SUBSET
    #------------------------------------------------------------
    local T0sub=$(time_nsec)
    # possibly subsample input
    if [[ -n "$_RUN_SCOREDIR" ]]; then
        verbose_info_msg "sub sample node"
        if ! is_dryrun; then
            map_subsample || return 255
        fi
    fi
    local dTsub=$(delta_time $T0sub)

    #------------------------------------------------------------
    # Run the mapping.
    #------------------------------------------------------------
    local T0map=$(time_nsec)
    # initialise todo file
    verbose_info_msg "create todo"
    is_dryrun || map_init_todo || return 255

    # check if ready to run
    verbose_info_msg "check if ready for run..."
    is_dryrun || map_ready_to_run || return 255

    # set todo file names
    # TODO == TODO1 -> labels to be mapped
    # TODO2 -> target node labels
    PL_TODO="$MAP_TODO"
    PL_TODO1="$MAP_TODO"
    PL_TODO2="$SOURCE_TODO"

    info_msg "-----------------------------------"
    info_msg "Calling update_todo with"
    info_msg " - PL_TODO=$(trim_name  $PL_TODO)"
    info_msg " - PL_TODO1=$(trim_name $PL_TODO1)"
    info_msg " - PL_TODO2=$(trim_name $PL_TODO2)"
    info_msg "-----------------------------------"
    # update todo variables
    is_dryrun || update_todo 1

    # redo this as TODO files have been changed
    is_dryrun || export_decl_doall
    
    # and the label
    local todolabel=$(file_basename $PL_TODO)

    is_dryrun && return 0

    # move to pipeline
    cd $DIR_PROJECT || { map_error_msg "failed entering $DIR_PROJECT"; return 255; }
    
    # make sure input cluster is there
    local inpog=$DIR_PROJECT/Cluster/${_RUN_NODEID}.${EXT_CLUSTER}
    [[ -f $inpog ]] || { map_error_msg "could not find source clustering $inpog"; pwd; return 255; }

    # subset of final clustering, contains ONLY genes from the mapped taxid
    local final_ext="odb"
    (( _RUN_IS_LOCAL )) && final_ext="org"
    local result_odb=$DIR_PROJECT/Results/${PL_TODO_NAME}.${EXT_CLUSTER}.$final_ext
    local result_hits=$DIR_PROJECT/Results/${PL_TODO_NAME}.${EXT_CLUSTER}.hits
    local result_desc=$DIR_PROJECT/Results/${PL_TODO_NAME}.${EXT_CLUSTER}.desc
    local result_annot=$DIR_PROJECT/Results/${PL_TODO_NAME}.${EXT_CLUSTER}.annotations
    local result_fs_dir=$DIR_PROJECT/Results/${PL_TODO_NAME}_fs
    local result_fsmap=$DIR_PROJECT/Results/${PL_TODO_NAME}.fs.maptxt

    # same but with a directory name that makes sense for user (relative DIR_PROJECT)
    local result_odb_ref=$(trim_name "$result_odb" "$DIR_PROJECT")
    local result_hits_ref=$(trim_name "$result_hits" "$DIR_PROJECT")
    local result_desc_ref=$(trim_name "$result_desc" "$DIR_PROJECT")
    local result_annot_ref=$(trim_name "$result_annot" "$DIR_PROJECT")
    local result_fs_dir_ref=$(trim_name "$result_fs_dir" "$DIR_PROJECT")
#    local result_fs_ref=$(trim_name "$result_fs" "$DIR_PROJECT")
#    local result_fsmap_ref=$(trim_name "$result_fsmap" "$DIR_PROJECT")
    
    # output cluster
    [[ -f $result_annot ]] && {
        map_info_msg "done - target clustering already exists : $result_annot_ref"
        map_write_metadata || { map_error_msg "failed writing metadata!"; return 255; }
        return 0
    }

    # flag if no user taxid was clustered
    local noData=0
    

    # run pipeline
    # note:
    # - stderr (2) : all output from orthologer.sh, then redirected to logfile
    # - stdout (1) : all messages outside orthologer.sh
    #
    map_info_msg "run pipeline; logfile at $(trim_name $logfile $DIR_PROJECT)"
    {
        # save TODO filenames
        save_todo
        local todoall="${PL_TODO1}:${PL_TODO2}"

        #-------------------------------------------------------------------------------------------------------------
        # make sure directories are present
        #-------------------------------------------------------------------------------------------------------------
        map_info_msg "pipeline start setup"
        is_dryrun || pipeline -t "$todoall" -C 1>&2
        [[ $? -eq 0 ]] || { map_error_msg "failed setting up the pipeline - see logfile $(trim_name $logfile $DIR_PROJECT)" 2>&1; return 255; }
        restore_todo
        
        #-------------------------------------------------------------------------------------------------------------
        # check if DB files for downloaded OrthoDB files exist
        #-------------------------------------------------------------------------------------------------------------
        # loop over at most the first N files
        check_db_files_and_run $PL_TODO2 10 || return 255

        # force checking for missing files up to SETINPARSEL
        if (( _RUN_CHECK > 0 )) || (( SET_MAKE_MISSING_ALL>0 )); then
            map_info_msg "reset done flag to MAKE MISSING for all steps, set POSTPROC to DONE"
            is_dryrun || {
                # make missing files
                base_pipeline -t "$todoall" -f ALIGNMENT-:3,ALIGNMENT+:$SET_ALIGNMENT_MAKE >& /dev/null
                base_pipeline -t "$PL_TODO" -f ALIGNMENT-:3,ALIGNMENT+:$SET_ALIGNMENT_MAKE >& /dev/null
            }
            ec=$?
            restore_todo
        fi
        [[ $ec -eq 0 ]] || { map_error_msg "failed setting up the pipeline - see logfile $(trim_name $logfile $DIR_PROJECT)" 2>&1; return 255; }
        
        # make sure POSTPROC is NOT run
        base_pipeline -t "$PL_TODO" -f POSTPROC:0 1>&2
        restore_todo
        
	    map_info_msg "pipeline setup passed"
        
        #-------------------------------------------------------------------------------------------------------------
        # run all steps upto SETINPARSEL, requiring taxid in all pairwise
        #-------------------------------------------------------------------------------------------------------------
        map_info_msg "pipeline pass 1 : run pipeline using $(trim_name $PL_TODO1 $DIR_PROJECT) vs $(trim_name $PL_TODO2 $DIR_PROJECT)"
        map_info_msg "run all before FORMATDB"
        ec=0
        ####is_dryrun || base_pipeline -t "$PL_TODO" -R "$_RUN_TAXID" -I "$inpog" -r STATS- 1>&2 || ec=$?
        is_dryrun || base_pipeline -t "$PL_TODO" -I "$inpog" -r STATS- 1>&2 || ec=$?
        [[ $ec -eq 0 ]] || { map_error_msg "failed running pipeline before STATS (ec=$ec)- see logfile $(trim_name $logfile $DIR_PROJECT)" 2>&1; return 255; }
        restore_todo
        
        local merge_opts=""
        # 
        ####(( _RUN_NODEID_IS_VIRIDAE == 1 )) && merge_opts="-m"
        
        ec=0
        local saveINPOG=$CLUSTER_INPOG
        if (( _RUN_DO_SUBSAMPLE == 1 )); then
            CLUSTER_INPOG=""
            map_info_msg "run STATS,FORMATDB on all taxids in todo - custom subsampling"
            is_dryrun || base_pipeline -t "$todoall" -r STATS,FORMATDB $merge_opts 1>&2 || ec=$?
        else
            map_info_msg "run STATS only on the user taxid - the other files are downloaded from OrthoDB"
            ###is_dryrun || base_pipeline -t "$PL_TODO" -R "$_RUN_TAXID" -I "$inpog" -r STATS 1>&2 || ec=$?
            is_dryrun || base_pipeline -t "$PL_TODO" -I "$inpog" -r STATS 1>&2 || ec=$?
            restore_todo
            
            #NOTE:: WHEN DOWNLOADED DATA IS UPDATED WITH diamond DB FILES -> REMOVE THIS
            CLUSTER_INPOG=""
            map_info_msg "run FORMATDB taxids to be mapped"
            ###is_dryrun || base_pipeline -t "$PL_TODO" -r FORMATDB -R "$_RUN_TAXID" $merge_opts 1>&2 || ec=$?
            is_dryrun || base_pipeline -t "$PL_TODO" -r FORMATDB $merge_opts 1>&2 || ec=$?
            restore_todo
        fi
        CLUSTER_INPOG="$saveINPOG"

        [[ $ec -eq 0 ]] || {
            map_error_msg "failed running pipeline step FORMATDB"
            map_error_msg "see logfile $(trim_name $logfile $DIR_PROJECT)" 2>&1
            return 255
        }

        map_info_msg "run FORMATDB done with ec=$ec"

        if [[ "$_RUN_SUBSAMPLE" != "0" ]]; then
            map_info_msg "DB fasta is subsampled - soft-link diagonal alignment files"
            ###map_softlink_subsampled_alignment "$PL_TODO" "$_RUN_TAXID" || { map_error_msg "failed softlinking subsampled alignment"; return 255; }

            #NOTE: the second arg is a dummy - to be removed once mapping using TODO1/2 works
            map_softlink_subsampled_alignment "$PL_TODO2" "weudf768whjkfv34ve768__" || {
                map_error_msg "failed softlinking subsampled alignment" 2>&1
                return 255
            }
        fi
        ###ec=0
        ###is_dryrun || base_pipeline -t "$PL_TODO" -R "$_RUN_TAXID" -I "$inpog" -r ALIGNMENT+ $merge_opts 1>&2 || ec=$?
        
        # Run the rest using TODO1/TODO2
        map_info_msg "run the remaining pairwise steps with todo specs: $todoall"
        ec=0
        is_dryrun || base_pipeline -t "$todoall" -I "$inpog" -r ALIGNMENT,MAKEBRH,MAKEINPAR $merge_opts 1>&2 || ec=$?
        restore_todo
        [[ $ec -eq 0 ]] || { map_error_msg "failed running pipeline steps after FORMATDB - see logfile $(trim_name $logfile $DIR_PROJECT)" 2>&1; return 255; }

        map_info_msg "run MAKEINPARSEL/SETINPARSEL only for mapped ids"
        ec=0
        is_dryrun || base_pipeline -t "$PL_TODO" -I "$inpog" -r MAKEINPARSEL,SETINPARSEL $merge_opts 1>&2 || ec=$?
        restore_todo
        [[ $ec -eq 0 ]] || { map_error_msg "failed running pipeline steps after FORMATDB - see logfile $(trim_name $logfile $DIR_PROJECT)" 2>&1; return 255; }

        map_info_msg "run remaining step"
        ec=0
        is_dryrun || base_pipeline -t "$todoall" -I "$inpog" -r CLUSTER+ $merge_opts 1>&2 || ec=$?
        restore_todo
        [[ $ec -eq 0 ]] || { map_error_msg "failed running pipeline steps after MAKEINPARSEL - see logfile $(trim_name $logfile $DIR_PROJECT)" 2>&1; return 255; }

	    map_info_msg "pipeline pass 1 : done"
    
        #-------------------------------------------------------------------------------------------------------------
        map_info_msg "pipeline pass 2 : map to real gene ids"
        if ! is_dryrun; then

            # create output with mapped gene ids:
            # 1. .og.odb|org : full cluster file with OrthoDB cluster ids and real gene ids
            # 2. .og.hits: same but only gene ids from mapped taxid
            local orgog="$DIR_PROJECT/Cluster/${PL_TODO_NAME}.${EXT_CLUSTER}"     # cluster result from the above mapping

            # check files
            [[ -f $orgog ]] || { map_error_msg "although it passed the clustering stage, it still cannot find the result : $orgog" 2>&1; return 255; }

            # mapping cluster id - only if map against OrthoDB node
            local oglevel="--oglevel $_RUN_NODEID"
            (( _RUN_IS_LOCAL )) && oglevel=""

            #---------------------------------------------
            # create file containing ONLY the mapped gene ids
            local tmpout=""
            tmpout=$(mktemp '/tmp/package_XXXXXX') || { map_error_msg "unable to create temp out for clustering"; return 255; }
            ###grep "\s${_RUN_TAXID}:" "$orgog" > "$tmpout" || {
            $DIR_BRHCLUS/getclus -C "$orgog" -s "$PL_TODO" > "$tmpout" || {
		        noData=1
		        map_warn_msg "no keys with taxids from $PL_TODO found in result $(pwd)/$orgog" 2>&1
		        echo "# No data found with taxids from $PL_TODO" > "$result_hits"
	        } 
            if (( noData == 0 )); then
                # remap
                ###local f="$DIR_PROJECT/Rawdata/${_RUN_TAXID}.fs.maptxt"
                map_info_msg "remapping ids"
                {
                    header_cluster && echo
                    ###$PYTHON3 $DIR_ORTHOLOGER/bin/remap.py -f "$tmpout" -m $f $oglevel --type 1 --outsep '\t'
                    $PYTHON3 $DIR_ORTHOLOGER/bin/remap.py -f "$tmpout" --labels "$PL_TODO" -d "$DIR_PROJECT/Rawdata" $oglevel --type 1 --outsep '\t'
                } > "$result_hits"
                [[ $? -eq 0 ]] || { map_error_msg "failed mapping ids for taxids from $PL_TODO"; return 255; }
            fi
            rm -f $tmpout
            chmod 644 "$result_hits"
            map_info_msg "generated cluster file with only user genes at $result_hits_ref"
            
            #---------------------------------------------
            # create cluster description file
            local hasDesc=0
            if (( _RUN_IS_LOCAL == 0 )) && (( noData == 0 )); then
                local fdescriptions=$MAP_ORTHODB_DATA/$DBI_DESCRIPTIONS/${_RUN_NODEID}.tsv
                if [[ -f $fdescriptions ]]; then
                    hasDesc=1
                    # extract all OG's (all but first line, first column)
                    local patternOGs=""
                    tmp_make patternOGs "patternOGs"

                    # The awk statement prints OG id preceeded by a '^' to mark the beginning of the line.
                    # It only processes non-empty lines.
                    # The sed statement prints all lines except the first one.
                    cat "$result_hits" | awk '!/^\s*$/{print "^"$1}' | sed -n '1d;p' | sort -u > $patternOGs
                    
                    # grep OG's from description tsv file
                    {
                        header_description && echo
                        grep -f $patternOGs $fdescriptions | sort -n
                    } > "$result_desc"
                fi
            fi
            # check if description is found
            if (( hasDesc == 0 )); then
                if (( _RUN_IS_LOCAL == 0 )); then
                    if (( noData != 0 )); then
                        map_warn_msg "no data found"
                        echo "# no data" > $result_desc
                    else
                        map_warn_msg "no description file found for clade ${_RUN_NODEID}"
                        map_warn_msg "file at: $fdescriptions"
                        echo "# missing OG descriptions for clade ${_RUN_NODEID}" > $result_desc
                    fi
                else
                    echo "# local map - no data" > $result_desc
                fi
            fi
            map_info_msg "generated description file at $result_desc_ref"

            #---------------------------------------------
            # merge user and description file
            if (( hasDesc == 0 )); then
                cp $result_hits $result_annot
            else
                {
                    ###header_cluster && printf '\t' && header_description_noclid && echo
                    $PYTHON3 $DIR_ORTHOLOGER/bin/create_annotated_clusters.py --fhits $result_hits --fdesc $result_desc
                } > $result_annot
            fi
            map_info_msg "generated annotated file at $result_annot_ref"
            
            #---------------------------------------------
            # remap all ids in the full result file
            local orgTodo="$DIR_PROJECT/todo/${_RUN_NODEID}.todo"
            local allMaps=""
            tmp_make allMaps "allMaps"
            map_info_msg "create map file in $allMaps"
            {
                while read line; do
                    local f="$DIR_PROJECT/Rawdata/${line}.fs.maptxt"
                    [[ -f $f ]] || continue
                    echo "$f"
                done < "$orgTodo"
                while read line; do
                    local f="$DIR_PROJECT/Rawdata/${line}.fs.maptxt"
                    [[ -f $f ]] || continue
                    echo "$f"
                done < "$PL_TODO"
            } > "$allMaps"

            #map_info_msg "DBG::all maps head n 30" 1>&2
            #head -n 30 $allMaps 1>&2
            {
                header_cluster && echo
                ###echo "# cmd $PYTHON3 $DIR_ORTHOLOGER/bin/remap.py -f $orgog -n $allMaps $oglevel --type 1 --outsep"
                $PYTHON3 $DIR_ORTHOLOGER/bin/remap.py -f "$orgog" -n $allMaps $oglevel --type 1 --outsep '\t'
            } > "$result_odb"
            [[ $? -eq 0 ]] || { map_error_msg "failed mapping all ids, file $result_odb_ref"; return 255; }
            chmod 644 "$result_odb"
            map_info_msg "generated cluster file with all genes at $result_odb_ref"

            #---------------------------------------------
            # rename fasta files and save in Results
            #---------------------------------------------
            map_info_msg "copying relevant fasta/map to Results"
            mkdir -p "$result_fs_dir" || {
                map_error_msg "failed creating result dir for fasta files: $result_fs_dir"
                return 255
            }
            #cp -f "$DIR_PROJECT/Rawdata/imported_${_RUN_TAXID}" "$result_fs" || { map_error_msg "failed copying $DIR_PROJECT/Rawdata/imported_${_RUN_TAXID} to $result_fs"; return 255; }
            for txid in ${_RUN_TAXIDS[@]}; do
                cp -f "$DIR_PROJECT/Rawdata/imported_$txid" "$result_fs_dir/" || {
                    map_error_msg "failed copying $DIR_PROJECT/Rawdata/imported_$txid to $result_fs_dir"
                    return 255
                }
            done
        fi
        
        map_info_msg "pipeline pass 2 : done"
        
        #-------------------------------------------------------------------------------------------------------------
        map_info_msg "check result"
        if ! is_dryrun; then
            map_get_ngenes "$PL_TODO_NAME"

            # create output
            local lines=()
            lines+=("Run mode               : $_RUN_MODE")
            lines+=("BUSCO auto lineage     : $usedBusco")
            if [[ $usedBusco == "yes" ]]; then
                lines+=("Complete BUSCOs        : $BUSCO_RES_COMPLETE")
            fi
            lines+=("Mapping node id        : $_RUN_NODEID")
            lines+=("Run label              : $PL_TODO_NAME")
            lines+=("N(taxids node)         : $(grep -v "^#" -c "$PL_TODO")")
            lines+=("N(subset)              : $_RUN_SUBSET")
            lines+=("N(subsample)           : $_RUN_SUBSAMPLE")
            lines+=("Subsample strategy     : $_RUN_SCOREDIR")
            lines+=("N(genes input)         : $RUN_NGENES")
            lines+=("N(genes mapped)        : $RUN_NGENES_MAPPED")
            lines+=("f(genes mapped) [%]    : $RUN_FMAPPED")
            lines+=("Time, busco (secs)     : $dTbusco")
            lines+=("Time, subsample (secs) : $dTsub")
            lines+=("Time, map (secs)       : $(delta_time $T0map)")
            lines+=("Time, total (secs)     : $(delta_time $T0busco)")
            
            local L
            map_info_msg "----------------------------------------------------------"
            for L in "${lines[@]}"; do
                map_info_msg "$L"
            done
            map_info_msg "----------------------------------------------------------"
            
            local fresult="$DIR_PROJECT/Results/${PL_TODO_NAME}.summary.txt"
            #if [[ -f $fresult ]]; then
            #    fresult="${fresult}.1"
            #    map_info_msg "earlier result file already exists - write with new name"
            #fi
            for L in "${lines[@]}"; do
                echo "$L"
            done > $fresult

            map_info_msg "summary in $fresult"
            map_write_metadata || { map_error_msg "failed writing metadata!"; return 255; }
        fi
        
    } >& $logfile
    ###} 2> $logfile
    
    map_info_msg "pipeline done"
    (( noData == 1 )) && map_warn_msg "no user data was mapped!"
    map_info_msg "final result at: $result_annot_ref"
    return 0
}

command_resetrun() {
    [[ $# -ge 1 ]] || { map_error_msg "usage: resetrun <run label>"; return 255; }
    map_check_pipeline || { map_error_msg "resetrun: pipeline is not setup, missing $CHECK_MISSING"; return 255; }
    local runlabel="$1"

    # get todo file
    map_set_todo "$runlabel"
    
    # move to pipeline
    cd $DIR_PROJECT || { map_error_msg "failed entering $DIR_PROJECT"; return 255; }

    # reset all steps to MAKE MISSING
    save_todo
    pipeline -t "$MAP_TODO" -f all:3
    restore_todo
}

command_delete() {
    if [[ $# -eq 0 ]]; then
        echo "available run labels:"
        map_get_metadata_list
        return $?
    fi
    local runlabel="$1"
    local opt=""
    [[ $# -ge 2 ]] && opt="$2"
    
    [[ ${opt,,} == "go" ]] || { DRY_RUN=1; map_warn_msg "add 'go' to the command line to actually perform the delete action"; }
    
    # goto pipeline root
    cd $DIR_PROJECT || { map_error_msg "failed entering $DIR_PROJECT"; return 255; }

    # loop over files
    local f=""
    local ndel=0
    for f in $(find $DIR_PROJECT/Cluster/ -maxdepth 1 -mindepth 1 -name "${runlabel}.*"); do
        if (( DRY_RUN == 1 )); then
            echo "$f"
            ndel=$((ndel+1))
        else
            rm -f "$f" && ndel=$((ndel+1))
        fi
    done
    for f in $(find $DIR_PROJECT/Results/ -maxdepth 1 -mindepth 1 -name "${runlabel}.*"); do
        if (( DRY_RUN == 1 )); then
            echo "$f"
            ndel=$((ndel+1))
        else
            rm -f "$f" && ndel=$((ndel+1))
        fi
    done
    for f in $(find $DIR_PROJECT/todo/ -maxdepth 1 -mindepth 1 -name "${runlabel}.pair*"); do
        if (( DRY_RUN == 1 )); then
            echo "$f"
            ndel=$((ndel+1))
        else
            rm -f "$f" && ndel=$((ndel+1))
        fi
    done
    for f in $(find $DIR_PROJECT/todo/ -maxdepth 1 -mindepth 1 -name "${runlabel}.todo"); do
        if (( DRY_RUN == 1 )); then
            echo "$f"
            ndel=$((ndel+1))
        else
            rm -f "$f" && ndel=$((ndel+1))
        fi
    done
    
    if (( DRY_RUN == 1 )); then
        if (( ndel==0 )); then
            map_info_msg "# found $ndel files"
        else
            map_info_msg "# found $ndel files - add 'go' to argument in order to delete them all"
        fi
    else
        map_info_msg "deleted $ndel files"
    fi
}

command_available() {
    # move to pipeline
    cd $DIR_PROJECT || { map_error_msg "failed entering $DIR_PROJECT"; return 255; }
    if ! chk_pid_file; then
        return 1
    fi
    return 0
}

status_busco() {
    map_info_msg "STATUS not yet implemented for BUSCO"
    return 0
}

command_status() {
    #[[ $# -ge 0 ]] || { map_error_msg "usage: status [<run label>]"; return 255; }
    #
    #_REQ_ARGS=0
    #parse_run_args status "$@" || return 255

    #if [[ "$_RUN_MODE" == "busco" ]]; then
    #    status_busco
    #    return $?
    #fi

    #cd $DIR_PROJECT || { map_error_msg "failed entering $DIR_PROJECT"; return 255; }
    local runlabel="${1-}"
    # find all todo's matching
    local t=""
    local tarr=()
    local tnid=""
    local ttxid=""
    local tsubn=""
    local tsubs=""
    printf "#%-64s %s\n" "label" "status"
    for t in $(map_get_todo_list); do
        local rlbl="$(basename $t .${EXT_TODO})"
        if [[ -n "$runlabel" ]]; then
            [[ "$rlbl" != "$runlabel" ]] && continue
        fi
        printf "%-64s" $rlbl
        pipeline -t "$t" -S
    done | sort
}

command_report() {
    [[ $# -ge 0 ]] || { map_error_msg "usage: report [<run label>]"; return 255; }
    map_check_pipeline || { map_error_msg "repor: pipeline is not setup, missing $CHECK_MISSING"; return 255; }
    if [[ $# -eq 0 ]]; then
        echo "available run labels:"
        map_get_metadata_list
        return $?
    fi
    local runlabel="$1"

    # get todo file with full path
    map_set_map_todo "$runlabel"
    [[ -f $MAP_TODO ]] || {
        map_error_msg "unknown runlabel <$runlabel> - a rerun command without a runlabel will print a list of all available"
        return 255
    }

    # read metadata for given runlabel
    map_read_metadata $runlabel 2>/dev/null || {
        map_error_msg "runlabel does not exist for metadata: $runlabel"
        return 255
    }
    
    # move to pipeline
    cd $DIR_PROJECT || { map_error_msg "failed entering $DIR_PROJECT"; return 255; }

    # find latest runlog
    local latestRunlog=$(ls $DIR_PROJECT_RUNLOGS/${runlabel}_*.log -tr | tail -n 1 2>/dev/null)
    local timeRunlog=""
    if [[ -f $latestRunlog ]]; then
        local tmpvar=$(grep 'Pipeline time elap' $latestRunlog 2>/dev/null)
	    [[ -z "$tmpvar" ]] || timeRunlog=$(echo "$tmpvar" | awk 'BEGIN{sum=0;} {sum+=$8;} END{print sum;}')
        if [[ -z "$timeRunlog" ]]; then
            #estimate from log message
            timeRunlog="N/A"
            local first=$(head -n1 $latestRunlog | awk '{print$1,$2}')
            local last=$(tail -n1 $latestRunlog | awk '{print$1,$2}')
            if [[ -n "$first" ]] && [[ -n "$last" ]]; then
                local T0=$(date --date="$first" +"%s" 2>/dev/null)
                local T1=$(date --date="$last" +"%s" 2>/dev/null)
                # there might be escape characters (color)
                [[ -z "$T0" ]] && T0=$(date --date="${first:5}" +"%s" 2>/dev/null)
                [[ -z "$T1" ]] && T1=$(date --date="${last:5}" +"%s" 2>/dev/null)
                [[ -n "$T0" ]] && [[ -n "$T1" ]] && timeRunlog=$((T1-T0))
            fi
        fi
    else
        latestRunlog="N/A"
        timeRunlog="N/A"
    fi
    # get results
    map_get_ngenes "$runlabel"
    #local f="$(calc 100.0*$nmapped/$ngenes)"
    printf "%-32s : %s\n"   "Label"               "$runlabel"
    printf "%-32s : %s\n"   "Mapping node id"     "$_RUN_NODEID"
    printf "%-32s : %s\n"   "N(taxids node)"      "$(grep -v "^#" -c "$MAP_TODO")"
    printf "%-32s : %s\n"   "Subset"              "$_RUN_SUBSET"
    printf "%-32s : %s\n"   "Subsample strategy"  "$_RUN_SCOREDIR"
    printf "%-32s : %s\n"   "N(subsample)"        "$_RUN_SUBSAMPLE"
    printf "%-32s : %s\n"   "N(genes input)"      "$RUN_NGENES"
    printf "%-32s : %s\n"   "N(genes mapped)"     "$RUN_NGENES_MAPPED"
    printf "%-32s : %.1f\n" "f(genes mapped) [%]" "$RUN_FMAPPED"
    printf "%-32s : %s\n"   "latest run log"      "$latestRunlog"
    printf "%-32s : %s\n"   "time according to log (secs)" "$timeRunlog"

    [[ "$latestRunlog" == "N/A" ]] || PIPELINE_RUNLOG="$latestRunlog"
    ###pipeline -t "$todo" -R "$_RUN_TAXID" -A all
    pipeline -t "$MAP_TODO" -A all
    restore_todo
}

# retrieves possible BUSCO result for given user taxid
command_busco_result() {
    if [[ $# -eq 0 ]]; then
        echo "available run labels:"
        map_get_metadata_list
        return $?
    fi
    cd $DIR_PROJECT || { map_error_msg "failed entering $DIR_PROJECT"; return 255; }
    
    # process args
    # [<label>] [taxid=<taxid>] [nodeid=<nodeid>] [nodeid=<nodeid>] [what=<nodeid|lineage|result|string|file>]
    local runlabel=""
    local nodeid=""
    local what="string"
    local check="0"
    local arg=""
    local arglist="$@"
    local n=0

    # reset global taxid
    _RUN_SINGLE_TAXID=""
    _RUN_NODEID=""
    
    # loop over args
    for arg in $arglist; do
        if [[ ${arg:0:5} == "what=" ]]; then
            what=${arg:5}
            continue
        fi
        if [[ ${arg:0:7} == "nodeid=" ]]; then
            nodeid=${arg:7}
            continue
        fi
        if [[ ${arg:0:6} == "check=" ]]; then
            check=${arg:6}
            [[ "$check" -eq "0" ]] || check="1"
            continue
        fi
        if [[ ${arg:0:6} == "taxid=" ]]; then
            _RUN_SINGLE_TAXID=${arg:6}
            [[ "$check" -eq "0" ]] || check="1"
            continue
        fi
        
        # non optional args
        (( n == 0 )) && runlabel="$1"
        n=$((n+1))
        shift
    done
    if [[ -n $runlabel ]]; then
        _RUN_SINGLE_TAXID=""
    fi
    if [[ -z $runlabel ]] && [[ -z $_RUN_SINGLE_TAXID ]]; then
        map_error_msg "busco_result: requires either a run label or a taxid (through taxid=<id>)"
        return 255
    fi

    if [[ -n $runlabel ]]; then
        # read metadata from pipeline/Results/<label>.metadata
        map_read_metadata $runlabel 2>/dev/null || {
            map_error_msg "runlabel does not exist for metadata: $runlabel"
            return 255
        }
    fi
    
    local ec=0
    # get BUSCO metadata , if it exists
    # The source is at pipeline/busco/<taxid>/.busco_metadata_<nodeid>.metadata
    # If nodeid is empty then it is set to 'auto'
    map_busco_set_dirs "$_RUN_SINGLE_TAXID"
    if [[ $check == "1" ]]; then
        # returns 1 if no metadata
        #         0 if ONE metadata file
        #         2 if more than one metadata file
        map_busco_get_metadata $_RUN_NODEID 2>/dev/null
        ec=$?
        if [[ $ec == 0 ]]; then
            map_info_msg "BUSCO result exist"
        else
            map_info_msg "no BUSCO result found"
        fi
        return $ec
    fi
    map_busco_read_metadata 2>/dev/null
    if [[ $? -ne 0 ]]; then
        echo "<NONE>"
        return 1
    fi
    # get the BUSCO result, labeled in two lines
    if [[ "$what" == "result" ]]; then
        echo "FBUS: $BUSCO_RES_FILE"
        echo "BUSC: $BUSCO_RES_COMPLETE"
        return 0
    fi

    # get the original BUSCO result string
    if [[ "$what" == "string" ]]; then
        echo "$BUSCO_RES_STRING"
        return 0
    fi

    # get the original BUSCO result file
    if [[ "$what" == "file" ]]; then
        echo "$BUSCO_RES_FILE"
        return 0
    fi
    
    # return the BUSCO name of selected lineage
    if [[ "$what" == "lineage" ]]; then
        echo "$BUSCO_LINEAGE"
        return 0
    fi

    # return the NCBI taxid of selected lineage
    if [[ "$what" == "nodeid" ]]; then
        echo "$BUSCO_LINEAGE_TAXID"
        return 0
    fi
    map_error_msg "busco_result: invalid 'what' keyword = $what"
    ARGS_PARSE_ERROR=4
    return 255
}

export RESULT_FOUND=0
export RESULT_ERRMSG=""
command_result() {
    RESULT_FOUND=0
    RESULT_ERRMSG=""

    if [[ $# -eq 0 ]]; then
        echo "# available run labels"
        map_get_metadata_list
        return $?
    fi
    local runlabel="$1"
    local what=${2-"what=ann"}
    if [[ $# -gt 1 ]]; then
        if [[ "${what:0:5}" != "what=" ]]; then
            map_error_msg "illegal extra option $what"
            map_error_msg "should be 'what=<all|fasta|map|cls|hits|odb|description|annotation>'"
            return 255
        fi
    fi
    # remove the 'what='
    what="${what:5}"
    
    # extract the first three letters, lower case
    what="${what:0:3}"
    what="${what,,}"

    # check runlabel argument
    if [[ -z $runlabel ]]; then
	    map_error_msg "empty runlabel argument"
	    return 255
    fi
    
    map_info_msg "retrieve results for $runlabel : $what"
    
    # read metadata with params for run
    map_read_metadata "$runlabel" 2>/dev/null || {
        map_error_msg "runlabel does not exist for metadata: $runlabel"
        return 255
    }

    if [[ "$_RUN_MODE" == "busco" ]]; then
        map_busco_set_dirs "$_RUN_SINGLE_TAXID"
        map_busco_read_metadata || { map_error_msg "missing BUSCO results"; return 255; }
        echo "FBUS: $BUSCO_RES_FILE"
        echo "BUSC: $BUSCO_RES_COMPLETE"
        return 0
    fi

    # mode is MAP

    # idem, for optional 'check' arg
    local onlyCheck=0

    cd $DIR_PROJECT || { map_error_msg "failed entering $DIR_PROJECT"; return 255; }
    local do_cluster=0
    local do_clusterhits=0
    local do_clusterdb=0
    local do_clusterdesc=0
    local do_clusterannot=0
    local do_fasta=0
    local do_map=0
    local do_any=0
    if   [[ $what == "all"   ]]; then
        do_cluster=1
        do_clusterhits=1
        do_clusterdb=1
        do_clusterdesc=1
        do_clusterannot=1
        do_fasta=1
        do_map=1
        do_any=1
    elif [[ $what == "fas" ]]; then
        do_fasta=1
        do_any=1
    elif [[ $what == "map" ]]; then
        do_map=1
        do_any=1
    elif [[ $what == "cls" ]]; then
        do_cluster=1
        do_any=1
    elif [[ $what == "hit" ]]; then
        do_clusterhits=1
        do_any=1
    elif [[ $what == "odb" ]]; then
        do_clusterdb=1
        do_any=1
    elif [[ $what == "des" ]]; then
        do_clusterdesc=1
        do_any=1
    elif [[ $what == "ann" ]]; then
        do_clusterannot=1
        do_any=1
    fi

    (( do_any > 0 )) || { ERROR_MSG="no valid 'what' argument is given ($what)"; return 255; }
    
    local f=""
    if (( do_cluster == 1 )); then
        f=$DIR_PROJECT/Cluster/${runlabel}.${EXT_CLUSTER}
        if [[ -f $f ]]; then
            printf "%s\n" "$f"
            RESULT_FOUND=$(( RESULT_FOUND += 1 ))
            (( onlyCheck == 1 )) && return 0
        fi
    fi
    if (( do_clusterhits == 1 )); then
        f=$DIR_PROJECT/Results/${runlabel}.${EXT_CLUSTER}.hits
        if [[ -f $f ]]; then
            printf "%s\n" "$f"
            RESULT_FOUND=$(( RESULT_FOUND += 1 ))
            (( onlyCheck == 1 )) && return 0
        fi
    fi
    if (( do_clusterdb == 1 )); then
        f=$DIR_PROJECT/Results/${runlabel}.${EXT_CLUSTER}.odb
        if [[ -f $f ]]; then
            printf "%s\n" "$f"
            RESULT_FOUND=$((RESULT_FOUND += 1))
            (( onlyCheck == 1 )) && return 0
        fi
    fi
    if (( do_clusterdesc == 1 )); then
        f=$DIR_PROJECT/Results/${runlabel}.${EXT_CLUSTER}.desc
        if [[ -f $f ]]; then
            printf "%s\n" "$f"
            RESULT_FOUND=$(( RESULT_FOUND += 1 ))
            (( onlyCheck == 1 )) && return 0
        fi
    fi
    if (( do_clusterannot == 1 )); then
        f=$DIR_PROJECT/Results/${runlabel}.${EXT_CLUSTER}.annotations
        if [[ -f $f ]]; then
            printf "%s\n" "$f"
            RESULT_FOUND=$(( RESULT_FOUND += 1 ))
            (( onlyCheck == 1 )) && return 0
        fi
    fi

    if (( do_fasta == 1 )); then
        if [[ ${#_RUN_TAXIDS[@]} -eq 0 ]]; then
            ERROR_MSG="taxids is required for 'fasta'"
            return 255
        fi
        for txid in ${_RUN_TAXIDS[@]}; do
            f=$DIR_PROJECT/Rawdata/${txid}.fs
            [[ -f $f ]] || { ERROR_MSG="missing fasta file at $f"; return 255; }
            printf "%s\n" "$f"
            RESULT_FOUND=$((RESULT_FOUND += 1))
        done
        (( onlyCheck == 1 )) && return 0
    fi
    if (( do_map == 1 )); then
        if [[ ${#_RUN_TAXIDS[@]} -eq 0 ]]; then
            ERROR_MSG="taxids is required for 'map'"
            return 255
        fi
        for txid in ${_RUN_TAXIDS[@]}; do
            f=$DIR_PROJECT/Rawdata/${txid}.fs.maptxt
            [[ -f $f ]] || { ERROR_MSG="missing fasta file at $f"; return 255; }
            RESULT_FOUND=$((RESULT_FOUND += 1))
            printf "%s\n" "$f"
            done
        (( onlyCheck == 1 )) && return 0
    fi
    return 0
}

command_hasresult() {
    command_result "$@" "check" >& /dev/null
    [[ $? -eq 0 ]] || return 1
    (( RESULT_FOUND == 0 )) && return 1
    return 0
}

command_dbinfo_user() {
    echo "NEEDS TO FIX THIS! TODO label is a arbitrary label"
    map_get_user_pipeline
    local todo=""
    print_separator '-'
    printf "%-24s %12s %12s\n" "clade" "node" "size"
    print_separator '-'

    for todo in $(find $DIR_PROJECT/todo -name "*.todo"); do
        local N=$(grep -c -v '^#' "$todo")
        local node=$(basename "$todo" .todo)
        [[ "${node:0:5}" == "node_" ]] && continue
        local clade=$(map_get_clade "$node")
        printf "%-24s %12s %12s\n" "$clade" "$node" "$N"
    done
}

command_dbinfo() {
    echo "NEEDS TO FIX THIS! TODO label is a arbitrary label"
    print_separator '-'
    printf "%-30s %s\n" "Data base root"  "$MAP_ORTHODB_DATA"
    printf "%-30s %s\n" "Max nr of nodes" "${#MAP_ALL_NODES[@]}"
    print_separator '-'
    echo
    if [[ -n $MAP_UID_HASH ]]; then
        command_dbinfo_user
        return $?
    fi
    
    local clade=""
    local todovar=""
    local rootdir=""
    local rootvar=""
    local flag=""
    local size=0

    print_separator '-'
    printf "%-20s %12s\n" "clade" "node"
    print_separator '-'
    for clade in $(find $MAP_ORTHODB_DATA/ -maxdepth 1 -mindepth 1 -type d); do
        [[ -d $clade/todo ]] || continue
        for todo in $(find $clade/todo -name "*.todo"); do
            local node="$(basename $todo .todo)"
            [[ "${node:0:1}" == "g" ]] && continue
            local C=$(map_get_clade "$node")
            printf "%-20s %12s\n" "$C" "$node"
        done
    done
    echo
}

command_findnodes() {
    local taxid="$1"
    local found=0
    for clade in ${MAP_ROOT_CLADES[@]}; do

        # get array variable name for given clade (defined in orthomapper_conf.sh)
        todovar="MAP_NODES_${clade^^}[@]"
        
        # get root directory for this clade
        rootdir="$(map_dir_clade ${clade,,})"
        tododir=${rootdir}/todo
        [[ -d $tododir ]] || continue
        #for x in $(grep '^'$taxid'_' $tododir/*.todo | cut -d ':' -f 1); do
        local x=
        local y=
        for x in $tododir/*.todo; do
            grep '^'$taxid'_' $x >& /dev/null
            if [[ $? -eq 0 ]]; then
                found=1
                basename $x .todo
                break
            fi
        done
        (( found == 0 )) || break
    done
    (( found == 0 )) && return 1
    return 0
}

command_package() {
    if [[ $# -eq 0 ]]; then
        echo "available run labels:"
        map_get_metadata_list
        return $?
    fi
    local runlabel=$1
    shift
    local options="$@"
    local opt=""
    local opt_check=0
    local olc=""
    for opt in $options; do
        olc=${opt,,}
        [[ $olc == "check" ]] && { opt_check=1; continue; }
        map_error_msg "unknown argument $opt"
        return 255
    done
    
    [[ $opt_check -eq 1 ]] && { DRY_RUN=1; map_info_msg "will make full check that all results exists"; }

    local ec=0

    # get metadata
    map_read_metadata $runlabel 2>/dev/null || {
        map_error_msg "runlabel does not exist for metadata: $runlabel"
        return 255
    }

    # set BUSCO dirs, if relevant
    map_busco_set_dirs "$_RUN_SINGLE_TAXID"
    map_busco_read_metadata 2>/dev/null || BUSCO_RES_FILE=""
    
    # result files
    local ftgz="$DIR_PROJECT/${runlabel}.tgz"
    [[ -f $ftgz ]] && { echo $ftgz; return 0; }

    local fann="$DIR_PROJECT/Results/${runlabel}.${EXT_CLUSTER}.annotations"
    local fdsc="$DIR_PROJECT/Results/${runlabel}.${EXT_CLUSTER}.desc"
    local fusr="$DIR_PROJECT/Results/${runlabel}.${EXT_CLUSTER}.hits"
    local fodb="$DIR_PROJECT/Results/${runlabel}.${EXT_CLUSTER}.odb"
    local ffst="$DIR_PROJECT/Results/${runlabel}.fs"
    local fsum="$DIR_PROJECT/Results/${runlabel}.summary.txt"
    
    [[ -f $fann   ]] || { map_error_msg "missing odb annotated file $fann";   return 255; }
    [[ -f $fdsc   ]] || { map_error_msg "missing odb desc file $fdsc";        return 255; }
    [[ -f $fodb   ]] || { map_error_msg "missing odb file $fodb";             return 255; }
    [[ -f $fusr   ]] || { map_error_msg "missing usr file $fusr";             return 255; }
    [[ -f $ffst   ]] || { map_error_msg "missing fasta file $ffst";         return 255; }
    [[ -f $fsum   ]] || { map_error_msg "missing summary file $fsum";         return 255; }

    if (( DRY_RUN == 1 )); then
        map_info_msg "all result files exists - rerun with 'go' in order to create package"
        return 0
    fi
    
    # create build directory
    local tmpdir=""
    tmpdir=$(mktemp -d '/tmp/package_XXXXXX') || { map_error_msg "unable to create root temp dir for package"; return 255; }
    local pkgroot=$tmpdir/$runlabel
    mkdir -p $pkgroot || { map_error_msg "unable to create temp dir for package at $pkgroot"; return 255; }

    # copy files there
    fbusco=${runlabel}.busco.summary.txt
    [[ -f $BUSCO_RES_FILE ]] && cp $BUSCO_RES_FILE $pkgroot/$fbusco
    cp $fann   $pkgroot/${runlabel}.og
    cp $ffst   $pkgroot/${runlabel}.fs
    cp $fsum   $pkgroot/

    command_report $runlabel > $pkgroot/${runlabel}.report
    
    local autostr="NO"
    (( BUSCO_RES_AUTO == 1 )) && autostr="YES"
    
    {
        printf "%-60s : %s\n" "Orthologer version" "$OP_VERSION"
        printf "%-60s : %s\n" "Time of run"     "$(date)"
        printf "%-60s : %s\n" "User"            "$SYS_WHOAMI"
        printf "%-60s : %s\n" "Auto lineage"    "$autostr"
        printf "%-60s : %s\n" "Lineage (taxid)" "$_RUN_NODEID"
        [[ -f $BUSCO_RES_FILE ]] && printf "%-60s %s\n" "$fbusco" "busco summary"
        printf "%-60s %s\n" "${runlabel}.og" "orthologs found with annotation"
        printf "%-60s %s\n" "${runlabel}.fs" "original fasta file"
        printf "%-60s %s\n" "${runlabel}.report" "detailed timing stats per step"
        printf "%-60s %s\n" $(basename "$fsum") "various stats"
    }> $pkgroot/README
    
    # create tarball
    cd $tmpdir
    tar -czf $(basename $ftgz) $runlabel  || { map_error_msg "failed creating package in $tmpdir"; rm -Rf $tmpdir/; return 255; }

    mv -f $(basename $ftgz) $ftgz || { map_error_msg "unable to move package from $tmpdir to $DIR_PROJECT"; return 255; }
    map_info_msg "created package in $ftgz"
    echo "$ftgz"
    rm -Rf $tmpdir

}

command_clear() {
    [[ $# -ge 2 ]] || { map_error_msg "usage: clear <node id> <taxid> [<subnode>]"; return 255; }
    map_error_msg "CLEAR - not yet implemented!"
    return 255

    #local nodeid=$1
    #local taxid=$2
    #local subnode=""
    #[[ $# -gt 2 ]] && subnode="$3"
    local runlabel="$1"
    
    local ec=0
    map_set_todo "$runlabel"

    # no todo file -> nothing to delete
    [[ -f $MAP_TODO ]] || return 0

    cd $DIR_PROJECT || { map_error_msg "failed entering $DIR_PROJECT"; return 255; }
}

dir_ok_if_empty() {
    local d="$1"
    [[ -z "$d" ]] && return 0
    [[ -d "$d" ]] && return 0
    return 1
}

file_ok_if_empty() {
    local d="$1"
    [[ -z "$d" ]] && return 0
    [[ -f "$d" ]] && return 0
    return 1
}

command_check() {
    local ec=0
    local N=0
    echo
    echo "----------------------------------------------------------------------------"
    [[ -f "$DIR_ORTHOLOGER/bin/common_base.sh" ]]; ec=$?; (( ec )) && N=255; xec_status_msg $ec "Orthologer installed"
    [[ -d "$MAP_ORTHODB_DATA" ]];                      ec=$?; (( ec )) && N=255; xec_status_msg $ec "orthomapper_conf.sh    : MAP_ORTHODB_DATA"
    dir_ok_if_empty  "$DBI_DOWNLOAD"                   ec=$?; (( ec )) && N=255; xec_status_msg $ec "orthomapper_conf.sh    : DBI_DOWNLOAD"
    dir_ok_if_empty  "$BUSCO_DATA"                     ec=$?; (( ec )) && N=255; xec_status_msg $ec "orthomapper_conf.sh    : BUSCO_DATA"
    
    
    local vdir=""
    vdir=$(file_getvar orthologer_conf.sh DIR_BRHCLUS )
    [[ -d "$vdir" ]]; ec=$?; (( ec )) && N=255; xec_status_msg $ec "orthologer_conf.sh : DIR_BRHCLUS"
    vdir=$(file_getvar orthologer_conf.sh DIR_BLAST )
    [[ -d "$vdir" ]]; ec=$?; (( ec )) && N=255; xec_status_msg $ec "orthologer_conf.sh : DIR_BLAST"
#    vdir=$(file_getvar orthologer_conf.sh DIR_MMSEQS )
#    [[ -d "$vdir" ]]; ec=$?; (( ec )) && N=255; xec_status_msg $ec "orthologer_conf.sh : DIR_MMSEQS"
    vdir=$(file_getvar orthologer_conf.sh DIR_DIAMOND )
    [[ -d "$vdir" ]]; ec=$?; (( ec )) && N=255; xec_status_msg $ec "orthologer_conf.sh : DIR_DIAMOND"
    vdir=$(file_getvar orthologer_conf.sh DIR_CDHIT )
    [[ -d "$vdir" ]]; ec=$?; (( ec )) && N=255; xec_status_msg $ec "orthologer_conf.sh : DIR_CDHIT"
    echo "----------------------------------------------------------------------------"
    echo
    return $N
}

run_mapping() {
    local projLabel="$1"
    parse_input_file "$2" || return 255

    # get target node
    local trgNodeId=""
    local extraArgs=""
    [[ $# -lt 3 ]] || trgNodeId="$3"
    [[ $trgNodeId == "-" ]] && trgNodeId=""

    # if input is several fasta files, explicit target node is required
    if [[ ${#FASTA_FILES[@]} -gt 1 ]] && [[ -z $trgNodeId ]]; then
        map_error_msg "target node ID is required with multiple fasta files given"
        return 255
    fi
    
    # shift to get extra args
    shift
    shift
    [[ $# -ge 1 ]] && shift
    local extraArgs="$@"
    if [[ ${#extraArgs} -gt 0 ]]; then
        map_info_msg "run map with extra args: $extraArgs"
        map_parse_special_args "$@" || return 255
        if (( _RUN_BY_PROFILE == 1 )); then
            map_info_msg "run mapping using mmseqs profiles"
        else
            local dbistr="SUBSAMPLED"
            (( DBI_USE_FULL == 1 )) && dbistr="FULL"
            map_info_msg "run mapping using $dbistr data"
        fi
        if [[ ${#_RUN_SUBSET} -gt 1 ]]; then
            map_info_msg "will use subset taxids: $_RUN_SUBSET"
        fi
    fi
    
    map_info_msg "---------------------------------------"
    map_info_msg "Initialise environment if needed"
    map_info_msg "---------------------------------------"
    command_init || return 255

    map_info_msg "---------------------------------------"
    map_info_msg "Load local config (orthologer_conf.sh)"
    map_info_msg "---------------------------------------"
    map_load_orthologer_conf || {
        map_error_msg "unable to read orthologer_conf.sh at $DIR_PROJECT/orthologer_conf.sh"
        return 255
    }

    map_info_msg "---------------------------------------"
    map_info_msg "Setup ODB environment"
    map_info_msg "---------------------------------------"
    setup_environment || return 255

    if map_node_is_orthodb "$trgNodeId"; then
        # download the requested OrthoDB node id
        map_info_msg "---------------------------------------"
        map_info_msg "Download ODB $trgNodeId"
        map_info_msg "---------------------------------------"
        command_download  "$trgNodeId" || return 255

        # import it
        map_info_msg "---------------------------------------"
        map_info_msg "Import ODB $trgNodeId to project"
        map_info_msg "---------------------------------------"
        command_dbimport  "$trgNodeId" || return 255
    fi
    
    # import the fasta file(s) with auto taxid
    map_info_msg "---------------------------------------"
    map_info_msg "Import ${#FASTA_FILES[@]} fasta file(s)"
    map_info_msg "---------------------------------------"
    local ffile=""
    local taxid=""
    local index=0
    for (( index=0; index<${#FASTA_FILES[@]}; index++ )); do
        ffile=${FASTA_FILES[$index]}
        taxid=${FASTA_TAXIDS[$index]}
        map_info_msg "import fasta file (id = $taxid) -> $(trim_name $ffile)"
        command_import "${taxid}:$ffile" || return 255
    done

    if [[ ${#FASTA_TAXIDS[@]} -gt 1 ]]; then
        error_msg "Run mapping - does not yet fully handle multiple input fasta files!!"
        return 255
    fi
    
    map_info_msg "---------------------------------------"
    map_info_msg "Run mapping"
    map_info_msg "---------------------------------------"
    if [[ -n "$trgNodeId" ]]; then
        # run using given OrthoDB node
        command_run nodeid="$trgNodeId" label="$projLabel" $extraArgs "GO" || return 255
    else
        # run using BUSCO to establish target node id
        command_run label="$projLabel" $extraArgs "GO" || return 255
    fi
    
    return 0
}


#-----------------------------------------------------------------------------------
# first arg - the command is retrieved at the top ( __command )
# shift out command for args
shift

runcmd=""

# flag what kind of uid; 0->raw, 1->hashed
whatuid=0

# check DB or not
checkdb=0

# nr of required arguments
nreqargs=0
cmdargs=""

# quiet - no log msgs
quiet=0

# req_project - 1 -> requires project a project to be setup (incl orthologer_conf.sh)
req_project=1

EXTENDED_HELP=0
case "${__command}" in
    "LIST"      ) checkdb=0
                  quiet=1
                  runcmd=command_list
                  nreqargs=0
                  cmdargs="[fasta|taxid|nodes]"
                  ;;
    "CREATE"    ) checkdb=0
                  runcmd=command_create
                  nreqargs=1
                  cmdargs="[<source label>] [go]"
                  ;;
    "INFO"      ) checkdb=0
                  quiet=1
                  req_project=0
                  cmdargs="[what]"
                  runcmd=command_info
                  ;;
    "IMPORT"    ) checkdb=0
                  runcmd=command_import
                  cmdargs="\"<txid1:file1>\" [\"<txid2:file2>\" \"<txid3:file3>\" ... ]"
		          nreqargs=1
                  ;;
    "FINDNODES" ) checkdb=0
                  quiet=1
                  runcmd=command_findnodes
                  cmdargs="<taxid>"
		          nreqargs=1
                  ;;
    "DOWNLOAD"  ) checkdb=0
                  quiet=0
                  runcmd=command_download
                  cmdargs="<node taxid | ?<clade> > [full]"
		          nreqargs=1
                  ;;
    "DBIMPORT"  ) checkdb=0
                  runcmd=command_dbimport
                  cmdargs="<nodeid> [go]"
		          nreqargs=1
                  ;;
    "INITRUN"   ) checkdb=1
                  runcmd=command_initrun
		          nreqargs=1
                  cmdargs="<taxid> [nodeid=<id>] [subsample=<N>] [using=<subnode|taxids>]"
                  ;;
    "RUN"       ) checkdb=1
                  quiet=0
                  runcmd=command_run
                  cmdargs="<taxid> [nodeid=<id>] [subsample=<N>] [using=<subnode|taxids>] [check] [go]"
		          nreqargs=1
                  ;;
    "MAP"       ) checkdb=1
                  quiet=0
                  runcmd=NONE
                  cmdargs="<label> <fasta file> [<target node id>]"
		          nreqargs=2
                  ;;
    "RUNLABEL"  ) checkdb=0
                  quiet=1
                  runcmd=command_runlabel
                  cmdargs="<taxid> [nodeid=<id>] [subsample=<N>] [using=<subnode|taxids>]"
		          nreqargs=1
                  ;;
    "RESETRUN"  ) checkdb=1
                  quiet=1
                  runcmd=command_resetrun
                  cmdargs="<runlabel> [check] [go]"
		          nreqargs=1
                  ;;
    "DELETE"    ) checkdb=0
                  quiet=0
                  runcmd=command_delete
		          nreqargs=0
                  cmdargs="[<run label> go]"
                  ;;
    "REPORT"    ) checkdb=0
                  quiet=1
                  nreqargs=0
                  runcmd=command_report
                  cmdargs="[<runlabel>]"
                  ;;
    "RESULT"    ) checkdb=0
		          quiet=1
                  nreqargs=0
                  runcmd=command_result
                  cmdargs="[<run label>] [what=fasta|map|cluster|all]"
                  ;;
    "BUSCO_RESULT") checkdb=0
		            quiet=1
                    nreqargs=0
                    runcmd=command_busco_result
                    cmdargs="[<run label>] [what=<nodeid|lineage|result|string|file>]"
                    ;;
    "HASRESULT" ) checkdb=0
		          quiet=1
                  nreqargs=1
                  runcmd=command_hasresult
                  cmdargs="<run label> [what=fasta|map|cluster|all]"
                  ;;
    "STATUS"    ) checkdb=0
                  quiet=1
                  nreqargs=0
                  runcmd=command_status
                  cmdargs="[<taxid>] [nodeid=<id>] [subsample=<N>] [using=<subnode|taxids>]"
                  ;;
    "AVAILABLE" ) checkdb=0
                  quiet=1
                  nreqargs=0
                  runcmd=command_available
                  cmdargs=""
                  ;;
    "DBINFO"    ) checkdb=0
                  quiet=1
                  nreqargs=0
                  runcmd=command_dbinfo
                  cmdargs=""
                  ;;
    "PACKAGE"   ) checkdb=0
                  quiet=1
                  runcmd=command_package
		          nreqargs=0
                  cmdargs="[<run label>] [check] [go]"
                  ;;
    "CONFIG"  )   checkdb=0
                  quiet=1
                  nreqargs=0
                  req_project=0
                  runcmd=command_config
                  ;;
    "CHECK"   )   checkdb=0
                  quiet=1
                  nreqargs=0
                  runcmd=command_check
                  ;;
    *           ) ARGS_PARSE_ERROR=4; map_error_msg "invalid command given ($__command)"; exit 255;;
esac

(( VERBOSE == 0 )) || quiet=0

if (( quiet == 1 )); then
    QUIET_INFO_MSG=1
fi
map_info_msg "--------------------------------------"
map_info_msg "mapping command : $__command"
map_info_msg "arguments       : $@"
map_info_msg "--------------------------------------"

#-----------------------------------------------------------------------------------
# check nr of args
nargs=$#
# check nr of arguments for command
(( nargs < nreqargs )) && {
    map_error_msg "usage: orthomapper.sh $__command $cmdargs"
    ARGS_PARSE_ERROR=4
    exit 255
}

#-----------------------------------------------------------------------------------
# MAP -> main simple command
#-----------------------------------------------------------------------------------
if [[ "$__command" == "MAP" ]]; then
    run_mapping "$@"
    exit $?
fi

#-----------------------------------------------------------------------------------
# process any other command
#-----------------------------------------------------------------------------------
map_info_msg "load orthologer config"
if ! map_load_orthologer_conf; then
    ###[[ "$__command" == "INIT" ]] || {
    (( req_project == 0 )) || {
        map_error_msg "failed loading project orthologer_conf.sh"
        exit 255
    }
    debug_msg "map_load_conf failed but the command is INIT so its ok"
fi
    
# setup environment
debug_msg "setup environment"
setup_environment || exit 255


# the original args have the command shifted out, as well as the user (if there)
debug_msg "starting command $runcmd"
$runcmd "$@"
ec=$?
if [[ $ec -ne 0 ]]; then
    [[ -n $ERROR_MSG ]] && map_error_msg "$ERROR_MSG"
fi
exit $ec
