#!/usr/bin/env bash

if [[ "${LOADED_ORTHOMAPPER_FUN_SH+1}" == "" ]]; then
    
# Mapping functions
set -u

# load pipeline
[[ -f "$DIR_ORTHOLOGER/bin/common_odb.sh" ]] || { echo "ERROR: file not found $DIR_ORTHOLOGER/bin/common_odb.sh" 1>&2; exit 255; }
source "$DIR_ORTHOLOGER/bin/common_odb.sh"

# make sure output is colorful
export PRETTY_PRINT=1
reset_attributes

# make sure it's an absolute path
export MAP_ORTHODB_DATA=${MAP_ORTHODB_DATA="$(pwd)/data"}
if [[ -d $MAP_ORTHODB_DATA ]]; then
    MAP_ORTHODB_DATA=$(cd $MAP_ORTHODB_DATA && pwd)
else
    MAP_ORTHODB_DATA=""
fi

update_orthomapper_config() {
    local conf="$DIR_PROJECT/orthomapper_conf.sh"
    if [[ -f $conf ]]; then
        file_setvar $conf MAP_ORTHODB_DATA    "$MAP_ORTHODB_DATA"
        file_setvar $conf BUSCO_ENABLED       "$BUSCO_ENABLED"
        file_setvar $conf BUSCO_DATA          "$BUSCO_DATA"
        file_setvar $conf BUSCO_ODB_VERSION   "$BUSCO_ODB_VERSION"
        file_setvar $conf BUSCO_NCPUS         "$BUSCO_NCPUS"
        file_setvar $conf BUSCO_OFFLINE       "$BUSCO_OFFLINE"
        file_setvar $conf BUSCO_DOCKER_IMAGE  "$BUSCO_DOCKER_IMAGE"
        file_setvar $conf BUSCO_CMD           "$BUSCO_CMD"
    fi
}

dump_orthomapper_config() {
    local conf="$DIR_PROJECT/orthomapper_conf.sh"
    {
        cat <<EOF        
#--------------------------------------------------------------------
# User config

# OrthoDB downloaded data directory - if not set, it will be in user /tmp
export DBI_DOWNLOAD=""

# Database root, should contain one directory per root clade defined (MAP_ROOT_CLADES), name in lower case
export MAP_ORTHODB_DATA="$MAP_ORTHODB_DATA"

# BUSCO enabled
export BUSCO_ENABLED=$BUSCO_ENABLED

# BUSCO command - to be used is docker image is not available
export BUSCO_CMD="$BUSCO_CMD"

# BUSCO docker image - to be used if BUSCO is not locally installed (BUSCO_CMD)
export BUSCO_DOCKER_IMAGE="$BUSCO_DOCKER_IMAGE"

# BUSCO data - directory where all BUSCO data is downloaded
export BUSCO_DATA="$BUSCO_DATA"

# BUSCO orthodb version string (eg odb10)
export BUSCO_ODB_VERSION="$BUSCO_ODB_VERSION"

# Nr of CPUs available for use
export BUSCO_NCPUS=$BUSCO_NCPUS

# Run BUSCO offline - assumes the lineages already exists in BUSCO_DATA
export BUSCO_OFFLINE=$BUSCO_OFFLINE

EOF
    } > $conf
    chmod +x orthomapper_conf.sh
}


#--------------------------------------------------------------------------------------------------------------------
# post-process MAP_ variable

# get clade for given node (NCBI taxid or 'auto')
# return:
#  0   - clade found
#  1   - node is 'auto'
#  255 - node is not in any OrthoDB clade
map_get_clade() {
    local node="$1"
    if [[ "$node" == "auto" ]]; then
        echo "<AUTO>"
        return 1
    fi
    local ec=0
    if [[ "${MAP_NODE2CLADE[$node]+_}" ]]; then
        echo "${MAP_NODE2CLADE[$node]}"
    else
        echo "<UNKOWN:$node>"
        ec=255
    fi
    return $ec
}

map_node_is_orthodb() {
    [[ " ${MAP_ALL_NODES[@]} " =~ " $1 " ]]
}

# check if given node is part of viridae tree
map_node_is_viridae() {
    [[ " ${MAP_NODES_VIRIDAE[@]} " =~ " $1 " ]]
}

map_set_nodeid() {
    [[ $# -eq 1 ]] || return 0
    _RUN_NODEID=$1
    _RUN_NODEID_IS_VIRIDAE=0
    map_node_is_viridae "$_RUN_NODEID" && _RUN_NODEID_IS_VIRIDAE=1
    return 0
}

# get default subsample label
map_subsample_label() {
    local nodeid=$_RUN_NODEID
    local subsample=$_RUN_SUBSAMPLE
    local scoredir=$_RUN_SCOREDIR
    if [[ $# -eq 3 ]]; then
        nodeid="$1"
        subsample="$2"
        scoredir="$3"
    fi
    if (( _RUN_NODEID_IS_VIRIDAE == 1 )); then
        echo $nodeid
    else
        echo "g${nodeid}_n${subsample}${scoredir}"
    fi
}
map_is_orthodb_nodeid() {
    local v="$1"
    for x in ${MAP_ALL_NODES[@]}; do
        [[ "$x" == "$v" ]] && return 0
    done
    return 1
}

#--------------------------------------------------------------------------------------------------------------------
export _RUN_CMD=""
export _RUN_LABEL_WAS_GIVEN=0
export _RUN_LABEL=""
export _RUN_MODE=""
export _RUN_NODEID=""
export _RUN_NODEID_IS_VIRIDAE=0
#-----------------------------------
# _RUN_SINGLE_TAXID -> _RUN_TAXIDS[0] is only one taxid (iff taxid is given by 'taxid=')
# _RUN_TAXIDS -> array of ALL taxids to be imported
# _RUN_HASHID -> hash of the above array
# Use _RUN_HASHID in _RUN_LABEL
# Set _RUN_HASHID to _RUN_TAXIDS[0] if only one taxid is being processed
# Set runlabel using the above hashid.
# - RUN_LABEL = "node_${_RUN_NODEID}_${sublbl}_label_${RUN_HASHID}"
#
export _RUN_SINGLE_TAXID=""
export _RUN_TAXIDS=()
export _RUN_HASHID=""
#-----------------------------------
export _RUN_SUBSET=""
export _RUN_SUBSAMPLE="8"
export _RUN_SCOREDIR="ssc"
export _RUN_SUBSAMPLE_LABEL=""
export _RUN_GO=0
export _RUN_CHECK=0
export _RUN_DO_SUBSAMPLE=0
export _RUN_BY_PROFILE=0
export _RUN_IS_LOCAL=0

map_parse_special_args() {
    
    _RUN_SINGLE_TAXID=""
    _RUN_TAXIDS=()
    _RUN_HASHID=""
    _RUN_LABEL_WAS_GIVEN=0
    _RUN_LABEL=""
    _RUN_MODE="map"
    _RUN_NODEID=""
    _RUN_SUBSET=""
    _RUN_SUBSAMPLE="$DBI_SAMPLED_N"
    _RUN_SCOREDIR="$DBI_SAMPLED_SCDIR"
    _RUN_SUBSAMPLE_LABEL=""
    _RUN_GO=0
    _RUN_CHECK=0
    _RUN_IS_LOCAL=0

    # this flags if subsampling is to be done
    # 0 -> no subsample -> DBI_USE_FULL = 1
    # 1 -> subsample using a given subset of taxids  -> DBI_USE_FULL = 1
    # 2 -> subsample using non-standard parameters  -> DBI_USE_FULL = 1
    # 3 -> subsampled data is downloaded (no local subsampling)  -> DBI_USE_FULL = 0
    _RUN_DO_SUBSAMPLE=0

    # only set if mode==mapprof
    _RUN_BY_PROFILE=0

    # flag if NODEID is part of viridae tree
    _RUN_NODEID_IS_VIRIDAE=0
    
    # browse options
    local options="$@"
    local opt=""
    local olc=""
    local res=()

    # flag what kind of subsampling
    # by default it uses official OrthoDB subsampled set
    local bySubset=0
    local bySubsample=0

    for opt in $options; do
        olc=${opt,,}
        [[ $olc == "check" ]] && { _RUN_CHECK=1; continue; }
        [[ $olc == "go"    ]] && { _RUN_GO=1;    continue; }
        res=($(tokenize "=" "$opt"))
        olc=${res[0]}
        olc=${olc,,}
        local val=""
        [[ ${#res[@]} -ne 2 ]] || val=${res[1]}
        local isauto=0
        [[ "${val,,}" == "auto" ]] && isauto=1
           
        if [[ "$olc" == "using" ]]; then
            #this is for mapping against a given set of taxids
            _RUN_SUBSET=""
            if (( isauto == 0 )); then
                _RUN_SUBSET=$(sort_csv "$val")
                bySubset=1
            fi
            continue

        elif [[ "$olc" == "subsample" ]]; then
            #this is for subsampling
            _RUN_SUBSAMPLE="$val"
            bySubsample=1
            continue

        elif [[ "$olc" == "score" ]]; then
            #this is for subsampling
            _RUN_SCOREDIR=$(map_get_scdir_value "${val,,}")
            [[ -n $_RUN_SCOREDIR ]] || { map_error_msg "parse_run_arg: illegal scdir argument $val"; return 255; }
            bySubsample=1
            continue

        elif [[ "$olc" == "mode" ]]; then
            # select mode, either map or busco
            if [[ "${val,,}" == "map" ]]; then
                _RUN_MODE="map"
            elif [[ "${val,,}" == "busco" ]]; then
                _RUN_MODE="busco"
            elif [[ "${val,,}" == "mapprof" ]]; then
                _RUN_MODE="mapprof"
                _RUN_BY_PROFILE=1
            else
                map_error_msg "unknown argument to 'mode' option $opt = ${val,,}"
                return 255
            fi
            
        elif [[ "$olc" == "taxid" ]]; then
            local taxid=$(map_cleanup_taxid $val)
            _RUN_TAXIDS=( "$taxid" )
            _RUN_HASHID="$taxid"
            #_RUN_HASHID=$(echo $taxid | md5sum)
            #_RUN_HASHID=${_RUN_HASHID:0:8}
            _RUN_SINGLE_TAXID="$_RUN_HASHID"
            map_info_msg "using explicit taxid = $taxid , hash id = $_RUN_HASHID"
            
        elif [[ "$olc" == "nodeid" ]]; then
            # select which node to use
            _RUN_NODEID=""
            if (( isauto == 0 )); then
                _RUN_NODEID="${val}"
                local prefix="${val:0:6}"
                if [[ "$prefix" == "local:" ]]; then
                    _RUN_IS_LOCAL=1
                    _RUN_NODEID="${val:6}"
                fi
            fi
            
        elif [[ "$olc" == "label" ]]; then
            # optional label for run
            _RUN_LABEL="${val}"
            _RUN_LABEL_WAS_GIVEN=1
        else
            map_error_msg "parse_run_arg: unknown argument $opt"
            return 255
        fi
    done

    if [[ ${#_RUN_TAXIDS[@]} -eq 0 ]]; then
        _RUN_TAXIDS=( ${FASTA_TAXIDS[@]} )
        _RUN_HASHID=$(echo $val | md5sum | cut -f 1 -d ' ')
        _RUN_HASHID=${_RUN_HASHID:0:8}
        map_info_msg "using full set of taxids, N = ${#_RUN_TAXIDS[@]}, hash id = $_RUN_HASHID"
    fi
    if [[ ${#_RUN_TAXIDS[@]} -eq 1 ]]; then
        _RUN_SINGLE_TAXID="${_RUN_TAXIDS[0]}"
    fi
    
    # special case - nodeid is local : 'local:<todo label>'
    if (( _RUN_IS_LOCAL )); then
        _RUN_NODEID_IS_VIRIDAE=0
        _RUN_BY_PROFILE=0
        _RUN_DO_SUBSAMPLE=0
        _RUN_SUBSAMPLE="0"
        _RUN_SCOREDIR=""
        _RUN_SUBSET=""
        return 0
    fi
    
    # special case if nodeid is in viridae tree
    if map_node_is_viridae "$_RUN_NODEID"; then
        map_info_msg "run using full data set with no subsampling - node ($_RUN_NODEID) is part of viridae tree"
        _RUN_NODEID_IS_VIRIDAE=1
        _RUN_BY_PROFILE=0
        _RUN_DO_SUBSAMPLE=0
        _RUN_SUBSAMPLE="0"
        _RUN_SCOREDIR=""
        _RUN_SUBSET=""
        _RUN_SUBSAMPLE_LABEL=""
        DBI_USE_FULL=1
        return 0
    fi

    # if profile, then subsampling is not relevant
    if (( _RUN_BY_PROFILE == 1 )); then
        map_info_msg "will run using new experimental MAPPROF - map using mmseqs profiles"
        _RUN_SUBSET=""
        _RUN_SUBSAMPLE="0"
        _RUN_SCOREDIR=""
        _RUN_SUBSAMPLE_LABEL=""
        _RUN_DO_SUBSAMPLE=0
        return 0
    fi

    # just to make sure
    [[ -z $_RUN_SUBSAMPLE ]] && _RUN_SUBSAMPLE="0"

    # set default mapping source
    if (( bySubset == 0 && bySubsample == 0 )); then
        bySubsample=1
    fi
    #-----------------------------------------------------
    # force DBI_USE_FULL if "using=" was requested
    # this has priority over subsample scenario
    #-----------------------------------------------------
    if (( bySubset == 1 )); then
        DBI_USE_FULL=1
        if [[ $_RUN_SUBSET == "__FULL__" ]]; then
            # flag subsample mode==0 -> no subsampling, use full set
            _RUN_DO_SUBSAMPLE=0
            map_info_msg "using the full set of node $_RUN_NODEID -> download FULL data"
        else
            # flag subsample mode==1 -> no subsampling, use given subset
            _RUN_DO_SUBSAMPLE=1
            map_info_msg "using a subset of node $_RUN_NODEID -> download FULL data"
        fi
        # clear bySubsample if set
        if (( bySubsample == 1 )); then
            bySubsample=0
            _RUN_SUBSAMPLE="0"
            _RUN_SCOREDIR=""
            map_warn_msg "'subsample' and 'score' are ignored as 'using' is set"
        fi
        return 0
    fi

    #-----------------------------------------------------
    # set DBI_USE_FULL == 0 if subsample params are the
    #                       same as official
    #-----------------------------------------------------
    if (( bySubsample == 1 )); then
        # check only subsample
        # scoredir is by default set to 'ssc'
        _RUN_SUBSET=""
        # check if subsample == 0 -> use full data set
        if (( _RUN_SUBSAMPLE == 0 )); then
            map_info_msg "no subsampling - using full data"
            _RUN_DO_SUBSAMPLE=0
            _RUN_SUBSAMPLE_LABEL=""
            DBI_USE_FULL=1
        else        
            if [[ $DBI_SAMPLED_N == $_RUN_SUBSAMPLE ]] && [[ $DBI_SAMPLED_SCDIR == $_RUN_SCOREDIR ]]; then
                map_info_msg "subsampling parameters are the same as official subsampling - will download pre-subsampled data"
                DBI_USE_FULL=0
                _RUN_DO_SUBSAMPLE=3
            else
                map_info_msg "subsampling parameters are not default - will download full data"
                DBI_USE_FULL=1
                _RUN_DO_SUBSAMPLE=2
            fi
            _RUN_SUBSAMPLE_LABEL="$(map_subsample_label)"
        fi
        return 0
    fi

    return 0

}
# create subset label
map_get_subset_label() {
    [[ -n $_RUN_SUBSET ]] || return 0

    # replace any ',' with '_'
    local v=$(echo "$sdata" | tr ',' '_')
    if [[ "${v:0:1}" == "+" ]]; then
        # special label prefixed with '+'
        echo ${v:1}
    else
        echo $v
    fi
}

map_set_runlabel() {
    (( _RUN_LABEL_WAS_GIVEN == 1 )) && return 0
    _RUN_LABEL=""
    if [[ -z $_RUN_NODEID ]]; then
        map_error_msg "failed setting run label, nodeid is not set,  with flag $_RUN_DO_SUBSAMPLE"
        return 255
    fi
    
    # its a mapping run using mmseqs profiles
    if (( _RUN_BY_PROFILE == 1 )); then
        _RUN_LABEL="node_${_RUN_NODEID}_prof"
        return 0
    fi
    
    # get sublabel depending on subsample mode
    local sublbl=""
    if (( _RUN_DO_SUBSAMPLE == 0 )); then
        _RUN_LABEL="node_${_RUN_NODEID}"
        return 0
        
    elif (( _RUN_DO_SUBSAMPLE == 1 )); then
        # create label using subset and nodeid
        sublbl=$(map_get_subset_label)
        
    elif (( _RUN_DO_SUBSAMPLE == 2 )); then
        # create label using non-default subsample params
        sublbl="nsub_${_RUN_SUBSAMPLE}_scdir_${_RUN_SCOREDIR}"
        
    elif (( _RUN_DO_SUBSAMPLE == 3 )); then
        # create label using default subsample params
        sublbl="nsub_${_RUN_SUBSAMPLE}_scdir_${_RUN_SCOREDIR}"
        
    fi
    if [[ -z $sublbl ]]; then
        map_error_msg "failed setting run label with flag $_RUN_DO_SUBSAMPLE"
        return 255
    fi

    # done - set label
    _RUN_LABEL="node_${_RUN_NODEID}_${sublbl}_label_${_RUN_HASHID}"
    return 0
}

map_parse_run_args() {
    _RUN_LABEL_WAS_GIVEN=0
    _RUN_LABEL=""
    _RUN_MODE="map"
    _RUN_NODEID=""
    _RUN_TAXIDS=()
    _RUN_HASHID=""
    _RUN_SUBSET=""
    _RUN_SUBSAMPLE="0"
    _RUN_SCOREDIR=""
    _RUN_GO=0
    _RUN_CHECK=0
    _RUN_IS_LOCAL=0
    _RUN_CMD="$1"
    shift

    # no more args - its OK
    [[ $# -gt 0 ]] || return 0

    # get user taxid -> now it is an argument taxid=
    #_RUN_TAXID="$1"
    #shift

    map_parse_special_args "$@" || return 255

    map_set_runlabel || return 255

    map_info_msg "set run label to <$_RUN_LABEL>"
    return 0

}

#--------------------------------------------------------------------------------------------------------------------
map_info_msg() {
    local m="$*"
    _MSG_SEL_PREFIX="<map> "
    info_msg "$m"
}

map_error_msg() {
    local m="$*"
    _MSG_SEL_PREFIX="<map> "
    error_msg "$m"
}

map_warn_msg() {
    local m="$*"
    _MSG_SEL_PREFIX="<map> "
    warn_msg "$m"
}

map_cleanup_taxid() {
    echo -n "$1" | tr -c '[:alnum:]' '_'
}

map_has_clade() {
    local clade="$1"
    local tstvar="MAP_NODE_ROOT_${clade^^}"
    isset_var $tstvar
}

map_dir_clade() {
    local clade=""
    [[ $# -gt 0 ]] && clade="$1"
    local rval=$MAP_ORTHODB_DATA
    if [[ -n $clade ]]; then
        clade="${clade^^}"
        [[ -z ${MAP_DIRS[$clade]+x} ]] || rval=$MAP_ORTHODB_DATA/${MAP_DIRS[$clade]}
    fi
    echo "$rval"
}

export ODB_CLADE=""
export ODB_TODO_FULL=""
export ODB_TODO_SUBSAMPLED=""
export ODB_TODO_USE=""
map_find_orthodb_todo() {
    ODB_CLADE=""
    ODB_TODO_FULL=""
    ODB_TODO_SUBSAMPLED=""
    ODB_TODO_USE=""
    
    # check args
    local nodeid=$_RUN_NODEID
    local check=""
    [[ $# -ge 1 ]] && nodeid="$1"
    [[ $# -eq 2 ]] && check="$2"
    check="${check^^}"
    
    local ec=0
    
    map_info_msg "find OrthoDB todo for node $nodeid"
    ODB_CLADE=$(map_get_clade "$nodeid")
    ec=$?
    if (( ec == 1 )); then
        map_warn_msg "map_find_orthodb_todo: node id is 'auto'"
        return 128
    elif (( ec > 1 )); then
        map_error_msg "map_find_orthodb_todo: node id ($nodeid) is not part of OrthoDB"
        return 255
    fi
    local srcdir="$(map_dir_clade $ODB_CLADE)"
    srcdir=$(cd "$srcdir" && pwd)
    ODB_TODO_FULL=$srcdir/todo/${nodeid}.todo
    ODB_TODO_SUBSAMPLED=$srcdir/todo/$(dbi_sampled_default_label "$nodeid").todo
    if [[ $check == "CHECK" ]]; then
        ec=0
        ###{ map_error_msg "map_find_orthodb_todo: OrthoDB full todo not found at $ODB_TODO"; ec=2; }
        ###{ map_error_msg "map_find_orthodb_todo: OrthoDB subsampled todo not found at $ODB_TODO_SUBSAMPLED"; ec=2; }
        if (( DBI_USE_FULL == 1 )); then
            [[ -f $ODB_TODO_FULL       ]] || ec=1
        else
            [[ -f $ODB_TODO_SUBSAMPLED ]] || ec=2
        fi
    fi
    if (( DBI_USE_FULL == 1 )); then
        ODB_TODO_USE=$ODB_TODO_FULL
    else
        ODB_TODO_USE=$ODB_TODO_SUBSAMPLED
    fi
    return $ec
}

# just retrieve info, do not check for existance
# SOURCE_TODO == the source todo file, possibly for subsampled source
# SOURCE_TODO_FULL == the original source todo file
export SOURCE_TODO=""
export SOURCE_TODO_FULL=""
map_find_source_todo() {

    map_info_msg "find source todo for node $_RUN_NODEID , clade $ODB_CLADE"
    map_info_msg "subsample label <$_RUN_SUBSAMPLE_LABEL>"
    # define source todo file
    local label="$_RUN_NODEID"
    if [[ -n $_RUN_SUBSAMPLE_LABEL ]]; then
        label="$_RUN_SUBSAMPLE_LABEL"
    fi
    SOURCE_TODO=$DIR_PROJECT/todo/${label}.todo
    SOURCE_TODO_FULL=$DIR_PROJECT/todo/${_RUN_NODEID}.todo

    [[ -f $SOURCE_TODO      ]] || { map_error_msg "map_find_source_todo: todo not found at $SOURCE_TODO"; return 255; }
    [[ -f $SOURCE_TODO_FULL ]] || { map_error_msg "map_find_source_todo: full todo not found at $SOURCE_TODO_FULL"; return 255; }
    map_info_msg "found source todo : $SOURCE_TODO"
    return 0
}

# find root node for a given clade
# e.g if clade is EUK it will return 2759
map_get_clade_rootnode() {
    local clade="$1"
    local rootvar="MAP_NODE_ROOT_${clade^^}"
    local nounset=0
    local rootnode=""
    isset_var $rootvar || return
    rootnode="${!rootvar}"
    local rootdir=$(map_dir_clade ${clade,,})
    echo "$rootdir/todo/${rootnode}.todo"
}


export CHECK_MISSING=""
map_check_pipeline() {
    CHECK_MISSING=""
    local nbad=0
    for d in busco Sequences PWC Cluster todo; do
        [[ -d $DIR_PROJECT/$d ]] || { nbad=1; CHECK_MISSING="$CHECK_MISSING $d"; break; }
    done
    [[ -f $DIR_PROJECT/orthologer.sh        ]] || { CHECK_MISSING="$CHECK_MISSING $DIR_PROJECT/orthologer.sh";       nbad=$((nbad+1)); }
    [[ -f $DIR_PROJECT/orthologer_conf.sh   ]] || { CHECK_MISSING="$CHECK_MISSING $DIR_PROJECT/orthologer_conf.sh";  nbad=$((nbad+1)); }
    [[ -f $DIR_PROJECT/orthomapper.sh       ]] || { CHECK_MISSING="$CHECK_MISSING $DIR_PROJECT/orthomapper.sh";      nbad=$((nbad+1)); }
    [[ -f $DIR_PROJECT/orthomapper_conf.sh  ]] || { CHECK_MISSING="$CHECK_MISSING $DIR_PROJECT/orthomapper_conf.sh"; nbad=$((nbad+1)); }
    return $nbad
}

# check config and output status
map_check_db_table() {
    local ec=0
    [[ -f "$DIR_ORTHOLOGER/bin/common_base.sh" ]]; ec=$?; xec_status_msg $ec "Orthologer"
    [[ -d "$MAP_ORTHODB_DATA" ]];                ec=$?; xec_status_msg $ec "Map source DB"

    # check dirs
    local d=""
    local rootdir=""
    local nok=0
    local nerrtot=0
    for clade in ${MAP_ROOT_CLADES[@]}; do
        local nerr=0
        # get root dir of clade
        rootdir=$(map_dir_clade ${clade,,})
        [[ -d "$rootdir" ]] || { nerr=$((nerr+1)); ec=$?; xec_status_msg $ec "<$clade> directory"; }
        
        # look for basic directories
        for d in Rawdata Sequences PWC Cluster todo; do 
            [[ -d "$rootdir/$d" ]] || { nerr=$((nerr+1)); xec_status_msg 1 "missing <$rootdir/$d> directory"; }
        done

        # todo file
        local todo="$(map_get_clade_rootnode ${clade,,})"
        [[ -f "$todo" ]] || { nerr=$((nerr+1)); xec_status_msg 1 "missing <$todo> file"; }
        ec=0
        (( nerr>0 )) && ec=2
        xec_status_msg $ec "Clade $clade"
    done

    return 0
}

# check non-user specific stuff
map_check_db() {
    [[ -d "$DIR_ORTHOLOGER"                         ]] || { map_error_msg "unable to read directory DIR_ORTHOLOGER=$DIR_ORTHOLOGER";            return 255; }
    [[ -f "$DIR_ORTHOLOGER/bin/common_base.sh"      ]] || { map_error_msg "DIR_ORTHOLOGER does not seem to point to a orthologer root";        return 255; }
    [[ -d "$MAP_ORTHODB_DATA"                     ]] || { map_error_msg "unable to read directory MAP_ORTHODB_DATA=$MAP_ORTHODB_DATA";            return 255; }

    #[[ -d "$DIR_TEMPLATES"                        ]] || { map_error_msg "unable to read directory DIR_TEMPLATES=$DIR_TEMPLATES";           return 255; }
    #[[ -f "$DIR_TEMPLATES/common.sh"              ]] || { map_error_msg "unable to read common.sh file in DIR_TEMPLATES=$DIR_TEMPLATES";   return 255; }
    #[[ -f "$DIR_TEMPLATES/orthologer.sh"            ]] || { map_error_msg "unable to read orthologer.sh file in DIR_TEMPLATES=$DIR_TEMPLATES"; return 255; }

    # set verbosity if any args are given
    local verb=0
    [[ $# -gt 0 ]] && verb=1

    # check dirs
    local d=""
    local rootdir=""
    local nok=0
    local nerrtot=0
    for clade in ${MAP_ROOT_CLADES[@]}; do
        local nerr=0
        # get root dir of clade
        rootdir=$(map_dir_clade ${clade,,})
        [[ "$rootdir" == "" ]] && { map_warn_msg "root dir is not defined for clade $clade"; nerr=$((nerr+1)); }
        
        # look for basic directories
        for d in busco Sequences PWC Cluster todo; do 
            [[ -d "$rootdir/$d" ]] || { map_warn_msg "the given orthodb source location does not contain the required data for the clade $clade (missing dir <$d>)"; nerr=$((nerr+1)); }
        done

        local todo="$(map_get_clade_rootnode ${clade,,})"
        [[ -f "$todo"   ]] || { map_warn_msg "unable to read root node at clade $clade ($todo)"; nerr=$((nerr+1)); }
        if (( verb == 1 )); then
            if (( nerr==0 )); then
                ok_status_msg "$clade"
            else
                xwarn_status_msg "$clade" "missing data"
            fi
        fi
        (( nerr == 0 )) && nok=$((nok+1))
        nerrtot=$((nerrtot+nerr))
    done
    if (( nerrtot > 0 )); then
        map_error_msg "mapping data missing - found $nok OK clade(s) and $nerrtot missing file(s)"
        return 255
    fi
    return 0
}

map_dump_setup() {
    local fmtstr="%-25s : %s"
    print_separator '~'
    printf "$fmtstr\n" "OrthoDB version"  "$API_ORTHODB_VERSION"
    printf "$fmtstr\n" "Orthologer version"       "$OP_VERSION_SHORT"
    printf "$fmtstr\n" "Orthologer installation"       "$DIR_ORTHOLOGER"
    printf "$fmtstr\n" "Database"         "$MAP_ORTHODB_DATA"
    printf "$fmtstr\n" "Work directory"   "$DIR_PROJECT"
    print_separator '~'
}

map_load_orthologer_conf() {
    # source orthologer conf
    if [[ -f "$DIR_PROJECT/orthologer_conf.sh" ]]; then
        debug_msg "loading $DIR_PROJECT/orthologer_conf.sh"
        source "$DIR_PROJECT/orthologer_conf.sh"
        #|| {
        #       error_msg "failed sourcing $DIR_PROJECT/orthologer_conf.sh"
        #       return 255
        #   }
        debug_msg "done loading"
    else        
        debug_msg "unable to find orthologer_conf.sh at : $DIR_PROJECT/orthologer_conf.sh"
        return 255
    fi
    debug_msg "refresh directory names"
    refresh_dir_project
}

map_softlink() {
    [[ $# -eq 1 ]] || return 255
    local f="$1"
    [[ -f $f ]] || { map_error_msg "required file not found ($f)"; return 255; }
    
    # skip silently if file is found locally
    local bf=$(basename $f)
    [[ -f $bf ]] && return 0
    ln -sf $f ./ || { map_error_msg "failed creating softlink"; return 255; }
    return 0
}

#
# The mapping requires from pipeline data the following:
# > Rawdata
#   - *.fs.maptxt            mapping between of fasta labels
# > Sequences
#   - *.fs_selected          final processed fasta
#   - *.fs_selected.mm*      mmseqs DB files, if they exist
#   - *.fs_selected.dmnd     diamond DB file, if it exists
#   - *.fs_selected.seqlen   sequence lengths
#   - *.fs_selected.stats    fasta stats
# > PWC
#   - PWC/A/A_A.align        diagonal of alignment matrix
# > Cluster
#   - Cluster/*.og           final cluster results, internal id's
#   - Cluster/*.og_map       maps cluster id to OrthoDB name NOOOO
#
# Input:
#   clade    - which root clade in MAP_ORTHODB_DATA
#   ftodo    - todo file to be used (official subsampled or full)
#   ffull    - official full todo, needs this todo file even if subsampled data
#
map_import_db() {
    [[ $# -eq 3 ]] || { map_error_msg "map_init_uid_import: wrong nr of args $# (3)"; return 255; }
    local clade="$1"
    local ftodo="$2"
    local ffull="$3"
    [[ -f "$ftodo" ]] || { map_error_msg "cannot find todo file for root node: $ftodo"; return 255; }
    [[ -f "$ffull" ]] || { map_error_msg "cannot find raw todo file for root node: $ffull"; return 255; }

    local node_is_viridae=0
    if [[ "${clade,,}" == "viridae" ]]; then
        map_info_msg "accept missing alignment files as clade is $clade"
        node_is_viridae=1
    fi

    # get root clade dir MAP_ORTHODB_DATA/$clade
    local rootdir=$(map_dir_clade ${clade,,})
    [[ -n $rootdir ]] || { map_error_msg "unable to read source for root clade ($clade)"; return 255; }
    
    # OrthoDB source directories
    local odb_rawdata="$rootdir/Rawdata"
    local odb_sequences="$rootdir/Sequences"
    local odb_pwc="$rootdir/PWC"
    local odb_cluster="$rootdir/Cluster"
    
    # make sure source directories exists
    [[ -d $odb_rawdata   ]] || { map_error_msg "unable to read source Rawdata at $odb_rawdata"; return 255; }
    [[ -d $odb_sequences ]] || { map_error_msg "unable to read source Sequences at $odb_sequences"; return 255; }
    [[ -d $odb_pwc       ]] || { map_error_msg "unable to read source PWC at $odb_pwc"; return 255; }
    [[ -d $odb_cluster   ]] || { map_error_msg "unable to read source Cluster at $odb_cluster"; return 255; }

    # get node id
    local nodeid=$(basename "$ffull" .todo)
    map_info_msg "will import DB for node $nodeid"
    
    # if accept missing files (viridae clade), set variables in common.sh
    if (( node_is_viridae == 1 )); then
        IGNORE_MISSING=1
        file_add_setvar $DIR_PROJECT/orthologer_conf.sh IGNORE_MISSING  "1" || { map_error_msg "failed setting IGNORE_MISSING in user orthologer_conf.sh";  return 255; }
    else
        file_add_setvar $DIR_PROJECT/orthologer_conf.sh IGNORE_MISSING  "0" || { map_error_msg "failed setting IGNORE_MISSING in user orthologer_conf.sh";  return 255; }
    fi

    # get all labels
    local labels=($(grep -v '^#' "$ftodo"))

    map_info_msg "importing data for ${#labels[@]} labels at $(pwd)"
    map_info_msg "using todo file: $(trim_name $ftodo)"
    
    local err=0
    local warn=0
    local t=""
    local src=""
    local base=""
    local trg=""
    local seqlen=""

    local ncp=0
    local ndb=0
    local p=""

    #------------------------------------------------------------
    # import todo file(s)
    #------------------------------------------------------------
    cd $DIR_PROJECT/todo || { map_error_msg "failed entering $DIR_PROJECT/todo"; return 255; }
    if [[ ! -f $(basename "$ftodo") ]]; then
        map_softlink "$ftodo" || return 255
        ncp=1
    fi
    if [[ $ftodo != $ffull ]]; then
        if [[ ! -f $(basename "$ffull") ]]; then
            map_softlink "$ffull" || return 255
            ncp=$((ncp+1))
        fi
    fi
    map_info_msg "imported $ncp todo file"
    cd ../

    #------------------------------------------------------------
    # import (softlink) all Rawdata/*.fs.maptxt
    #------------------------------------------------------------
    map_info_msg "importing rawdata, in total ${#labels[@]} label(s)"
    [[ -d Rawdata ]] || { verbose_info_msg "creating missing Rawdata directory"; mkdir -p Rawdata || return 255; }
    ###cd Rawdata || { map_error_msg "failed entering $(pwd)/Rawdata"; return 255; }
    ncp=0
    local ext=0
    for t in ${labels[@]}; do
        #---------------------------------------------
        # get raw fasta and gene map file
        # for ext in fs.maptxt; do
        # Note - they are never with subsampled label
        local label=""
        if (( DBI_USE_FULL == 0 )); then
            # label is like g9443_n8ssc_9606_0
            # remove g9443_
            t=${t#*_}
            # ... remove n8ssc_
            t=${t#*_}
        fi
        ext="fs.maptxt"
        trgbase="${t}.$ext"
        src="$odb_rawdata/${trgbase}"
        trg="Rawdata/$trgbase"
        [[ -f $trg ]] && continue

        # check source
        [[ -f $src ]] || { err=$((err+1)); map_error_msg "expected file not found ($src)"; continue; }
        ln -sf "$src" "$trg" || { map_error_msg "failed creating softlink for $src at $trg"; return 255; }
        ####( cd Rawdata && map_softlink "$src" ) || return 255
        ncp=$((ncp+1))
        ####done
    done
    [[ $err -eq 0 ]] || { map_error_msg "missing $err files in Rawdata"; return 255; }
    map_info_msg "imported rawdata $ncp files with $err error(s)"
    ##cd ../ >& /dev/null

    #------------------------------------------------------------
    # import (softlink) all Sequences/*.fs_selected and
    # corresponding formatdb files
    #------------------------------------------------------------
    map_info_msg "importing sequences"
    [[ -d Sequences ]] || { verbose_info_msg "creating missing Sequences directory"; mkdir -p Sequences || return 255; }
    ###cd Sequences || { map_error_msg "failed entering $(pwd)/Sequences"; return 255; }
    for t in ${labels[@]}; do
        #-----------------------------------
        # get final fasta *.fs_selected
        #-----------------------------------
        trgbase="${t}.fs_selected"
        src="$odb_sequences/${trgbase}"
        trg="Sequences/$trgbase"

        [[ -f $trg ]] && continue
        
        # check source
        [[ -f $src ]] || { err=$((err+1)); map_error_msg "expected file not found ($src)"; continue; }
        
        # check target
        ln -sf "$src" "$trg" || { map_error_msg "failed creating softlink for $src at $trg"; return 255; }
        ncp=$((ncp+1))

        #-----------------------------------
        # get corresponding DB files *.fs_selected.*
        #-----------------------------------
        # They may be either from formatdb, makeblastdb or mmseqs
        # get all formatdb files
        ndb=0
        src="$odb_sequences/${trgbase}"
        if [[ -f ${src}.mm ]]; then
            # mmseqs DB files
            trg="Sequences/${trgbase}.mm"
            if [[ ! -f $trg ]]; then
                for ext in mm mm.dbtype mm_h.dbtype mm.index mm.source mm_h mm_h.index mm.lookup; do
                    trg="Sequences/${trgbase}.$ext"
                    src="$odb_sequences/${trgbase}.$ext"
                    [[ -f $src ]] || { err=$((err+1)); map_error_msg "expected file not found ($src)"; continue; }
                    ln -sf "$src" "$trg" || { map_error_msg "failed creating softlink for $src at $trg"; return 255; }
                    ncp=$((ncp+1))
                done
            fi
            ndb=1
        else
            # check here for diamond source - should be just one file
            #warn=1
            if [[ -f ${src}.dmnd ]]; then
                trg="Sequences/${trgbase}.dmnd"
                src="$odb_sequences/${trgbase}.dmnd"
                [[ -f $src ]] || { err=$((err+1)); map_error_msg "expected file not found ($src)"; continue; }
                if [[ ! -f $trg ]]; then
                    ln -sf "$src" "$trg" || { map_error_msg "failed creating softlink for $src at $trg"; return 255; }
                    ncp=$((ncp+1))
                fi
                ndb=1
            fi
        fi
        (( ndb == 1 )) || { err=$((err+1)); map_error_msg "expected DB file(s) not found for $src"; }
        
        #-----------------------------------
        # get .seqlen file
        #-----------------------------------
        src="$odb_sequences/${trgbase}.seqlen"
        trg="Sequences/${trgbase}.seqlen"
        [[ -f $src ]] || { err=$((err+1)); map_error_msg "expected file not found ($src)"; continue; }

        # get seqlen
        if [[ ! -f "$trg" ]]; then
            ln -sf "$src" "$trg" || { map_error_msg "failed creating softlink for $src at $trg"; return 255; }
            ncp=$((ncp+1))
        fi
        
        #-----------------------------------
        # get .stats file
        #-----------------------------------
        src="$odb_sequences/${trgbase}.stats"
        trg="Sequences/${trgbase}.stats"
        [[ -f $src ]] || { err=$((err+1)); map_error_msg "expected file not found ($src)"; continue; }

        # get stats
        if [[ ! -f "$trg" ]]; then
            ln -sf "$src" "$trg" || { map_error_msg "failed creating softlink for $src at $trg"; return 255; }
            ncp=$((ncp+1))
        fi                
    done

    [[ $err -eq 0 ]] || { map_error_msg "missing $err files in Sequences"; return 255; }
    map_info_msg "imported sequences $ncp files with $err error(s)"

    #------------------------------------------------------------
    # get alignment diagonal elements
    #------------------------------------------------------------
    # they are used by the inparalog search
    map_info_msg "importing diagonal alignments from PWC"
    err=0
    ncp=0    
    ###cd ../PWC >& /dev/null || { map_error_msg "failed entering $(pwd)/PWC"; return 255; }
    for t in ${labels[@]}; do
        # get file
        src="$odb_pwc/$t/${t}_${t}.align"
        [[ -f $src ]] || {
            err=$((err+1))
            map_warn_msg "expected file not found ($src)"
            continue
        }

        # skip silently if target already exists
        trg=PWC/$t/$(basename "$src")
        [[ -f "$trg" ]] && continue
        
        # soft link
        ln -sf "$src" "$trg" || { map_error_msg "failed creating softlink in $(pwd)/PWC/$t"; return 255; }
        
        ncp=$((ncp+1))
    done
    if [[ $err -gt 0 ]]; then
        if [[ "$node_is_viridae" == "1" ]]; then
            map_info_msg "missing $err files in PWC - accepted as clade is $clade"
        else
            map_error_msg "missing $err files in PWC"
            return 255
        fi
    fi
    
    map_info_msg "imported $ncp PWC files with $err error(s)"

    #------------------------------------------------------------
    # import cluster data
    #------------------------------------------------------------
    map_info_msg "importing cluster data"
    ncp=0
    
    ###cd ../ >& /dev/null
    [[ -d Cluster ]] || { verbose_info_msg "creating missing Cluster directory"; mkdir -p Cluster || return 255; }
    ###cd Cluster || { map_error_msg "failed entering $(pwd)/Cluster"; return 255; }

    src=$odb_cluster/${nodeid}.og
    [[ -f "$src" ]] || { map_error_msg "expected file not found ($src)"; return 255; }
        
    # skip silently if file already exists
    trg=Cluster/$(basename $src)
    if [[ ! -f "$trg" ]]; then
        # soft link
        ln -sf "$src" "$trg" || { map_error_msg "failed creating softlink $src to $trg"; return 255; }
        ncp=$((ncp+1))
    fi

    # get spcov - required if subsampling with custom params, skip if viridae clade
    if [[ "$node_is_viridae" == "0" ]]; then
        srcCov=$odb_cluster/${nodeid}.spcov
        if [[ -f "$srcCov" ]]; then
            trg=Cluster/$(basename $srcCov)
            if [[ ! -f "$trg" ]]; then
                # soft link
                ln -sf $srcCov $trg || { map_error_msg "failed creating softlink $src to $trg"; return 255; }
                ncp=$((ncp+1))
            fi
        else
            map_warn_msg "missing spcov file ($srcCov) - skip"
        fi
    fi

    map_info_msg "imported $ncp cluster files"
}

map_init_uid() {

    # init base stuff
    map_info_msg "initialise mapping"
    ###mkdir -p $DIR_PROJECT/runlogs || { map_error_msg "failed creating $DIR_PROJECT/runlogs"; return 255; }
    ###mkdir -p $DIR_PROJECT/import  || { map_error_msg "failed creating $DIR_PROJECT/import";  return 255; }
    
    #-------------------------------------------------------------------------
    # set environment variables
    export PL_TODO=""                     # not set here 
    ###export DATA_TYPE="PROT"               # data type - always proteins

    # make sure DIR_PROJECT_XXX are correct
    refresh_dir_project

    # create project dirs
    debug_msg "create project dirs"
    map_info_msg "map_init - create project directories"

    create_project_dirs || { map_error_msg "failed creating project directories"; return 255; }

    return 0
}

# check if taxid exists within any OrthoDB clade
# return 1 if taxid is found
# otherwise return 0
map_check_taxid() {
    [[ $# -eq 1 ]] || return 0
    local taxid="$1"
    [[ -n "$taxid" ]] || return 0
    
    local froot=""
    local clade=""
    for clade in ${MAP_ROOT_CLADES[@]}; do
        froot=$(map_get_clade_rootnode ${clade,,})
        [[ -f $froot ]] || return 0
        local match='^\s*'$taxid'\s*$'
        grep -q "$match" "$froot" && return 1
    done
    return 0
}

#-------------------------------------------------------
# subsampled taxids
#-------------------------------------------------------

map_softlink_subsampled_alignment() {
    # soft link subsampled alignment files (diagonal)
    debug_msg "map_softlink_subsampled_alignment nr args = $#"
    [[ $# -eq 2 ]] || return 255
    local ftodo="$1"    # todo with subsampled taxon ids
    local taxidMap="$2"
    [[ -f $ftodo ]] || { map_error_msg "missing todo file at $ftodo"; return 255; }
    
    local key=""
    local nerr=0
    local nok=0

    debug_msg "map_softlink_subsampled_alignment: ftodo = $ftodo"
    # read keys from todo file
    while read key; do
        [[ "${key:0:1}" == "#" ]] && continue
        [[ -z $key ]] && continue
        [[ "$key" == "$taxidMap" ]] && continue

	    # check if softlink already exists
        local fpwc="$DIR_PROJECT/PWC/${key}/${key}_${key}.$EXT_ALIGN"
	    [[ -f "$fpwc" ]] && continue
	    
        local elems=($(map_split_subsampled_taxid "$key"))
        [[ ${#elems[@]} -ge 4 ]] || { map_info_msg "wrong nr of elements return from split ${#elems[@]}"; nerr=1; continue; }
        local taxid="${elems[3]}"
        local src="$DIR_PROJECT/PWC/${taxid}/${taxid}_${taxid}.$EXT_ALIGN"
        [[ -f $src ]] || { map_error_msg "missing source PWC - mapping for subsampled taxid $taxid"; nerr=1; break; }
        ln -sf "$src" "$fpwc" || { map_error_msg "failed softlink $src to $fpwc"; nerr=1; break; }
	    nok=$((nok+1))
    done < "$ftodo"
    map_info_msg "created $nok softlink(s) for diagonal alignment of subsampled taxids"
    return $nerr
}

# get score scan direction
# * des     sort genes by ascending score
# * asc     sort genes by ascending score
# * sco     sort genes first by overall species coverage and then by score
# * sde     des, select stepwise
# * sas     asc, select stepwise
# * ssc     sco, select stepwise
map_get_scdir_value() {
    [[ $# -eq 1 ]] || return 255
    [[ -n "$1" ]] || { echo "ssc"; return 0; }
    local v="$1"
    v=${v,,}
    if [[ ${v:0:3} == "des" ]]; then
        # sort genes by decreasing score
        echo "des"
    elif [[ ${v:0:3} == "asc" ]]; then
        # sort genes by ascending score
        echo "asc"
    elif [[ ${v:0:3} == "sde" ]]; then
        # sort genes by decreasing score, select stepwise
        echo "sde"
    elif [[ ${v:0:3} == "sas" ]]; then
        # sort genes by ascending score, select stepwise
        echo "sas"
    elif [[ ${v:0:3} == "sco" ]]; then
        # sort genes first by overall species coverage and then by score
        echo "sco"
    elif [[ ${v:0:3} == "ssc" ]]; then
        # like sco, select stepwise
        echo "ssc"
    else
        return 255
    fi
    return 0
}


# create subsampled taxid
map_subsample_taxid() {
    echo "${_RUN_SUBSAMPLE_LABEL}_$taxid"
}

# subsampled taxid has form:
# g<node>_n<subsample>[dec|asc)]_<taxid>
# the extension (dec|asc) of subsample part is optional
# if not given, assume 'dec'
map_split_subsampled_taxid() {
    [[ $# -eq 1 ]] || return 0
    local stxid="$1"
    [[ ${stxid:0:1} == "g" ]] || return 255
    local tarr=($(tokenize '_' "$stxid"))
    local N=${#tarr[@]}
    (( N > 2 )) || return 255
    
    local node="${tarr[0]}"
    local subsample="${tarr[1]}"
    local scdir="des" # default value
    local taxid="${tarr[2]}"
    if (( N > 3 )); then
        for (( i=3; i<N; i++ )); do
	        taxid="${taxid}_${tarr[$i]}"
	    done
    fi
    if [[ $subsample == *dec ]] || [[ $subsample == *asc ]]; then
        # example: 'n12dec'
        # get last 3 characters
        scdir=${subsample:(-3)}
        # extract numerical value
        subsample=${subsample:1:-3}
    else
        # example: 'n12'
        subsample=${subsample:1}
    fi
    node="${node:1}"
    scdir=$(map_get_scdir_value "$scdir")
    [[ -n $scdir ]] || { map_error_msg "subsample: illegal scdir argument $3"; return 255; }
    echo "$node $subsample $scdir $taxid"
    return 0
}


#
# Parse subset; input is a CSV list of node taxids and/or orthodb taxids
# - expand any node ids to corresponding taxids
# - make sure it's valid and that relevant files exists for the current user
# - prints space separated list
#
map_parse_subset() {
    # check CSV list of OrthoDB style taxids or clade ids
    [[ "$_RUN_SUBSET" != "" ]] || return 0
    
    local subarr=($(tokenize ',' "$_RUN_SUBSET"))
    local nsub=${#subarr[@]}
    local Nnodes=0
    local Ntaxids=0
    local elem=""
    local csvList=""

    for elem in ${subarr[@]}; do
        if [[ ${elem:0:3} == "+fr" ]]; then
            if (( nsub > 1 )); then
                map_error_msg "subset allows only one element with +frxx"
                return 255
            fi
            # HERE define species ids using staclus spcov
            #/statclus -T 64 -C /data/elk/orthodb/v11/mapping/data/eukaryota/Cluster/7742.og --specCov -O 7742.spcov
            #python3 ./get_statclus_best_cov.py -f data/eukaryota/Cluster/6960.spcov -x 0.99

            # extract numeric part
            local fcov="0.${elem:3}"
            # test if numeric
            printf "%.2f" "$fcov" >& /dev/null || { map_error_msg "+frxx requires that 0.xx is numerical! Got $elem"; return 255; }
            local clade=""
            clade=$(map_get_clade ${_RUN_NODEID,,})
            local dirclade=$(map_dir_clade ${clade,,})
            [[ -d $dirclade ]] || { map_error_msg "missing directory for node ($_RUN_NODEID) at $dirclade"; return 255; }
            local fspcov=$dirclade/Cluster/${_RUN_NODEID}.spcov
            [[ -f $fspcov ]] || { map_error_msg "missing spcov file for clade ($_RUN_NODEID) at $fspcov"; return 255; }

            local cmd=$DIR_BRHCLUS/get_statclus_best_cov.py
            [[ -f $cmd ]] || { map_error_msg "missing command $cmd"; return 255; }

            # print all labels up to given coverage
            $PYTHON3 $cmd -f $fspcov -x $fcov
            continue
        fi

        # check if clade label
        local ftodo=$DIR_PROJECT/todo/${elem}.todo
        if [[ -f $ftodo ]]; then
            if (( Nnodes==1 )); then
                map_error_msg "only one node is allowed in sub node ($_RUN_SUBSET)"
                return 255
            fi
            Nnodes=$((Nnodes+1))
            (( Nnodes==1 )) && grep -v '^#' $ftodo
            continue
        fi

        # check if fasta label
        local ffasta=$DIR_PROJECT/Sequences/${elem}.fs_selected
        if [[ -f $ffasta ]]; then
            echo "$elem"
            Ntaxids=$((Ntaxids+1))
            continue
        fi
        
        # failed label
        map_error_msg "cannot find a valid taxid or node id matching $elem"
        map_error_msg "missing fasta file $ffasta"
        return 255
    done
    
    return 0
}

# Fill todo files:
# MAP_TODO (set by map_set_map_todo)
# SOURCE_TODO : possibly subsampled todo , name set in this function
# SOURCE_TODO_FULL : source todo (not subsampled)
map_fill_todo() {

    touch "$MAP_TODO" >& /dev/null || {
        map_error_msg "new todo file not writeable at $MAP_TODO"
        return 255
    }

    map_info_msg "creating map todo file at $(basename $MAP_TODO)"
    local txid=
    for txid in ${_RUN_TAXIDS[@]}; do
        echo $txid
    done > "$MAP_TODO"

    # its a mapping run using mmseqs profiles
    # todo file only consists of the fasta to be mapped
    if (( _RUN_BY_PROFILE == 1 )); then
        map_info_msg "mode is 'mapprof' -> no source todo"
        return 0
    fi
    
    #-------------------------------------------------------------------------
    # In new mapper, use TODO1/2 in orthologer:
    # PL_TODO  = MAP_TODO
    # PL_TODO1 = MAP_TODO
    # PL_TODO2 = SOURCE_TODO
    #-------------------------------------------------------------------------

    # OBSOLETE CODE - keep for now..
    #-------------------------------------------------------------------------
    # handle potential subset
    #-------------------------------------------------------------------------
    #if [[ -n $_RUN_SUBSET ]]; then
    #    map_info_msg "creating new todo file with subset from source $SOURCE_TODO"
    #    local fsubsetdata=""
    #    tmp_make fsubsetdata "subsetdata" || {
    #        map_error_msg "map_init_todo: failed creating tmp file"
    #        return 255
    #    }
    #    map_parse_subset > $fsubsetdata || {
    #        map_error_msg "failed parsing subset $_RUN_SUBSET"
    #        rm -f "$newtodo"
    #        return 255
    #    }
    #    {
    #        # select labels common between source and subset
    #        comm -12 <(sort "$SOURCE_TODO") <(sort "$fsubsetdata")
    #        echo "$_RUN_TAXID"
    #    } | sort -u > "$newtodo"
    #    return 0
    #fi
    
    #-------------------------------------------------------------------------
    # no subset - concatenate source todo and run taxid
    #-------------------------------------------------------------------------
    #map_info_msg "creating new todo file with taxid $_RUN_TAXID and source $SOURCE_TODO"
    ## just src todo plus taxid
    #{
    #    cat "$SOURCE_TODO"
    #    echo "$_RUN_TAXID"
    #} | sort -u > "$newtodo"
    
    return 0
}

# gets todo file full path for given tax id and node
MAP_TODO=""
map_set_map_todo() {
    MAP_TODO=""
    local lbl="$_RUN_LABEL"
    [[ $# -eq 1 ]] && lbl="$1"
    MAP_TODO="$DIR_PROJECT/todo/${lbl}.todo"
    return 0
}


map_get_todo_list() {
    for x in $(find $DIR_PROJECT/todo/ -type f -name "*.todo" 2>/dev/null); do
        echo $x
    done
    return 0
}

map_get_todo_label_list() {
    for x in $(find $DIR_PROJECT/todo/ -name "*.todo" 2>/dev/null); do
        echo $(basename $x .todo)
    done
    return 0
}

map_get_metadata_list() {
    for x in $(find $DIR_PROJECT/Results/ -name "*.metadata" 2>/dev/null); do
        echo $(basename $x .metadata)
    done
    return 0
}

map_split_todo_name() {
    [[ $# -eq 1 ]] || return 0
    local t=$(file_basename "$1")
    local tarr=($(tokenize '_' "$t"))
    local N=${#tarr[@]}
    local what=0
    local x=""
    local node=""
    local subset=""
    local subsample=""
    local scdir=""
    local taxid=""
    for x in ${tarr[@]}; do
        if   [[ "$x" == "node" ]]; then
            what=1
            continue
        elif [[ "$x" == "subset" ]]; then
            what=2
            continue
        elif [[ "$x" == "taxid" ]]; then
            what=3
            continue
        elif [[ "$x" == "nsub" ]]; then
            what=4
            continue
        elif [[ "$x" == "scdir" ]]; then
            what=5
            continue
        fi
        if (( what == 1 )); then
            node="$x"
            continue
        elif (( what == 2 )); then
            [[ -z "$subset" ]] || subset="${subset}_"
            subset="$subset$x"
            continue
        elif (( what == 3 )); then
	    [[ -z $taxid ]] || taxid="${taxid}_"
            taxid="$taxid$x"
            continue
        elif (( what == 4 )); then
            subsample="$x"
            continue
        elif (( what == 5 )); then
            scdir="$x"
            continue
        fi
    done
    # quit in error if taxid or node are not found
    [[ -z $taxid ]] && return 255
    [[ -z $node  ]] && return 255
    
    [[ -z $subset   ]] && subset="-"
    [[ -z $subsample ]] && subsample="-"
    [[ -z $scdir     ]] && scdir="des"

    scdir=$(map_get_scdir_value "$scdir")
    [[ -n $scdir ]] || { map_error_msg "map_split_todo_name: illegal scdir argument $3"; return 255; }
    
    # all ok
    # dont check subset for now -> compat with older versions without 'subset' in the name
    echo "$node $taxid $subset $subsample $scdir"
    return 0
}

map_write_metadata() {
    local fmeta=$DIR_PROJECT/Results/${_RUN_LABEL}.metadata
    {
        echo "_RUN_LABEL=$_RUN_LABEL"
        echo "_RUN_MODE=$_RUN_MODE"
        echo "_RUN_NODEID=$_RUN_NODEID"
        echo "_RUN_NODEID_IS_VIRIDAE=$_RUN_NODEID_IS_VIRIDAE"
        echo "_RUN_TAXIDS=( ${_RUN_TAXIDS[@]} )"
        echo "_RUN_HASHID=$_RUN_HASHID"
        echo "_RUN_SUBSET=$_RUN_SUBSET"
        echo "_RUN_SUBSAMPLE=$_RUN_SUBSAMPLE"
        echo "_RUN_SCOREDIR=$_RUN_SCOREDIR"
    } > $fmeta
}

map_read_metadata() {
    local runlabel="$1"
    local fmeta=$DIR_PROJECT/Results/${runlabel}.metadata
    if [[ -f $fmeta ]]; then
        map_info_msg "reading metadata file at $fmeta"
        source $fmeta
        map_set_nodeid "$_RUN_NODEID"
        if [[ ${#_RUN_TAXIDS[@]} -eq 1 ]]; then
            _RUN_SINGLE_TAXID=${_RUN_TAXIDS[0]}
        fi
    else
        map_error_msg "no metadata file found at $fmeta"
        return 255
    fi
}

map_update_run_nodeid() {
    [[ $# -eq 1 ]] || { map_error_msg "map_update_run_nodeid() wrong nr of arguments = $#"; return 255; }
    map_set_nodeid "$1"
    map_info_msg "update run nodeid to $_RUN_NODEID"
    # if run label was not given in the arguments, rebuild it since NODEID has probably been updated
    if ((_RUN_LABEL_WAS_GIVEN == 0 )); then
        map_set_runlabel || return 255
    fi
    _RUN_SUBSAMPLE_LABEL="$(map_subsample_label)"
    map_info_msg "run label is updated to <$_RUN_LABEL>"
    map_info_msg "run subsample label is updated to <$_RUN_SUBSAMPLE_LABEL>"
}
    
# create all todo files for the given taxid
map_init_todo() {
    
    local ec=0

    map_info_msg "map_init_todo: make sure the todo files are there"
    
    # set filename for todo file to be created (no error possible...)
    map_set_map_todo
    if is_dryrun; then
        dryrun_info_msg "create todo file at $MAP_TODO"
        return 0
    fi

    # get source todo file - returns SOURCE_TODO
    map_find_source_todo || return 255
    if is_dryrun; then
        dryrun_info_msg "found target todo file at $SOURCE_TODO"
    fi
    map_info_msg "-------------------------------------------"
    map_info_msg "Map todo    : $(trim_name $MAP_TODO)"
    map_info_msg "Source todo : $(trim_name $SOURCE_TODO)"
    map_info_msg "-------------------------------------------"
    
    # if todo already exists exit happy
    [[ -f "$MAP_TODO" ]] && [[ -f "$SOURCE_TODO" ]] && {
        map_info_msg "skip init todo - map todo at $(trim_name $MAP_TODO)"
        map_info_msg "as well as source todo already exists at $(trim_name $SOURCE_TODO)"
        return 0
    }

    # initialise new todo files
    # MAP_TODO    -> contains the labels to be mapped
    # SOURCE_TODO -> contains the labels of the target node
    map_fill_todo
    ec=$?
    [[ -s "$MAP_TODO" && $ec -eq 0 ]] || {
        map_error_msg "map_init_todo: failed creating a new TODO file"
        map_error_msg "               todo      = $MAP_TODO"
        map_error_msg "               mode      = $_RUN_MODE"
        map_error_msg "               label     = $_RUN_LABEL"
        map_error_msg "               hashid    = $_RUN_HASHID"
        map_error_msg "               source    = $_RUN_NODEID"
        map_error_msg "               scdir     = $_RUN_SCOREDIR"
        map_error_msg "               subsample = $_RUN_SUBSAMPLE"
        map_error_msg "               subset    = $_RUN_SUBSET"
        rm -f "$MAP_TODO"
        return 255
    }
    return 0
}


#map_split_runlabel() {
#    [[ $# -eq 1 ]] || {
#        map_error_msg "usage: map_split_runlabel <runlabel>]"
#        return 255
#    }
#    local runlabel="$1"
#    local elems=($(tokenize '_' "$runlabel"))
#}

export IMPORT_MSG=""
#
# Import a fasta file
# Arguments: <taxid> <fasta file>
#
# 1. check if <taxid> is free, ie not in ODB (not in the root node todo file)
# 2. check if Rawdata/imported_<label>_<fasta label>.fs exists; if so, quit with a warning - <label> = basename <fasta>
# 3. check taxid; return with error if it's already used (ie Rawdata/<taxid>.fs exists)
# 4. create import/<taxid>.txt
# 5. import the fasta through create_project_odb.sh
# 6. if last step failed, return error
#
# It returns an error code
#  0   - OK
#  1   - warning
#  255 - error
# In addition it prints a msg to stdout related to the error code.
#
map_import_fasta() {
    [[ $# -eq 2 ]] || { map_error_msg "usage: map_import_fasta <taxid> <fasta>"; return 255; }
    IMPORT_MSG=""
    local taxid="$1"
    local fasta=$(expand_name "$2")
    
    [[ -n $taxid ]] || { IMPORT_MSG="map_import_fasta requires a non-empty taxid"; return 255; }

    [[ -f $fasta ]] || { IMPORT_MSG="file not found"; return 255; } #{ map_error_msg "unable to find user fasta file"; return 255; }

    # get label of fasta
    local label=$(file_basename "$fasta")

    # 1. check taxid
    map_check_taxid "$taxid" || { IMPORT_MSG="taxid already in odb"; return 255; }
    
    # 2. check if already imported
    local fraw="$DIR_PROJECT/Rawdata/imported_${taxid}_${label}.fs"
    local frawok=0
    [[ -f "$fraw" ]] && frawok=1 #{ IMPORT_MSG="a file with the same label is already imported ($label)"; return 1; }

    # 3. check taxid
    local ftax="$DIR_PROJECT/Rawdata/${taxid}.fs"
    local ftaxok=0
    [[ -f "$ftax" ]] && ftax=1 #{ IMPORT_MSG="taxid already used ($taxid)"; return 255; }

    # 4. check import file
    local fimp="$DIR_PROJECT/import/${taxid}.txt"
    local fimpok=0
    [[ -f "$fimp" ]] && fimpok=1 #{ IMPORT_MSG="taxid already imported (import/${taxid}.txt)"; return 255; }

    # 5. check how much is there
    # if anything other than 0 or 3 -> something wrong
    local ftotal=$((frawok+ftaxok+fimpok))
    (( ftotal > 3 )) && ftotal=3 # dumb check - presumably this cannot happen...
    
    # all three there -> already imported, return with a warning
    (( ftotal == 3 )) && { IMPORT_MSG="a file with the same label ($label) and taxid ($taxid) is already imported"; return 1; }

    # 6. ftotal is non-zero and <3 -> remove all files and re-import
    # at this point ftotal is guaranteed to be < 3
    if (( ftotal<3 && ftotal>0 )); then
        ftotal=0
        rm -f "$fraw" "$ftax" "$fimp"
    fi

    # 7. create import file if not all exists already
    if (( ftotal == 0 )); then
        echo "+$taxid $fasta" > "$fimp"
    fi
    
    # 8. import the file and force relabling ( the '+' before taxid)
    #    import by creating a soft link ('0' in the args)
    source "$DIR_ORTHOLOGER/bin/common_manage_project.sh"
    import_fasta 0 "$fasta" "+$taxid"
    if [[ $? -gt 1 ]]; then
	    rm -f "$fimp"
	    IMPORT_MSG="create failed"
	    return 255
    fi
    return 0
}

subsample_fasta() {
    (( _RUN_DO_SUBSAMPLE == 2 )) || return 0
    local srcdir="$1"
    local trgdir="$2"
    local scdirOpt=""
    local Nmax=$_RUN_SUBSAMPLE
    
    # input cluster
    local fcluster=$srcdir/Cluster/${_RUN_NODEID}.og
    
    # input spcov, if used
    local fspcov=$srcdir/Cluster/${_RUN_NODEID}.spcov
    
    # start from best score
    [[ "$_RUN_SCOREDIR" == "des" ]] && scdirOpt='-r'
    
    # start from best score, subsample by steps
    [[ "$_RUN_SCOREDIR" == "sde" ]] && scdirOpt='-r -d'

    # start from worst score, subsample by steps
    [[ "$_RUN_SCOREDIR" == "sas" ]] && scdirOpt='-d'

    # select by sorting genes in cluster first by species coverage and then by score
    local checkSpcov=0
    [[ "$_RUN_SCOREDIR" == "sco" ]] && { scdirOpt="-c $fspcov"; checkSpcov=1; }

    # sco, but stepwise selection
    [[ "$_RUN_SCOREDIR" == "ssc" ]] && { scdirOpt="-c $fspcov -d"; checkSpcov=1; }

    [[ -f $fcluster ]] || { map_error_msg "missing cluster file $fcluster"; return 255; }

    local facc=0.97
    if (( checkSpcov == 1 )); then
	    [[ -f $fspcov   ]] || { map_error_msg "missing coverage file $fspcov";  return 255; }
        Nmax=$($PYTHON3 $DIR_ORTHOLOGER/bin/get_subsample_nmax.py -c $fspcov -m $_RUN_SUBSAMPLE -f $facc)
    fi

    
    verbose_info_msg "subsample from cluster $fcluster"
    
    [[ -d $srcdir           ]] || { map_error_msg "missing source dir at $srcdir"; return 255; }
    [[ -d $srcdir/Sequences ]] || { map_error_msg "missing Sequences dir at $srcdir/Sequences"; return 255; }
    [[ -d $srcdir/Cluster   ]] || { map_error_msg "missing Cluster dir at $srcdir/Cluster"; return 255; }
    [[ -d $trgdir           ]] || { map_error_msg "missing target dir at $trgdir"; return 255; }
    [[ -f $fcluster         ]] || { map_error_msg "missing cluster file at $fcluster"; return 255; }

    # use the input N=subsample for label and the deduced Nmax for calculation
    map_info_msg "subsample from node $node , label $_RUN_SUBSAMPLE_LABEL with options -n $Nmax $scdirOpt"

    local fgenes=( $($PYTHON3 $DIR_ORTHOLOGER/bin/subsample_cluster_genes.py -o $fcluster -n "$Nmax" $scdirOpt | sort | $PYTHON3 $DIR_ORTHOLOGER/bin/group_by_species.py -d $trgdir --label $_RUN_SUBSAMPLE_LABEL ) )
    [[ $? -eq 0 ]] || { map_error_msg "failed creating fgenes"; return 255; }

    map_info_msg "done subsampling - now build fasta files"

    local subsampledTodo="$trgdir/../todo/${_RUN_SUBSAMPLE_LABEL}.todo"
    local data=
    local saveIFS=$IFS
    for data in ${fgenes[@]}; do
        IFS=','
        elems=( $data )
        IFS=$saveIFS
        local taxid=${elems[0]}
        local fgeneList=${elems[1]}
        b=$(basename $fgeneList .txt)
        ffasta=$trgdir/${b}.fs_selected
        fsrc=$srcdir/Sequences/${taxid}.fs_selected
        Nfsrc=$(grep -c '>' $fsrc)
        [[ -f $fgeneList ]] || { map_error_msg "missing genes file $fgeneList" 1>&2;   continue; }
        [[ -f $fsrc      ]] || { map_error_msg "missing source file $fsrc" 1>&2;  continue; }
        [[ -f $ffasta    ]] && continue #{ echo "warning: skip existing file $ffasta" 1>&2; continue; }
        {
            echo "# N(src)      $Nfsrc"
            echo "# f(spcov)    $facc"
            echo "# Subsample   $Nmax ($subsample)"
            grep -f $fgeneList --no-group-separator -A 1 $fsrc
        }> $ffasta
        Nfasta=$(grep -c '>' $ffasta)
        {
            echo "# fgenes    $fgenes"
            echo "# fsrc      $fsrc"
            echo "# N(fasta)  $Nfasta"
            echo "#END"
        }>> $ffasta

        # output label for subsampled todo
        echo $b
    done > $subsampledTodo

}

#---------------------------------------
# subsample fasta file
#
# Args: <nodeid> <subsample> <scdir>
#       <nodeid>     which node to subsample
#       <subsample>  max nr of genes per cluster (different taxids)
#       <scdir>      score selection strategy (asc,ssc,des,...)
#
# Skip if <subsample> equals zero.
#---------------------------------------
map_subsample() {
    (( _RUN_DO_SUBSAMPLE == 2 )) || return 0

    map_info_msg "subsample fasta for node $_RUN_NODEID using N=$_RUN_SUBSAMPLE and score dir='$_RUN_SCOREDIR'"

    # to get OrthoDB clade info for current nodeid
    map_find_orthodb_todo || return 255

    # get source todo
    map_find_source_todo  || return 255

    # get source and target directories
    local srcdir=$(map_dir_clade "$ODB_CLADE")
    local trgdir="$DIR_PROJECT/Sequences"
    verbose_info_msg "subsample node <$_RUN_NODEID> , N = $_RUN_SUBSAMPLE, scdir = $_RUN_SCOREDIR , clade $ODB_CLADE, source $srcdir"

    is_dryrun && return 0
    
    # check if subsampled files are already there
    local nmissing=0
    local ntot=0
    local subsampledTodo="$trgdir/../todo/${_RUN_SUBSAMPLE_LABEL}.todo"
    if [[ -f $subsampledTodo ]]; then
        while read sstx; do
            [[ ${sstx:0:1} == "#" ]] && continue
            [[ ${#sstx} -eq 0 ]] && continue
            local f=$trgdir/${sstx}.fs_selected
            [[ -f $f ]] || nmissing=$((nmissing+1))
            ntot=$((ntot+1))
        done < "$subsampledTodo"
        if (( nmissing > 0 )); then
            map_info_msg "generate $nmissing subsampled fasta files out of $ntot"
        fi
    else
        # force regenerate all by setting nmissing>0
        map_info_msg "generate all subsampled fasta files"
        nmissing=1
    fi

    # generate subsamped fasta
    if (( nmissing > 0 )); then
        subsample_fasta "$srcdir" "$trgdir"
    else
        map_info_msg "all subsampled files are already generated"
    fi
}

map_list_taxids() {
    # get user path
    local x=""
    local taxid=""
    for x in $(find "$DIR_PROJECT/Rawdata" -name "*.fs"); do
        taxid=$(basename "$x" .fs)
        printf '%s\n' "$taxid"
    done
}

map_cleanup_fasta() {
    [[ $# -eq 2 ]] || { map_error_msg "usage: map_cleanup_fasta <taxid> <action>"; return 255; }
    local taxid=$1
    local action="$2"
    echo "NOT YET IMPLEMENTED"
    return 255
}

map_runlog_name() {
    local datestr=$(date '+%Y%m%d_%H%M%S')
    echo ${DIR_PROJECT_RUNLOGS}/${_RUN_LABEL}_${datestr}.log
    return 0
}

map_ready_to_run() {
    map_info_msg "check if ready to run at project dir $DIR_PROJECT"
    
    # dry run check
    is_dryrun && info_msg "check if ready to run for node=$_RUN_NODEID hashid=$_RUN_HASHID" && return 0
    
    # check system
    local fp=$DIR_PROJECT/orthologer.sh
    [[ -x $fp ]] || { map_error_msg "missing local <orthologer.sh> - did the init fail?"; return 1;}
    
    local fp=$DIR_PROJECT/orthologer_conf.sh
    [[ -f $fp ]] || { map_error_msg "missing local <orthologer_conf.sh> - did the init fail?"; return 1;}

    local fp=$DIR_PROJECT/orthomapper.sh
    [[ -x $fp ]] || { map_error_msg "missing local <orthomapper.sh> - did the init fail?"; return 1;}
    
    local fp=$DIR_PROJECT/orthomapper_conf.sh
    [[ -f $fp ]] || { map_error_msg "missing local <orthomapper_conf.sh> - did the init fail?"; return 1;}

    # check required data files
    local txid=
    local ff=
    for txid in ${_RUN_TAXIDS[@]}; do
        ff=$DIR_PROJECT/Sequences/${txid}.fs
        [[ -f $ff    ]] || { map_error_msg "missing fasta file ($(trim_name $ff))";   return 1; }
    done
    
    return 0
}

export RUN_NGENES=0
export RUN_NGENES_MAPPED=0
export RUN_FMAPPED=""
map_get_ngenes() {
    RUN_NGENES=0
    RUN_NGENES_MAPPED=0
    RUN_FMAPPED="0.0"

    local runlabel="$1"
    local result_hits="$DIR_PROJECT/Results/${runlabel}.${EXT_CLUSTER}.hits"

    # nr of mapped genes
    RUN_NGENES_MAPPED=$(grep -c -v '^#' $result_hits)
    RUN_NGENES_MAPPED=$((RUN_NGENES_MAPPED-1))

    # get the original nr of genes
    while read line; do
        local f="$DIR_PROJECT/Rawdata/${line}.fs.maptxt"
        [[ -f $f ]] || continue
        local n=$(grep -c -v '^#' $f)
        RUN_NGENES=$(( RUN_NGENES + n ))
    done < "$DIR_PROJECT/todo/${runlabel}.$EXT_TODO"
    
    # compute fraction of genes mapped
    RUN_FMAPPED="$(calc 100.0*$RUN_NGENES_MAPPED/$RUN_NGENES)"
}

header_cluster() {
    printf '#cluster_id\tgene_id\ttype\tn\tali_start\tali_end\tpid\tscore\tevalue'
}

header_description_noclid() {
    printf '#cog\tannotation\tmf_go\tbp_go\tec\tkegg\tinterpro'
}

header_description() {
    printf '#cluster_id\t'
    header_description_noclid
}

LOADED_ORTHOMAPPER_FUN_SH=0
fi
