#!/usr/bin/env bash
#
# orthologer - Pipeline for deducing orthology from fasta files.
# Copyright (C) 2005-2023 E. Zdobnov lab: F. Tegenfeldt
# <fredrik.tegenfeldt@unige.ch> based on code by E. Kriventseva
#
# This file is part of orthologer.
#
# orthologer is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# orthologer is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with orthologer.  If not, see <http://www.gnu.org/licenses/>.
#
#--------------------------------------------------------------------------

# A long if statement that spans over the whole file - do not indent...
if [[ "${LOADED_COMMON_ORTHODB_SH+1}" == "" ]]; then
    #--------------------------------------------------------------------------
    # make sure DIR_ORTHOLOGER is set
    export DIR_ORTHOLOGER=${DIR_ORTHOLOGER:="$(cd -P "$(dirname "${BASH_SOURCE[0]}")"/../ && pwd)"}
    [[ -d "$DIR_ORTHOLOGER" ]] || { echo "FATAL: unable to determine DIR_ORTHOLOGER! Got <$DIR_ORTHOLOGER>." 1>&2; exit 255; }

    #--------------------------------------------------------------------------
    # Check prerequisites (bash version)
    source $DIR_ORTHOLOGER/bin/check_prerequisites.sh

    # Check user id - possibly fail if root
    source $DIR_ORTHOLOGER/bin/check_runas_root.sh

    # Define OrthoDB version strings - set_orthobe_api()
    # URL_ORTHODB_API_BASE    - base URL
    # URL_ORTHODB_API_VERSION - OrthoDB version string
    source $DIR_ORTHOLOGER/bin/set_orthodb_api.sh

    #--------------------------------------------------------------------------
    # check for common
    [[ -f "$DIR_ORTHOLOGER/bin/common_pipeline.sh" ]] || { echo "FATAL: unable to find main pipeline at $(pwd)/bin/common_pipeline.sh"; exit 255; }

    # source the file
    source "$DIR_ORTHOLOGER/bin/common_pipeline.sh"

    debug_msg "loaded common_pipeline.sh"


    #===============================================================
    # call main common file
    #===============================================================
    CMD_PIPELINE="orthologer.sh"
        
    ###    source $DIR_ORTHOLOGER/bin/profiler.sh
    ###    profile_on

    # do not allow unset variables
    set -u

    #-----------------------------------------------
    # set global variables
    #-----------------------------------------------
    export LOADED_COMMON_ORTHODB_SH=${LOADED_COMMON_ORTHODB_SH=}
    
    #---------------------------------------
    # external software directories
    export DIR_BRHCLUS=${DIR_BRHCLUS:=}
    export DIR_BLAST=${DIR_BLAST:=}
    export DIR_SWIPE=${DIR_SWIPE:=}
    export DIR_DIAMOND=${DIR_DIAMOND:=}
    export DIR_MMSEQS=${DIR_MMSEQS:=}
    export DIR_CDHIT=${DIR_CDHIT:=}

    #---------------------------------------
    # special software
    export DIR_CLUSTALO=${DIR_CLUSTALO:=}
    export DIR_HMMER=${DIR_HMMER:=}

    # special for simple gene prediction
    export DIR_PRODIGAL=${DIR_PRODIGAL:=}

    #---------------------------------------
    # external software execs
    # cdhit
    export EXE_CDHIT_NUCL=${EXE_CDHIT_NUCL:=}
    export EXE_CDHIT_PROT=${EXE_CDHIT_PROT:=}

    # blast
    export EXE_MAKEBLASTDB=${EXE_MAKEBLASTDB:=}
    export EXE_DUSTMASKER=${EXE_DUSTMASKER:=}
    export EXE_SEGMASKER=${EXE_SEGMASKER:=}
    export EXE_BLASTP=${EXE_BLASTP:=}
    export EXE_BLASTN=${EXE_BLASTN:=}

    # aligners
    export EXE_SWIPE=${EXE_SWIPE:=}
    export EXE_DIAMOND=${EXE_DIAMOND:=}
    export EXE_MMSEQS=${EXE_MMSEQS:=}
    
    #brhclus tools
    export EXE_GETINPAR=${EXE_GETINPAR:=}
    export EXE_MERGEINPAR=${EXE_MERGEINPAR:=}
    export EXE_BRHCLUS=${EXE_BRHCLUS:=}
    export EXE_FINDBRH=${EXE_FINDBRH:=}
    export EXE_SPLITGENES=${EXE_SPLITGENES:=}
    export EXE_EXTRACTPAIRS=${EXE_EXTRACTPAIRS:=}

    # prodigal - gene prediction
    export EXE_PRODIGAL=${EXE_PRODIGAL:=}

    # gene prediction - max allowed size of input fasta in MB
    export GENEPRED_MAXSIZE=${GENEPRED_MAXSIZE:=20}
    
    #------------------------------------------------------------
    # HUGEMEM
    # For very large runs the memory usage may exceed the max nr
    # of 4k memory pages even though there is still memory remaining.
    # In such cases HUGEPAGES are required (typical pagesize of 2M).
    # This is only relevant for brhclus and mergeinpar tools on very large datasets.
    
    # flag if BRHCLUS/MERGEINPAR uses hugemem (0 NO, 1 YES)
    export BRHCLUS_USE_HUGEMEM=${BRHCLUS_USE_HUGEMEM:=0}
    export MERGEINPAR_USE_HUGEMEM=${MERGEINPAR_USE_HUGEMEM:=0}
    
    # full path to libhugetlbfs.so
    export LIB_HUGEMEM=${LIB_HUGEMEM:=}
    #------------------------------------------------------------

    
    # data type FOR NOW DISABLE THE OPTION "NUCL"
    ###export DATA_TYPE=PROT ####${DATA_TYPE:=PROT}

    # extensions - declare as global, export and read-only
    $DECLARE_G -xr EXT_TREE="nw"
    $DECLARE_G -xr EXT_TODO="todo"
    $DECLARE_G -xr EXT_PAIRS="pairs"
    $DECLARE_G -xr EXT_PAIRS_TREE="pairs_tree"
    $DECLARE_G -xr EXT_FASTA="fs"
    $DECLARE_G -xr EXT_FASTA_MAPTXT="fs.maptxt"
    $DECLARE_G -xr EXT_FASTA_CLEAN="${EXT_FASTA}_clean"
    $DECLARE_G -xr EXT_FASTA_SELECTED="${EXT_FASTA}_selected"
    $DECLARE_G -xr EXT_FASTA_SELCLSTR="${EXT_FASTA_SELECTED}.clstr"
    $DECLARE_G -xr EXT_FASTA_STATS="${EXT_FASTA_SELECTED}.stats"
    $DECLARE_G -xr EXT_FASTA_SEQLEN="${EXT_FASTA_SELECTED}.seqlen"
    $DECLARE_G -xr EXT_FASTA_MASKED="${EXT_FASTA}_masked"
    $DECLARE_G -xr EXT_INPAR="inpar"
    $DECLARE_G -xr EXT_INPAR_SELECTED="${EXT_INPAR}_selected"
    $DECLARE_G -xr EXT_BRH="brh"
    $DECLARE_G -xr EXT_WEAK_BRH="weak_brh"
    $DECLARE_G -xr EXT_SPLITS_BRH="splits.$EXT_BRH"
    $DECLARE_G -xr EXT_SPLITS="splits"
    $DECLARE_G -xr EXT_CLUSTER_RAW="og_raw"
    $DECLARE_G -xr EXT_CLUSTER_INPAR="og_inpar"
    $DECLARE_G -xr EXT_CLUSTER="og"
    $DECLARE_G -xr EXT_CLUSTER_MAP="og_map"
    $DECLARE_G -xr EXT_CLUSTER_OGXML="og_xml"
    $DECLARE_G -xr EXT_ALIGN="align"
    
    # extensions: formatdb - set in set_db_ext()
    export EXT_DBPRE=${EXT_DBPRE:="x"}
    export EXT_DB=${EXT_DB:="xxx"}
    export EXT_DB1=${EXT_DB1:="xxx"}
    export EXT_DB2=${EXT_DB2:="xxx"}
    export EXT_DB_LOG=${EXT_DB_LOG:="xxx"}

    # if this file exists, it is used as the source for pairwise runs
    export PAIR_FILE=${PAIR_FILE:=""}

    # in case the pairwise scan is not a simple scan over a matrix
    export GENERATED_PAIR_FILE=""
    export GENERATED_PAIR_FILE_OPTS=""
    export GENERATED_PAIR_FILE_HEADER=""

    # flag, set to 1 if fasta is to be merged before alignment
    export ODB_ALIGN_MERGED=${ODB_ALIGN_MERGED:=0}

    # in case merge of all fasta into one is requested for ALIGNMENT step
    export MERGED_LABEL=""
    export MERGED_FASTA=""
    
    # run mode - sensitivity/stringency
    #  FAST      - mmseqs sensitivy=3.0
    #  XFAST     - mmseqs sensitivy=1.0
    #  SENSITIVE - mmseqs sensitivy=7.0
    #  USER      - params set by user
    export ODB_RUN_MODE=${ODB_RUN_MODE:=DEFAULT}

    # general cutoff options
    export MIN_OVERLAP=${MIN_OVERLAP:=}        # sequence minimum overlap
    export MIN_COVERAGE=${MIN_COVERAGE:=}      # sequence minimum coverage
    export MAX_EVALUE=${MAX_EVALUE:=}          # max e-value in the final BRH
    
    # select step, percentage id for match (close to 100)
    export SELECT_PID=${SELECT_PID:=}          # percentage id in SELECT step
    
    # aligner: scaling parameter for e-values
    # should be >=1.0 as this is the max eval allowed by the aligner

    # aligner: effective DB size used for calculating the e-value
    ###export ALIGNMENT_EFFDBSZ=0 #${ALIGNMENT_EFFDBSZ=20000} DEPRECATED

    # aligner: maximum nr of align hits to show (if not cut by e-value)
    export ALIGNMENT_NUMALIGNMENTS=${ALIGNMENT_NUMALIGNMENTS:=100}

    # aligner: calculate full (0) or half matrix (1)
    # default is full (0)
    export ALIGNMENT_MATRIX=${ALIGNMENT_MATRIX:=1}

    # makebrh:
    # force running MAKEBRH even if they were computed in the alignment step
    export FORCE_MAKEBRH=${FORCE_MAKEBRH:=0}
    
    # makebrh: options for fuzzy BRH's
    # NALT>2 -> generates alternatives to the main BRH, ie BRH's that are not exactly BRH but with score close to the best
    export MAKEBRH_NALT=${MAKEBRH_NALT:=}
    
    # if NALT>1, this tells how much the alternatives can differ in score
    #   negative -> as if NALT=1, only the best
    #   0.0 -> only the best one (or possibly with identical score)
    #   1.0 -> all alternatives
    export MAKEBRH_FSEPMAX=${MAKEBRH_FSEPMAX:=}
    
    # input cluster in case of merging using brhclus
    export CLUSTER_INPOG=${CLUSTER_INPOG:=}
    
    # flag -> 0- no weak brh, 1- has weak brh
    export CLUSTER_INPOG_WEAKBRH=${CLUSTER_INPOG_WEAKBRH:=0}
    
    # opts for BRHCLUS , constructed from the two above
    export CLUSTER_INPOG_OPTS=
    
    # brhclus options - TRIEVAL and PAIREVAL are not user settable - will be deduced
    # pair eval scale should be <1.0 as this is the max eval allowed for forming a
    # pair wise cluster when no triangle is found
    export BRHCLUS_PAIREVAL_SCALE=${BRHCLUS_PAIREVAL_SCALE:=0.001}
    export BRHCLUS_PAIREVAL=${BRHCLUS_PAIREVAL=}
    export BRHCLUS_TRIEVAL=${BRHCLUS_TRIEVAL=}
    export BRHCLUS_MAX_SPRATIO=${BRHCLUS_MAX_SPRATIO:=}
    export BRHCLUS_MERGE_FNLINKS=${BRHCLUS_MERGE_FNLINKS:=}
    ####export BRHCLUS_MINCONN=${BRHCLUS_MINCONN:=0.8}    # merge cut-off
    ####export BRHCLUS_MINLINK=${BRHCLUS_MINLINK:=0.7}    # merge cut-off, fraction links (>BRHCLUS-1.4.5)

    # this one is set in step_cluster(_tree) - not user settable
    export BRHCLUS_DO_WEAK=0
    
    # mergeinpar options
    ####export MERGEINPAR_CLFUNC=${MERGEINPAR_CLFUNC:=5}           # cut-off function - recommended is 5 (as of BRHCLUS-1.1.0)
    export MERGEINPAR_FHITSMAX=${MERGEINPAR_FHITSMAX:=0.6}     # inparalog link resolution
    export MERGEINPAR_FSPECMIN=${MERGEINPAR_FSPECMIN:=}     # merge: species overlap, merge above
    export MERGEINPAR_FCLUSTMIN=${MERGEINPAR_FCLUSTMIN:=}   # merge: fraction of links, merge above
    ####export MERGEINPAR_FLINKEDMIN=${MERGEINPAR_FLINKEDMIN:=0.6} # OBSOLETE (clfunc 2 or 3)

    # splitgenes
    # input split file to be used in eg MAKEINPAR if SPLITBRH is enabled
    # NOTE: this is set by step_makeinpar()
    ###export SPLIT_FILE=${SPLIT_FILE=}

    
    # flag if clustering using tree
    export TREE_ENABLED=${TREE_ENABLED:=0}
    
    # root clade - todo label for the root clade
    export TREE_ROOT_CLADE=${TREE_ROOT_CLADE:=""}

    # tree input file
    export TREE_INPUT_FILE=${TREE_INPUT_FILE:=""}

    # OrthoDB URL to tree as a json - set in 'set_tree_source()'
    export TREE_INPUT_ODB=""

    # tree: run over leaves and/or nodes
    # 0 - run all
    # 1 - run only leaf nodes
    # 2 - run only internal nodes
    export TREE_CLUSTER_RUN=${TREE_CLUSTER_RUN:=0}
    
    # extra options for tools
    export OPTS_DUSTMASKER=${OPTS_DUSTMASKER=}   # MASKER:
    export OPTS_SEGMASKER=${OPTS_SEGMASKER=}     # MASKER:
    export OPTS_CDHIT=${OPTS_CDHIT:="-M 4000"}    # SELECT: cdhit memory requirement: -M 4000
    ###    export OPTS_BLASTCLUST=${OPTS_BLASTCLUST=}   # SELECT:
    export OPTS_SWIPE=${OPTS_SWIPE:=}             # ALIGNMENT: swipe aligner
    export OPTS_DIAMOND=${OPTS_DIAMOND:=}         # ALIGNMENT: diamond aligner
    export OPTS_MMSEQS=${OPTS_MMSEQS:=}           # ALIGNMENT: mmseqs2 aligner
    export OPTS_BLAST=${OPTS_BLAST:=}             # ALIGNMENT: blast+
    export OPTS_BRHCLUS=${OPTS_BRHCLUS:=""}       # CLUSTER
    export OPTS_MERGEINPAR=${OPTS_MERGEINPAR:=""}         # MERGEINPAR
    export OPTS_MERGEINPARSEL=${OPTS_MERGEINPARSEL:=""}   # MERGEINPARSEL

    # temp: special MMSEQS flag : align (0), search (1), rbh -> search or BRH (2)
    export MMSEQS_ALIGN_MODE=${MMSEQS_ALIGN_MODE:=0}
    
    # temp: for testing: score scaling modes - see alignment_mmseqs.sh
    # use 2 -> score' = score/seqlen
    export ALIGNMENT_SCORE_MODE=${ALIGNMENT_SCORE_MODE:=2}
    
    # temp: mmseqs search rescore mode
    export MMSEQS_RESCORE_MODE=${MMSEQS_RESCORE_MODE:=0}
    
    # mmseqs prefiltering sensitivity [1.0,7.5]
    export MMSEQS_SENSITIVITY=${MMSEQS_SENSITIVITY:=}
    
    # plugin scripts
    export PREPROC_SCRIPT=${PREPROC_SCRIPT:=}
    export ALIGNMENT_SCRIPT=${ALIGNMENT_SCRIPT:=}
    export MASKER_SCRIPT=${MASKER_SCRIPT:=}
    export SELECT_SCRIPT=${SELECT_SCRIPT:=}
    export CLUSTER_SCRIPT=${CLUSTER_SCRIPT:=}

    # plugin labels
    export PREPROC_LABEL=${PREPROC_LABEL:="CHECKFASTA"}
    export ALIGNMENT_LABEL=${ALIGNMENT_LABEL:="DIAMOND"}
    export MASKER_LABEL=${MASKER_LABEL:="SEGMASKER"}
    export SELECT_LABEL=${SELECT_LABEL:="CDHIT"}
    export CLUSTER_LABEL=${CLUSTER_LABEL:="BRHCLUS"}

    # supported plugin labels
    export PREPROC_SUPPORT="CHECKFASTA MAKEGENES"
    export ALIGNMENT_SUPPORT="BLAST SWIPE MMSEQS DIAMOND"
    export MASKER_SUPPORT="NONE SEGMASKER SEG+XNU DUSTMASKER"
    export SELECT_SUPPORT="NONE CDHIT BLASTCLUST"
    export CLUSTER_SUPPORT="BRHCLUS"

    # requirements per plugin
    #$DECLARE_G -A PREPROC_REQ
    #$DECLARE_G -A ALIGNMENT_REQ
    #$DECLARE_G -A MASKER_REQ
    #$DECLARE_G -A SELECT_REQ
    #$DECLARE_G -A CLUSTER_REQ

    # array of fasta files
    #$DECLARE_G -a FASTA_BASENAMES

    # label selection for pairwise processing
    export OP_LABELA_FIRST=${OP_LABELA_FIRST:=""}
    export OP_LABELA_LAST=${OP_LABELA_LAST:=""}
    export OP_LABELB_FIRST=${OP_LABELB_FIRST:=""}
    export OP_LABELB_LAST=${OP_LABELB_LAST:=""}

    # POSTPROC
    # csv list of postproc label(s)
    export POSTPROC_LABELS=${POSTPROC_LABELS:=}

    # set in postproc_X.sh scripts
    export POSTPROC_NTARGETS=0
    export POSTPROC_TRGS=()
    export POSTPROC_NAME=""

    # special ones for POSTPROC:cmpref
    export POSTPROC_CMPREF_CLUSTER=${POSTPROC_CMPREF_CLUSTER:=""}
    export POSTPROC_CMPREF_REFERENCE=${POSTPROC_CMPREF_REFERENCE:=""}
    
    # csv list of postproc script(s) found, with full filenames
    export OP_POSTPROC_SCRIPTS_FOUND=""

    # internally defined postproc scripts
    export OP_POSTPROC_INTERN=""

    # project directories
    export DIR_PROJECT_IMPORT=${DIR_PROJECT_IMPORT:=""}
    export DIR_PROJECT_BUSCO=${DIR_PROJECT_BUSCO:=""}
    export DIR_PROJECT_JOBLOG=${DIR_PROJECT_JOBLOG:=""}
    export DIR_PROJECT_RUNLOGS=${DIR_PROJECT_RUNLOGS:=""}
    export DIR_PROJECT_TODO=${DIR_PROJECT_TODO:=""}
    export DIR_PROJECT_RAWDATA=${DIR_PROJECT_RAWDATA:=""}
    export DIR_PROJECT_SEQUENCES=${DIR_PROJECT_SEQUENCES:=""}
    export DIR_PROJECT_PWC=${DIR_PROJECT_PWC:=""}
    export DIR_PROJECT_CLUSTER=${DIR_PROJECT_CLUSTER:=""}
    export DIR_PROJECT_RESULTS=${DIR_PROJECT_RESULTS:=""}
    export DIR_PROJECT_BUSCO=${DIR_PROJECT_BUSCO:=""}

    # ignore missing PWC's when running SETXXX steps
    export IGNORE_MISSING=${IGNORE_MISSING:=0}

    # dump more detailed config if set
    export DUMP_COMMON_ADVANCED=${DUMP_COMMON_ADVANCED:=0}
    
    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    
    
    #-----------------------------------------------
    # dump ODB specific part of user config
    #-----------------------------------------------
    proj_dump_conf_sh() {
        set_run_mode
        
        local comment=""
        [[ $ODB_RUN_MODE == USER ]] || comment="#"
        
        echo_title_reset
        echo "#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
        echo "# ORTHODB specific setup"
        echo "#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
        echo
        echo_title "OrthoDB API base URL and version string"
        echo "URL_ORTHODB_API_BASE=$URL_ORTHODB_API_BASE"
        echo "URL_ORTHODB_API_VERSION=$URL_ORTHODB_API_VERSION"
        echo
        echo_title "Orthologer run mode"
        echo "#    DEFAULT   - more specific, faster"
        echo "#    XFAST     - least sensitive, fastest"
        echo "#    SENSITIVE - most sensitive"
        echo "#    USER      - use parameters in this file"
        echo "#    This sets the following parameters:"
        echo "#    * MIN_OVERLAP          - sequence overlap/min length"
        echo "#    * MIN_COVERAGE         - sequence match coverage"
        echo "#    * MAX_EVALUE           - max e-value cutoff (alignment)"
        echo "#    * SELECT_PID           - min percentage id for match in SELECT"
        echo "#    * MMSEQS_SENSITIVITY   - alignment (mmseqs): sensitivity parameter"
        echo "#    * MERGEINPAR_FCLUSTMIN - mergeinpar: cluster merge param 1"
        echo "#    * MERGEINPAR_FSPECMIN  - mergeinpar: cluster merge param 2"
        echo "#    * MAKEBRH_NALT         - makebrh   : fuzzy BRHs"
        echo "#    * MAKEBRH_FSEPMAX      - makebrh   : fuzzy BRHs"
        echo "ODB_RUN_MODE=$ODB_RUN_MODE"
        echo
        
        echo_title "Tool dirs"
        echo "DIR_BRHCLUS=\"$DIR_BRHCLUS\""
        echo "DIR_BLAST=\"$DIR_BLAST\""
        echo "DIR_SWIPE=\"$DIR_SWIPE\""
        echo "DIR_DIAMOND=\"$DIR_DIAMOND\""
        echo "DIR_MMSEQS=\"$DIR_MMSEQS\""
        echo "DIR_CDHIT=\"$DIR_CDHIT\""
        echo
        echo_title "ODB plugin labels"
        echo "PREPROC_LABEL=$PREPROC_LABEL"
        echo "ALIGNMENT_LABEL=$ALIGNMENT_LABEL"
        echo "MASKER_LABEL=$MASKER_LABEL"
        echo "SELECT_LABEL=$SELECT_LABEL"
        echo "CLUSTER_LABEL=$CLUSTER_LABEL"
        echo
        #echo_title "Data type"
        #echo "DATA_TYPE=$DATA_TYPE"
        #echo
        echo_title "Possibly ignore missing files (if set to 1)"
        echo "IGNORE_MISSING=$IGNORE_MISSING"
        echo ""
        echo_title "Tree parameters"
        echo "# Enable tree mode by setting TREE_ENABLED=1."
        echo "# The tree can be specified in many ways:"
        echo "#    If both are set, the pairwise steps will run PL_TODO1 against PL_TODO2."
        echo "#    1. TREE_INPUT_FILE -> newick file containing the tree"
        echo "#    2. TREE_ROOT_CLADE -> set to taxid which is extracted from OrthoDB tree"
        echo "#    3. PL_TODO -> label is taxid -> proceed as in 1"
        echo "#"
        echo "#    Select what clusters to run using TREE_CLUSTER_RUN"
        echo "#    - 0 : do all"
        echo "#    - 1 : do only leaf nodes"
        echo "#    - 2 : do only internal nodes"
        echo "TREE_ENABLED=$TREE_ENABLED"
        echo "TREE_INPUT_FILE=$TREE_INPUT_FILE"
        echo "TREE_ROOT_CLADE=$TREE_ROOT_CLADE"
        echo "TREE_CLUSTER_RUN=$TREE_CLUSTER_RUN"
        echo
        echo_title "Homology/sequence match parameters <ODB_RUN_MODE=USER>"
        echo "${comment}MIN_OVERLAP=$MIN_OVERLAP"
        echo "${comment}MIN_COVERAGE=$MIN_COVERAGE"
        echo "${comment}MAX_EVALUE=$MAX_EVALUE"
        echo
        echo "#-----------------------------------------------------"
        echo "# Step SELECT"
        echo "#-----------------------------------------------------"
        echo_title "min percentage ID when selecting representatives <ODB_RUN_MODE=USER>"
        echo "${comment}SELECT_PID=$SELECT_PID"
        echo
        echo_title "cdhit opts"
        echo "OPTS_CDHIT=\"$OPTS_CDHIT\""
        echo
        echo "#-----------------------------------------------------"
        echo "# Step MASKER"
        echo "#-----------------------------------------------------"
        echo_title "dustmasker options"
        echo "OPTS_DUSTMASKER=\"$OPTS_DUSTMASKER\""
        echo
        echo_title "segmasker options"
        echo "OPTS_SEGMASKER=\"$OPTS_SEGMASKER\""
        echo
        echo "#-----------------------------------------------------"
        echo "# Step ALIGNMENT"
        echo "#-----------------------------------------------------"
        echo_title "alignment matrix"
        echo "#    0 - calculate full matrix"
        echo "#    1 - calculate half matrix (M11...MN1,M12...MN2,...,MNN)"
        echo "ALIGNMENT_MATRIX=$ALIGNMENT_MATRIX"
        echo
        echo_title "nr of alignments to show"
        echo "ALIGNMENT_NUMALIGNMENTS=$ALIGNMENT_NUMALIGNMENTS"
        echo
        
        #--------
        # The below were used for devel - now just use the default
        #echo_title "alignment score mode - set which score is retained"
        #echo "#    0 - S = raw score*alignlen/seqlen"
        #echo "#    1 - deprecated"
        #echo "#    2 - S = raw score/seqlen (default)"
        #echo "#    3 - S = raw score"
        #echo "#    4 - S = bit score"
        #echo "ALIGNMENT_SCORE_MODE=$ALIGNMENT_SCORE_MODE"
        #echo
        #echo_title "mmseqs: alignment mode"
        #echo "#    0 - align (default)"
        #echo "#    1 - search"
        #echo "#    2 - rbh (finds directly the BRH pairs)"
        #echo "MMSEQS_ALIGN_MODE=\"$MMSEQS_ALIGN_MODE\""
        #echo
        #--------
        
        echo_title "mmseqs: prefilter sensitivity <ODB_RUN_MODE=USER>"
        echo "${comment}MMSEQS_SENSITIVITY=\"$MMSEQS_SENSITIVITY\""
        echo
        #        echo_title "mmseqs: --rescore-mode used when MMSEQS_ALIGN_MODE>0"
        #        echo "#  0 - hamming"
        #        echo "#  1 - local alignment (score only)"
        #        echo "#  2 - local alignment"
        #        echo "#  3 - global alignment"
        #        echo "#  4 - longest alignment fullfilling window quality criterion [0]"
        #        echo "MMSEQS_RESCORE_MODE=$MMSEQS_RESCORE_MODE"
        #        echo
        echo_title "mmseqs extra opts"
        echo "OPTS_MMSEQS=\"$OPTS_MMSEQS\""
        echo
        echo_title "diamond extra opts"
        echo "OPTS_DIAMOND=\"$OPTS_DIAMOND\""
        echo
        echo_title "swipe extra opts"
        echo "OPTS_SWIPE=\"$OPTS_SWIPE\""
        echo
        echo_title "blastplus extra opts"
        echo "OPTS_BLAST=\"$OPTS_BLAST\""
        echo
        echo "#-----------------------------------------------------"
        echo "# Step MAKEBRH"
        echo "#-----------------------------------------------------"
        echo_title "Fuzzy BRHs : max nr of BRH candidates <ODB_RUN_MODE=USER>"
        echo "#    If NALT>1 then the BRH finder will output similar candidates."
        echo "${comment}MAKEBRH_NALT=\"$MAKEBRH_NALT\""
        echo
        echo_title "Fuzzy BRHs : max separation between the best and the candidates <ODB_RUN_MODE=USER>"
        echo "#    Only used if NALT>1"
        echo "${comment}MAKEBRH_FSEPMAX=\"$MAKEBRH_FSEPMAX\""
        echo
        #        echo "#-----------------------------------------------------"
        #        echo "# Step SPLITBRH"
        #        echo "#-----------------------------------------------------"
        #        echo_title "define split file - normally from the root node"
        #        echo "SPLIT_FILE=\"${SPLIT_FILE}\""
        #        echo
        echo "#-----------------------------------------------------"
        echo "# Step CLUSTER"
        echo "#-----------------------------------------------------"
        echo
        echo_title "set cluster input seed (for mapping)"
        echo "CLUSTER_INPOG=\"$CLUSTER_INPOG\""
        echo "CLUSTER_INPOG_WEAKBRH=\"$CLUSTER_INPOG_WEAKBRH\""
        echo
        echo_title "brhclus: extra options for brhclus"
        echo "OPTS_BRHCLUS=\"$OPTS_BRHCLUS\""
        echo
        echo_title "brhclus: max triangle evalue - if not set, MAX_EVALUE will be used"
        echo "BRHCLUS_TRIEVAL=$BRHCLUS_TRIEVAL"
        echo
        echo_title "brhclus: scale factor for max pair e-value (<= 1.0)"
        echo "BRHCLUS_PAIREVAL_SCALE=$BRHCLUS_PAIREVAL_SCALE"
        echo
        echo_title "brhclus: max species ratio in cluster merge attempts [0.0,1.0] <ODB_RUN_MODE=USER>"
        echo "#    This will prevent merging of clusters with similar species content."
        echo "#    A value of 0.0 will only allow merging of clusters with no species overlap."
        echo "${comment}BRHCLUS_MAX_SPRATIO=$BRHCLUS_MAX_SPRATIO"
        echo
        echo_title "brhclus: fraction defining the min nr of extra links required for merge attempts <ODB_RUN_MODE=USER>"
        echo ""
        echo "#    This counts in both directions for a seed link [A0,B0] in two different clusters."
        echo "#    The value is val=0 implies just the seed link."
        echo "#    val=N implies N links from A0 and N from B0 resulting in 2N+1 links."
        echo "${comment}BRHCLUS_MERGE_FNLINKS=$BRHCLUS_MERGE_FNLINKS"
        echo
        echo_title "mergeinpar: extra options"
        echo "OPTS_MERGEINPAR=\"$OPTS_MERGEINPAR\""
        echo
        echo_title "mergeinpar: inparalog resolution - second best vs best candidate"
        echo "MERGEINPAR_FHITSMAX=\"$MERGEINPAR_FHITSMAX\""
        echo
        echo_title "mergeinpar: inparalog merge 1 - min nr of linked vs nr of members in smallest cluster <ODB_RUN_MODE=USER>"
        echo "${comment}MERGEINPAR_FCLUSTMIN=\"$MERGEINPAR_FCLUSTMIN\""
        echo
        echo_title "mergeinpar: inparalog merge 2 - min species overlap <ODB_RUN_MODE=USER>"
        echo "${comment}MERGEINPAR_FSPECMIN=\"$MERGEINPAR_FSPECMIN\""
        echo
        echo "#-----------------------------------------------------"
        echo "# Step POSTPROC"
        echo "#-----------------------------------------------------"
        echo_title "define script(s)"
        echo "POSTPROC_LABELS=\"$POSTPROC_LABELS\""
        echo
        echo_title "Load system ODB common setup"
        echo "[[ -f \"\${DIR_ORTHOLOGER}/bin/common_odb.sh\" ]] || { echo \"ERROR unable to read system common_odb.sh : \$DIR_ORTHOLOGER/bin/common_odb.sh\" 1>&2; exit 255; }"
        echo "source \"\${DIR_ORTHOLOGER}/bin/common_odb.sh\""
        echo "#"
        echo
        echo "#-----------------------------------------------------"
        echo_title "Pipeline per step config"
        echo "#-----------------------------------------------------"
        echo "# Set OP_STEP_xxx variables to defaults"
        echo "op_step_setup"
        echo
        ###echo "# OP_STEP_COMPRESS[label]   : compression flag; 1->enable, 0->disable"
        echo "# OP_STEP_NPARALLEL[label]  : nr of jobs that should be run in parallel"
        echo "# OP_STEP_NTHREADS[label]   : nr of CPUs to be allocated for each job"
        echo "# OP_STEP_NMERGE[label]     : nr of jobs merged into one"
        echo "# OP_STEP_ENABLE[label]     : enable flag; 1->enable, 0->disable"
        echo "# OP_STEP_RUNMODE[label]    : run mode - either BATCH (run with scheduler) or LOCAL (run on local machine)"
        echo "# OP_STEP_SCHEDOPTS[label]  : option string for job scheduler, per step"
        echo ""
        dump_op_config_steps 0

        echo ""
        echo "# Select slice/block of the pairwise steps (ALIGNMENT,MAKEBRH and MAKEINPAR"
        echo "# If left empty, the full matrix is calculated."
        echo "# Note that the convention for half-matrix is that"
        echo "#   labelA <(=) labelB"
        echo "# This is a case insensitive string comparison (using locale LANG=C)."
        echo "OP_LABELA_FIRST=$OP_LABELA_FIRST"
        echo "OP_LABELA_LAST=$OP_LABELA_LAST"
        echo "OP_LABELB_FIRST=$OP_LABELB_FIRST"
        echo "OP_LABELB_LAST=$OP_LABELB_LAST"
        echo
        echo "# set/check variables based on ODB_RUN_MODE"
        echo "set_run_mode"
        echo
        echo "#--- END OF USER CONFIG ---"
        echo
        debug_msg "<--- proj_dump_conf_sh()"
    }


    #===============================================================
    #                        CONFIGURATION
    #===============================================================

    #-----------------------------------------------
    # get data type of fasta file(s)
    #-----------------------------------------------
    find_data_type() {
        # make sure TODO exists and is non empty
        [[ -f "$PL_TODO" ]] || return

        # this should be non-zero if TODO exists
        isempty FASTA_BASENAMES && return
        local type=
        local i=0
        for ((i=0; i<${#FASTA_BASENAMES[@]}; i++ )); do
            local fs="${FASTA_BASENAMES[$i]}.fs"
            [[ -f "$fs" ]] || continue

            type=$($PYTHON3 "${DIR_ORTHOLOGER}/bin/check_fasta.py" -f "$fs" --type --spsep ":")
            [[ "$type" == "PROT" ]]  && break
            [[ "$type" == "NUCL" ]]  && break
            [[ "$type" == "BADT" ]]  && break
        done

        echo "$type"
    }

    #-----------------------------------------------
    # establish which BLAST to use according to
    # setup
    #-----------------------------------------------
    set_blast_exe() {
        EXE_MAKEBLASTDB="" #"NOT USED"
        EXE_BLASTP="" #"NOT USED"
        EXE_BLASTN="" #"NOT USED"
        EXE_DUSTMASKER=""
        EXE_SEGMASKER=""
        local ec=0

        # if no DIR_BLAST check if in path
        if [[ ! -d "${DIR_BLAST}" ]]; then
            local fexe=$(which blastn 2>/dev/null)
            [[ -f $fexe ]] || return 1
            DIR_BLAST=$(dirname $fexe)
        fi
        
        if [[ -d "$DIR_BLAST" ]]; then
            # check new ncbi blast+
            EXE_DUSTMASKER=$DIR_BLAST/dustmasker
            EXE_SEGMASKER=$DIR_BLAST/segmasker
            EXE_MAKEBLASTDB=$DIR_BLAST/makeblastdb
            EXE_BLASTP=$DIR_BLAST/blastp
            EXE_BLASTN=$DIR_BLAST/blastn

            [[ -x $EXE_DUSTMASKER ]]    || EXE_DUSTMASKER=
            [[ -x $EXE_SEGMASKER ]]     || EXE_SEGMASKER=
            [[ -x $EXE_MAKEBLASTDB ]]   || EXE_MAKEBLASTDB=
            [[ -x $EXE_BLASTP ]]        || EXE_BLASTP=
            [[ -x $EXE_BLASTN ]]        || EXE_BLASTN=

            [[ -z $EXE_DUSTMASKER ]]  && ec=1
            [[ -z $EXE_SEGMASKER ]]   && ec=1
            [[ -z $EXE_MAKEBLASTDB ]] && ec=1
            [[ -z $EXE_BLASTP ]]      && ec=1
            [[ -z $EXE_BLASTN ]]      && ec=1
        fi
        return $ec
    }

    set_special_exe() {
        if [[ ! -f $EXE_PRODIGAL ]]; then
            EXE_PRODIGAL=
            if [[ ! -d $DIR_PRODIGAL ]]; then
                local fexe=$(which prodigal 2>/dev/null)
                if [[ -f $fexe ]]; then
                    DIR_PRODIGAL=$(dirname $fexe)
                    EXE_PRODIGAL="$fexe"
                fi
            else
                EXE_PRODIGAL="$DIR_PRODIGAL/prodigal"
                [[ -f $EXE_PRODIGAL ]] || EXE_PRODIGAL=
            fi
        fi
    }
    #-----------------------------------------------
    # set executables
    #-----------------------------------------------
    find_exe_dirs() {
        info_msg "searching for executables in path $DIR_EXTERNAL_PATH"
        [[ -z $DIR_BRHCLUS ]] && DIR_BRHCLUS=$(find_exe_dir brhclus $DIR_EXTERNAL_PATH)

        [[ -z $DIR_BLAST   ]] && DIR_BLAST=$(find_exe_dir makeblastdb $DIR_EXTERNAL_PATH)

        # search the below only if requested
        #[[ "$ALIGNMENT_LABEL" == "SWIPE"    ]]  &&
        [[ -z $DIR_SWIPE    ]]  && DIR_SWIPE=$(find_exe_dir swipe $DIR_EXTERNAL_PATH)
        #[[ "$ALIGNMENT_LABEL" == "DIAMOND"  ]]  &&
        [[ -z $DIR_DIAMOND  ]]  && DIR_DIAMOND=$(find_exe_dir diamond $DIR_EXTERNAL_PATH)
        #[[ "$ALIGNMENT_LABEL" == "MMSEQS"   ]]  &&
        [[ -z $DIR_MMSEQS   ]]  && DIR_MMSEQS=$(find_exe_dir mmseqs $DIR_EXTERNAL_PATH)
        #[[ "$SELECT_LABEL" == "CDHIT"       ]]  &&
        [[ -z $DIR_CDHIT    ]]  && DIR_CDHIT=$(find_exe_dir cd-hit $DIR_EXTERNAL_PATH)
    }
    
    set_executables() {
        # set BLAST executables
        set_blast_exe || warn_msg "unable to find BLAST related executables"

        # set special software exe
        set_special_exe
        
        # set all other executables
        if [[ -n $DIR_CDHIT ]]; then
            [[ -z $EXE_CDHIT_NUCL ]] && EXE_CDHIT_NUCL="${DIR_CDHIT}/cd-hit-est"
            [[ -z $EXE_CDHIT_PROT ]] && EXE_CDHIT_PROT="${DIR_CDHIT}/cd-hit"
        fi
        if [[ -n $DIR_SWIPE ]]; then
            [[ -z $EXE_SWIPE ]] && EXE_SWIPE="${DIR_SWIPE}/swipe"
        fi
        if [[ -n $DIR_DIAMOND ]]; then
            [[ -z $EXE_DIAMOND ]] && EXE_DIAMOND="${DIR_DIAMOND}/diamond"
        fi
        if [[ -n $DIR_MMSEQS ]]; then
            [[ -z $EXE_MMSEQS   ]] && EXE_MMSEQS="${DIR_MMSEQS}/mmseqs"
        fi
        if [[ -n $DIR_BRHCLUS ]]; then
            [[ -z $EXE_MERGEINPAR   ]] && EXE_MERGEINPAR="${DIR_BRHCLUS}/mergeinpar"
            [[ -z $EXE_GETINPAR     ]] && EXE_GETINPAR="${DIR_BRHCLUS}/getinpar"
            [[ -z $EXE_BRHCLUS      ]] && EXE_BRHCLUS="${DIR_BRHCLUS}/brhclus"
            [[ -z $EXE_FINDBRH      ]] && EXE_FINDBRH="${DIR_BRHCLUS}/findbrh"
            [[ -z $EXE_SPLITGENES   ]] && EXE_SPLITGENES="${DIR_BRHCLUS}/splitgenes"
            [[ -z $EXE_EXTRACTPAIRS ]] && EXE_EXTRACTPAIRS="${DIR_BRHCLUS}/extractpairs"
        fi
        return 0
    }
    
    proj_print_version() {
        local vbrhclus=""
        if [[ -x $DIR_BRHCLUS/brhclus ]]; then
            vbrhclus=$($DIR_BRHCLUS/brhclus  --version | grep version: | cut -d ':' -f2)
        fi
        local vcdhit=""
        if [[ -x $DIR_CDHIT/cd-hit ]]; then
            vcdhit=$($DIR_CDHIT/cd-hit | head -n1 | cut -d '=' -f 7-7)
        fi
        local vmmseqs=""
        if [[ -x $DIR_MMSEQS/mmseqs ]]; then
            vmmseqs=$(DIR_MMSEQS/mmseqs -h | grep Version: | cut -d ':' -f 2)
        fi
        local vdiamond=""
        if [[ -x $DIR_DIAMOND/diamond ]]; then
            vdiamond=$($DIR_DIAMOND/diamond version | cut -d ' ' -f 3)
        fi
        local vblast=""
        if [[ -x $DIR_BLAST/blastn ]]; then
            vblast=$($DIR_BLAST/blastn -version | grep Package | cut -d ' ' -f 4-)
        fi
        printf "%-40s : %s\n" "OrthoDB API" "$URL_ORTHODB_API_VERSION"
        printf "%-40s : %s\n" "brhclus" "$(trim $vbrhclus)"
        printf "%-40s : %s\n" "cdhit" "$(trim $vcdhit)"
        printf "%-40s : %s\n" "diamond" "$(trim $vdiamond)"
        printf "%-40s : %s\n" "mmseqs" "$(trim $vmmseqs)"
        printf "%-40s : %s\n" "blast" "$(trim $vblast)"

    }
    #-----------------------------------------------
    # check BRHCLUS version
    #-----------------------------------------------
    check_brhclus_version() {
        isnewer=0
        "$EXE_BRHCLUS" -h | grep -q minlink >& /dev/null
        if [[ $? -eq 0 ]]; then
            info_msg "using older version of brhclus: $EXE_BRHCLUS"
        else
            isnewer=1
            info_msg "using newer version of brhclus: $EXE_BRHCLUS"
        fi

        if [[ "$TREE_ENABLED" == "1" ]]; then
            if (( isnewer == 0 )); then
                warn_msg "disabling TREE_ENABLED as it is not allowed with the installed BRHCLUS version"
                warn_msg "BRHCLUS: $EXE_BRHCLUS"
            fi
        fi
    }
    
    #-----------------------------------------------
    # set data type
    #-----------------------------------------------
    check_data_type() {
        return 0
    }       
#        [[ -n $DATA_TYPE ]] || { warn_msg "DATA_TYPE is not set"; return 0; }
#        
#        # get data type and make sure it is in upper case
#        local dtype=${DATA_TYPE^^}
#        DATA_TYPE=${dtype:0:4}
#
#        if [[ "$DATA_TYPE" != "PROT" ]] && [[ "$DATA_TYPE" != "NUCL" ]]; then
#
#            error_msg "Bad data type DATA_TYPE=<$DATA_TYPE> -> please check in common.sh"
#            error_msg "Valid settings are PROT and NUCL - if BADT -> bad label or sequence found"
#            return 2
#        fi
#        debug_msg "Set DATA_TYPE = $DATA_TYPE"
#        return 0
#    }

    #--------------------------------------------------
    # set fasta files based on current todo
    # output:
    #  FASTA_BASENAMES      - all selected fasta file
    #  SELECTED_LABELS  - all selected todo labels
    #  SELECTED_MISSING - nr of missing fasta files
    #--------------------------------------------------
    export SELECTED_LABELS=()
    set_fasta_files() {
        SELECTED_LABELS=()
        FASTA_BASENAMES=()

        # no todo file, return 0
        [[ -n $PL_TODO ]] || return 0
        
        local nlbls=${#PL_TODO_LABELS[@]}
        [[ $nlbls -gt 0 ]] || return 0 #{ info_msg "no TODO labels defined!! PL_TODO = $PL_TODO"; return 255; }
        ###restore_todo_arrays
        nlbls=${#PL_TODO_LABELS[@]}
        [[ $nlbls -gt 0 ]] || return 0


        local Nsel=0
        
        local t=
        local f=
        for t in "${PL_TODO_LABELS[@]}"; do
            has_required_label $t || continue
            f="$DIR_PROJECT_SEQUENCES/${t}"
            #file_has_data "$f" || { Nempty=$((Nempty+1)); continue; }

            FASTA_BASENAMES[Nsel]="$f"
            SELECTED_LABELS[Nsel]="$t"
            Nsel=$((Nsel+1))
        done
        debug_msg "$FUNCNAME found $Nsel labels"
        (( Nsel > 0 )) || step_fatal_msg "no valid labels for fasta files were found - was required labels option used? empty todo file?"

        return 0
    }

    #--------------------------------------------------
    # DEPRECATED  update e-value cutoffs from user given info
    # see set_run_mode
    #--------------------------------------------------
    set_evalue_cutoffs() {
        fatal_msg "set_evalue_cutoffs : deprecated!"
        # if no max evalue is set, do nothing
        [[ ${#MAX_EVALUE} -gt 0 ]] || return 0

        # max e-value for triangles
        BRHCLUS_TRIEVAL=${BRHCLUS_TRIEVAL:=$MAX_EVALUE}
        # max e-value for pairs, must be lower than for triangles
        BRHCLUS_PAIREVAL=$(calc "${BRHCLUS_TRIEVAL}*$BRHCLUS_PAIREVAL_SCALE")
    }

    #--------------------------------------------------
    # DEPRECATED  set data type dependent cutoffs
    # see set_run_mode
    #--------------------------------------------------
    set_datatype_defs() {
        fatal_msg "set_datatype_defs : deprecated!"
        
        # if brhclus eval and overlap are not set, set them to defaults accordning to type
        # if DATA_TYPE is not set or PROT, set the below values
        [[ ${#MIN_OVERLAP}             -gt 0 ]] || MIN_OVERLAP=30
        [[ ${#MIN_COVERAGE}            -gt 0 ]] || MIN_COVERAGE="0.8"
        [[ ${#MAX_EVALUE}              -gt 0 ]] || MAX_EVALUE="1.0e-3"
        [[ ${#BRHCLUS_PAIREVAL_SCALE}  -gt 0 ]] || BRHCLUS_PAIREVAL_SCALE="1.0e-3"
        [[ ${#BRHCLUS_MAX_SPRATIO}     -gt 0 ]] || BRHCLUS_MAX_SPRATIO="0.5"

        # nucleotides - small e-value since fewer symbols
        # also short overlap allowed
        #if [[ "$DATA_TYPE" == "NUCL" ]]; then
        #    [[ ${#MIN_OVERLAP}             -gt 0 ]] || MIN_OVERLAP=20
        #    [[ ${#MAX_EVALUE}              -gt 0 ]] || MAX_EVALUE="1.0e-6"
        #    [[ ${#BRHCLUS_PAIREVAL_SCALE}  -gt 0 ]] || BRHCLUS_PAIREVAL_SCALE="1.0e-4"
        #fi
        # update e-value cut-offs
        set_evalue_cutoffs
    }

    #--------------------------------------------------
    # set run mode opts
    #--------------------------------------------------
    export FOUND_WARN_VAR=0
    check_var_ifset() {
        local var=$1
        local value="$2"
        [[ -n ${!var} ]] && {
            if [[ "${!var}" != "$value" ]]; then
                FOUND_WARN_VAR=1
                warn_msg "overwrite user set variable $var - old value <${!var}> to new value <$value>";
            fi
        }
        eval "$var"="$value"
        ####set_var "$var" "$value"
    }

    check_var_ifnotset() {
        local var=$1
        local value="$2"
        [[ -n ${!var} ]] || {
            FOUND_WARN_VAR=2
            warn_msg "variable $var is not set - setting var to <$value>";
            eval "$var"="$value"
        }
        ####set_var "$var" "$value"
    }

    set_run_mode() {
        # change to upper case
        ODB_RUN_MODE="${ODB_RUN_MODE^^}"
        local modestr="${ODB_RUN_MODE:0:4}"
        FOUND_WARN_VAR=0
        if [[ $modestr == "USER" ]]; then
            check_var_ifnotset MMSEQS_SENSITIVITY "3.0"
            check_var_ifnotset MERGEINPAR_FCLUSTMIN "0.5"
            check_var_ifnotset MERGEINPAR_FSPECMIN "0.4"
            check_var_ifnotset MAKEBRH_NALT "2"
            check_var_ifnotset MAKEBRH_FSEPMAX "0.03"
            check_var_ifnotset MIN_OVERLAP "30"
            check_var_ifnotset MIN_COVERAGE "0.8"
            check_var_ifnotset MAX_EVALUE "1.0e-3"
            check_var_ifnotset SELECT_PID "97"
            check_var_ifnotset BRHCLUS_MAX_SPRATIO "0.5"
            check_var_ifnotset BRHCLUS_MERGE_FNLINKS "0.3"
        else
            check_var_ifset MERGEINPAR_FCLUSTMIN "0.5"
            check_var_ifset MERGEINPAR_FSPECMIN "0.4"
            check_var_ifset MAKEBRH_NALT "2"
            check_var_ifset MAKEBRH_FSEPMAX "0.03"
            check_var_ifset MIN_OVERLAP "30"
            check_var_ifset MIN_COVERAGE "0.8"
            check_var_ifset MAX_EVALUE "1.0e-3"
            check_var_ifset SELECT_PID "97"
            check_var_ifset BRHCLUS_MAX_SPRATIO "0.5"
            check_var_ifset BRHCLUS_MERGE_FNLINKS "0.3"

            if [[ $modestr == "DEFA" ]]; then
                # defaults - lower sensitivity for speed
                check_var_ifset MMSEQS_SENSITIVITY "3.0"
            elif [[ $modestr == "XFAS" ]]; then
                # lowest sensitivity, fastest alignments
                check_var_ifset MMSEQS_SENSITIVITY "1.0"
            elif [[ $modestr == "SENS" ]]; then
                # highest sensitivity, slowest alignments
                check_var_ifset MMSEQS_SENSITIVITY "7.0"
            else
                fatal_msg "unknown ODB_RUN_MODE = $ODB_RUN_MODE, must be one of [DEFAULT,XFAST,SENSITIVE]"
            fi
        fi

        if (( FOUND_WARN_VAR==1 )); then
            warn_msg "some variables that were expected to be set (ODB_RUN_MODE=USER), are not set and are therefore set to default values"
        elif (( FOUND_WARN_VAR==2 )); then
            warn_msg "some variables that were expected NOT to be set (ODB_RUN_MODE=$ODB_RUN_MODE), are set in common.sh and are therefore overwritten"
        fi
        # brhclus: max e-value for triangles
        BRHCLUS_TRIEVAL=${BRHCLUS_TRIEVAL:=$MAX_EVALUE}
        # brhclus: max e-value for pairs, must be lower than for triangles
        BRHCLUS_PAIREVAL=$(calc "${BRHCLUS_TRIEVAL}*$BRHCLUS_PAIREVAL_SCALE")
    }
    
    #--------------------------------------------------
    # set default opts
    #--------------------------------------------------
    set_opts_defs() {
        OPTS_BRHCLUS=${OPTS_BRHCLUS=""} # extra options
        OPTS_MERGEINPAR=${OPTS_MERGEINPAR:=}
        OPTS_CDHIT=${OPTS_CDHIT="-M 4000"} # memory requirement
        OPTS_DUSTMASKER=${OPTS_DUSTMASKER:=}
        OPTS_SEGMASKER=${OPTS_SEGMASKER:=}
        ALIGNMENT_NUMALIGNMENTS=${ALIGNMENT_NUMALIGNMENTS=100}  # nr of alignments to show
        ALIGNMENT_MATRIX=${ALIGNMENT_MATRIX=0}
        OPTS_SWIPE=${OPTS_SWIPE:=}
        OPTS_MMSEQS=${OPTS_MMSEQS:=} 
        OPTS_BLAST=${OPTS_BLAST:=}
    }

    #--------------------------------------------------
    # set formatdb extensions
    #--------------------------------------------------
    set_db_ext() {

        # formatdb extensions
        EXT_DBPRE="p"
        EXT_DB="phr"
        EXT_DB1="pin"
        EXT_DB2="psq"
        EXT_DB3=""
        EXT_DB_LOG="formatdb_${ALIGNMENT_LABEL,,}.log"

        if   [[ "$ALIGNMENT_LABEL" == "MMSEQS" ]]; then
            # mmseqs aligner - special DB
            EXT_DBPRE="mm"
            EXT_DB="mm.index"
            EXT_DB1="mm.lookup"
            EXT_DB2="mm_h"
            EXT_DB3="mm.dbtype"
        elif [[ "$ALIGNMENT_LABEL" == "DIAMOND" ]]; then
            EXT_DBPRE="dmnd"
            EXT_DB="dmnd"
            EXT_DB1=""
            EXT_DB2=""
            EXT_DB3=""            
        fi

    }

    set_alignment_label() {
        [[ $# -eq 1 ]] || return 0
        ALIGNMENT_LABEL="$1"
        set_db_ext
    }
    
    #    set_split_out_file() {
    #        [[ -n $SPLIT_OUT_FILE ]] || SPLIT_OUT_FILE=${DIR_PROJECT_CLUSTER}/${PL_TODO_NAME}.$EXT_SPLITS
    #        info_msg "split output file used: $SPLIT_OUT_FILE"
    #    }
    #    set_split_in_file() {
    #        if [[ -z $SPLIT_FILE ]]; then
    #            SPLIT_FILE=${DIR_PROJECT_CLUSTER}/${TREE_ROOT_CLADE}.$EXT_SPLITS
    #            [[ -n $TREE_ROOT_CLADE ]] || SPLIT_FILE=""
    #        fi
    #    }
    
    #--------------------------------------------------
    # refresh DIR_PROJECT_xxx variables
    #--------------------------------------------------
    refresh_dir_project() {
        DIR_PROJECT_RUNLOGS=${DIR_PROJECT}/RunLogs
        DIR_PROJECT_IMPORT=${DIR_PROJECT}/import
        DIR_PROJECT_BUSCO=${DIR_PROJECT}/busco
        DIR_PROJECT_TODO=${DIR_PROJECT}/todo
        DIR_PROJECT_RAWDATA=${DIR_PROJECT}/Rawdata
        DIR_PROJECT_SEQUENCES=${DIR_PROJECT}/Sequences
        DIR_PROJECT_PWC=${DIR_PROJECT}/PWC
        DIR_PROJECT_JOBLOG=${DIR_PROJECT}/JobLog
        DIR_PROJECT_CLUSTER=${DIR_PROJECT}/Cluster
        DIR_PROJECT_RESULTS=${DIR_PROJECT}/Results
        DIR_PROJECT_BUSCO=${DIR_PROJECT}/busco

        add_project_dir DIR_PROJECT_BUSCO
        add_project_dir DIR_PROJECT_RUNLOGS
        add_project_dir DIR_PROJECT_IMPORT
        add_project_dir DIR_PROJECT_BUSCO
        add_project_dir DIR_PROJECT_TODO
        add_project_dir DIR_PROJECT_RAWDATA
        add_project_dir DIR_PROJECT_SEQUENCES
        add_project_dir DIR_PROJECT_PWC
        add_project_dir DIR_PROJECT_JOBLOG
        add_project_dir DIR_PROJECT_CLUSTER
        add_project_dir DIR_PROJECT_RESULTS
    }
    
    #--------------------------------------------------
    # Configure the environment
    #--------------------------------------------------
    proj_configure_env() {
        debug_msg "calling proj_configure_env()"
        local rval=0
        local ec=0
        #---------------------------------------------------------
        # Make sure all user settables are set to something
        #---------------------------------------------------------
        # add orthologer to path and cleanup the path (remove non-existing paths)
        DIR_ORTHOLOGER_PATH="${DIR_ORTHOLOGER_PATH}:${DIR_ORTHOLOGER}/bin:$DIR_PROJECT/bin"
        DIR_ORTHOLOGER_PATH="$(cleanup_path $DIR_ORTHOLOGER_PATH)"

        # add new plugins
        debug_msg "adding plugins"
        add_plugin PREPROC
        add_plugin ALIGNMENT
        add_plugin MASKER
        add_plugin SELECT
        add_plugin CLUSTER

        # make plugin labels upper case
        ALIGNMENT_LABEL=${ALIGNMENT_LABEL^^}
        MASKER_LABEL=${MASKER_LABEL^^}
        SELECT_LABEL=${SELECT_LABEL^^}
        CLUSTER_LABEL=${CLUSTER_LABEL^^}

        # requirements
        # use "$PLUGIN_NONE_REQ" if no requirement
        # if using environment variables that should be expanded later
        # in check_plugin(), escape the '$'
        PREPROC_REQ[CHECKFASTA]="$PLUGIN_NONE_REQ"

        ALIGNMENT_REQ[BLAST]="\$EXE_BLASTP \$EXE_BLASTN"
        ALIGNMENT_REQ[SWIPE]="\$EXE_SWIPE"
        ALIGNMENT_REQ[MMSEQS]="\$EXE_MMSEQS"
        ALIGNMENT_REQ[DIAMOND]="\$EXE_DIAMOND"

        MASKER_REQ[NONE]="$PLUGIN_NONE_REQ"
        MASKER_REQ[SEGMASKER]="\$EXE_SEGMASKER"
        MASKER_REQ[DUSTMASKER]="\$EXE_DUSTMASKER"

        SELECT_REQ[NONE]="$PLUGIN_NONE_REQ"
        SELECT_REQ[CDHIT]="\$EXE_CDHIT_PROT \$EXE_CDHIT_NUCL"

        CLUSTER_REQ[BRHCLUS]="\$EXE_BRHCLUS"

        # set project directories
        refresh_dir_project

        # check data type - don't try to guess it
        # fails if DATA_TYPE is not empty and not in ('PROT' or 'NUCL', first four letters)
        ####check_data_type || fatal_msg "failed check_data_type DATA_TYPE=$DATA_TYPE"

        # set e-value cutoffs
        # if MAX_EVAL is not set, it will be set later in set_datatype_defs
        ####debug_msg "set e-value cutoffs"
        ####set_datatype_defs
        
        # set DB extensions (formatdb/makeblastdb)
        # allows for empty DATA_TYPE
        debug_msg "set fasta DB extensions (formatdb/makeblastdb)"
        set_db_ext

        # set run mode based on ODB_RUN_MODE
        #debug_msg "set run mode (ODB_RUN_MODE=$ODB_RUN_MODE)"
        #set_run_mode
        
        # set all executables
        debug_msg "set executables"
        set_executables || fatal_msg "failed set_executables"

        # set all splitfile
        #debug_msg "set splitfile"
        #set_split_in_file
        
        # initialise postproc scripts
        debug_msg "init postproc scripts"
        init_postproc_scripts || fatal_msg "failed init_postproc_scripts"

        # fix OP_ variables
        debug_msg "proj OP fixes"
        proj_op_fix

        # set API url
        debug_msg "set orthodb API url/versions"
        set_orthodb_api || error_msg "failed setting OrthoDB API"
        
        # done!
        debug_msg "DONE proj_configure_env"
        return 0
    }

    #--------------------------------------------------
    # finalise configuration and do checks
    #--------------------------------------------------

    #finalise_configure() {
    #    local args=${@}
    #
    #    #ODB set DATA_TYPE and dependent variables
    #    set_data_type
    #    set_db_ext
    #    set_datatype_defs
    #
    #    #ODB find all executables
    #    set_executables
    #
    #    #GEN configure job scheduler
    #    configure_jobsched || return 255
    #
    #    # verify setup
    #    verify_setup || return 255
    #
    #    return 0
    #}


    #--------------------------------------------------
    # Verify that the setup is OK.
    #--------------------------------------------------
#    verify_datatype() {
#        [[ "$DATA_TYPE" == "PROT" ]] && return 0
#        [[ "$DATA_TYPE" == "NUCL" ]] && return 0
#        error_msg "Invalid DATA_TYPE = $DATA_TYPE"
#        return 1
#    }

    verify_parameters() {
        local rval=0
        inrange_int   $MIN_OVERLAP 10 x                      || { rval=1; error_msg "MIN_OVERLAP is low (<10)"; }
        inrange_float $MIN_COVERAGE 0.0 1.0                  || { rval=1; error_msg "MIN_COVERAGE is greater than 1.0 or negative"; }
        inrange_int   $SELECT_PID 90 100                     || { rval=1; error_msg "SELECT_PID is outside range [90,100]"; }

        inrange_float $MAX_EVALUE  0.0 x                     || { rval=1; error_msg "MAX_EVALUE is negative"; }

        inrange_float $BRHCLUS_PAIREVAL_SCALE  0.0 1.0       || { rval=1; error_msg "BRHCLUS_PAIREVAL_SCALE is greater than 1.0 or negative"; }
        inrange_float $BRHCLUS_TRIEVAL         0.0 x         || { rval=1; error_msg "BRHCLUS_TRIEVAL is negative ($BRHCLUS_TRIEVAL) - check MAX_EVALUE"; }
        inrange_float $BRHCLUS_MAX_SPRATIO     0.0 1.0       || { rval=1; error_msg "BRHCLUS_MAX_SPRATIO is outside range [0,1] ($BRHCLUS_MAX_SPRATIO)"; }

        
        inrange_int   $ALIGNMENT_MATRIX 0 1                    || { rval=1; error_msg "ALIGNMENT_MATRIX must 0 or 1"; }
        inrange_int   $ALIGNMENT_NUMALIGNMENTS 0 x             || { rval=1; error_msg "ALIGNMENT_NUMALIGNMENTS is negative"; }
        #inrange_int   $ALIGNMENT_EFFDBSZ -1 x                  || { rval=1; error_msg "ALIGNMENT_EFFDBSZ is <-1"; }
        
        return $rval
    }

    #--------------------------------------------------
    # verify the setup
    #--------------------------------------------------
    proj_verify_setup() {
        local rval=0
        local ec=0
        debug_msg "verify common.sh setup"

        # verify data type
        ####verify_datatype || rval=1

        # parameters
        verify_parameters || rval=1

        return $rval
    }

    #--------------------------------------------------
    # get PWC directory
    #--------------------------------------------------
    R_PWC_DIR=""
    get_pwc_dir() {
        [[ $# -eq 1 ]] || { echo $DIR_PROJECT_PWC; return 0; }
        local label=$1
        R_PWC_DIR="${DIR_PROJECT_PWC}/$label"
    }
    v_get_pwc_dir() {
        [[ $# -ge 1 ]] || return 255
        local var="$1"
        local label=""
        [[ $# -eq 2 ]] && label="$2"
        local "$var" && set_var "$var" "${DIR_PROJECT_PWC}/$label"
    }

    #--------------------------------------------------
    # get PWC file name
    #--------------------------------------------------
    get_pwc_file() {
        [[ $# -eq 3 ]] || return
        local A="$1"
        local B="$2"
        local ext="$3"
        get_pwc_dir "$A"
        echo "$R_PWC_DIR/${A}_${B}.${ext}"
    }
    v_get_pwc_file() {
        [[ $# -eq 4 ]] || return
        local var="$1"
        local A="$2"
        local B="$3"
        local ext="$4"
        local v=""
        v_get_pwc_dir v $A
        local "$var" && set_var "$var" "$v/${A}_${B}.${ext}"
    }
    #--------------------------------------------------
    # get and check PWC file
    #--------------------------------------------------
    get_pwc_file_chk() {
        [[ $# -eq 3 ]] || return 255
        local A="$1"
        local B="$2"
        local ext="$3"
        
        local f=""
        f="$(get_pwc_file $A $B $ext)"
        [[ -f "$f" ]] || f=""
        echo "$f"
    }
    v_get_pwc_file_chk() {
        [[ $# -eq 4 ]] || return 255
        local var="$1"
        local A="$2"
        local B="$3"
        local ext="$4"
        local f=""
        v_get_pwc_file f $A $B $ext
        [[ -f "$f" ]] || f=""
        local "$var" && set_var "$var" "$f"
    }

    #--------------------------------------------------
    # get and check PWC file
    # will check for both A,B and B,A
    # and will return the first one found
    #--------------------------------------------------
    get_pwc_bifile_chk() {
        [[ $# -eq 3 ]] || return
        local A="$1"
        local B="$2"
        local ext="$3"
        
        local fp=""
        fp="$(get_pwc_file_chk $A $B $ext)"
        [[ ${#fp} -eq 0 ]] && fp="$(get_pwc_file_chk $B $A $ext)"

        echo $fp
    }
    v_get_pwc_bifile_chk() {
        [[ $# -eq 4 ]] || return 255
        local var="$1"
        local A="$2"
        local B="$3"
        local ext="$4"

        local fp=""
        v_get_pwc_file_chk fb $A $B $ext
        [[ ${#fp} -eq 0 ]] && v_get_pwc_file_chk fp $B $A $ext
        local "$var" && set_var "$var" "$fp"
    }

    #--------------------------------------------------
    # create PWC directory tree
    #--------------------------------------------------
    create_pwc_dirs() {
        [[ -d "$DIR_PROJECT_PWC" ]] || { _N_CREATED_DIRS=$((_N_CREATED_DIRS+1)); create_dir "$DIR_PROJECT_PWC" || return 255; }
        [[ -f "$PL_TODO" ]] || return 0
        
        local t=""
        #for t in $PL_TODO_LBLSTR; do
        for t in ${PL_TODO_LABELS[@]}; do
            [[ -d "$DIR_PROJECT_PWC/$t" ]] || { _N_CREATED_DIRS=$((_N_CREATED_DIRS+1)); create_dir "$DIR_PROJECT_PWC/$t" || return 255; }
        done
        return 0
    }


    #--------------------------------------------------
    # check PWC directory tree
    #--------------------------------------------------
    check_pwc_dirs() {
        [[ -d "$DIR_PROJECT_PWC" ]] || { error_msg "DIR_PROJECT_PWC is not readable"; return 255; }
        [[ -f "$PL_TODO" ]] || return 0
        if (( PL_TODO_NLABELS < 10 )); then
            debug_msg "checking DIR_PROJECT_PWC/xxx"
            local t=""
            #for t in $PL_TODO_LBLSTR; do
            for t in ${PL_TODO_LABELS[@]}; do
                [[ -d "$DIR_PROJECT_PWC/$t" ]] || { error_msg "DIR_PROJECT_PWC/$t is not readable"; return 255; }
            done
        fi
        return 0
    }

    #--------------------------------------------------
    # get PWC dir but from a different root
    #--------------------------------------------------
    get_external_pwc_dir() {
        [[ $# -lt 2 ]] && return 255
        local label=$1
        local srcdir="$2"
        local f=""
        [[ $# -eq 3 ]] && f="$3"        
        local pdir="$srcdir/PWC"
        if [[ -d "$pdir/$label" ]]; then
            # new PWC dir tree
            echo "$pdir/$label/$f"
            return 2
        elif [[ -d "$pdir" ]]; then
            # old PWC dir tree
            echo "$pdir/$f"
            return 1
        fi
        return 254
    }
    #--------------------------------------------------
    # get a PWC file from a different root
    #--------------------------------------------------
    get_external_pwc_file() {
        [[ $# -eq 4 ]] || return 255
        local A="$1"
        local B="$2"
        local ext="$3"
        local srcdir="$4"
        get_external_pwc_dir "$A" "$srcdir" "${A}_${B}.${ext}"
    }
    #--------------------------------------------------
    # get and check PWC file from a different root
    #--------------------------------------------------
    get_external_pwc_file_chk() {
        [[ $# -eq 4 ]] || return 255
        local A="$1"
        local B="$2"
        local ext="$3"
        local srcdir="$4"
        local f=""
        f="$(get_external_pwc_file $A $B $ext $srcdir)"
        [[ -f "$f" ]] || f=""
        echo "$f"
    }

    #--------------------------------------------------
    # link result files to DIR_PROJECT_RESULTS
    # <ext> <file1> <file2> ...
    #
    # new file1 in Results:
    #   <label1><ext>
    #--------------------------------------------------
    link_results() {
        [[ $# -ge 2 ]] || return 0
        [[ -d $DIR_PROJECT_RESULTS ]] || { error_msg "missing DIR_PROJECT_RESULTS at $DIR_PROJECT_RESULTS"; return 255; }
        local ext=$1
        [[ "$ext" == "$NONE_VALUE" ]] && ext=""
        shift
        local oldpwd="$(pwd)"
        
        cd $DIR_PROJECT_RESULTS
        local f=""
        local t=""
        local s=""
        local b=""
        for f in "$@"; do
            b="$(basename $f)"
            s="../Cluster/$b"
            if [[ -n $ext ]]; then
                t="$(file_basename $b)$ext"
            else
                t="$b"
            fi
            [[ -f $s ]] || { warn_msg "skipping missing file $s"; continue; }
            ln -sf "$s" "$t"
        done
        cd "$oldpwd"
        return 0
    }
    
    #--------------------------------------------------
    # create project directories
    #--------------------------------------------------

    proj_create_dirs() {
        # create PWC dir structure - one per species
        create_pwc_dirs
    }

    proj_check_dirs() {
        # check PWC dir structure - one per species
        check_pwc_dirs
    }


    #===============================================================
    # ------ PIPELINE.SH START
    #===============================================================

    # todo file as given by orthologer_conf.sh
    COMMON_TODO_NAME=
    COMMON_TODO=
    MAX_MESSAGES=10

    #-----------------------------------------------
    # usage message
    # see proj_args()
    #-----------------------------------------------
    proj_usage() {
        echo
        help_header_msg "OrthoDB Specific Options"
        echo
        help_opt_msg "-I <file>"  "input cluster(s) as a CSV list of labels (or files)"
        help_opt_msg "-G"         "print tree info"
        help_opt_msg "-D"         "force TREE_ENABLED==0, run according to given PL_TODO"
        help_opt_msg "-M"         "align on all fasta (from SELECTED) merged into one"
    }

    #-----------------------------------------------
    # project specific options
    #
    # PROJ_OPTS_HOOK[<opt>]=<function>
    # PROJ_OPTS_ARGHELP[<opt>]=<help string for option>
    #
    # Note that if ARGHELP is non empty, the hook
    # function expects an argument.
    #
    # For each option, both has to be defined.
    #-----------------------------------------------
    proj_args() {

        PROJ_OPTS_ARGHELP[I]="<input cluster/CSV>"
        PROJ_OPTS_HOOK[I]=set_input_cluster

        PROJ_OPTS_ARGHELP[G]=""
        PROJ_OPTS_HOOK[G]=print_tree_info

        PROJ_OPTS_ARGHELP[D]=""
        PROJ_OPTS_HOOK[D]=disable_tree

        PROJ_OPTS_ARGHELP[m]=""
        PROJ_OPTS_HOOK[m]=set_align_merged
        return 0
    }

    #-----------------------------------------------
    # variable help
    #-----------------------------------------------
    proj_variables_help() {
        local cnt=$1
        help_header_msg "${cnt}. Run mode" && cnt=$((cnt+1))
        variable_def "ODB_RUN_MODE" "set preset parameters for different modes"
        variable_def ""             "DEFAULT     more specific, faster"
        variable_def ""             "XFAST       least sensitive, fastest"
        variable_def ""             "SENSITIVE   most sensitive"
        variable_def ""             "USER        use relevant parameters set in common.sh"
        variable_def ""             "Variables controlled by the run mode:"
        variable_def "" "* MIN_OVERLAP          - sequence overlap/min length ($MIN_OVERLAP)"
        variable_def "" "* MIN_COVERAGE         - sequence match coverage ($MIN_COVERAGE)"
        variable_def "" "* MAX_EVALUE           - max e-value cutoff in alignment ($MAX_EVALUE)"
        variable_def "" "* SELECT_PID           - min percentage id for match in SELECT ($SELECT_PID)"
        variable_def "" "* MMSEQS_SENSITIVITY   - alignment (mmseqs): sensitivity parameter ($MMSEQS_SENSITIVITY)"
        variable_def "" "* MERGEINPAR_FCLUSTMIN - mergeinpar: cluster merge param 1 ($MERGEINPAR_FCLUSTMIN)"
        variable_def "" "* MERGEINPAR_FSPECMIN  - mergeinpar: cluster merge param 2 ($MERGEINPAR_FSPECMIN)"
        variable_def "" "* MAKEBRH_NALT         - makebrh   : fuzzy BRHs, the nr of near BRHs including the main one ($MAKEBRH_NALT)"
        variable_def "" "* MAKEBRH_FSEPMAX      - makebrh   : fuzzy BRHs, max separation wrt the best BRH ($MAKEBRH_FSEPMAX)"
        
        #help_header_msg "${cnt}. Data" && cnt=$((cnt+1))
        #variable_def "DATA_TYPE" "Input data type - either PROT(eins) or NUCL(eotides)."

        help_header_msg "${cnt}. Pairwise runs" && cnt=$((cnt+1))
        variable_def "PAIR_FILE"       "A file with explicit pairs to run over. Overrides pairwise files created by PL_TODO/1/2."
      
        help_header_msg "${cnt}. Tree" && cnt=$((cnt+1))
        variable_def "TREE_ENABLED"      "flag enabling tree based clustering (0 -> off, 1-> on)"
        variable_def ""                  "If enabled, the tree is obtained in one of two ways."
        variable_def ""                  "1. User provided tree file in newick format."
        variable_def ""                  "   The TREE_ROOT_CLADE is defined as the root of the newick tree."
        variable_def ""                  "2. Tree from OrthoDB and root defined by TREE_ROOT_CLADE."
        variable_def ""                  "   The user sets TREE_ROOT_CLADE which must be a numerical taxid."
        variable_def ""                  "If TREE_INPUT_FILE is not set, it will search for a .$EXT_TREE file matching the PL_TODO."
        variable_def ""                  "If still no file, option 2 is considered."
        variable_def ""                  "All clusters are created starting from the leaf clades."
        variable_def ""                  "The step CLUSTER will actually also run MERGEINPAR and MERGEINPARSEL."
        variable_def ""                  "Note that this is only supported using a newer version of brhclus."
        variable_def "TREE_INPUT_FILE"   "File with tree in newick format."
        variable_def "TREE_ROOT_CLADE"   "Root clade either derived from the given tree file OR set by user (required if tree is derived from OrthoDB)."

        
        help_header_msg "${cnt}. Flags" && cnt=$((cnt+1))
        variable_def "IGNORE_MISSING" "Ignore missing source files"
        
        help_header_msg "${cnt}. Tool binary directories" && cnt=$((cnt+1))
        variable_def "DIR_BRHCLUS"   "BRHCLUS package root directory"
        variable_def "DIR_BLAST"     "NCBI blast+"
        variable_def "DIR_CDHIT"     "cd-hit package"
        variable_def "DIR_SWIPE"     "swipe aligner"
        variable_def "DIR_MMSEQS"    "mmseqs aligner"

        help_header_msg "${cnt}. Plugin labels" && cnt=$((cnt+1))
        echo "NOTE: each label below defines a plugin for a given step id. It is expected that a given label <L>"
        echo "      corresponds to a script <step>_<L>.sh either in DIR_ORTHOLOGER/bin or local ./bin ."
        echo "      User defined scripts should be stored in the local project ./bin directory."
        echo
        variable_def "PREPROC_LABEL" "Pre-processing of the fasta files"
        variable_def ""              "CHECKFASTA - check and clean up the input fasta files"
        variable_def "ALIGNMENT_LABEL" "Which aligner to use (SWIPE,MMSEQS,BLAST,user)."
        variable_def "MASKER_LABEL"    "Low-complexity masker. Apart from below, NONE is also allowed."
        variable_def ""                "SEGMASKER  - for proteins, uses 'segmasker' from blast package"
        ###variable_def ""                "DUSTMASKER - for nucleotides, uses 'dustmasker' from blast package"
        variable_def "SELECT_LABEL"    "Method to use for selecting representatives of closely related genes within a species."
        variable_def ""                "CDHIT      - preferred method, uses cd-hit tool"
        variable_def ""                "NONE       - dont select any representatives"
        variable_def "CLUSTER_LABEL"   "Clustering method. Supported is 'BRHCLUS'."
        variable_def "POSTPROC_LABELS"  "CSV list of labels or full script names"

        help_header_msg "${cnt}. Masker step options" && cnt=$((cnt+1))
        #variable_def "OPTS_DUSTMASKER"   
        variable_def "OPTS_SEGMASKER"    "Options added to the segmasker call."

        help_header_msg "${cnt}. Alignment step options" && cnt=$((cnt+1))
        variable_def "ALIGNMENT_MATRIX"  "Calculate full or half alignment matrix."
        variable_def ""                  "0 - calculate full matrix"
        variable_def ""                  "1 - calculate half matrix (M11,M12,...M1n,M22,...,Mnn)"
        variable_def "ALIGNMENT_NUMALIGNMENTS" "Max nr of alignments per query given by the aligner (typically 100)."
        #variable_def "ALIGNMENT_EFFDBSZ"  "The effective database size. This defines a fixed DB size for each alignment."
        #variable_def ""                   "Any change here will affect the e-values calculated. A larger size will shift"
        #variable_def ""                   "the e-values to higher values."
        variable_def "OPTS_SWIPE"         "Extra options to swipe."
        variable_def "OPTS_MMSEQS"        "Extra options to mmseqs."
        variable_def "OPTS_BLAST"         "Extra options to blast."

        help_header_msg "${cnt}. Homology" && cnt=$((cnt+1))
        variable_def "MIN_OVERLAP"  "Minimum overlap (and sequence) length considered."
        variable_def "MIN_COVERAGE" "Minimum sequence coverage allowed for a match."
        variable_def "MAX_EVALUE"   "Maximum e-value allowed."
        variable_def "SELECT_PID"   "Percentage identity lower bound used in the SELECT step. Should be something close to 100% (default 97)."

        help_header_msg "${cnt}. Make BRH" && cnt=$((cnt+1))
        variable_def "MAKEBRH_NALT"       "if >1 then generate fuzzy BRHs"
        variable_def "MAKEBRH_FSEPMAX"    "if NALT>1 then this is the max separation between the candidates and the best BRH, a nr in [0..1], should be small"

        help_header_msg "${cnt}. Clustering" && cnt=$((cnt+1))
        variable_def "CLUSTER_INPOG"      "seed cluster(s), either a cluster file or a CSV list of labels for cluster"
        variable_def "CLUSTER_INPOG_WEAKBRH"       "if '1' then each CLUSTER_INPOG has a weak BRH file"
        variable_def "OPTS_BRHCLUS"       "extra options for brhclus"
        
        variable_def "BRHCLUS_TRIEVAL"           "max evalue in triangles"
        variable_def "BRHCLUS_PAIREVAL_SCALE"    "scale factor (<1.0) relative TRIEVAL for max evalue in pairs/chains"
        variable_def "BRHCLUS_MAX_SPRATIO"    "maximum species ratio allowed in cluster merges"
        variable_def "BRHCLUS_MAX_SPRATIO"       "brhclus merge: max species ratio allowed; [0..1]"
        variable_def ""                          "0.0 -> no species overlap is allowed"
        variable_def ""                          "1.0 -> species overlap is ignored"
        variable_def "BRHCLUS_MERGE_FNLINKS"     "brhclus merge: nr of links required"
        variable_def ""                          "0 -> require just the seed BRH pair"
        variable_def ""                          "N -> require N links in both direction, hence 2N+1 links"
        
        help_header_msg "${cnt}. Inparalogs" && cnt=$((cnt+1))
        variable_def "OPTS_MERGEINPAR"           "extra options for mergeinpar command"
        variable_def "MERGEINPAR_FHITSMAX"       "mergeinpar inparalog resolution - second best vs best candidate [0..1]"
        variable_def ""                          "a high value will resolve more inparalogs"
        variable_def "MERGEINPAR_FCLUSTMIN"      "mergeinpar cluster merge: min fraction of linked vs nr of members in smallest cluster"
        variable_def ""                          "a small value will merge more"
        variable_def "MERGEINPAR_FSPECMIN"       "mergeinpar cluster merge: minimum species overlap - exclude merges with to small species overlap"
        
        help_header_msg "${cnt}. Post-processing" && cnt=$((cnt+1))
        variable_def "POSTPROC_LABELS" "This may be a label or directly a script name."
        variable_def "" "If a label, it expects to find a script postproc_<label>.sh in DIR_ORTHOLOGER_PATH"

        help_header_msg "${cnt}. Per-step variables" && cnt=$((cnt+1))
        ###variable_def "OP_STEP_COMPRESS[step]"  "compression flag; 1->enable, 0->disable"
        variable_def "OP_STEP_NPARALLEL[step]" "nr of jobs that should be run in parallel"
        variable_def "OP_STEP_NTHREADS[step]"  "nr of CPUs to be allocated for each job"
        variable_def "OP_STEP_NMERGE[step]"    "nr of jobs merged into one"
        variable_def "OP_STEP_ENABLE[step]"    "enable flag; 1->enable, 0->disable"
        variable_def "OP_STEP_RUNMODE[step]"   "run mode - either BATCH (run with scheduler) or LOCAL (run on local machine)"
        variable_def "OP_STEP_SCHEDOPTS[step]" "option string for job scheduler"

        help_header_msg "${cnt}. Subset selection of pairwise matrix (alignment, brh, inpar)" && cnt=$((cnt+1))
        variable_def "OP_LABELA_FIRST" "Start label for A in pairwise steps [A,B] "
        variable_def "OP_LABELA_LAST"   "Idem end label"
        variable_def "OP_LABELB_FIRST" "Start label for B in pairwise steps [A,B] "
        variable_def "OP_LABELB_LAST"   "Idem end label"
    }
    
    #-----------------------------------------------
    # extended help
    #-----------------------------------------------
    proj_extended_help() {
        echo
        #    echo -e "${ALWAYS_BOLD}III. OrthoDB Specific Options${ALWAYS_NORMAL}"
    }

    #-----------------------------------------------
    # usage message
    #-----------------------------------------------
    help_param() {
        local what="$1"
        local desc=""
        [[ $# -eq 1 ]] || desc="$2"
        if [[ ${#desc} -eq 0 ]]; then
            printf '   %-30s - %s\n' "" "$what"
        else
            printf ' * %-30s - %s\n' "$what" "$desc"
        fi
    }

    help_step_preparetree() {
        echo -e "${ALWAYS_BOLD}PREPARETREE${ALWAYS_NORMAL}"
        echo "Prepare for tree mode processing"

        echo -e "$str_help_script"
        echo " <none>"

        echo -e "$str_help_input"
        help_param "todo/<todo>.$EXT_TREE " "tree file (TREE_INPUT_FILE)"

        echo -e "$str_help_output"
        help_param "todo/<nodes>.$EXT_TODO " "todo files for nodes in tree"
        help_param "todo/<nodes>.$EXT_PAIRS_TREE " "pair files for nodes in tree"

        echo -e "$str_help_fmt"
        echo " tree in newick format"

        echo -e "$str_help_desc\n" \
             "It takes the input newick tree and creates\n" \
             "* todo files for each node\n" \
             "* for each internal node, a pair file which contains all species pairs to be considered"
        
    }
    help_step_preproc() {
        echo -e "${ALWAYS_BOLD}PREPROC${ALWAYS_NORMAL}"
        echo "Preprocessing of given input fasta files"

        echo -e "$str_help_input"
        help_param 'Sequences/*'".$EXT_FASTA"  "fasta files"

        echo -e "$str_help_output"
        help_param "Sequences/*.$EXT_FASTA_CLEAN"  "fasta files"

        echo -e "$str_help_fmt"
        echo " regular fasta format"

        echo -e \
             "$str_help_desc\n" \
             "This step performs a preprocessing on all input files.\n"\
             "It will run the script bin/preproc_<PREPROC_LABEL>.\n"\
             "Currently the following PREPROC_LABEL are defined:\n"\
             " 1. CHECKFASTA\n"\
             "    Checks each fasta file for\n" \
             "    * proper fasta type" "protein or nucleotide\n" \
             "    * bad sequences and/or labels\n" \
             "\n" \
             "    The files are also filtered for\n"\
             "    * stop codon characters - replaced with masking character\n" \
             "    * masked regions -> masked using a single character\n" \
             "    * sequence lengths less than MIN_OVERLAP are excluded"
    }
    help_step_masker() {
        echo -e "${ALWAYS_BOLD}MASKER${ALWAYS_NORMAL}"
        echo -e "$str_help_script"
        echo " masker_<MASKER_LABEL>.sh"
        
        echo -e "$str_help_input"
        help_param "Sequences/*.$EXT_FASTA_CLEAN " "cleaned fasta files"
        
        echo -e "$str_help_output"
        help_param "Sequences/*.$EXT_FASTA_MASKED " "masked fasta files"

        echo -e "$str_help_vars"
        help_param "MASKER_LABEL" "defines what algo is used, is set depending on sequence type (PROT or NUCL), may be NONE"

        echo -e "$str_help_fmt"
        echo " regular fasta format"
                
        echo -e "$str_help_desc\n" \
             "This step masks low-complexity regions in fasta files."
    }
    help_step_select() {
        echo -e "${ALWAYS_BOLD}SELECT${ALWAYS_NORMAL}"
        echo "Select representatives"
        echo -e "$str_help_script"
        echo " select_<SELECT_LABEL>.sh"
        echo -e "$str_help_input"
        help_param "Sequences/*.$EXT_FASTA_MASKED " "masked fasta files"
        
        echo -e "$str_help_output"
        help_param "Sequences/*.$EXT_FASTA_SELECTED " "filtered fasta files plus similar gene clusters"
        
        echo -e "$str_help_vars"
        help_param "SELECT_LABEL" "defines what algo is used, currently only CDHIT and NONE are defined"
        help_param "SELECT_PID" "degree of similarity to be used, set in the range [90,100] where 100 is perfect identity"

        echo -e "$str_help_fmt"
        echo " input : regular fasta format"
        echo " output: .fs_selected       - regular fasta format"
        echo "         .fs_selected.clstr - cd-hit clusters"
        echo
        echo " CD-hit cluster example:"
        echo "  >Cluster 9"
        echo "  0       7604aa, >hobbit:000047... at 1:7604:137:7732/97.62%"
        echo "  1       7732aa, >hobbit:00008d... *"
        echo
        echo " Each cluster consists of one selected representative (marked with a '*')."
        echo " For the other genes, they have a string with with alignment range for representative and the similar gene."
        echo " The final percentage nr denotes the percentage identity."
        echo
        echo -e "$str_help_desc\n" \
             "This step searches for very similar genes in a fasta file.\n" \
             "For each cluster of similar genes, one is selected as a representative\n" \
             "while the rest is saved to later (included as \"inparalogs\" in MERGEINPARSEL)."
    }
    help_step_stats() {
        echo -e "${ALWAYS_BOLD}STATS${ALWAYS_NORMAL}"
        echo -e "$str_help_input"
        help_param "Sequences/*.$EXT_FASTA_SELECTED " "filtered fasta files"
        echo -e "$str_help_output"
        help_param "Sequences/*.$EXT_FASTA_STATS, $EXT_FASTA_SEQLEN" "fasta stats"
        echo -e "$str_help_fmt"
        echo " input : regular fasta file"
        echo " output: .fs_selected.stats  - global statistics on sequences in the file"
        echo "         .fs_selected.seqlen - sequence length and fraction non-masked per sequence"
        echo -e "$str_help_desc\n" \
             "Generate basic statistics for each fasta file.\n"\
             "Currently not used - it was there for the alignment step in order to rescale e-values."
    }
    
    help_step_makeinparsel() {
        echo -e "${ALWAYS_BOLD}MAKEINPARSEL${ALWAYS_NORMAL}"
        echo -e "$str_help_input"
        help_param "Sequences/*.$EXT_FASTA_SELCLSTR " "clusters from the SELECT step"
        echo -e "$str_help_output"
        help_param "Sequences/*.$EXT_INPAR_SELECTED " "\"inparalogs\" based on the clusters"
        echo -e "$str_help_fmt"
        echo " same as for MAKEINPAR with the exception that the second column is always '-'."
        echo -e "$str_help_desc\n" \
             "Extracts all sequences replaced by a representative in step SELECT.\n" \
             "They are later treated as inparalogs and merged into the clustering."
    }
    help_step_formatdb() {
        echo -e "${ALWAYS_BOLD}FORMATDB${ALWAYS_NORMAL}"
        echo -e "$str_help_script"
        echo " run_formatdb.sh - runs the appropriate program for the given aligner (ALIGNMENT_LABEL)"
        echo -e "$str_help_input"
        help_param "Sequences/*.$EXT_FASTA_SELECTED " "filtered fasta files"
        echo -e "$str_help_output"
        help_param "Sequences/*.${EXT_FASTA_SELECTED}.${EXT_DB}x " "blast or mmseqs DB files"
        echo -e "$str_help_fmt"
        echo " depends on DB formatter"
        echo -e "$str_help_desc\n" \
             "Creates databases of each selected fasta file for the alignment step."
    }

    help_step_alignment() {
        echo -e "${ALWAYS_BOLD}ALIGNMENT${ALWAYS_NORMAL}"
        echo -e "$str_help_script"
        echo " alignment_<ALIGNMENT_LABEL>.sh"
        echo -e "$str_help_input"
        help_param "Sequences/*.${EXT_FASTA_SELECTED}.${EXT_DB}x " "blast or mmseqs DB files"
        echo -e "$str_help_output"
        help_param "PWC/A/A_B.$EXT_ALIGN " "pairwise alignments, species A,B"
        echo -e "$str_help_vars"
        
        help_param "ALIGNMENT_LABEL        " "which aligner to use (default MMSEQS)"
        help_param "ALIGNMENT_MATRIX       " "flag doing half (1) or full (0) matrix"
        help_param "ALIGNMENT_NUMALIGNMENTS" "nr of alignments to show per sequence"
        help_param "MIN_OVERLAP           "  "minimum length allowed"
        help_param "OPTS_<aligner>       "   "user supplied additional options for the specific aligner"
        help_param "OP_LABELA_FIRST/END   "  "if set, first and last label to process for A"
        help_param "OP_LABELB_FIRST/END   "  "if set, first and last label to process for B"

        echo -e "$str_help_fmt"
        echo " <id A> <id B> <score> <e-value> <PID> <startA> <lastA> <startB> <lastB>"
        
        echo -e "$str_help_desc\n" \
             "Does the full alignment on all possible pairs (half or full matrix).\n" \
             "The output is reformatted to\n" \
             "  id1  id2  score  evalue  percentid  start1  end1  start2  end2"

    }
    help_step_makebrh() {
        echo -e "${ALWAYS_BOLD}MAKEBRH${ALWAYS_NORMAL}"
        echo -e "$str_help_input"
        help_param "PWC/A/A_B.$EXT_ALIGN " "pairwise alignments, species pair A,B"
        help_param "PWC/B/B_A.$EXT_ALIGN " "idem for pair B,A - only if ALIGNMENT_MATRIX=0 (full)"

        echo -e "$str_help_output"
        help_param "PWC/A/A_B.$EXT_BRH " "best reciprocal hits, species A,B"

        echo -e "$str_help_vars"
        help_param "MAX_EVALUE         " "max e-value allowed"
        help_param "MAKEBRH_NALT       " "generate alternative BRHs if NALT>1"
        help_param "MAKEBRH_FSEPMAX    " "how much an alternative BRH can differ in score"

        echo -e "$str_help_fmt"
        echo " <id A> <id B> <score> <e-value> <PID> <startA> <lastA> <startB> <lastB> [two columns related to alt BRHs]"
        
        echo -e "$str_help_desc\n" \
             "For each sequence pair [Ax,By], it searches for [By,Ax] in the alignment files.\n" \
             "If both are the best found, it is saved as a BRH.\n" \
             "With half aligner matrix (ALIGNMENT_MATRIX=1) the [B,A] is estimated.\n" \
             "Output format is same as for ALIGNMENT."
    }
    help_step_makeinpar() {
        echo -e "${ALWAYS_BOLD}MAKEINPAR${ALWAYS_NORMAL}"

        echo -e "$str_help_input"
        help_param "PWC/A/A_B.$EXT_BRH " "BRH file, species A,B"
        help_param "PWC/A/A_A.$EXT_ALIGN " "alignment file A,A"
        help_param "PWC/B/B_B.$EXT_ALIGN " "alignment file B,B"

        echo -e "$str_help_output"
        help_param "PWC/A/A_B.$EXT_INPAR " "inparalog files, species A,B"

        echo -e "$str_help_vars"
        help_param "MIN_OVERLAP          " "for sequence matching"
        help_param "MIN_COVERAGE         " "for sequence matching"
        
        echo -e "$str_help_fmt"
        echo " Output format (wrt a pair [A0,B0]):"
        echo "   A0 B0 A1;data A2;data ..."
        echo
        echo " All pairs [A0,A1], [A0,A2] etc have lower scores than the main pair [A0,B0]."
        echo " The same is true for [B0,B1]..."
        echo " (data) is a semicolon delimeted string containing:"
        echo "   <score>,<start0>,<end0>,<start1>,<end1>"
        echo
        echo " where score is relative the main gene."

        echo -e "$str_help_desc\n" \
             "For each BRH [Ax,By] it will scan for alignments [Ax,Az] and [By,Bw] that are\n" \
             "closer than the BRH. Each found Az and Bw will be saved as inparalogs wrt the\n" \
             "given BRH.\n"
        
    }
    help_step_setbrh() {
        echo -e "${ALWAYS_BOLD}SETBRH${ALWAYS_NORMAL}"

        echo -e "$str_help_input"
        help_param "PWC/*/*.$EXT_BRH " "all BRH files"

        echo -e "$str_help_output"
        help_param "Cluster/<todo>.$EXT_BRH " "all BRHs"

        echo -e "$str_help_fmt"
        echo " see MAKEBRH"
        
        echo -e "$str_help_desc\n" \
             "Collects all BRHs into one file for a given todo file.\n"\
             "This step is disabled in tree mode."
    }
    help_step_splitbrh() {
        echo -e "${ALWAYS_BOLD}SPLITBRH${ALWAYS_NORMAL}"
        echo "====> CURRENTLY DISABLED <===="

        echo -e "$str_help_input"
        help_param "Cluster/${PL_TODO_NAME}.$EXT_BRH " "full BRH file"

        echo -e "$str_help_output"
        help_param "Cluster/${PL_TODO_NAME}.$EXT_SPLITS_BRH " "full BRH file, now with split genes"
        help_param "Cluster/${PL_TODO_NAME}.$EXT_SPLITS " "all split genes with parameters"

        echo -e "$str_help_fmt"
        echo " -"
        
        echo -e "$str_help_desc\n" \
             "The brhclus tool \'splitgenes\' is run on the full BRH file.\n" \
             "Genes split will have _00 or _01 added to their ids."
    }

    help_step_setinpar() {
        echo -e "${ALWAYS_BOLD}SETINPAR${ALWAYS_NORMAL}"

        echo -e "$str_help_input"
        help_param "PWC/*/*.$EXT_INPAR " "all inparalog files"

        echo -e "$str_help_output"
        help_param "Cluster/<todo>.$EXT_INPAR " "all inparalogs"

        echo -e "$str_help_fmt"
        echo " see MAKEINPAR"

        echo -e "$str_help_desc\n" \
             "Collects all inparalogs into one file for a given todo file.\n"\
             "This step is disabled in tree mode."
    }

    help_step_setinparsel() {
        echo -e "${ALWAYS_BOLD}SETINPARSEL${ALWAYS_NORMAL}"

        echo -e "$str_help_input"
        help_param "PWC/*/*.$EXT_INPAR_SELECTED " "all \"inparalogs\" from SELECT step"

        echo -e "$str_help_output"
        help_param "Cluster/<todo>.$EXT_INPAR_SELECTED " "merge of input files"

        echo -e "$str_help_fmt"
        echo " see MAKEINPARSEL"
        
        echo -e "$str_help_desc\n" \
             "Merge all inparalogs deduced from SELECT step into one file for a given todo file."
    }
    
    help_step_cluster() {
        echo -e "${ALWAYS_BOLD}CLUSTER${ALWAYS_NORMAL}"

        echo -e "$str_help_script"
        echo " cluster_<CLUSTER_LABEL>.sh"

        echo -e "$str_help_input"
        help_param "Cluster/<todo>.$EXT_BRH " "all BRHs"

        echo -e "$str_help_output"
        help_param "Cluster/<todo>.$EXT_CLUSTER_RAW " "raw cluster file"

        echo -e "$str_help_vars"
        help_param "CLUSTER_LABEL         " "label defining the clustering algo - only BRHCLUS is defined"
        help_param "MIN_OVERLAP           " "minimum overlap allowed in the clustering"
        help_param "MIN_COVERAGE          " "minimum sequence coverage for a match"
        help_param "MAX_EVALUE            " "max e-value allowed in cluster algo"
        help_param "BRHCLUS_PAIREVAL_SCALE" "evalue scaling factor for pairs in brhclus algo (<=1)"
        help_param "BRHCLUS_MAX_SPRATIO   " "maximum species ratio allowed in cluster merges"
        help_param "BRHCLUS_MERGE_FNLINKS " "fraction defining the min nr of extra links required for cluster merges"
        help_param "CLUSTER_INPOG"          "mapping: input source cluster(s)"
        help_param "CLUSTER_INPOG_WEAKBRH"  "mapping: flag if weak BRH files exists"
        help_param "genes added will have 10 added to the regular types"
        help_param "this requires also the -R option to be set (defines species that should be merged into the source cluster)"
        help_param "this is ignored if tree mode is enabled"

        echo -e "$str_help_fmt"
        echo "  <cluster id> <gene id> <cluster type> <nr of edges> <align start> <align last> <PID> <score> <e-value>"
        echo
        echo " The metrics at the end are evaluated as a mean over all edges."
        echo " Depending on how the element entered the cluster it will get a different cluster type."
        echo " In particular 7 refers to inparalogs and 9 to genes filtered in the SELECT step (cd-hit)."
        echo " See the README.md file for BRHCLUS."
        
        echo -e "$str_help_desc"
        echo -e "In this step the collected BRHs are clustered using the cluster algorithm\n" \
             "defined by CLUSTER_LABEL.\n" \
             "If TREE_ENABLED=1 then it will go through the whole tree, running also the MERGEINPAR[SEL] step."


    }
    help_step_mergeinpar() {
        echo -e "${ALWAYS_BOLD}MERGEINPAR${ALWAYS_NORMAL}"

        echo -e "$str_help_input"
        help_param "Cluster/<todo>.$EXT_CLUSTER_RAW " "raw cluster file"
        help_param "Cluster/<todo>.$EXT_INPAR_SELECTED " "inparalog file"

        echo -e "$str_help_output"
        help_param "Cluster/<todo>.$EXT_CLUSTER_INPAR " "clustering with inparalogs"

        echo -e "$str_help_vars"
        help_param "MIN_OVERLAP        " "minimum overlap"
        help_param "MIN_COVERAGE       " "minimum sequence coverage for a match"
        help_param "MERGEINPAR_FHITSMAX" "inparalog link resolution"
        help_param "fraction of scores between 2nd best and best inparalog link"
        help_param "high value (<1.0) implies less resolution needed for resolving an inparalog"
        help_param "low value  (>0.0) implies more stringent separation between the two best candidates"
        ####help_param "MERGEINPAR_CLFUNC  " "cluster merge function - do not change (default 5)"
        help_param "MERGEINPAR_FCLUSTMIN" "cluster merge 1: min fraction of inparalog links vs source cluster size"
        help_param "small value will favor merging of clusters"
        help_param "MERGEINPAR_FSPECMIN" "cluster merge 2: min species overlap between cluster merge candidates (relative the smallest one in terms of species content)"
        help_param "small value will favor merging of clusters"
        help_param "Note that the two cutoffs above are both required to trigger a merge"
        echo -e "NOTE: the MERGEINPAR_X values are set to default values and should be changed with care."

        echo -e "$str_help_fmt"
        echo " see CLUSTER"
        
        echo -e "$str_help_desc\n" \
             "Merge inparalogs into the clustering. This step may change the nr of clusters as\n" \
             "it may cause clusters to merge. Sequences entering the clustering through this step\n" \
             "will have type 7 in the output file.\n"\
             "This step is disabled from explicit running in tree mode."
    }

    help_step_mergeinparsel() {
        echo -e "${ALWAYS_BOLD}MERGEINPARSEL${ALWAYS_NORMAL}"

        echo -e "$str_help_input"
        help_param "Cluster/<todo>.$EXT_CLUSTER_INPAR " "clustering with main inparalogs merged"
        help_param "Cluster/<todo>.$EXT_INPAR_SELECTED " "\"inparalogs\" from SELECT step"

        echo -e "$str_help_output"
        help_param "Cluster/<todo>.$EXT_CLUSTER " "final cluster file"

        echo -e "$str_help_fmt"
        echo " see CLUSTER"

        echo -e "$str_help_desc\n" \
             "Merge \"inparalogs\" from the SELECT step. Note that since these are not associated with\n" \
             "a BRH, they cannot merge clusters. They will be of type 9.\n"\
             "This step is disabled from explicit running in tree mode."
    }

    #help_step_weakbrh() {
    #    echo -e "${ALWAYS_BOLD}WEAKBRH${ALWAYS_NORMAL}"
    #    echo -e "$str_help_script"
    #    echo " run_weakbrh.sh"
    #    echo -e "$str_help_input"
    #    help_param "Cluster/<node>.og " "final cluster file"
    #    help_param "todo/<todo>.$EXT_TREE " "tree file (TREE_INPUT_FILE)"
    #    echo -e "$str_help_output"
    #    help_param "Cluster/<node>.weak_brh " "weak brh file"
    #    echo -e "$str_help_desc\n" \
    #         "Extract BRHs that contain weakly clustered genes.\n" \
    #         "A BRH is considered weak if at least one is clustered as type 1,2,3,4 or 5 (ie not triangle or inparalog).\n" \
    #         "They are then used as input for the raw clustering by a parent node in the tree.\n" \
    #         "This is only enabled in tree mode."
    #}

    #-----------------------------------------------
    #
    #-----------------------------------------------
    help_step_postproc() {
        local label=""
        [[ $# -eq 1 ]] && label="$1"
        [[ -z $label ]] || POSTPROC_LABELS=$label
        
        # set postproc script variables
        init_postproc_scripts
        
        echo -e "${ALWAYS_BOLD}POSTPROC${ALWAYS_NORMAL}\n"
        echo -e "possible labels: $OP_POSTPROC_INTERN"
        echo
        [[ ${#OP_POSTPROC_SCRIPTS_FOUND} -gt 0 ]] || {
            echo -e "${ALWAYS_BOLD}---> DISABLED${ALWAYS_NORMAL}"
            echo -e "$str_help_script"
            echo " postproc_<POSTPROC_LABELS>.sh"

            echo -e "$str_help_desc\n" \
                 "Post processing defined by POSTPROC_LABELS.\n" \
                 "The scripts(s) is given as a comma separated list.\n" \
                 "Valid input are any of the predefined:\n\n" \
                 "  $OP_POSTPROC_INTERN\n\n" \
                 "or a filename postproc_<label>.sh .\n" \
                 "Templates are found in sbin/postproc_template*.sh .\n" \
                 "\n"
            return 0
        }
        local scripts=()
        eval $(create_array scripts ':' "$OP_POSTPROC_SCRIPTS_FOUND")
        local ind=
        for (( ind=0; ind<${#scripts[@]}; ind++ )); do
            echo "-----------------------------------------"
            local f="${scripts[$ind]}"
            if set_postproc "$f"; then
                echo -e "${ALWAYS_BOLD}NAME   :${ALWAYS_NORMAL} $POSTPROC_NAME"
                echo -e "${ALWAYS_BOLD}Script :${ALWAYS_NORMAL} $f"
                postproc_help
            else
                echo -e "\n${ALWAYS_BOLD}---> NAME: $f${ALWAYS_NORMAL}"
                echo -e "" \
                     "$str_help_input\n" \
                     " * user defined\n" \
                     "$str_help_output\n" \
                     " * user defined\n" \
                     "$str_help_desc\n" \
                     "Runs script in POSTPROC_LABELS.\n"
            fi
        done

    }
    #-----------------------------------------------
    # cleanup_step_XXX() functions
    #-----------------------------------------------
    cleanup_done_file() {
        [[ $# -eq 1 ]] || return 255
        local step="$1"
        local donefile=""
        donefile=$(get_done_file "$step")
        [[ -f $donefile ]] && rmcmd "$donefile"
        return 0
    }
    cleanup_step_preproc() {
        RMCMD_REMOVED=0
        cleanup_done_file $STEP_ID
        cleanup_step_per_todo_label  "$DIR_PROJECT_SEQUENCES" "$EXT_FASTA_CLEAN"
        step_info_msg "removed $RMCMD_REMOVED files"
    }
    cleanup_step_masker() {
        RMCMD_REMOVED=0
        cleanup_done_file $STEP_ID
        cleanup_step_per_todo_label  "$DIR_PROJECT_SEQUENCES" "$EXT_FASTA_MASKED"
        step_info_msg "removed $RMCMD_REMOVED files"
    }
    cleanup_step_select() {
        RMCMD_REMOVED=0
        cleanup_done_file $STEP_ID
        cleanup_step_per_todo_label "$DIR_PROJECT_SEQUENCES" "$EXT_FASTA_SELECTED" "${EXT_FASTA_SELECTED}.clstr"
        step_info_msg "removed $RMCMD_REMOVED files"
    }
    cleanup_step_stats() {
        RMCMD_REMOVED=0
        cleanup_done_file $STEP_ID
        cleanup_step_per_todo_label "$DIR_PROJECT_SEQUENCES" "$EXT_FASTA_STATS" "$EXT_FASTA_SEQLEN"
        
        local sumfile="$DIR_PROJECT_SEQUENCES/${PL_TODO_NAME}_fasta_totlen.dat"
        rmcmd "$sumfile"
        step_info_msg "removed $RMCMD_REMOVED files"
    }
    cleanup_step_makeinparsel() {
        RMCMD_REMOVED=0
        cleanup_done_file $STEP_ID
        cleanup_step_per_todo_label "$DIR_PROJECT_SEQUENCES" "$EXT_INPAR_SELECTED"
        step_info_msg "removed $RMCMD_REMOVED files"
    }
    cleanup_step_formatdb() {
        RMCMD_REMOVED=0
        cleanup_done_file $STEP_ID
        local efmt=""
        efmt="$(get_formatdb_exts $EXT_FASTA_SELECTED)"
        cleanup_step_per_todo_label "$DIR_PROJECT_SEQUENCES" "${EXT_FASTA_SELECTED}_formatdb.log" $efmt
        step_info_msg "removed $RMCMD_REMOVED files"
    }

    cleanup_step_pairwise() {
        RMCMD_REMOVED=0
        cleanup_done_file $STEP_ID

        # initialise pairwise run
        PW_SCRIPT=$(get_run_script $stepid)
        if is_dryrun; then
            PW_NPAIRS=0
        else
            PW_NPAIRS=$(grep -c -v "^#" $GENERATED_PAIR_FILE)
        fi
        
        # check if no pairs - return
	    [[ $PW_NPAIRS -gt 0 ]] || return 0
        
        # get extension
        local extwc=""
        
        if [[ $STEP_ID == ALIGNMENT ]]; then
            extwc="$EXT_ALIGN"
        elif [[ $STEP_ID == MAKEBRH ]]; then
            extwc="$EXT_BRH"
        elif [[ $STEP_ID == MAKEINPAR ]]; then
            extwc="$EXT_INPAR"
        fi

        [[ ${#extwc} -gt 0 ]] || { error_msg "cleanup_step_pairwise() illegal stepid <$STEP_ID>"; return 255; }

        # counters
        local indA=0
        local indB=0
        local A=""
        local B=""
        local pwcdir=""
        local joblogdir=""
        local f=""
        # loop over first label A
        for (( indA=PW_RANGEA[0]; indA<PW_RANGEA[1]+1; indA++ )); do
            A=${PL_TODO_LABELS[$indA]}
            
            # get PWC/JobLog dirs
            get_pwc_dir "$A"
            get_joblog_dir "$A"

            # update B range
            pw_update_range_B $indA
            
            # loop over second label B
            for (( indB=PW_RANGEB[0]; indB<PW_RANGEB[1]+1; indB++ )); do
                
                # get label B
                B=${PL_TODO_LABELS[$indB]}
                
                has_required_label $A $B || continue

                sj_joblog_base_nopid "$PL_TODO_NAME" "$STEP_ID" "${A}-${B}"

                # find files - ignores compression
                f="$R_PWC_DIR/${A}_${B}.$extwc"
                [[ -f "$f"      ]] && rmcmd "$f"
                [[ -f "${f}.gz" ]] && rmcmd "${f}.gz"
                for f in $(find $R_JOBLOG_DIR/ -name "${SJ_R_JOBLOG_NAME}*.log"); do
                    rmcmd "$f"
                done
            done
        done
        step_info_msg "removed $RMCMD_REMOVED files"
    }
    
    cleanup_step_alignment() {
        RMCMD_REMOVED=0
        cleanup_step_pairwise
    }
    cleanup_step_makebrh() {
        RMCMD_REMOVED=0
        cleanup_step_pairwise
    }
    cleanup_step_makeinpar() {
        RMCMD_REMOVED=0
        cleanup_step_pairwise
    }
    cleanup_step_setbrh() {
        RMCMD_REMOVED=0
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.$EXT_BRH.gz"
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.$EXT_BRH"
        rmcmd "$DIR_PROJECT/$BASE_JOBLOG/${PL_TODO_NAME}_${STEP_ID}_*.log"
        step_info_msg "removed $RMCMD_REMOVED files"
    }
    cleanup_step_splitbrh() {
        RMCMD_REMOVED=0
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.${EXT_SPLITS}.gz"
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.${EXT_SPLITS}"
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.${EXT_SPLITS_BRH}"
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.${EXT_SPLITS_BRH}.gz"
        rmcmd "$DIR_PROJECT/$BASE_JOBLOG/${PL_TODO_NAME}_${STEP_ID}_*.log"
        step_info_msg "removed $RMCMD_REMOVED files"
    }
    cleanup_step_setinpar() {
        RMCMD_REMOVED=0
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.$EXT_INPAR.gz"
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.$EXT_INPAR"
        rmcmd "$DIR_PROJECT/$BASE_JOBLOG/${PL_TODO_NAME}_${STEP_ID}_*.log"
        step_info_msg "removed $RMCMD_REMOVED files"
    }
    cleanup_step_setinparsel() {
        RMCMD_REMOVED=0
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.$EXT_INPAR_SELECTED.gz"
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.$EXT_INPAR_SELECTED"
        rmcmd "$DIR_PROJECT/$BASE_JOBLOG/${PL_TODO_NAME}_${STEP_ID}_*.log"
        step_info_msg "removed $RMCMD_REMOVED files"
    }
    cleanup_step_preparetree() {
        # dont remove anything here
        RMCMD_REMOVED=0
        step_info_msg "removed $RMCMD_REMOVED files"
    }    
    cleanup_step_weakbrh() {
        # dont remove anything here
        RMCMD_REMOVED=0
        step_info_msg "removed $RMCMD_REMOVED files"
    }    
    cleanup_step_cluster() {
        RMCMD_REMOVED=0
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.${EXT_CLUSTER_RAW}.gz"
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.${EXT_CLUSTER_RAW}"
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.${EXT_CLUSTER_RAW}.log"
        rmcmd "$DIR_PROJECT/$BASE_JOBLOG/${PL_TODO_NAME}_${STEP_ID}_*.log"
        step_info_msg "removed $RMCMD_REMOVED files"
    }
    cleanup_step_mergeinpar() {
        RMCMD_REMOVED=0
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.${EXT_CLUSTER_INPAR}.gz"
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.${EXT_CLUSTER_INPAR}"
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.${EXT_INPAR}.unresolved.gz"
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.${EXT_INPAR}.unresolved"
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.${EXT_CLUSTER_INPAR}.log"
        rmcmd "$DIR_PROJECT/$BASE_JOBLOG/${PL_TODO_NAME}_${STEP_ID}_*.log"
        step_info_msg "removed $RMCMD_REMOVED files"
    }
    cleanup_step_mergeinparsel() {
        RMCMD_REMOVED=0
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.${EXT_CLUSTER}.gz"
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.${EXT_CLUSTER}"

        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.${EXT_INPAR_SELECTED}.unresolved.gz"
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.${EXT_INPAR_SELECTED}.unresolved"
        rmcmd "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.${EXT_CLUSTER}.log"
        rmcmd "$DIR_PROJECT/$BASE_JOBLOG/${PL_TODO_NAME}_${STEP_ID}_*.log"
        step_info_msg "removed $RMCMD_REMOVED files"
    }
    cleanup_step_postproc() {
        RMCMD_REMOVED=0
        local scripts=()
        eval $(create_array scripts ':' "$OP_POSTPROC_SCRIPTS_FOUND")
        local ind=
        for (( ind=0; ind<${#scripts[@]}; ind++ )); do
            local s="${scripts[$ind]}"
            debug_msg "cleanup postproc : $s"
            set_postproc "$s" || continue
            if isfunc postproc_cleanup; then
                postproc_cleanup
            fi
        done
        step_info_msg "removed $RMCMD_REMOVED files"
    }



    #-----------------------------------------------
    # time_step_XXX() functions
    #-----------------------------------------------
    
    time_step_preproc() {
        time_step_per_todo_label  "$DIR_PROJECT_SEQUENCES" "$EXT_FASTA_CLEAN"
    }
    time_step_masker() {
        time_step_per_todo_label  "$DIR_PROJECT_SEQUENCES" "$EXT_FASTA_MASKED"
    }
    time_step_select() {
        time_step_per_todo_label "$DIR_PROJECT_SEQUENCES" "$EXT_FASTA_SELECTED" "${EXT_FASTA_SELECTED}.clstr"
    }
    time_step_stats() {
        time_step_per_todo_label "$DIR_PROJECT_SEQUENCES" "$EXT_FASTA_STATS" "$EXT_FASTA_SEQLEN"
    }
    time_step_makeinparsel() {
        time_step_per_todo_label "$DIR_PROJECT_SEQUENCES" "$EXT_INPAR_SELECTED"
    }
    time_step_formatdb() {
        time_reset
        time_step_per_todo_label "$DIR_PROJECT_SEQUENCES" "${EXT_FASTA_SELECTED}_formatdb.log"
    }


    time_step_pairwise() {
        time_reset
        [[ $# -eq 1 ]] || fatal_msg_wrong_nargs "$#" "$FUNCNAME"
        set_stepid "$1"

        local ec=0
        # check for pair file - do not use modificators or tree info
        set_info_msg_quiet
        create_pairwise_file "$STEP_ID" "$PL_TODO_NAME" "0" "0" || {
            reset_info_msg_quiet
            return 255
        }
        reset_info_msg_quiet
        
        [[ -f $GENERATED_PAIR_FILE ]] || { error_msg "pair file not found <$GENERATED_PAIR_FILE>"; return 255; }
        
        debug_msg "collect timing for pairwise step for $STEP_ID"
        
	    # initialise pairwise run
        PW_SCRIPT=$(get_run_script $STEP_ID)
        PW_NPAIRS=$(grep -c -v "^#" $GENERATED_PAIR_FILE)

	    # check if no pairs - return
	    [[ $PW_NPAIRS -gt 0 ]] || return 0

        # get extension
        local extwc=""
        
        if [[ $STEP_ID == ALIGNMENT ]]; then
            extwc="$EXT_ALIGN"
        elif [[ $STEP_ID == MAKEBRH ]]; then
            extwc="$EXT_BRH"
        elif [[ $STEP_ID == MAKEINPAR ]]; then
            extwc="$EXT_INPAR"
        fi

        [[ ${#extwc} -gt 0 ]] || { error_msg "time_step_pairwise() illegal stepid <$STEP_ID>"; return 255; }
        
        # indices and labels for A and B
        local allDone=0
        local saveIFS="$IFS"
        local line=""

        ###while read line; do
        until (( allDone==1 )); do
            IFS=$'\n'
            read line || allDone=1
            IFS=$saveIFS

            if (( allDone==0 )); then
                
                # skip comment and empty lines
                [[ ${line:0:1} == '#' ]] && continue
                [[ -n $line ]] || continue

                # set IFS to single space
                IFS=' '
                # process line
                local elems=( $line )
                local A=${elems[0]}
                local B=${elems[1]}
                local ftrg="$DIR_PROJECT_PWC/$A/${A}_${B}.$extwc"
                time_set_from_footer "$ftrg"
            fi

            # restore IFS to newline
            IFS=$'\n'
            
        done < $GENERATED_PAIR_FILE

        # restore IFS
        IFS="$saveIFS"

        # compute mean time
        time_set_mean
        return 0
    }
    
    time_step_alignment() {
        time_step_pairwise ALIGNMENT
    }

    time_step_makebrh() {
        time_step_pairwise MAKEBRH
    }
    
    time_step_makeinpar() {
        time_step_pairwise MAKEINPAR
    }
    
    time_step_setbrh() {
        time_reset
        time_set_from_footer "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.$EXT_BRH"
    }
    time_step_setinpar() {
        time_reset
        time_set_from_footer "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.$EXT_INPAR"
    }
    time_step_setinparsel() {
        time_reset
        time_set_from_footer "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.$EXT_INPAR_SELECTED"
    }
    time_step_preparetree() {
        # nothing to do
        time_reset
    }    
    time_step_weakbrh() {
        # nothing to do
        time_reset
    }    
    time_step_cluster() {
        time_reset
        time_set_from_footer "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.$EXT_CLUSTER_RAW"
    }
    time_step_mergeinpar() {
        time_reset
        time_set_from_footer "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.$EXT_CLUSTER_INPAR"
    }
    time_step_mergeinparsel() {
        time_reset
        time_set_from_footer "$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.$EXT_CLUSTER"
    }
    time_step_postproc() {
        time_reset
        local scripts=()
        eval $(create_array scripts ':' "$OP_POSTPROC_SCRIPTS_FOUND")
        local ind=
        for (( ind=0; ind<${#scripts[@]}; ind++ )); do
            local s="${scripts[$ind]}"
            debug_msg "cleanup postproc : $s"
            set_postproc "$s" || continue
            if isfunc postproc_timing; then
                postproc_timing
            fi
        done
    }

    #-----------------------------------------------
    # get nodes:
    # 1. internal nodes
    # 2. idem excluding lowest node
    # 3. leaf nodes
    # 4. leaf organisms
    #-----------------------------------------------
    export _NODES=()
    get_internal_nodes() {
        local f=${1-$TREE_INPUT_FILE}
        [[ -f $f ]] || return 255
        _NODES=($($PYTHON3 $DIR_ORTHOLOGER/bin/get_orthodb_tree.py -n -i "$f" -N))
    }
    get_internal_nodes_noleaves() {
        local f=${1-$TREE_INPUT_FILE}
        [[ -f $f ]] || return 255
        _NODES=($($PYTHON3 $DIR_ORTHOLOGER/bin/get_orthodb_tree.py -n -i "$f" -N -k))
    }
    get_leaf_nodes() {
        local f=${1-$TREE_INPUT_FILE}
        [[ -f $f ]] || return 255
        _NODES=($($PYTHON3 $DIR_ORTHOLOGER/bin/get_orthodb_tree.py -n -i "$f" -L -k))
    }
    get_leaf_organisms() {
        local f=${1-$TREE_INPUT_FILE}
        [[ -f $f ]] || return 255
        _NODES=($($PYTHON3 $DIR_ORTHOLOGER/bin/get_orthodb_tree.py -n -i "$f" -L))
    }
    get_branches() {
        local f=${1-$TREE_INPUT_FILE}
        local n=${2-}
        [[ -f $f ]] || return 255
        local args=""
        [[ -n $n ]] && args="-o -r $n"
        _NODES=($($PYTHON3 $DIR_ORTHOLOGER/bin/get_orthodb_tree.py -n -i "$f" -C -k $args))
    }
    
    print_root_clade() {
        local f=${1-$TREE_INPUT_FILE}
        [[ -f $f ]] || return 255
        $PYTHON3 $DIR_ORTHOLOGER/bin/get_orthodb_tree.py -n -i "$f" -R
    }
    print_newick_odb() {
        local url=$1
        local root=$2
        local opt="-u"
        [[ -f "$url" ]] && opt="-i"
        $PYTHON3 $DIR_ORTHOLOGER/bin/get_orthodb_tree.py -n $opt "$url" -r "$root"
    }
    print_leaf_organisms() {
        local f=$1
        local node=$2
        [[ -f $f ]] || return 255
        $PYTHON3 $DIR_ORTHOLOGER/bin/get_orthodb_tree.py -n -i "$f" -L -r "$node"
    }
    #-----------------------------------------------
    # set tree root if not set
    #-----------------------------------------------
    set_tree_root() {
        if [[ -z $TREE_ROOT_CLADE ]]; then
            [[ -f $TREE_INPUT_FILE ]] || return 255
            # get root from newick tree
            TREE_ROOT_CLADE=$(print_root_clade) || {
                step_error_msg "set_tree_root: failed finding root node in tree $(trim_name $TREE_INPUT_FILE) - missing file or wrong format (should be newick)?"
                return 255
            }
        fi
        return 0
    }

    #-----------------------------------------------
    # hard configs that require change if TODO is
    # changed
    # called by update_todo() in main pipeline
    # NOTE this updates OP_LABELB_FIRST/END if
    # PL_TODO_LABELS_REQ contains one id.
    # That speeds up the processing in step_pairwise().
    #-----------------------------------------------
    proj_update_todo() {
        debug_msg "proj_update_todo - nothing to do"
    }
    
    #-----------------------------------------------
    # configure readonly part of OP_ variables
    #-----------------------------------------------
    configure_step_readonly() {
        [[ "${OP_ALL_STEPS[@]+1}" == "" ]] || { debug_msg "skipping configure readonly"; return 0; }
        debug_msg "setting readonly OP_STEP_xxx"
        # array of all steps
        # removed: SPLITBRH
        OP_ALL_STEPS=( "PREPARETREE" \
                           "PREPROC" \
                           "MASKER" \
                           "SELECT" \
                           "STATS" \
                           "FORMATDB" \
                           "ALIGNMENT" \
                           "MAKEBRH" \
                           "MAKEINPAR" \
                           "MAKEINPARSEL" \
                           "SETBRH" \
                           "SETINPAR" \
                           "SETINPARSEL" \
                           "CLUSTER" \
                           "MERGEINPAR" \
                           "MERGEINPARSEL" \
                           "POSTPROC" )

        
        # save all steps in string
        OP_ALL_STEPS_STR="${OP_ALL_STEPS[@]}"
        OP_ALL_STEPS_N=${#OP_ALL_STEPS[@]}

        # descriptions
        OP_STEP_DESCRIPTION[PREPARETREE]="if TREE mode, prepare tree related files (.$EXT_PAIRS_TREE,.poor.brh)"
        OP_STEP_DESCRIPTION[PREPROC]="pre-process fasta files"
        OP_STEP_DESCRIPTION[MASKER]="mask out low complexity regions"
        OP_STEP_DESCRIPTION[SELECT]="select sequences that are similar and replace with a representative"
        OP_STEP_DESCRIPTION[STATS]="get statistics on selected fasta file"
        OP_STEP_DESCRIPTION[FORMATDB]="make blast DB files"
        OP_STEP_DESCRIPTION[ALIGNMENT]="do pair wise alignment"
        OP_STEP_DESCRIPTION[MAKEBRH]="find the Best Reciprocal Hits (BRH)"
        OP_STEP_DESCRIPTION[MAKEINPAR]="find inparalogs from within species alignment"
        OP_STEP_DESCRIPTION[MAKEINPARSEL]="collect the sequences removed in the SELECT step"
        OP_STEP_DESCRIPTION[SETBRH]="collect all BRHs into one file"
        #OP_STEP_DESCRIPTION[SPLITBRH]="splits genes in final BRH file"
        OP_STEP_DESCRIPTION[SETINPAR]="collect all inparalogs into one file"
        OP_STEP_DESCRIPTION[SETINPARSEL]="collect all from MAKEINPARSEL"
        OP_STEP_DESCRIPTION[CLUSTER]="cluster all BRHs"
        OP_STEP_DESCRIPTION[MERGEINPAR]="merge in inparalogs"
        OP_STEP_DESCRIPTION[MERGEINPARSEL]="merge in sequences from SETINPARSEL"
        #OP_STEP_DESCRIPTION[WEAKBRH]="if TREE mode, extract weak BRHs"
        OP_STEP_DESCRIPTION[POSTPROC]="user defined post processing of final cluster (POSTPROC_LABELS)"
        
        local depend="COMMON"
        local step=
        for step in "${OP_ALL_STEPS[@]}"; do
            # change default values as of SETBRH (ie when there is only one job to run per step)"
            [[ "$step" == "SETBRH" ]] && depend="LABEL"
            # as of SETBRH step, result has TODO label in name
            OP_STEP_DATA[$step]=$depend
        done
        
        OP_STEP_DEPENDS[PREPARETREE]=""
        OP_STEP_DEPENDS[PREPROC]=""
        OP_STEP_DEPENDS[MASKER]=PREPROC
        OP_STEP_DEPENDS[SELECT]=MASKER
        OP_STEP_DEPENDS[STATS]=SELECT
        OP_STEP_DEPENDS[FORMATDB]=SELECT
        OP_STEP_DEPENDS[ALIGNMENT]=FORMATDB
        OP_STEP_DEPENDS[MAKEBRH]=ALIGNMENT
        OP_STEP_DEPENDS[MAKEINPAR]=MAKEBRH
        OP_STEP_DEPENDS[MAKEINPARSEL]=SELECT
        OP_STEP_DEPENDS[SETBRH]=MAKEBRH
        #OP_STEP_DEPENDS[SPLITBRH]=SETBRH
        OP_STEP_DEPENDS[SETINPAR]=MAKEINPAR
        OP_STEP_DEPENDS[SETINPARSEL]=MAKEINPARSEL
        OP_STEP_DEPENDS[CLUSTER]=SETBRH
        OP_STEP_DEPENDS[MERGEINPAR]=CLUSTER
        OP_STEP_DEPENDS[MERGEINPARSEL]=MERGEINPAR
        #OP_STEP_DEPENDS[WEAKBRH]=MERGEINPARSEL
        OP_STEP_DEPENDS[POSTPROC]=MERGEINPARSEL

        OP_STEP_REQBY[PREPROC]=MASKER
        OP_STEP_REQBY[MASKER]=SELECT
        OP_STEP_REQBY[SELECT]=STATS,MAKEINPARSEL
        OP_STEP_REQBY[STATS]=ALIGNMENT
        OP_STEP_REQBY[FORMATDB]=ALIGNMENT
        OP_STEP_REQBY[ALIGNMENT]=MAKEBRH
        OP_STEP_REQBY[MAKEBRH]=SETBRH
        OP_STEP_REQBY[SETBRH]=CLUSTER
        OP_STEP_REQBY[PREPARETREE]=SETBRH
        #OP_STEP_REQBY[SETBRH]=SPLITBRH
        #OP_STEP_REQBY[SPLITBRH]=MAKEINPAR,CLUSTER
        OP_STEP_REQBY[MAKEINPAR]=SETINPAR
        OP_STEP_REQBY[MAKEINPARSEL]=SETINPARSEL
        
        OP_STEP_REQBY[SETINPAR]=MERGEINPAR
        OP_STEP_REQBY[SETINPARSEL]=MERGEINPARSEL
        OP_STEP_REQBY[CLUSTER]=MERGEINPAR
        OP_STEP_REQBY[MERGEINPAR]=""
        OP_STEP_REQBY[MERGEINPARSEL]=""
        #OP_STEP_REQBY[WEAKBRH]=""
        OP_STEP_REQBY[POSTPROC]=""

        OP_STEP_TYPE[PREPROC]=$OP_STEP_TYPE_LABEL
        OP_STEP_TYPE[MASKER]=$OP_STEP_TYPE_LABEL
        OP_STEP_TYPE[SELECT]=$OP_STEP_TYPE_LABEL
        OP_STEP_TYPE[STATS]=$OP_STEP_TYPE_LABEL
        OP_STEP_TYPE[FORMATDB]=$OP_STEP_TYPE_LABEL

        OP_STEP_TYPE[ALIGNMENT]=$OP_STEP_TYPE_PWC
        OP_STEP_TYPE[MAKEBRH]=$OP_STEP_TYPE_PWC
        OP_STEP_TYPE[MAKEINPAR]=$OP_STEP_TYPE_PWC
        
        OP_STEP_TYPE[MAKEINPARSEL]=$OP_STEP_TYPE_LABEL

        OP_STEP_TYPE[PREPARETREE]=$OP_STEP_TYPE_SINGLE
        OP_STEP_TYPE[SETBRH]=$OP_STEP_TYPE_SINGLE
        OP_STEP_TYPE[SETINPAR]=$OP_STEP_TYPE_SINGLE
        OP_STEP_TYPE[SETINPARSEL]=$OP_STEP_TYPE_SINGLE
        OP_STEP_TYPE[CLUSTER]=$OP_STEP_TYPE_SINGLE
        OP_STEP_TYPE[MERGEINPAR]=$OP_STEP_TYPE_SINGLE
        OP_STEP_TYPE[MERGEINPARSEL]=$OP_STEP_TYPE_SINGLE

        OP_STEP_TYPE[POSTPROC]=$OP_STEP_TYPE_SINGLE
        
        # internally defined postproc scripts
        OP_POSTPROC_INTERN=""
        local s=""
        local lbl=""
        for s in $DIR_ORTHOLOGER_BIN/postproc_*.sh; do
            lbl=$(get_postproc_name $s)
            # tmp remove siblings postproc (requires TODO_LBLIND - see common_pipeline.sh)
            [[ $lbl == "siblings" ]] && continue
            OP_POSTPROC_INTERN="$OP_POSTPROC_INTERN $lbl"
        done

        # make some read only
        readonly OP_ALL_STEPS
        readonly OP_ALL_STEPS_N
        readonly OP_ALL_STEPS_STR
        readonly OP_STEP_DESCRIPTION
        readonly OP_STEP_DATA
        readonly OP_STEP_DEPENDS
        readonly OP_STEP_REQBY
        readonly OP_POSTPROC_INTERN

        # export the declarations
        #export_decl OP_ALL_STEPS
        #export_decl OP_STEP_DESCRIPTION
        #export_decl OP_STEP_DATA
        #export_decl OP_STEP_DEPENDS
        #export_decl OP_STEP_REQBY
        #export_decl OP_POSTPROC_INTERN
    }

    #-----------------------------------------------
    # configure OP_ variables to default
    # called by op_step_setup()
    # which is called in the config file: orthologer_conf.sh
    #-----------------------------------------------
    proj_step_setup() {
        local runmode="BATCH"
        [[ "$SCHEDULER_LABEL" == "NONE" ]] && runmode="LOCAL"

        # only do this if it has not been done before
        isempty OP_STEP_NPARALLEL || { debug_msg "skipping proj_step_setup()"; return 0; }
        debug_msg "running proj_step_setup()"

        # configure readonly vars
        configure_step_readonly

        # set OP_STEP arrays to default values
        set_op_step_defaults

        # customise OP_STEP arrays
        # set some run mode defaults to BATCH
        OP_STEP_RUNMODE[SELECT]="$runmode"
        OP_STEP_RUNMODE[ALIGNMENT]="$runmode"
        OP_STEP_RUNMODE[MAKEBRH]="$runmode"
        OP_STEP_RUNMODE[MAKEINPAR]="$runmode"
        OP_STEP_RUNMODE[MAKEINPARSEL]="$runmode"
        OP_STEP_RUNMODE[CLUSTER]="$runmode"
        OP_STEP_RUNMODE[MERGEINPAR]="$runmode"
        OP_STEP_RUNMODE[MERGEINPARSEL]="$runmode"

        OP_STEP_RUNMODE[PREPARETREE]="LOCAL"
        #OP_STEP_RUNMODE[WEAKBRH]="LOCAL"
        
        # set required executables
        # format : "<dir var> <exec var>"

        # MASKER
        OP_STEP_REQTOOLS[MASKER]=""
        if   [[ "$MASKER_LABEL" == "SEGMASKER" ]]; then
            OP_STEP_REQTOOLS[MASKER]="DIR_BLAST segmasker EXE_SEGMASKER"
        elif [[ "$MASKER_LABEL" == "DUSTMASKER" ]]; then
            OP_STEP_REQTOOLS[MASKER]="DIR_BLAST dustmasker EXE_DUSTMASKER"
        fi

        # SELECT
        OP_STEP_REQTOOLS[SELECT]=""
        if   [[ "$SELECT_LABEL" == "CDHIT" ]]; then
            #if [ "$DATA_TYPE" == "NUCL" ]; then
            #    OP_STEP_REQTOOLS[SELECT]="DIR_CDHIT cd-hit-est EXE_CDHIT_NUCL"
            #elif [ "$DATA_TYPE" == "PROT" ]; then
            OP_STEP_REQTOOLS[SELECT]="DIR_CDHIT cd-hit EXE_CDHIT_PROT"
            #fi
        fi

        # FORMATDB
        OP_STEP_REQTOOLS[FORMATDB]=""
        if [[ ${ALIGNMENT_LABEL} == "SWIPE" ]]; then
            OP_STEP_REQTOOLS[FORMATDB]="DIR_BLAST makeblastdb EXE_MAKEBLASTDB"
        elif [[ ${ALIGNMENT_LABEL} == "MMSEQS" ]]; then
            OP_STEP_REQTOOLS[FORMATDB]="DIR_MMSEQS mmseqs EXE_MMSEQS"
        elif [[ ${ALIGNMENT_LABEL} == "DIAMOND" ]]; then
            OP_STEP_REQTOOLS[FORMATDB]="DIR_DIAMOND diamond EXE_DIAMOND"
        fi

        # ALIGNMENT
        OP_STEP_NMERGE[ALIGNMENT]=1
        OP_STEP_REQTOOLS[ALIGNMENT]=""
        if [[ "$ALIGNMENT_LABEL" == "SWIPE" ]]; then
            OP_STEP_REQTOOLS[ALIGNMENT]="DIR_SWIPE swipe EXE_SWIPE"
        elif [[ "$ALIGNMENT_LABEL" == "MMSEQS" ]]; then
            OP_STEP_REQTOOLS[ALIGNMENT]="DIR_MMSEQS mmseqs EXE_MMSEQS"
        elif [[ "$ALIGNMENT_LABEL" == "DIAMOND" ]]; then
            OP_STEP_REQTOOLS[ALIGNMENT]="DIR_DIAMOND diamond EXE_DIAMOND"
        elif [[ "$ALIGNMENT_LABEL" == "BLAST" ]]; then
            OP_STEP_REQTOOLS[ALIGNMENT]="DIR_BLAST blastp EXE_BLASTP"
        fi

        # MAKEBRH
        OP_STEP_REQTOOLS[MAKEBRH]="DIR_BRHCLUS findbrh EXE_FINDBRH"
        OP_STEP_REQTOOLS[MAKEINPAR]="DIR_BRHCLUS getinpar EXE_GETINPAR"
        #OP_STEP_REQTOOLS[WEAKBRH]="DIR_BRHCLUS extractpairs EXE_EXTRACTPAIRS"
        OP_STEP_REQTOOLS[PREPARETREE]=""

        # SPLITBRH
        ###OP_STEP_REQTOOLS[SPLITBRH]="DIR_BRHCLUS splitgenes EXE_SPLITGENES"
        
        # CLUSTER
        OP_STEP_REQTOOLS[CLUSTER]=""
        if   [[ "$CLUSTER_LABEL" == "BRHCLUS" ]]; then
            OP_STEP_REQTOOLS[CLUSTER]="DIR_BRHCLUS brhclus EXE_BRHCLUS"
        fi
        
        # MERGEINPAR(SEL)
        OP_STEP_REQTOOLS[MERGEINPAR]="DIR_BRHCLUS mergeinpar EXE_MERGEINPAR"
        OP_STEP_REQTOOLS[MERGEINPARSEL]="${OP_STEP_REQTOOLS[MERGEINPAR]}"
        
        # export runmode dictionary - required in op_step_adjust below... REALLY???
        #export_dict OP_STEP_RUNMODE
        #export_decl OP_STEP_RUNMODE
    }
    
    #-----------------------------------------------
    # fix OP_ variables depending on setup
    # called by proj_configure_env()
    #-----------------------------------------------
    proj_op_fix() {
        # adjust certain parameters depending on PL_TODO setting etc
        #op_step_adjust

        # make sure NMERGE is not more than 1 in certain cases
        # removed SPLITBRH
        for step in SETBRH SETINPAR SETINPARSEL CLUSTER MERGEINPAR MERGEINPARSEL POSTPROC; do
            if [[ ${OP_STEP_NMERGE[$step]} -ne 1 ]]; then
                debug_msg "setting OP_STEP_NMERGE[$step]=1 - job merge is not supported (or relevant) here"
                OP_STEP_NMERGE[$step]=1
            fi
        done
        #if is_step_enabled SPLITBRH; then
        #    warn_msg "disabling SPLITBRH (enabled in common.sh) - not yet fully implemented"
        #    disable_step SPLITBRH
        #fi
        if (( TREE_ENABLED == 1 )); then
            OP_STEP_ENABLE[PREPARETREE]=1
            #OP_STEP_ENABLE[WEAKBRH]=0
            OP_STEP_ENABLE[SETBRH]=0
            OP_STEP_ENABLE[SETINPAR]=0
            info_msg "tree mode enabled -> enable (PREPARETREE) and disable (SETBRH,SETINPAR)"
        else
            OP_STEP_ENABLE[PREPARETREE]=0
            #OP_STEP_ENABLE[WEAKBRH]=0
        fi
    }

    #-----------------------------------------------
    # Set default OP_STEP_NPARALLEL .
    # This just provides a default setup for various
    # steps.
    # Currently just used by manage_project_odb.sh
    #-----------------------------------------------
#    set_default_nparallel() {
#        fatal_msg "set_default_nparallel() should not be used!"###

        
#        local nspec=$PL_TODO_NLABELS
#        [[ -n $nspec ]] || nspec=0
#        local nspec2=$((nspec*nspec))
#        local nhalf=$((nspec*(nspec-1)/2))
#        [[ $nspec -gt 0 ]] || { nspec=1; nspec2=1; nhalf=1; }

#        # set max nr of jobs for select and alignment
#        OP_STEP_NPARALLEL[SELECT]=$nspec
#        OP_STEP_NPARALLEL[ALIGNMENT]=$nspec2
#        OP_STEP_NPARALLEL[MAKEBRH]=$nhalf
#        OP_STEP_NPARALLEL[MAKEINPAR]=$nhalf
#        OP_STEP_NPARALLEL[MAKEINPARSEL]=$nspec
#    }

    set_step_nparallel() {
        [[ $# -eq 2 ]] || return 0
        local oldstep=$STEP_ID
        STEP_ID="$1"
        local N=$2
        local Nmax=0
        
        if   is_runmode_batch; then
            Nmax=$OP_NJOBMAX_BATCH
        elif is_runmode_local; then
            Nmax=$OP_NJOBMAX_LOCAL
        elif is_runmode_interactive; then
            Nmax=1
        fi
        if (( N == 0 )); then
            N=$Nmax
        else
            (( N > Nmax )) && N=$Nmax
        fi
        OP_STEP_NPARALLEL[$STEP_ID]=$N
        STEP_ID="$oldstep"
    }

    #-----------------------------------------------
    # This will adjust OP_STEP_ variables
    # to values relevant to the selected TODO
    #-----------------------------------------------
    op_step_adjust() {
        # skip if OP_STEP is not setup
        isempty OP_STEP_NPARALLEL && return 0

        # get nr of species
        local nspec=$PL_TODO_NLABELS
        local nspec2=0
        local nhalf=0
        [[ -n $nspec ]] || nspec=0
        if (( nspec > 0 )); then
            nspec2=$((nspec*nspec))
            nhalf=$((nspec*(nspec-1)/2))
        else
            # set all to the same
            nspec=$(get_nthreads_conservative)
            nspec2=$nspec
            nhalf=$nspec
        fi

        #- OP_STEP_NPARALLEL
        #  nr of jobs in parallel to be submitted
        set_step_nparallel SELECT       $nspec
        set_step_nparallel ALIGNMENT    $nspec2
        set_step_nparallel MAKEBRH      $nhalf
        set_step_nparallel MAKEINPAR    $nhalf
        set_step_nparallel MAKEINPARSEL $nspec

        #- OP_STEP_NMERGE
        #  nr of jobs to merge into one job
        local nmergemax=50
        local nmerge=$nspec
        [[ $nmerge -gt $nmergemax ]] && nmerge=$nmergemax

        # if there are very many alignments between short sequences
        # then this should be set to a larger value
        ####OP_STEP_NMERGE[ALIGNMENT]=1

        # these are very quick jobs - best be merged
        [[ ${OP_STEP_NMERGE[MAKEBRH]}   -le 1 ]] && OP_STEP_NMERGE[MAKEBRH]=$nmerge
        [[ ${OP_STEP_NMERGE[MAKEINPAR]} -le 1 ]] && OP_STEP_NMERGE[MAKEINPAR]=$nmerge
        return 0
    }

    #-----------------------------------------------
    # get postproc label from full script name
    #-----------------------------------------------
    get_postproc_name() {
        [[ $# -eq 1 ]] || return 0
        local lbl=""
        lbl=$(file_basename "$1")
        lbl=${lbl/postproc_/}
        echo $lbl
    }

    #-----------------------------------------------
    # reset postproc functions
    #-----------------------------------------------
    reset_postproc() {
        unset postproc_cleanup
        unset postproc_conf
        unset postproc_help
        unset postproc
        unset POSTPROC_NTARGETS
        unset POSTPROC_TRGS
        unset POSTPROC_NAME
    }

    #-----------------------------------------------
    # check postproc variables
    #-----------------------------------------------

    check_postproc_setup() {
        [[ $# -eq 1 ]] || fatal_msg "check_postproc() wrong arguments!"
        local lbl="$1"
        local ec=0
        isfunc postproc          || { ec=1; error_msg "postproc ($lbl) script missing postproc() function"; }
        isfunc postproc_cleanup  || { warn_msg "postproc ($lbl) script missing postproc_cleanup() function - ignored"; }
        isfunc postproc_help     || { ec=1; error_msg "postproc ($lbl) script missing postproc_help() function"; }
        isfunc postproc_conf     || { ec=1; error_msg "postproc ($lbl) script missing postproc_conf() function"; }

        isset  POSTPROC_NTARGETS || { ec=1; error_msg "postproc ($lbl) script missing POSTPROC_NTARGETS variable"; }
        isset  POSTPROC_TRGS     || { ec=1; error_msg "postproc ($lbl) script missing POSTPROC_TRGS variable"; }
        isset  POSTPROC_NAME     || POSTPROC_NAME=$lbl

        [[ $ec -eq 0 ]] || return 255
        return 0
    }

    #-----------------------------------------------
    # set postproc
    #-----------------------------------------------
    set_postproc() {
        # check if provided script is in list
        [[ $# -eq 1 ]] || return 0
        local script="$1"
        ###        inrange_set_delim "$script" ":" "$OP_POSTPROC_SCRIPTS_FOUND" || { error_msg "postproc script not in list $script"; return 255; }
        ###        [[ -x $script ]] || { error_msg "postproc script $script not executable by user $SYS_WHOAMI"; return 255; }
        [[ -f "$script" ]] || { error_msg "missing postproc script '$script'"; return 255; }

        # reset postproc env
        reset_postproc

        # set env
        source $script

        # get postproc label
        local lbl=""
        lbl=$(get_postproc_name "$script")

        # check postproc
        check_postproc_setup "$lbl"
        return $?
    }

    #-----------------------------------------------
    # determine the postproc script setting
    #-----------------------------------------------
    init_postproc_scripts() {
        # no post proc script by default
        POSTPROC_LABELS=${POSTPROC_LABELS:="$STEP_ID_FLAG"}

        debug_msg "init postproc scripts with POSTPROC_LABELS=<$POSTPROC_LABELS> and STEP_ID_FLAG=<$STEP_ID_FLAG>"
        
        # clear this one
        OP_POSTPROC_SCRIPTS_FOUND=""
        
        # check post proc script
        [[ ${#POSTPROC_LABELS} -ne 0 ]] || return 0 #{ debug_msg "POSTPROC_LABELS is not set -> disabling POSTPROC"; OP_STEP_ENABLE[POSTPROC]=0; return 0; }

        # loop over all scripts
        local snew=""
        local s=""
        local lbl=""
        local foundss=""
        local ind=0
        local labels=()
        eval $(create_array labels ',' "$POSTPROC_LABELS")
        for lbl in ${labels[@]}; do
            local ss=$lbl
            lbl=${lbl,,}
            # check if internal postproc
            if inrange_set "$lbl" $OP_POSTPROC_INTERN; then
                # it's internal - get full filename
                ss="postproc_${lbl}.sh"
            fi
            debug_msg "using postproc script $ss ($OP_POSTPROC_INTERN)"
            lbl=$(get_postproc_name "$s")

            foundss=$(find_script $ss)
            [[ $? -eq 0 ]] || { warn_msg "postproc ($lbl): unable to find script $ss"; continue; }
            debug_msg "postproc ($lbl): found script $foundss"

            [[ ${#snew} -eq 0 ]] || snew="${snew}:"
            snew="${snew}$foundss"
        done
        # check that at least one script is found
        [[ ${#snew} -gt 0 ]] || { warn_msg "unable to find any valid POSTPROC_LABELS = $POSTPROC_LABELS -> disabling POSTPROC"; OP_STEP_ENABLE[POSTPROC]=0; return 0; }

        OP_POSTPROC_SCRIPTS_FOUND="$snew"
        return 0
    }

    #-----------------------------------------------
    # check all postproc scripts
    #-----------------------------------------------

    check_postproc_scripts() {
        local ec=0
        local nbad=0
        local nok=0
        local s=""
        local lbl=""
        local scripts=()
        eval $(create_array scripts ':' "$OP_POSTPROC_SCRIPTS_FOUND")
        local ind=0
        for (( ind=0; ind<${#scripts[@]}; ind++ )); do
            s="${scripts[$ind]}"
            debug_msg "checking postproc script : $s"

            set_postproc "$s" || { nbad=$((nbad+1)); continue; }

            lbl=$(get_postproc_name "$s")
            check_postproc_setup "$lbl" || { nbad=$((nbad+1)); continue; }
            nok=$((nok+1))
        done
        [[ $nok -gt 0 ]] || return 255
        return 0
    }



    
    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    
    #-----------------------------------------------
    # get preproc label from full script name - OK
    #-----------------------------------------------
    get_preproc_name() {
        [[ $# -eq 1 ]] || return 0
        local lbl=""
        lbl=$(file_basename "$1")
        lbl=${lbl/preproc_/}
        echo $lbl
    }

    #-----------------------------------------------
    # reset preproc functions
    #-----------------------------------------------
    reset_preproc() {
        unset preproc_cleanup
        unset preproc_conf
        unset preproc_help
        unset preproc
        unset PREPROC_NTARGETS
        unset PREPROC_TRGS
        unset PREPROC_NAME
    }

    #-----------------------------------------------
    # check preproc variables
    #-----------------------------------------------

    check_preproc_setup() {
        [[ $# -eq 1 ]] || fatal_msg "check_preproc() wrong arguments!"
        local lbl="$1"
        local ec=0
        isfunc preproc          || { ec=1; error_msg "preproc ($lbl) script missing preproc() function"; }
        isfunc preproc_cleanup  || { warn_msg "preproc ($lbl) script missing preproc_cleanup() function - ignored"; }
        isfunc preproc_help     || { ec=1; error_msg "preproc ($lbl) script missing preproc_help() function"; }
        isfunc preproc_conf     || { ec=1; error_msg "preproc ($lbl) script missing preproc_conf() function"; }

        isset  PREPROC_NTARGETS || { ec=1; error_msg "preproc ($lbl) script missing PREPROC_NTARGETS variable"; }
        isset  PREPROC_TRGS     || { ec=1; error_msg "preproc ($lbl) script missing PREPROC_TRGS variable"; }
        isset  PREPROC_NAME     || PREPROC_NAME=$lbl

        [[ $ec -eq 0 ]] || return 255
        return 0
    }

    #-----------------------------------------------
    # set preproc
    #-----------------------------------------------

    set_preproc() {
        # check if provided script is in list
        [[ $# -eq 1 ]] || return 0
        local script="$1"
        inrange_set "$script" "$OP_PREPROC_SCRIPTS_FOUND" || { error_msg "preproc script not in list $script"; return 255; }
        [[ -x $script ]] || { error_msg "preproc script $script not executable by user $SYS_WHOAMI"; return 255; }

        # reset preproc env
        reset_preproc

        # set env
        source $script

        # get preproc label
        local lbl=""
        lbl=$(get_preproc_name "$script")

        # check preproc
        check_preproc_setup "$lbl"
        return $?
    }

    #-----------------------------------------------
    # determine the preproc script setting
    #-----------------------------------------------

    init_preproc_scripts() {
        # no post proc script by default
        OP_PREPROC_SCRIPT=${OP_PREPROC_SCRIPT=""}

        # clear this one
        OP_PREPROC_SCRIPTS_FOUND=""

        # check post proc script
        [[ ${#OP_PREPROC_SCRIPT} -ne 0 ]] || { debug_msg "OP_PREPROC_SCRIPT is not set -> disabling PREPROC"; OP_STEP_ENABLE[PREPROC]=0; return 0; }

        # loop over all scripts
        local snew=""
        local s=""
        local lbl=""
        local foundss=""
        local scripts=($(tokenize ',' $OP_PREPROC_SCRIPT))
        #local scripts=()
        #v_tokenize scripts ',' $OP_PREPROC_SCRIPT
        
        for s in "${scripts[@]}"; do
            local ss=$s 
            # check if internal preproc
            if inrange_set "$s" $OP_PREPROC_INTERN; then
                # it's internal - get full filename
                ss="preproc_${s}.sh"
            fi
            debug_msg "using preproc script $ss"
            lbl=$(get_preproc_name "$s")

            foundss=$(find_script $ss)
            [[ $? -eq 0 ]] || { warn_msg "preproc ($lbl): unable to find script $ss"; continue; }
            debug_msg "preproc ($lbl): found script $foundss"

            #TODO: this does not allow for spaces in file name!!!
            snew="$snew $foundss"
        done

        # check that at least one script is found
        [[ ${#snew} -gt 0 ]] || { warn_msg "unable to find any valid OP_PREPROC_SCRIPT = $OP_PREPROC_SCRIPT -> disabling PREPROC"; OP_STEP_ENABLE[PREPROC]=0; return 0; }

        OP_PREPROC_SCRIPTS_FOUND="$snew"
        return 0
    }

    #-----------------------------------------------
    # check all preproc scripts
    #-----------------------------------------------

    check_preproc_scripts() {
        local ec=0
        local nbad=0
        local nok=0
        local s=""
        local lbl=""
        for s in $OP_PREPROC_SCRIPTS_FOUND; do
            debug_msg "checking preproc script : $s"

            set_preproc $s
            ec=$?
            [[ $ec -eq 0 ]] || { nbad=$((nbad+1)); continue; }

            lbl=$(get_preproc_name "$s")
            check_preproc_setup "$lbl"
            ec=$?
            [[ $ec -eq 0 ]] || { nbad=$((nbad+1)); continue; }
            nok=$((nok+1))
        done
        [[ $nok -gt 0 ]] || return 255
        return 0
    }
    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~





    

    #-----------------------------------------------
    # common format header line for alignment
    # and BRH files
    #-----------------------------------------------
    common_fmt_header() {
        echo "#FMT id1 id2 score evalue pid start1 end1 start2 end2"
    }

    #-----------------------------------------------
    # get blast DB extensions
    # depends on DATA_TYPE
    #-----------------------------------------------
    get_formatdb_exts() {
        local prefix=""
        [[ $# -ge 1 ]] && prefix="$1"
        local exts=""
        if [[ $ALIGNMENT_LABEL == "MMSEQS" ]]; then
            prefix="${prefix}.${EXT_DBPRE}"
            exts="${prefix} ${prefix}.index ${prefix}.lookup ${prefix}_h ${prefix}_h.index ${prefix}.dbtype ${prefix}_h.dbtype"
        elif [[ $ALIGNMENT_LABEL == "DIAMOND" ]]; then
            exts=".dmnd"
        else
            prefix="${prefix}.${EXT_DBPRE}"
            exts="${prefix}db ${prefix}hr ${prefix}in ${prefix}og ${prefix}os ${prefix}ot ${prefix}sq ${prefix}tf ${prefix}to"
        fi
        [[ ${#exts} -gt 0 ]] || return 255
        echo "$exts"
        return 0
    }
    #-----------------------------------------------
    # check for formatdb files for the given fasta
    # file it expects them to be in the same
    # directory
    #-----------------------------------------------
    check_formatdb_files() {
        [[ $# -eq 2 ]] || { error_msg "check_formatdb_files() wrong nr of args $#"; return 255;}
        local fsfile="$1"
        local silent="$2"
        local exts=""
        local ec=0
        local e=""
        exts="$(get_formatdb_exts)"
        for e in $exts; do
            local f="${fsfile}$e"
            [[ -f "$f" ]] || { ec=1; break; }
        done
        if [[ $ec -ne 0 ]]; then
            [[ $silent -eq 0 ]] && error_msg "need to run formatdb for ${fsfile}"
            return 255
        fi
        return 0
    }

    #-----------------------------------------------
    # make soft links of formatdb/makeblastdb files
    #-----------------------------------------------
    link_formatdb_files() {
        [[ $# -eq 2 ]] || { error_msg "link_formatdb_files() wrong nr of args $#"; return 255;}

        local src="$1"
        local trg="$2"
        local exts=""
        local ec=0
        local e=""
        local errmsg=""
        exts="$(get_formatdb_exts)" || fatal_msg "failed retrieving formatdb extensions!"
        for e in ${exts}; do
            local f="${src}$e"
            [[ -f "$f" ]] || { errmsg="file not found $f"; ec=1; break; }
            ###local e="${f##*.}" # get the last extension
            local t="${trg}$e"
            ###        [[ -f "$t" ]] && continue
            rm -f "$t"
            ln -sf "$f" "$t"
            [[ $? -eq 0 ]] || { errmsg="failed soft link $f"; ec=2; break; }
        done
        [[ $ec -eq 0 ]] && return 0

        error_msg "link_formatdb_files() failed with ec=$ec"
        error_msg "link_formatdb_files() msg    = $errmsg"
        error_msg "link_formatdb_files() source = $src"
        error_msg "link_formatdb_files() target = $trg"
        return 255
    }

    remake_pair_target_makebrh() {
        [[ $# -eq 2 ]] || fatal_msg "remake_pair_target() wrong nr of arguments <$*>"

        # if no remake check, return ok
        [[ $SKIP_REMAKE_CHECK -eq 0 ]] || { debug_msg "remake_pair_target() skipped"; return 0; }

        # labels
        local A="$1"
        local B="$2"

        local ftrg="$DIR_PROJECT_PWC/$A/${A}_${B}.$EXT_BRH"
        [[ -f $ftrg ]] && return 1
        
        # get alignment file(s)
        local fAB="$DIR_PROJECT_PWC/$A/${A}_${B}.$EXT_ALIGN"
        [[ -f "$fAB" ]] || { step_error_msg "missing source: $fAB"; return 255; }
        if [[ $ALIGNMENT_MATRIX -eq 0 ]]; then
            fBA="$DIR_PROJECT_PWC/$B/${B}_${A}.$EXT_ALIGN"
            [[ -f "$fBA" ]] || { step_error_msg "missing source: $fBA"; return 255; }
        fi
        return 0
    }

    remake_pair_target_makeinpar() {
        [[ $# -eq 2 ]] || fatal_msg "remake_pair_target() wrong nr of arguments <$*>"

        # if no remake check, return ok
        [[ $SKIP_REMAKE_CHECK -eq 0 ]] || { debug_msg "remake_pair_target() skipped"; return 0; }

        # labels
        local A="$1"
        local B="$2"

        local ftrg="$DIR_PROJECT_PWC/$A/${A}_${B}.$EXT_INPAR"
        [[ -f $ftrg ]] && return 1
        
        # get BRH,AA,BB files
        local fBRH="$DIR_PROJECT_PWC/$A/${A}_${B}.$EXT_BRH"
        [[ -f "$fBRH" ]] || { step_error_msg "missing source: $fBRH"; return 255; }
        
        local fAA="$DIR_PROJECT_PWC/$A/${A}_${A}.$EXT_ALIGN"
        [[ -f "$fAA" ]] || { step_error_msg "missing source: $fAA"; return 255; }
        
        local fBB="$DIR_PROJECT_PWC/$B/${B}_${B}.$EXT_ALIGN"
        [[ -f "$fBB" ]] || { step_error_msg "missing source: $fBB"; return 255; }
        
        return 0
    }

    #-----------------------------------------------
    # Get source for given step id and species pair
    #-----------------------------------------------
    remake_pair_target() {
        [[ $# -eq 2 ]] || fatal_msg "remake_pair_target() wrong nr of arguments <$*>"
        #----------------------------------------------------------------
        # The below in this section can be used if compression is NOT used
        #
        #if [[ "$STEP_ID" == "MAKEBRH" ]]; then
        #    step_warn_msg "NOTE - simple version of remake_pair_target is used - no check for compression"
        #    remake_pair_target_makebrh "$1" "$2"
        #    return $?
        #fi
        #if [[ "$STEP_ID" == "MAKEINPAR" ]]; then
        #    step_warn_msg "NOTE - simple version of remake_pair_target is used - no check for compression"
        #    remake_pair_target_makeinpar "$1" "$2"
        #    return $?
        #fi
        #----------------------------------------------------------------
        # if no remake check, return ok
        [[ $SKIP_REMAKE_CHECK -eq 0 ]] || { debug_msg "remake_pair_target() skipped"; return 0; }

        # labels
        local A="$1"
        local B="$2"

        local equal=0
        [[ $A == $B ]] && equal=1

        # target and source
        local target=""
        local sources=""

        # string of missing files separated by ':'
        local missing=""
        local nmissing=0
        
        # error code
        local ec=0

        # variables used below
        local inp=""
        local fsA=""
        local fsB=""
        local fAA=""
        local fAB=""
        local fBA=""
        local dbA=""
        local dbB=""
        local brh=""
        local finpar=""
        local trgerr=0

        local allow_missing="0"
        # start checking per possible step id
        if   [[ "$STEP_ID" == "ALIGNMENT" ]]; then
            allow_missing="0"
            
            # source files
            fsA="${DIR_PROJECT_SEQUENCES}/${A}.${EXT_FASTA_SELECTED}"
            [[ -f $fsA ]] || { nmissing=$((nmissing+1)); missing="${missing}:$fsA"; }
            
            fsB="${DIR_PROJECT_SEQUENCES}/${B}.${EXT_FASTA_SELECTED}"
            [[ -f $fsB ]] || { nmissing=$((nmissing+1)); missing="${missing}:$fsB"; }

            debug_msg "$STEP_ID : source file A = $fsA"
            debug_msg "$STEP_ID : source file B = $fsB"
            
            # note DB files are wrt to non-compressed files, hence no extra extension
            local ndberr=0
            dbA="${DIR_PROJECT_SEQUENCES}/${A}.${EXT_FASTA_SELECTED}.$EXT_DB"
            [[ -f "$dbA" ]] || { nmissing=$((nmissing+1)); missing="${missing}:$dbA"; cstep_error_msg "missing DB files for A $fsA ; $dbA"; }
            dbB="${DIR_PROJECT_SEQUENCES}/${B}.${EXT_FASTA_SELECTED}.$EXT_DB"
            [[ -f "$dbB" ]] || { nmissing=$((nmissing+1)); missing="${missing}:$dbB"; cstep_error_msg "missing DB files for B $fsB ; $dbB"; }

            if (( ALIGNMENT_MATRIX == 0 )) || (( equal )); then
                # target pair file is alignment
                fAB="$DIR_PROJECT_PWC/$A/${A}_${B}.${EXT_ALIGN}"
            else
                fAB="$DIR_PROJECT_PWC/$A/${A}_${B}.${EXT_BRH}"
            fi
            
            # set source/target
            sources="${fsA}:${fsB}:${dbA}:${dbB}"
            target="$fAB"
        elif [[ "$STEP_ID" == "MAKEBRH" ]]; then
            if [[ "$A" == "$B" ]]; then
                # this can happen if TODO1 and TODO2 contains overlapping labels
                debug_msg "skip since A == B"
                return 1
            fi
            allow_missing=$IGNORE_MISSING
            # source files
            fBA=""
            fAB="$DIR_PROJECT_PWC/$A/${A}_${B}.${EXT_ALIGN}"
            [[ -f $fAB ]] || { nmissing=$((nmissing+1)); missing="${missing}:$fAB"; }
            
            if [[ $ALIGNMENT_MATRIX -eq 0 ]]; then
                fBA="$DIR_PROJECT_PWC/$B/${B}_${A}.${EXT_ALIGN}"
                [[ -f $fBA ]] || { nmissing=$((nmissing+1)); missing="${missing}:$fBA"; }
            else
                fsA="${DIR_PROJECT_SEQUENCES}/${A}.${EXT_FASTA_SEQLEN}"
                [[ -f $fsA ]] || { nmissing=$((nmissing+1)); missing="$missing:$fsA"; cstep_error_msg "missing from STATS step fasta sequence length file <$fsA>"; }
                fsB="${DIR_PROJECT_SEQUENCES}/${B}.${EXT_FASTA_SEQLEN}"
                [[ -f $fsB ]] || { nmissing=$((nmissing+1)); missing="$missing:$fsB"; cstep_error_msg "missing from STATS step fasta sequence length file <$fsB>"; }
            fi

            # source file(s)
            sources="${fAB}:$fBA"
            
            # target file
            target="$DIR_PROJECT_PWC/$A/${A}_${B}.${EXT_BRH}"

        elif [[ "$STEP_ID" == "MAKEINPAR" ]]; then
            if [[ "$A" == "$B" ]]; then
                # this can happen if TODO1 and TODO2 contains overlapping labels
                debug_msg "skip since A == B"
                return 1
            fi
            allow_missing=$IGNORE_MISSING
            fAB="$DIR_PROJECT_PWC/$A/${A}_${B}.${EXT_BRH}"
            [[ -f $fAB ]] || { nmissing=$((nmissing+1)); missing="${missing}:$(reduce_name $fAB)"; }
            
            fAA="$DIR_PROJECT_PWC/$A/${A}_${A}.${EXT_ALIGN}"
            [[ -f $fAA ]] || { nmissing=$((nmissing+1)); missing="${missing}:$(reduce_name $fAA)"; }
            
            fBB="$DIR_PROJECT_PWC/$B/${B}_${B}.${EXT_ALIGN}"
            [[ -f $fBB ]] || { nmissing=$((nmissing+1)); missing="${missing}:$(reduce_name $fBB)"; }
            
            sources="${fAB}:${fAA}:$fBB"
            
            # target file
            target="$DIR_PROJECT_PWC/$A/${A}_${B}.${EXT_INPAR}"
        else
            fatal_msg "remake_pair_target() got an invalid step id: $STEP_ID"
        fi
        
        if [[ $trgerr -ne 0 ]]; then
            debug_msg "target exists but with wrong compression <$target>"
            return 11
        fi

        # check if any files are missing or target is wrong
        if [[ ${#missing} -gt 0 ]]; then
            if [[ $allow_missing == "0"  ]]; then
                debug_msg "missing $nmissing source files: $missing"
                return 10
            fi
            info_msg "found $nmissing missing source file(s) - will ignore this and continue regardless"
            debug_msg "files: $missing ..."
        fi

        # debug messages
        debug_msg "remake sources: $sources"
        debug_msg "remake target : $target"

        # check if target needs remake
        if [[ ${#missing} -eq 0 ]]; then
            remake_target "$target" "$sources"
            ec=$?
            debug_msg "remake_target ec = $ec"
        else
            ec=1
            debug_msg "missing files found - ignore and return ec=$ec"
        fi
        return $ec
    }


    #-----------------------------------------------
    # like remake_target but it scans through all
    # pairwise compinations in PWC
    # done code:
    # - 0 : return 1 -> DONE, never remake
    # - 1 : check all PWC files wrt target
    # - 2 : return 0 -> always remake
    # - 3 : return 0 if target does not exist, 1 otherwise
    #-----------------------------------------------
    remake_target_from_pairwise() {
        # if nothing todo, quit!
        ###isempty PL_TODO_LABELS && return 255

        [[ $# -eq 2 ]] || fatal_msg "remake_target_from_pairwise: wrong nr of arguments <$@>"
        
        # arguments
        local target="$1"
        local ext="$2"


        # if no remake check, quit with ok status
        [[ $SKIP_REMAKE_CHECK -eq 0 ]] || { debug_msg "remake_target_from_pairwise() skipped"; return 0; }

        # never make if done code is 0
        [[ $STEP_DONE_CODE -eq 0 ]] && return 1

        # always make if done code is 2
        [[ $STEP_DONE_CODE -eq 2 ]] && return 0
        
        # check if target exists (done code 3)
        if [[ $STEP_DONE_CODE -eq 3 ]]; then
            # return 1 if target exists
            [[ -f "$target" ]] && return 1
            # return 0 if target does not exist
            return 0
        fi
        
        # set vars
        local foundany=0
        local ec=0
        local A=""
        local B=""
        local srcfile=""
        local fAB=""
        local fBA=""
        local indA=0
        local indB=0
        local m=""
        # loop over first label
        for (( indA=0; indA<PL_TODO_NLABELS; indA++ )); do
            # label of A
            A=${PL_TODO_LABELS[$indA]}

            # loop over second label
            # do the upper half matrix fo all possible pairs (excluding [B,A] and [A,A])
            indB=0
            for (( indB=indA+1; indB<PL_TODO_NLABELS; indB++ )); do
                # label of B
                B=${PL_TODO_LABELS[$indB]}

                # get BRH file name for [A,B] (try also [B,A] if prev does not exist)
                #HERE
                ec=0
                fAB="$DIR_PROJECT_PWC/$A/${A}_${B}.$ext"
                [[ -f $fAB ]] || {
                    fBA="$DIR_PROJECT_PWC/$B/${B}_${A}.$ext"
                    [[ -f $fBA ]] || ec=2
                }

                # if missing or problem with compression state, return error!
                if (( ec > 0 )); then
                    if (( IGNORE_MISSING == 0 )); then
                        error_msg "missing file $(trim_name $fAB) (and swapped)"
                        return 255
                    fi
                    # otherwise continue
                    continue
                fi

                # flag that at least one file was found
                foundany=1

                # check if src is newer than target and return 0 if it's true
                remake_target "$target" "$srcfile"
                ec=$?

                # remake target
                (( ec == 0 )) && return 0
            done
        done

        return $foundany
    }

    #-----------------------------------------------
    # Get median DB size for given fasta collection
    #-----------------------------------------------
    get_median_dbsz() {
        local sumfile="$DIR_PROJECT_SEQUENCES/${PL_TODO_NAME}_fasta_totlen.dat"
        [[ -f "$sumfile" ]] || return 255
        local mstr=""
        mstr=$(grep "^#MEDIAN" "$sumfile")
        [[ ${#mstr} -gt 0 ]] || return 255
        local tok=()
        tok=( $(tokenize " " "$mstr") )
        ###v_tokenize tok " " "$mstr"
        [[ ${#tok[@]} -eq 2 ]] || return 255
        echo "${tok[1]}"
    }

    #-----------------------------------------------
    # get effective DB size
    # valid sizes are
    #  -1 : get median DB size of all fasta files
    #   0 : use actual size
    #  >0 : use given fixed size
    #
    # The 'size' referred to above is the full length
    # of all the sequences together in the file,
    # including masked elements.
    #-----------------------------------------------
    get_effdbsz() {
        # NOTE IS NOW OBSOLETE
        echo 0
        return 0
        #------------------------
        #        local effdbsz=0
        #        if [[ $ALIGNMENT_EFFDBSZ -eq -1 ]]; then
        #            effdbsz=$(get_median_dbsz)
        #            [[ $? -eq 0 ]] || { warn_msg "unable to retrieve effective DB size - reverting to default"; effdbsz=$ALIGNMENT_EFFDBSZ; }
        #        else
        #            effdbsz=$ALIGNMENT_EFFDBSZ
        #        fi
        #        # if get_median_dbsz fails (no stats files), restore dbsz value (-1)
        #        [[ ${#effdbsz} -gt 0 ]] || effdbsz=$ALIGNMENT_EFFDBSZ
        #
        #        # make value integer
        #        effdbsz=$(calc "int($effdbsz)")
        #
        #        echo $effdbsz
        #        return 0
    }

    #-----------------------------------------------
    # check if label is OK wrt the range [LABELX_START,LABELX_END]
    #
    #-----------------------------------------------
    isok_label() {
        [[ $# -eq 2 ]] || return 0
        # test against label A or B?
        local what=$1
        what=${what^^}

        # label to test
        local L=$2

        # set range
        local use0=$OP_LABELA_FIRST
        local use1=$OP_LABELA_LAST
        if [[ $what == "B" ]]; then
            local use0=$OP_LABELB_FIRST
            local use1=$OP_LABELB_LAST
        fi

        [[ ${#use0} -gt 0 ]] && [[ "$L" < "$use0" ]] && { debug_msg "skipping A=$L"; return 1; }
        [[ ${#use1} -gt 0 ]] && [[ "$L" > "$use1" ]] && { debug_msg "skipping A=$L"; return 1; }
        return 0
    }


    #======================================================
    #
    # Below follows code for each processing step
    #
    # The template is:
    #
    # step_<step id>() {
    #   # set current step (sets STEP_ID)
    #   set_stepid "<step id>"
    #
    #   # check if step should be run
    #   check_done_flag || { step_done; return 0; }
    #
    #   # set source files (or actually script args) and job labels
    #   # NOTE: JOBLABELS should be of same size
    #   # as SRCFILES, or zero size.
    #   SRCFILES=( "${FASTA_BASENAMES[@]}" )
    #   JOBLABELS=( "${PL_TODO_LABELS[@]}" )
    #   JOBSUBLBL=( "${PL_TODO_LABELS[@]}" )
    #
    #   # set the _CHECK arrays - corresponds to labels to be error checked
    #   JOBLABELS_CHECK=( "${PL_TODO_LABELS[@]}" )
    #   JOBSUBLBL_CHECK=( "${PL_TODO_LABELS[@]}" )
    #
    #   # Announce the start of the step
    #   step_start ${#SRCFILES[@]}
    #
    #   # check if dry run
    #   is_dryrun && return 0
    #
    #   # run job
    #   run_cmd "<script>"
    #   local ec=$?
    #
    #   # check for errors
    #   check_error $ec "failed fasta full check"
    #   [[ $? -eq 0 ]] || return 1
    #
    #   # update done flag for step
    #   force_remake_none
    #
    #   return 0
    #
    # 
    # The step is run using run_cmd().
    # In order to include the flexibility of chosing
    # job submission engine, commands should be run:
    #
    #   run_cmd <job script>
    #
    # Effectively it will run like the following
    #
    #   njobs=0
    #   for s in srcfiles and lbl in labels
    #     run_job <script> <lbl> <s>
    #     njobs=njobs+1
    #     if njobs==OP_STEP_NPARALLEL[$STEP_ID] then
    #        wait until jobs are done
    #        njobs=0
    #     endif
    #   endfor
    #
    # The function run_job() will expand to a job submitter
    # according to the setting of OP_STEP_RUNMODE[STEP_ID].
    #
    #======================================================

    #-----------------------------------------------
    # Get (main) file extension
    #-----------------------------------------------
    get_extension() {
        [[ $# -eq 1 ]] || return 0
        local stepid="$1"
        local rval=""
        if [[ $stepid == PREPROC ]]; then
            rval=$EXT_FASTA_CLEAN
        elif [[ $stepid == MASKER ]]; then
            rval=$EXT_FASTA_MASKED
        elif [[ $stepid == SELECT ]]; then
            rval=$EXT_FASTA_SELECTED
        elif [[ $stepid == STATS ]]; then
            rval=$EXT_FASTA_STATS
        elif [[ $stepid == ALIGNMENT ]]; then
            rval=$EXT_ALIGN
        elif [[ $stepid == MAKEBRH ]]; then
            rval=$EXT_BRH
        elif [[ $stepid == MAKEINPAR ]]; then
            rval=$EXT_INPAR
        elif [[ $stepid == SETBRH ]]; then
            rval=$EXT_BRH
        elif [[ $stepid == SETINPAR ]]; then
            rval=$EXT_INPAR
        elif [[ $stepid == CLUSTER ]]; then
            rval=$EXT_CLUSTER_RAW
        elif [[ $stepid == MERGEINPAR ]]; then
            rval=$EXT_CLUSTER_INPAR
        elif [[ $stepid == MERGEINPARSEL ]]; then
            rval=$EXT_CLUSTER
        #elif [[ $stepid == WEAKBRH ]]; then
        #    rval=$EXT_WEAK_BRH
        fi
        echo $rval
        return 0
    }

    
    #-----------------------------------------------
    # set source/target for steps depending on
    # fasta files
    #-----------------------------------------------
    set_todo_source_target() {
        [[ $# -eq 2 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local srcext="$1"
        local trgext="$2"
        [[ ${#trgext} -gt 0 ]] || fatal_msg "invalid empty target extension (step $STEP_ID)"

        [[ ${srcext:0:1} == '.' ]] || srcext=".$srcext"
        [[ ${trgext:0:1} == '.' ]] || trgext=".$trgext"
        
        local x=
        local nf=${#FASTA_BASENAMES[@]}
        local ind=0
        local nmiss=0
        SRCFILES=()
        for ((ind=0; ind<nf; ind++)); do
            x="${FASTA_BASENAMES[$ind]}${srcext}"
            if [[ -f $x ]]; then
                SRCFILES[ind]="$x"
            else
                (( nmiss <  10 )) && error_msg "missing source file $x"
                (( nmiss == 10 )) && { error_msg "more than 10 missing source files..."; return 255; }
                nmiss=$((nmiss+1))
            fi
        done
        (( nmiss > 0 )) && { error_msg "missing $nmiss source file(s)"; return 255; }
        TRGFILES=()
        for ((ind=0; ind<nf; ind++)); do
            x="${FASTA_BASENAMES[$ind]}${trgext}"
            TRGFILES[ind]="$x"
        done
        return 0
    }

    #-----------------------------------------------
    # common init for all per-todo-label steps
    #
    # this will set the arrays
    #   SRCFILES == ( array of all selected sources )
    #   TRGFILES == ( array of all selected targets )
    #   JOBARGS  == ( array of all selected labels )
    #
    # JOBLABELS and JOBSUBLBL are reset to ().
    #
    # 
    #-----------------------------------------------
    init_step_per_todo_label() {
        #VERBOSE=1111
        [[ $# -eq 3 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local stepid="$1"
        local srcext="$2"
        local trgext="$3"
        # for CLUSTER, nothing needs to be done
        [[ "$stepid" == "CLUSTER" ]] && return 0
        verbose_info_msg "init per todo label for source ext: $srcext and target ext: $trgext"
        
        reset_joblists
        if ! is_step_done $stepid; then
            verbose_info_msg "set fasta file list"
            set_fasta_files || { error_msg "failed setting fasta files";         return 255; }
            verbose_info_msg "set source/target files"
            set_todo_source_target "$srcext" "$trgext" || { error_msg "failed setting source/target files"; return 255; }
            verbose_info_msg "nr of selected labels: ${SELECTED_LABELS[@]}"
            # set jobargs
            eval $(create_array_eol JOBARGS   "${SELECTED_LABELS[@]}")
            eval $(create_array_eol JOBLABELS "${SELECTED_LABELS[@]}")
            eval $(create_array_eol JOBSUBLBL "${SELECTED_LABELS[@]}")
        fi
        
        return 0
    }
    

    #-----------------------------------------------
    # get matrix type
    #
    #  'f' - full matrix
    #  'n' - half matrix, excluding diagonal
    #  'h' - half matrix, including diagonal
    #-----------------------------------------------
    get_matrix_type() {
        [[ $# -eq 1 ]] || return 255
        local stepid=$1
        local rval='f'
        if [[ $stepid == ALIGNMENT ]]; then
            rval='f'                                # full matrix
            (( ALIGNMENT_MATRIX == 1 )) && rval='h' # half matrix by request
        elif [[ $stepid == MAKEBRH ]]; then
            rval='n'
        elif [[ $stepid == MAKEINPAR ]]; then
            rval='n'
        elif [[ $stepid == SETBRH ]]; then
            rval='n'
        elif [[ $stepid == SETINPAR ]]; then
            rval='n'
        fi
        echo $rval
    }

    #-----------------------------------------------
    # Set pairwise labels name
    # GENERATED_PAIR_FILE       - pair file name
    # GENERATED_PAIR_FILE_OPTS  - options if it does not exist
    # The options are for the script generate_pairs.py
    #-----------------------------------------------
    set_pairwise_file_name() {
        [[ $# -eq 4 ]] || fatal_msg_wrong_nargs "$#" "$FUNCNAME"
        local stepid="$1"
        local label="$2"
        local usemods="$3"
        local use_pairs_tree="$4"

        # set possible todo files
        local ftodo="$DIR_PROJECT_TODO/${label}.${EXT_TODO}"
        local ftodo1="$PL_TODO1"
        local ftodo2="$PL_TODO2"

        # options and pair filename
        GENERATED_PAIR_FILE_OPTS=""
        GENERATED_PAIR_FILE=""

        # possibly use pairs tree
        if (( use_pairs_tree == 1 )); then
            # TREE mode - use EXT_PAIRS_TREE which was generated in the PREPARETREE step
            if [[ -n $label ]]; then
                # use original pair file
                GENERATED_PAIR_FILE="$DIR_PROJECT_TODO/${label}.${EXT_PAIRS_TREE}"
                return 0
            fi
            # if no label given -> error
            error_msg "failed setting pair file name - tree mode enabled but label is not set"
            return 255
        fi

        # TREE mode is NOT active
        
        # Check what combination of todo file(s) is given
        if [[ -f "$ftodo2" ]]; then
            [[ -f "$ftodo1" ]] || ftodo1="$ftodo"
            ftodo=""
        fi

        # if only todo file is given, add to options
        if [[ -f "$ftodo" ]]; then
            GENERATED_PAIR_FILE_OPTS="${GENERATED_PAIR_FILE_OPTS} -t $ftodo -m $(get_matrix_type $stepid)"
            ftodo1=""
            ftodo2=""
            debug_msg "generate pairwise file using todo: $ftodo"
        else
            if [[ ! -f "$ftodo1" ]] && [[ ! -f "$ftodo2" ]]; then
                error_msg "failed setting pair file name - not enough valid info"
                error_msg "label    : $label"
                error_msg "ftodo1   : $ftodo1"
                error_msg "ftodo2   : $ftodo2"
                return 255
            fi
            ftodo=""
            label=$(basename "$PL_TODO1" ".$EXT_TODO")
            GENERATED_PAIR_FILE_OPTS="${GENERATED_PAIR_FILE_OPTS} -t $ftodo1 -e $ftodo2"
            if (( PL_TODO_ONE2ONE == 0 )); then
                # set matrix type - 'half' will be ignored and set to 'full'
                # type 'nodiag' will exclude self-pairs from todo1 and todo2
                GENERATED_PAIR_FILE_OPTS="${GENERATED_PAIR_FILE_OPTS} -m $(get_matrix_type $stepid)"
            else
                # one2one -> ignores matrix type, hence no -m option
                GENERATED_PAIR_FILE_OPTS="${GENERATED_PAIR_FILE_OPTS} --one2one"
            fi

            debug_msg "generate pairwise file using todo1: $ftodo1 and todo2: $ftodo2"   
        fi
        
        # now either ftodo or both ftodo1+ftodo2 valid file names
        
        # base name of fpairs
        local fpairs="$DIR_PROJECT_TODO/${label}.${EXT_PAIRS}"

        # check if modificators should be used
        # and if so, check if they exist
        if (( usemods == 1 )); then
            # check for range modifiers and add to OPTS which are used when calliing generate_pairs.py
            [[ -n $OP_LABELA_LAST$OP_LABELA_FIRST ]] && { GENERATED_PAIR_FILE_OPTS="${GENERATED_PAIR_FILE_OPTS} -A ${OP_LABELA_FIRST},${OP_LABELA_LAST}"; }
            [[ -n $OP_LABELB_LAST$OP_LABELB_FIRST ]] && { GENERATED_PAIR_FILE_OPTS="${GENERATED_PAIR_FILE_OPTS} -B ${OP_LABELB_FIRST},${OP_LABELB_LAST}"; }
            [[ -n $PL_TODO_LABELS_REQ             ]] && { GENERATED_PAIR_FILE_OPTS="${GENERATED_PAIR_FILE_OPTS} -r $PL_TODO_LABELS_REQ"; }
        fi

        # create hashed pair file
        GENERATED_PAIR_FILE_HEADER="# Todo            : $ftodo\n \
            # Todo1           : $PL_TODO1\n \
            # Todo2           : $PL_TODO2\n \
            # One2one         : $PL_TODO_ONE2ONE\n \
            # Pairs           : $fpairs\n \
            # Range A         : [${OP_LABELA_FIRST},${OP_LABELA_LAST}]\n \
            # Range B         : [${OP_LABELB_FIRST},${OP_LABELB_LAST}]\n \
            # Required Labels : $PL_TODO_LABELS_REQ\n \
            # Matrix type     : $(get_matrix_type $stepid)"
        local hash=$(echo $GENERATED_PAIR_FILE_HEADER | md5sum | cut -f 1 -d ' ')

        # set generated file name
        GENERATED_PAIR_FILE="${fpairs}_${hash}"
        return 0
    }

    #-----------------------------------------------
    # Create a pairwise file based on
    # * todo file
    # * pair file (optional)
    # * second todo file (optional)
    # * range A, range B
    # * required key(s)
    #-----------------------------------------------
    create_pairwise_file() {
        [[ $# -eq 4 ]] || fatal_msg_wrong_nargs "$#" "$FUNCNAME"
        is_dryrun && return 0
        local stepid="$1"
        local label="$2"
        local usemods="$3"
        local use_pairs_tree="$4"
        
        # set pairwise file
        # if pair file already exists and no constraints -> this will be just the $PAIR_FILE
        set_pairwise_file_name "$stepid" "$label" "$usemods" "$use_pairs_tree" || {
            if is_dryrun; then
                debug_msg "unable to establish a pairs filename"
                return 0
            fi
            step_error_msg "unable to establish a pairs filename"
            return 255
        }

        # if it exists, nothing to do!
        if [[ -f "$GENERATED_PAIR_FILE" ]]; then
            step_info_msg "(generated) pair file already exists: $(trim_name $GENERATED_PAIR_FILE)"
            return 0
        fi

        # build generated file
        VERBOSE=111111
        debug_msg "generate cmd: $DIR_ORTHOLOGER/bin/generate_pairs.py $GENERATED_PAIR_FILE_OPTS"
        VERBOSE=0
        $DIR_ORTHOLOGER/bin/generate_pairs.py $GENERATED_PAIR_FILE_OPTS > "$GENERATED_PAIR_FILE"
        [[ $? -eq 0 ]] || {
            step_error_msg "failed generating pair file, setup:"
            echo -e "$GENERATED_PAIR_FILE_HEADER" 1>&2
            rm -f "$GENERATED_PAIR_FILE"
            return 255
        }
        step_info_msg "Generated pairwise file at $(trim_name $GENERATED_PAIR_FILE)"
    }

    #-----------------------------------------------
    # merge fasta name
    #-----------------------------------------------
    set_merged_fasta_name() {
        MERGED_LABEL=ALL_${PL_TODO_NAME}
        MERGED_FASTA=$DIR_PROJECT_SEQUENCES/${MERGED_LABEL}.fs_selected
    }
    
    #-----------------------------------------------
    # Run alignment on all fasta merged into one.
    # That is, run only A vs A and then split it into
    # individual per species pair alignments.
    # As a consequence there will be files for the
    # whole matrix.
    #-----------------------------------------------
    step_alignment_file_merge() {
        step_info_msg "---------------------------------------------------------"
        step_info_msg "- will run alignment on all fasta files merged into one -"
        step_info_msg "---------------------------------------------------------"
        local ec=0

        reset_joblists
        
        step_start 1

        #-----------------------------------------------
        # first create merged fasta file if needed
        #-----------------------------------------------
        
        # set merged filename
        set_merged_fasta_name

        step_info_msg "merged fasta file label: $MERGED_LABEL"
        
        # dry run -> return
        is_dryrun && return 0

        local T0=$(time_nsec)
        local ec=0

        # make sure pairwise dirs are there
        mkdir -p $DIR_PROJECT_PWC/$MERGED_LABEL || {
            step_error_msg "unable to create dir for label at $DIR_PROJECT/PWC"
            return 255
        }
        mkdir -p $DIR_PROJECT_JOBLOG/$MERGED_LABEL || {
            step_error_msg "unable to create dir for label at $DIR_PROJECT/$BASE_JOBLOG"
            return 255
        }

        # check if merged fasta file should be created
        remake_target $MERGED_FASTA
        ec=$?

        if (( ec == 0 )); then
           # create merged fasta
           step_info_msg "create merged fasta file"
           local label=
           local nerr=0
           {
               for label in ${PL_TODO_LABELS[@]}; do
                   local f=$DIR_PROJECT_SEQUENCES/${label}.$EXT_FASTA_SELECTED
                   if [[ -f $f ]]; then
                       grep -v '#' $f
                   else
                       echo "#MISSING: $f"
                       nerr=$((nerr+1))
                   fi
               done
               echo '#END'
           } > $MERGED_FASTA
           if (( nerr > 0 )); then
               step_error_msg "missing $nerr fasta file(s) for merging"
               return 255
           fi
        else
            step_info_msg "skip creating merged fasta file, ec=$ec"
        fi

        if [[ ! -f $MERGED_FASTA ]]; then
            step_error_msg "missing merged fasta file at $MERGED_FASTA"
            step_error_msg "make sure it is remade by setting the done code (2)"
        fi
        #-----------------------------------------------
        # run formatdb on merged fasta
        #-----------------------------------------------
        step_info_msg "run formatdb on label: $MERGED_LABEL"
        JOBARGS=("$MERGED_LABEL")
        JOBLABELS=("$MERGED_LABEL")
        JOBSUBLBL=("$MERGED_LABEL")
        run_cmd "$(get_run_script FORMATDB)"
        ec=$?
        if (( ec != 0 )); then
            step_error_msg "failed running formatdb on $MERGED_LABEL"
            return 255
        fi
        
        #-----------------------------------------------
        # align merged fasta
        #-----------------------------------------------
        # run full alignment
        step_info_msg "run alignment on label: $MERGED_LABEL"
        JOBARGS=("$MERGED_LABEL,$MERGED_LABEL,1")
        
        # run alignment - note it will not create any BRH's as it only aligns A vs A
        run_cmd "$(get_run_script ALIGNMENT)"
        ec=$?
        if (( ec != 0 )); then
            step_error_msg "failed running alignment on $MERGED_LABEL"
            return 255
        fi

        #-----------------------------------------------
        # split alignment result into pairs
        #-----------------------------------------------
        step_info_msg "split alignment into per species pairs, label: $MERGED_LABEL"
        local fali=$DIR_PROJECT/PWC/$MERGED_LABEL/${MERGED_LABEL}_${MERGED_LABEL}.$EXT_ALIGN
        [[ -f $fali ]] || {
            step_error_msg "missing alignment file at $fali"
            return 255
        }
        $PYTHON3 $DIR_ORTHOLOGER/bin/split_global_align.py --file "$fali" -d "$DIR_PROJECT_PWC" || {
            step_error_msg "failed splitting merged alignment file at $fali"
            return 255
        }

        local T1=$(time_nsec)
        step_timing $T0 $T1 $naddedTot
        return 0
    }

    #-----------------------------------------------
    # Run alignments on pairs defined by pair file.
    # See GENERATED_PAIR_FILE.
    # The filename is defined in: set_pairwise_file_name()
    # It is creader in: create_pairwise_file()
    #-----------------------------------------------
    step_pairwise_file() {
        [[ $# -eq 1 ]] || fatal_msg_wrong_nargs "$#" "$FUNCNAME"
        local stepid="$1"
        local ec=0

        # check for pair file
        if ! is_dryrun; then
            [[ -f $GENERATED_PAIR_FILE ]] || { error_msg "pair file not found $GENERATED_PAIR_FILE"; return 255; }
        fi
        
        local T0=$(time_nsec)
        
        debug_msg "starting pairwise step for $stepid"
        
        # reset job lists
        reset_joblists

	    # initialise pairwise run
        PW_SCRIPT=$(get_run_script $stepid)
        if is_dryrun; then
            PW_NPAIRS=0
        else
            PW_NPAIRS=$(grep -c -v "^#" $GENERATED_PAIR_FILE)
        fi
        
        # announce start of step
        step_start $PW_NPAIRS

        # check if dry run
        is_dryrun && return 0

	    # check if no pairs - return
	    [[ $PW_NPAIRS -gt 0 ]] || return 0
	    
        # job submission/threads
        local nsubmax=$OP_NJOBMAX_BATCH
        
        # br of threads per alignement
        local nthreads=${OP_STEP_NTHREADS[$STEP_ID]}

        # max nr of merged pairs
        local maxnmerge=${OP_STEP_NMERGE[$STEP_ID]}
        (( maxnmerge > 0 )) || maxnmerge=1
        
        # counters/flags
        local nproc=0     # nr of processed pairs
        local nadded=0    # nr of added pairs per job
        
        local nmerged=0   # nr of merged pairs into one job
        local nsub=0      # nr of submitted jobs
        local nfailed=0   # nr of submitted jobs that failed
        local naddedTot=0 # nr of added pairs, accumulated
        
        local perc=0      # current percentage done
        local founderr=0  # an error is found
        local lastA=0     # last A label
        local lastAB=0    # last A,B pair
        local allmerged=0 # flag when last merged is added to a job

        # batch flag
        local isbatch=0
        is_runmode_batch && isbatch=1
        
        local nperc=0     # counter for percentage output
        local noneperc=$(((PW_NPAIRS+100)/100)) # nr of alignments corresponding to one percent
        [[ $noneperc -gt 0 ]] || noneperc=1
        
        # start time of loop
        local totTrun=0
        local Tchk0=0
        local dTchk=0
        local totTchk=0
        
        # indices and labels for A and B
        local A=""
        local B=""

        # various vars in loop
        local addPair=0
        local njobs=0
        local allDone=0

        [[ -n $OP_LABELA_LAST   ]] && lastLblA=$OP_LABELA_LAST
        [[ -n $OP_LABELB_LAST   ]] && lastLblB=$OP_LABELB_LAST

        step_info_msg "will generate pair wise using pair file: $(trim_name $GENERATED_PAIR_FILE)"
        step_info_msg "ignore missing: $IGNORE_MISSING"
        
        Tchk0=$(time_nsec)
        
        local saveIFS="$IFS"
        local line=""

        ###while read line; do
        until (( allDone==1 )); do
            IFS=$'\n'
            read line || allDone=1
            IFS=$saveIFS

            if (( allDone==0 )); then
                
                # skip comment and empty lines
                [[ ${line:0:1} == '#' ]] && continue
                [[ -n $line ]] || continue

                # set IFS to single space
                IFS=' '
                # process line
                local elems=( $line )
                A=${elems[0]}
                B=${elems[1]}

                nproc=$((nproc+1))
                
                addPair=0
                #
                # this will check if all source files are there
                # returns:
                #  0   : all exists
                #  1   : some are missing, ignore this error 
                # 10+  : some are missing, error"
                remake_pair_target $A $B
                ec=$?
                if [[ $ec -eq 0 ]]; then
                    # target should be made -> add pair
                    addPair=1
                else
                    # ec>0 -> target should not be made
                    addPair=0
                    # ec>=10 -> error: missing source file(s)
                    [[ $ec -ge 10 ]] && {
                        if (( founderr < 10 )); then
                            if (( IGNORE_MISSING == 0 )); then
                                step_error_msg "missing file for labels $A and $B"
                            else
                                step_warn_msg "ignoring missing file for labels $A and $B"
                            fi
                        fi
                        founderr=$((founderr+1))
                    }
                fi
                
                if (( addPair == 1 )); then
                    # add pair to JOBARGS
                    # if nmerged>0 then the pair is added to the last one in JOBARGS
                    # if nmerged==0 then a new element is added in JOBARGS
                    add_jobargs "$A,$B,$nthreads" $nmerged "${A}-${B}" "$A"

                    nadded=$((nadded+1))
                    nmerged=$((nmerged+1))
                    nperc=$((nperc+1))
                    if (( nmerged == maxnmerge )); then
                        debug_msg "($A,$B) merged $maxnmerge jobs into one"
                        allmerged=1
                        nmerged=0
                    fi
                    # calc perentage done
                    if [[ $nperc -ge $noneperc ]]; then
                        perc=$(calc "int(100.0*$nproc/$PW_NPAIRS)")
                        printf -v msgstr "processed %8s (%3s%%) and added %6s/max %6s, njobs = %s" "$nproc" "$perc" "$nadded" "$PW_NPAIRS" "${#JOBARGS[@]}"
                        MSG_PREFIX="($A,$B)"
                        step_info_msg "$msgstr"
                        MSG_PREFIX=""
                        nperc=0
                    fi

                fi
            fi
            # nr of job definitions collected
            njobs=${#JOBARGS[@]}
            # check if submit condition is reached
            # - allDone==1 OR
            # - njobs==nsubmax AND allmerged
            dosubmit=$allDone
            if (( dosubmit==0 )) && (( allmerged==1 )) && (( njobs==nsubmax )); then
                dosubmit=1
            fi
            
            # check if dosubmit
            if (( dosubmit==1 )); then
                # if no jobs, quit
                (( njobs > 0 )) || continue

                # nr of submissions
                nsub=$((nsub+1))

                # current label (pair)
                MSG_PREFIX="$(printf "%-22s" "(${A},${B})")"
                
                # timer
                dTchk=$(delta_time $Tchk0)

                totTchk=$(sum_time "$totTchk" "$dTchk")
                
                # dump check time
                step_info_msg "done scanning after $dTchk secs"
                
                # start jobs
                step_info_msg "starting $njobs job(s) with in total $nadded pair(s)"

                # run the jobs
                dispatch_cmd "$PW_SCRIPT" || nfailed=$((nfailed+1))
                
                # summarize the timings
                step_timing_dispatch $nadded "rate (secs/pair):"

                #check_error $ec "failed pairwise job"
                #ec=$?
                
                # accumulate run time
                totTrun=$( calc "$totTrun+$T_RUN_JOBS" )

                # total nr of added pairs
                naddedTot=$((naddedTot+nadded))
                
                # clear job arrays
                reset_joblists
                
                # reset nr of pair counter
                nadded=0
                nmerged=0

                # for next check
                Tchk0=$(time_nsec)

                if (( SIGNAL_SOFT_ABORT==1 )); then
                    local sep="$(print_separator)"
                    step_info_msg $sep
                    step_info_msg "abort requested!"
                    step_info_msg "last pair processed: $A $B"
                    step_info_msg $sep
                    exit 0
                fi
            fi
            
            # restore IFS to newline
            IFS=$'\n'
            MSG_PREFIX=""
            
        done < $GENERATED_PAIR_FILE
        ####<(sort -u $PAIR_FILE && echo "@ENDOFPAIRS")

        # restore IFS
        IFS="$saveIFS"


        #----------------------------------------------------------------
        # dump accumulated times
        totTrun=$( calc "$totTrun*1.0e-9" )
        local tmstr=""
        printf -v tmstr "N(submit jobs) = %d , N(submit fail) = %d , N(pairs) = %d/%d , T(chk/run) = %.2f/%.2f\n" $nsub $nfailed $naddedTot $nproc $totTchk $totTrun
        step_info_msg "$tmstr"
        step_info_msg "submitted $nsub job(s) with $nfailed failed submission(s)"

        # clear job arrays
        reset_joblists

        # check for bad results
        if [[ $nfailed -gt 0 ]]; then
            step_error_msg "failed submitting $nfailed job(s)"
            force_remake_missing
            return 3
        fi

        # flag as DONE if rno errors were found
        if [[ $IGNORE_MISSING == 1 ]]; then
            [[ $founderr -eq 0 ]] || step_warn_msg "found $founderr missing files (see above) -> continuing anyway as IGNORE_MISSING is set" 1
        else
            [[ $founderr -eq 0 ]] || { step_error_msg "found $founderr errors (see above) -> aborting run"; return 255; }
        fi
        local T1=$(time_nsec)
        step_timing $T0 $T1 $naddedTot
        step_done

        if [[ "$STEP_ID" == "ALIGNMENT" ]] && (( ALIGNMENT_MATRIX == 0 )) && (( ODB_ALIGN_MERGED != 1 )); then
            # alignment done with BRH only if
            # 1. step is alignment
            # 2. alignment matrix is set to 'half'
            # 3. alignment is NOT done on the merged fasta file 
            step_done MAKEBRH
        fi
        
        ###profile_stop_timing 00FULL
        ###profile_report
        return 0
    }


    #-----------------------------------------------
    # generic pairwise step
    #-----------------------------------------------
    step_pairwise() {
        [[ $# -eq 1 ]] || fatal_msg_wrong_nargs "$#" "$FUNCNAME"

        # set/create pairwise file
        #    "$1"   -> step id
        #    "PL_TODO_NAME" -> main todo label, may use PL_TODO1/2
        #    "1"    -> flag that modificators should be used
        #    "0"    -> assume no tree (dont use EXT_PAIRS_TREE)
        create_pairwise_file "$1" "$PL_TODO_NAME" 1 0 || { step_error_msg "unable to create pairwise file"; return 255; }
        step_pairwise_file   "$1"   || { step_error_msg "failed running pairwise step";   return 255; }
    }

    #-----------------------------------------------
    # CLUSTER TREE
    # create newick tree for given root in PL_TODO
    #-----------------------------------------------
    tree_create() {
        # default orthodb tree file name
        local odbtree="orthodb_${API_ORTHODB_VERSION_NODOT}.nw"

        # if step is PREPARETREE, then take care of done code MAKE_ALL
        if [[ "$STEP_ID" == "PREPARETREE" ]] && [[ -f "$TREE_INPUT_FILE" ]]; then
            if [[ $(basename "$TREE_INPUT_FILE") == "$odbtree" ]]; then
                local dc=$(get_done_code)
                if [[ "$dc" == "$DC_MAKE_ALL" ]]; then
                    # rm file -> force recreation of the file
                    # NOTE: only if the tree is an OrthoDB tree
                    # NOTE: it's determined by name only...
                    step_info_msg "removing tree file: $TREE_INPUT_FILE"
                    step_info_msg "will be recreated later"
                    rm -f "$TREE_INPUT_FILE"
                fi
            fi
        fi
        [[ -n $TREE_INPUT_FILE ]] && [[ -f $TREE_INPUT_FILE ]] && {
            step_info_msg "tree file already exists: $TREE_INPUT_FILE"
            return 0
        }
        #---------------------------------------------
        # No valid input tree - use
        TREE_INPUT_FILE="$DIR_PROJECT_TODO/$odbtree"
        step_info_msg "create tree from OrthoDB at root $TREE_ROOT_CLADE, url $TREE_INPUT_ODB"


        if [[ -f "$TREE_INPUT_FILE" ]]; then
            step_info_msg "tree file exists: $TREE_INPUT_FILE"
        else
            step_info_msg "generating OrthoDB tree at level $root"
            step_info_msg "tree url  : $TREE_INPUT_ODB"
            step_info_msg "tree file : $TREE_INPUT_FILE"

            # get tree, include leaves (ie species taxids)
            local root=$(file_basename "$TREE_INPUT_FILE")
            local nwstr=
            nwstr=$(print_newick_odb "$TREE_INPUT_ODB" "$root")
            [[ $? -eq 0 ]] || {
                rm -f $TREE_INPUT_FILE >& /dev/null
                step_error_msg "unable to create tree for label <$root>"
                step_error_msg "command used: $cmd -n -u $TREE_INPUT_ODB -r $root --keep"
                step_error_msg "note that the label must be a valid taxid present in the OrthoDB tree"
                step_error_msg "this error implies that the taxid in NOT present"
                return 255
            }
            [[ "${nwstr:0:-1}" == ";" ]] || nwstr="${nwstr};"
            echo "$nwstr" > "$TREE_INPUT_FILE"
        fi
        return 0
    }

    #-----------------------------------------------
    # CLUSTER TREE
    # create todo files from tree
    #-----------------------------------------------    
    tree_create_todos() {
        is_dryrun && return 0

        [[ -f $TREE_INPUT_FILE ]] || {
            step_error_msg "cannot find tree at $TREE_INPUT_FILE"
            step_error_msg "tree_create_todos() must be run before tree_create_todos()"
            return 255
        }
        local nfound=0
        local ncreated=0
        local nskip=0
        local x=

        # retrieve all internal nodes
        get_internal_nodes || {
            step_error_msg "tree_create_todo: failed retrieving all internal nodes from $TREE_INPUT_FILE"
            return 255
        }
        for x in ${_NODES[@]}; do
            local f=${DIR_PROJECT_TODO}/${x}.$EXT_TODO
            nfound=$((nfound+1))
            if [[ "$STEP_ID" == "PREPARETREE" ]]; then
                remake_target "$f" "$TREE_INPUT_FILE"
                [[ $? -eq 0 ]] || {
                    step_info_msg "skip - file already exists: $(trim_name $f)"
                    nskip=$((nskip+1))
                    continue
                }
            else
                [[ -f $f ]] && {
                    nskip=$((nskip+1))
                    step_info_msg "skip - file already exists: $(trim_name $f)"
                    continue
                }
            fi
            step_info_msg "generating: $(trim_name $f)"
            print_leaf_organisms "$TREE_INPUT_FILE" "$x" | sort -u > "$f"
            [[ $? -eq 0 ]] || {
                rm -f $f >& /dev/null
                step_error_msg "unable to create todo file $(trim_name $f)"
                return 255
            }
            ncreated=$((ncreated+1))
        done
        step_info_msg "created $ncreated pair files and skipped $nskip , out of $nfound possible"
        return 0        
    }
    
    #-----------------------------------------------
    # CLUSTER TREE
    # create pair files for each node (non-leaf)
    # the pair file contains pairs of taxids to be
    # considered in the clusteriong of the node
    #-----------------------------------------------
    tree_create_pairs() {
        is_dryrun && return 0
        
        # check args
        local onlyRoot="0"
        [[ $# -ge 1 ]] && onlyRoot="1"
        
        # check tree file
        [[ -f $TREE_INPUT_FILE ]] || {
            step_error_msg "expected tree file is not readable: $TREE_INPUT_FILE"
            return 255
        }
        local dirtodo=$(dirname $PL_TODO)
        
        # get all nodes and generate pair files
        # loop over nodes and generate a pair file for each node
        # local fpairs=${fnode/.$EXT_TODO/.$EXT_TREE}
        local ncreated=0
        local ntot=0
        local nskip=0
        local nodedef=()
        local nodes=()
        local ec=0
        local args=""
        [[ "$onlyRoot" == "1" ]] && args="$PL_TODO_NAME"
        local saveIFS="$IFS"

        # get_orthodb_tree.py returns one line per node consisting of
        #   <root>,<child1>,<child2> ....
        get_branches "$TREE_INPUT_FILE" "$args" || {
            step_error_msg "tree_create_pairs: failed finding nodes in tree $(trim_name $TREE_INPUT_FILE) - missing file or wrong format (should be newick)?"
            return 255
        }
        local line=
        step_info_msg "generating pairs for all nodes (including leaves)"
        for line in ${_NODES[@]}; do
            # each line is comma separated <root>,<child1>,<child2>, ...
            IFS=','
            eval nodes=( $line )
            IFS="$saveIFS"
            # skip lines with only one node
            ###[[ ${#nodes[@]} -gt 1 ]] || continue
            ntot=$((ntot+1))
            local root=$dirtodo/${nodes[0]}.$EXT_TODO
            local fpairs=${root/.$EXT_TODO/.$EXT_PAIRS_TREE}

            if [[ "$STEP_ID" == "PREPARETREE" ]]; then
                remake_target "$fpairs" "$TREE_INPUT_FILE"
                [[ $? -eq 0 ]] || {
                    nskip=$((nskip+1))
                    continue
                }
            else
                [[ -f $fpairs ]] && {
                    nskip=$((nskip+1))
                    step_info_msg "skip - file already exists: $(trim_name $fpairs)"
                    continue
                }
            fi
            
            step_info_msg "generating $(trim_name $fpairs)"
            
            ##remake_target "$fpairs" "$TREE_INPUT_FILE"
            ##[[ $? -eq 0 ]] || { continue; }

            ###arg="-r $root"
            args="--todo $root"
            [[ ${#nodes[@]} -gt 1 ]] && args="$args -c "
            local nv=
            for nv in ${nodes[@]:1}; do
                args="$args $dirtodo/${nv}.$EXT_TODO"
            done

            ###eval $PYTHON3 $DIR_ORTHOLOGER/bin/get_pairs.py $arg > $fpairs
            debug_msg "generate pairs: generate_pairs.py $args > $(basename $fpairs)"
            eval $PYTHON3 $DIR_ORTHOLOGER/bin/generate_pairs.py $args > $fpairs

            ncreated=$((ncreated+1))
        done
        IFS="$saveIFS"
        
        step_info_msg "created $ncreated pair files and skipped $nskip , out of $ntot possible"
        return 0
    }
    
    #-----------------------------------------------
    # PREPARETREE
    # create files related to tree clustering
    #-----------------------------------------------
    step_preparetree() {
        local ec=0
        ###init_step "PREPARETREE"
        ###ec=$?
        # step is disabled?
        #[[ $ec -eq 1   ]] && return 0
        # step id done?
        #[[ $ec -eq 2   ]] && return 0
        # step dependancy?
        #[[ $ec -eq 255 ]] && return 255

        # flag start of step
        step_start 1
        
        # check if dry run
        is_dryrun && return 0

        local T0=$(time_nsec)
        step_info_msg "tree: prepare tree clustering"
        init_tree || return 255
        local T1=$(time_nsec)
        step_timing $T0 $T1 1
        step_done
        return 0
    }
    
    #-----------------------------------------------
    # PREPROC
    # Input : fasta files
    # Output: cleaned/processed fasta files
    #-----------------------------------------------
    step_preproc() {
        is_dryrun || init_step_per_todo_label "PREPROC" "$EXT_FASTA" "$EXT_FASTA_CLEAN" || return 255
        step_per_todo_label "PREPROC" "$(get_run_script PREPROC)"
    }

    #-----------------------------------------------
    # MASKER
    # Input : <species>.<EXT_FASTA>
    # Output: <species>.<EXT_FASTA_MASKED>
    #-----------------------------------------------
    step_masker() {
        # warn if NONE is used
        if ! is_dryrun; then
            [[ "$MASKER_LABEL" == "NONE" ]] && step_warn_msg "will run without sequence masking"
            init_step_per_todo_label "MASKER" "$EXT_FASTA_CLEAN" "$EXT_FASTA_MASKED" || return 255
        fi
        step_per_todo_label "MASKER" "$(get_run_script MASKER)"
    }

    #-----------------------------------------------
    # SELECT
    # Input : <species>.<EXT_FASTA_MASKED>
    # Output: <species>.<EXT_FASTA_SELECTED> (.clstr)
    #-----------------------------------------------
    step_select() {
        # warn if NONE is used
        if [[ "$SELECT_LABEL" == "NONE" ]]; then
            step_warn_msg "will run without filtering very similar genes"
            step_warn_msg "it is recommended to set SELECT_LABEL=\"CDHIT\""
        fi
        is_dryrun || init_step_per_todo_label "SELECT" "$EXT_FASTA_MASKED" "$EXT_FASTA_SELECTED" || return 255
        step_per_todo_label "SELECT" "$(get_run_script SELECT)"
    }

    #-----------------------------------------------
    # STATS
    # Input : <species>.<EXT_FASTA_SEQLEN>
    # Output: <species>.<EXT_FASTA_STATS>
    #         <todo_label>_fasta_totlen.dat
    #-----------------------------------------------
    step_stats() {
        tmp_setup_trap
        
        local old_dc=$(get_done_code "STATS")
        
        # init step
        is_dryrun || init_step_per_todo_label "STATS" "${EXT_FASTA_SELECTED}" "${EXT_FASTA_STATS}" || return 255
        
        # run stats per todo label
        step_per_todo_label "STATS" "$(get_run_script STATS)"
        [[ $? -eq 0 ]] || return 255

        # skip the rest if dry run
        is_dryrun && return 0

        # restore done code from before
        update_done_file $old_dc

        # if step is done - skip the following
        is_step_done && return 0
        
        # create length summary
        local nval=0
        local sumfile="$DIR_PROJECT_SEQUENCES/${PL_TODO_NAME}_fasta_totlen.dat"

        # check if sumfile should be remade
        remake_target "$sumfile" || return 0

        local tmpfile=""
        tmp_make tmpfile "statsum"
        {
            local N=0
            local A=""
            local f=""
            local indA=0
            echo "# Label TOTLEN"
            #for A in "${PL_TODO_LABELS[@]}"; do -> not IFS safe
            for (( indA=0; indA<PL_TODO_NLABELS; indA++ )); do
                A=${PL_TODO_LABELS[$indA]}
                has_required_label $A || continue
                f="$DIR_PROJECT_SEQUENCES/${A}.${EXT_FASTA_STATS}"
                [[ -f "$f" ]] || { warn_msg "missing file for stats <$f>"; continue; }
                N=$(grep "#TOTLEN" "$f" | cut -d ":" -f 2)
                [[ "$N" == "-1" ]] || { echo $A $N; nval=$((nval+1)); }
            done
        } > "$tmpfile"
        
        # if empty return error
        (( nval > 0 )) || return 255

        # append the median
        {
            cat "$tmpfile"
            if (( nval > 0 )); then
                local median=$($PYTHON3 "$DIR_ORTHOLOGER_BIN/stats.py" -f "$tmpfile" -c 1 -M)
                echo "#MEDIAN $median"
            else
                echo "#MEDIAN NA"
                error_msg "no stats to get median!"
            fi
        } > "$sumfile"
        
        # copy file
        chmod 644 "$sumfile"

        # mark step as done
        step_done
        return 0
    }

    #-----------------------------------------------
    # MAKEINPARSEL
    # Input :
    # Output:
    #-----------------------------------------------
    step_makeinparsel() {

        is_dryrun || init_step_per_todo_label "MAKEINPARSEL" "$EXT_FASTA_SELCLSTR" "$EXT_INPAR_SELECTED" || return 255
        step_per_todo_label "MAKEINPARSEL" "$(get_run_script MAKEINPARSEL)"
    }

    #-----------------------------------------------
    # FORMATDB
    # Input :
    # Output:
    #-----------------------------------------------
    step_formatdb() {
        local ec=0
        is_dryrun || init_step_per_todo_label "FORMATDB" "$EXT_FASTA_SELECTED" "${EXT_FASTA_SELECTED}.${EXT_DB}"
        if (( ODB_ALIGN_MERGED == 0 )); then
            step_per_todo_label "FORMATDB" "$(get_run_script FORMATDB)"
            ec=$?
        else
            # format db is called in step_alignment_file_merge()
            info_msg "FORMATDB is skipped since ODB_ALIGN_MERGED is set"
            step_done
        fi
        return $ec
    }

    #-----------------------------------------------
    # ALIGNMENT
    # Input : <species>.<EXT_FASTA_SELECTED>
    # Output: pair wise files in PWC directory
    #-----------------------------------------------
    step_alignment() {
        local ec=0
        if (( ODB_ALIGN_MERGED == 1 )); then
            step_alignment_file_merge
            ec=$?
        else
            step_pairwise ALIGNMENT
            ec=$?
        fi
        return $ec
    }

    #-----------------------------------------------
    # MAKEBRH
    # Input :
    # Output:
    #-----------------------------------------------
    step_makebrh() {
        if (( FORCE_MAKEBRH == 0 )); then
            if (( ALIGNMENT_MATRIX == 1 )); then
                set_stepid MAKEBRH
                step_done
                step_info_msg "half alignment matrix -> BRHs already computed in ALIGNMENT step"
                return 0
            fi
        fi
        step_pairwise MAKEBRH
    }

    #-----------------------------------------------
    # MAKEINPAR
    # Input :
    # Output:
    #-----------------------------------------------
    step_makeinpar() {
        #        if is_step_enabled SPLITBRH; then
        #            if is_dryrun; then
        #                [[ -n $TREE_ROOT_CLADE ]] || warn_msg "TREE_ROOT_CLADE is not set - should be set to the label of the root clade"
        #            else
        #                [[ -n $TREE_ROOT_CLADE ]] || fatal_msg "TREE_ROOT_CLADE is not set - should be set to the label of the root clade"
        #                set_split_in_file
        #                [[ -f $SPLIT_FILE ]] || fatal_msg "missing root clade ($TREE_ROOT_CLADE) input split file at $SPLIT_FILE"
        #            fi
        #        fi
        step_pairwise MAKEINPAR
    }

    #-----------------------------------------------
    # run setxxx over leaves in a tree
    # called by run_setxxx()
    #-----------------------------------------------
    run_setxxx_leaves() {
        [[ $# -eq 3 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        (( TREE_ENABLED == 0 )) && { step_warn_msg "run_setxxx_leaves should not be called when tree is disabled"; return 0; }
        local stepsrc=$1
        local ext=$2
        local ispwc=$3

        # get directory of todo file
        local dirtodo=$(dirname $PL_TODO)

        # get all leaves (script in brhclus get_runs...)
        local leaves=()
        local ec=0

        if (( TREE_ENABLED == 0 )); then
            # no tree - a single node given by PL_TODO
            leaves=( $PL_TODO_NAME )
        else
            # tree - get all leaves
            # check tree file
            [[ -f $TREE_INPUT_FILE ]] || { step_error_msg "missing tree file $TREE_INPUT_FILE"; return 255; }

            # get a list of leaf internal nodes
            get_leaf_nodes || {
                step_error_msg "failed finding leaves in tree $TREE_INPUT_FILE - missing file or wrong format (should be newick)?"
                return 255
            }
            leaves=( ${_NODES[@]} )
            [[ $? -eq 0 ]]
            debug_msg "found ${#leaves[@]} leaves in the tree $TREE_INPUT_FILE"
            [[ ${#leaves[@]} -eq 0 ]] && step_fatal_msg "found ${#leaves[@]} leaves in the tree $TREE_INPUT_FILE"
        fi

        #--------------------------------------------------------------
        # setbrh all leaves
        # reset jobarg arrays
        reset_joblists

        # loop over all leaves and fill job array args
        local node=
        for node in ${leaves[@]}; do
            # set src/target
            local ftodo=$dirtodo/${node}.$EXT_TODO
            local ftrg=$DIR_PROJECT_CLUSTER/${node}.$ext

            step_info_msg "-> adding leaf node $node"
            local fsrc="$ftodo"
            if [[ "$ispwc" == "1" ]]; then
                # set/create pair file
                # -> WITHOUT modificators (0) if no TREE
                # -> pairfile from PREPARETREE for given node
                create_pairwise_file "$stepsrc" "$node" "0" "$TREE_ENABLED"
                fsrc="$GENERATED_PAIR_FILE"
            fi
            SRCFILES+=("$fsrc")

            TRGFILES+=("$ftrg")
            # set JOB specs
            JOBARGS+=( "${stepsrc},${fsrc},${ftrg},${ext}" )
            JOBLABELS+=( "$node" )
            JOBSUBLBL+=( "$JOBLOGNOSPEC" )
        done
        
        local ec=0
        step_info_msg "start collecting files for each node (${#leaves[@]} jobs)"

        # third argument flags that it should NOT call init_step() and step_done()
        step_per_label $STEP_ID "${DIR_ORTHOLOGER_BIN}/run_merge_files.sh" "0"
        ec=$?
        check_error $ec "failed step_leaves with ec=$ec"
        ec=$?
        [[ $ec -eq 0 ]] || return $ec
    }

    #-----------------------------------------------
    # run over internal nodes in a tree
    # called by run_setxxx()
    #-----------------------------------------------
    run_setxxx_nodes() {
        [[ $# -eq 3 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        (( TREE_ENABLED == 0 )) && { step_warn_msg "run_setxxx_nodes should not be called when tree is disabled"; return 0; }

        # args
        local stepsrc=$1
        local ext=$2
        local ispwc=$3
        
        # get directory of todo file
        local dirtodo=$(dirname $PL_TODO)

        # get all leaves (script in brhclus get_runs...)
        local ec=0

        # tree - get all leaves
        # check tree file
        [[ -f $TREE_INPUT_FILE ]] || { step_error_msg "missing tree file $TREE_INPUT_FILE"; return 255; }

        # get a list of all internal nodes, excluding leaf nodes (ie the lowest nodes, not organisms)
        get_internal_nodes_noleaves || {
            step_error_msg "run_setxxx_nodes: failed finding nodes in tree $TREE_INPUT_FILE - missing file or wrong format (should be newick)?"
            return 255
        }
        debug_msg "found ${#_NODES[@]} internal nodes in the tree $TREE_INPUT_FILE"
        [[ ${#_NODES[@]} -eq 0 ]] && step_fatal_msg "found ${#_NODES[@]} nodes in the tree $TREE_INPUT_FILE"

        #--------------------------------------------------------------
        # setbrh all leaves
        # reset jobarg arrays
        reset_joblists

        local node=
        # loop over all leaves and fill job array args
        for node in ${_NODES[@]}; do
            # get root node for each line - first elem in csv
            # set src/target
            local ftodo=$dirtodo/${node}.$EXT_TODO
            local ftrg=$DIR_PROJECT_CLUSTER/${node}.$ext

            step_info_msg "-> adding node $node"
            local fsrc="$ftodo"
            if [[ "$ispwc" == "1" ]]; then
                # set/create pair file
                # -> WITHOUT modificators (0) if no TREE
                # -> else pairfile from PREPARETREE for given node
                create_pairwise_file "$stepsrc" "$node" "0" "$TREE_ENABLED"
                fsrc="$GENERATED_PAIR_FILE"
            fi
            SRCFILES+=("$fsrc")
            TRGFILES+=("$ftrg")
            # set JOB specs
            JOBARGS+=( "${stepsrc},${fsrc},${ftrg},${ext}" )
            JOBLABELS+=( "$node" )
            JOBSUBLBL+=( "$JOBLOGNOSPEC" )
        done
        
        local ec=0
        step_info_msg "start collecting files for each node (${#_NODES[@]} jobs)"

        # third argument flags that it should NOT call init_step() and step_done()
        step_per_label $STEP_ID "${DIR_ORTHOLOGER_BIN}/run_merge_files.sh" "0"
        ec=$?
        check_error $ec "failed step_nodes with ec=$ec"
        ec=$?
        [[ $ec -eq 0 ]] || return $ec
    }

    #-----------------------------------------------
    # run over a generated pair file, no tree
    # called by run_setxxx()
    #-----------------------------------------------
    run_setxxx_notree() {
        [[ $# -eq 3 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local stepsrc=$1
        local ext=$2
        local ispwc=$3
        local fsrc=""
        if [[ "$ispwc" -ne "0" ]]; then
            # get generated pair file using pair file from TREE (last "1")
	        create_pairwise_file "$stepsrc" "$PL_TODO_NAME" "1" "1" || return 255
            [[ -f $GENERATED_PAIR_FILE ]] || { step_error_msg "missing generated pair file at $GENERATED_PAIR_FILE"; return 255; }
            fsrc="$GENERATED_PAIR_FILE"
        else
            fsrc="$PL_TODO"
        fi
        
        local ec=0


        #--------------------------------------------------------------
        # setbrh all leaves
        # reset jobarg arrays
        reset_joblists
        
        local label=$(file_basename $PL_TODO)
        local ftrg=$DIR_PROJECT_CLUSTER/${label}.$ext
        SRCFILES=("$fsrc")
        TRGFILES=("$ftrg")
        
        # set JOB specs
        JOBARGS=( "${stepsrc},${fsrc},${ftrg},${ext}" )
        JOBLABELS=( "$label" )
        JOBSUBLBL=( "$JOBLOGNOSPEC" )        
        
        local ec=0
        step_info_msg "start collecting files for node $label"
        
        # submit script
        dispatch_cmd "${DIR_ORTHOLOGER_BIN}/run_merge_files.sh"

        step_timing_dispatch 1 "timing (secs):"
        return $DISPATCH_ERR
    }

    #-----------------------------------------------    
    # build BRH,INPAR or INPAR_SELECTED collections per node
    # it builds each node from scratch (i.e original pairwise BRHs)
    #
    # An earlier version transversed the tree, from the leaves to the root
    # and built the BRH files (uses previous nodes and .$EXT_PAIRS files)
    # Issue: have seen on e.g ZFS (not NFS mounted) that it sporadically fails
    #        - file not found or file detected as binary by grep.
    # This could potentially also be a problem in case of NFS mounts.
    #-----------------------------------------------    
    run_setxxx() {
        [[ $# -eq 3 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local stepsrc=$1
        local ext=$2
        local ispwc=$3

        if (( TREE_ENABLED == 0 )); then
            run_setxxx_notree "$stepsrc" "$ext" "$ispwc"
            return $?
        fi
        run_setxxx_leaves "$stepsrc" "$ext" "$ispwc" || return 255
        run_setxxx_nodes  "$stepsrc" "$ext" "$ispwc" || return 255
        return 0
    }

    #-----------------------------------------------
    # generic for SETBRH,SETINPAR and SETINPARSEL
    # supports with or with tree
    #-----------------------------------------------
    step_setxxx() {
        [[ $# -eq 1 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local stepid=$1
        is_step_enabled $stepid || return 0
            
        local T0=$(time_nsec)
        
        # initialise step
        local ec=0

        # in order to print the step info
        step_start 1
        
        # check if dry run
        is_dryrun && return 0

        local ext=
        local stepsrc=
        local ispwc=1
        if [[ "$STEP_ID" == "SETBRH" ]]; then
            ext=$EXT_BRH
            stepsrc="MAKEBRH"
        elif [[ "$STEP_ID" == "SETINPAR" ]]; then
            ext=$EXT_INPAR
            stepsrc="MAKEINPAR"
        elif [[ "$STEP_ID" == "SETINPARSEL" ]]; then
            ispwc=0
            ext=$EXT_INPAR_SELECTED
            stepsrc="MAKEINPARSEL"
        else
            fatal_msg "call step_setxxx for the wrong step $STEP_ID"
        fi
        # run at each node starting closest to the leafs and moving upwards
        step_info_msg "run $STEP_ID per node"
        run_setxxx "$stepsrc" "$ext" "$ispwc"  || { step_error_msg "error: failed running setxxx";  return 255; }

        local T1=$(time_nsec)
        step_timing $T0 $T1 1
        
        # ran ok
        step_done
        
        return 0

    }
    
    #-----------------------------------------------
    # SETBRH
    # Input : A_B.brh
    # Output:
    #-----------------------------------------------
    step_setbrh() {
        step_setxxx SETBRH
    }

    #-----------------------------------------------
    # SETINPAR
    # Input : A_B.inpar
    # Output:
    #-----------------------------------------------
    step_setinpar() {
        step_setxxx SETINPAR
    }

    #-----------------------------------------------
    # SETINPARSEL
    # Input : A.inpar_selected
    # Output:
    #-----------------------------------------------
    step_setinparsel() {
        step_setxxx SETINPARSEL
    }

    #-----------------------------------------------
    # SPLITBRH
    # Input : BRH full file
    # Output: BRH full file with splits
    #         split file
    #-----------------------------------------------
    step_splitbrh() {
        fatal_msg "step SPLITBRH is disabled!"

        local T0=$(time_nsec)
        # initialise step
        local ec=0
        #        init_step "SPLITBRH"
        #        ec=$?       
        #        # step is disabled?
        #        [[ $ec -eq 1   ]] && return 0
        #        # step id done?
        #        [[ $ec -eq 2   ]] && return 0
        #        # step dependancy?
        #        [[ $ec -eq 255 ]] && return 255

        # check for PL_TODO_LABELS_REQ
        #if [[ ${#PL_TODO_LABELS_REQ} -gt 0 ]]; then
        #    # get CSV list of labels
        #    local lbls=${PL_TODO_LABELS_REQ// /,}
        #fi
        

        JOBARGS=( "${PL_TODO_NAME},${OP_STEP_NTHREADS[$STEP_ID]}" )
        JOBLABELS=( "splitbrh" )
        JOBSUBLBL=( "$JOBLOGNOSPEC" )

        # start step
        step_start 1

        # check if dry run
        is_dryrun && return 0

        # check source/target files
        local src="${DIR_PROJECT_CLUSTER}/${PL_TODO_NAME}.${EXT_BRH}"
        local target="${DIR_PROJECT_CLUSTER}/${PL_TODO_NAME}.${EXT_SPLITS_BRH}"
        [[ -f $src ]] || { step_error_msg "missing source file $(trim_name $src)"; }

        remake_target "$target" "$src"
        ec=$?
        [[ $ec -eq 1 ]] && { step_done; return 0; }
        [[ $ec -eq 2 ]] && return 0

        # check that scheduler is free
        check_scheduler_status 1

        # run the cluster job
        dispatch_cmd "$(get_run_script $STEP_ID)"

        # clear job arrays
        reset_joblists

        (( DISPATCH_ERR == 0 )) && step_done

        step_timing_dispatch 1 "timing (secs):"
        
        return $DISPATCH_ERR
    }

    #-----------------------------------------------
    # get input pairs file for MERGEINPAR step
    #-----------------------------------------------
    _MERGEINPAR_INPUT_PAIRS=""
    get_mergeinpar_input_pairs() {
        local label=$1
        _MERGEINPAR_INPUT_PAIRS="${DIR_PROJECT_CLUSTER}/${label}.$EXT_INPAR"
        if ! is_step_enabled SETINPAR; then
            create_pairwise_file SETINPAR "$label" "0" "$TREE_ENABLED" || return 255
            _MERGEINPAR_INPUT_PAIRS="$GENERATED_PAIR_FILE"
        fi
    }
    
    #-----------------------------------------------
    # Mergeinpar for cluster
    # only called when not in tree mode
    #-----------------------------------------------
    common_mergeinpar_basic() {
        (( TREE_ENABLED == 0 )) || return 0
        [[ $# -eq 1 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        
        local stepid="$1"
        stepid=${stepid^^} # upper case
        
        local dirtodo=$(dirname $PL_TODO)
        
        # get extensions depending on step (MERGEINPAR or MERGEINPARSEL
        local extclin=${EXT_CLUSTER_RAW}
        local extclout=${EXT_CLUSTER_INPAR}
        _MERGEINPAR_INPUT_PAIRS="<NONE>"
        if [[ "$stepid" == "MERGEINPARSEL" ]]; then
            extclin=$EXT_CLUSTER_INPAR
            extclout=$EXT_CLUSTER
        elif [[ "$stepid" == "MERGEINPAR" ]]; then
            get_mergeinpar_input_pairs "$PL_TODO_NAME"
            # now _MERGEINPAR_INPUT_PAIRS is set to the proper input file
        fi

        # set SRCFILES/JOBARGS array for each node
        reset_joblists
        SRCFILES=( "${DIR_PROJECT_CLUSTER}/${PL_TODO_NAME}.${extclin}" )
        TRGFILES=( "${DIR_PROJECT_CLUSTER}/${PL_TODO_NAME}.${extclout}" )
        JOBARGS=( "${PL_TODO_NAME},${_MERGEINPAR_INPUT_PAIRS},${OP_STEP_NTHREADS[$STEP_ID]}" )
        JOBLABELS=( "mergeinpar-$PL_TODO_NAME")
        JOBSUBLBL=( "$JOBLOGNOSPEC" )

        local ec=0
        step_info_msg "start (in)paralog merge for a single node ($PL_TODO_NAME)"
        
        # third argument flags that it should call init_step() and step_done()
        step_per_label $stepid "$(get_run_script MERGEINPAR)" "0"
        ec=$?

        # step_per_label resets JOBLABELS etc, restore them
        JOBLABELS=( "mergeinpar-$PL_TODO_NAME")
        JOBSUBLBL=( "$JOBLOGNOSPEC" )

        return $ec
    }
    
    #-----------------------------------------------
    # Mergeinpar for leaves
    # in tree mode this is called in the cluster step
    #-----------------------------------------------
    common_mergeinpar_leaves() {
        [[ $# -eq 1 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        (( TREE_ENABLED != 0 )) || return 0
        
        local stepid="$1"
        stepid=${stepid^^} # upper case
        
        local dirtodo=$(dirname $PL_TODO)
        
        # get extensions depending on step (MERGEINPAR or MERGEINPARSEL
        local extclin=${EXT_CLUSTER_RAW}
        local extclout=${EXT_CLUSTER_INPAR}
        if [[ "$stepid" == "MERGEINPARSEL" ]]; then
            extclin=$EXT_CLUSTER_INPAR
            extclout=$EXT_CLUSTER
        fi

        reset_joblists
        
        # get all nodes from the tree
        [[ -f "$TREE_INPUT_FILE" ]] || step_fatal_msg "tree enabled but tree file missing at $TREE_INPUT_FILE"

        # get all leaf nodes
        get_leaf_nodes || {
            step_error_msg "failed retrieving leaves from tree file $TREE_INPUT_FILE"
            return 255
        }
        
        # sort leaves by todo file size
        local todosort=()
        todosort=$(ls -Sr $(for leaf in ${_NODES[@]}; do echo $dirtodo/${leaf}.$EXT_TODO; done))
        
        _MERGEINPAR_INPUT_PAIRS="<NONE>"
        local setpairs=0
        [[ "$stepid" == "MERGEINPAR" ]] && setpairs=1
        
        # the leaves have no pair files, uses the todo files
        local ftodo=
        for ftodo in ${todosort[@]}; do
            local leaf=$(basename $ftodo .$EXT_TODO)
            (( setpairs )) && get_mergeinpar_input_pairs "$leaf"
            SRCFILES+=( "${DIR_PROJECT_CLUSTER}/${leaf}.${extclin}" )
            TRGFILES+=( "${DIR_PROJECT_CLUSTER}/${leaf}.${extclout}" )
            JOBARGS+=( "${leaf},${_MERGEINPAR_INPUT_PAIRS},${OP_STEP_NTHREADS[$STEP_ID]}" )
            JOBLABELS+=( "mergeinpar-$leaf")
            JOBSUBLBL+=( "$JOBLOGNOSPEC" )
        done

        local ec=0
        step_info_msg "start (in)paralog merge for each leaf node (${#JOBARGS[@]} jobs)"

        # third argument flags that it should call init_step() and step_done()
        step_per_label $stepid "$(get_run_script MERGEINPAR)" "0"
        ec=$?
        check_error $ec "failed mergeinpar_leaves with ec=$ec"
        return $?
    }

    #-----------------------------------------------
    # Mergeinpar for nodes
    # in tree mode this is called in the cluster step
    #-----------------------------------------------
    common_mergeinpar_nodes() {
        [[ $# -eq 1 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        (( TREE_ENABLED != 0 )) || return 0
        
        local stepid="$1"
        stepid=${stepid^^} # upper case
        
        local dirtodo=$(dirname $PL_TODO)
        
        # get extensions depending on step (MERGEINPAR or MERGEINPARSEL
        local extclin=${EXT_CLUSTER_RAW}
        local extclout=${EXT_CLUSTER_INPAR}
        if [[ "$stepid" == "MERGEINPARSEL" ]]; then
            extclin=$EXT_CLUSTER_INPAR
            extclout=$EXT_CLUSTER
        fi

        # set SRCFILES/JOBARGS array for each node
        reset_joblists

        # get all nodes
        # get all nodes from the tree
        [[ -f "$TREE_INPUT_FILE" ]] || step_fatal_msg "tree enabled but tree file missing at $TREE_INPUT_FILE"

        # set pair file, or not
        _MERGEINPAR_INPUT_PAIRS="<NONE>"
        local setpairs=0
        [[ "$stepid" == "MERGEINPAR" ]] && setpairs=1
        
        local ec=0
        local rval=0
        local n=
        
        # retrieve all internal nodes, excluding the lowest nodes (not organisms)
        get_internal_nodes || {
            step_error_msg "failed retrieving nodes from tree file $TREE_INPUT_FILE"
            return 255
        }

        for n in ${_NODES[@]}; do
            (( setpairs )) && get_mergeinpar_input_pairs "$n"
            SRCFILES=( "${DIR_PROJECT_CLUSTER}/${n}.${extclin}" )
            TRGFILES=( "${DIR_PROJECT_CLUSTER}/${n}.${extclout}" )
            JOBARGS=( "${n},${_MERGEINPAR_INPUT_PAIRS},${OP_STEP_NTHREADS[$STEP_ID]}" )
            JOBLABELS=( "mergeinpar_node-$n")
            JOBSUBLBL=( "$JOBLOGNOSPEC" )
            step_info_msg "start (in)paralog merge for node $n)"
            # third argument flags that it should call init_step() and step_done()
            step_per_label $stepid "$(get_run_script MERGEINPAR)" "0"
            ec=$?
            check_error $ec "failed mergeinpar_nodes at $n with ec=$ec"
            [[ $? -eq 0 ]] || rval=1
        done
        return $rval
    }

    #-----------------------------------------------
    # Mergeinpar - call all possible cases
    #-----------------------------------------------
    common_mergeinparx() {
        #[[ $# -eq 1 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        #local stepid="$1"
        #stepid=${stepid^^} # upper case
        [[ "$STEP_ID" == "MERGEINPAR" || "$STEP_ID" == "MERGEINPARSEL" ]] || fatal_msg "common_mergeinparx - expected MERGEINPAR or MERGEINPARSEL"

        # initialise step - should be either MERGEINPAR or MERGEINPARSEL
        local ec=0
        #        init_step "$stepid"
        #        ec=$?
        #        # step is disabled?
        #        [[ $ec -eq 1   ]] && return 0
        #        # step id done?
        #        [[ $ec -eq 2   ]] && return 0
        #        # step dependancy?
        #        [[ $ec -eq 255 ]] && return 255

        # in order to print the step info
        step_start 1
        
        # check if dry run
        is_dryrun && return 0

        # call mergeinpar:
        #  no tree   : basic
        #  with tree : leaves, nodes
        common_mergeinpar_basic  $STEP_ID || return $?
        common_mergeinpar_leaves $STEP_ID || return $?
        common_mergeinpar_nodes  $STEP_ID || return $?
        
        # step is done
        step_done
    }
    
    #-----------------------------------------------
    # WEAKBRH for leaves
    # in tree mode this is called in the cluster step
    #-----------------------------------------------
    common_weakbrh_leaves() {
        
        [[ $# -eq 1 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        (( TREE_ENABLED != 0 )) || return 0
        
        local stepid="$1"
        stepid=${stepid^^} # upper case
        
        local dirtodo=$(dirname $PL_TODO)
        
        # get extensions
        local extclin=${EXT_CLUSTER}
        local extclout=${EXT_WEAK_BRH}

        reset_joblists
        
        # get all nodes from the tree
        [[ -f "$TREE_INPUT_FILE" ]] || step_fatal_msg "tree enabled but tree file missing at $TREE_INPUT_FILE"

        # get all leaf nodes
        get_leaf_nodes || {
            step_error_msg "failed retrieving leaves from tree file $TREE_INPUT_FILE"
            return 255
        }

        # sort leaves by todo file size
        local todosort=()
        todosort=$(ls -Sr $(for leaf in ${_NODES[@]}; do echo $dirtodo/${leaf}.$EXT_TODO; done))
        
        # the leaves have no pair files, uses the todo files
        local ftodo=
        for ftodo in ${todosort[@]}; do
            local leaf=$(basename $ftodo .$EXT_TODO)
            SRCFILES+=( "${DIR_PROJECT_CLUSTER}/${leaf}.${EXT_CLUSTER}" )
            TRGFILES+=( "${DIR_PROJECT_CLUSTER}/${leaf}.${EXT_WEAK_BRH}" )
            JOBARGS+=( "${leaf},${OP_STEP_NTHREADS[$STEP_ID]}" )
            JOBLABELS+=( "weakbrh-$leaf")
            JOBSUBLBL+=( "$JOBLOGNOSPEC" )
        done

        local ec=0
        step_info_msg "start extracting weak BRHs for each leaf node (${#JOBARGS[@]} jobs)"
        
        # third argument flags that it should call init_step() and step_done()
        step_per_label $stepid "$(get_run_script WEAKBRH)" "0"
        ec=$?
        check_error $ec "failed weakbrh_leaves at $n with ec=$ec"
        return $?
    }

    #-----------------------------------------------
    # PARSE CLUSTER_INPOG
    #-----------------------------------------------
    parse_cluster_inpog() {
        local check=${1-0}
        CLUSTER_INPOG_OPTS=""
        [[ -n $CLUSTER_INPOG ]] || return 0
        local saveIFS="$IFS"
        IFS=','
        local labels=( $CLUSTER_INPOG )
        IFS="$saveIFS"

        local lbl=
        for lbl in ${labels[@]}; do
            if [[ "${lbl: -3}" != ".$EXT_CLUSTER" ]]; then
                lbl="$DIR_PROJECT/Cluster/${lbl}.$EXT_CLUSTER"
            fi
            (( check )) && [[ ! -f $lbl ]] && {
                CLUSTER_INPOG_OPTS="missing file: $lbl"
                return 255
            }
            CLUSTER_INPOG_OPTS="$CLUSTER_INPOG_OPTS -C $lbl"
            
            local f=
            if (( CLUSTER_INPOG_WEAKBRH )); then
                f="${lbl//$EXT_CLUSTER/$EXT_WEAK_BRH}"
                (( check )) && [[ ! -f $f ]] && {
                    CLUSTER_INPOG_OPTS="missing weak brh file: $f"
                    return 255
                }
                CLUSTER_INPOG_OPTS="$CLUSTER_INPOG_OPTS -P $f"
            fi
        done
    }


    #-----------------------------------------------
    # CLUSTER TREE
    # run leaf nodes
    # they can all be run in parallel
    # required BRH files generated in MAKEBRH step
    #
    # Job mapping
    # - <node>.brh : <node>.og_raw
    #
    #
    #-----------------------------------------------
    run_cluster_leaves() {
        # get directory of todo file
        local dirtodo=$(dirname $PL_TODO)
        
        # check tree file
        [[ -f $TREE_INPUT_FILE ]] || { step_error_msg "missing tree file $TREE_INPUT_FILE"; return 255; }

        # get all leaves (script in brhclus get_runs...)
        get_leaf_nodes || {
            step_error_msg "run_cluster_leaves: failed finding leaves in tree $TREE_INPUT_FILE - missing file or wrong format (should be newick)?"
            return 255
        }
        step_info_msg "found ${#_NODES[@]} leaf nodes in the tree"

        #--------------------------------------------------------------
        # cluster all leaves
        # -> .og_raw files
        
        # reset jobarg arrays
        reset_joblists

        # sort all leaves according to size
        local sortedLeaves=()
        sortedLeaves=$(ls -Sr $(for leaf in ${_NODES[@]}; do echo $dirtodo/${leaf}.$EXT_TODO; done))

        # loop over all leaves and fill job array args
        local ftodo=
        local nfailed=0
        for ftodo in ${sortedLeaves[@]}; do
            # set src/target
            local leaf=$(basename $ftodo .$EXT_TODO)
            local fpairs=$DIR_PROJECT_TODO/${leaf}.$EXT_PAIRS_TREE
            local ftrg=$DIR_PROJECT_CLUSTER/${leaf}.$EXT_CLUSTER_RAW

            # check input file
            [[ -f $fpairs ]] || { nfailed=$((nfailed+1)); step_error_msg "missing source file $(trim_name $fpairs)"; continue; }

            # CLUSTER step
            if remake_target "$ftrg" "$fpairs"; then
                SRCFILES+=($fpairs)
                TRGFILES+=($ftrg)
                # set JOB specs <root,node,nthreads>
                JOBARGS+=( "${leaf},$fpairs,$ftrg,$BRHCLUS_DO_WEAK,${OP_STEP_NTHREADS[$STEP_ID]}" )
                JOBLABELS+=( "cluster-$leaf" )
                JOBSUBLBL+=( "$JOBLOGNOSPEC" )
            fi
        done
        
        local ec=0

        if [[ ${#JOBARGS[@]} -gt 0 ]]; then
            step_info_msg "start clustering for each leaf (${#JOBARGS[@]} jobs)"

            # third argument flags that it should NOT call init_step() and step_done()
            step_per_label $STEP_ID "$(get_run_script $STEP_ID)" "0"
            ec=$?
            check_error $ec "failed clustering leaves with ec=$ec"
            ec=$?
            [[ $ec -eq 0 ]] || return $ec
        else
            (( nfailed == 0 )) && step_info_msg "all leaves are already clustered"
        fi

        if (( nfailed > 0 )); then
            step_error_msg "some nodes failed clustering!"
            return 1
        fi

        #--------------------------------------------------------------------------
        # Merge inparalogs to all leaves
        #--------------------------------------------------------------------------
        # save step info
        local stepid="$STEP_ID"
        local stepid_flag="$STEP_ID_FLAG"
        # run mergeinpar steps on leafs
        # -> .og_inpar -> .og files
        local step=
        step_info_msg "run mergeinpar per leaf"
        for step in MERGEINPAR MERGEINPARSEL; do
            ec=0
            #if [[ $step == WEAKBRH ]]; then
            #    # if doweak==0 then it's not produced in brhclus
            #    if [[ "$BRHCLUS_DO_WEAK" == "0" ]]; then
            #        common_weakbrh_leaves $STEP_ID
            #        ec=$?
            #    fi
            #else
            set_stepid $step
            common_mergeinpar_leaves $STEP_ID
            ec=$?
            check_error $ec "failed running $step for leaves with ec=$ec"
            ec=$?
            if [[ $ec -ne 0 ]]; then
                set_stepid $stepid
                return 255
            fi
        done        
        set_stepid "$stepid" "$stepid_flag"
    }

    #-----------------------------------------------
    # copy child files to target
    #-----------------------------------------------
    make_copy_of_child() {
        local trgNode="$1"
        local child="$2"
        local todoParent="$3"
        local todoChild="$4"

        local ftrg=$DIR_PROJECT_CLUSTER/${trgNode}.$EXT_CLUSTER_RAW
        step_info_msg "node $(basename $ftrg) and single child node $child are identical"
        step_info_msg "copying raw cluster file"
        
        # perfect match - parent cluster is the same as the child
        local fkid=$DIR_PROJECT_CLUSTER/${child}.$EXT_CLUSTER_RAW
        [[ -f $fkid ]] || fatal_msg "unable to find child cluster file at $fkid"
        make_copy "$fkid" "$ftrg"
        [[ $? -eq 0 ]] || fatal_msg "failed copying trivial case from $fkid to $ftrg"
        echo "# copy of $fkid" > ${ftrg}.log
        
        # do the same for weak BRH
        local fkid=$DIR_PROJECT_CLUSTER/${child}.$EXT_WEAK_BRH
        [[ -f $fkid ]] || fatal_msg "unable to find child cluster file at $fkid"
        ftrg=$DIR_PROJECT_CLUSTER/${trgNode}.$EXT_WEAK_BRH
        make_copy "$fkid" "$ftrg"
        [[ $? -eq 0 ]] || fatal_msg "failed copying trivial case from $fkid to $ftrg"
        echo "# copy of $fkid" > ${ftrg}.log

        # now do the same for mergeinpar and mergeinparsel
        set_stepid MERGEINPAR
        step_info_msg "copying mergeinpar file"
        fkid=$DIR_PROJECT_CLUSTER/${child}.${EXT_CLUSTER_INPAR}
        ftrg=$DIR_PROJECT_CLUSTER/${trgNode}.$EXT_CLUSTER_INPAR
        [[ -f $fkid ]] || fatal_msg "unable to find child cluster file at $fkid"
        make_copy "$fkid" "$ftrg"
        [[ $? -eq 0 ]] || fatal_msg "failed copying trivial case from $fkid to $ftrg"
        echo "# copy of $fkid" > ${ftrg}.log
        
        set_stepid MERGEINPARSEL
        step_info_msg "copying mergeinparsel file"
        fkid=$DIR_PROJECT_CLUSTER/${child}.${EXT_CLUSTER}
        [[ -f $fkid ]] || fatal_msg "unable to find child cluster file at $fkid"
        ftrg=$DIR_PROJECT_CLUSTER/${trgNode}.$EXT_CLUSTER
        make_copy "$fkid" "$ftrg"
        [[ $? -eq 0 ]] || fatal_msg "failed copying trivial case from $fkid to $ftrg"
        echo "# copy of $fkid" > ${ftrg}.log
        return 0
    }
    
    #-----------------------------------------------
    # CLUSTER TREE
    # run nodes that depend on previously generated
    # clusters
    # BRHs to be used are obtained from
    # * list of pairs for given node (script in brhclus get_pairs.py)
    #-----------------------------------------------
    run_cluster_nodes() {
        # check tree file
        local dirtodo=$(dirname $PL_TODO)

        # get all internal nodes
        local ec=0
        local nodes=()
        local saveIFS="$IFS"

        get_branches "$TREE_INPUT_FILE" || {
            step_error_msg "run_cluster_nodes: failed finding nodes in tree $TREE_INPUT_FILE - missing file or wrong format (should be newick)?"
            return 255
        }

        step_info_msg "found ${#_NODES[@]} nodes in the tree"

        # each element in _NODES is a CSV <root>,<child1>,...
        
        # get script - run_cluster.sh
        local script="$(get_run_script $STEP_ID)"

        # counter
        nfailed=0

        # error code
        local ec=0
        
        # loop over all nodes and submit jobs
        # note that they cannot all be run in parallel as some depend on others
        reset_joblists

        local stepid_flag="$STEP_ID_FLAG"
        local node=
        for node in "${_NODES[@]}"; do
            # split up in root node and children cluster
            local allnodes=()
            
            IFS=','
            eval allnodes=( $node )
            IFS="$saveIFS"
            
            local trgNode="${allnodes[0]}"

            # check if node should generate weak BRH file
            # note that BRHLUS_DO_WEAK is set to 0 if it is not supported by the version of BRHCLUS
            local doweakbrh=$BRHCLUS_DO_WEAK
            local isroot=0
            [[ "$trgNode" == "$TREE_ROOT_CLADE" ]] && { doweakbrh=0; isroot=1; }
            
            step_info_msg "clustering node <$trgNode> (root+children ${allnodes[@]})"
            is_dryrun && continue

            # set src/target
            local fpairs=$dirtodo/${trgNode}.$EXT_PAIRS_TREE
            local fbrh=$DIR_PROJECT_CLUSTER/${trgNode}.$EXT_BRH
            local ftrg=$DIR_PROJECT_CLUSTER/${trgNode}.$EXT_CLUSTER_RAW
            local finputPairs="$fpairs"
            [[ -f "$fbrh" ]] && finputPairs="$fbrh"
            # check input file
            [[ -f $finputPairs ]] || {
                nfailed=$((nfailed+1))
                step_error_msg "missing source file $(trim_name $finputPairs)"
                continue
            }
            step_info_msg "selected pair file: $(trim_name $finputPairs)"
            
            # src/trg files
            SRCFILES=($finputPairs)
            TRGFILES=($ftrg)

            # loop over kids, add them as a CSV string
            local nodeargs="${trgNode},$finputPairs,$ftrg"
            local nv=""
            local nkids=0
            for nv in ${allnodes[@]:1}; do
                nodeargs="${nodeargs},$nv"
                nkids=$((nkids+1))
            done

            #-----------------------------------------
            # Cluster node
            #-----------------------------------------
            # check target and source
            remake_target "$ftrg" "$finputPairs"
            if [[ $? -ne 0 ]]; then
                # raw clustering is done - set ec=0 it is checked further down
                step_info_msg "raw clustering already ok"
                ec=0
            else
                # raw clustering is not done - do it!
                # special case if just one child node and identical set of species
                isidentical=0
                if (( nkids == 1 )); then
                    local child=${allnodes[1]}
                    local todoParent=${dirtodo}/${trgNode}.$EXT_TODO
                    local todoBaby=${dirtodo}/${child}.$EXT_TODO
                    step_info_msg "node $trgNode has only one child node $child"
                    diff <(grep -v '#' $todoBaby) <(grep -v '#' $todoParent) >& /dev/null
                    if [[ $? -eq 0 ]]; then
                        isidentical=1
                        make_copy_of_child "$trgNode" "$child" "$todoParent" "$todoBaby"
                        
                        # restore step id
                        set_stepid "CLUSTER" "$stepid_flag"
                        continue
                    fi
                fi
                #--------------------------------------------------------------
                # set JOB specs for clustering
                JOBARGS=( "${nodeargs},${doweakbrh},${OP_STEP_NTHREADS[$STEP_ID]}" )
                JOBLABELS=( "cluster-$trgNode" )
                JOBSUBLBL=( "$JOBLOGNOSPEC" )

                # run clustering for one level with one or more children
                dispatch_cmd "$script"
                ec=$DISPATCH_ERR
                step_timing_dispatch 1 "node timing (secs):"
            fi

            #-----------------------------------------
            # Merginpar for node
            #-----------------------------------------
            # check error
            if [[ $ec -ne 0 ]]; then
                step_error_msg "failed cluster node $trgNode with ec=$ec"
                nfailed=$((nfailed+1))
            else
                # if tree based (no kids>0), then apply mergeinpar here on trgNode
                if (( nkids > 0 )); then
                    local stepid=$STEP_ID
                    step_info_msg "run mergeinpar+weakbrh for node $trgNode"
                    local extclin=${EXT_CLUSTER_RAW}
                    local extclout=${EXT_CLUSTER_INPAR}
                    

                    for step in MERGEINPAR MERGEINPARSEL; do
                        set_stepid $step
                        _MERGEINPAR_INPUT_PAIRS="<NONE>"
                        [[ "$STEP_ID" == "MERGEINPAR" ]] && get_mergeinpar_input_pairs "$trgNode"

                        SRCFILES=( "${DIR_PROJECT_CLUSTER}/${trgNode}.${extclin}" )
                        TRGFILES=( "${DIR_PROJECT_CLUSTER}/${trgNode}.${extclout}" )
                        JOBARGS=( "${trgNode},${_MERGEINPAR_INPUT_PAIRS},${OP_STEP_NTHREADS[$STEP_ID]}" )
                        JOBLABELS=( "mergeinpar_node-$trgNode")
                        JOBSUBLBL=( "$JOBLOGNOSPEC" )
                        step_info_msg "start $step merge for node $trgNode"
                        
                        # third argument flags that it should call init_step() and step_done()
                        step_per_label $stepid "$(get_run_script MERGEINPAR)" "0"
                        ec=$?
                        if [[ $ec -ne 0 ]]; then
                            step_error_msg "failed running $step for nodes with ec=$ec"
                            set_stepid $stepid
                            return 255
                        fi
                        extclin=$EXT_CLUSTER_INPAR
                        extclout=$EXT_CLUSTER
                    done
                    #-----------------------------------------
                    # extract weak BRHs
                    if (( isroot )); then
                        [[ "$BRHCLUS_DO_WEAK" == "0" ]] && step_info_msg "skip WEAKBRH for root ($trgNode)"
                    else
                        if [[ "$BRHCLUS_DO_WEAK" == "0" ]]; then
                            step_info_msg "run WEAKBRH step for node $trgNode - root $TREE_ROOT_CLADE"
                            set_stepid "WEAKBRH"
                            SRCFILES=( "${DIR_PROJECT_CLUSTER}/${trgNode}.${EXT_CLUSTER}" )
                            TRGFILES=( "${DIR_PROJECT_CLUSTER}/${trgNode}.${EXT_WEAK_BRH}" )
                            JOBARGS=( "${trgNode},${OP_STEP_NTHREADS[$STEP_ID]}" )
                            JOBLABELS=( "weakbrh_node-$trgNode")
                            JOBSUBLBL=( "$JOBLOGNOSPEC" )
                            step_info_msg "start WEAKBRH for node $trgNode)"#

                            # third argument flags that it should call init_step() and step_done()
                            step_per_label $stepid "$(get_run_script WEAKBRH)" "0"
                            ec=$?
                            if [[ $ec -ne 0 ]]; then
                                step_error_msg "failed running $step for nodes with ec=$ec"
                                set_stepid $stepid
                                return 255
                            fi
                        else
                            step_info_msg "skip WEAKBRH step - already done in clustering via the brhclus tool"
                        fi
                    fi
                    
                    #-----------------------------------------
                    # reset step id back to cluster
                    set_stepid "$stepid" "$stepid_flag"
                fi
            fi
        done
        if (( nfailed==0 )); then
            return 0
        fi
        return 1
    }

    #-----------------------------------------------
    # CLUSTER TREE
    # Input : BRH files
    # Output: Raw cluster - without inparalogs
    #-----------------------------------------------
    step_cluster_tree() {
        # cluster by tree
        [[ $CLUSTER_LABEL == "BRHCLUS" ]] || fatal_msg "cluster from tree only supported by BRHCLUS"

        # set DO_WEAK flag for brhclus
        "$EXE_BRHCLUS" -h | grep -q dumpweak
        if [[ $? -eq 0 ]]; then
            verbose_info_msg "will generate weak brh files using brhclus"
            BRHCLUS_DO_WEAK=1
        else
            verbose_info_msg "will generate weak brh files in WEAKBRH step"
        fi
        
        # initialise step
        local ec=0

        # in order to print the step info
        step_start 1
        
        # check if dry run
        is_dryrun && return 0

        step_info_msg "cluster by tree"

        # run clustering at the leaf levels
        if [[ "$TREE_CLUSTER_RUN" == "2" ]]; then
            step_info_msg "skip clustering per leaf (TREE_CLUSTER_RUN = $TREE_CLUSTER_RUN)"
        else
            step_info_msg "---> run clustering per leaf"
            run_cluster_leaves || { step_error_msg "error: failed running clustering for leaves"; return 255; }
        fi

        if [[ "$TREE_CLUSTER_RUN" != "1" ]]; then
            # run at each node starting closest to the leafs and moving upwards
            step_info_msg "---> run clustering per node"
            run_cluster_nodes  || { step_error_msg "error: failed running clustering for nodes";  return 255; }

            #TODO HERE -> this needs to be added to cluster_nodes

            # both ran ok - step is done
            step_done
            
            local stepid_flag="$STEP_ID_FLAG"
            ####[[ "$BRHCLUS_DO_WEAK" == "1" ]] && set_stepid WEAKBRH && step_done
            set_stepid MERGEINPAR    && step_done
            set_stepid MERGEINPARSEL && step_done
            set_stepid CLUSTER "$stepid_flag"
        else
            step_info_msg "skip clustering per node (TREE_CLUSTER_RUN = $TREE_CLUSTER_RUN)"
        fi
        
        return 0
    }


    #-----------------------------------------------
    # CLUSTER
    # Input : BRH files
    # Output: Raw cluster - without inparalogs
    #-----------------------------------------------
    step_cluster() {
        
        if (( TREE_ENABLED == 1 )); then
            step_cluster_tree
            return $?
        fi

        # no tree -> no weak BRHs
        BRHCLUS_DO_WEAK=0

        # initialise step
        local ec=0
        
        # check source/target files
        # source is SETBRH or pair file
        local inputSrc="${DIR_PROJECT_CLUSTER}/${PL_TODO_NAME}.$EXT_BRH"
        if ! is_step_enabled SETBRH; then
            create_pairwise_file SETBRH "$PL_TODO_NAME" "0" "$TREE_ENABLED" || return 255
            inputSrc="$GENERATED_PAIR_FILE"
        fi
        local target="${DIR_PROJECT_CLUSTER}/${PL_TODO_NAME}.${EXT_CLUSTER_RAW}"

        remake_target "$target" "$inputSrc"
        ec=$?
        [[ $ec -eq 1 ]] && { step_done; return 0; }
        [[ $ec -eq 2 ]] && return 0

        #-----------------------------------------------------------------------------------------
        # Cluster seed
        #-----------------------------------------------------------------------------------------
        if [[ -n $CLUSTER_INPOG ]]; then
            parse_cluster_inpog || {
                step_error_msg "failed parsing input CLUSTER_INPOG = $CLUSTER_INPOG"
                step_error_msg "$CLUSTER_INPOG_OPTS"
                CLUSTER_INPOG_OPTS=""
                return 255
            }
            OPTS_BRHCLUS="$OPTS_BRHCLUS --nomerges $CLUSTER_INPOG_OPTS --keepweak"
            if [[ ${#PL_TODO_LABELS_REQ} -gt 0 ]]; then
                local lbls="${PL_TODO_LABELS_REQ// /,}"
                OPTS_BRHCLUS="$OPTS_BRHCLUS --absorb $lbls"
            else
                OPTS_BRHCLUS="$OPTS_BRHCLUS --absorball"
            fi
	        OPTS_MERGEINPAR="$OPTS_MERGEINPAR --nomerges --nograb"
            info_msg "CLUSTER step will map user taxid(s) with cluster file(s) $CLUSTER_INPOG"            
        fi

        #-----------------------------------------------------------------------------------------
        # check for PL_TODO_LABELS_REQ
        #if [[ ${#PL_TODO_LABELS_REQ} -gt 0 ]]; then
        #    [[ $CLUSTER_LABEL == "BRHCLUS" ]] || { error_msg "merge in required taxids ($PL_TODO_LABELS_REQ) is for now only supported by BRHCLUS"; return 255; }
	    #    [[ -n $CLUSTER_INPOG ]] || { error_msg "merge taxids are defined ($PL_TODO_LABELS_REQ) -> requires CLUSTER_INPOG to be set as the input cluster file name"; return 255; }
        #    if ! is_dryrun; then
        #        [[ -f $CLUSTER_INPOG ]] || { error_msg "cluster step is missing input cluster $CLUSTER_INPOG"; return 255; }
        #    fi
        #    # get CSV list of labels
        #    local lbls=${PL_TODO_LABELS_REQ// /,}
        #    OPTS_BRHCLUS="$OPTS_BRHCLUS --nomerges --absorb $lbls --inpog $CLUSTER_INPOG --keepweak"
	    #    OPTS_MERGEINPAR="$OPTS_MERGEINPAR --nomerges --nograb"
        #    info_msg "CLUSTER step will merge taxid(s) $PL_TODO_LABELS_REQ with cluster file $CLUSTER_INPOG"
        #fi
        #-----------------------------------------------------------------------------------------
        
        # the JOBARGS should contain <root,node,nthreads>
        # where root==node here
        JOBARGS=( "${PL_TODO_NAME},${inputSrc},${target},${BRHCLUS_DO_WEAK},${OP_STEP_NTHREADS[$STEP_ID]}" )
        JOBLABELS=( "cluster" )
        JOBSUBLBL=( "$JOBLOGNOSPEC" )

        # start step
        step_start 1

        # check if dry run
        is_dryrun && return 0

        step_info_msg "cluster by todo file"
        
        info_msg "run BRHCLUS with options: $OPTS_BRHCLUS"
                    
        # run the cluster job
        dispatch_cmd "$(get_run_script $STEP_ID)"
        ec=$?
        
        # clear job arrays
        reset_joblists
        step_timing_dispatch 1 "timing (secs):"
        
        [[ $ec -eq 0 ]] || return 1

        step_done

        return 0
    }

    #-----------------------------------------------
    # MERGEINPAR
    # Input :
    # Output:
    #-----------------------------------------------
    step_mergeinpar() {
        (( TREE_ENABLED == 1 )) && return 0
        common_mergeinparx
    }
    step_mergeinparsel() {
        (( TREE_ENABLED == 1 )) && return 0
        OPTS_MERGEINPAR="$OPTS_MERGEINPAR --cleanseeds"
        common_mergeinparx
    }

    #-----------------------------------------------
    # WEAKBRH
    # Input :
    # Output:
    #-----------------------------------------------
    step_weakbrh() {
        (( TREE_ENABLED == 0 )) || { step_info_msg "WEAKBRH step cannot me run explicitely in TREE mode."; return 0; }
        return 0
    }

    #-----------------------------------------------
    # POSTPROC
    # Input :
    # Output:
    #-----------------------------------------------
    step_postproc() {
        local ec=0
        local flag=""
        [[ $# -lt 1 ]] || flag="$1"
        
        #        # initialise step
        #        init_step "POSTPROC" "$flag"
        #        ec=$?
        #        # step is disabled?
        #        [[ $ec -eq 1   ]] && return 0
        #        # step id done?
        #        [[ $ec -eq 2   ]] && return 0
        #        # step dependancy?
        #        [[ $ec -eq 255 ]] && return 255

        init_postproc_scripts

        #echo "DBG nps ${#OP_POSTPROC_SCRIPTS_FOUND}"
        # no script -> disabled
        [[ ${#OP_POSTPROC_SCRIPTS_FOUND} -eq 0 ]]  && {
            step_warn_msg "disable POSTPROC - no scripts found in labels: $POSTPROC_LABELS"
            disable_step
            step_status_msg 0
            return 0
        }

        # reset postproc variables/functions
        reset_postproc

        # loop over postproc labels
        local nfail=0
        local scripts=()
        eval $(create_array scripts ':' "$OP_POSTPROC_SCRIPTS_FOUND")
        local ind=0
        for ((ind=0; ind<${#scripts[@]}; ind++ )); do
            p="${scripts[$ind]}"
            
            # set postproc
            set_postproc "$p" || continue;

            # set stepid and flag
            # the label POSTPROC_NAME is set by the postproc script
            set_stepid "POSTPROC" "$POSTPROC_NAME"

            # clear variables
            reset_joblists

            # call postproc_conf if defined
            if isfunc postproc_conf; then
                postproc_conf || { step_error_msg "failed config for $POSTPROC_NAME"; return 255; }
            fi

            # make sure POSTPROC_NTARGETS is set
            POSTPROC_NTARGETS=${POSTPROC_NTARGETS=1}

            # mark step start
            step_start $POSTPROC_NTARGETS

            # check if dry run
            is_dryrun && continue

            # run the post processing
            postproc
            local ec=$?
            if [[ $ec -ne 0 ]]; then
                step_error_msg "failed postproc <$POSTPROC_NAME> with error code $ec"
                nfail=$((nfail+1))
                continue
            fi
        done
        # dry run
        is_dryrun && return 0

        reset_joblists

        # reset stepid to general postproc, no flag
        set_stepid "POSTPROC"

        # check if failed
        [[ $nfail -eq 0 ]] || { error_msg "failed postproc"; force_remake_new; return 255; }

        # otherwise mark as OK
        step_done
        return 0
    }

    #-----------------------------------------------
    # check status of pipeline
    #-----------------------------------------------
    proj_load_check_pipeline() {
        local save="$PL_TODO_LABELS_REQ"
        PL_TODO_LABELS_REQ=""
        set_fasta_files
        source "${DIR_ORTHOLOGER_BIN}/check_pipeline_odb.sh"
        PL_TODO_LABELS_REQ="$save"
    }

    #-----------------------------------------------
    # check setup
    #-----------------------------------------------
    proj_check_setup() {
        local save="$PL_TODO_LABELS_REQ"
        PL_TODO_LABELS_REQ=""
        set_fasta_files
        # defines all check_<step> functions
        source "${DIR_ORTHOLOGER_BIN}/check_setup_odb.sh"
        odb_check_setup
        PL_TODO_LABELS_REQ="$save"
    }

    #-----------------------------------------------
    # print estimated disk usage TODO
    #-----------------------------------------------
    print_estimated_usage_step() {
        local stepid="$1"
        local njobs="$2"
        printf " %-20s%20d\n" "$stepid" "$njobs"
    }

    #-----------------------------------------------
    # print estimated resource usage
    #-----------------------------------------------
    print_estimated_usage() {
        local nspec=${#PL_TODO_LABELS[@]}
        local npair=$((nspec*nspec))
        local nbrh=$((nspec*(nspec-1)/2))
        local ntodo=1
        [[ -f "$PL_TODO" ]] || ntodo=0

        echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
        printf " %-20s%20s\n" "STEP ID" "N(JOBS)"
        echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
        print_estimated_usage_step PREPROC       $nspec
        print_estimated_usage_step SELECT        $nspec
        print_estimated_usage_step STATS         $nspec
        print_estimated_usage_step MASKER        $nspec
        print_estimated_usage_step FORMATDB      $nspec
        print_estimated_usage_step ALIGNMENT     $npair
        print_estimated_usage_step MAKEBRH       $nbrh
        print_estimated_usage_step MAKEINPAR     $nbrh
        print_estimated_usage_step MAKEINPARSEL  $nspec
        print_estimated_usage_step SETBRH        $ntodo
        print_estimated_usage_step SETINPAR      $ntodo
        print_estimated_usage_step SETINPARSEL   $ntodo
        print_estimated_usage_step CLUSTER       $ntodo
        print_estimated_usage_step MERGEINPAR    $ntodo
        print_estimated_usage_step MERGEINPARSEL $ntodo
        print_estimated_usage_step POSTPROC      $ntodo
    }

    # set input cluster
    set_input_cluster() {
        [[ $# -eq 1 ]] || return 255
        CLUSTER_INPOG="$1"
    }

    #-----------------------------------------------
    # set tree source based on user input
    # if no TREE_INPUT_FILE:
    # * set TREE_INPUT_FILE to <todo dir>/<todo label>.$EXT_TREE if it exists
    # * set source to tree in OrthoDB URL
    #
    # if URL:
    # * make sure TREE_ROOT_CLADE is set and it is an integer (taxid)
    # * make sure URL is valid
    # if FILE:
    # * get root clade from file (if not set)
    #-----------------------------------------------
    set_tree_source() {
        (( TREE_ENABLED == 1 )) || return 0
        if [[ -n $PL_TODO1 ]] && [[ -n $PL_TODO2 ]]; then
            fatal_msg "PL_TODO1 and PL_TODO2 cannot be set when tree is enabled!"
        fi

        # If no input file is given, check for a file with extension .$EXT_TREE and the same base as the todo file.
        # This means that if you have only specified PL_TODO and TREE_ENABLED=1, it will search
        # for a .$EXT_TREE file. Only if not found it will use the OrthoDB URL.
        local tree2todo=0
        if [[ -z $TREE_INPUT_FILE ]]; then
            local base=$PL_TODO_NAME
            local fdir=$(dirname $PL_TODO)
            local ftree=$fdir/${base}.$EXT_TREE
            [[ -f $ftree ]] && {
                tree2todo=1
                TREE_INPUT_FILE="$ftree"
                info_msg "tree source set to existing file: $ftree"
            }
        fi

        # set URL for retrieving tree
        TREE_INPUT_ODB="$URL_ORTHODB_API/tree"
        
        # if tree file is still not set, assume source is OrthoDB tree
        if   [[ -n $TREE_INPUT_FILE ]]; then
            # it's a file, set TREE_ROOT_CLADE if not set
            set_tree_root || fatal_msg "tree file not found or bad format TREE_INPUT_FILE=$TREE_INPUT_FILE"
            [[ -z $TREE_ROOT_CLADE ]] && TREE_ROOT_CLADE=$PL_TODO_NAME
            
        elif [[ -n $TREE_INPUT_ODB  ]]; then
            # no input tree file is set, use TREE_ROOT_CLADE plus tree from OrthoDB
            [[ -z $TREE_ROOT_CLADE ]] && fatal_msg "TREE_ROOT_CLADE is not set in common.sh (should be a level taxid present in OrthoDB)"
            is_integer "$TREE_ROOT_CLADE" || fatal_msg "TREE_ROOT_CLADE ($TREE_ROOT_CLADE) is set but not numerical (should be a level taxid present in OrthoDB)"

            # check URL
            [[ "${TREE_INPUT_ODB:0:4}" != "http" ]] && fatal_msg "invalid input tree OrthoDB source : $TREE_INPUT_ODB"
            TREE_INPUT_FILE=""
            check_url "$TREE_INPUT_ODB" || fatal_msg "input tree is assumed to be an URL but it is not valid $TREE_INPUT_ODB"
            info_msg "tree source set to OrthoDB via URL: $TREE_INPUT_ODB"
        else
            fatal_msg "no tree source to set!"
        fi
    }

    # disable tree
    disable_tree() {
        [[ $TREE_ENABLED == 0 ]] || info_msg "disabling tree"
        TREE_ENABLED=0
        OP_STEP_ENABLE[PREPARETREE]=0
        #OP_STEP_ENABLE[WEAKBRH]=0
    }

    set_align_merged() {
        # set alignment using merge of fasta files
        ODB_ALIGN_MERGED=1
    }
    
    #---------------------------------------------
    # check input specs - PL_TODO/1/2, TREE mode...
    #---------------------------------------------
    proj_check_input_specs() {
        
        # if tree is not enabled
        if (( TREE_ENABLED == 0 )); then
            [[ -f $PL_TODO ]] || {
		        #warn_msg "no PL_TODO file specified!"
		        return 0
	        }
            return 0
        fi

        #---------------------------------------
        # the rest is ONLY when tree is enabled
        # ensure that TREE_CLUSTER_RUN is valid
        if [[ $TREE_CLUSTER_RUN != "0" ]] && [[ $TREE_CLUSTER_RUN != "1" ]] && [[ $TREE_CLUSTER_RUN != "2" ]]; then
            info_msg "resetting TREE_CLUSTER_RUN"
            TREE_CLUSTER_RUN=0
        fi
        # tree is enabled
        # check priority:
        # 1. TREE_INPUT_FILE
        # 2. TREE_ROOT_CLADE (needs TREE_INPUT_ODB to be defined)
        # 3. PL_TODO
        if [[ -z $TREE_INPUT_FILE ]] && [[ -z $TREE_ROOT_CLADE ]] && [[ -z $PL_TODO ]]; then
            error_msg "at least one of TREE_INPUT_FILE, TREE_ROOT_CLADE or PL_TODO must be set"
            error_msg "  PL_TODO          -  basename must be a valid taxid in OrthoDB"
            error_msg "  TREE_INPUT_FILE  -  basename must be a valid taxid in OrthoDB as well as the root node in the file"
            error_msg "  TREE_ROOT_CLADE  -  must be a valid taxid in OrthoDB"
            return 255
        fi
        
        # get root
        local root=""
        if   [[ -n $TREE_INPUT_FILE ]]; then
            [[ -f $TREE_INPUT_FILE ]] || {
                error_msg "given TREE_INPUT_FILE is not readable $TREE_INPUT_FILE (from $(pwd))"
                return 255
            }
            root=$(print_root_clade) || {
                step_error_msg "failed retrieving root node from tree file $TREE_INPUT_FILE"
                return 255
            }
            
            info_msg "defining input tree based on TREE_INPUT_FILE=$(trim_name $TREE_INPUT_FILE) with root node $root"
            local tbase=$DIR_PROJECT_TODO/$(basename $TREE_INPUT_FILE)
            if [[ ! -f $tbase ]]; then
                info_msg "copying tree file to $tbase"
                cp $TREE_INPUT_FILE $tbase
                TREE_INPUT_FILE="$tbase"
            else
                local md1=$(cat $TREE_INPUT_FILE | md5sum | cut -f 1 -d ' ')
                local md2=$(cat $tbase | md5sum | cut -f 1 -d ' ')
                if [[ "$md1" != "$md2" ]]; then
                    error_msg "Given input tree file (TREE_INPUT_FILE=$TREE_INPUT_FILE) is not the same as file in $tbase"
                    return 255
                fi
            fi
            file_setvar "$DIR_PROJECT/orthologer_conf.sh" "TREE_INPUT_FILE" "$tbase"
            
        elif [[ -n $TREE_ROOT_CLADE ]]; then
            root=$TREE_ROOT_CLADE
            info_msg "defining input tree based of TREE_ROOT_CLADE=$TREE_ROOT_CLADE assuming file root is a valid taxid in OrthoDB"
        else
            root=$(file_basename "$PL_TODO")
            info_msg "defining input tree based on PL_TODO=$(trim_name $PL_TODO) and assuming file basename is a valid taxid in OrthoDB"
        fi
        
        # set input variables
        [[ "$PL_TODO" == "$DIR_PROJECT_TODO/${root}.$EXT_TODO" ]] || [[ -z $PL_TODO ]] || {
            warn_msg "resetting PL_TODO from <$PL_TODO> to todo/${root}.$EXT_TODO"
            PL_TODO="$DIR_PROJECT_TODO/${root}.$EXT_TODO"
            PL_TODO_NAME="$root"
            update_todo 1
        }


        if [[ -z $TREE_INPUT_FILE ]]; then
            TREE_INPUT_FILE="$DIR_PROJECT_TODO/${root}.$EXT_TREE"
            info_msg "setting TREE_INPUT_FILE to $TREE_INPUT_FILE"
        fi
        
        
        [[ "$TREE_ROOT_CLADE" == "$root" ]] || [[ -z $TREE_ROOT_CLADE ]] || {
            warn_msg "resetting TREE_ROOT_CLADE from $TREE_ROOT_CLADE to $root"
        }
        TREE_ROOT_CLADE=$root
        # now everything should be consistent
        return 0
    }


    
    # initialise tree structure
    init_tree() {
        (( TREE_ENABLED == 1 )) || { step_warn_msg "tree clustering is not enabled - set TREE_ENABLED=1 in common.sh"; return 0; }
        
        is_dryrun && return 0

        # ensure that tree environment vars are consistent
        set_tree_source
        
        # create tree
        step_info_msg "tree: create tree"
        tree_create  || { step_error_msg "failed creating tree"; return 255;}

        # create todos
        step_info_msg "tree: create todos based on tree"
        tree_create_todos || { step_error_msg "failed creating todo files"; return 255;}

        # set todo to root clade
        PL_TODO="$DIR_PROJECT/todo/${TREE_ROOT_CLADE}.todo"
        PL_TODO_NAME="${TREE_ROOT_CLADE}"
        update_todo 1

        # create pair files
        step_info_msg "tree: create pair files for each internal node"
        tree_create_pairs || { step_error_msg "failed creating pair files"; return 255;}
        
        return 0
    }

    proj_print_setup() {
        local amat="HALF"
        (( ALIGNMENT_MATRIX == 0 )) && amat="FULL"
        printf "%-25s : %s\n" "alignment matrix" "$amat"
        #    printf "%-25s : %s\n" "pairwise A range" "$PL_TODO_NLABELS"
        #    printf "%-25s : %s\n" "pairwise B range" "$PL_TODO_NLABELS"
       

        printf "%-25s : %s\n" "run mode" "$ODB_RUN_MODE"
        #printf "%-25s : %s\n" "MMSEQS_SENSITIVITY"   "$MMSEQS_SENSITIVITY"
        #printf "%-25s : %s\n" "MERGEINPAR_FCLUSTMIN" "$MERGEINPAR_FCLUSTMIN"
        #printf "%-25s : %s\n" "MERGEINPAR_FSPECMIN"  "$MERGEINPAR_FSPECMIN"
        echo

        #if [[ -n $PL_TODO1 ]] && [[ -n $PL_TODO2 ]]; then
        #    printf "%-25s : %s\n" "pairwise todo 1" "$PL_TODO1"
        #    printf "%-25s : %s\n" "pairwise todo 2" "$PL_TODO2"
        #    echo
        #fi

        #printf "%-25s : %s\n"
    }

    print_tree_info() {
        print_separator "="
        (( TREE_ENABLED == 1 )) || { echo "-> tree clustering not enabled"; return; }
        printf "%-25s : %s\n" "tree OrtoDB" "$TREE_INPUT_ODB"
        printf "%-25s : %s\n" "tree file" "$TREE_INPUT_FILE"
        printf "%-25s : %s\n" "tree root clade" "$TREE_ROOT_CLADE"
        print_separator "="
    }

    #===============================================================
    # call main common file
    #===============================================================
    # check for common
    #    [[ -f "$DIR_ORTHOLOGER/bin/common_pipeline.sh" ]] || { echo "FATAL: unable to find main pipeline at $DIR_ORTHOLOGER/bin/common_pipeline.sh"; exit 255; }

    # source the file
    #    source "$DIR_ORTHOLOGER/bin/common_pipeline.sh"



    # A function that summarizes all software used
    # Possibly run that as a postproc plugin
    #dump_exec_versions() {
    #    $DIR_BRHCLUS/brhclus --version
    #}
    
    #===============================================================
    #                EXPORT VARIABLES AND FUNCTIONS
    #===============================================================
    #
    # Export functions
    #
    export -f get_internal_nodes
    export -f get_internal_nodes_noleaves
    export -f get_leaf_nodes
    export -f get_leaf_organisms
    export -f get_branches
    export -f print_root_clade
    export -f print_newick_odb
    export -f print_leaf_organisms
    
    export -f set_tree_root
    
    export -f proj_dump_conf_sh
    export -f find_data_type
    export -f set_blast_exe
    export -f set_executables
    export -f find_exe_dirs
    ###export -f check_data_type

    export -f set_fasta_files
    #export -f set_evalue_cutoffs
    #export -f set_datatype_defs
    export -f set_opts_defs
    export -f set_db_ext
    export -f set_alignment_label
    export -f refresh_dir_project
    export -f proj_check_input_specs
    export -f proj_configure_env
    ###export -f verify_datatype
    export -f verify_parameters
    export -f proj_verify_setup
    
    export -f get_pwc_dir
    export -f get_pwc_file
    export -f get_pwc_file_chk
    export -f get_pwc_bifile_chk

    export -f v_get_pwc_dir
    export -f v_get_pwc_file
    export -f v_get_pwc_file_chk
    export -f v_get_pwc_bifile_chk
    
    export -f create_pwc_dirs
    export -f check_pwc_dirs
    export -f get_external_pwc_dir
    export -f get_external_pwc_file
    export -f get_external_pwc_file_chk
    export -f link_results
    export -f proj_create_dirs
    export -f proj_check_dirs
    export -f proj_usage
    export -f proj_variables_help
    export -f proj_extended_help

    export -f help_step_preproc
    export -f help_step_masker
    export -f help_step_select
    export -f help_step_stats
    export -f help_step_makeinparsel
    export -f help_step_formatdb
    export -f help_step_alignment
    export -f help_step_makebrh
    export -f help_step_makeinpar
    export -f help_step_setbrh
    export -f help_step_setinpar
    export -f help_step_setinparsel
    export -f help_step_preparetree
    export -f help_step_cluster
    export -f help_step_mergeinpar
    export -f help_step_mergeinparsel
    export -f help_step_postproc

    export -f cleanup_step_preproc
    export -f cleanup_step_masker
    export -f cleanup_step_select
    export -f cleanup_step_stats
    export -f cleanup_step_makeinparsel
    export -f cleanup_step_formatdb
    export -f cleanup_step_alignment
    export -f cleanup_step_makebrh
    export -f cleanup_step_makeinpar
    export -f cleanup_step_setbrh
    export -f cleanup_step_setinpar
    export -f cleanup_step_setinparsel
    export -f cleanup_step_preparetree
    export -f cleanup_step_cluster
    export -f cleanup_step_mergeinpar
    export -f cleanup_step_mergeinparsel
    export -f cleanup_step_postproc

    export -f time_step_preproc
    export -f time_step_masker
    export -f time_step_select
    export -f time_step_stats
    export -f time_step_makeinparsel
    export -f time_step_formatdb
    export -f time_step_alignment
    export -f time_step_makebrh
    export -f time_step_makeinpar
    export -f time_step_setbrh
    export -f time_step_setinpar
    export -f time_step_setinparsel
    export -f time_step_preparetree
    export -f time_step_cluster
    export -f time_step_mergeinpar
    export -f time_step_mergeinparsel
    export -f time_step_postproc

    #export -f proj_update_todo
    export -f configure_step_readonly
    export -f proj_step_setup
    export -f proj_op_fix
    ###export -f set_default_nparallel
    export -f set_step_nparallel
    export -f op_step_adjust

    export -f get_preproc_name
    export -f reset_preproc
    export -f check_preproc_setup
    export -f set_preproc
    export -f init_preproc_scripts
    export -f check_preproc_scripts

    export -f get_postproc_name
    export -f reset_postproc
    export -f check_postproc_setup
    export -f set_postproc
    export -f init_postproc_scripts
    export -f check_postproc_scripts

    export -f common_fmt_header
    export -f get_formatdb_exts
    export -f check_formatdb_files
    export -f link_formatdb_files
    export -f remake_pair_target
    export -f remake_target_from_pairwise
    export -f get_median_dbsz
    export -f get_effdbsz
    export -f isok_label
    export -f get_extension
    export -f step_preproc
    export -f step_masker
    export -f step_select
    export -f step_stats
    export -f step_makeinparsel
    export -f step_formatdb
    export -f get_matrix_type
    export -f set_pairwise_file_name
    export -f create_pairwise_file
    export -f step_preparetree
    export -f step_pairwise_file
    export -f step_pairwise
    export -f step_alignment
    export -f step_makebrh
    export -f step_makeinpar
    export -f step_setbrh
    export -f step_setinpar
    export -f step_setinparsel
    export -f tree_create
    export -f tree_create_todos
    export -f tree_create_pairs
    export -f parse_cluster_inpog
    export -f step_cluster
    export -f step_mergeinpar
    export -f step_mergeinparsel
    export -f step_postproc
    export -f proj_load_check_pipeline
    export -f proj_check_setup
    export -f proj_update_todo
    export -f print_estimated_usage_step
    export -f print_estimated_usage
    export -f init_tree
    export -f print_tree_info
    export -f proj_print_setup
    export -f proj_args
    export -f set_input_cluster

    export -f set_run_mode
    export -f check_var_ifset

    export -f proj_print_version

    # flag it's loaded
    LOADED_COMMON_ORTHODB_SH=0

    # setup OP_STEP variables
    op_step_setup

    # the re-load protection
fi
