#!/usr/bin/env bash
#
# orthologer - Pipeline for deducing orthology from fasta files.
# Copyright (C) 2005-2023 E. Zdobnov lab: F. Tegenfeldt
# <fredrik.tegenfeldt@unige.ch> based on code by E. Kriventseva
#
# This file is part of orthologer.
#
# orthologer is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# orthologer is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with orthologer.  If not, see <http://www.gnu.org/licenses/>.
#
# set pipeline directory, ie the dir of this file
export DIR_ORTHOLOGER=${DIR_ORTHOLOGER:="$(cd -P "$(dirname "${BASH_SOURCE[0]}")"/../ && pwd)"}
[[ -d "$DIR_ORTHOLOGER" ]] || { echo "FATAL: unable to determine DIR_ORTHOLOGER! Got <$DIR_ORTHOLOGER>." 1>&2; exit 255; }

#--------------------------------------------------------------------------
# Check prerequisites (bash version)
source $DIR_ORTHOLOGER/bin/check_prerequisites.sh

# Check user id - possibly fail if root
source $DIR_ORTHOLOGER/bin/check_runas_root.sh
#--------------------------------------------------------------------------

# A long if statement that spans over the whole file - do not indent...
if [[ "${LOADED_COMMON_PIPELINE_SH+1}" == "" ]]; then
    # before anything - make sure version.sh exists
    source $DIR_ORTHOLOGER/bin/generate_version.sh
    
    # do not allow unset variables
    set -u
    export SETUP_ENVIRONMENT_STATUS=255

    # pipeline config file (is changed in common_odb.sh)
    export CMD_PIPELINE=${CMD_PIPELINE:="pipeline.sh"}
    
    ###source $DIR_ORTHOLOGER/bin/profiler.sh
    ###profile_on

    #-----------------------------------------------
    # load common base
    #-----------------------------------------------
    [[ -f ${DIR_ORTHOLOGER}/bin/common_base.sh ]] || { echo "FATAL: unable to find common_base.sh in setup at ${DIR_ORTHOLOGER}" 1>&2; exit 255; }

    source "${DIR_ORTHOLOGER}/bin/common_base.sh"

    debug_msg "loaded common_pipeline.sh" 
    # expand the name to full path
    DIR_ORTHOLOGER=$(expand_name "$DIR_ORTHOLOGER")

    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    # VARIABLES
    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    #-----------------------------------------------
    # set global variables
    #-----------------------------------------------
    # flags if common_xxx.sh has been loaded, if so it also gives the error code
    export LOADED_COMMON_PIPELINE_SH=${LOADED_COMMON_PIPELINE_SH:=255}

    # is set to the PID of the root bash processes
    export PID_PIPELINE=
    
    # set in base_pipeline in case of non-fatal (!=255) argument parse error
    export ARGS_PARSE_ERROR=0
    
    # for time_steps - import a pipeline timing from external source (eg parsing the log)
    export PIPELINE_RUNLOG=${PIPELINE_RUNLOG:=}
    
    # pipeline bin
    $DECLARE_G -xr DIR_ORTHOLOGER_BIN="$DIR_ORTHOLOGER/bin"

    # set project base directory and it's sub directories
    export DIR_PROJECT=${DIR_PROJECT:=$(pwd)}

    DIR_PROJECT=$(expand_name "$DIR_PROJECT")
    export DIR_PROJECT_TODO=""   #$DIR_PROJECT/todo

    $DECLARE_G -xr BASE_JOBLOG="JobLog"
    $DECLARE_G -xr BASE_SCRATCH=".scratch"
    export ALL_DIR_PROJECT="DIR_PROJECT"

    # pipeline scrips path
    export DIR_ORTHOLOGER_PATH=${DIR_ORTHOLOGER_PATH=}

    # external tools path
    export DIR_EXTERNAL_PATH=${DIR_EXTERNAL_PATH=}

    # define python3
    export PYTHON3=${PYTHON3:=python3}

    # todo file
    export PL_TODO=${PL_TODO=}
    
    # special TODO files for cases when running one against another
    # if they are set the pairwise alignments will be done on TODO1 against TODO2
    # if A is a label from TODO1 and B from TODO2 then pairs will be run
    # for A_B if A<B and B_A otherwise.
    # Cases when A==B are ignored.
    export PL_TODO1=${PL_TODO1:=""}
    export PL_TODO2=${PL_TODO2:=""}
    # if the above two are set, this flag determines how pairs are generated
    # assume A is from PL_TODO1 and B from PL_TODO2
    # 0 - AA, AB
    # 1 - AA, AB, BB
    export PL_TODO_ALGO=${PL_TODO_ALGO:=0}
    #------------------------------------------------------------
    # OBSOLETE
    # Now has one2one and one2many as options.
    # flag how pairs are constructed from PL_TODO1 and PL_TODO2
    #  0 -> all against all
    #  1 -> pairs are one-to-one, first key of each file form the first pair, second key the second pair, etc
    #       requires that TODO1 and TODO2 have the same nr of keys
    ####export PL_TODO_ONE2ONE=${PL_TODO_ONE2ONE:=0}
    #------------------------------------------------------------
    
    # derived from PL_TODO
    export PL_TODO_NAME=${PL_TODO_NAME:=""}
    export PL_TODO_NLABELS=${PL_TODO_NLABELS:=0}
    export PL_TODO_EXT=${PL_TODO_EXT:=""}
    
    # get version
    export OP_VERSION="${OP_VERSION:=???}"
    export OP_VERSION_SHORT="${OP_VERSION_SHORT:=???}"
    export OP_VERSION_HASH="${OP_VERSION_HASH:=???}"
    export OP_VERSION_DATE="${OP_VERSION_DATE:=???}"
    if [[ -f ${DIR_ORTHOLOGER}/bin/version.sh ]]; then
        # extract only relevant lines... in case someone is nasty...
        op=$(grep -e "OP_VERSION=" -e "OP_VERSION_SHORT=" -e "OP_VERSION_HASH=" -e "OP_VERSION_DATE=" "${DIR_ORTHOLOGER}/bin/version.sh")
        [[ -z $op ]] || eval "$op"
    fi

    # ignore missing files flag
    # 0 -> fail when an input file is missing
    # if this is set to 1, a run request will run even though pending jobs exists
    export FORCE_RUN=${FORCE_RUN:=0}

    # 0 -> always check if target needs to be (re)made
    # 1 -> never check
    export SKIP_REMAKE_CHECK=${SKIP_REMAKE_CHECK:=0}

    # 0 -> check error logs
    # 1 -> do not check
    export SKIP_CHECK_ERROR_LOGS=${SKIP_CHECK_ERROR_LOGS:=0}

    # in get_run_script it will optionally check for a _nochecks.sh alternative
    # 0 -> get regular script, e.g run_makebrh.sh
    # 1 -> get nochecks script, e.g run_makebrh_nochecks.sh
    export USE_NOCHECKS=${USE_NOCHECKS:=0}

    # done codes
    declare -xr DC_DONE=0
    declare -xr DC_MAKE_NEW=1
    declare -xr DC_MAKE_ALL=2
    declare -xr DC_MAKE_MISSING=3
    declare -xr DC_UNKNOWN=9

    #-----------------------------------------------
    # variables specific for pipeline
    #-----------------------------------------------

    # implementation options and hooks/help
    export PROJ_OPTS=""             # options without args
    export PROJ_OPTS_WARGS=""       # ... with args
    $DECLARE_G -A PROJ_OPTS_HOOK    # ... what to run
    $DECLARE_G -A PROJ_OPTS_ARGHELP # ... arg help

    # global variables
    export PL_TODO_LABELS_REQ=${PL_TODO_LABELS_REQ=}  # if set, the pairwise steps will require this species

    # step specific variables - should these really be settable before???
    export STEP_ID=${STEP_ID:=""}              # contains the current step
    export STEP_ID_FLAG=${STEP_ID_FLAG:=""}    # contains the current step flag
    export STEP_FULL_ID=${STEP_FULL_ID:=""}    # same as STEP_ID but may be of the format <STEP_ID>:<sub id>
    export STEP_FULL_LBL=${STEP_FULL_LBL:=""}  # same as STEP_ID_FULL but with ':' replaced by '_'
    export STEP_ID_STATUS=${STEP_ID_STATUS:="0"} # contains the current status flag (<STEP>=<status>)
    export STEP_DONE_CODE=${STEP_DONE_CODE:=$DC_DONE} # global remake flag

    # time to sleep between steps
    export STEP_SLEEP=${STEP_SLEEP:="0"}
    
    # label used to mark a job log not related to any species
    # can be any weird meaningless string
    declare -xr JOBLOGNOSPEC="$NONE_VALUE"
    
    # general job arrays
    # JOBARGS
    #  array of arguments, specifies the nr of jobs that need to be run.
    #  This defines the argument given to each call to the job script.
    #  It is used in
    #     run_cmd()
    #
    # JOBLABELS
    #  array of job labels, used in run_cmd()
    $DECLARE_G -a JOBARGS         # list of job arguments for given list of source files
    $DECLARE_G -a JOBLABELS       # list of job labels
    $DECLARE_G -a JOBSUBLBL       # list of main species per job label
    $DECLARE_G -a SRCFILES        # list of source files
    $DECLARE_G -a TRGFILES        # list of target files

    # for save_jobspecs()
    $DECLARE_G -a _save_JOBARGS         # list of job arguments for given list of source files
    $DECLARE_G -a _save_JOBLABELS       # list of job labels
    $DECLARE_G -a _save_JOBSUBLBL       # list of main species per job label
    
    # todo file related variables - set by set_todo_vars() function
    $DECLARE_G -xa PL_TODO_LABELS

    # the LBLIND dict is used only by run_siblings*.sh
    # disable it as it takes time to create when handling large arrays
    # KEEP IT
    ###$DECLARE_G -A PL_TODO_LBLIND

    # possibly remove this one
    ###export PL_TODO_LBLSTR=${PL_TODO_LBLSTR=""}
    
    # dont export theses as they can generate too long arguments to commands in bash
    ###export_decl_add_assoc_noval PL_TODO_LBLIND
    export_decl_add_noval       PL_TODO_LABELS
    
    #-----------------------------------------------
    # Declare and configure the associative arrays
    # for configuring each step in the pipeline
    #-----------------------------------------------
    declare -xr OP_STEP_TYPE_PWC=0
    declare -xr OP_STEP_TYPE_LABEL=1
    declare -xr OP_STEP_TYPE_SINGLE=2
    declare -xr OP_STEP_TYPE_TREE=3

    source $DIR_ORTHOLOGER/bin/declare_arrays.sh
    
    # the below just marks the (associative) arrays for export using export_decl_stdout()
    export_decl_add OP_ALL_STEPS
    export_decl_add OP_STEP_NPARALLEL
    export_decl_add OP_STEP_NTHREADS
    export_decl_add OP_STEP_NMERGE
    export_decl_add OP_STEP_ENABLE
    export_decl_add OP_STEP_RUNMODE
    export_decl_add OP_STEP_DATA
    ####export_decl_add OP_STEP_COMPRESS
    export_decl_add OP_STEP_DEPENDS
    export_decl_add OP_STEP_REQBY
    export_decl_add OP_STEP_REQTOOLS
    export_decl_add OP_STEP_DESCRIPTION
    export_decl_add OP_STEP_SCHEDOPTS
    
    export OP_ALL_STEPS_STR=""        # string version of array - required in order to be accesible in sub-shells (notably xargs)
    export OP_ALL_STEPS_N=0           # nr of elements in array 

    export OP_SAVE_JOBLOG=${OP_SAVE_JOBLOG=1}      # if 1, save all job logs, otherwise only in case of error

    # scheduler support - set in base_configure_env()
    export SCHEDULER_SUPPORT=${SCHEDULER_SUPPORT=}
    export SCHEDULER_LABEL=${SCHEDULER_LABEL:="NONE"}
    export SCHEDULER_SCRIPT=${SCHEDULER_SCRIPT=}
    
    export OP_NJOBMAX_BATCH=${OP_NJOBMAX_BATCH:=100}
    export OP_NJOBMAX_LOCAL=${OP_NJOBMAX_LOCAL:=0}
    
    # plugins
    export ALL_PLUGINS="SCHEDULER"

    # plugin empty requirement string
    $DECLARE_G -xr PLUGIN_NONE_REQ="<none>"

    # help msg strings
    $DECLARE_G -xr str_help_input="\n${ALWAYS_BOLD}Input:${ALWAYS_NORMAL}"
    $DECLARE_G -xr str_help_script="\n${ALWAYS_BOLD}Script:${ALWAYS_NORMAL}"
    $DECLARE_G -xr str_help_output="\n${ALWAYS_BOLD}Output:${ALWAYS_NORMAL}"
    $DECLARE_G -xr str_help_desc="\n${ALWAYS_BOLD}Description:${ALWAYS_NORMAL}"
    $DECLARE_G -xr str_help_vars="\n${ALWAYS_BOLD}Variables:${ALWAYS_NORMAL}"
    $DECLARE_G -xr str_help_fmt="\n${ALWAYS_BOLD}Format:${ALWAYS_NORMAL}"

    # project functions
    # 1. required
    $DECLARE_G -xr PROJ_FUNCTIONS_REQUIRED="proj_dump_conf_sh proj_configure_env proj_create_dirs proj_args proj_verify_setup"
    # 2. optional
    $DECLARE_G -xr PROJ_FUNCTIONS_OPTIONAL="proj_update_todo proj_usage proj_extended_help"

    # set level of complexity for finding a filename
    # 0 -> default: checks if there are ambiguous files (fname.gz AND fname)
    # 1 -> just checks for existance of expected file
    #####export CHECK_FILENAME_LEVEL=${CHECK_FILENAME_LEVEL=0}
    
    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    # FUNCTIONS
    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    set_python3() {
        if [[ -x $PYTHON3 ]]; then
            #info_msg "python3 executable at $PYTHON3 and version $($PYTHON3 --version)"
            return 0
        fi
        [[ $PYTHON3 == python3 ]] && return 0
        [[ $PYTHON3 == python  ]] && return 0

        PYTHON3=$(/usr/bin/which python3 2>/dev/null)
        if [[ -z $PYTHON3 ]]; then
            # still not defined - try /usr/bin/which on python and check version
            PYTHON3=$(/usr/bin/which python 2>/dev/null)
            if [[ -x $PYTHON3 ]]; then
                local vtst=$($PYTHON3 --version)
                [[ "${vtst::8}" == "Python 3" ]] || PYTHON3=""
                    
            fi
        fi
        if [[ -z $PYTHON3 ]]; then
            fatal_msg "no valid python v3.x found on $(uname -a)"
        fi
        if [[ ! -x $PYTHON3 ]]; then
            fatal_msg "unable to set PYTHON3 -> got $PYTHON3"
        fi
        #info_msg "python3 executable at $PYTHON3 and version $($PYTHON3 --version)"
        return 0
    }
    
    #-----------------------------------------------
    # Guess file type
    # It will skip commented lines, ie starting with '#'
    # The result of a very simple analysis is summarized in
    # _FILE_TYPE:
    # * NONE     - accompanied by a non-zero error code
    # * FASTA    - regular fasta file, just checking the first entry
    # * 1COL     - a file with a single column of labels
    # * 2COL     - like prervious but with two columns
    # * UNKNOWN  - unclassified
    #-----------------------------------------------
    export _FILE_TYPE=""
    file_type() {
        _FILE_TYPE=""
        local f="$1"
        [[ -f $f ]] || {
            debug_msg "file_type: cannot read file $f"
            return 255
        }
        local aa_pattern='^[\*-ABCDEFGHIJKLMNOPQRSTUVWXY]+$'
        local ws_pattern='[[:space:]]'
        local -u line="" # always upper case
        #declare -a elems=
        local countdown=0
        local rval=10
        
        while read line; do
            [[ ${line:0:1} == "#" ]] && continue
            [[ ${#line} == 0 ]] && continue
            debug_msg "check line: <$line>"
            if [[ -z $_FILE_TYPE ]]; then
                if [[ ${line:0:1} == ">" ]]; then
                    debug_msg "file_type: prelim FASTA"
                    _FILE_TYPE="_FASTA"
                    continue
                fi
                read -r -a elems <<< "$line"
                if [[ ${#elems[@]} -eq 1 ]]; then
                    debug_msg "file_type: prelim 1COL"
                    _FILE_TYPE="_1COL"
                    countdown=5
                elif [[ ${#elems[@]} -eq 2 ]]; then
                    debug_msg "file_type: prelim 2COL"
                    _FILE_TYPE="_2COL"
                    countdown=5
                else
                    debug_msg "file_type: UNKNOWN"
                    debug_msg "nr of columns = ${#elems[@]}"
                    _FILE_TYPE="UNKNOWN"
                    rval=1
                    break
                fi
            fi
            if [[ ${_FILE_TYPE} == "_FASTA" ]]; then
                if [[ "$line" =~ $aa_pattern ]]; then
                    _FILE_TYPE="FASTA"
                    rval=0
                    debug_msg "file_type: FASTA"
                else
                    _FILE_TYPE="UNKNOWN"
                    rval=1
                    debug_msg "file_type: FASTA -> UNKNOWN"
                fi
                break
            fi
            if (( countdown > 0 )); then
                read -ra elems <<< "$line"
                if [[ $_FILE_TYPE == "_1COL" ]]; then
                    if [[ ${#elems[@]} -eq 1 ]]; then
                        countdown=$((countdown-1))
                        if (( countdown == 0 )); then
                            debug_msg "file_type: 1COL"
                            _FILE_TYPE="1COL"
                            rval=0
                            break
                        fi
                    else
                        debug_msg "file_type: 1COL -> UNKNOWN"
                        _FILE_TYPE="UNKNOWN"
                        rval=1
                    fi
                    continue
                fi
                if [[ $_FILE_TYPE == "_2COL" ]]; then
                    if [[ ${#elems[@]} -eq 2 ]]; then
                        countdown=$((countdown-1))
                        if (( countdown == 0 )); then
                            debug_msg "file_type: 2COL"
                            _FILE_TYPE="2COL"
                            rval=0
                            break
                        fi
                    else
                        debug_msg "file_type: 2COL -> UNKNOWN"
                        _FILE_TYPE="UNKNOWN"
                        rval=1
                    fi
                    continue
                fi
            fi
        done < <(grep -v '#' "$f")
        (( countdown > 0 )) && {
            _FILE_TYPE=${_FILE_TYPE//_/}
            rval=0
        }
        debug_msg "file_type: return $_FILE_TYPE with ec=$rval"
        return $rval
    }
    
    #-----------------------------------------------
    # Trim filename directory for output
    #-----------------------------------------------
    trim_name() {
        [[ $# -ge 1 ]] || return 0
        local trim=${2-"$DIR_PROJECT"}
        reduce_name "$1" "$trim"
        return $?
    }
    
    #-----------------------------------------------
    # add a plugin name
    # name is case insensitive (will be forced
    # to upper case)
    #-----------------------------------------------
    add_plugin() {
        [[ $# -eq 1 ]] || return 0
        local name="$1"
        name=${name^^}
        inrange_set "$name" $ALL_PLUGINS && return 1
        ALL_PLUGINS="$ALL_PLUGINS $name"
        return 0
    }

    #-----------------------------------------------
    # Add project dir variable name
    #
    # NOTE: one should add the env variable name
    # and not the actual directory
    #-----------------------------------------------
    add_project_dir() {
        [[ $# -eq 1 ]] || return 0
        local name="$1"
        inrange_set "$name" $ALL_DIR_PROJECT && return 1
        ALL_DIR_PROJECT="$ALL_DIR_PROJECT $name"
        export $name
        return 0
    }


    #-----------------------------------------------
    # check implementation specific functions
    #-----------------------------------------------
    check_proj_functions() {
        # first check required functions
        local nmissing=0
        local f=
        for f in $PROJ_FUNCTIONS_REQUIRED; do
            check_function "$f" 1 || nmissing=$((nmissing+1))
        done

        # check non-required functions
        for f in $PROJ_FUNCTIONS_OPTIONAL; do
            check_function "$f" 0
        done

        return $nmissing
    }

    #-----------------------------------------------
    # dump the status of the possible project
    # dependent functions
    #-----------------------------------------------
    dump_proj_functions() {
        # load all check functions
        ###    load_check_pipeline

        print_separator '~'
        header_status_msg "Function"
        print_separator '~'

        local nmissing=0
        local fun="proj_dump_conf_sh proj_configure_env proj_create_dirs"
        fun="$fun proj_usage proj_extended_help proj_update_todo proj_args"
        fun="$fun proj_verify_setup"

        local f=
        for f in $PROJ_FUNCTIONS_REQUIRED; do
            check_function_status "$f" 1 && continue
            nmissing=$((nmissing+1))
        done

        print_separator '-'
        for f in $PROJ_FUNCTIONS_OPTIONAL; do
            check_function_status "$f" 0 && continue
        done

        print_separator '-'
        local s=
        for s in "${OP_ALL_STEPS[@]}"; do
            local slow=${s,,}
            f=help_step_$slow
            check_function_status "$f" 0 && continue
        done

        print_separator '-'
        for s in "${OP_ALL_STEPS[@]}"; do
            local slow=${s,,}
            f=check_step_$slow
            check_function_status "$f" 0 && continue
        done

        print_separator '-'
        for s in "${OP_ALL_STEPS[@]}"; do
            local slow=${s,,}
            f=cleanup_step_$slow
            check_function_status "$f" 0 && continue
        done
        print_separator '~'
        local ec=0
        local nmissing=0

        return $nmissing
    }


    #-----------------------------------------------
    # load user config file
    # NOT SURE IF THIS IS USEFUL
    #-----------------------------------------------
    reload_usercommon() {
        local f="${DIR_PROJECT}/orthologer_conf.sh"
        [[ -f "$f" ]] || fatal_msg "unable to reload user orthologer_conf.sh at <$f>"

        # force reload
        LOADED_COMMON_PIPELINE_SH=
        source "$f"
    }

    #-----------------------------------------------
    # lock dir name
    #-----------------------------------------------
    lock_dir_name() {
        [[ $# -eq 1 ]] || return
        local label="$1"
        echo "${DIR_PROJECT}/${BASE_SCRATCH}/lock_${PL_TODO_NAME}_${label}"
    }
    v_lock_dir_name() {
        [[ $# -eq 2 ]] || return
        local var="$1"
        local label="$2"
        local "$var" && set_var "$var" "${DIR_PROJECT}/${BASE_SCRATCH}/lock_${PL_TODO_NAME}_${label}"
    }

    #-----------------------------------------------
    # lock request
    #
    # It uses mkdir to create a lock directory.
    #-----------------------------------------------
    request_lock() {
        check_nargs $# 3 "$FUNCNAME"

        local label="$1"
        local tsleep=$2
        local ntries=$3

        local lock=""
        ###        lock=$(lock_dir_name "$label")
        v_lock_dir_name lock "$label"

        # possible return values:
        # 0 : lock achieved
        # 1 : bad parameters
        # 2 : time out (no more trials)
        local rval=2
        local N=0
        while [[ $N -lt $ntries ]]; do
            # the lock file is a directory
            # this means that test for existance and creation is in one command
            if  mkdir "$lock" >&/dev/null; then
                rval=0
                break
            fi
            sleep $tsleep
        done
        # return code
        return $rval
    }

    #-----------------------------------------------
    # remove lock file
    #-----------------------------------------------
    remove_lock() {
        check_nargs $# 1 "$FUNCNAME"

        local label="$1"
        local lock=""
        ###lock=$(lock_dir_name "$label")
        v_lock_dir_name lock "$label"
        rmdir "$lock"
    }

    #-----------------------------------------------
    # check if lock file is active
    #-----------------------------------------------
    is_locked() {
        check_nargs $# 1 "$FUNCNAME"
        local label="$1"
        local lock=""
        ###lock=$(lock_dir_name "$label")
        v_lock_dir_name lock "$label"
        [[ -d "$lock" ]] && return 0
        return 1
    }

    #-----------------------------------------------
    # handle jobid files
    #
    # Jobid files are created one per pipeline run.
    # The filename template is:
    #   DIR_PROJECT/BASE_SCRATCH/jobid_<TODO>_<label>
    #
    # It contains the job id's of all submitted
    # jobs.
    #-----------------------------------------------
    get_jobid_file_templ() {
        local d="$DIR_PROJECT/$BASE_SCRATCH"
        iswriteable "$d" || fatal_msg "jobid file directory is not writable <$d>"
        local f="jobid_${PL_TODO_NAME}_"
        echo "$d/$f"
    }
    v_get_jobid_file_templ() {
        local var="$1"
        local d="$DIR_PROJECT/$BASE_SCRATCH"
        iswriteable "$d" || fatal_msg "jobid file directory is not writable <$d>"
        local f="jobid_${PL_TODO_NAME}_"
        local "$var" && set_var "$var" "$d/$f"
    }

    get_jobid_file() {
        [[ $# -eq 1 ]] || return 1
        local label="$1"
        echo "$(get_jobid_file_templ)${label}"
    }
    v_get_jobid_file() {
        [[ $# -eq 2 ]] || return 1
        local var="$1"
        local label="$2"
        ###echo "$(get_jobid_file_templ)${label}"
        local ftmp=""
        v_get_jobid_file_templ ftmp
        local "$var" && set_var "$var" "${ftmp}${label}"
    }

    add_jobid_to_file() {
        [[ $# -eq 2 ]] || return 1
        local label="$1"
        local jobid="$2"
        local fjid=""
        ###fjid=$(get_jobid_file "$label")
        v_get_jobid_file fjid "$label"
        debug_msg "add id $jobid to file $fjid"
        # request lock
        #  > sleep  = 2s
        #  > ntries = 50
        request_lock "$label" 2 50
        if [[ $? -ne 0 ]]; then
            error_msg "failed obtaining jobid file lock for $label"
            exit 255
        fi
        # add jobid
        echo "$jobid" >> "$fjid"
        # remove lock
        remove_lock "$label"
        debug_msg "added job id $jobid to file $fjid"
    }

    replace_jobid_file() {
        # replace jobid file content
        [[ $# -eq 2 ]] || return 1
        local label="$1"
        shift
        local jobids="$*"
        local fjid=""
        ###fjid=$(get_jobid_file "$label")
        v_get_jobid_file fjid "$label"
        # request lock
        #  > sleep  = 2s
        #  > ntries = 50
        request_lock "$label" 2 50
        if [[ $? -ne 0 ]]; then
            error_msg "failed obtaining jobid file lock for $label"
            exit 255
        fi

        # replace contents
        local j=""
        local N=0
        for j in $jobids; do
            echo "$j"
            N=1
        done > "$fjid"

        # if file is empty, remove it
        [[ $N -gt 0 ]] || rm -f "$fjid"

        # remove lock
        remove_lock "$label"
    }

    update_jobid_file() {
        # update jobid file with running/pending jobs
        [[ $# -eq 1 ]] || return 1
        local label="$1"
        local jobids=""
        jobids=$(sj_check_jobids "$label")
        replace_jobid_file "$label" $jobids
        return $?
    }

    clear_jobid_file() {
        [[ $# -eq 1 ]] || return
        local label="$1"
        local fjid="";
        ###fjid=$(get_jobid_file "$label")
        v_get_jobid_file fjid "$label"
        [[ -f "$fjid" ]] || return
        rm -f "$fjid"
    }

    get_jobids() {
        # gets the content of the job id file
        [[ $# -eq 1 ]] || return
        local label="$1"
        local fjid
        ###fjid=$(get_jobid_file "$label")
        v_get_jobid_file fjid "$label"
        [[ -f "$fjid" ]] && cat "$fjid"
    }

    get_njobs() {
        # gets the nr of jobs without checking if they are running/pending
        [[ $# -eq 1 ]] || { echo 0; return; }
        local label="$1"
        local fjid
        ###fjid=$(get_jobid_file "$label")
        v_get_jobid_file fjid "$label"
        [[ -f "$fjid" ]] || { echo 0; return; }
        wc -l "$fjid" | cut -f 1 -d " "
    }

    #-----------------------------------------------
    # scheduler
    #-----------------------------------------------
    uses_scheduler() {
        # if job scheduler label is none or empty, return 1 since none is used
        [[ "$SCHEDULER_LABEL" == "NONE" ]] && return 1
        [[ "$SCHEDULER_LABEL" == "" ]]     && return 1
        return 0
    }

    #-----------------------------------------------
    # create a header in a temporary file
    #-----------------------------------------------
    print_header() {
        [[ $# -eq 2 ]] || return 1
        local title="$1"
        local hdate="$2"
        
        # get job id
        local jid="<NOT FOUND>"
        if [[ -z $SJ_BATCH_JOBID_NAME ]]; then
            jid=$SHELL
        else
            set +u
            jid=${!SJ_BATCH_JOBID_NAME}
            set -u
            [[ -n $jid ]] || jid=$SHELL
        fi
        # dump every thing into one file
        echo "#----------------------------------------------------------------------"
        echo "# Version : $OP_VERSION"
        echo "# Title   : $title"
        echo "# Date    : $hdate"
        echo -n "# System  : " && uname -a
        echo "# User    : $SYS_WHOAMI"
        echo "# Job id  : $jid"
        echo "# Work dir: $DIR_PROJECT"
        echo "#----------------------------------------------------------------------"
    }
    
    create_header() {
        [[ $# -eq 2 ]] || return 1
        local title="$1"
        local hdate="$2"
        local tmpfile=
        tmp_make tmpfile "header" || { error_msg "create_header() failed creating tmp header"; return 255; }

        print_header "$title" "$hdate" > "$tmpfile" || { error_msg "create_header() failed writing tmp header"; return 255; }
        
        echo "$tmpfile"
    }

    #-----------------------------------------------
    # adds a given text to the top of a file
    #-----------------------------------------------
    add_to_top() {
        [[ $# -eq 3 ]] || return 1
        local inpf=$1
        local outf=$2
        local text="$3"
        local ec=0
        local tmpfile="";
        tmp_make tmpfile "header"
        [[ $? -eq 0 ]] || return 255
        local tmpout="";
        tmp_make tmpout "header"
        [[ $? -eq 0 ]] || { rm -f "$tmpfile"; return 255; }
        echo -e "$text" > "$tmpfile"
        [[ $? -eq 0 ]] || { rm -f "$tmpfile" "$tmpout"; return 255; }
        cat "$tmpfile" "$inpf" > "$tmpout"
        [[ $? -eq 0 ]] || { rm -f "$tmpfile" "$tmpout"; return 255; }
        mv -f "$tmpout" "$outf" || return 255
        rm -f "$tmpfile"
    }
    #-----------------------------------------------
    # add a header
    # input and output may be the same files
    #-----------------------------------------------
#    OLD_add_header() {
#        [[ $# -eq 4 ]] || return 1
#        local inpf="$1"
#        local outf="$2"
#        local date="$3"
#        local title="$4"
#        if [[ ! -f "$inpf" ]]; then
#	        touch "$inpf" || fatal_msg "failed creating file $inpf"
#        fi
#        
#        local tmpfile=""
#        tmpfile=$(create_header "$title" "$date")  || { rm -f "$tmpfile"; return 2; }
#
#        local tmpout=""
#        tmp_make tmpout "header" || return 3
#        
#        cat "$tmpfile" "$inpf" > "$tmpout"
#        if [[ $? -ne 0 ]]; then
#            rm -f "$tmpout"
#            return 4
#        fi
#
#        mv -f "$tmpout" "$outf" || return 255
#        rm -f "$tmpfile"
#    }

    add_header() {
        [[ $# -eq 3 ]] || return 1
        local inpf="$1"
        local date="$2"
        local title="$3"
        if [[ ! -f "$inpf" ]]; then
	        touch "$inpf" || fatal_msg "failed creating file $inpf"
        fi
        local tmpout=""
        tmp_make tmpout "header" || return 3
        {
            print_header "$title" "$date" || return 4
            cat "$inpf" || return 4
        } > "$tmpout"

        mv -f "$tmpout" "$inpf" || return 255
        chmod 644 "$inpf"
    }

    export TMP_HEADER_FILE=""
    add_header_tmp() {
        [[ $# -eq 3 ]] || return 1
        local inpf="$1"
        local date="$2"
        local title="$3"
        if [[ ! -f "$inpf" ]]; then
	        touch "$inpf" || fatal_msg "failed creating file $inpf"
        fi

        tmp_make TMP_HEADER_FILE "header" || return 3
        {
            print_header "$title" "$date" || return 4
            cat "$inpf" || return 4
        } > "$TMP_HEADER_FILE"
    }
    #-----------------------------------------------
    # add footer
    # format is (all times in seconds)
    # #TIME : <node> <time> <bogomips scaled time>
    #-----------------------------------------------
    export _FOOTER_TIME=0
    get_footer_time() {
        _FOOTER_TIME=0
        local f="$1"
        [[ -f "$f" ]] || return 255
        local saveIFS=$IFS
        IFS=' '
        local A=($(tail -n 2 $f | head -n 1))
        [[ "${A[0]}" == "#TIME" ]] && _FOOTER_TIME=${A[3]}
        IFS=$saveIFS
        return 0
    }
    
    print_footer() {
        local tval="$1"
        echo "#TIME : $(uname -n) $tval"
        echo "#END at $(date)"
    }
    
    add_footer() {
        [[ $# -gt 2 ]] && return 1
        [[ $# -eq 0 ]] && return 1
        local outf="$1"
        [[ -f "$outf" ]] || return 255
        
        local ttext=""
        if [[ $# -eq 2 ]]; then
            local tval=$2
            #        local bogomips
            #        bogomips=$(get_bogomips)
            #        local bogoref=5000.0
            # scale time linearly with bogomips value
            # makes a more comparable time over different machines
            #        local trel=0
            #        trel=$(calc "$tval*$bogoref/$bogomips")
            #	    ttext="#TIME : $(uname -n) $tval $trel"
            ttext="#TIME : $(uname -n) $tval"
        fi
        local endtxt=
        endtxt="#END at $(date)"

        # append the footer to the file
        {
            echo $ttext
            echo $endtxt
        } >> "$outf"
        [[ $? -eq 0 ]] || { error_msg "failed appending footer to $outf"; return 255; }
        chmod 644 "$outf"
        return 0
    }

    #-----------------------------------------------
    # add header and footer
    # input and output may be the same files
    #-----------------------------------------------
    add_header_footer() {
        [[ $# -eq 6 ]] || return 1
        local label="$1"
        local input="$2"
        local output="$3"
        local hdrdate="$4"
        local deltime="$5"
        local cmd="$6"
        local ec=0
        
        # add header to input
        add_header_tmp "$input" "$hdrdate" "$cmd"
        ec=$?
        if [[ $ec -ne 0 ]]; then
            error_msg "$label : unable to add header with ec=$ec"
            return $ec
        fi

        # add footer...
        add_footer "$TMP_HEADER_FILE" "$deltime"
        ec=$?
        if [[ $ec -ne 0 ]]; then
            error_msg "$label : unable to add footer"
            return $ec
        fi
        # move to output if it's not the same string
        if [[ ${#output} -gt 0 ]]; then
            mv -f "$TMP_HEADER_FILE" "$output" || { error_msg "$label : unable to save updated file at $output"; return 5; }
        fi
        chmod 644 "$output"
        
        return 0
    }

    #-----------------------------------------------
    # check footer
    #
    # This will check for the end tag in a file.
    # The input file is either a plain or
    # compressed (gzip) text file.
    #
    # Returns:
    #  0 - iff end tag is found as the last line
    #  1 - otherwise (also if file is not found)
    #-----------------------------------------------
    check_footer() {
        [[ $# -eq 1 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local f="$1"
        [[ -f "$f" ]] || return 1

        tail -n 1 "$f" | grep -q "#END"
        [[ $? -eq 0 ]] && return 0
        return 1
    }
    #-----------------------------------------------
    # create a copy with updated headers
    #-----------------------------------------------
    make_copy() {
        [[ $# -eq 2 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local fsrc="$1"
        local ftrg="$2"
        local ec=0
        remake_target "$ftrg" "$fsrc" || { step_info_msg "skip copying $fsrc to $ftrg"; return 0; }
        {
            print_header "indentical result -> copied from $fsrc" "$(date)"
            grep -v "^#" "$fsrc"
            echo "#END at $(date)"
        } > "$ftrg"
        [[ $? -eq 0 ]] || fatal_msg "failed copying trivial case from $fsrc to $ftrg"
    }
    #-----------------------------------------------
    # create a config file, printing to stdout
    #-----------------------------------------------
    pl_dump_base_conf_sh() {
        echo_title_reset
        echo "#!/usr/bin/env bash"
        echo "#====================================================="
        echo "# USER PROJECT CONFIG"
        echo "#-----------------------------------------------------"
        echo "# Created by: $SYS_WHOAMI"
        echo "# Date      : $(date)"
        echo "#"
        echo "# NOTE - for detailed help on variables do:"
        echo "#  ./$CMD_PIPELINE -H .variables"
        echo "#"
        echo "#-----------------------------------------------------"
        echo
        echo_title "Verbose mode"
        echo "#    0 - none"
        echo "#    1 - verbose"
        echo "#   >1 - debug"
        echo "VERBOSE=$VERBOSE"
        echo
        echo_title "# limit the nr of error/warnings per step - set to 0 if no limit"
        echo "MAX_MESSAGES=$MAX_MESSAGES"
        echo
        echo_title "Python3"
        echo "PYTHON3=\"$PYTHON3\""
        echo
        echo_title "Pipeline directory"
        echo "DIR_ORTHOLOGER=\"$DIR_ORTHOLOGER\""
        echo
        echo_title "Temp files directory"
        echo "TMP_DIR_BASE=\"$TMP_DIR_BASE\""
        echo
        echo_title "Todo file - defines all labels to be used"
        echo "PL_TODO=\"$PL_TODO\""
        echo
        echo_title "Pair of todo files"
        echo "#    If both are set, the pairwise steps will run PL_TODO1 against PL_TODO2."
        echo "#    Original PL_TODO is used in all non-pairwise steps and is used as project label."
        echo "PL_TODO1=\"$PL_TODO1\""
        echo "PL_TODO2=\"$PL_TODO2\""
        echo

        # OBSOLETE
        #echo "# flag how pairs are copnstructed from PL_TODO1 and PL_TODO2"
        #echo "#  0 -> all against all"
        #echo "#  1 -> pairs are one-to-one, first key of each file form the first pair, second key the second pair, etc"
        #echo "#       requires that TODO1 and TODO2 have the same nr of keys"
        #echo "PL_TODO_ONE2ONE=$PL_TODO_ONE2ONE"

        #        echo_title "Orthologer root dir - do not change"
        #        echo "DIR_ORTHOLOGER=\"\$DIR_ORTHOLOGER\""
        echo
        #echo_title "Enable/disable data compression"
        #echo "#    0 -> no compression"
        #echo "#    1 -> compression using gzip"
        #echo "COMPRESS_DATA=$COMPRESS_DATA"
        echo
        echo_title "Running: scheduler label"
        echo "SCHEDULER_LABEL=$SCHEDULER_LABEL"
        echo
        echo_title "Running: maximum nr of jobs in one batch (running/pending)"
        echo "OP_NJOBMAX_BATCH=$OP_NJOBMAX_BATCH"
        echo
        echo_title "Running: maximum nr of threads available on the local machine"
        echo "OP_NJOBMAX_LOCAL=$(get_njobmax_local)"
        echo
        echo_title "Running: keep JOBLOG flag"
        echo "#    useful flag if the nr of jobs is expected to be very large"
        echo "#    0 -> delete all logs if no error"
        echo "#    1 -> keep logs (delete empty error logs)"
        echo "OP_SAVE_JOBLOG=$OP_SAVE_JOBLOG"
        echo
        echo_title "Running: wait poll time in seconds"
        echo "#    This only affects how often the progress is updated."
        echo "SJ_CHECK_POLLTIME=$SJ_CHECK_POLLTIME"
        echo
        echo_title "Running: skip check for remake of step - overrides whatever done flag is set"
        echo "#    For very large runs the checking can take a significant amount of time."
        echo "#    Disable (set to 1) if this is the case."
        echo "#    0 -> do remake check if required"
        echo "#    1 -> NEVER do a remake check, useful when running over large data sets"
        echo "SKIP_REMAKE_CHECK=$SKIP_REMAKE_CHECK"
        echo
        echo_title "Running: skip check of error logs"
        echo "#    The check assumes there was an error if there is anything in the error log."
        echo "#    It may be useful to skip this check as in some cases"
        echo "#    programs may output to stderr even though it's not an error."
        echo "#    0 -> check contents of error logs"
        echo "#    1 -> skip the check"
        echo "SKIP_CHECK_ERROR_LOGS=$SKIP_CHECK_ERROR_LOGS"
        echo
        echo_title "Running: sleep time between steps"
        echo "#    If set to an integer greater than 0, the process will sleep for that amount of seconds"
        echo "#    between each step."
        echo "#    This is useful if steps fail due to files not found, although they were successfully created in a previous step."
        echo "#    It can e.g happen if the NFS lookupcache is not updated sufficiently fast."
        echo "STEP_SLEEP=$STEP_SLEEP"
        echo
    }

    #-----------------------------------------------
    # dump full config file
    #-----------------------------------------------
    pl_dump_conf_sh() {
        pl_dump_base_conf_sh
        isfunc proj_dump_conf_sh || { error_msg "no proj_dump_conf_sh defined!"; return 255; }
        proj_dump_conf_sh
    }

    #-----------------------------------------------
    # create local pipeline script
    #-----------------------------------------------
    pl_dump_pipeline_sh() {
        echo "#!/usr/bin/env bash"
        echo "#~~~~~~~~~~~~~~~"
        echo "# RUN PIPELINE"
        echo "#~~~~~~~~~~~~~~~"
        echo
        echo "# Make sure it is run as an executable and not using 'source'"
        echo "src0=\"\$0\""
        echo "src1=\"\${BASH_SOURCE[@]}\""
        echo "if [[ \"\$src0\" == \"\$src1\" ]]; then"
        echo
        echo "# Make sure the orthologer environment is not re-loaded"
        echo "[[ \"\${LOADED_COMMON_PIPELINE_SH+1}\" == \"\" ]] || { echo \"the common environment has been loaded in this shell\"; echo \"please exit this shell - probably the local config file or system common_pipeline.sh were sourced directly in this shell\"; exit 255; }"
        echo
        echo "# set DIR_PROJECT"
        echo "__dp=\$(dirname \${BASH_SOURCE[0]})"
        echo "__dp=\$(cd \$__dp && pwd)"
        echo "export DIR_PROJECT=\$__dp"
        echo "unset __dp"
        echo
        echo "# Load user config (orthologer_conf.sh) as well as the orthologer environment"
        echo "[[ -f orthologer_conf.sh ]] || { echo \"ERROR unable to read local orthologer_conf.sh : \$(pwd)/orthologer_conf.sh\" 1>&2; exit 255; }"
        echo "source orthologer_conf.sh"
        echo "[[ \$? -eq 0 ]] || exit 255"
        echo
        echo "# run pipeline"
        echo "pipeline \"\$@\""
        echo "exit \$?"
        echo
        echo "# Capture attempts to run with 'source'."
        echo "else"
        echo "echo \"ERROR: do not run \$src1 using 'source'. Make sure it is executable (chmod +x \$(basename \$0)) and run it as an executable.\""
        echo "fi"
    }

    #===============================================================
    #                        CONFIGURATION
    #===============================================================
    #-----------------------------------------------
    # print the setup (basic info)
    #-----------------------------------------------
    print_setup() {
        # get pid file
        pid_file
        
        local pyver="???"
        if [[ -x $PYTHON3 ]]; then
            pyver=$($PYTHON3 --version)
            pyver=${pyver:7}
        fi
        local steps="$1"
        print_repeated '~' 132 && echo
        printf "%-25s : %s\n" "version"        "$OP_VERSION"
        printf "%-25s : %s\n" "tag date"       "$OP_VERSION_DATE"
        printf "%-25s : %s\n" "python version" "$pyver"
        
        printf "%-25s : %s\n" "software root"  "$DIR_ORTHOLOGER"
        printf "%-25s : %s\n" "project home"   "$DIR_PROJECT"
        echo
        printf "%-25s : %s\n" "dry run"        "$(yesno_str $DRY_RUN)"
        printf "%-25s : %s\n" "verbose"        "$VERBOSE"
        printf "%-25s : %s\n" "scheduler"      "$SCHEDULER_LABEL"
        #    printf "%-25s : %s\n"
        #    printf "%-25s : %s\n"
        #    printf "%-25s : %s\n"
        echo
        printf "%-25s : %s\n" "todo file"        "$PL_TODO"
        if [[ -n $PL_TODO1 ]]; then
            printf "%-25s : %s\n" "todo file 1"      "$PL_TODO1"
            printf "%-25s : %s\n" "todo file 2"      "$PL_TODO2"
            ###printf "%-25s : %s\n" "todo 1 vs 2"      "$(yesno_str $PL_TODO_ONE2ONE)"
        fi
        echo
        printf "%-25s : %s\n" "tree enabled" "$(yesno_str $TREE_ENABLED)"
        if (( TREE_ENABLED == 1 )); then
            printf "%-25s : %s\n" "tree root clade" "$TREE_ROOT_CLADE"
            printf "%-25s : %s\n" "tree OrtoDB" "$TREE_INPUT_ODB"
            printf "%-25s : %s\n" "tree file" "$TREE_INPUT_FILE"
        fi
        echo
        printf "%-25s : %s\n" "todo name"        "$PL_TODO_NAME"
        if [[ -n $OP_LABELA_LAST$OP_LABELA_FIRST$OP_LABELB_LAST$OP_LABELB_FIRST ]]; then
            printf "%-25s : %s\n" "todo range A"        "${OP_LABELA_FIRST},${OP_LABELA_LAST}"
            printf "%-25s : %s\n" "todo range B"        "${OP_LABELB_FIRST},${OP_LABELB_LAST}"
        fi
        printf "%-25s : %s\n" "todo N(lables)"   "$PL_TODO_NLABELS"
        ###printf "%-25s : %s\n" "required labels"  "$PL_TODO_LABELS_REQ"
        echo
        printf "%-25s : %s\n" "selected steps"   "$steps"
        echo
        printf "%-25s : %s\n" "max nr of parallel jobs"   "$OP_NJOBMAX_LOCAL"
        printf "%-25s : %s\n" "max nr of jobs in batch"   "$OP_NJOBMAX_BATCH"
        echo
        isfunc proj_print_setup && proj_print_setup
        echo
        printf "%-25s : %s\n" "PID file"   "$(trim_name $R_PIDFILE)"
        echo
        print_repeated '~' 132 && echo
    }

    #-----------------------------------------------
    # get the script name
    # args: <label> <plugin>
    #
    # echo's the name as <plugin>_<label>.sh
    # all lower case
    #-----------------------------------------------
    script_name() {
        [[ $# -eq 2 ]] || return 0
        local label="$1"
        local plugin="$2"
        echo "${plugin,,}_${label,,}.sh"
    }

    #-----------------------------------------------
    # find script in pipeline path
    #
    # It will look for executable script in
    # DIR_ORTHOLOGER_PATH
    #
    #-----------------------------------------------
    find_script() {
        [[ $# -eq 1 ]] || return 0
        local name="$1"
        [[ -n $DIR_ORTHOLOGER_PATH ]] || set_script_path

        # find executable (-x) in path
        find_in_path "$name" "$DIR_ORTHOLOGER_PATH" "-x"
        [[ $? -eq 0 ]] && return 0

        # not found - quit
        return 1
    }

    #-----------------------------------------------
    # Get the todo name
    # It replaces any spaces in todo base file name
    #-----------------------------------------------
    get_todo_name() {
        [[ $# -eq 1 ]] || return 0

        # get filename without dir and extension
        local todo=$(file_basename "$PL_TODO")

        # replace all spaces with '_'
        replace_char "$todo" " " "_"
    }

    #-----------------------------------------------
    # get max nr of jobs in current machine
    # if OP_NJOBMAX_LOCAL is given and non-zero
    #   cap it at the nr of threads available
    # else
    #   if hyperthreading
    #     set to nr of cores
    #   else
    #     set to nr of cores / 2
    #-----------------------------------------------
    get_njobmax_local() {
        # by default, local machine max
        get_nthreads_conservative $OP_NJOBMAX_LOCAL
    }

    #-----------------------------------------------
    # get nr of jobs to submit in parallel
    #-----------------------------------------------
    get_step_nbatch() {
        local stepid="$STEP_ID"
        [[ $# -eq 1 ]] && stepid="$1"
        local n="$OP_NJOBMAX_BATCH"
        if [[ "${OP_STEP_TYPE[$stepid]}" == "$OP_STEP_TYPE_SINGLE" ]]; then
            n=1
        fi
        echo "$n"
    }
    
    get_step_nparallel() {
        local stepid="$STEP_ID"
        [[ $# -eq 1 ]] && stepid="$1"

        # by default, local machine max
        local rval=${OP_STEP_NPARALLEL[$stepid]}

        # check if batch (or no jobscheduler), if so use NPARALLEL
        local nmax=$(get_njobmax_local)
        (( rval > nmax )) && rval=$nmax

        echo $rval
    }

    get_step_nmerge() {
        local stepid="$STEP_ID"
        [[ $# -eq 1 ]] && stepid="$1"
        is_stepid_valid "$stepid" || fatal_msg "$FUNCNAME invalid STEP_ID = $stepid"

        # by default, local machine max
        local rval=${OP_STEP_NMERGE[$stepid]}
        (( rval > 0 )) || rval=1
        echo $rval
    }

    get_step_minjobs() {
        local nparallel=$(get_step_nparallel POSTPROC)
        local maxnmerge=$(get_step_nmerge POSTPROC)
        local minjobs=$((nparallel*maxnmerge))
        # limit arbitrarily at 10000
        (( minjobs < 10000 )) || minjobs=10000
        echo $minjobs
    }
    
    #-----------------------------------------------
    # check for label TPA
    # it has a special meaning in blast tools
    # and it will cause the masker step to silently fail
    #-----------------------------------------------
    check_todo_special_labels() {
        local ftodo="$1"
        [[ -f $ftodo ]] || return 0
        grep -q "^TPA\$" "$ftodo" && return 1
        return 0
    }
    
    #-----------------------------------------------
    # check todo file
    # 1. PL_TODO is set
    # 2. PL_TODO is set and file readable
    # 3. base name does not contain spaces
    # 4. one species label per line
    # 5. labels contain only a-z,0-9 or '_', case-insensitive
    #
    # return OK (0) if:
    # - PL_TODO is empty (warn msg)
    # - PL_TODO is set but file is not readable (warn msg)
    #
    # return error (1) if:
    # - PL_TODO contains labels with illegal characters
    #
    # return fail (255) if:
    # - any operation failed
    #
    # fatal if:
    # - todo file resorted and not writable
    #
    #-----------------------------------------------
    check_todo_file() {
        local ftodo="$1"
        [[ -n $ftodo ]] || return 0

        local btodo=$(basename "$ftodo")
        [[ "$btodo" == *\ * ]] && fatal_msg "TODO file base name must not contain spaces!"
        
        [[ -f "$ftodo" ]] || return 0
        
        # check characters in file
        verbose_info_msg "check_todo_file: checking for bad characters in labels"
        local badlbl=()
        badlbl=($(grep -v '^#' "$ftodo" | good_label_chr))
        if [[ ${#badlbl[@]} -gt 0 ]]; then
            error_msg "found labels in todo file ($ftodo) with bad characters"
            error_msg "allowed characters are : a-z,0-9 and '_', case-insensitive"
            error_msg "bad labels:"
            local x=""
            for x in "${badlbl[@]}"; do
                echo "$x"
            done
            return 1
        fi
        # check for special labels
        # if fail, it will will call exit 255
        verbose_info_msg "check_todo: checking for special labels"
        check_todo_special_labels "$ftodo" || fatal_msg "todo file ($ftodo) contains a bad label (TPA) - it has a special meaning for the blast tools and it will cause the pipeline to break - please change label!"
        
        # dir name of ftodo
        verbose_info_msg "check_todo: sort todo file"
        local pldir=''
        pldir=$(dirname "$ftodo")

        local tmp=""
        # check that nr of words equals nr of lines
        local nw=0
        local nl=0
        nw=$(grep -v '^#' "$ftodo" | wc -w) # nr of words
        nl=$(grep -c -v '^#' "$ftodo") # nr of lines
        if [[ $nl -ne $nw ]]; then
            
            # nr of lines does not equal to nr of words!
            if iswriteable "$pldir"; then
                # recreate file
                debug_msg "recreate todo file"
                tmp_make tmp "todo"
                todolbls=($(get_lines "$ftodo"))
                local s=""
                # the following will force one word per line
                for s in "${todolbls[@]}"; do
	                # use each non-empty word
	                [[ -z "$s" ]] || echo "$s"
                done | sort -u > "$tmp"

                # copy new to ftodo, sorted and duplicates removed
                cp -f "$tmp" "$ftodo"

                tmp_delete "$tmp"
                verbose_info_msg "restructured and sorted todo file <$ftodo>"
            else
                error_msg "TODO file <$ftodo> does not contain one species per line"
                fatal_msg "unable to update TODO file -> not writeable!"
            fi
        else
            # one label per line - ok
            # check if sorted and unique
            tmp_make tmp "todo"
            
            # only replace if not ok
            sort -u "$ftodo" > "$tmp"
            diff "$ftodo" "$tmp" >& /dev/null
            if [[ $? -ne 0 ]]; then
                if iswriteable "$pldir"; then
                    cp -f "$tmp" "$ftodo"
                    verbose_info_msg "sorted todo file <$ftodo>"
                fi
            fi
            tmp_delete "$tmp"
        fi
        verbose_info_msg "check_todo: done checking $ftodo"
        return 0
    }

    #-----------------------------------------------
    # get TODO label indexes in PL_TODO_LABELS
    #-----------------------------------------------
    get_todo_indexes() {
        [[ $PL_TODO_NLABELS -gt 0 ]] || { echo "0 0"; return 255; }
        
        local startlbl=${1-"${PL_TODO_LABELS[0]}"}
        local stoplbl=${2-""}

        local start=0
        local stop=$PL_TODO_NLABELS
        local startok=0
        local stopok=0
        local ix=0
        for ((ix=0; ix<PL_TODO_NLABELS; ix++)); do
            local A=${PL_TODO_LABELS[$ix]}
            if [[ "$A" == "$startlbl" ]]; then
                startok=1
                start=$ix
                [[ ${#stoplbl} -gt 0 ]] || { stopok=1; break; }
            fi
            if [[ "$B" == "$stoplbl"  ]]; then
                stopok=1
                stop=$((ix+1))
                break
            fi
        done

        echo "$start $stop"
        [[ $stopok -eq 1 && $startok -eq 1 ]] || return 1
        return 0
    }

    #-----------------------------------------------
    # restore PL_TO_LABELS and PL_TODO_LABELIND
    # from PL_TODO_LBLSTR
    # this is useful in subshells where arrays
    # are not exported
    #-----------------------------------------------
    #restore_todo_arrays() {
    #    PL_TODO_LABELS=( $(grep -v "^#" "$PL_TODO" | sort -u --ignore-case ) )
    #    PL_TODO_NLABELS=${#PL_TODO_LABELS[@]}
        #local lbl=""
        #local i=0
        ## loop over all, make sure they are uppercase
        #for ((i=0; i<PL_TODO_NLABELS; i++)); do
        #    lbl=${PL_TODO_LABELS[$i]}
        #    PL_TODO_LABELS[$i]=$lbl
        #    PL_TODO_LBLIND[$lbl]=$i
        #done
    #}
    
    #-----------------------------------------------
    # set todo file related variables
    #-----------------------------------------------
    clear_todo_vars() {
        PL_TODO_LABELS=()
        PL_TODO_NAME=""
        PL_TODO_EXT=""
#        PL_TODO_LBLSTR=""
        #set +u
        #if [[ ${#PL_TODO_LBLIND[@]} -gt 0 ]]; then
        #    local x
        #    for x in ${!PL_TODO_LBLIND[@]}; do
        #        unset PL_TODO_LBLIND[$x]
        #    done
        #    #clear_assoc PL_TODO_LBLIND <- this does not work for bash <4.2
        #fi
        #set -u
    }

    #-----------------------------------------------
    # Will check todo filenames.
    #
    # If tree is enabled:
    #   set PL_TODO from tree name
    # If PL_TODO is NOT set:
    #   if PL_TODO1 is set:
    #     PL_TODO=PL_TODO1
    #
    # Return 0 if todo files are consistent.
    # Return 255 is otherwise.
    #-----------------------------------------------
    set_todo_files() {
        # first priority -> TREE
        if (( TREE_ENABLED == 1 )); then
            local b=$(file_basename "$TREE_INPUT_FILE")
            PL_TODO="${DIR_PROJECT_TODO}/${b}.${EXT_TODO}"
            debug_msg "todo file set (PL_TODO = $PL_TODO) - based on given tree file"
            return 0
        fi

        # second priority -> PL_TODO1/2
        if [[ -n $PL_TODO1 ]]; then
            if [[ -z $PL_TODO2 ]]; then
                error_msg "PL_TODO2 is required when PL_TODO1 is set"
                return 255
            fi
            if [[ "$PL_TODO" != "$PL_TODO1" ]]; then
                error_msg "both PL_TODO1 and PL_TODO are set but to different names"
                error_msg "clear PL_TODO in the config file if PL_TODO1 should be used"
                return 255
            fi
            PL_TODO="$PL_TODO1"
            debug_msg "todo file set (PL_TODO = $PL_TODO) - set equal to PL_TODO"
            debug_msg "todo file PL_TODO2 = $PL_TODO2"
            return 0
        fi

        # last option - now PL_TODO must be set
        if [[ -z $PL_TODO ]]; then
            error_msg "cannot establish a PL_TODO"
            return 255
        fi
        debug_msg "todo file set (PL_TODO = $PL_TODO)"
        return 0
    }
    #-----------------------------------------------
    # fill todo labels array
    # retrives all unique todo labels from PL_TODO/1/2
    #-----------------------------------------------
    set_todo_arrays() {
        debug_msg "create array of todo labels"
        PL_TODO_LABELS=( $(grep -h -v "^#" $PL_TODO $PL_TODO1 $PL_TODO2 | sort -u --ignore-case ) )
        PL_TODO_NLABELS=${#PL_TODO_LABELS[@]}
    }

    #-----------------------------------------------
    # set todo file related variables
    # skip if PL_TODO_NAME is already set
    # called by
    # |- set_todo_vars_with_check
    #    |- update_todo_pipeline
    #       |- update_todo
    #          |- base_configure_env
    #             |- setup_environment
    #                |- base_pipeline
    #                   |- pipeline
    #-----------------------------------------------
    set_todo_vars() {
        debug_msg "set_todo_vars()"
        [[ -n $PL_TODO_NAME ]] && { debug_msg "skipping set_todo_vars as PL_TODO_NAME is set ($PL_TODO_NAME)"; return 0; }
        
        # clear todo variables
        clear_todo_vars
        
        # return ok if no PL_TODO is defined
        if [[ -z $PL_TODO ]]; then
            if (( TREE_ENABLED == 0 )); then
                if [[ -n $TREE_INPUT_FILE ]]; then
                    local b=$(file_basename "$TREE_INPUT_FILE")
                    PL_TODO="${DIR_PROJECT_TODO}/${b}.${EXT_TODO}"
                    debug_msg "todo file set (PL_TODO = $PL_TODO) - based on given tree file"
                fi
            fi
        fi

        if [[ -z $PL_TODO ]]; then
            debug_msg "no todo file set (PL_TODO = $PL_TODO)"
            return 0
        fi
        
        # set todo label
        PL_TODO_NAME=$(get_todo_name "$PL_TODO")
        PL_TODO_EXT=$(file_extension "$PL_TODO")

        # PL_TODO does not exist or empty
        if [[ ! -f "$PL_TODO" ]]; then
            debug_msg "no todo file found (PL_TODO = $PL_TODO), PL_TODO_NAME=$PL_TODO_NAME"
            return 0
        fi

        # sort todo labels
        debug_msg "todo file set (PL_TODO = $PL_TODO)"
        set_todo_arrays

        debug_msg "found ${#PL_TODO_LABELS[@]} todo labels)"
        debug_msg "PL_TODO_NAME = $PL_TODO_NAME"
        return 0
    }

    #-----------------------------------------------
    # the same as previous but does checking of the
    # todo file -> it may change the todo file
    #-----------------------------------------------
    set_todo_vars_with_check() {
        [[ -n $PL_TODO ]] || return 0

        # check and potentially fix todo-file
        check_todo_file "$PL_TODO"  || { error_msg "check_todo_file failed; PL_TODO=$PL_TODO"; return 255; }
        check_todo_file "$PL_TODO1" || { error_msg "check_todo_file failed; PL_TODO1=$PL_TODO1"; return 255; }
        check_todo_file "$PL_TODO2" || { error_msg "check_todo_file failed; PL_TODO2=$PL_TODO2"; return 255; }
        
        # set the variables
        set_todo_vars || return 255
        
        return 0
    }

    #-----------------------------------------------
    # get todo label index for a given label
    # it will print empty string and return 1
    # if nothing is found
    #-----------------------------------------------
    #get_todo_lblind() {
    #    [[ $# -eq 1 ]] || return 0
    #    local lbl=$1
	#    local rval=1
	#    local ind=0
	#    if [[ ${#PL_TODO_LBLIND[@]} -eq $PL_TODO_NLABELS ]]; then
    #        set +u
    #        ind="${PL_TODO_LBLIND[$lbl]}"
    #        [[ -z $ind ]] || rval=0
	#        set -u
	#    else
	#        rval=1
	#        for (( ind=0; ind<PL_TODO_NLABELS; ind++ )); do
    #		        local A="${PL_TODO_LABELS[$ind]}"
	#[[ "$A" == "$lbl" ]] && { rval=0; break; }
	#        done
    #    fi
	#    [[ $rval -eq 0 ]] && echo "$ind"
    #    return $rval
    #}


    #-----------------------------------------------
    # update everything which depends on TODO
    # to be called if TODO is changed
    # args: [<force flag>]
    #-----------------------------------------------
    update_todo_pipeline() {
        debug_msg "base configure -> todo file -> update args = $*"
        # check force arg
        # anything but 0 will force update by clearing all todo variables
        local force=0
        [[ $# -eq 1 ]] && force=$1
        [[ "$force" == "0" ]] || force="1"
        [[ $force -eq 1 ]] && clear_todo_vars
        if [[ -z $PL_TODO ]]; then
            debug_msg "PL_TODO is empty -> set it to PL_TODO1=$PL_TODO1"
            PL_TODO=$PL_TODO1
        fi
        [[ -n $PL_TODO ]] || {
            debug_msg "PL_TODO as well as PL_TODO1 are empty"
            return 255
        }
        
        debug_msg "base configure -> todo file -> update $force"

        # check todo file (may alter it) and set todo related variables
        debug_msg "base configure -> todo file -> check file and set todo vars"
        set_todo_vars_with_check || {
            error_msg "failed checking todo file and/or setting the related variables"
            return 255
        }

        debug_msg "base configure -> todo file -> check file and set todo vars - DONE"
        return $force
    }

    update_todo() {
        # set PL_TODO etc
        set_todo_files || return 255
        
        debug_msg "update_todo with:"
        debug_msg "   PL_TODO  = $PL_TODO"
        debug_msg "   PL_TODO1 = $PL_TODO1"
        debug_msg "   PL_TODO2 = $PL_TODO2"
        [[ $# -eq 1    ]] || return 0
        local ec=0
        local force="$1"
        # update general todo related variables
        # will return 1 if a full was update was done
        is_dryrun && dryrun_info_msg "update todo"
        update_todo_pipeline "$force"
        ec=$?
        [[ $ec -eq 255 ]] && return 255
        
        if [[ $ec -eq 1 ]]; then
            # TODO: SHOULD BE PHASED OUT
            # call implementation specific
            isfunc proj_update_todo && proj_update_todo
        fi
        
        return 0
    }

    #--------------------------------------------------
    # Parse TODO file name input (option -t)
    #--------------------------------------------------
    parse_todo_spec() {
        local input="$1"
        local elems=()
        [[ -n $input ]] || return 0
        
        # split line by ':'
        IFS=':' read -ra elems <<< "$input"
        if [[ ${#elems[@]} -eq 1 ]]; then
            PL_TODO="${elems[0]}"
            PL_TODO1=""
            PL_TODO2=""
        elif [[ ${#elems[@]} -eq 2 ]]; then
            PL_TODO="${elems[0]}"
            PL_TODO1="$PL_TODO"
            PL_TODO2="${elems[1]}"
        else
            error_msg "invalid TODO spec -t : $input"
            return 255
        fi

        # disable tree
        TREE_ENABLED=0

        return 0
    }
    
    #--------------------------------------------------
    # Set NJOBMAX for scheduler
    #--------------------------------------------------
    set_scheduler_njobmax() {
        local nsub=10
        [[ $# -eq 1 ]] && nsub=$1

        # General job control variables
        # 1. OP_NJOBMAX_BATCH - max nr of jobs allowed to be running/queueing
        #                        may be important to limit this depending on scheduler setup
        # 2. OP_NJOBMAX_LOCAL  - max nr of jobs on local machine
        #                        is limited by the nr of cores - leaves always one free

        # 1. maximum nr of jobs pending/running in scheduler queue
        OP_NJOBMAX_BATCH=${OP_NJOBMAX_BATCH:="$nsub"}
        (( OP_NJOBMAX_BATCH > 0 )) || OP_NJOBMAX_BATCH=3000

        # 2. maximum nr of jobs running on local machine
        OP_NJOBMAX_LOCAL=${OP_NJOBMAX_LOCAL:=0}
        OP_NJOBMAX_LOCAL=$(get_njobmax_local)
    }

    #--------------------------------------------------
    # check that SCHEDULER_SCRIPT defines every
    # required function
    #--------------------------------------------------
    check_scheduler_setup() {
        check_function sj_scheduler_init  || return 255
        check_function sj_submit_job      || return 255
        check_function sj_interactive_job || return 255
        check_function sj_list_jobs       || return 255
        check_function sj_cancel_jobid    || return 255
        if ! is_decimal $SJ_CHECK_POLLTIME; then
            SJ_CHECK_POLLTIME=5
        fi
    }
    #--------------------------------------------------
    # set script search path
    #--------------------------------------------------
    set_script_path() {
        # if its already set, return
        [[ -n $DIR_ORTHOLOGER_PATH ]] && return 0
        
        # set path
        DIR_ORTHOLOGER_PATH="${DIR_ORTHOLOGER_BIN}:$DIR_PROJECT/bin"
        DIR_ORTHOLOGER_PATH=$(cleanup_path "$DIR_ORTHOLOGER_PATH")
    }
    
    #--------------------------------------------------
    # Configure the environment
    #--------------------------------------------------
    base_configure_env() {
        debug_msg "base configure"
        #---------------------------------------------------------
        # Set paths
        #---------------------------------------------------------
        # path is fixed in order to avoid confusion
        set_script_path
        
        # cleanup external path
        DIR_EXTERNAL_PATH=$(cleanup_path "$DIR_EXTERNAL_PATH")

        #---------------------------------------------------------
        # Set TODO related
        #---------------------------------------------------------
        debug_msg "base configure -> todo file"

        # update TODO/1/2 settings
        if [[ -n $PL_TODO ]]; then
            debug_msg "base configure -> todo file -> update PL_TODO=$PL_TODO"
            update_todo 1 || {
                error_msg "failed initializing todo files"
                return 255
            }
        else
            debug_msg "PL_TODO is not set"
        fi

        #---------------------------------------------------------
        # Configure job scheduler
        #---------------------------------------------------------
        debug_msg "base configure -> scheduler"

        # it's a plugin...
        add_plugin SCHEDULER
        
        # set scheduler opts
        SCHEDULER_LABEL=${SCHEDULER_LABEL:="NONE"}
        SCHEDULER_LABEL=${SCHEDULER_LABEL^^}
        SCHEDULER_SUPPORT="NONE SLURM LSF"
        SCHEDULER_REQ[NONE]="$PLUGIN_NONE_REQ"
        SCHEDULER_REQ[SLURM]="sbatch"
        SCHEDULER_REQ[LSF]="bjobs"

        # make sure it's NONE if empty
        [[ "$SCHEDULER_LABEL" == "" ]] && SCHEDULER_LABEL="NONE"

        # check if scheduler is one supported, if so check requirement
        local chk="${SCHEDULER_REQ[$SCHEDULER_LABEL]+1}"
        if [[ ${#chk} -gt 0 ]]; then
            local req="${SCHEDULER_REQ[$SCHEDULER_LABEL]}"
            if [[ $req != "$PLUGIN_NONE_REQ" ]]; then
                local cmdok=1
                /usr/bin/which "$req" >& /dev/null || cmdok=0
                if [[ $cmdok -eq 0 ]]; then
                    warn_msg "required command ($req) for given scheduler ($SCHEDULER_LABEL) is not found in the system path"
                    warn_msg "will proceed but without scheduler"
                    SCHEDULER_LABEL="NONE"
                fi
            fi
        fi
        # find scheduler script
        local jobsched_label=${SCHEDULER_LABEL,,}
        SCHEDULER_SCRIPT=$(find_script scheduler_${jobsched_label}.sh)
        
        # check scheduler_common.sh
        [[ -x "${DIR_ORTHOLOGER_BIN}/scheduler_common.sh" ]] || fatal_msg "unable to find scheduler_common.sh script"
        

        # source scheduler scripts
        source  "${DIR_ORTHOLOGER_BIN}/scheduler_common.sh"
        [[ $? -eq 0 ]] || return 255

        # check selected scheduler script
        if [[ -x "${SCHEDULER_SCRIPT}" ]]; then
            source "${SCHEDULER_SCRIPT}"
            [[ $? -eq 0 ]] || return 255

            # make sure scheduler script has everything required defined
            check_scheduler_setup || return 255

            # run scheduler init function
            sj_scheduler_init || return 255

            # set scheduler nr of job limits
            local nsub=1
            [[ -n $PL_TODO_NLABELS ]] && nsub=$((PL_TODO_NLABELS*PL_TODO_NLABELS))
            set_scheduler_njobmax $nsub

        else
            warn_msg "unable to find scheduler script scheduler_${SCHEDULER_LABEL,,}.sh"
            warn_msg "create script or change to a valid scheduler SCHEDULER_LABEL in orthologer_conf.sh (eg NONE if none is available)"
            ###            return 255
        fi

        debug_msg "base configure -> DONE"
                        
        return 0
    }

    #--------------------------------------------------
    # check requirement
    # that is, check if given argument:
    # - expands to a valid file/dir
    # - is an executable
    #--------------------------------------------------
    check_requirement() {
        # one requirement
        [[ $# -eq 1 ]] || return 0

        # check if given requirement is ok
        local requirement="$1"
        [[ "${requirement}" == "$PLUGIN_NONE_REQ" ]] && return 0
        [[ -n $requirement ]] || { echo "given requirement is empty for a supported plugin!"; return 1; }

        # check if req is an environment variable
        local reqvar="$requirement"
        local firstchr="${requirement:0:1}"
        [[ "$firstchr" == "\$" ]] && { reqvar=$(eval echo "$requirement"); }
        local emsg=

        # rval=1 -> requirement not found, rval=0 ->requirement found
        rval=1
        # check if empty - should only happen if req is an unset environment variable
        if [[ -z $reqvar ]]; then
            emsg="given requirement variable is empty :"
        else
            # check if dir
            [[ -d "$reqvar" ]] || { emsg="<$reqvar>given requirement is not a directory :"; rval=1; }

            # if reqok=0 then check if file is executable
            local expreq
            expreq=$(/usr/bin/which --skip-alias "$reqvar" 2> /dev/null)
            [[ $rval -eq 1 ]] && [[ -x "$reqvar" ]] && rval=0 # || { emsg="given requirement is not an executable :"; reqok=0; } }
            [[ $rval -eq 1 ]] && [[ -x "$expreq" ]] && rval=0 # || { emsg="given requirement is not an executable :"; reqok=0; } }
            [[ $rval -eq 1 ]] && emsg="given requirement is neither an executable nor a directory"
        fi
        [[ $rval -eq 0 ]] && emsg=
        echo "$emsg"
        return $rval
    }

    check_requirement_list() {
        [[ $# -ge 3 ]] || return 0
        local plugin="$1"
        local label="$2"
        shift
        shift
        local emsg=
        local ec=0
        local r=
        for r in $*; do
            emsg=$(check_requirement "$r")
            [[ $? -eq 0 ]] || ec=1
            [[ -z $emsg ]] || error_msg "$plugin plugin $label $emsg $r"
        done
        return $ec
    }
    #--------------------------------------------------
    # Check directories
    # Input is a list of environment variable names
    #--------------------------------------------------
    check_directories() {
        [[ $# -gt 1 ]] || return 0
        local flag=$1
        shift

        # loop over space separated  env name list
        local nbad=0
        local v=
        for v in $*; do
            isset "$v" || { warn_msg "directory variable $v not set"; continue; }
            local expv
            expv=$(evalvar "$v")

            # check existance of directory
            check_dir "$v" "$expv" || { nbad=$((nbad+1)); continue; }
            
            # check if its writeability should be checked
            [[ $flag -ne 0 ]] || continue
            
            # yes, check if writeable
            iswriteable "$expv" || { nbad=$((nbad+1)); continue; }
        done

        [[ $nbad -eq 0 ]] || return 255
        return 0
    }

    #--------------------------------------------------
    # Check executables
    # Input is a list of environment variable names
    #--------------------------------------------------
    check_executables() {
        local nbad=0
        local v=""
        for v in $*; do
            local expv=""
            expv=$(evalvar "$v")
            check_file_exe "$v" "$expv" || nbad=$((nbad+1))
        done

        [[ $nbad -eq 0 ]] || return 255
        return 0
    }

    #--------------------------------------------------
    # return value of ${label}_SCRIPT variable
    # will always return 0
    #--------------------------------------------------
    get_run_script() {
        [[ $# -eq 1 ]] || return 0
        local label="$1"
        local fbase="$DIR_ORTHOLOGER/bin/run_${label,,}"
        local f="${fbase}.sh"
        if (( USE_NOCHECKS == 1 )); then
            [[ -f ${fbase}_nochecks.sh ]] && f=${fbase}_nochecks.sh
        fi
        echo "$f"
    }

    #--------------------------------------------------
    # return value of ${label}_SCRIPT variable
    # will always return 0
    #--------------------------------------------------
    get_plugin_script() {
        [[ $# -eq 1 ]] || return 0
        local label="$1"
        local scriptvar=${label}_SCRIPT
        isset_var $scriptvar || return 0
        echo "${!scriptvar}"
    }

    #--------------------------------------------------
    # find plugin scripts
    #--------------------------------------------------
    find_plugin_scripts() {
        local p=
        for p in $ALL_PLUGINS; do
            local pup=${p^^}
            local plow=${p,,}

            # get label used for plugin
            local labelvar=${pup}_LABEL
            local label
            label=$(evalvar $labelvar)

            # set label to lower case
            label=${label,,}
            
            local scriptvar=${pup}_SCRIPT

            # find the script via variable <label>_SCRIPT
            local script=$(get_plugin_script "$pup")
            [[ -z $script ]] || continue
            
            # not set, find the script
            script=$(find_script "${plow}_${label}.sh")

            # assign script to <step>_SCRIPT
            assign $scriptvar "$script"

            debug_msg "$scriptvar = $script"
            
            # export the variable
            export $scriptvar
        done

    }

    #--------------------------------------------------
    # parse error code as returned by check_plugin()
    #--------------------------------------------------
    parse_check_plugin_ec() {
        [[ $# -eq 2 ]] || return 255
        local plugin="$1"  # plugin
        local ec=$2      # error code from check_plugin

        # upper/lower case versions
        local plup="${plugin^^}"
        local pllow="${plugin,,}"
        local labelvar="${plup}_LABEL"

        # get the plugin label
        local label
        label="$(evalvar $labelvar)"
        label="${label^^}"

        # get the expected script name
        local scriptname
        scriptname=$(script_name "$label" "$pllow")

        local msg=
        case $ec in
            0)
                msg=""
                ;;
            10)
                msg="plugin var ${plup}_LABEL is not set"
                ;;
            11)
                msg="plugin var ${plup}_SCRIPT is not set"
                ;;
            12)
                msg="plugin var ${plup}_SUPPORT is not set"
                ;;
            13)
                msg="plugin var ${plup}_REQ is not set"
                ;;
            20)
                msg="missing requirement(s) for plugin ${plup} with label $label"
                ;;
            30)
                msg="$plup supported plugin $label missing its script ($scriptname) - not set!"
                ;;
            31)
                msg="$plup unsupported plugin $label missing its script ($scriptname) - create script in DIR_ORTHOLOGER_PATH"
                ;;
            40)
                msg="$plup supported plugin $label missing its script ($scriptname) - pipeline properly installed?"
                ;;
            41)
                msg="$plup unsupported plugin $label missing its script $scriptname - check in path DIR_ORTHOLOGER_PATH"
                ;;
            *)
                msg="Plugin check gave unknown error code $ec"
                ;;
        esac
        echo "$msg"
    }


    #--------------------------------------------------
    # check plugin
    #
    # checks the following
    # - label/script/support/req variables are set
    # - check if supported
    # - check if script exists
    # - check for special cases
    #--------------------------------------------------
    check_plugin() {
        [[ $# -eq 1 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"

        # get plugin
        local plugin="$1"
        local plup="${plugin^^}"
        local pllow="${plugin,,}"

        # plugin variables to check
        # The plugin label - always upper case
        local labelvar=${plup}_LABEL
        # The plugin script found by find_script()
        local scriptvar=${plup}_SCRIPT
        # Array of directly supported plugins
        local supportvar=${plup}_SUPPORT
        # Requirement(s)
        local reqvar=${plup}_REQ

        # check if variables are set
        isset "$labelvar"   || return 10
        isset "$scriptvar"  || return 11
        isset "$supportvar" || return 12
        isset "$reqvar"     || return 13

        # get the plugin label
        local label
        label=$(evalvar "$labelvar")
        label="${label^^}"

        # get the expected script name
        local scriptname
        scriptname=$(script_name "$label" "$pllow")

        # get the full script name - will be an empty string if nothing is present
        local script
        script=$(evalvar "$scriptvar")

        # check if it is supported
        local support
        support=$(evalvar "$supportvar")

        local supported=0
        inrange_set "$label" "$support" && supported=1

        # check requirements
        # if plugin is not supported, user is assumed to have control -> skip check
        # accept requirement if it is
        #   1. the string "$PLUGIN_NONE_REQ"
        #   2. a directory
        #   3. an executable (with absolute path or found using 'which')
        if (( supported==1 )); then
            local requirement
            requirement=$(evalvar "${reqvar}[$label]")
            check_requirement_list "$plup" "$label" "$requirement"
            [[ $? -eq 0 ]] || return 20
        fi

        # check if script is there
        [[ -z $script ]] && {
            (( supported==1 )) || return 31
            return 30
        }
        [[ -x "$script" ]] || {
            (( supported==1 )) || return 41
            return 40
        }
        return 0
    }

    #--------------------------------------------------
    # Get JobLog directory
    #--------------------------------------------------
    R_JOBLOG_DIR=""
    get_joblog_dir() {
        local label=
        [[ $# -eq 1 ]] && label="/$1"
        [[ "$label" == "/$JOBLOGNOSPEC" ]] && label=
        R_JOBLOG_DIR="$DIR_PROJECT/$BASE_JOBLOG$label"
    }
    
    v_get_joblog_dir() {
        local var="$1"
        local label=
        [[ $# -eq 2 ]] && label="/$2"
        [[ "$label" == "/$JOBLOGNOSPEC" ]] && label=
        local "$var" && set_var "$var" "$DIR_PROJECT/$BASE_JOBLOG$label"
    }


    #--------------------------------------------------
    # check JobLog directory tree
    #--------------------------------------------------
    check_joblog_dirs() {
        [[ -d "$DIR_PROJECT/$BASE_JOBLOG" ]] || { error_msg "DIR_PROJECT/$BASE_JOBLOG is not readable"; return 255; }
        [[ -f "$PL_TODO" ]] || return 0
        local t=""
        for t in $(get_lines "$PL_TODO"); do
            [[ -d "$DIR_PROJECT/$BASE_JOBLOG/$t" ]] || { error_msg "DIR_PROJECT/$BASE_JOBLOG/$t is not readable"; return 255; }
        done
        return 0
    }

    export _N_CREATED_DIRS=0
    #--------------------------------------------------
    # create joblog directory tree
    #--------------------------------------------------
    create_joblog_dirs() {
        debug_msg "---> create_joblog_dirs at $DIR_PROJECT/$BASE_JOBLOG"
        [[ -d "$DIR_PROJECT/$BASE_JOBLOG" ]] || { _N_CREATED_DIRS=$((_N_CREATED_DIRS+1)); create_dir "$DIR_PROJECT/$BASE_JOBLOG" || return 255; }

        # create missing dirs
        [[ -f "$PL_TODO" ]] || return 0
        
        local t=""
        for t in $(get_lines "$PL_TODO"); do
            [[ -d "$DIR_PROJECT/$BASE_JOBLOG/$t" ]] || { _N_CREATED_DIRS=$((_N_CREATED_DIRS+1)); create_dir "$DIR_PROJECT/$BASE_JOBLOG/$t" || return 255; }
        done
        return 0
    }

    #-----------------------------------------------
    # create project directories
    #-----------------------------------------------
    create_project_dirs() {
        local basedir=
        [[ $# -eq 0 ]] || { basedir="$*"; [[ -d "$basedir" ]] || basedir=""; }

        debug_msg "create project dirs:  $ALL_DIR_PROJECT"
        info_msg "create project dirs at $DIR_PROJECT"
        local ncr=0
        local var=""
        for var in $ALL_DIR_PROJECT; do
            local dname="${!var}"
            
            if [[ -z $basedir ]]; then
                if [[ ! -d "$dname" ]]; then
                    info_msg "create dir given by $var = $(trim_name ${dname})"
                    ncr=$((ncr+1))
                    create_dir "$dname" || return 255
                fi
            else
                local bname
                bname=$(basename "$dname")
                local newdir="${basedir}/$bname"
                [[ -d "$newdir" ]] || { ncr=$((ncr+1)); create_dir  "$newdir"           || return 255; }
                [[ -f "$dname"  ]] || { ncr=$((ncr+1)); create_link "$newdir"  "$dname" || return 255; }
            fi
        done

        # local bin
        [[ -d "${DIR_PROJECT}/bin" ]] || { ncr=$((ncr+1)); create_dir "${DIR_PROJECT}/bin" || return 255; }

        # local scratch
        [[ -d "${DIR_PROJECT}/.scratch" ]] || { ncr=$((ncr+1)); create_dir "${DIR_PROJECT}/.scratch" || return 255; }

        # create link to pipeline bin
        [[ -d "${DIR_PROJECT}/sbin" ]] || { ncr=$((ncr+1)); create_link "${DIR_ORTHOLOGER}/bin" "${DIR_PROJECT}/sbin" || return 255; }

        _N_CREATED_DIRS=$ncr
        
        # joblog dir(s)
        create_joblog_dirs || return 255

        # implementation specific directories
        isfunc proj_create_dirs && { dryrun_info_msg "calling proj_create_dirs"; proj_create_dirs || return 255; }

        info_msg "created $_N_CREATED_DIRS directories"

        # make sure it returns 0 as if the isfunc fails, it will return the error code
        return 0
    }

    #-----------------------------------------------
    # check if project dirs are valid
    #-----------------------------------------------
    check_project_dirs() {
        local var=
        for var in $ALL_DIR_PROJECT; do
            debug_msg "checking project dir $var"
            local dirname
            dirname="$(evalvar $var)"

            # error if undefined
            [[ -n $dirname ]] || { error_msg "$var is undefined"; return 1; }

            # error if non-existant
            check_dir "$var" "$dirname" || return 1

        done

        # check joblog dir
        debug_msg "checking DIR_PROJECT/$BASE_JOBLOG"
        check_dir "DIR_PROJECT/$BASE_JOBLOG" "$DIR_PROJECT/$BASE_JOBLOG" || return 1
        if [[ -f "$PL_TODO" ]]; then
            # only do detailed check if few species
            if (( PL_TODO_NLABELS < 10 )); then
                local t=''
                debug_msg "checking DIR_PROJECT/$BASE_JOBLOG/xxx"
                ####for t in ${PL_TODO_LBLSTR}; do
                for t in ${PL_TODO_LABELS[@]}; do
                    check_dir "DIR_PROJECT/$BASE_JOBLOG/$t" "$DIR_PROJECT/$BASE_JOBLOG/$t" || return 1
                done
            fi
        fi

        # project specific check
        if isfunc proj_check_dirs; then
            debug_msg "running project specific check"
            proj_check_dirs || return 1
        fi

        return 0
    }

    #-----------------------------------------------
    # check if project directories exists
    #-----------------------------------------------
    has_project_dirs() {
        local rval=0
        [[ -n $ALL_DIR_PROJECT ]] || return 1

        # loop over all project dirs
        # break if dir is undefined
        local var=
        for var in $ALL_DIR_PROJECT; do
            local dirname
            dirname="$(evalvar $var)"
            [[ -d "$dirname" ]] || { rval=1; break; }
        done
        # return 1 if some bad dirs were found
        [[ $rval -eq 0 ]] || return 1
        return 0
    }

    help_header_msg() {
        [[ $# -ge 1 ]] || return 0
        local msg="$*"
        echo
        echo -e "${ALWAYS_BOLD}$msg${ALWAYS_NORMAL}"
        echo
    }
    
    help_opt_msg() {
        [[ $# -eq 2 ]] || return 0
        local opt="$1"
        local msg="$2"
        printf " %-20s %s\n" "$opt" "$msg"
    }
    #-----------------------------------------------
    # usage message TODO: update
    #-----------------------------------------------
    usage_base() {
        help_header_msg "Usage: $CMD_PIPELINE [manage|odbmap] [options]"
        help_header_msg "Special commands (options below are not valid, add 'help' to obtain more info)"
        help_opt_msg "manage" "run in orthologer project management mode"
        help_opt_msg "odbmap" "run OrthoDB mapper (simply wraps ./orthomapper.sh)"
        help_header_msg "General"
        help_opt_msg "-h"        "this message"
        help_opt_msg "-H <arg>"  "extended help:"
        help_opt_msg ""   "'<stepid>'        help for given step id"
        help_opt_msg ""   "'.'               extended help"
        help_opt_msg ""   "'.variables'      a summary of user-settable variables (in orthologer_conf.sh)"
        help_opt_msg ""   "'.examples'       a collection of examples"
        help_opt_msg ""   "'.steps'          print all steps in one line"
        help_opt_msg "-v"        "print version"
        help_opt_msg "-t <todo>|<todo1>:<todo2>" "use the given todo file(s) instead of the one in orthologer_conf.sh"
        help_opt_msg ""                          "if two files are given separated by a ';' they are used by PL_TODO1 and PL_TODO2"
        help_opt_msg ""                          "if tree mode is enabled, this option will be ignored"
        help_opt_msg "-T <tree>" "run using a tree (newick format), overrides -t and settings in orthologer_conf.sh"
        help_opt_msg ""          "if an empty string is given, tree mode is disabled"

        help_header_msg "Output"
        help_opt_msg "-d"              "dry run, just print what will be done but do nothing"
        help_opt_msg "-p"              "use color in output messages"
        help_opt_msg "-V"              "verbose output, if given twice -> debug output mode"

        help_header_msg "Running/killing"
        help_opt_msg "-L <npar>"       "max nr of processes in parallel - overrides settings in config (NJOBMAX_LOCAL)"
        help_opt_msg "-B <nbat>"       "max nr of jobs in one batch - overrides settings in config (NJOBMAX_BATCH)"
        help_opt_msg "-R <spec>"       "space separated labels; for all steps, require this species label"
        help_opt_msg ""                "note: for pairwise the requirement is at least ONE of the pair"
        help_opt_msg "-r <step>"       "run the pipeline for step(s) <stepid>[+/-] (for <step> see extended help)"
        help_opt_msg "-k <step>"       "kill any pending jobs for given step(s)"
        help_opt_msg "-K <hard/soft>"  "kill pipeline; hard -> kill immediately; soft -> kill after next batch of jobs are done; does not kill pending or running jobs"

        help_header_msg "Check status"
        help_opt_msg "-e <step>"  "check the pipeline status - details in extended help '-H .'"
        help_opt_msg "-X <step>"  "check the step(s) done flag - returns the latest status as an error code (\$?)"
        
        help_header_msg "Step states/cleaning"
        help_opt_msg "-f <step>"  "force state of step(s) -> <stepid>[+/-][:<state>]"
        help_opt_msg "-F <state>" "with -f, specify state either as a string or numeric code"
        help_opt_msg "-c <step>"  "clear contents of step(s) <stepid>[+/-]"
        help_opt_msg "-E"         "cleanup all empty joblog files"

        help_header_msg "Special"
        help_opt_msg "-M <nr>"    "max nr of messages (only with -e) - sets variable MAX_MESSAGES"

        help_header_msg "Project setup and check"
        help_opt_msg "-O"         "print per step setup"
        help_opt_msg "-x"         "check pipeline setup, gives a non-zero return code when a problem is encountered"
        help_opt_msg "-Z"         "check project directories"
        help_opt_msg "-S"         "one-line status message - prints the last DONE step"

        help_header_msg "Diverse"
        help_opt_msg "-o"         "refresh $CMD_PIPELINE"
        help_opt_msg "-u"         "update orthologer_conf.sh"
        help_opt_msg "-C"         "create any missing project directories"
    }
    #-----------------------------------------------
    # usage function
    #-----------------------------------------------
    usage() {
        usage_base
        isfunc proj_usage && proj_usage

        help_header_msg "For further help"
        echo "> ./${CMD_PIPELINE} -H ."
        echo 
        echo "or for a given step, e.g:"
        echo
        echo "> ./${CMD_PIPELINE} -H makebrh"
        echo
        echo "or for a summary of user setable variables:"
        echo
        echo "> ./${CMD_PIPELINE} -H .variables"
        echo
        echo "or for a list of all possible steps:"
        echo
        echo "> ./${CMD_PIPELINE} -H .steps"
        echo
        echo "For a description of the final results:"
        echo
        echo "> ./${CMD_PIPELINE} -H .results"
        echo
        echo "For any option requiring a step id list, all step ids possible is obtained using '.':"
        echo
        echo "> ./${CMD_PIPELINE} -r ."
        echo
        help_header_msg "Managing orthologer project"
        echo "Import files (soft-link/copy from other projects, add new fasta files, etc)"
        echo
        echo "> ./${CMD_PIPELINE} manage"
        echo
        help_header_msg "Running OrthoDB mapper"
        echo "Map a given fasta file to OrthoDB official data"
        echo
        echo "> ./${CMD_PIPELINE} odbmap"
        echo "  or"
        echo "> ./orthomapper.sh"
        echo
        return 0
    }

    #-----------------------------------------------
    # extended help
    #-----------------------------------------------
    extended_help() {
        local step=
        echo
        echo "${ALWAYS_BOLD}I. Table of valid steps${ALWAYS_NORMAL}"
        echo
        print_steps_table
        echo
        echo
        echo "${ALWAYS_BOLD}II. Details on options${ALWAYS_NORMAL}"
        echo
        echo
        echo -e "${ALWAYS_BOLD}  Expand step lists <step id>:${ALWAYS_NORMAL}"
        echo "  -> format is:"
        echo "     <step1>[+|-][:val1][,<step2>[:val2],...]"
        echo "  -> use ALL to run over all steps"
        echo "  -> a range can be given by adding one of the following:"
        echo "     '+' - expand to all following steps"
        echo "     '-' - expand to all preceding steps"
        echo "  -> a value associated with the step may be added (used with -f option)"
        echo
        echo -e "${ALWAYS_BOLD}  Valid states for -f option are:${ALWAYS_NORMAL}"
        echo "  0 - do not remake"
        echo "  1 - remake if source is newer than target, or target is missing"
        echo "  2 - always remake"
        echo "  3 - remake only if target is missing - does not check time stamps (DEFAULT)"
        echo
        echo -e "${ALWAYS_BOLD}  If -F option is given, both string and numerical representations are allowed:${ALWAYS_NORMAL}"
        echo "  0 - \"DONE\""
        echo "  1 - \"MAKE NEW\""
        echo "  2 - \"MAKE ALL\""
        echo "  3 - \"MAKE MISSING\""
        echo
        echo
#        echo "${ALWAYS_BOLD}III. Compression${ALWAYS_NORMAL}"
#        echo
#        echo "In order to add data compression it must be flagged both globally and per step."
#        echo "To allow compression set:"
#        echo "   COMPRESS_DATA=1"
#        echo
#        echo "Per step, set:"
#        echo "   OP_STEP_COMPRESS[stepid]=1"
#        echo
#        echo "By default, the compression runs gzip in --fast mode."
#        echo
#        echo
        echo "${ALWAYS_BOLD}III. Examples${ALWAYS_NORMAL}"
        echo
        echo "1. Run all steps before STEPX:"
        echo " ./${CMD_PIPELINE} -r STEPX-"
        echo
        echo "2. Run all steps after and including STEPX:"
        echo " ./${CMD_PIPELINE} -r STEPX+"
        echo
        echo "3. Force a step to create only missing files:"
        echo " ./${CMD_PIPELINE} -f STEPX:3"
        echo "OR"
        echo " ./${CMD_PIPELINE} -f STEPX -F \"MAKE MISSING\""
        echo
        isfunc proj_extended_help && proj_extended_help
        return 0
    }

    #-----------------------------------------------
    # help on variables
    #-----------------------------------------------
    variable_def() {
        local var="$1"
        shift
        local descr="$*"
        printf "%-25s %s\n" "$var" "$descr"
    }
    
    variables_help() {
        local cnt=1
        help_header_msg "${cnt}. General" && cnt=$((cnt+1))
        variable_def "VERBOSE"       "verbose output if >0; debug if >1"
        variable_def "MAX_MESSAGES"  "maximum nr of warning/error messages shown with ./${CMD_PIPELINE} -x"
        variable_def "PYTHON3"       "full path to python3"
        
        help_header_msg "${cnt}. Directories" && cnt=$((cnt+1))
        variable_def "DIR_ORTHOLOGER" "Pipeline package root directory."
        variable_def "TMP_DIR_BASE" "Temporary files root directory."

        help_header_msg "${cnt}. Organism selection" && cnt=$((cnt+1))
        variable_def "PL_TODO"      "'todo' file - contains the labels defining which fasta files to use."
        variable_def ""             "For each <label> there should be a fasta file Sequences/<label>.fs ."
        help_header_msg "${cnt}. Pairwise organism selection" && cnt=$((cnt+1))
        variable_def "PL_TODO1/2"      "Pair of 'todo' files - run over pairs formed by labels between TODO1 and TODO2."
        #variable_def "PL_TODO_ONE2ONE" "Defines how pairs are created from TODO1 and TODO2."
        #variable_def ""                "  0->create all possible pairs"
        #variable_def ""                "  1->create only one-to-one (requires both todo files to have the same nr of keys)"
#        variable_def ""
#        variable_def "COMPRESS_DATA" "Global compression flag."
#        variable_def ""              "The compression per step is then controlled by OP_STEP_COMPRESS[stepid] ."
#        variable_def ""              "0 - no compression allowed"
#        variable_def ""              "1 - use compression when requested by step"

        help_header_msg "${cnt}. Job control" && cnt=$((cnt+1))
        variable_def "SCHEDULER_LABEL"  "Define which scheduler to use."
        variable_def ""                 "Predefined are 'SLURM','LSF' and 'NONE' ."
        variable_def "OP_NJOBMAX_BATCH" "Maximum allowed nr of running/pending jobs with the scheduler."
        variable_def "OP_NJOBMAX_LOCAL" "Maximum allowed nr of jobs running locally."
        variable_def "OP_SAVE_JOBLOG"   "To keep or not to keep joblogs."
        variable_def ""                 "0 - delete all logs if no error"
        variable_def ""                 "1 - keep logs (delete empty error logs)"
        variable_def "SJ_CHECK_POLLTIME" "Job status checking - poll time in seconds."
        variable_def "SKIP_REMAKE_CHECK" "Possibly skip remake check."
        variable_def ""                 "This is useful with large data sets when you know all files needs to be redone."
        variable_def ""                 "0 - do remake check if required"
        variable_def ""                 "1 - never do remake check -> always remake"
        variable_def "SKIP_CHECK_ERROR_LOGS" "Skip the check of errorlogs."
        variable_def ""                 "May be useful if a tool outputs non-error messages to stderr."
        variable_def ""                 "A step enters error state if either the returned error code is non-zero or"
        variable_def ""                 "the errorlog is not empty"
        variable_def ""                 "0 - always check error logs"
        variable_def ""                 "1 - never check error logs"
        variable_def "STEP_SLEEP"       "sleep the given nr of seconds between each step"
        echo
        if isfunc proj_variables_help; then
            proj_variables_help $cnt
        fi
        echo
    }

    exline() {
        local c="$1"
        if [[ $# -eq 2 ]]; then
            local ex="$2"
            printf "   %-40s  # %s\n" "$c" "$ex"
        else
            printf "   %s\n" "$c"
        fi
    }
    
    examples_help() {
        local cnt=1
        echo
        help_header_msg "EXAMPLES"
        help_header_msg "${cnt}. Force step state" && cnt=$((cnt+1))
        exline "./${CMD_PIPELINE} -f FORMATDB:3"  "make sure that FORMATDB will only run if output is missing"
        exline "./${CMD_PIPELINE} -f FORMATDB+:3" "idem but for all steps starting at FORMATDB"
        exline "./${CMD_PIPELINE} -f FORMATDB-:3" "idem but for all steps before FORMATDB"
                
        help_header_msg "${cnt}. Run a selection of steps" && cnt=$((cnt+1))
        exline "./${CMD_PIPELINE} -r FORMATDB-" "run all steps up to the step before FORMATDB"
        exline "./${CMD_PIPELINE} -r FORMATDB+" "run all steps starting at FORMATDB"
        exline "./${CMD_PIPELINE} -r MAKEBRH,ALIGNMENT" "run steps MAKEBRH followed by ALIGNMENT"

        help_header_msg "${cnt}. Run with special requests" && cnt=$((cnt+1))
        exline "./${CMD_PIPELINE} -t another.todo -r MAKEBRH" "run step with a specific todo file"
        exline "./${CMD_PIPELINE} -R \"HSAPI MMULA\" -r MAKEBRH" "run step requiring two labels (must be in the todo file)"
        exline "./${CMD_PIPELINE} -R \"TAXID\" -r CLUSTER -I Cluster/mybase.og" "map given label into the cluster given by -I"

        help_header_msg "${cnt}. Run pairwise steps over a section of the alignment matrix" && cnt=$((cnt+1))
        exline "Modify variables in orthologer_conf.sh:"
        echo
        exline " OP_LABELA_FIRST=AFLOR"   "start label"
        exline " OP_LABELA_END=MMULA"     "end label (alphabetic order)"
        echo
        exline "The above change will force a loop over AFLOR-MMULA for 'A' in the pairwise steps (like MAKEBRH)."
        exline "There is also the equivalent for 'B' which allows selecting a rectangle in the pairwise matrix."
        exline "This is useful when running on different machines without common storage."
        echo

        #help_header_msg "${cnt}. Run ALIGNMENT over a section of the alignment matrix" && cnt=$((cnt+1))
    }

    results_help() {
        local cnt=1
        echo
        help_header_msg "Results"
        echo "The results are summerized in 'Results' directory."
        echo "Three types of files are generated by default from the POSTPROC step."
        help_header_msg "${cnt}. Cluster file(s)" && cnt=$((cnt+1))
        exline "Filename: <node>_orthogroups.txt"
        exline "If tree mode, one cluster file per node."
        exline "Each file contains the original gene ids."
        echo
        help_header_msg "${cnt}. Cluster file, orthoXML format" && cnt=$((cnt+1))
        exline "Filename: <node>.orthoxml"
        exline "If tree mode, only one file is created at the root level."
        exline "However it uses the full tree result to generate an tree hierarchy at cluster level."
        exline "Definition of the format is at http://orthoxml.org/0.3/orthoxml_doc_v0.3.html"
        echo
        help_header_msg "${cnt}. Cluster statistics" && cnt=$((cnt+1))
        exline "Filename: <node>_stats.txt"
        exline "This produces one cluster stat file per node using 'statclus' from brhclus package."
        exline "It contains stats like cluster mean size, connectivity, modularity etc"
        echo
    }

    list_steps() {
        echo "${OP_ALL_STEPS[@]}"
    }
    #-----------------------------------------------
    # help message
    #-----------------------------------------------

    #-----------------------------------------------
    # general help function, calls help for
    # each step given
    #-----------------------------------------------
    help_xxx() {
        [[ $# -gt 0 ]] || { extended_help; return 0; }
        local arg="$1"
        arg="${arg^^}"
        [[ "$arg"       == "."     ]] && { extended_help;  return 0; }
        [[ "${arg:0:5}" == ".VARI" ]] && { variables_help; return 0; }
        [[ "${arg:0:5}" == ".EXAM" ]] && { examples_help;  return 0; }
        [[ "${arg:0:5}" == ".STEP" ]] && { list_steps;     return 0; }
        [[ "${arg:0:5}" == ".RESU" ]] && { results_help;   return 0; }
        
        # make step list
        local ec=0
        expand_step_list_check "$arg"
        ec=$?
        [[ $ec -eq 1 ]] && return 0
        [[ $ec -eq 0 ]] || return 244
        
        local sep
        sep=$(print_repeated '-' 80)
        local i=
        local si
        local sf
        for (( i=0; i<${#STEP_ID_LIST[@]}; i++ )); do
            local si=${STEP_ID_LIST[$i]}        # step id
            local sf=${STEP_ID_LIST_FLAGS[$i]}  # step id flag
            # execute help if help function exists
            echo
            echo $sep
            local fun=help_step_${si,,}
            if isfunc $fun; then
                if [[ "$sf" == ":" ]]; then
                    $fun
                else
                    $fun $sf
                fi
            else
                warn_msg "missing help for step $si"
            fi
        done
        echo $sep
    }
    #-----------------------------------------------
    # set nr of jobs limits
    # args: [<nparallel> [<nbatch>]]
    #
    # <nparallel> -> nr of jobs to run in parallel (if local run mode)
    # <nbatch> -> nr of jobs to submit to scheduler (xargs or slurm...)
    #
    # 1. If no args, NJOBMAX_LOCAL is set to get_nthreads_conservative()
    # 2. If one arg, NJOBMAX_LOCAL = arg 
    # 3. If two args, NJOBMAX_LOCAL = arg1 and NJOBMAX_BATCH = arg2
    #    It is ensured that BATCH>LOCAL.
    #-----------------------------------------------
    set_njobmax() {
        local npar=0
        local nbat=0
        [[ $# -ge 1 ]] && npar="$1"
        [[ $# -ge 2 ]] && nbat="$2"

        OP_NJOBMAX_LOCAL=$npar
        (( nbat == 0 )) || OP_NJOBMAX_BATCH=$nbat

        (( OP_NJOBMAX_LOCAL == 0 )) && OP_NJOBMAX_LOCAL=$(get_nthreads_conservative)
        (( OP_NJOBMAX_BATCH == 0 )) && OP_NJOBMAX_BATCH=$OP_NJOBMAX_LOCAL
        local nmax=$(get_nthreads_total)
        (( OP_NJOBMAX_LOCAL > nmax             )) && OP_NJOBMAX_LOCAL=$nmax
        local nminbatch=$(( 10*OP_NJOBMAX_LOCAL ))
        (( OP_NJOBMAX_BATCH < nminbatch )) && OP_NJOBMAX_BATCH=$nminbatch
    }

    #-----------------------------------------------
    # set multiprocessing parameters for given step
    # args: <stepid> [<npar> [<nthr> [<nmer>]]]
    # <npar>    nr of jobs in parallel
    # <nthr>    nr of threads per job
    # <nmer>    nr of jobs merged into one job
    #
    # If not all args are given the corresponding values
    # are taken from OP_STEP_xxx.
    #
    # If <npar> == 0 then set it to nr of threads available (a bit below, see fkn).
    # The nr of threads is set to 1.
    #
    # If <nthr> == 0 then the reverse logic is true.
    #
    # If step type is 'SINGLE' then <npar> is set to 1.
    #-----------------------------------------------
    set_multiprocessing_stepid() {
        local stepid="$1"
        local npar=0 #${OP_STEP_NPARALLEL[$stepid]}
        local nthr=0 #${OP_STEP_NTHREADS[${stepid}]} #nr of threads per job
        local nmer=0 #${OP_STEP_NMERGE[${stepid}]} #nr of threads per job

        [[ $# -ge 2 ]] && npar="$2"
        [[ $# -ge 3 ]] && nthr="$3"
        [[ $# -ge 4 ]] && nmer="$4"

        local nthrmax=$(get_nthreads_total)
        
        # ensure npar and nthr are non-zero
        (( npar == 0 )) && { npar=$(get_nthreads_conservative); nthr=1; }
        (( nthr == 0 )) && { nthr=$(get_nthreads_conservative); npar=1; }
        (( nmer == 0 )) && nmer=4

        # if step type is single, npar = 1
        [[ "${OP_STEP_TYPE[$stepid]}" == "$OP_STEP_TYPE_SINGLE" ]] && npar=1

        if is_runmode_local $stepid; then
            # local- possibly correct npar
            local ntot=$((npar*nthr))
            if (( ntot > nthrmax )); then
                (( npar > nthrmax )) && npar=$nthrmax
                nthr=1
            fi
        fi
        # set the variables
        OP_STEP_NPARALLEL[${stepid}]=$npar
        OP_STEP_NTHREADS[${stepid}]=$nthr
        OP_STEP_NMERGE[${stepid}]=$nmer
    }
    
    #-----------------------------------------------
    # set OP_STEP_xxx arrays to default values
    #-----------------------------------------------
    set_op_step_defaults() {
        debug_msg "set all OP_STEP_XXX to default values"
        isempty OP_ALL_STEPS && fatal_msg "set_op_step_defaults() OP_ALL_STEPS not set"
        
        # set default NJOBMAX_xxx
        set_njobmax

        # loop over all steps and set defaults
        local njobmax=$(get_njobmax_local)
        local step=
        for step in "${OP_ALL_STEPS[@]}"; do
            # enable all jobs
            OP_STEP_ENABLE[$step]=1

            # run on local machine (batch will be set below)
            OP_STEP_RUNMODE[$step]="LOCAL"

	        # job scheduler option strings
	        OP_STEP_SCHEDOPTS[$step]=""

            # compress flag
            #OP_STEP_COMPRESS[$step]=0

            # required tools "<dir> <exec>"
            OP_STEP_REQTOOLS[$step]=""

            OP_STEP_NPARALLEL[$step]=1
            OP_STEP_NTHREADS[$step]=1
            OP_STEP_NMERGE[$step]=4
            
            # set multiproc
            set_multiprocessing_stepid $step $njobmax 1 4
        done
        
        return 0
    }

    #-----------------------------------------------
    # print table of valid steps
    #-----------------------------------------------
    print_steps_table() {
        isempty OP_ALL_STEPS && fatal_msg "print_steps_table() OP_ALL_STEPS not set"
        local userr=0
        [[ $# -eq 0 ]] || userr=$1
        [[ $userr -eq 0 ]] || userr=1

        useecho=echo
        useprint=printf
        if [[ $userr -eq 1 ]]; then
            useecho=echoerr
            useprint=printferr
        fi
        local sep
        sep=$(print_repeated '-' 80)
        $useecho $sep
        $useprint "%-20s%-20s%s\n" "Step" "Dependency" "Description"
        $useecho $sep
        $useprint "%-20s%-20s%s\n" ALL "-" "All steps below"
        local step=
        for step in "${OP_ALL_STEPS[@]}"; do
            $useprint "%-20s%-20s%s\n" "$step" "${OP_STEP_DEPENDS[$step]}" "${OP_STEP_DESCRIPTION[$step]}"
        done
    }

    #-----------------------------------------------
    # dump OP_STEP_ variables
    #-----------------------------------------------
    dump_op_config_steps() {
        local commented=1
        [[ $# -eq 1 ]] && commented=$1
        [[ "$commented" == "0" ]] || [[ "$commented" == "1" ]] || { warn_msg "dump_op_config_steps() got an invalid flag <$commented> - will default to 1"; commented=1; }

        # check if commented
        local cmstr="#"
        [[ $commented -eq 0 ]] && cmstr=""

        local step=
        for step in "${OP_ALL_STEPS[@]}"; do
            echo "${cmstr}OP_STEP_ENABLE[$step]=${OP_STEP_ENABLE[$step]}"
            #echo "${cmstr}OP_STEP_COMPRESS[$step]=${OP_STEP_COMPRESS[$step]}"
            echo "${cmstr}OP_STEP_NPARALLEL[$step]=${OP_STEP_NPARALLEL[$step]}"
            echo "${cmstr}OP_STEP_NTHREADS[$step]=${OP_STEP_NTHREADS[$step]}"
            echo "${cmstr}OP_STEP_NMERGE[$step]=${OP_STEP_NMERGE[$step]}"
            echo "${cmstr}OP_STEP_RUNMODE[$step]=${OP_STEP_RUNMODE[$step]}"
            echo "${cmstr}OP_STEP_SCHEDOPTS[$step]=${OP_STEP_SCHEDOPTS[$step]}"
            echo
        done
    }

    #-----------------------------------------------
    #
    #-----------------------------------------------
    dump_config_steps() {
        # print table

        print_separator
        fmtstr="%-20s%-6s%10s%10s%10s%10s  %s\n"
        printf "$fmtstr" "STEP" "ENABLE" "RUNMODE" "NPARALLEL" "NTHREADS" "NMERGE" "SCHEDOPTS"
        print_separator

        local step=
        for step in "${OP_ALL_STEPS[@]}"; do
            printf "$fmtstr" "$step" "${OP_STEP_ENABLE[$step]}" "${OP_STEP_RUNMODE[$step]}" "${OP_STEP_NPARALLEL[$step]}" "${OP_STEP_NTHREADS[$step]}" "${OP_STEP_NMERGE[$step]}" "\"${OP_STEP_SCHEDOPTS[$step]}\""
        done
    }

    #-----------------------------------------------
    #
    #-----------------------------------------------
    get_step_depend() {
        [[ $# -eq 1 ]] || return 0
        local step=$1
        is_stepid_valid $step || return 255
        
        local depends=${OP_STEP_DEPENDS[$step]}
        local ec=1
        if [[ "$depends" == "" ]]; then
            ec=0
            echo ""
        else
            is_stepid_valid $depends && { echo $depends; ec=0; }
        fi
        return $ec
    }
    #-----------------------------------------------
    # check all OP_STEP arrays
    #-----------------------------------------------
    check_op_steps() {
        isempty OP_ALL_STEPS && { error_msg "OP_ALL_STEPS is empty"; return 1; }

        local ec=0
        local nexp=${#OP_ALL_STEPS[@]}
        # check global arrays
        set +u
        [[ ${#OP_ALL_STEPS[@]}        -eq $nexp ]] || { error_msg "size of OP_ALL_STEPS is wrong = ${#OP_ALL_STEPS}"; ec=1; }
        [[ ${#OP_STEP_NPARALLEL[@]}   -eq $nexp ]] || { error_msg "size of OP_STEP_NPARALLEL is wrong = ${#OP_STEP_NPARALLEL[@]}"; ec=1; }
        [[ ${#OP_STEP_NTHREADS[@]}    -eq $nexp ]] || { error_msg "size of OP_STEP_NTHREADS is wrong = ${#OP_STEP_NTHREADS[@]}"; ec=1; }
        [[ ${#OP_STEP_NMERGE[@]}      -eq $nexp ]] || { error_msg "size of OP_STEP_NMERGE is wrong = ${#OP_STEP_NMERGE[@]}"; ec=1; }
        [[ ${#OP_STEP_ENABLE[@]}      -eq $nexp ]] || { error_msg "size of OP_STEP_ENABLE is wrong = ${#OP_STEP_ENABLE[@]}"; ec=1; }
        [[ ${#OP_STEP_RUNMODE[@]}     -eq $nexp ]] || { error_msg "size of OP_STEP_RUNMODE is wrong = ${#OP_STEP_RUNMODE[@]}"; ec=1; }
        [[ ${#OP_STEP_DATA[@]}        -eq $nexp ]] || { error_msg "size of OP_STEP_DATA is wrong = ${#OP_STEP_DATA[@]}"; ec=1; }
        [[ ${#OP_STEP_DESCRIPTION[@]} -eq $nexp ]] || { error_msg "size of OP_STEP_DESCRIPTION is wrong = ${#OP_STEP_DESCRIPTION[@]}"; ec=1; }
        [[ ${#OP_STEP_DEPENDS[@]}     -eq $nexp ]] || { error_msg "size of OP_STEP_DEPENDS is wrong = ${#OP_STEP_DEPENDS[@]}"; ec=1; }
        [[ ${#OP_STEP_SCHEDOPTS[@]}   -eq $nexp ]] || { error_msg "size of OP_STEP_SCHEDOPTS is wrong = ${#OP_STEP_SCHEDOPTS[@]}"; ec=1; }
        [[ ${#OP_STEP_REQBY[@]}       -eq $nexp ]] || { error_msg "size of OP_STEP_REQBY is wrong = ${#OP_STEP_REQBY[@]}"; ec=1; }
        [[ ${#OP_STEP_REQTOOLS[@]}    -eq $nexp ]] || { error_msg "size of OP_STEP_REQTOOLS is wrong = ${#OP_STEP_REQTOOLS[@]}"; ec=1; }
        set -u


        # check per step
        local globec=0

        local step=
        for step in "${OP_ALL_STEPS[@]}"; do

            # check step function
            local f=step_${step,,}
            isfunc "$f" || fatal_msg "function defining step $step is missing (step_${step,,})"

            # check run mode - make sure it's upper case
            local runmode=${OP_STEP_RUNMODE[$step]}
            runmode=${runmode^^}
            OP_STEP_RUNMODE[$step]=$runmode
            ec=1
            [[ "$runmode" == "BATCH" ]]       && ec=0
            [[ "$runmode" == "LOCAL" ]]       && ec=0
            [[ "$runmode" == "INTERACTIVE" ]] && ec=0
            [[ $ec -eq 0 ]] || { globec=1; error_msg "illegal OP_STEP_RUNMODE for step $step -> $runmode (must be BATCH, INTERACTIVE or LOCAL)"; }


            # check and modify NPARALLEL if needed
            local npar=${OP_STEP_NPARALLEL[$step]}
            if [[ "$runmode" == "LOCAL" ]] && [[ $npar -gt $OP_NJOBMAX_LOCAL ]]; then
                verbose_warn_msg "changing OP_STEP_NPARALLEL[$step] = $npar to OP_NJOBMAX_LOCAL = $OP_NJOBMAX_LOCAL"
                OP_STEP_NPARALLEL[$step]=$OP_NJOBMAX_LOCAL
            fi

            if [[ "$runmode" == "BATCH" ]] && [[ $npar -gt $OP_NJOBMAX_BATCH ]]; then
                verbose_warn_msg "changing OP_STEP_NPARALLEL[$step] = $npar to OP_NJOBMAX_BATCH=$OP_NJOBMAX_BATCH"
                OP_STEP_NPARALLEL[$step]=$OP_NJOBMAX_BATCH
            fi

            # at least 1 job
            [[ $npar -lt 1 ]] && OP_STEP_NPARALLEL[$step]=1

            # check step data
            local sd=${OP_STEP_DATA[$step]}
            ec=1
            [[ "$sd" == "COMMON" ]] && ec=0
            [[ "$sd" == "LABEL" ]]  && ec=0
            [[ $ec -eq 0 ]] || { globec=1; error_msg "illegal OP_STEP_DATA for step $step (must be COMMON or LABEL) -> $sd"; }

            # check threads
            local nt=${OP_STEP_NTHREADS[$step]}
            [[ $nt -gt 0 ]] || { globec=1; error_msg "illegal OP_STEP_NTHREADS for step $step -> $s (must be >0)"; }
            
            # check NMERGE
            local nt=${OP_STEP_NMERGE[$step]}
            [[ $nt -gt 0 ]] || { globec=1; error_msg "illegal OP_STEP_NMERGE for step $step -> $s (must be >0)"; }
            
            # check step enable
            local enable=${OP_STEP_ENABLE[$step]}
            ec=1
            (( enable == 0 )) && ec=0
            (( enable == 1 )) && ec=0
            (( ec == 0 )) || { globec=1; error_msg "illegal OP_STEP_ENABLE for step $step (must be 0 or 1)} -> $enable"; }

            # check depends
            local depends=${OP_STEP_DEPENDS[$step]}
            ec=1
            if [[ "$depends" == "" ]]; then
                ec=0
            else
                is_stepid_valid $depends && ec=0
            fi
            [[ $ec -eq 0 ]] || { globec=1; error_msg "illegal OP_STEP_DEPENDS for step $step (must be a valid step id)} -> $depends"; }

        done


        # check joblog flag
        [[ $OP_SAVE_JOBLOG -le 1 ]] || { error_msg "illegal OP_SAVE_JOBLOG (0 or 1)"; globec=1; }


        return $globec
    }

    #-----------------------------------------------
    # verify given list of plugins
    #-----------------------------------------------
    verify_plugins() {
        local plugins="$ALL_PLUGINS"
        local rval=0
        local ec=0
        # Check plugins
        local p=
        for p in $plugins; do
            check_plugin "$p"
            ec=$?
            [[ $ec -eq 0 ]] || { 
                local msg
                msg=$(parse_check_plugin_ec "$p" "$ec")
                error_msg "msg"
                rval=1
            }
        done

        return $rval
    }

    #-----------------------------------------------
    # Check if label is required.
    # PL_TODO_LABELS_REQ has the format:
    # S1,S2-S5,...
    # where S2-S5 == S2,S3,S4,S5
    #-----------------------------------------------
    TSTis_required_label() {
        [[ $# -ne 1 ]] && return 1
        # if no req species is defined, return success
        [[ -n $PL_TODO_LABELS_REQ ]] || return 0

        # first split by '-'
        local lblarr=()
        eval $(create_array lblarr '-' "$PL_TODO_LABELS_REQ")
        ###lblarr=($(tokenize '-' $PL_TODO_LABELS_REQ))
        
        local i=0
        local ilast=$((${#lblarr[@]}-1))
        for ((i=0; i<ilast+1; i++)); do
            local lbls=()
            lbls=("$(tokenize ',' ${lblarr[$i]})")
            ###v_tokenize lbls ',' ${lblarr[$i]}
            inrange_set "$A" "${lbls[@]}" && return 0
            [[ $i -lt $ilast ]] || continue

        done
        return 1
    }

    #-----------------------------------------------
    # Check if label is a required label
    # Returns true (code==0) if either one is true:
    # 1. given label is part of PL_TODO_LABELS_REQ
    # 2. PL_TODO_LABELS_REQ is empty
    #-----------------------------------------------
    is_required_label() {
        [[ $# -ne 1 ]] && return 1
        
        # if no req species is defined, return success
        [[ -n $PL_TODO_LABELS_REQ ]] || return 0

        # check if given equals required
        local A="$1"
        local P=",${PL_TODO_LABELS_REQ},"
        [[ $P =~ ",${A}," ]] && return 0
        return 1
    }

    #-----------------------------------------------
    # check if the pair has a required label
    #-----------------------------------------------
    has_required_label() {
        [[ $# -ge 1 ]] || return 1
        debug_msg "check if required labels in [$PL_TODO_LABELS_REQ]"
        # no required labels defined - return OK
        [[ -n $PL_TODO_LABELS_REQ ]] || return 0
        # check each label, at least one has to a required label
        is_required_label "$1" && return 0
        [[ $# -ge 2 ]] && is_required_label "$2" && return 0
        # no required label
        debug_msg "NOPE"
        return 1
    }

    #-----------------------------------------------
    # set step id and label
    #-----------------------------------------------
    set_stepid() {
        [[ $# -ge 1 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local stepid="$1"
        local flag=""
        [[ $# -eq 2 ]] && flag="$2"
        [[ "$flag" == ":" ]] && flag=""

        STEP_ID="$stepid"
        STEP_ID_FLAG="${flag^^}"

        STEP_FULL_ID="$stepid"
        STEP_FULL_LBL="$stepid"
        if [[ -n $flag ]]; then
            STEP_FULL_ID="${stepid}:$flag"
            STEP_FULL_LBL="${stepid}_$flag"
        fi
        debug_msg "setting step id <$STEP_ID> with flag <$STEP_ID_FLAG>"
        is_stepid_valid "$STEP_ID" || fatal_msg "$FUNCNAME attempting to set invalid step id: $STEP_ID"
        return 0
    }

    #-----------------------------------------------
    # Check if a step is valid
    #-----------------------------------------------
    is_stepid_valid() {
        [[ $# -eq 1 ]] || return 1
        local step="$1"
        [[ -z $step ]] && return 1
        [[ "$step" == "ALL" ]] && return 0
        local allSteps=($(tokenize "," "${step}"))
        local s=""
        for s in ${allSteps[@]}; do
            inrange_set "$s" "$OP_ALL_STEPS_STR" || return 1
        done
        return 0
    }

    #-----------------------------------------------
    # check if step is batch
    #-----------------------------------------------
    is_runmode_batch() {
        local stepid="$STEP_ID"
        [[ $# -eq 1 ]] && stepid="$1"
        is_stepid_valid "$stepid" || fatal_msg "is_runmode_batch() invalid STEP_ID = $stepid"
        
        # if job scheduler is none, return 1 since none is used
        uses_scheduler || return 1

        local str=""
        str=$(get_runmode_str "$stepid")

        # check if run mode is batch
        [[ "$str" == "BATCH" ]] && return 0
        return 1
    }

    #-----------------------------------------------
    # check if step is interactive
    #-----------------------------------------------
    is_runmode_interactive() {
        local stepid="$STEP_ID"
        [[ $# -eq 1 ]] && stepid="$1"
        is_stepid_valid "$stepid" || fatal_msg "is_runmode_interactive() invalid STEP_ID = $stepid"

        # if job scheduler is none, return 1 since none is used
        uses_scheduler || return 1

        local str
        str=$(get_runmode_str "$stepid")
        
        # check if run mode is batch
        [[ "$str" == "INTERACTIVE" ]] && return 0
        return 1
    }

    #-----------------------------------------------
    # check if step is interactive
    #-----------------------------------------------
    is_runmode_local() {
        local stepid="$STEP_ID"
        [[ $# -eq 1 ]] && stepid="$1"
        is_stepid_valid "$stepid" || fatal_msg "is_runmode_local() invalid STEP_ID = $stepid"

        # if job scheduler is none, return 0
        uses_scheduler || return 0

        local str
        str=$(get_runmode_str "$stepid")
        
        # check if run mode is batch
        [[ "$str" == "LOCAL" ]] && return 0
        return 1
    }

    #-----------------------------------------------
    # get run mode
    #  0 - local
    #  1 - batch
    #  2 - interactive
    #-----------------------------------------------
    get_runmode() {
        local stepid="$STEP_ID"
        [[ $# -eq 1 ]] && stepid="$1"
        is_stepid_valid "$stepid" || fatal_msg "get_runmode() invalid STEP_ID = $stepid"

        # if no scheduler is used -> local
        uses_scheduler || { echo 0; return 0; }

        [[ "${OP_STEP_RUNMODE[$stepid]+1}" == "" ]] && fatal_msg "$FUNCNAME : OP_STEP_RUNMODE[$stepid] is not defined!"
        local X=${OP_STEP_RUNMODE[$stepid]}
        [[ "$X" == "BATCH" ]]       && { echo 1; return 0; }
        [[ "$X" == "INTERACTIVE" ]] && { echo 2; return 0; }
        return 255
    }

    get_runmode_str() {
        local stepid="$STEP_ID"
        [[ $# -eq 1 ]] && stepid="$1"
        is_stepid_valid "$stepid" || fatal_msg "get_runmode_str() invalid STEP_ID = $stepid"

        # if no scheduler is used -> local
        uses_scheduler || { echo "LOCAL"; return 0; }

        [[ "${OP_STEP_RUNMODE[$stepid]+1}" == "" ]] && fatal_msg "$FUNCNAME : OP_STEP_RUNMODE[$stepid] is not defined!"
        echo ${OP_STEP_RUNMODE[$stepid]}
        return 0
    }

    #-----------------------------------------------
    # check if step is done
    #-----------------------------------------------
    is_step_done() {
        local step=$STEP_ID
        [[ $# -eq 1 ]] && step=$1
        local donecode=0
        donecode=$(get_done_code $step)
        [[ $donecode -eq $DC_DONE ]] && return 0
        return 1
    }

    #-----------------------------------------------
    # check if step is enabled
    #-----------------------------------------------
    is_step_enabled() {
        local step=$STEP_ID
        [[ $# -eq 1 ]] && step=$1
        [[ ${OP_STEP_ENABLE[$step]} -eq 1 ]] && return 0
        return 1
    }

    disable_step() {
        local step=$STEP_ID
        [[ $# -eq 1 ]] && step=$1
        OP_STEP_ENABLE[$step]=0
    }
    enable_step() {
        local step=$STEP_ID
        [[ $# -eq 1 ]] && step=$1
        OP_STEP_ENABLE[$step]=1
    }
    
    #-----------------------------------------------
    # check step dependancy
    #-----------------------------------------------
    check_step_dependancy() {
        local stepid="$STEP_ID"
        [[ $# -eq 1 ]] && stepid="$1"
        is_stepid_valid "$stepid" || fatal_msg "$FUNCNAME invalid STEP_ID = $stepid"

        is_dryrun && return 0

        # get list of required steps
        # if step is not enabled, ignore its state
        local steps=($(tokenize "," "${OP_STEP_DEPENDS[$stepid]}"))
        # following check is needed for bash versions 4.x (?)
        # earlier versions treat steps[@] as an undefined variable if the array is empty
        # hence the need for checking its size
        if [[ ${#steps[@]} -gt 0 ]]; then
            local s=""
            for s in ${steps[@]}; do
                [[ ${OP_STEP_ENABLE[$s]} -eq 1 ]] || continue
                is_step_done $s || { error_msg "$s not done as required by $stepid"; return 1; }
            done
        fi
        return 0
    }

    #-----------------------------------------------
    # get steps required by given step
    # list is space separated and in execution order
    #-----------------------------------------------
    _get_reqby() {
        local step="$1"
        local allsteps="$2"
        [[ ${#step} -gt 0 ]] || { echo "$allsteps"; return 0; }

        
        local next=${OP_STEP_REQBY[$step]}
        [[ ${#next} -gt 0 ]] || { echo "$allsteps"; return 0; }
        [[ ${#allsteps} -eq 0 ]] || allsteps="$allsteps,"
        
        allsteps="$allsteps$next"
        
        ###local nsteps=($(tokenize ',' "$next"))
        local nsteps=()
        nsteps=( $(tokenize "," "$next") )
        ###v_tokenize nsteps ',' "$next"
        local s=
        for s in ${nsteps[@]}; do
            allsteps=$(_get_reqby "$s" "$allsteps")
        done
        echo "$allsteps"
        return 0
    }
    
    get_steps_required_by() {
        [[ $# -eq 1 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local step="$1"
        
        [[ "$step" == "" ]] && return 0
        is_stepid_valid "$step" || fatal_msg "get_steps_required_by() invalid step id <$step>"
        local reqstr=$(_get_reqby "$step" "")
        expand_step_list "$reqstr"
        return 0
    }

    # a test function for the above
    tst_reqby() {
        local step=""
        for step in ${OP_ALL_STEPS[@]}; do
            get_steps_required_by "$step"
            printf '%-20s -> %s\n' "$step" "${STEP_ID_LIST[@]}"
        done
    }
    
    #--------------------------------------------------
    # check all OP_STEP_REQTOOLS
    #--------------------------------------------------
    check_step_reqtools() {
        [[ $# -eq 1 ]] || return 0
        local step="$1"
        local toolvar=(${OP_STEP_REQTOOLS[$step]})

        [[ ${#toolvar[@]} -eq 3 ]] || return 0
        local dirvar=${toolvar[0]} # e.g DIR_BLAST
        local execmd=${toolvar[1]} # e.g segmasker
        local exevar=${toolvar[2]} # e.g EXE_SEGMASKER

        if isset_var "$exevar"; then
            local tool="${!exevar}"
            if [[ ! -x "$tool" ]]; then
                local dirtool="${!dirvar}"
                local errmsg=""
                local accept=1
                if [[ -d "$dirtool" ]]; then
                    errmsg="$step: missing required tool ($execmd), check directory $dirvar = ${!dirvar}"
                    accept=0
                elif [[ ${#dirtool} -eq 0 ]] ; then
                    errmsg="$step: no directory path for required tool ($execmd), make sure to set $dirvar in your orthologer_conf.sh"
                else
                    errmsg="$step: the directory for the required tool ($execmd) is not readable, check directory $dirvar = ${!dirvar}"
                fi
                if ((accept == 0)); then
                    if is_runmode_batch "$step"; then
                        warn_msg "$errmsg"
                        warn_msg "above warning is accepted as this step ($step) is expected to run in batch mode"
                        warn_msg "make sure that $dirvar is correct on whatever computing node the scripts are expected to run"
                        return 0
                    fi
                fi
                error_msg "$errmsg"
                is_dryrun || return 1
                return 0
            fi
            verbose_info_msg "$step: tool found $exevar = $tool"
            return 0
        fi
        return 0
    }

    #--------------------------------------------------
    # check all OP_STEP_REQTOOLS
    #--------------------------------------------------
    check_step_reqtools_all() {
        local step=""
        for step in ${OP_ALL_STEPS[@]}; do
            check_step_reqtools "$step"
        done
    }
    
    #-----------------------------------------------
    # check scheduler status
    # args: <nr of jobs to be submitted>
    # It checks the nr of jobs pending/running for
    # the current step. If it is less than
    # OP_NJOBMAX_BATCH - N(to be submitted), then
    # return OK.
    # Otherwise wait until the condition
    # is fullfilled.
    #-----------------------------------------------
    check_scheduler_status() {
        # not enough args - quit
        [[ $# -eq 1 ]] || return 0

        # if no max job count is given - always allow new jobs
        [[ $OP_NJOBMAX_BATCH -gt 0 ]] || return 0

        # quit if not batch
        is_runmode_batch || return 0

        # get args
        local ntobe=$1  # nr of jobs to be submitted

        # update jobid file - remove jobids that are done
        update_jobid_file $STEP_ID

        # nr of jobs running/pending
        # don't need to do sj_check_jobids() here since jobid file is updated
        # just get the nr of jobs as given by the job id file
        local nrunall
        nrunall=$(get_njobs $STEP_ID)

        # nr of jobs after submission
        # if it is greater than the largest allowed, wait!
        # otherwise return OK -> scheduler is available
        local nrunall2=$((nrunall+ntobe))
        [[ $nrunall2 -gt $OP_NJOBMAX_BATCH ]] || return 0

        # get the max nr of running/pending jobs allowed before submitting
        local maxrun=$((OP_NJOBMAX_BATCH-ntobe))
        [[ $maxrun -ge 0 ]] || { step_error_msg "fatal pblm -> OP_NJOBMAX_BATCH=$OP_NJOBMAX_BATCH is smaller than nr of jobs to submit ($ntobe)"; exit 255; }

        info_msg "waiting for batch queue to be sufficiently free (less than or equal to $maxrun jobs running/pending - currently at $nrunall)"

        # wait for scheduler maxrun jobs are running/pending
        sj_wait_scheduler $STEP_ID $maxrun || { step_error_msg "aborting step since job id wait failed"; exit 255; }

        return 0
    }

    #-----------------------------------------------
    # print start status
    #-----------------------------------------------
    step_start() {
        [[ $# -eq 1 ]] || return 255
        # clear jobid file if it exists
        is_dryrun || clear_jobid_file $STEP_ID

        # count nr of sources using the nr of commas
        local nsrc=$1
        [[ $nsrc -eq 0 ]] && { is_dryrun || force_remake_none; }

        # step status
        step_status_msg $nsrc
    }

    #-----------------------------------------------
    # print step status
    #-----------------------------------------------
    step_status_msg() {
        [[ $# -eq 1 ]] || return 255
        local nsrc=$1
        local donecode=0
        donecode=$(get_done_code)

        local donestr=$(get_done_code_str $donecode)
        local nparallel=$(get_step_nparallel)
        local runmode=$(get_runmode_str)
        local nbatch=$(get_step_nbatch)
        
        local enabledstr="YES"
        is_step_enabled || { donestr="DISABLED"; enabledstr="NO"; }
        local datestr=
        is_dryrun || datestr=$(printf "%-25s" "$(date_locale)")

        ##    local infostr=$(printf "%-8s %8d %8d %8d" ${runmode:0:5} $nsrc $nparallel ${OP_STEP_NTHREADS[$STEP_ID]})
        local infostr=
        infostr=$(printf "${datestr}%-25s%-8s %-8s %8d %8d %8d %8d %8d    %s" "$STEP_FULL_ID" $enabledstr ${runmode:0:5} $nsrc ${OP_STEP_NMERGE[$STEP_ID]} $nbatch $nparallel ${OP_STEP_NTHREADS[$STEP_ID]} "$donestr")
        printf "${COLOR_GREEN}${infostr}${COLOR_RESET}\n"
    }

    #-----------------------------------------------
    # set status to DC_DONE
    #-----------------------------------------------
    step_done() {
        is_dryrun || force_remake_none
        step_status_msg 0
        cnt_sum_msg
    }

    #-----------------------------------------------
    # print step timing
    #-----------------------------------------------
    # give a T range (integer)
    step_timing() {
        [[ $# -ge 2 ]] || return 0
        # start and end time
        local T0=$1
        local T1=$2
        local N=0
        [[ $# -ge 3 ]] && N=$3
        # delta time in seconds
        local dT=$(delta_time "$T0" "$T1")        
        if [[ "$N" == "0" ]]; then
            step_info_msg "DONE time $dT seconds"
            return 0
        fi
        
        # text
        local txt="rate (secs/job):"
        [[ $# -gt 3 ]] && txt="$4"

        # rate (sec/jobs), set to N if dT=0
        local R=$(awk "BEGIN{ printf \"%6.4f\n\",(1.0*${dT})/($N); }")
        step_info_msg "$txt $R"
        #
    }

    # give a single dT (time_nsec)
    step_timing_dt() {
        [[ $# -ge 2 ]] || return 0
        # start and end time
        local dT=$1
        # nr of jobs
        local N=$2
        # text
        local txt="rate (secs/job):"
        [[ $# -gt 3 ]] && txt="$4"

        # rate (sec/jobs), set to N if dT=0
        local R=0
        [[ $N -gt 0 ]] && R=$(awk "BEGIN{ printf \"%6.4f\n\",(1.0*${dT})/(1.0e9*${N}); }")
        step_info_msg "$txt $R"
    }

    # get timing from T_*_JOBS
    step_timing_dispatch() {
        [[ $# -ge 1 ]] || return 0
        # nr of jobs
        local N=$1
        # text
        local msg="rate (secs/job):"
        [[ $# -gt 1 ]] && msg="$2"
        # rate (sec/jobs), set to N if dT=0
        local R=0
        [[ $N -gt 0 ]] && R=$(awk "BEGIN{ printf \"%6.4f\n\",(1.0*${T_RUN_JOBS})/($N); }")
        if (( DISPATCH_ERR == 0 )); then
            step_info_msg "done $msg $R"
        else
            step_error_msg "done with error(s) $msg $R"
        fi
    }

    #-----------------------------------------------
    # initialise a step
    # to be called in the beginning of any
    # step_<STEPID>() function
    # Return cases:
    # - arg error  : fatal msg
    # - disabled   : return 1
    # - dependancy : return 255
    # - done       : return 2
    # - all ok     : return 0
    #-----------------------------------------------
    init_step() {
        [[ $# -ge 1 ]] || fatal_msg "init_step() given the wrong nr of args ($*)"
        local stepid="$1"
        local flag=""
        [[ $# -eq 2 ]] && flag="$2"
        [[ "$flag" == ":" ]] && flag=""

        debug_msg "init_step for $stepid with flag $flag"

        # clear msg counter for warning messages
        CHECK_COUNTER=0

        # set step id
        set_stepid "$stepid" "$flag"

        # check if enabled
        is_step_enabled || { step_status_msg 0; debug_msg "step disabled $stepid"; return 1; }

        # check dependancy (done flag from previous step, not source files)
        check_step_dependancy || { debug_msg "step dependancy $stepid"; return 255; }

        # remake defined by done flag
        check_done_flag || { step_done; debug_msg "step done $stepid"; return 2; }

        # check tool
        check_step_reqtools $STEP_ID || { debug_msg "step missing required tool(s) $stepid"; return 255; }

        debug_msg "step ok $stepid with flag $flag"
        return 0
    }

    save_jobspecs() {
        _save_JOBLABELS=(${JOBLABELS[@]})
        _save_JOBSUBLBL=(${JOBSUBLBL[@]})
        _save_JOBARGS=(${JOBARGS[@]})
    }

    restore_jobspecs() {
        JOBLABELS=(${_save_JOBLABELS[@]})
        JOBSUBLBL=(${_save_JOBSUBLBL[@]})
        JOBARGS=(${_save_JOBARGS[@]})
    }
    #-----------------------------------------------
    #
    #-----------------------------------------------
    check_step_joblists() {
        # check sources and labels
        set +u
        #[[ ${#SRCFILES[@]}  -gt 0 ]] || fatal_msg "$FUNCNAME empty SRCFILES at step $STEP_ID"
        [[ ${#JOBARGS[@]}   -gt 0 ]] || fatal_msg "$FUNCNAME empty JOBARGS at step $STEP_ID"
        [[ ${#JOBLABELS[@]} -gt 0 ]] || fatal_msg "$FUNCNAME empty JOBLABELS at step $STEP_ID"
        [[ ${#JOBSUBLBL[@]} -gt 0 ]] || fatal_msg "$FUNCNAME empty JOBSUBLBL at step $STEP_ID"

        #[[ ${#SRCFILES[@]}  -eq ${#JOBLABELS[@]} ]] || fatal_msg "$FUNCNAME SRCFILES and JOBLABELS have different size at step $STEP_ID"
        [[ ${#JOBARGS[@]}   -eq ${#JOBLABELS[@]} ]] || fatal_msg "$FUNCNAME JOBARGS and JOBLABELS have different size at step $STEP_ID"
        [[ ${#JOBSUBLBL[@]} -eq ${#JOBLABELS[@]} ]] || fatal_msg "$FUNCNAME JOBSUBLBL and JOBLABELS have different size at step $STEP_ID"
        set -u
    }

    #-----------------------------------------------
    # rm command
    #-----------------------------------------------
    export RMCMD_REMOVED=0
    rmcmd() {
        [[ $# -gt 0 ]] || return 0
        local opt="--preserve-root -f"
        [[ $VERBOSE -gt 0 ]] && opt="$opt -v"
        local f=""
        for f in $*; do
            [[ -f "$f" ]] && rm $opt "$f" && RMCMD_REMOVED=$((RMCMD_REMOVED+1))
        done
    }

    #-----------------------------------------------
    # cleanup step per todo label
    #-----------------------------------------------
    cleanup_step_per_todo_label() {
        [[ $# -ge 2 ]] || return 0
        local srcdir="$1"
        shift
        local extwc="$*"
        [[ -d $srcdir ]] || return 0
        
        local t=""
        local f=""
        local ext=""
        for t in "${PL_TODO_LABELS[@]}"; do
            has_required_label $t || continue
            for ext in $extwc; do
                f=$srcdir/${t}.$ext
                [[ -f "$f"      ]] && rmcmd "$f"
                [[ -f "${f}.gz" ]] && rmcmd "${f}.gz"
            done
            local joblogdir=""
            ###joblogdir=$(get_joblog_dir $t)
            v_get_joblog_dir joblogdir "$t"

            for f in $joblogdir/${PL_TODO_NAME}_${STEP_ID}_${t}_*.log; do
                [[ -f "$f" ]] && rmcmd "$f"
            done
        done
    }

    #-----------------------------------------------
    # cleanup one step
    #-----------------------------------------------
    export CLEANUP_INTERACTIVE=${CLEANUP_INTERACTIVE=1}
    cleanup_single_step() {
        # check cleanup function
        isfunc cleanup_step_${STEP_ID,,} || { step_warn_msg "missing cleanup function"; return 0; }

        # do the cleanup
        debug_msg "Cleanup of $STEP_ID"

        # if interactive, ask user
        if [[ $CLEANUP_INTERACTIVE -eq 1 ]]; then
            local -u answer="N"
            echo
            echo "About to remove result files (step: $STEP_ID , todo label : $PL_TODO_NAME)"
            read -r -p "Proceed (y/N/a(ll))? " answer
            if [[ "$answer" != "Y" ]] && [[ "$answer" != "A" ]]; then
                echo "Aborting cleanup"
                return 0
            fi
            [[ "$answer" == "A" ]] && CLEANUP_INTERACTIVE=0
            echo
        fi

        # remove file(s)
        cleanup_step_${STEP_ID,,}
    }

    #-----------------------------------------------
    # cleanup step(s)
    #-----------------------------------------------
    cleanup_steps() {
        local ec=0
        expand_step_list_check "$1"
        ec=$?
        [[ $ec -eq 1 ]] && return 0
        [[ $ec -eq 0 ]] || return 255

        # loop over given steps
        local querycommon=0
        local s=
        for s in "${STEP_ID_LIST[@]}"; do
            set_stepid "${s^^}"
            # check if step should be cleaned up
            # protect against cleaning up of common data (alignments etc) when a specific TODO file is given.
            local docleanup=0
            if [[ "$PL_TODO_NAME" != "$COMMON_TODO_NAME" ]]; then
                if [[ ${OP_STEP_DATA[$STEP_ID]} == "COMMON" ]]; then
                    # check first time encountered
                    if [[ $querycommon -eq 0 ]]; then
                        local answer="N"
                        echo
                        echo "--------------------------"
                        echo " Explicit TODO file given "
                        echo "--------------------------"
                        echo "Todo        : $PL_TODO"
                        echo "Common todo : $COMMON_TODO"
                        echo
                        echo "Do you want to clean up todo label dependent steps?"
                        echo "That is, all steps preceeding SETBRH." #TODO no ref to ODB specifics here
                        echo "If yes, it will still query per step."
                        echo
                        read -r -p "Proceed (y/N)? " answer
                        answer=${answer^^}
                        [[ "$answer" == "Y" ]] && docleanup=1
                        querycommon=1
                        echo
                    fi
                else
                    # it's label dependent part - possible to cleanup safely
                    docleanup=1
                fi
            else
                # not a specific TODO file
                docleanup=1
            fi
            [[ $docleanup -eq 1 ]] && cleanup_single_step
        done
    }

    #-----------------------------------------------
    # variables set in time_step_xxx
    #-----------------------------------------------
    export TIME_MEAN=0
    export TIME_SUM=0
    export TIME_MIN=
    export TIME_MAX=0
    export TIME_N=0
    export TIME_NMISS=0
    time_reset() {
        TIME_N=0
        TIME_SUM=0
        TIME_MEAN=0
        TIME_NMISS=0
        TIME_MIN=
        TIME_MAX=0
    }

    time_set_minmax() {
        local T=$1
        if [[ -z $TIME_MIN ]]; then
            TIME_MIN=$T
            TIME_MAX=$T
        else
            float_lt $T $TIME_MIN && TIME_MIN=$T
            float_gt $T $TIME_MAX && TIME_MAX=$T
        fi
    }
    time_set_from_footer() {
        local f="$1"
        [[ -f $f ]] || return 0
        if get_footer_time "$f"; then
            TIME_SUM=$(calc ${_FOOTER_TIME}+${TIME_SUM})
            TIME_N=$((TIME_N+1))
            time_set_minmax $_FOOTER_TIME
        else
            TIME_NMISS=$((TIME_NMISS+1))
        fi
    }
    time_set_mean() {
        (( TIME_N == 0 )) || TIME_MEAN=$(calc $TIME_SUM/$TIME_N)
    }
    #----------------------------------------------------------------------
    # Compute time used per todo label from '#TIME' tags
    #----------------------------------------------------------------------
    time_step_per_todo_label() {
        time_reset
        [[ $# -ge 2 ]] || return 0
        local srcdir="$1"
        shift
        local extwc="$*"
        [[ -d $srcdir ]] || return 0
        
        local t=""
        local f=""
        local ext=""
        for t in "${PL_TODO_LABELS[@]}"; do
            has_required_label $t || continue
            for ext in $extwc; do
                f="$srcdir/${t}.$ext"
                time_set_from_footer "$f"
            done
        done
        time_set_mean
    }
    
    #----------------------------------------------------------------------
    # parse run log
    #
    # For current version of log , there will be a line at the very end
    # giving the elapsed time, minus a small overhead.
    # If not, it will use the times from the first and last log message.
    #----------------------------------------------------------------------
    time_parse_log() {
        [[ $# -eq 1 ]] || return 0
        local flog="$1"
        local timeRunlog="N/A"
        [[ -f $flog ]] || return 0
        timeRunlog=$(grep 'Pipeline time elap' $latestRunlog | awk 'BEGIN{sum=0;} {sum+=$8;} END{print sum;}')
        if [[ -z $timeRunlog ]]; then
            local first=$(head -n1 $flog | awk '{print$1,$2}')
            local last=$(tail -n1 $flog | awk '{print$1,$2}')
            if [[ -n "$first" ]] && [[ -n "$last" ]]; then
                local T0=$(date --date="$first" +"%s" 2>/dev/null)
                local T1=$(date --date="$last" +"%s" 2>/dev/null)
                # there might be escape characters (color)
                [[ -z "$T0" ]] && T0=$(date --date="${first:5}" +"%s" 2>/dev/null)
                [[ -z "$T1" ]] && T1=$(date --date="${last:5}" +"%s" 2>/dev/null)
                [[ -n "$T0" ]] && [[ -n "$T1" ]] && timeRunlog=$((T1-T0))
            fi
        fi
        echo $timeRunlog
    }
    
    #----------------------------------------------------------------------
    # Compute time used per step using the '#TIME' tag at the end.
    # Note that these times do not include overhead for scheduling
    # and error checking.
    #----------------------------------------------------------------------
    time_steps() {
        local ec=0
        local seplen=170
        expand_step_list_check "$1"
        ec=$?
        [[ $ec -eq 1 ]] && return 0
        [[ $ec -eq 0 ]] || return 255

        # header for table
        print_separator '-' $seplen
        printf "%-25s %16s %16s %16s %16s %16s %16s %16s %16s\n" "STEP" "N" "TSUM" "TMEAN" "TMIN" "TMAX" "TREAL" "TRMAX" "NMISS"
        print_separator '-' $seplen

        # loop over given steps and accumulate #TIME tags
        local s=
        local Ttot=0
        local Treal=0
        local Trmax=0
        for s in "${STEP_ID_LIST[@]}"; do
            [[ ${OP_STEP_ENABLE[$s]} -eq 1 ]] || continue
            isfunc time_step_${s,,} || continue
            time_step_${s,,}
            local nmerge=${OP_STEP_NMERGE[$s]}
            local nparallel=${OP_STEP_NPARALLEL[$s]}
            # estimated time assuming time per job is TIME_MEAN
            local T=0
            # estimated time assuming time per job is TIME_MAX
            local TMAX=0
            if [[ "$nmerge" -ge "$TIME_N" ]]; then
                T="$TIME_SUM"
                TMAX="$TIME_SUM"
            else
                # more than one batch
                local nbatch=$(calc_nbatch "$TIME_N" "$nmerge")
                local nparjobs=$(calc $nbatch/$nparallel)
                local njobs=$(calc "1+int($nparjobs)")
                #local njobsPart=$(calc "${nparjobs}-$njobsFull")
                local Tmerge=$(calc "${TIME_MEAN}*$nmerge")
                T=$(calc "${Tmerge}*$njobs")
                TMAX=$(calc "${TIME_MAX}*${nmerge}*$njobs")
            fi
            printf "%-25s %16s %16.2f %16.2f %16.2f %16.2f %16.2f %16.2f %16s\n" "$s" "$TIME_N" "$TIME_SUM" "$TIME_MEAN" "$TIME_MIN" "$TIME_MAX" "$T" "$TMAX" "$TIME_NMISS"
            # accumulate times
            Ttot=$(calc "${Ttot}+${TIME_SUM}")
            Treal=$(calc "${Treal}+$T")
            Trmax=$(calc "${Trmax}+$TMAX")
            
        done

        # get time from log file, if given
        local tlog=""
        [[ -f $PIPELINE_RUNLOG ]] && tlog=$(time_parse_log $PIPELINE_RUNLOG)

        # print timing summary
        print_separator '-' $seplen
        printf "%-48s %16.2f seconds - if all jobs were run sequentially (TSUM)\n" "Total accumulated time" "$Ttot"
        printf "%-48s %16.2f seconds - corrected for parallel/merged jobs (TREAL)\n" "Real time spent in jobs assuming T(job)=mean" "$Treal"
        printf "%-48s %16.2f seconds - corrected for parallel/merged jobs (TRMAX)\n" "Real time spent in jobs assuming T(job)=max" "$Trmax"
        if [[ -n $tlog ]]; then
            local Toverhead=$(calc "$tlog - $Treal")
            if [[ "${Toverhead:0:1}" == "-" ]]; then
                printf "%-48s %16.2f seconds -> is probably wrong as it is less that the estimated time\n" "Total time from pipeline run log" "$tlog"
            else
                printf "%-48s %16.2f seconds\n" "Total time from pipeline run log" "$tlog"
                printf "%-48s %16.2f seconds\n" "Overhead from scheduling etc" "$Toverhead"
            fi
        fi
        print_separator '-' $seplen
    }
    
    #-----------------------------------------------
    # add to jobargs
    # arguments are:
    # 1. addjargs : job arg string to be added
    # 2. nmerged  : nr of args already merged
    # 3. label    : if nmerged==0 then set JOBLABELS[]=label
    # 4. sublbl   : if nmerged==0 then set JOBSUBLBL[]=sublbl
    #-----------------------------------------------
    add_jobargs() {
        [[ $# -eq 4 ]] || fatal_msg "add_jobargs: wrong nr of arguments"
        local addjargs="$1"
        local nmerged="$2"
        local label="$3"
        local sublbl="$4"
        
        # get nr of JOBARGS
        local jobind=${#JOBARGS[@]}
        # if nmerged is > 0 -> JOBARGS must contain atleast one element
        if [[ $nmerged -gt 0 ]]; then
            # bug capture
            [[ $jobind -gt 0 ]] || fatal_msg "negative jobind in add_jobargs()"
            jobind=$((jobind-1))
        fi

        # add job args
        local args="$addjargs"
        [[ $nmerged -eq 0 ]] || args="${JOBARGS[$jobind]};$addjargs"

        # set job args
        JOBARGS[$jobind]="$args"
        
        if [[ $nmerged -eq 0 ]]; then
            # set these only for the first one
            # the consequence of this is that the joblog file
            # will be named after the first pair
            JOBLABELS[$jobind]="$label"
            JOBSUBLBL[$jobind]="$sublbl"
        fi
    }
    #-----------------------------------------------
    # step per label
    #
    # It assumes the following arrays are set:
    #   SRCFILES == ( array of all selected sources )
    #   TRGFILES == ( array of all selected targets )
    #   JOBARGS  == ( array of all job arguments )
    #
    # The JOBLABELS and JOBSUBLBL are of the same size
    # as JOBARGS and set previously
    #
    #-----------------------------------------------
    step_per_label() {
        # get arguments
        [[ $# -ge 2 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local stepid="$1"
        local script="$2"
        local doinit="0"
        local single="N"
        [[ $# -ge 3 ]] && doinit="$3"
        [[ $# -ge 4 ]] && single="$4"
        # single -> flag if script at this level should be run in single lane (not parallel)
        # 'Y' -> yes do that
        # 'N' -> no, default behaviour
        single=${single^^}
        [[ $single == "N" ]] || single="Y"
        
        # make sure stepid is upper case
        stepid=${stepid^^} # upper case

        debug_msg "starting step_per_label for stepid $stepid"
        # check script
        [[ -x "$script" ]] || fatal_msg "step_per_label() given script is not executable : <$script>"

        # error code
        local ec=0
        local firstA=''        
        # initialise step
        if [[ $doinit == "1" ]]; then
            init_step $stepid
            ec=$?
        
            debug_msg "$stepid - init_step with ec = $ec"
            # step is disabled?
            [[ $ec -eq 1   ]] && return 0
            # step id done?
            [[ $ec -eq 2   ]] && return 0
            # step dependancy?
            [[ $ec -eq 255 ]] && return 255
        else
            load_done_flag || return 255
        fi

        # check job lists - fatal msg if fail
        # this ensures that the following arrays are of same size
        # - SRCFILES
        # - TRGFILES
        # - JOBARGS
        # - JOBLABELS
        # - JOBSUBLBL
        # Any failures will result in a fatal msg and exit
        is_dryrun || check_step_joblists

        # nr of labels
        local nlabels=${#JOBARGS[@]}

        # start step
        if [[ $doinit == "1" ]]; then
            step_start $nlabels || fatal_msg "step_per_label: step_start() failed (arg=$nlabels)"
        fi

        # check if dry run
        is_dryrun && return 0

        # check if batch mode
        local isbatch=0
        is_runmode_batch && isbatch=1

        # nr of jobs to submit in one go
        local nsubmax=$OP_NJOBMAX_BATCH

        # max nr of merged pairs
        local maxnmerge=${OP_STEP_NMERGE[$stepid]}

        if [[ "$single" == "Y" ]]; then
            info_msg "running step $STEP_ID in single lane - leaving parallel jobs to sub script"
            nsubmax=1
            maxnmerge=1
        fi
        # Note that JOBARGS, JOBLABELS and JOBSUBLBL all contain the list of SELECTED todo labels.
        # They are thus a subset of PL_TODO_LABELS.
        #
        # save JOBARGS - note need this change in IFS in order to handle
        # arguments containing spaces
        local full_jobargs=()
        local full_joblabels=()
        local full_jobsublbl=()
        local full_src=()
        local full_trg=()
        eval $(create_array_eol full_jobargs   "${JOBARGS[@]}")
        eval $(create_array_eol full_joblabels "${JOBLABELS[@]}")
        eval $(create_array_eol full_jobsublbl "${JOBSUBLBL[@]}")
        eval $(create_array_eol full_src       "${SRCFILES[@]}")
        eval $(create_array_eol full_trg       "${TRGFILES[@]}")

        save_jobspecs
        
        # reset job lists
        reset_joblists

        # get which step id this step depends on...
        local stepsrc=''
        stepsrc=$(get_step_depend $stepid)

        # counters
        local nsub=0
        local nadded=0  # accumulated nmerged
        local nmerged=0 # reset after each submit
        local nfailed=0

        # start time of loop
        ##local TL0=$(time_nsec)

        # loop over all TODO labels
        local nlabelsm1=$((nlabels-1))    
        local addlabel=0

        [[ -z $stepsrc ]] || debug_msg "Source ($stepsrc)"
        debug_msg "Target ($stepid)"

        local ind=0
        for (( ind=0; ind<nlabels; ind++ )); do

            local fsrc="${full_src[$ind]}"
            local ftrg="${full_trg[$ind]}"
            
            # flag last one
            local last=0
            [[ $ind -eq $nlabelsm1 ]] && last=1

            # get label
            local A=${full_joblabels[$ind]}
            debug_msg "got job label $A"

            # flag if the label should be added
            addlabel=0
            
            # select job if target is requested to be (re)made
            debug_msg "check source and target names"
            if [[ -f $fsrc ]]; then
                debug_msg "check remake target=$(basename $ftrg)   source=$(basename $ftrg)"
                remake_target "$ftrg" "$fsrc" && addlabel=1
                debug_msg "---->>>>>  remake_target $fsrc -> $ftrg -> $addlabel"
            else
                error_msg "missing source file $fsrc"
                nfailed=$((nfailed+1))
            fi
            debug_msg "set label $(basename ${fsrc}) with jobargs <${full_jobargs[$ind]}> , joblabel <${full_joblabels[$ind]}>, sublabel <${full_jobsublbl[$ind]}>"
            # add label to JOBARGS
            if [[ $addlabel -eq 1 ]]; then
                step_info_msg "<${full_joblabels[$ind]}> add label"
                # note JOBARGS will get a new element if nmerged==0
                # otherwise the args are merged
                add_jobargs "${full_jobargs[$ind]}" "$nmerged" "${full_joblabels[$ind]}" "${full_jobsublbl[$ind]}"

                # set first label if mergeing jobs
                (( nmerged == 0 )) && firstA="$A"
                
                # global counter
		        nadded=$((nadded+1))

                # counter for nr of merged pairs
                nmerged=$((nmerged+1))
            else
                step_info_msg "skipping <${full_joblabels[$ind]}>"
            fi

            # check if its time to submit jobs
            local nsrc=${#JOBARGS[@]}
            local dosubmit=0
            
            # if nr of JOBARGS is nsubmax, we should probably submit now
            [[ $nsrc -lt $nsubmax ]] || dosubmit=1

            # but if not all merged, we should wait and see
            [[ $nmerged -eq $maxnmerge ]] || dosubmit=0

            # reached last element - submit now!
            [[ $last -eq 0 ]] || dosubmit=1
            
            # ahh, but if there is nothing in the first place...
            [[ $nsrc -gt 0 ]] || dosubmit=0
            
            # submit if conditions are met
            if [[ $dosubmit -eq 1 ]]; then
                step_info_msg "submit $nsrc jobs"
                # make sure nr of srcs is non-zero
                # how can this condition be true??
                [[ $nsrc -gt 0 ]] || { nmerged=0; continue; }

                # check if scheduler is ready
                check_scheduler_status $nsrc

                # nr of submissions
                nsub=$((nsub+1))

                # dispatch batch of jobs
                dispatch_cmd "$script"
                ec=$?
                [[ $ec -eq 0 ]] || nfailed=$((nfailed+1))
                                
#                # submit script
#                run_cmd "$script"
#                ec=$?
#                [[ $ec -eq 0 ]] || nfailed=$((nfailed+1))
#
#                # wait
#                if [[ $isbatch -eq 1 ]]; then
#		            sj_wait || { step_error_msg "aborting step since job id wait failed"; exit 255; }
#	            fi
#
#                # get timing
#                local TL1=0
#                TL1=$(time_nsec)
#
#                # check error (execution of alignments have finished)
#                check_error $ec "failed $stepid"
#                [[ $? -eq 0 ]] || { restore_jobspecs; return 1; }

                #
                (( ec == 0 )) || { restore_jobspecs; return $ec; }
                
                # clear job arrays
                reset_joblists

                # print timing
                local lblstr="$A"
                (( maxnmerge>1 )) && lblstr="${firstA} - $A"
                step_timing_dispatch "$nsrc" "($lblstr , N(proc)=$nsrc) rate (secs/job):"

                ## reset start time
                #TL0=$(time_nsec)

                # reset nmerged counter
                nmerged=0

                if (( SIGNAL_SOFT_ABORT==1 )); then
                    local sep="$(print_separator)"
                    step_info_msg $sep
                    step_info_msg "abort requested!"
                    step_info_msg "last label processed: $A"
                    step_info_msg $sep
                    exit 0
                fi

            fi
            # if maxmerged -> reset nmerged=0
            # -> this will force the add_jobargs to add a new element in the JOBARGS array
            [[ $nmerged -eq $maxnmerge ]] && nmerged=0
            
        done
        debug_msg "submitted $nsub job(s) with $nfailed failed job(s)"

        # BUG catching
        [[ ${#JOBARGS[@]} -eq 0 ]] || fatal_msg "nr of unprocessed sources is ${#JOBARGS[@]}"

        # check for bad results
        if [[ $nfailed -gt 0 ]]; then
            step_error_msg "failed submitting/running $nfailed job(s)"
            # suggest where to look for a problem
            step_info_msg "Look for files JobLog/*${STEP_ID}* or if it's a pair wise step (eg ALIGNENT) in JobLog/<fasta label>/."
            step_info_msg "In particular, look for log files from stderr (has 'error' in the name)."
            step_info_msg "If no JobLog files are found, it means the job submission itself failed."
            step_info_msg "In such a case, check the scheduler settings:"
            if [[ "$SCHEDULER_LABEL" == "NONE" ]]; then
                step_info_msg "-> seems like no scheduler is used - contact OrthoDB support"
            else
                step_info_msg "-> check your scheduler settings ($SCHEDULER_LABEL)."
                step_info_msg "-> if the scheduler requires some extra options, add them to OP_STEP_SCHEDOPTS (one per step)."
            fi
            
            force_remake_missing
            restore_jobspecs
            return 3
        fi

        if [[ $doinit == "1" ]]; then
            # mark step as done
            step_done
        fi
        
        restore_jobspecs
        return 0
    }

    #-----------------------------------------------
    # step per todo label
    #
    # It assumes the following arrays are set (init_step_per_todo_label)
    #   SRCFILES == ( array of all selected sources )
    #   TRGFILES == ( array of all selected targets )
    #   JOBARGS  == ( array of all selected todo labels )
    #
    # The JOBLABELS and JOBSUBLBL are of the same size
    # as JOBARGS (check_step_joblists)
    #-----------------------------------------------
    step_per_todo_label() {
        # get arguments
        [[ $# -ge 2 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local stepid="$1"
        local script="$2"
        local doinit="1"
        [[ $# -ge 3 ]] && doinit="$3"

        local ec=0

        # start time of step
        local T0=$(time_nsec)


        # make sure stepid is upper case
        stepid=${stepid^^} # upper case

        debug_msg "starting step_per_todo_label for stepid $stepid"
        
        # check script
        [[ -x "$script" ]] || fatal_msg "step_per_todo_label() given script is not executable : <$script>"

        # initialise step
        if [[ $doinit == "1" ]]; then
            init_step $stepid
            ec=$?
            debug_msg "$stepid - init_step with ec = $ec"
            # step is disabled?
            [[ $ec -eq 1   ]] && return 0
            # step id done?
            [[ $ec -eq 2   ]] && return 0
            # step dependancy?
            [[ $ec -eq 255 ]] && return 255
        else
            load_done_flag || return 255
        fi
        
        # check job lists - fatal msg if fail
        # this ensures that the following arrays are of same size
        # - SRCFILES
        # - TRGFILES
        # - JOBARGS
        # - JOBLABELS
        # - JOBSUBLBL
        # Any failures will result in a fatal msg and exit
        is_dryrun || check_step_joblists

        # nr of labels
        local nlabels=1
        is_dryrun || nlabels=${#JOBARGS[@]}

        # start step
        step_start $nlabels || fatal_msg "step_per_todo_label: step_start() failed (arg=$nlabels)"

        # check if dry run
        is_dryrun && return 0

        # check if batch mode
        local isbatch=0
        is_runmode_batch && isbatch=1

        # nr of jobs to submit in one go
        local nbatchmax=$(get_step_nbatch)

        # max nr of merged pairs
        local maxnmerge=${OP_STEP_NMERGE[$STEP_ID]}

        # Note that JOBARGS, JOBLABELS and JOBSUBLBL all contain the list of SELECTED todo labels.
        # They are thus a subset of PL_TODO_LABELS.
        #
        # save JOBARGS - note need this change in IFS in order to handle
        # arguments containing spaces
        local full_jobargs=()
        local full_src=()
        local full_trg=()
        eval $(create_array_eol full_jobargs "${JOBARGS[@]}")
        eval $(create_array_eol full_src     "${SRCFILES[@]}")
        eval $(create_array_eol full_trg     "${TRGFILES[@]}")

        save_jobspecs
        
        # reset job lists
        reset_joblists

        # get which step id this step depends on...
        local stepsrc=''
        stepsrc=$(get_step_depend $STEP_ID)

        # counters
        local nsub=0
        local nadded=0  # accumulated nmerged
        local nmerged=0 # reset after each submit
        local nfailed=0

        # first label, in case of merge
        local firstA=''

        # loop over all TODO labels
        local nlabelsm1=$((nlabels-1))    
        local addlabel=0
        local ind=0
        for (( ind=0; ind<nlabels; ind++ )); do

            # flag last one
            local last=0
            [[ $ind -eq $nlabelsm1 ]] && last=1

            # get label
            ###            local A=${PL_TODO_LABELS[$ind]}
            local A=${full_jobargs[$ind]}
            debug_msg "check label $A"

            has_required_label $A || continue
            
            # flag if the label should be added
            addlabel=0
            
            # select job if target is requested to be (re)made
            if [[ -f "${full_src[$ind]}" ]]; then
                remake_target "${full_trg[$ind]}" "${full_src[$ind]}" && addlabel=1
                debug_msg "remake_target ${full_trg[$ind]} - ${full_src[$ind]} -> $addlabel"
            else
                debug_msg "missing source file $(trim_name ${full_src[$ind]})"
                nfailed=$((nfailed+1))
            fi

            # add label to JOBARGS
            if [[ $addlabel -eq 1 ]]; then
                # note JOBARGS will get a new element if nmerged==0
                # otherwise the args are merged
                add_jobargs "${full_jobargs[$ind]}" "$nmerged" "$A" "$A"

                # check if first label
                (( nmerged == 0 )) && firstA="$A"
                
                # global counter
		        nadded=$((nadded+1))

                # counter for nr of merged pairs
                nmerged=$((nmerged+1))
            fi

            # check if its time to submit jobs
            local nsrc=${#JOBARGS[@]}
            local dosubmit=0
            
            # if nr of JOBARGS is nbatchmax, we should probably submit now
            [[ $nsrc -lt $nbatchmax ]] || dosubmit=1

            # but if not all merged, we should wait and see
            [[ $nmerged -eq $maxnmerge ]] || dosubmit=0

            # reached last element - submit now!
            [[ $last -eq 0 ]] || dosubmit=1

            # ahh, but if there is nothing in the first place...
            [[ $nsrc -gt 0 ]] || dosubmit=0
            
            # submit if conditions are met
            if [[ $dosubmit -eq 1 ]]; then
                # make sure nr of srcs is non-zero
                # how can this condition be true??
                [[ $nsrc -gt 0 ]] || { nmerged=0; continue; }
                local labelRange=""
                if [[ ${#JOBARGS[@]} -eq 1 ]]; then
                    labelRange="${JOBARGS[0]}"
                elif [[ ${#JOBARGS[@]} -eq 2 ]]; then
                    labelRange="${JOBARGS[0]} ${JOBARGS[-1]}"
                else
                    labelRange="${JOBARGS[0]} ... ${JOBARGS[-1]}"
                fi
                [[ ${#labelRange} -gt 32 ]] && labelRange="${labelRange:0:32} ..."
                step_info_msg "starting ${#JOBARGS[@]} job(s), in total $nadded labels, [$labelRange]"

                # check if scheduler is ready
                check_scheduler_status $nsrc

                # nr of submissions
                nsub=$((nsub+1))

                # submit script
                dispatch_cmd "$script"
                ec=$?
                [[ $ec -eq 0 ]] || nfailed=$((nfailed+1))
                
                # clear job arrays
                reset_joblists

                # print timing
                local lblstr="$A"
                #####(( maxnmerge>1 )) && lblstr="${firstA} - $A"
                ###step_timing $TL0 $TL1 $nsrc "($lblstr , N(jobs)=$nmerged) rate (secs/job):"
                step_timing_dispatch $nsrc "($lblstr , N(jobs)=$nsrc) rate (secs/job):"

                # reset nmerged counter
                nmerged=0

                if (( SIGNAL_SOFT_ABORT==1 )); then
                    local sep="$(print_separator)"
                    step_info_msg $sep
                    step_info_msg "abort requested!"
                    step_info_msg "last label processed: $A"
                    step_info_msg $sep
                    exit 0
                fi

            fi
            # if maxmerged -> reset nmerged=0
            # -> this will force the add_jobargs to add a new element in the JOBARGS array
            [[ $nmerged -eq $maxnmerge ]] && nmerged=0
            
        done
        debug_msg "submitted $nsub job(s) with $nfailed failed job(s)"

        # BUG catching
        [[ ${#JOBARGS[@]} -eq 0 ]] || fatal_msg "nr of unprocessed sources is ${#JOBARGS[@]}"

        # check for bad results
        if [[ $nfailed -gt 0 ]]; then
            step_error_msg "failed submitting $nfailed job(s)"
            force_remake_new
            restore_jobspecs
            return 3
        fi

        # mark step as done
        if [[ $doinit == "1" ]]; then
            local T1=$(time_nsec)
            step_timing $T0 $T1 1 "total time (secs):"
            step_done
        fi
        
        restore_jobspecs
        return 0
    }

    #-----------------------------------------------
    # create the done filename for the current step
    #-----------------------------------------------
    get_done_file() {
        local step=$STEP_ID
        local flag=$STEP_ID_FLAG #TODO: POSSIBLY ADD IN THE FUTURE - NOT USED
        [[ $# -eq 1 ]] && { step=$1; flag=""; }
        [[ $# -eq 2 ]] && flag=$2
        [[ -z $flag ]] || step="${step}_$flag"
        echo "${DIR_PROJECT}/.done_${PL_TODO_NAME}_${step^^}"
    }

    #-----------------------------------------------
    # Check if a step done flag is valid
    #-----------------------------------------------
    is_valid_done() {
        [[ $# -eq 0 ]] && return 1
        local stepflag="$1"
        [[ -z $stepflag ]]  && return 1
        [[ $stepflag -lt 0 ]] && return 1
        [[ $stepflag -gt 3 ]] && return 1
        return 0
    }

    #-----------------------------------------------
    # return the done code in done file
    #  0 - remake none - done
    #  1 - remake newer or missing
    #  2 - remake always
    #  3 - remake only missing, ignore time stamps (DEFAULT)
    #-----------------------------------------------
    get_done_code() {
        local step=$STEP_ID
        local subid="" # POSSIBLY ADD IN THE FUTURE
        [[ $# -eq 1 ]] && { step=$1; subid=""; }
        [[ $# -eq 2 ]] && subid=$2

        local donefile=""
        donefile=$(get_done_file $step)
        [[ -f "$donefile" ]] || { echo $DC_MAKE_MISSING; return 0; }

        local donecode=0
        donecode=$(<"$donefile")
        # file empty - return 0
        [[ -n $donecode ]] || { echo $DC_DONE; return 0; }

        # return valid done code
        [[ $donecode -le $DC_MAKE_MISSING ]] && { echo $donecode; return 0; }

        # done code is invalid - return 0 - remake nothing
        echo $DC_DONE
        return 0
    }

    #-----------------------------------------------
    # get string representative of done code
    #-----------------------------------------------
    get_done_code_str() {
        [[ $# -eq 1 ]] || fatal_msg "missing argument to get_done_code_str()"
        local donecode=$1
        local str=""
        is_valid_done $donecode || error_msg "invalid done code given : $donecode"
        case $donecode in
            $DC_DONE)  str="DONE"
                ;;
            $DC_MAKE_NEW)  str="MAKE NEW"
                ;;
            $DC_MAKE_ALL)  str="MAKE ALL"
                ;;
            $DC_MAKE_MISSING)  str="MAKE MISSING"
                ;;
            *)  str="UNKNOWN"
                ;;
        esac
        echo $str
    }

    get_done_code_num() {
        [[ $# -eq 1 ]] || fatal_msg "missing argument to get_done_code_str()"
        local str="$1"
        local code=9
        case "$str" in
            "DONE")          code=$DC_DONE
                             ;;
            "MAKE NEW")      code=$DC_MAKE_NEW
                             ;;
            "MAKE ALL")      code=$DC_MAKE_ALL
                             ;;
            "MAKE MISSING")  code=$DC_MAKE_MISSING
                             ;;
            *)  code=$DC_UNKNOWN
                ;;
        esac
        echo $code
    }

    parse_done_code() {
        [[ $# -eq 1 ]] || fatal_msg "missing argument to get_done_code_str()"
        local inp="$1"
        [[ ${#inp} -gt 0 ]] || { echo $DC_UNKNOWN; return 0; }
        
        local code=$DC_UNKNOWN
        if is_integer $inp; then
            code=$inp
        else
            code=$(get_done_code_num "$inp")
        fi
        (( code < 0 )) && return 255
        (( code > 3 )) && return 255
        echo $code
        return 0
    }

    #-----------------------------------------------
    # update done file
    #   0 -> make none
    #   1 -> make new
    #   2 -> make all
    #   3 -> make missing
    #
    # anything else -> ignore
    #-----------------------------------------------
    update_done_file() {
        local flag="$1"
        [[ -n $flag ]] || return 0
        is_valid_done $flag || { error_msg "trying to set invalid step flag $flag"; return 1;}
        
        local donefile=""
        donefile=$(get_done_file $STEP_ID)
        echo $flag > "$donefile"
    }

    #-----------------------------------------------
    # force remake of missing files
    #-----------------------------------------------
    force_remake_missing() {
        update_done_file $DC_MAKE_MISSING
        return $?
    }

    #-----------------------------------------------
    # force remake by removing done file
    #-----------------------------------------------
    force_remake_all() {
        update_done_file $DC_MAKE_ALL
        return $?
    }

    #-----------------------------------------------
    # force remake of new by setting done file
    # content to 1
    #-----------------------------------------------
    force_remake_new() {
        update_done_file $DC_MAKE_NEW
        return $?
    }

    #-----------------------------------------------
    # force disabling of remake by setting
    # done file content to 0
    #-----------------------------------------------
    force_remake_none() {
        update_done_file $DC_DONE
        return $?
    }
    #-----------------------------------------------
    # dump done file status for each step (used??)
    #-----------------------------------------------
    dump_done_status() {
        local s
        local donecode
        local donestr
        local i
        for ((i=0; i<${#OP_ALL_STEPS[@]}; i++)); do
            s="${OP_ALL_STEPS[$i]}"
            set_stepid $s
            donecode=$(get_done_code)
            donestr=$(get_done_code_str $donecode)
            printf "%-20s%-15s\n" $STEP_ID $donestr
        done
    }

    #-----------------------------------------------
    # Set the STEP_DONE_CODE variable to the
    # contents of the done file for the given step.
    #
    # Return:
    #   0 : remake all (2) or partial (1)
    #   1 : do not remake (done code=0)
    #-----------------------------------------------
    load_done_flag() {
        # make sure needed info is there
        isset STEP_ID || { error_msg "STEP_ID is not set! $STEP_ID"; return 255; }
        isset PL_TODO || { error_msg "PL_TODO is not set!"; return 254; }

        # get done file name
        STEP_DONE_CODE=$(get_done_code)
        return 0
    }
    check_done_flag() {
        load_done_flag || return 255
        case $STEP_DONE_CODE in
            $DC_DONE)  # never remake
                return 1
                ;;
            $DC_MAKE_NEW)  # remake if target does not exist or target is older than source
                return 0
                ;;
            $DC_MAKE_ALL)  # always remake
                return 0
                ;;
            $DC_MAKE_MISSING)  # remake if missing
                return 0
                ;;
            *)  # bug capture
                fatal_msg "check_done_flag: unknown done code $donecode"
                ;;
        esac
        STEP_DONE_CODE=$DC_DONE
        return 252
    }


    #-----------------------------------------------
    # step msgs
    #-----------------------------------------------
    step_fatal_msg() {
        fatal_msg "$*" 1
    }
    step_error_msg() {
        error_msg "$*" 1
    }
    step_warn_msg() {
        warn_msg "$*" 1
    }
    step_info_msg() {
        info_msg "$*" 1
    }
    cstep_error_msg() {
        cnt_error_msg "$*" 1
    }
    cstep_warn_msg() {
        cnt_warn_msg "$*" 1
    }

    #-----------------------------------------------
    # Test if a target should be remade.
    # The arguments are:
    #  <target> [[ <src1>:<src2>:... ]
    #
    # Note that the different source files are ':' separated.
    # Do not remake if:
    #   * STEP_DONE_CODE==0
    #   * target is newer than source
    #
    # Remake if:
    #   * target does not exist
    #   * target is older than any of the sources
    #   * STEP_DONE_CODE==2
    #
    # Return values:
    #   0  : remake target
    #   >0 : do not remake
    #   1  : target is OK, no remake
    #   2  : never remake (DONE_CODE=0)
    # 255  : 
    #
    #-----------------------------------------------
    remake_target() {
        # if no remake check, return ok
        [[ $SKIP_REMAKE_CHECK -eq 0 ]] || { debug_msg "skipping remake check as SKIP_REMAKE_CHECK=1"; return 0; }

        #    # check that check_done_flag has been run
        #    isset STEP_DONE_CODE || step_fatal_msg "remake_target -> STEP_DONE_CODE is not set!"

        # make sure enough args are given
        [[ $# -lt 1 ]]  && step_fatal_msg "remake_target -> invalid nr of args $#"

        # if no step id set
        [[ -n $STEP_ID ]] || { STEP_DONE_CODE=$DC_UNKNOWN; return 0; }
        
        # done code = 2 -> always make
        [[ $STEP_DONE_CODE -eq $DC_MAKE_ALL ]] && { debug_msg "remake as DONE code is 2"; return 0; }

        # done code = 0 -> never make
        [[ $STEP_DONE_CODE -eq $DC_DONE ]] && { debug_msg "NO remake as DONE code is 0"; return 2; }

        # get args
        local target="$1"

        # done code = 3 -> make iff target does not exist
        if [[ $STEP_DONE_CODE -eq $DC_MAKE_MISSING ]]; then
            # check target file
            [[ -f "$target" ]] || { debug_msg "target does not exist $target"; return 0; }
            # target file exists, no remake
            debug_msg "target exists $target"
            return 1        
        fi
        #--------------------------------------------------------
        # at this point STEP_DONE_CODE == 1 or UNKNOWN (9)
        if [[ $STEP_DONE_CODE -ne $DC_MAKE_NEW ]] && [[ $STEP_DONE_CODE -ne $DC_UNKNOWN ]]; then
            step_fatal_msg "remake_target -> illegal STEP_DONE_CODE = $STEP_DONE_CODE"
        fi
        # remake if target does not exist or target is older than source

        # remake if target does not exist
        [[ -f "$target" ]] || return 0

        # target exists, check source files
        # first get a list of all sources
        local srcfiles=()
        if [[ $# -gt 1 ]]; then
            shift
            eval $(create_array srcfiles ':' "$*")
        fi
        [[ ${#srcfiles[@]} -eq 0 ]] && step_fatal_msg "remake_target -> no source files given"
        # source given - remake IF source is newer than target
        local found_newer=0
        local i=0
        for ((i=0; i<${#srcfiles[@]}; i++)); do
            src="${srcfiles[$i]}"
            local ec=0
            check_if_newer_source "$src" "$target"
            ec=$?
            # ec=0 -> found newer source
            # ec=1 -> target is more recent
            # ec=255 -> src does not exist
            # TODO: dont die here - count missing source files ...
            [[ $ec -le 1 ]] || fatal_msg "missing source file ($src)"
            
            # remake (source is newer)
            [[ $ec -eq 0 ]] && found_newer=1
        done
        [[ $ec -le 1 ]] || return $ec

        # found a src file newer than target -> remake
        [[ $found_newer -eq 1 ]] && return 0
        
        # target exists and is newer than source - do not remake (ec=1)
        return 1
    }

    #-----------------------------------------------
    # check existance of srcfiles
    # prints error msg and returns as soon as the
    # first missing file is found
    #-----------------------------------------------
    check_sources() {
        local i=0
        for ((i=0; i<${#SRCFILES[@]}; i++)); do
            local src="${SRCFILES[$i]}"
            [[ -f "$src" ]] || { step_error_msg "missing source file: $src"; return 1; }
        done
        return 0
    }

    #-----------------------------------------------
    # Mask out files from SRCFILES whose target is
    # uptodate.
    # This requires there is a 1-1 correspondence
    # between SRCFILES and TRGFILES.
    #-----------------------------------------------
    mask_ok_sources() {
        # return if empty list of files
        isempty SRCFILES && return 0
        isempty TRGFILES && return 0

        # remake nothing - clear SRCFILES
        if [[ $STEP_DONE_CODE -eq 0 ]]; then
            SRCFILES=()
            JOBARGS=()
            TRGFILES=()
            JOBLABELS=()
            JOBSUBLBL=()
            return 0
        fi

        # return now if DRY_RUN
        is_dryrun && return 0

        # check sources
        check_sources
        [[ $? -eq 0 ]] || return 1

        # remake everything - keep SRCFILES
        [[ $STEP_DONE_CODE -eq $DC_MAKE_ALL ]] && return 0

        # if target list differs from source in size, return full source
        [[ ${#SRCFILES[@]} -ne ${#TRGFILES[@]} ]] && return 0

        # loop over source and check that each target is more recent than the source
        local newsrc=""
        local newarg=""
        local newlbl=""
        local newspc=""
        local newtrg=""
        local ec=0
        local ind=0
        for ((ind=0; ind<${#SRCFILES[@]}; ind++)); do
            # remove each file that do not need remaking
            local src="${SRCFILES[$ind]}"
            local arg="${JOBARGS[$ind]}"
            local target="${TRGFILES[$ind]}"
            local joblabel="${JOBLABELS[$ind]}"
            local jobspecies="${JOBSUBLBL[$ind]}"

            # skip if no remake is required
            remake_target "$target" "$src"
            [[ $? -ge 1 ]] && continue

            # merge sources that are newer than target
            newsrc="${newsrc}${src}:"
            newarg="${newarg}${arg}:"
            newlbl="${newlbl}${joblabel}:"
            newspc="${newspc}${jobspecies}:"
            newtrg="${newtrg}${target}:"
        done

        # recreate arrays using ':' as delimiter
        SRCFILES=()
        TRGFILES=()
        JOBARGS=()
        JOBLABELS=()
        JOBSUBLBL=()

        eval $(create_array SRCFILES  ':' "$newsrc")
        eval $(create_array TRGFILES  ':' "$newtrg")
        eval $(create_array JOBARGS   ':' "$newarg")
        eval $(create_array JOBLABELS ':' "$newlbl")
        eval $(create_array JOBSUBLBL ':' "$newspc")

        return 0
    }

    #-----------------------------------------------
    # Run a command through scheduler using xargs.
    #
    # It will loop over all JOBARGS.
    # NOTE that there is no check of OP_STEP_NPARALLEL.
    #
    # Arguments:
    #   <label>  : step label
    #   <cmd>    : command to be run
    #
    #  Script must take three args in the order:
    #   <label>  : label to be used with the
    #              submit job script
    #   <spc>    : species label, JOBLOGNOSPEC if none
    #   <src>    : script arguments, comma separated
    #
    # The script will be run in a sub-shell, preceeded by
    # a call to tmp_setup_trap() -> no need to do
    # that in the script to be run!
    #-----------------------------------------------
    run_cmd_batch() {
        # check args and JOBARGS
        [[ $# -ge 1 ]] || return 0

        debug_msg "submitting job for $STEP_ID"

        # make sure sj_prepare_script does not read exported arrays/dicts from a file
        export EXPORT_DECL_FILE=""
        
        # get scheduler opts
        local opts=
        [[ $OP_SAVE_JOBLOG -ne 0 ]] || opts="-j"
        opts="$opts -n ${OP_STEP_NTHREADS[$STEP_ID]}"

        # get script to be run
        local script="$1"

        # process scheduler args common for all JOBARGS
        sj_process_args_common $opts "$script"
        
        # submit jobs
        local src=
        local ec=0
        local joblabels=0
        local jobsublbl=0
        [[ ${#JOBLABELS[@]} -eq ${#JOBARGS[@]} ]] && joblabels=1
        [[ ${#JOBSUBLBL[@]} -eq ${#JOBARGS[@]} ]] && jobsublbl=1

        # loop over all jobargs
        local i=0
        for (( i=0; i<${#JOBARGS[@]}; i++ )); do
            local args="${JOBARGS[$i]}"
            local lbl="$i"
            local spc="$JOBLOGNOSPEC"
            [[ $joblabels -eq 1 ]] && lbl=${JOBLABELS[$i]}
            [[ $jobsublbl -eq 1 ]] && spc=${JOBSUBLBL[$i]}
            

            # check for odd cases
            [[ -n $lbl  ]] || { warn_msg "got a job with empty label! args = $args"; continue; }
            [[ -n $args ]] || { warn_msg "got a job with empty args! lbl = $lbl"; continue; }
            
            # submit job with scheduler, batch or interactive
            #debug_msg "submitting job"
            #debug_msg "-> script  : $script"
            #debug_msg "-> lbl,spc : $lbl,$spc"
            #debug_msg "-> args    : $args"

            #------------------------------------
            # submit command
            #------------------------------------
            # sets SJ_SCRIPT="$script"
            #      SJ_JOBSTR="$lbl" (spaces replaced with '_')
            #      SJ_SUBLBL="$spc"
            #
            # $args becomes the arguments to SJ_SCRIPT
            # OLD: sj_submit $opts "$script" "$lbl" "$spc" "$args"
            sj_submit "$lbl" "$spc" "$args"
            ec=$?
            
            # quit if error is non-zero
            [[ $ec -eq 0 ]] || break
        done

        return $ec
    }

    #-----------------------------------------------
    # Run script locally.
    #
    # It uses the function sj_submit to run
    # locally. This ensures correct log files
    # and automatic tmp file deletion.
    #
    # The script will be run in a sub-shell, preceeded by
    # a call to tmp_setup_trap() -> no need to do
    # that in the script to be run!
    #-----------------------------------------------
    run_cmd_local() {
        # check args and JOBARGS
        [[ $# -ge 1 ]]    || return 0
        debug_msg "running local job for $STEP_ID"
        
        # get scheduler opts
        local opts=
        [[ $OP_SAVE_JOBLOG -ne 0 ]] || opts="-j"
        opts="$opts -n ${OP_STEP_NTHREADS[$STEP_ID]}"

        # get script to be run
        local script="$1"

        # how many jobs to submit in parallel
        local nparallel=$(get_step_nparallel)
        
        # set args for xargs
        local optP=""
        [[ $nparallel -gt 0 ]] && optP="-P $nparallel"
        
        # run xargs with command
        local src=
        local ec=0
        local joblabels=0
        local jobsublbl=0
        [[ ${#JOBLABELS[@]} -eq ${#JOBARGS[@]} ]] && joblabels=1
        [[ ${#JOBSUBLBL[@]} -eq ${#JOBARGS[@]} ]] && jobsublbl=1

        # make sure sj_prepare_script reads exported arrays/dicts from a file
        export EXPORT_DECL_FILE=""
        tmp_make EXPORT_DECL_FILE "expdecl"      || {
            error_msg "failed creating tmp file"
            return 255
        }
        export_decl_stdout > "$EXPORT_DECL_FILE" || {
            error_msg "failed creating export decl file"
            return 255
        }

        # process scheduler args common for all JOBARGS
        debug_msg "processing arguments (common)"
        sj_process_args_common $opts "$script"
        
        # loop over all src files and pipe job label + src into xargs
        # NOTE -> piping creates a sub-shell
        local i=0
        for (( i=0; i<${#JOBARGS[@]}; i++ )); do
            local args="${JOBARGS[$i]}"
            local lbl="$i"
            local spc="$JOBLOGNOSPEC"
            [[ $joblabels  -eq 1 ]] && lbl=${JOBLABELS[i]}
            [[ $jobsublbl  -eq 1 ]] && spc=${JOBSUBLBL[i]}
            # The args for sj_process_args() function.
            # What is run here for each xargs thread is:
            #
            #  sj_submit <job opts> SCRIPT JOBSTR SPECIES SCRIPT_ARGS
            #
            # where lbl==JOBSTR, spc=SPECIES and args==SCRIPT_ARGS
            echo "$lbl" "$spc" "$args"
        done | xargs $optP -I{} --no-run-if-empty bash -c "sj_submit \"{}\""
        #done | xargs $optP -I{} --no-run-if-empty bash -c "sj_submit $opts \"$script\" \"{}\""

        # get error code
        ec=$?

        if [[ $ec -ne 0 ]]; then
            error_msg "==============================="
            error_msg "ERROR IN SCRIPT: $script"
            error_msg "==============================="
        fi
        # remove the expdecl file
        tmp_delete $EXPORT_DECL_FILE
        return $ec
    }

    #-----------------------------------------------
    # Run a script.
    #
    # If on local machine:
    #
    # It will loop over all JOBARGS.
    # Nr of jobs to be run in parallel is given by
    # OP_STEP_NPARALLEL.
    #
    # Arguments:
    #   <label>  : step label
    #   <cmd>    : command to be run
    #
    #  Script must take three args in the order:
    #   <label>  : label to be used with the
    #              submit job script
    #   <spc>    : species label, JOBLOGNOSPEC if none
    #   <src>    : script arguments, comma separated
    #
    # The script will be run in a sub-shell, preceeded by
    # a call to tmp_setup_trap() -> no need to do
    # that in the script to be run!
    #
    # Returns:
    #  0      OK (or empty JOBARGS)
    #  >0     error in xargs (local) or in job submission (batch)
    #  255    unexpected error
    #-----------------------------------------------
    run_cmd() {
        # check args and JOBARGS
        [[ $# -ge 1 ]]    || return 0

        ##debug_msg "check JOB arrays"
        #isempty JOBARGS   && { debug_msg "JOBARGS is empty! ${JOBARGS[@]}"; return 0; }
        #isempty JOBLABELS && { debug_msg "JOBLABELS is empty!"; return 0; }
        #isempty JOBSUBLBL && { debug_msg "JOBSUBLBL is empty!"; return 0; }

        ## make sure all job lists are of the same size
        #debug_msg "check JOB array sizes"
        #check_step_joblists || return 255

        debug_msg "get runmode"
        local isbatch=0
        is_runmode_batch       && isbatch=1
        is_runmode_interactive && isbatch=1

        local ec=0
        if [[ $isbatch -eq 0 ]]; then
            # local - submit using xargs
            debug_msg "run local"
            run_cmd_local "$@"
            ec=$?
        else
            # check if scheduler is available
            debug_msg "run batch"
            check_scheduler_status ${#JOBARGS[@]}
            # submit jobs in a loop
            run_cmd_batch "$@"
            ec=$?
        fi
        verbose_info_msg "run_cmd ended with error code = $ec"
        return $ec
    }

    #-----------------------------------------------
    # check scheduler, run script and check for errors
    # 0    - all OK
    # 1    - jobs ran but at least one signalled an error
    # 255  - sj_wait failed (batch mode)
    #-----------------------------------------------
    export T_DISPATCH_JOBS=0
    export T_RUN_JOBS=0
    export DISPATCH_ERR=0
    dispatch_cmd() {
        local script=$1
        local njobs=${#JOBARGS[@]}
        
        T_DISPATCH_JOBS=0
        T_RUN_JOBS=0
        DISPATCH_ERR=0
        DISPATCH_NJOBS=$njobs
        
        (( njobs == 0 )) && {
            step_info_msg "no jobs dispatched - empty job args"
            return 0
        }
        
        # wait for scheduler
        check_scheduler_status $njobs

        step_info_msg "dispatching script $(reduce_name $script) with $njobs jobs"
        local T0=$(time_nsec)
        # run script - arguments in JOBARGS, logfile defined by JOBLABELS+JOBSUBLBL
        run_cmd "$script"
        ec=$?

        # time dispatching
        T_DISPATCH_JOBS=$(delta_time "$T0")
        
        # wait for job
        if is_runmode_batch; then
            sj_wait || { step_error_msg "aborting step since job id wait failed"; DISPATCH_ERR=255; return 255; }
        fi

        # time dispatching+running
        T_RUN_JOBS=$(delta_time "$T0")

        # check error (execution of alignments have finished)
        check_error $ec "error(s) found in batch $STEP_ID"
        ec=$?
        [[ $? -eq 0 ]] || {
            restore_jobspecs
            DISPATCH_ERR=$ec
            return $DISPATCH_ERR
        }
        return $ec
    }
    
    #-----------------------------------------------
    # parse error code from xargs
    #-----------------------------------------------
    parse_xargs_ec() {
        local ec=$1
        local ecstr=""
        if [[ $ec -eq 123 ]]; then
            ecstr="command exited with code 1-125"
        elif [[ $ec -eq 124 ]]; then
            ecstr="command exited with code 255"
        elif [[ $ec -eq 125 ]]; then
            ecstr="command is killed by a signal"
        elif [[ $ec -eq 126 ]]; then
            ecstr="command cannot be run"
        elif [[ $ec -eq 127 ]]; then
            ecstr="command is not found"
        elif [[ $ec -eq 1 ]]; then
            ecstr="unknown error"
        fi
        echo $ecstr
    }

    #-----------------------------------------------
    # check error logs from step
    #-----------------------------------------------
    OLD_check_error_logs() {
        local rval=0
        [[ $SKIP_CHECK_ERROR_LOGS -eq 0 ]] || { step_info_msg "skipping check of error logs"; return 0; }
        verbose_info_msg "checking error logs"
        
        # check arrays
        [[ ${#JOBLABELS[@]} -eq 0 ]] && { step_error_msg  "check_error_logs() does not see any labels to check!"; return 255; }
        [[ ${#JOBSUBLBL[@]} -eq 0 ]] && { step_error_msg  "check_error_logs() does not see any sub-labels to check!"; return 255; }

        # loop over all labels to check
        local i=0
        for ((i=0; i<${#JOBLABELS[@]}; i++)); do
            local s=${JOBLABELS[$i]}
            # get joblog base filename for job log files
            # <PL_TODO_NAME>_<STEP_ID>_<s>_*_<pid>
            # <pid> is the process id of this run
            # It will set SJ_R_JOBLOG_NAME
            sj_joblog_base "$PL_TODO_NAME" "$STEP_ID" "$s"
            
            # get joblog dir
            get_joblog_dir ${JOBSUBLBL[$i]}
            [[ -d "$R_JOBLOG_DIR/" ]] || fatal_msg "check_error_logs() dir not found or readable : $R_JOBLOG_DIR"

            # assume several matching job logs - should not happen
            # since process ID of shell is in the pattern
            local fout=()
            local ferr=()
            local file=""
            while IFS= read -r -d '' file; do
                fout+=("$file")
                ferr+=("${file//$SJ_JOBLOG_STDOUT_EXT/$SJ_JOBLOG_STDERR_EXT}")
            done < <(find "$R_JOBLOG_DIR/" -mindepth 1 -maxdepth 1 -name "${SJ_R_JOBLOG_NAME}_${SJ_JOBLOG_STDOUT_EXT}_*.log" -print0 )
            
            # check nr of logs found, skip if none found
            [[ ${#fout[@]} -eq 0 ]] && continue

            local foutsel=""
            local ferrsel=""
            # warn if more than one
            if [[ ${#fout[@]} -gt 1 ]]; then
                warn_msg "several job logs matching pattern (${SJ_R_JOBLOG_NAME}_*.log) - using the latest one"

                # get latest log matching
                local f=""
                local foutsel=""
                local indf=0
                for (( indf=0; indf<${#fout[@]}; indf++ )); do
                    f=${fout[$indf]}
                    foutsel=$(select_newest_file "$f" "$foutsel")
                    [[ "$foutsel" == "$f" ]] && ferrsel="${ferr[$indf]}"
                done
            else
                foutsel=${fout[0]}
                ferrsel=${ferr[0]}
            fi
            [[ -f "$foutsel" ]] || fatal_msg "did not find job output log - ${SJ_R_JOBLOG_NAME}"

            local ec=0
            if ! grep -q "^#JOBDONEOK" "$foutsel" >& /dev/null; then
                step_error_msg "--> job ended with error"
                step_error_msg "stdout file : $foutsel"
                step_error_msg "stderr file : $ferrsel"
                rval=1
                ec=255
            fi

            # check if stderr is non-empty, ignore lines with <VERBOSE> or <DEBUG>
            if ! grep -q -v -e '<VERBOSE>' -e '<DEBUG>' "$ferrsel"; then
                if [[ $ec -eq 0 ]]; then
                    # stderr is empty and job ended in OK state
     		        rm -f "$ferrsel"
                    [[ $OP_SAVE_JOBLOG -eq 0 ]] && rm -f "$foutsel"
                else
                    # stderr is empty but job ended in ERROR state
                    step_error_msg "$s -> in error state but empty error log - check $(trim_name $foutsel)"
                fi
		        continue
	        fi
	    
            # SPECIAL CHECK FOR slurmstep:
            if [[ $SCHEDULER_LABEL == SLURM ]]; then
                grep -q "slurmstepd: error:" "$ferrsel" && {
                    step_info_msg "skip - got a slurmstepd error - see $ferrsel"
                    continue
                }
            fi
            
            # we have a non-empty error log
            step_error_msg "check non-empty error log: $ferrsel"
            rval=1
        done
        
        return $rval
    }




    check_error_logs() {
        local rval=0
        [[ $SKIP_CHECK_ERROR_LOGS -eq 0 ]] || { step_info_msg "skipping check of error logs"; return 0; }
        
        # check arrays
        [[ ${#JOBLABELS[@]} -eq 0 ]] && { step_error_msg  "check_error_logs() does not see any labels to check!"; return 255; }
        [[ ${#JOBSUBLBL[@]} -eq 0 ]] && { step_error_msg  "check_error_logs() does not see any sub-labels to check!"; return 255; }

        # loop over all labels to check
        local i=0
        for ((i=0; i<${#JOBLABELS[@]}; i++)); do
            local s=${JOBLABELS[$i]}
            # get joblog base filename for job log files
            # <PL_TODO_NAME>_<STEP_ID>_<s>_*_<pid>
            # <pid> is the process id of this run
            # It will set SJ_R_JOBLOG_NAME
            sj_joblog_base "$PL_TODO_NAME" "$STEP_ID" "$s"
            
            # get joblog dir R_JOBLOG_DIR
            get_joblog_dir ${JOBSUBLBL[$i]}
            ####[[ -d "$R_JOBLOG_DIR/" ]] || fatal_msg "check_error_logs() dir not found or readable : $R_JOBLOG_DIR"

            # get log files
            local fout=()
            local ferr=()
            local file=""
            while IFS= read -r -d '' file; do
                fout+=("$file")
                ferr+=("${file//$SJ_JOBLOG_STDOUT_EXT/$SJ_JOBLOG_STDERR_EXT}")
            done < <(find "$R_JOBLOG_DIR/" -mindepth 1 -maxdepth 1 -name "${SJ_R_JOBLOG_NAME}_${SJ_JOBLOG_STDOUT_EXT}_*.log" -print0 )
            
            # check nr of logs found, skip if none found
            [[ ${#fout[@]} -eq 0 ]] && continue

            local foutsel=""
            local ferrsel=""
            # warn if more than one
            if [[ ${#fout[@]} -gt 1 ]]; then
                warn_msg "several job logs matching pattern (${SJ_R_JOBLOG_NAME}_*.log) - using the latest one"

                # get latest log matching
                local f=""
                local foutsel=""
                local indf=0
                for (( indf=0; indf<${#fout[@]}; indf++ )); do
                    f=${fout[$indf]}
                    foutsel=$(select_newest_file "$f" "$foutsel")
                    [[ "$foutsel" == "$f" ]] && ferrsel="${ferr[$indf]}"
                done
            else
                foutsel=${fout[0]}
                ferrsel=${ferr[0]}
            fi
            [[ -f "$foutsel" ]] || fatal_msg "did not find job output log - ${SJ_R_JOBLOG_NAME}"

            local ec=0
            if ! grep -q "^#JOBDONEOK" "$foutsel" >& /dev/null; then
                step_error_msg "--> job ended with error"
                step_error_msg "stdout file : $foutsel"
                step_error_msg "stderr file : $ferrsel"
                rval=1
                ec=255
            fi

            # if no error file exists - return OK
            [[ -f "$ferrsel" ]] || return 0

            # check if stderr is non-empty, ignore lines with <VERBOSE> or <DEBUG>
            if ! grep -q -v -e '<VERBOSE>' -e '<DEBUG>' "$ferrsel"; then
                if [[ $ec -eq 0 ]]; then
                    # stderr is empty and job ended in OK state
     		        rm -f "$ferrsel"
                    [[ $OP_SAVE_JOBLOG -eq 0 ]] && rm -f "$foutsel"
                else
                    # stderr is empty but job ended in ERROR state
                    step_error_msg "$s -> in error state but empty error log - check $(trim_name $foutsel)"
                fi
		        continue
	        fi
	    
            # SPECIAL CHECK FOR slurmstep:
            if [[ $SCHEDULER_LABEL == SLURM ]]; then
                grep -q "slurmstepd: error:" "$ferrsel" && {
                    step_info_msg "skip - got a slurmstepd error - see $ferrsel"
                    continue
                }
            fi
            
            # we have a non-empty error log
            step_error_msg "check non-empty error log: $ferrsel"
            rval=1
        done
        
        return $rval
    }

    #-----------------------------------------------
    # check error from step
    #-----------------------------------------------
    check_error() {
        step_info_msg "check for errors"
        local xargec=$1
        shift
        if [[ $xargec -gt 0 ]]; then
            local msg="$*"
            local emsg
            emsg="$(parse_xargs_ec $ec)"
            step_error_msg "$msg - xargs error ($ec): $emsg"
        fi

        # check error logs
        local ec=0
        check_error_logs
        ec=$?

        # if no error in xarg ec or from error logs, return ok
        [[ $xargec -eq 0 ]] && [[ $ec -eq 0 ]] && return 0

        if [[ $STEP_DONE_CODE -eq $DC_DONE ]]; then
            step_info_msg "error(s) found -> changing done code from <$(get_done_code_str $DC_DONE)> to <$(get_done_code_str $DC_MAKE_MISSING)>"
            force_remake_missing
        else
            step_warn_msg "error(s) found -> current done code is <$(get_done_code_str $STEP_DONE_CODE)>"
            step_warn_msg "- if step finished partially, make sure it is set to <$(get_done_code_str $DC_MAKE_MISSING)>"
            step_warn_msg "- note that if output files were corrupt, make sure to remove the bad files"
        fi

        return 1
    }

    #-----------------------------------------------
    #
    #-----------------------------------------------
    reset_joblists() {
        JOBLABELS=()
        JOBSUBLBL=()
        SRCFILES=()
        TRGFILES=()
        JOBARGS=()
    }

    #-----------------------------------------------
    # the PID file - removed todo label in name
    #-----------------------------------------------
    export R_PIDFILE=
    pid_file() {
        local ext="$PL_TODO_NAME"
        [[ -n $ext ]] && ext="_$ext"
        R_PIDFILE="${DIR_PROJECT}/.pid_pipeline$ext"
    }

    #-----------------------------------------------
    # get PID in pid file
    # if none found, return 0
    #-----------------------------------------------
    export R_PID=
    get_pid() {
        pid_file
        [[ -f "$R_PIDFILE" ]] && R_PID=$(<"$R_PIDFILE")
    }

    #-----------------------------------------------
    # set PID file
    #-----------------------------------------------
    set_pid_file() {
        PID_PIPELINE="$$"
        # get PID - this also sets R_PIDFILE
        get_pid
        if [[ $R_PID -gt 0 ]]; then
            if ! kill -0 $R_PID >& /dev/null; then
                # stale PID - remove it
                info_msg "removing stale PID file (pid=$R_PID)"
                rm -f "$R_PIDFILE"
            else
                PID_PIPELINE=""
                error_msg "another pipeline is already running at PID=$R_PID"
                return 255
            fi
        fi
        echo "$PID_PIPELINE" > "$R_PIDFILE"
        return 0
    }


    #-----------------------------------------------
    # check PID file
    #-----------------------------------------------
    chk_pid_file() {
        get_pid
        if [[ $R_PID -gt 0 ]]; then
            ps -p $R_PID >& /dev/null
            if [[ $? -eq 0 ]]; then
                error_msg "another pipeline is already running at PID=$R_PID"
                return 255
            fi
            warn_msg "removing a stale pid file with PID=$R_PID"
            rem_pid_file
        fi
        return 0
    }

    #-----------------------------------------------
    # remove PID file
    #-----------------------------------------------
    rem_pid_file() {
        pid_file
        [[ -f "$R_PIDFILE" ]] || return 0
        rm -f "$R_PIDFILE"
        return $?
    }

    #-----------------------------------------------
    # cleanup PID file
    # will check if stale PID file exists
    # and, if so, cleanup
    #-----------------------------------------------
    cleanup_pid_file() {
        get_pid
        [[ -f "$R_PIDFILE" ]] || return 0
        ps -p $R_PID >& /dev/null
        [[ $? -eq 0 ]] || rm -f "$R_PIDFILE"
        return $?
    }

    #-----------------------------------------------
    #
    #-----------------------------------------------
    split_step_data() {
        [[ $# -eq 1 ]] || return 0
        local step=$1
        [[ -n $step ]] || return 0
        step=${step^^} # make upper case
        tokenize ":" $step
    }

    #-----------------------------------------------
    #
    #-----------------------------------------------
    get_step_id() {
        local astep=()
        astep=( $(split_step_data "$@") )
        isempty astep && return 1
        echo ${astep[0]}
    }

    #-----------------------------------------------
    #
    #-----------------------------------------------
    get_step_val() {
        local astep=()
        astep=( $(split_step_data "$@") )
        isempty astep && return 1
        echo ${astep[1]}
    }

    #-----------------------------------------------
    # expand a step id
    # - ALL[:<flag>]
    # - <step>+[:<flag>]
    # - <step>-[:<flag>]
    #
    # They are added to
    # - STEP_ID_LIST -> expanded steps
    # - STEP_ID_FLAGS -> optional flags ( ':' means none )
    #-----------------------------------------------
    export STEP_ID_LIST=()
    export STEP_ID_LIST_FLAGS=()
    declare -A STEP_ID_MAP
    expand_step() {
        [[ $# -eq 1 ]] || fatal_msg "expand_step requires two args: <stepid>"
        local step="$1"
        STEP_ID_LIST=()
        STEP_ID_LIST_FLAGS=()
        [[ -n $step ]] || return 0
        step=${step^^} # make upper case

        local allsteps=""
        local delim=""
        local doall=0
        local dorest=0
        local dobefore=0
        local astep=()
        astep=( $(tokenize ":" $step) )
        ###v_tokenize astep ":" $step

        # split into id and flag
        isempty astep && return 1
        [[ ${#astep[@]} -le 2 ]] || return 1
        local stepid="${astep[0]}"
        
        # set flag if there or not ignored
        local stepflag=":"
        [[ ${#astep[@]} -eq 2 ]] && stepflag="${astep[1]}"

        # if stepid ends with a 'plus' it means that all subsequent steps should be added
        local lc=$((${#stepid}-1))
        local last="${stepid:$lc:1}"
        if [[ "$last" == "+" ]]; then
	        stepid=${stepid:0:$lc}
            dorest=1
        elif [[ "$last" == "-" ]]; then
	        stepid=${stepid:0:$lc}
            dobefore=1
        fi

        is_stepid_valid "$stepid" || return 2

        # if id is ALL -> use all
        [[ "${stepid}" == "ALL" ]] && doall=1 && dorest=$doall

        local foundid=0
        local state=0
        local s=""
        local i
        local add=0
        for ((i=0; i<${#OP_ALL_STEPS[@]}; i++)); do
            s="${OP_ALL_STEPS[$i]}"
            foundid=0
            [[ "$s" == "$stepid" ]] && { foundid=1; state=1; }
            add=0
            if (( doall==1 )); then
                # add all steps
                add=1
            elif (( dobefore==1 )); then
                # add all before given step
                add=$((1-state))
            elif (( dorest==1 )); then
                # add all after and including the step
                add=$state
            else
                # add the step
                add=$foundid
	        fi
            
            (( add==1 )) || continue
            STEP_ID_LIST+=( $s )
            STEP_ID_LIST_FLAGS+=( $stepflag )
        done

        return 0
    }

    #-----------------------------------------------
    # expand step list
    #-----------------------------------------------
    expand_step_list() {
        # reset lists
        STEP_ID_LIST=()
        STEP_ID_LIST_FLAGS=()
        
        [[ $# -eq 1 ]] || fatal_msg "expand_step_list requires two args: <steps>"
        local steps="$1"

        steps="${steps^^}"
        local steparr=()
        eval $(create_array steparr ',' "$steps")
        [[ ${#steparr[@]} -gt 0 ]] || return 2
        
        # check if query
        [[ "$steparr" == "." ]] && return 1

        local ec=0
        local s
        local i
        declare -A stepid_dict
        for s in "${OP_ALL_STEPS[@]}"; do
            stepid_dict[$s]=""
        done
        # loop over all steps and scan for ALL, <step>+ and <step>-
        for s in ${steparr[@]}; do
            expand_step "$s"
            ec=$?
            [[ $ec -eq 0 ]] || { step_error_msg "invalid step specifier <$s>"; return 255; }

            for (( i=0; i<${#STEP_ID_LIST[@]}; i++ )); do
                local si=${STEP_ID_LIST[$i]}   # step id
                local sf=${STEP_ID_LIST_FLAGS[$i]}  # new step id flag
                local sfd=${stepid_dict[$si]}  # flag previously set
                # empty flag in dict, set new
                [[ -z $sfd       ]] && { stepid_dict[$si]=$sf; continue; }
                # previous flag is none (':')
                [[ "$sfd" == ":" ]] && { stepid_dict[$si]=$sf; continue; }
                # set to latest given flag
                stepid_dict[$si]=$sf
            done
        done
        STEP_ID_LIST=()
        STEP_ID_LIST_FLAGS=()
        for s in "${OP_ALL_STEPS[@]}"; do
            local sf=${stepid_dict[$s]}
            [[ -n $sf ]] || continue;
            STEP_ID_LIST+=($s)
            STEP_ID_LIST_FLAGS+=($sf)
        done
        
        return 0
    }

    #-----------------------------------------------
    # expand step list
    #-----------------------------------------------
    expand_step_list_check() {
        [[ $# -eq 1 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local ec=0
        expand_step_list "$@"
        ec=$?
        
        (( ec>1 )) && { error_msg "invalid step list given - use '.' as step indicator to get a full list of valid steps"; return 255; }
        
        if (( ec==1 )); then
            print_steps_table 1
            echoerr ""
            echoerr "Examples"
            echoerr ""
            echoerr "  <STEP>+           all steps starting from <STEP>"
            echoerr "  <STEP>-           all steps before <STEP>"
            echoerr "  <STEP1>,<STEP2>   steps <STEP1> and <STEP2>"
            echoerr ""
            return 1
        fi

        return 0
    }

    #-----------------------------------------------
    # Update done file for given step(s).
    # The step string is assumed to be of the form:
    #   <step1>:<doneflag1>,<step2>:<doneflag2>,...
    # If no flag is given, the previous one is used.
    #-----------------------------------------------
    update_done_steps() {
        [[ $# -eq 1 ]] || return 0
        local defcode="$1"
        
        echo "--------------------------------------------------------"
        printf "%-20s%-20s%-20s\n" "Step id" "Old status" "New status"
        echo "--------------------------------------------------------"
        # loop over all steps and set done flag
        local stepid=""
        local newcode=""
        for (( i=0; i<${#STEP_ID_LIST[@]}; i++ )); do
            stepid=${STEP_ID_LIST[$i]}

            newcode=$defcode
            [[ -n $code ]] || newcode=${STEP_ID_LIST_FLAGS[$i]}
            [[ "$newcode" == ":" ]] && newcode=""

            [[ -n $newcode ]] || { warn_msg "no done flag given for step <$stepid>"; continue; }
            
            is_stepid_valid "$stepid" || { error_msg "skipping unknown label <$stepid>"; continue; }

            # set step id
            set_stepid $stepid

            # get current done status
            local oldcode=$(get_done_code)

            # update done flag if given
            [[ -z $newcode ]] || { update_done_file $newcode || return 255; }

            # print current status of done flag
            printf "%-20s%-20s%-20s\n" "$stepid" "$(get_done_code_str $oldcode)" "$(get_done_code_str $newcode)"
        done
    }

    #-----------------------------------------------
    # Update done file for given step.
    # The step string is assumed to be of the form:
    #   <step>:<doneflag>
    # If no doneflag is given, then the step is
    # ignored.
    #-----------------------------------------------
    update_done_from_step_list() {
        local stepstr="$1"
        local status="$2"
        local ec=0
        expand_step_list_check "$stepstr"
        ec=$?
        [[ $ec -eq 1 ]] && return 0
        [[ $ec -eq 0 ]] || return 255

        # override code parsed from optional status
        local code=""

        # check if a common status is given (-F)
        if [[ -n $status ]]; then
            # if status is not given then code==9
            code=$(parse_done_code "${status^^}")
            [[ $? -eq 0 ]] || { error_msg "invalid step status given \"$status\""; return 255; }
        fi

        update_done_steps "$code"
    }

    #-----------------------------------------------
    # check status of pipeline
    #-----------------------------------------------
    check_done_pipeline() {
        [[ $# -gt 0 ]] || return 0
        local ec=0
        expand_step_list_check "$1"
        ec=$?
        [[ $ec -eq 1 ]] && return 0
        [[ $ec -eq 0 ]] || return 255

        local fmt="%-20s%-20s\n"
        local retval=0

        # header
        echo "---------------------------------"
        printf "$fmt" "Step id" "Status"
        echo "---------------------------------"
        # loop over all steps and get done flag
        local s=""
        local code=0
        local cstr=""
        for s in "${STEP_ID_LIST[@]}"; do
            s=${s^^}
            set_stepid $s
            is_step_enabled || continue
            code=$(get_done_code)
            cstr=$(get_done_code_str $code)
            printf "$fmt" "$s" "$cstr"
            if [[ $code -ne 0 ]]; then
                # save the first non-zero code, in case of a list
                [[ $retval -eq 0 ]] && retval=$code
            fi
        done
        return $retval
    }



    #-----------------------------------------------
    # cleanup scratch
    #-----------------------------------------------
    cleanup_scratch() {
        [[ -d "$DIR_PROJECT/$BASE_SCRATCH" ]] || return 255

        # remove jobid files
        rm -f "$(get_jobid_file_templ)*"

        # remove submit files
        rm -f "$(sj_submit_tmp_file_templ)*"

        # remove any pending locks
        rm -Rf "$DIR_PROJECT/$BASE_SCRATCH/lock_*"
    }

    #-----------------------------------------------
    # cleanup empty log files
    #-----------------------------------------------
    clean_empty_logs() {
        # make sure joblog does not resolve to root
        local logdir=$(cd -P "$DIR_PROJECT/$BASE_JOBLOG" && pwd)
        [[ -d "$logdir" ]] || fatal_msg "DIR_PROJECT/$BASE_JOBLOG is not a readable directory!"
        [[ "$logdir" == "/" ]] && fatal_msg "DIR_PROJECT/$BASE_JOBLOG resolves to the root directory - not allowed!"
        
        # find all empty files *.log and remove them, ok for names with spaces
        find "$DIR_PROJECT/$BASE_JOBLOG/" -name "*.log" -empty -exec rm -f '{}' \;
    }

    #-----------------------------------------------
    # Check if any pending jobs are running
    # arg: <verbose flag>
    #      - 0 -> echo only nr of runningjobs found
    #      - 1 -> print a detailed warning msg
    #-----------------------------------------------
    check_pending_jobs() {
        local verbose=0
        [[ $# -eq 1 ]] && verbose=$1
        local nrun=0
        local s=
        for s in "${OP_ALL_STEPS[@]}"; do
            local nrunning=0
            nrunning=$(sj_running_jobs $s)
            # check for empty result - ignore if empty
            [[ -n $nrunning ]] || continue
            # print msg if nrunning>0
            if [[ $nrunning -gt 0 ]]; then
                nrun=$((nrun+1))
                [[ $verbose -eq 0 ]] || warn_msg "step $s has $nrunning running jobs"
            fi
        done
        [[ $verbose -ne 0 ]] || echo $nrun
        [[ $nrun -gt 0 ]] && return 1
        return 0
    }

    #-----------------------------------------------
    # produce a simple one-liner for the status
    #-----------------------------------------------
    pipeline_status() {
        local ec=0
        local step=""
        
        local isrun=0      # flag if pipeline is running
        local nrun=0       # if running, the nr of jobs queued/running
        local steprun=""   # if running, the step
        local stepdone=""  # last step completed
        local alldone=0    # flag if all is done

        # 1. check PID file
        debug_msg "check pid file"
        chk_pid_file >& /dev/null
        [[ $? -eq 0 ]] || isrun=1

        # 2. check running steps
        if [[ $isrun -ne 0 ]]; then
            debug_msg "pipeline is running"
            local nrun=0
            for step in "${OP_ALL_STEPS[@]}"; do
                debug_msg "test step $step"
                njobs=$(sj_get_njobs $step)
                # check for empty result - ignore if empty
                [[ -n $njobs ]] || continue
                [[ $njobs -gt 0 ]] || continue
                steprun=$step
                break
            done
            # step = running step, nrun = nr of jobs running
        fi

        # 3. check all done files
        local dc=0
        local ndone=0
        local ntot=0
        debug_msg "check done file"
        if (( isrun == 0 )); then
            for step in "${OP_ALL_STEPS[@]}"; do
                # skip disabled steps
                is_step_enabled $step || continue
                ntot=$((ntot+1))
                dc=$(get_done_code $step)
                [[ $dc -eq 0 ]] || continue
                stepdone=$step
                ndone=$((ndone+1))
            done
            [[ $ndone -eq $ntot ]] && alldone=1
        fi

        # compose final message
        local statusmsg="NOT STARTED"
        if   [[ $isrun -eq 1 ]]; then
            statusmsg="RUNNING:$steprun"
        elif [[ $alldone -eq 1 ]]; then
            statusmsg="DONE"
        elif [[ $ndone -gt 0 ]]; then
            statusmsg="DONE:$stepdone"
        fi
        echo "$statusmsg"

    }
    
    #-----------------------------------------------
    # Kill pipeline but not pending jobs
    #-----------------------------------------------
    kill_pipeline() {
        local killopt=$1
        get_pid
        if [[ $R_PID -gt 0 ]]; then
            ps -p $R_PID >& /dev/null
            if [[ $? -ne 0 ]]; then
                warn_msg "pid file found with invalid PID=$R_PID - removing file"
                rem_pid_file
                return 0
            fi
            local question="Immediately kill pipeline with PID = $R_PID ?"
            local statement="Aborting pipeline with PID $R_PID"
            local killsig="TERM"
            if [[ $killopt == "S" ]]; then
                question="Soft kill pipeline (when next batch of jobs are done) with PID = $R_PID ?"
                statement="Sending a soft-abort to pipeline with PID $R_PID"
                killsig="USR1"
            fi
            local answer="N"
            echo $question
            read -r -p "Proceed (y/N)? " answer
            answer=${answer^^}
            if [[ "$answer" == "Y" ]]; then
                echo $statement
                kill -$killsig $R_PID
                return 0
            fi
        else
            warn_msg "no pipeline is running for the requested todo label : $PL_TODO_NAME"
        fi
    }

    #-----------------------------------------------
    # Kill any pending jobs are running
    #-----------------------------------------------
    kill_pending_jobs() {
        [[ $# -eq 0 ]] && return

        local s=""
        local nrun=0

        # get step list
        local ec=0
        expand_step_list_check "$1"
        ec=$?
        [[ $ec -eq 1 ]] && return 0
        [[ $ec -eq 0 ]] || return 255

        # loop over steps
        local s
        local nrem
        local sjname
        for s in "${STEP_ID_LIST[@]}"; do
            nrem=$(sj_running_jobs $s)
            if [[ $nrem -gt 0 ]]; then
	            # ask, with No as default
	            ask_noyes "$s: cancel $nrem jobs"
                [[ $? -eq 1 ]] && { sj_cancel_jobs $s; sjname="$(sj_submit_tmp_file_templ $s)*"; rm -f "$sjname"; rm -f "$(get_jobid_file $s)"; }
            fi
        done
    }

    #-----------------------------------------------
    # A wrapper to exit.
    # If the given error code is empty or 0,
    # it will remove the EXIT trap and exit.
    # Otherwise an exit will pass by the trap_exit
    # function
    #-----------------------------------------------
    exitfunc() {
        __exit_value__=$?
        trap - EXIT
        exit $__exit_value__
    }

    print_exit_info() {
        local ec="$1";     shift
        local signal="$1"; shift
        local script="$1"; shift
        local funcnm="$1"; shift
        local lineno="$1"; shift
        local command="$*"
        echoerr " Bash PID     : $BASHPID" 
        echoerr " Signal       : $signal"
        echoerr " Step ID      : $STEP_ID"
        echoerr " Function     : $funcnm"
        echoerr " Line nr      : $lineno"
        echoerr " Command      : $command"
        echoerr " Error code   : $ec"
        echoerr " Tmp files    : ${TMP_FILES:0:32}"
    }

    print_extended_exit_info() {
        local npend=0
        npend=$(check_pending_jobs 0)
        echoerr " Pending jobs : $npend"
        echoerr ""
        echoerr " Any still running jobs on batch"
        echoerr " will continue running."
        echoerr ""
        echoerr " Note that scripts submitted but"
        echoerr " not yet running will need the"
        echoerr " contents in the .scratch dir."
        echoerr ""
        echoerr " Running jobs can be cancelled with"
        echoerr " > ./pipeline -k <STEPID>"
    }

    exit_cleanup() {
        # remove PID file
        rem_pid_file

        # clear all locks
        rm -Rf "$DIR_PROJECT/$BASE_SCRATCH/lock_${PL_TODO_NAME}_*"

        # cleanup tmp files
        tmp_cleanup_files
    }

    #-----------------------------------------------
    # like trap_exit() but does only
    # tmp file cleanup
    #-----------------------------------------------
    trap_exit_lite() {
        local ec=$?
        if [[ $ARGS_PARSE_ERROR == 0 ]]; then
            if [[ $ec -gt 0 ]]; then

                # non-zero error-code
                echoerr ""
	            echoerr "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
                echoerr " SCRIPT ENDED IN ERROR"
	            echoerr "----------------------------------------"
                print_exit_info "$ec" "$@"
                echoerr "----------------------------------------"
                echo
            fi
        fi
        
        tmp_cleanup_files
        return $ec
    }

    #-----------------------------------------------
    # This function is called when a function
    # calls exit
    #-----------------------------------------------
    trap_exit() {
        ec=$?
        # if 0 error code -> return silently
        if [[ $ec -gt 0 ]]; then

            # non-zero error-code
            echoerr ""
	        echoerr "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
            echoerr " PIPELINE ENDED IN ERROR"
            echoerr "----------------------------------------"
            print_exit_info "$ec" "$@"
            echoerr "----------------------------------------"
            echo
        fi
        
        exit_cleanup
        return $ec
    }

    #-----------------------------------------------
    # Function to be called when ctrl-c is pressed
    #
    # It will print in which step the interuption
    # occured.
    # The done file of the step will be set such
    # that a full check is enforced the next time.
    #-----------------------------------------------
    trap_ctrlc() {
        ec=$?
        echoerr ""
        echoerr "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
        echoerr " PIPELINE STOPPED "
        echoerr "----------------------------------------"
        print_exit_info "$?" "$@"
        print_extended_exit_info
        echoerr "----------------------------------------"
        echoerr ""
        
        # cleanup
        exit_cleanup

        # unset EXIT trap
        trap - EXIT

        # exit
        exit 255
    }

    #-----------------------------------------------
    # to be able to trap passing the signal
    #-----------------------------------------------
    trap_with_arg() {
        func="$1" ; shift
        for sig ; do
            trap "$func $sig \"\$0\" \"\${FUNCNAME-}\" \"\$LINENO\" \"\$BASH_COMMAND\""  "$sig"
        done
    }

    #-----------------------------------------------
    # If pipeline receives a SIGUSR1, it flags this
    # and the process can do a clean shutdown.
    #-----------------------------------------------
    export SIGNAL_SOFT_ABORT=0
    trap_sigusr1_fun() {
        info_msg "***** will soft abort the pipeline *****"
        SIGNAL_SOFT_ABORT=1
    }
    trap_soft_abort() {
        trap trap_sigusr1_fun SIGUSR1
    }
    #-----------------------------------------------
    # In order for the automatic tmp cleaning to work in a sub shell
    # this function must be called first.
    # Note that if called twice in the SAME shell, previously defined tmp files
    # will not be removed.
    #-----------------------------------------------
    tmp_setup_trap() {
        TMP_FILES=""
        trap_with_arg 'trap_exit_lite' EXIT SIGINT SIGQUIT SIGTERM
    }

    #-----------------------------------------------
    # load check pipeline code
    #-----------------------------------------------
    load_check_pipeline() {
        source "${DIR_ORTHOLOGER_BIN}/check_pipeline_common.sh"
        isfunc proj_load_check_pipeline && proj_load_check_pipeline
        return 0
    }

    #-----------------------------------------------
    # check pipeline
    #-----------------------------------------------
    check_full_pipeline() {
        local ec=0
        expand_step_list_check "$1"
        ec=$?
        [[ $ec -eq 1 ]] && return 0
        [[ $ec -eq 0 ]] || return 255


        local ec=0
        chk_pid_file 2>/dev/null
        ec=$?
        [[ $ec -eq 0 ]] && info_msg "pipeline is NOT running"
        [[ $ec -ne 0 ]] && info_msg "pipeline is running"

        check_pending_jobs 1 && info_msg "no pending jobs"

        # source main check code
        load_check_pipeline
        
        # run the actual check - function defined in check_pipeline_common.sh
        check_pipeline "${STEP_ID_LIST[@]}"

        return $?
    }
    #-----------------------------------------------
    # check setup
    #-----------------------------------------------
    check_full_setup() {
        # source main check code
        source "${DIR_ORTHOLOGER_BIN}/check_setup_common.sh"
        local err=0
        # run the check
        debug_msg "run check_setup"
        check_setup
        err=$?
        return $err
    }

    print_version() {
        printf "%-40s : %s\n" "orthologer"  "$OP_VERSION_SHORT ; $OP_VERSION_HASH ; $OP_VERSION_DATE"
        isfunc proj_print_version && proj_print_version
    }
    
    #-----------------------------------------------
    # Run the full pipline
    #-----------------------------------------------
    run_pipeline() {
        local ec=0
        local steps="$1"
        local seplen=132
        print_setup "$steps"

        #    if [[ ${#PL_TODO_LABELS[@]} -eq 0 ]]; then
        #        error_msg "TODO file contains no valid labels"
        #        error_msg "make sure file has one label per line, all lines starting with '#' are ignored"
        #        return 255
        #    fi
        
        # make step list
        expand_step_list_check "$steps"
        ec=$?
        [[ $ec -eq 1 ]] && return 0    # special step '.' was given
        [[ $ec -eq 0 ]] || return 255  # error!

        if ! is_dryrun; then

            # make sure no other pipeline is running
            set_pid_file || return 255

            # setup trap for signals
            # trap function     tmp     pidfile   lockfiles
            #------------------------------------------------
            # trap_ctrlc         X         X          X
            # trap_exit_lite     X
            #
            # by default, trap_exit_lite is used.
            # Since PID file is used here, change trap
            trap_with_arg trap_ctrlc EXIT SIGINT SIGQUIT SIGTERM

            # trap soft abort - triggered by SIGUSR1
            trap_soft_abort

            # check pending jobs according to jobid_xxx files
            check_pending_jobs 1
            ec=$?
            if [[ $ec -ne 0 ]]; then
                [[ $FORCE_RUN -eq 0 ]] && { error_msg "aborting run since pending/running jobs were found"; return 255; }
                warn_msg "Forced run even though pending/running jobs were found."
                warn_msg "Make sure files are not written twice."
            else
                # cleanup scratch dir
                cleanup_scratch
            fi
        fi

        # fill all steps
#        local -A do_step
#        local s=""
#        for s in "${OP_ALL_STEPS[@]}"; do
#            do_step[$s]=0
#        done
#        
#        # loop over given steps
#        for s in "${STEP_ID_LIST[@]}"; do
#            do_step[$s]=1
#        done

        # check scheduler
        uses_scheduler
        if [[ $? -eq 1 ]]; then
            local nparallel=$(get_njobmax_local)
            warn_msg "No job scheduler is used! All jobs will be run locally."
            warn_msg "All steps set to use BATCH will run locally with a maximum of $nparallel jobs in parallel."
        fi

        # warn if OP_SAVE_JOBOLOG=0, ie job logging is turned off
        if [[ $OP_SAVE_JOBLOG -eq 0 ]]; then
            echo
            warn_msg "General job logging is turned off - only logs where errors occured will be retained."
        else
            # warn if nr of species is over 100
            if [[ ${#PL_TODO_LABELS[@]} -ge 100 ]]; then
                echo
                print_separator '=' $seplen
                warn_msg "Running over many species (N=${#PL_TODO_LABELS[@]}) while full job logging is activated."
                warn_msg "Consider disabling full logging by setting OP_SAVE_JOBLOG=0 in the local orthologer_conf.sh ."
                warn_msg "Otherwise the JobLog directory will be swamped by many short files."
                warn_msg "Note that logging will always be retained in the cases when a job ends in error."
                print_separator '=' $seplen
                echo
            fi
        fi

        # skip check to remake -> essentially always make
        local sep=""
        sep="$(print_separator)"
        
        # warn if SKIP_REMAKE_CHECK
        if [[ "$SKIP_REMAKE_CHECK" == "1" ]]; then
            warn_msg "$sep"
            warn_msg "SKIP_REMAKE_CHECK==1 -> remake check disabled -> will ignore the remake status (unless step is marked as DONE)"
            warn_msg "$sep"
            echo
        fi

        # warn if SKIP_CHECK_ERROR_LOGS
        if [[ "$SKIP_CHECK_ERROR_LOGS" == "1" ]]; then
            warn_msg "$sep"
            warn_msg "SKIP_CHECK_ERROR_LOGS==1 -> normally a non-empty error log would trigger an error. This is now disabled."
            warn_msg "$sep"
            echo
        fi
        
        if (( USE_NOCHECKS == 1 )); then
            warn_msg "$sep"
            warn_msg "USE_NOCHECKS==1 -> Will use scripts where the checking has been reduced."
            warn_msg "Steps affected: MAKEBRH, MAKEINPAR and SETBRH/SETINPAR"
            warn_msg "$sep"
            echo
        fi

        print_separator "-" $seplen
        local datestr=""
        is_dryrun || datestr=$(printf "%-25s" "Time")
        printf "${datestr}%-25s%-8s %-8s %8s %8s %8s %8s %8s %8s    %-8s\n" "Label" "Enabled" "Runmode" "Nsrc" "Nmerge" "Nbatch" "Npar" "Nthread" "Status"
        print_separator "-" $seplen

        # check step sleep time
        local tsleep=$STEP_SLEEP
        if ! is_integer "$tsleep"; then
            tsleep=0
        fi

        local T0=$(time_nsec)
        # run requested steps
        local step=""
        local flag=""
        local i=0
        for (( i=0; i<${#STEP_ID_LIST[@]}; i++ )); do
            step=${STEP_ID_LIST[$i]}
            flag=${STEP_ID_LIST_FLAGS[$i]}
            # get step function
            local f=step_${step,,}
            isfunc "$f" || fatal_msg "function defining step $step is missing (step_${step,,})"

            # initialise step
            debug_msg "init $step, function $f and flag $flag"
            init_step "$step" "$flag"
            ec=$?
            (( ec == 255 )) && return $ec
            (( ec == 1   )) && continue # step disabled
            (( ec == 2   )) && continue # step flagged as done
            
            # run step
            debug_msg "runstep $step, function $f and flag $flag"
            "$f" "$flag" || return $?
            
            # if sleep after each step
            if is_runmode_batch; then
                (( tsleep==0 )) || {
                    step_info_msg "process sleeping for $STEP_SLEEP seconds"
                    is_dryrun || sleep $tsleep
                }
            fi
        done
        local T=$(delta_time $T0)
        info_msg "============================================"
        info_msg "Pipeline total time elapsed : $T seconds"
        info_msg "============================================"

        #unset do_step

        trap_with_arg 'trap_exit_lite' EXIT
        return 0
    }

    #-----------------------------------------------
    # Process and check implementation specific
    # arguments.
    #-----------------------------------------------
    define_args() {
        PROJ_OPTS=""
        PROJ_OPTS_WARGS=""
        # define project args
        isfunc proj_args || { warn_msg "no orthologer arguments defined!"; return 0; }

        proj_args || { warn_msg "proj_args returned non-zero error"; return 0; }

        # no hooks -> no args
        isempty PROJ_OPTS_HOOK && { warn_msg "PROJ_OPTS_HOOK is empty!? ${#PROJ_OPTS_HOOK[@]}"; return 0; }

        # hooks and arghelp should be of same size
        set +u
        [[ ${#PROJ_OPTS_HOOK[@]} -eq ${#PROJ_OPTS_ARGHELP[@]} ]] || fatal_msg "PROJ_OPTS_HOOK and PROJ_OPTS_ARGHELP differ in size!"
        set -u

        # get all keys in arghelp
        local allhelp="${!PROJ_OPTS_ARGHELP[*]}"

        # loop over all args and check
        local allargs=""
        local k=
        for k in "${!PROJ_OPTS_HOOK[@]}"; do
            inrange_set $k $allhelp || fatal_msg "key <$k> in PROJ_OPTS_HOOK is not defined in PROJ_OPTS_ARGHELP!"
            inrange_set $k $allargs && fatal_msg "duplicate definition of user option <$k> !"
            allargs="$allargs $k"
            if [[ ${#PROJ_OPTS_ARGHELP[$k]} -eq 0 ]]; then
                PROJ_OPTS="$PROJ_OPTS $k"
            else
                PROJ_OPTS_WARGS="$PROJ_OPTS_WARGS $k"
            fi
        done
        # now the PROJ_OPTS... arrays should be consistent
    }

    #-----------------------------------------------
    # Base pipeline parser
    # setup_environment() must not be called before
    #-----------------------------------------------
    base_pipeline() {
        clear_last_msg
        [[ $# -eq 0 ]] && { usage; return 0; }

        if [[ "${LOADED_COMMON_PIPELINE_SH+1}" == "" ]]; then
            echo "ERROR $FUNCNAME : BUG environment has not been loaded!!!"
            echo ""
            return 255
        fi
        
        debug_msg "base_pipeline() start"

        DIR_PROJECT_TODO=${DIR_PROJECT}/todo
        add_project_dir DIR_PROJECT_TODO
        
        # save PL_TODO from config as it may be overridden
        COMMON_TODO="$PL_TODO"
        COMMON_TODO_NAME=$(get_todo_name "$COMMON_TODO")

        #----------------------
        # parse arguments
        #----------------------
        local ec=0

        # flag that any action will be taken
        local do_anything=0

        # group 1 - general
        local do_clean_joblogs=0
        local do_todo_update=0
        local do_todo_parse=""
        local set_opt_t=0  # -t option was set
        local set_opt_T=0  # -T option was set
        
        # group 2 - check
        local do_check_pipeline=0
        local do_check_setup=0
        local do_check_projdirs=0
        local do_status=0
        
        # group 3 - steps
        local do_update_steps=0
        local do_update_status=0
        local do_clear_steps=0
        local do_time_steps=0
        local do_check_steps=0
        local do_dump_step_setup=0

        # group 4 creation...
        local do_create_dirs=0

        # group 5 - run/kill pipeline
        local do_run_pipeline=0
        local do_kill_pipeline=0
        local do_kill_pending=0

        local do_update_conf_sh=0

        # arguments
        local arg_run_pipeline=""
        local arg_update_steps=""
        local arg_update_status=""
        local arg_clear_steps=""
        local arg_time_steps=""
        local arg_check_steps=""
        local arg_kill_pending=""
        local arg_kill_pipeline=""
        local arg_check_pipeline=""
        
        # list of options without/with args
        local opts_noargs="h E v p V d O u x C Z N S o Q"
        local opts_withargs="c r f M t k K R T H e F X A L B J"

        # help for options with argument
        declare -A opts_args
        opts_args[H]="<step id, '.', '.variables', '.examples'>"
        opts_args[c]="<step id>"
        opts_args[A]="<step id>"
        opts_args[r]="<step id>"
        opts_args[e]="<step id>"
        opts_args[f]="<step id[:update code]>"
        opts_args[F]="<state>"
        opts_args[L]="<npar>"
        opts_args[B]="<nsub>"
        opts_args[J]="<nthr>"

        opts_args[M]="<nr of error messages before supressing>"
        opts_args[t]="<todo file(s)>"
        opts_args[k]="<step id>"
        opts_args[K]="<soft/hard>"
        opts_args[R]="<species>"
        opts_args[T]="<tree file>"
        opts_args[X]="<step id>"

        # add proj specific arguments defined by define_args()
        local key=
        for key in "${!PROJ_OPTS_ARGHELP[@]}"; do
            haskey opts_args $key && fatal_msg "duplicate definition of option <$key> !"
            opts_args[$key]="${PROJ_OPTS_ARGHELP[$key]}"
        done

        # make option string
        # NOTE that the first character in the opts_str
        # must be ':' in order for the missing arg functionality
        # to work.
        local opts_str=":"
        local o=
        for o in $opts_noargs; do
            opts_str=${opts_str}$o
        done
        for o in $opts_withargs; do
            opts_str=${opts_str}${o}:
        done
        # add implementation specific options and check that they are unique
        if [[ -n $PROJ_OPTS ]]; then
            for o in $PROJ_OPTS; do
                inrange_set $o $opts_noargs   && fatal_msg "duplicate definition of option <$o> !"
                inrange_set $o $opts_withargs && fatal_msg "duplicate definition of option <$o> !"
                opts_str=${opts_str}$o
            done
        fi
        if [[ -n $PROJ_OPTS_WARGS ]]; then
            for o in $PROJ_OPTS_WARGS; do
                inrange_set $o $opts_noargs   && fatal_msg "duplicate definition of option <$o> !"
                inrange_set $o $opts_withargs && fatal_msg "duplicate definition of option <$o> !"
                opts_str=${opts_str}${o}:
            done
        fi
        # parse options
        local OPT=

        # assoc array with hooks to run
        # PROJ_RUN[hook]=args
        declare -A PROJ_RUN=()

        debug_msg "base_pipeline() parse arguments"

        # option -P and -B
        # -P -> override
        #       1. OP_STEP_NPARALLEL if TYPE is not SINGLE
        #       2. OP_STEP_NTHREADS  if TYPE is SINGLE

        local force_nthr=0
        local force_npar=0
        local force_nbat=0
        local do_set_njobmax=0
        
        # reset certain variables to default
        #TREE_ENABLED=0
        PRETTY_PRINT=0
        PL_TODO_LABELS_REQ=""

        # parse error flag
        ARGS_PARSE_ERROR=0

        # better reset OPTIND in case getopts is called elsewhere
        OPTIND=1
        while getopts $opts_str OPT; do
            case "$OPT" in
                # general informative options
	            h)	usage
		            return 0
		            ;;
                H)  help_xxx $OPTARG
                    return 0
                    ;;
	            v)	print_version
		            return 0
		            ;;
                Q)  dump_proj_functions
                    return 0
                    ;;
                # some general variables
                p)  PRETTY_PRINT=1
                    reset_attributes
                    ;;
                V)  if (( VERBOSE==1 )); then
                        # a second -V option increases the verbosity
                        VERBOSE=1111
                    else
                        VERBOSE=1
                    fi
                    ;;
                d)  set_dryrun
                    ;;
                M)  MAX_MESSAGES=$OPTARG
                    ;;
                R)  PL_TODO_LABELS_REQ="${OPTARG}"
                    ;;
                # options controlling PL_TODO/1/2 file(s)
                t)  do_todo_update=1
                    do_todo_parse="$OPTARG"
                    set_opt_t=1
                    ;;
                T)  if [[ -n "$OPTARG" ]]; then
                        TREE_ENABLED=1
                    else
                        TREE_ENABLED=0
                    fi
                    TREE_INPUT_FILE="$OPTARG"
                    do_todo_update=1
                    set_opt_T=1
                    ;;
                # check pipeline/setup
                Z)  do_check_projdirs=1
                    do_anything=1
                    ;;
                e)  do_check_pipeline=1
                    do_anything=1
                    arg_check_pipeline="${OPTARG^^}"
                    ;;
                x)  do_check_setup=1
                    do_anything=1
                    ;;
                S)  do_status=1
                    do_anything=1
                    ;;
                # create project dirs/update common
                C)  do_create_dirs=1
                    do_anything=1
                    ;;
                o)  pl_dump_pipeline_sh > ${CMD_PIPELINE}
                    return 0
                    ;;
                u)  do_update_conf_sh=1
                    do_anything=1
                    ;;

                # update/check steps
                f)  do_update_steps=1
                    arg_update_steps="${OPTARG^^}"
                    do_anything=1
                    ;;
                F)  do_update_status=1
                    arg_update_status="${OPTARG^^}"
                    do_anything=1
                    ;;
                c)  do_clear_steps=1
                    arg_clear_steps="${OPTARG^^}"
                    do_anything=1
                    ;;
                A)  do_time_steps=1
                    arg_time_steps="${OPTARG^^}"
                    do_anything=1
                    ;;
                N)  CLEANUP_INTERACTIVE=0
                    ;;
                E)  do_clean_joblogs=1
                    do_anything=1
                    ;;
                X)  do_check_steps=1
                    arg_check_steps="${OPTARG^^}"
                    do_anything=1
                    ;;

                # dump/check pipeline setup
	            O)  do_dump_step_setup=1
                    do_anything=1
	                ;;
		
                # run/kill options
	            r)  do_run_pipeline=1
                    arg_run_pipeline="${OPTARG^^}"
                    do_check_projdirs=1
                    do_anything=1
	                ;;
                L)  force_npar="${OPTARG}"
                    do_set_njobmax=1
                    ;;
                B)  force_nbat="${OPTARG}"
                    do_set_njobmax=1
                    ;;
                J)  force_nthr="${OPTARG}"
                    do_set_njobmax=1
                    ;;
                k)  do_kill_pending=1
                    arg_kill_pending="${OPTARG}"
                    do_anything=1
                    ;;
                K)  do_kill_pipeline=1
                    arg_kill_pipeline="${OPTARG^^}"
                    arg_kill_pipeline=${arg_kill_pipeline:0:1}
                    do_anything=1
                    ;;

                # special/user options
	            \?)	# getopts issues an error message
	                error_msg "argument parse error - for help, do ./${CMD_PIPELINE} -h"
                    ARGS_PARSE_ERROR=2
		            return 2
		            ;;
                :)  # missing argument - note that the option is in OPTARG
                    local missingarg=${opts_args[$OPTARG]}
                    error_msg "missing argument -$OPTARG $missingarg"
                    ARGS_PARSE_ERROR=2
                    return 2
                    ;;
                *)  # implementation specific options
                    inrange_set "$OPT" "$PROJ_OPTS $PROJ_OPTS_WARGS"
                    [[ $? -eq 0 ]] || {
                        set -u
                        error_msg "unknown argument - for help, do ./${CMD_PIPELINE} -h"
                        ARGS_PARSE_ERROR=2
                        return 2
                    }
                    # get arg if it exists
                    set +u
                    local arg="$OPTARG"
                    set -u
                    # check if hook is a valid function
                    isfunc "${PROJ_OPTS_HOOK[$OPT]}" || { error_msg "no hook defined for option $OPT"; return 255; }

                    # save hook
                    PROJ_RUN[${PROJ_OPTS_HOOK[$OPT]}]="$arg"
                    do_anything=1
                    ;;
            esac
            
        done

        [[ $do_anything -eq 1 ]] || { info_msg "no command is given!"; return 0; }

        if (( VERBOSE==1 )); then
            info_msg "==> verbose message mode"
        fi
        if (( VERBOSE>1 )); then
            info_msg "==> debug message mode"
        fi

        
        # set NJOBMAX
        if (( do_set_njobmax>0 )); then
            set_njobmax "$force_npar" "$force_nbat"
            njobmax=$(get_njobmax_local)
            for step in "${OP_ALL_STEPS[@]}"; do
                nthr=${OP_STEP_NTHREADS[$step]}
                nmer=${OP_STEP_NMERGE[$step]}
                (( force_nthr > 0 )) && nthr=$force_nthr
                set_multiprocessing_stepid $step $njobmax $nthr $nmer
            done
        fi
        
        #--------------------------------------
        # check options for bad combinations
        #--------------------------------------
        if (( set_opt_t == 1 )) && (( set_opt_T == 1 )); then
            error_msg "both -t and -T cannot be set"
            ARGS_PARSE_ERROR=2
            return 2
        fi
        
        if [[ $do_run_pipeline -eq 1 ]]; then
            [[ $do_kill_pipeline -eq 0 ]] || { error_msg "both -r and -K are set"; return 255; }
            [[ $do_kill_pending  -eq 0 ]] || { error_msg "both -r and -k are set"; return 255; }
        fi

        if [[ $do_kill_pipeline -ne 0 ]]; then
            [[ $arg_kill_pipeline == "S" ]] || [[ $arg_kill_pipeline == "H" ]] || {
                error_msg "invalid argument given to option -K"
                error_msg "must be either s(oft) or h(ard) where only the first letter is significant"
                ARGS_PARSE_ERROR=2
                return 2
            }
        fi
        
        # update steps has highest priority
        if [[ $do_update_steps -eq 1 ]]; then
            [[ $do_check_steps   -eq 0 ]] || warn_msg "ignoring option -X since -f is set"
            [[ $do_clear_steps   -eq 0 ]] || warn_msg "ignoring option -c since -f is set"
            [[ $do_clean_joblogs -eq 0 ]] || warn_msg "ignoring option -E since -f is set"
            do_check_steps=0
            do_clear_steps=0
            do_time_steps=0
	        do_clean_joblogs=0
        fi

        if [[ $do_check_steps -eq 1 ]]; then
            [[ $do_clear_steps   -eq 0 ]] || warn_msg "ignoring option -c since -X is set"
            [[ $do_clean_joblogs -eq 0 ]] || warn_msg "ignoring option -E since -X is set"
            do_clear_steps=0
            do_time_steps=0
	        do_clean_joblogs=0
        fi

        if [[ $do_time_steps -eq 1 ]]; then
            do_clear_steps=0
	        do_clean_joblogs=0
        fi
        
        if (( do_update_status == 1 && do_update_steps == 0 )); then
            warn_msg "ignoring option -F as -f is not set"
            do_update_status=0
        fi
        
        #--------------------------------------
        # only allow MAX_MESSAGES in verbose mode
        #--------------------------------------
        ###    [[ $VERBOSE -eq 0 ]] && MAX_MESSAGES=0

        # parse todo spec if given
        if [[ -n $do_todo_parse ]]; then
            parse_todo_spec "$do_todo_parse" || {
                debug_msg "parse_todo_spec failed"
                return 255
            }
            update_todo 1 || {
                error_msg "failed updating todo file(s)"
                return 255
            }
            verbose_info_msg "TODO files set by options:"
            verbose_info_msg "- PL_TODO         $PL_TODO"
            verbose_info_msg "- PL_TODO1        $PL_TODO1"
            verbose_info_msg "- PL_TODO2        $PL_TODO2"
        else
            if [[ $set_opt_T -eq 1 ]]; then
                verbose_info_msg "tree mode - set PL_TODO accordingly"
                update_todo 1
            fi
        fi
        #--------------------------------------
        # User defined options
        #--------------------------------------
        debug_msg "check project args"
        if ! isempty PROJ_RUN; then
            local key=
            for key in "${!PROJ_RUN[@]}"; do
                local arg=${PROJ_RUN[$key]}
                ###is_dryrun && { info_msg "DRY RUN -> run \"$key $arg\""; continue; }
                $key $arg
            done
        fi
        unset PROJ_RUN
	
        #--------------------------------------
        # Setup environment
        #--------------------------------------
        debug_msg "base_pipeline() setup environment"

        # note that if pipeline() is called twice, the second one will not pass through setup_environment()
        # to force a setup_environment(), set SETUP_ENVIRONMENT_STATUS=255
        if (( $SETUP_ENVIRONMENT_STATUS != 0 )); then
            setup_environment || return 255
        fi
        # make sure todo is updated
        #[[ $do_todo_update -eq 0 ]] || update_todo 1
        #fi
        
        #--------------------------------------
        # cleanup PID (defined by PL_TODO_NAME)
        #--------------------------------------
        debug_msg "check pid file"
        cleanup_pid_file
	
        #--------------------------------------
        # Do things requiring TODO to be set
        #--------------------------------------
        ###[[ -f "$PL_TODO"   ]] || fatal_msg "no valid TODO file given (PL_TODO=$PL_TODO)"

        #--------------------------------------
        # Option -C -> create project dirs
        #--------------------------------------
        debug_msg "check option -C"
        if [[ $do_create_dirs -eq 1 ]]; then
            create_project_dirs || { error_msg "create project dirs failed!"; return 255; }
            return 0
        fi

        #--------------------------------------
        # Option -u
        #--------------------------------------
        debug_msg "check option -u"
        if [[ $do_update_conf_sh -eq 1 ]]; then
            local fcom=$(readlink orthologer_conf.sh)
            [[ -n $fcom ]] || fcom=orthologer_conf.sh
            local fbck=${fcom}.bck_pipeline
            local ftmp=
            tmp_make ftmp "newconf"
            pl_dump_conf_sh > $ftmp
            if ! diff -q <(grep -v "^# Date" "$ftmp") <(grep -v "^# Date" "$fcom") > /dev/null; then
                info_msg "updating orthologer_conf.sh (old in orthologer_conf.sh.bck_pipeline)"
                mv $fcom $fbck || return 255
                cp $ftmp $fcom || return 255
                chmod 755 $fcom
            else
                info_msg "config file is already up to date: orthologer_conf.sh"
            fi
            return 0
        fi

        #--------------------------------------
        # Option -Z -> check project directories
        # also run for any command that requires
        # all directories to be correct
        #--------------------------------------
        debug_msg "check option -Z"
        [[ $do_check_projdirs -eq 0 ]] || {
            verbose_info_msg "checking project directories"
            if check_project_dirs; then
                verbose_info_msg "project directories are OK"
            else
                error_msg "missing project directory - run with -V to see what and -C to create missing directories"
                return 255
            fi
        }

        #--------------------------------------
        # Option -e -> check pipeline
        #--------------------------------------
        debug_msg "check option -e"
        if [[ $do_check_pipeline -eq 1 ]]; then
            check_full_pipeline "$arg_check_pipeline"
            return 0
        fi
        #--------------------------------------
        # Option -S -> status
        #--------------------------------------
        debug_msg "check option -S"
        if [[ $do_status -eq 1 ]]; then
            debug_msg "run pipeline_status"
            pipeline_status
            return 0
        fi
        #--------------------------------------
        # Option -x -> check setup
        #--------------------------------------
        debug_msg "check option -x"
        if [[ $do_check_setup -eq 1 ]]; then
            check_full_setup
            return $?
        fi

        #--------------------------------------
        # Option -X
        #--------------------------------------
        debug_msg "check option -X"
        if [[ $do_check_steps -eq 1 ]]; then
            is_dryrun && { dryrun_info_msg "check status for step(s) ($arg_check_steps)"; return 0; }
            debug_msg "checking done flags for steps $arg_check_steps"
            check_done_pipeline $arg_check_steps 1
            return 0
        fi

        #--------------------------------------
        # Option -O
        #--------------------------------------
        if [[ $do_dump_step_setup -eq 1 ]]; then
            is_dryrun && { info_msg "DRY RUN -> print per step setup"; return 0; }
            debug_msg "print setup"
            dump_config_steps
            return $?
        fi

        #--------------------------------------
        # Option -E
        #--------------------------------------
        # cleanup empty logs
        if [[ $do_clean_joblogs -eq 1 ]]; then
            is_dryrun && { dryrun_info_msg "clean empty joblogs"; return 0; }
            info_msg "cleaning up all empty joblog files"

            chk_pid_file         || return 255
            check_pending_jobs 1 || { error_msg "aborting run since pending jobs were found"; return 255; }

	        clean_empty_logs
            return $?
        fi

        #--------------------------------------
        # Option -A
        #--------------------------------------
        # print time report per step
        if [[ $do_time_steps -eq 1 ]]; then
            time_steps $arg_time_steps
            return $?
        fi 
        #--------------------------------------
        # Option -c
        #--------------------------------------
        # clear given steps
        if [[ $do_clear_steps -eq 1 ]]; then
            is_dryrun && { info_msg "DRY RUN -> clear data for step(s) ($arg_clear_steps)"; return 0; }
            chk_pid_file         || return 255
            check_pending_jobs 1 || { error_msg "aborting run since pending jobs were found"; return 255; }

            debug_msg "clearing step flags $arg_clear_steps"
            cleanup_steps $arg_clear_steps
	        [[ $do_clean_joblogs -eq 0 ]] || clean_empty_logs
            return $?
        fi
        
        #--------------------------------------
        # Option -f and optionally -F
        #--------------------------------------
        # update done file per step
        if [[ $do_update_steps -eq 1 ]]; then
            is_dryrun && { dryrun_info_msg "update step flags ($arg_update_steps)"; return 0; }
            chk_pid_file || return 255
            debug_msg "updating done flags $arg_update_steps"
	        update_done_from_step_list "$arg_update_steps" "$arg_update_status"
            return $?
        fi

        #--------------------------------------
        # Option -r
        #--------------------------------------
        # run pipeline
        if [[ $do_run_pipeline -eq 1 ]]; then
            if is_dryrun; then
                chk_pid_file
            else
                chk_pid_file || return 255
                debug_msg "run pipeline $arg_run_pipeline"
            fi
            local ec=0
            run_pipeline $arg_run_pipeline
            ec=$?
            is_dryrun || rem_pid_file

            return $ec
        fi

        #--------------------------------------
        # Option -K
        #--------------------------------------
        if [[ $do_kill_pipeline -eq 1 ]]; then
            is_dryrun && { dryrun_info_msg "kill pipeline"; return 0; }
            debug_msg "kill pipeline"
            kill_pipeline $arg_kill_pipeline
            return $?
        fi

        #--------------------------------------
        # Option -k
        #--------------------------------------
        if [[ $do_kill_pending -eq 1 ]]; then
            is_dryrun && { dryrun_info_msg "kill pending jobs ($arg_kill_pending)"; return 0; }
            chk_pid_file || return 255
            debug_msg "kill pending jobs for $arg_kill_pending"
            kill_pending_jobs $arg_kill_pending
            return $?
        fi

        return 0
    }

    #-----------------------------------------------
    # setup default OP_xxx variables
    # called by the config file: orthologer_conf.sh
    #-----------------------------------------------
    op_step_setup() {
        [[ "${OP_STEP_NTHREADS[@]+1}" == "" ]] || { debug_msg "OP_STEP_XXX variables are not reset as they are already defined"; return 0; }
        if isfunc proj_step_setup; then
            debug_msg "calling proj_step_setup()"
            proj_step_setup
            return $?
        fi
        verbose_warn_msg "no proj_step_setup() found"
        return 0
    }

    check_input_specs() {
        if isfunc proj_check_input_specs; then
            proj_check_input_specs
            return $?
        fi
        return 0
    }
    #-----------------------------------------------
    # Setup environment
    #-----------------------------------------------
    setup_environment() {
        # only proceed if not done before
        [[ $SETUP_ENVIRONMENT_STATUS -gt 0 ]] || { debug_msg "setup_environment already called before - skipping!"; return 0; }
        
        local docheck=1
        if [[ $# -eq 1 ]]; then
            [[ $1 == nocheck ]] && docheck=0
        fi
        
        debug_msg "--> set OP_STEP_XXX variables"
        op_step_setup
        
        # configure basic env
        # -> set paths
        # -> set todo (update_todo)
        # -> setup scheduler
        debug_msg "--> configure basic environment"
        base_configure_env || { error_msg "base_configure_env() failed"; return 255; }

        debug_msg "--> setup environment PL_TODO=$PL_TODO"

        if (( docheck )); then
            debug_msg "--> check input specs"
            check_input_specs || return 255
        else
            debug_msg "--> skip checking input specs"
        fi
        

        # warn if PL_TODO is not set or not a file
        if [[ -z $PL_TODO ]]; then
            verbose_warn_msg "PL_TODO not set!"
        elif [[ ! -f "$PL_TODO" ]]; then
            debug_msg "PL_TODO is set but cannot find file ($PL_TODO)"
        fi
        
        # update all todo variables
        ###update_todo 1 || return 255 DONE ALREADY in base_configure_env()

        # configure implementation env
        # -> set paths
        # -> add plugins
        # -> define additional project dirs
        # -> whatever else needs setup
        debug_msg "--> configure project environment"
        isfunc proj_configure_env || { error_msg "no proj_configure_env() function defined!"; return 255; }
        proj_configure_env
        ec=$?
        [[ $ec -eq 0 ]] || fatal_msg "failed configuring environment with code $ec . Check local orthologer_conf.sh ."

        ### THIS FAILS WITH SOME ERRORS
        ###info_msg "verify setup"
        ###verify_setup || fatal_msg "verify setup failed"
        
        # find plugin scripts - depends on DIR_ORTHOLOGER_PATH
        debug_msg "--> find plugin scripts"
        find_plugin_scripts || { error_msg "find_plugin_scripts() failed!"; return 255; }
        
        # call op_step_setup - if the arrays are already defined it will do nothing
        op_step_setup

        # make sure OP_STEPS values are reasonable
        check_op_steps || return 255
        
        #---------------------------------------------------------
        # Checks - maybe move these, especially proj_func
        #---------------------------------------------------------
        # check all functions that are supposed to be defined by the implementation
        #TODO - really do this here???
        debug_msg "--> check project functions"
        check_proj_functions || { error_msg "missing proj_ functions!"; return 255; }

        # export all dictionaries/arrays using declare -p
        debug_msg "--> export arrays"
        export_decl_doall

        SETUP_ENVIRONMENT_STATUS=0
        debug_msg "--> setup environment DONE"
        return 0
    }

    #-----------------------------------------------
    # verify setup
    # NOTE: NOT USED FOR NOW
    #-----------------------------------------------
    verify_setup() {
        local rval=0
        local ec=0
        debug_msg "verify pipeline setup"

        # Project directories
        debug_msg "checking project directories"
        check_project_dirs || rval=1

        # Check OP_STEP variables
        debug_msg "checking op_steps arrays"
        check_op_steps || rval=1

        # plugins
        debug_msg "checking plugins"
        verify_plugins || rval=1

        # implementation specific verify_setup
        if isfunc proj_verify_setup; then
            debug_msg "project verify setup"
            proj_verify_setup || rval=1
        fi

        return $rval
    }

    #-----------------------------------------------
    # Parse input file argument.
    # 
    # It may be one of
    # 1 [<taxid>:]<fasta file> where the prefixed taxid is optional
    # 2 <file with list of fasta files>
    #
    # In case 1 the id asspciated with file is set to <taxid>
    # if it is given. Otherwise it's set to the basename of the
    # fasta file without extension.
    #
    # In case 2 each fasta file will get the id in the same manner
    # as in case 1.
    #
    # Results are written to three arrays:
    # - FASTA_FILES
    # - FASTA_TAXIDS
    # - FASTA_REMAP_IDS
    #
    # They are checked such that they are consistent.
    #
    #-----------------------------------------------
    FASTA_FILES=()
    FASTA_TAXIDS=()
    FASTA_REMAP_IDS=()
    parse_input_file() {
        FASTA_FILES=()
        FASTA_TAXIDS=()
        FASTA_REMAP_IDS=()
        
        # parse input file specification
        # it may be either
        # 1. <id>:<file>
        # 2. <file>
        local oldIFS=$IFS
        IFS=':'
        local inputArr=( $1 )
        IFS=$oldIFS

        # check what we got when split
        local inputTaxid=""
        local inputFile=""
        if [[ ${#inputArr[@]} -eq 1 ]]; then
            # just a file is given
            inputFile=$(expand_name "${inputArr[0]}")
        else
            # a taxid is prefixed
            inputTaxid="${inputArr[0]}"
            inputFile=$(expand_name "${inputArr[1]}")
        fi

        # get file type
        file_type "$inputFile"
        local inputFileType=$_FILE_TYPE
        if [[ "$inputFileType" == "NONE" ]]; then
            error_msg "input fasta file is not readable : $inputFile"
            return 255
        fi
        if [[ "$inputFileType" == "UNKNOWN" ]]; then
            error_msg "input file is readable but format is not recognized: $inputFile"
            return 255
        fi
        info_msg "input file is of type: $inputFileType"

        # flag indicating if index was prefixed by a '+'
        # fc=1 if yes
        local fc=0
        
        # inputTaxid with '+' removed
        local itx=""
        
        # save input
        if [[ "$inputFileType" == "FASTA" ]]; then
            FASTA_FILES=( "$inputFile" )
            fc=0
            if [[ -n $inputTaxid ]]; then
                itx="${inputTaxid##*+}"
                [[ "$itx" == "$inputTaxid" ]] && fc=1
                FASTA_TAXIDS=( "$itx" )
            else
                FASTA_TAXIDS=( $(file_basename "$inputFile") )
            fi
            FASTA_REMAP_IDS=( $fc )
            #taxid=$( echo $(basename "$fastaFile") | md5sum )
            #taxid=${taxid:0:16}

        elif [[ "$inputFileType" == "1COL" ]]; then
            fc=0
            [[ -n $inputTaxid ]] && {
                itx="${inputTaxid##*+}"
                [[ "$itx" == "$inputTaxid" ]] || fc=1                
                inputTaxid="${itx}_"
                info_msg "note all taxids will be prefixed with $inputTaxid"
            }
            # it potentially contains filenames
            FASTA_FILES=( $(grep -v '^#' "$inputFile") )
            for ffile in ${FASTA_FILES[@]}; do
                [[ -f $ffile ]] || {
                    error_msg "at least one of the provided input files in $inputFile does not exist"
                    error_msg "file: $ffile"
                    return 255
                }
                FASTA_TAXIDS+=( ${inputTaxid}$(file_basename "$ffile") )
                FASTA_REMAP_IDS+=( $fc )
            done
            
        elif [[ "$inputFileType" == "2COL" ]]; then
            local fc0=0
            [[ -n $inputTaxid ]] && {
                itx="${inputTaxid##*+}"
                [[ "$itx" == "$inputTaxid" ]] || fc0=1
                inputTaxid="${itx}_"
                info_msg "note all taxids will be prefixed with $inputTaxid"
            }
            
            # it potentially contains filenames
            array=( $(grep -v '^#' "$inputFile") )
            indarr=0
            for data in ${array[@]}; do
                if (( indarr == 0 )); then
                    indarr=1
                    itx="${data##*+}"
                    # set map if inputTaxid OR the label is prefixed by a '+'
                    fc=0
                    if (( fc0 == 1 )) || [[ ${#itx} -lt ${#data} ]]; then
                        fc=1
                    fi
                    FASTA_TAXIDS+=( "${inputTaxid}$data" )
                    FASTA_REMAP_IDS+=( $fc )
                else
                    [[ -f $data ]] || {
                        error_msg "at least one of the provided input files in $(trim_name $inputFile) does not exist"
                        error_msg "file: $data"
                        return 255
                    }
                    FASTA_FILES+=( $data )
                    indarr=0
                fi
            done
        fi
        
        # check that the arrays are consistent
        if [[ ${#FASTA_FILES[@]} -ne ${#FASTA_TAXIDS[@]} ]]; then
            error_msg "mismatch in input file: $inputFile"
            error_msg "make sure all non-comment lines are consistent"
            return 255
        fi

        [[ ${#FASTA_FILES[@]} -gt 0 ]] || {
            error_msg "no valid fasta file found : $inputFile"
            return 255
        }

        # make sure all taxids are unique
        [[ ${#FASTA_TAXIDS[@]} -gt 1 ]] || {
            local tmparr=()
            local oldIFS=$IFS
            IFS=$'\n' tmparr=($(sort -u <<<"${FASTA_TAXIDS[*]}"))
            IFS=$oldIFS
            if [[ ${#tmparr[@]} -lt ${#FASTA_TAXIDS[@]} ]]; then
                error_msg "input file contains multiple files with the same label"
                return 255
            fi
        }
            
               
        return 0
    }
    
    #-----------------------------------------------
    # run pipeline
    #-----------------------------------------------
    pipeline() {
        # Now ensure that pipeline tmp dir is compatible
        # TMP_DIR_BASE should always be non-empty.
        TMP_DIR_BASE=${TMP_DIR_BASE:=$TMPDIR}
        if [[ -n "$TMP_DIR_BASE" ]]; then
            [[ -d "$TMP_DIR_BASE" ]] || TMP_DIR_BASE=$TMPDIR
        else
            TMP_DIR_BASE=$TMPDIR
        fi
        [[ -z $TMP_DIR_BASE ]] && TMP_DIR_BASE=/tmp

        # setup trap for tmp files
        tmp_setup_trap

        # find python3
        set_python3

        # check for incompatabilities in user config
        # it will typically check for variables with changed names
        debug_msg "test_compatability"
        [[ -f $DIR_ORTHOLOGER/bin/test_compatability.sh ]] || fatal_msg "cannot find file <$DIR_ORTHOLOGER/bin/test_compatability.sh>"
        source $DIR_ORTHOLOGER/bin/test_compatability.sh
        debug_msg "DONE test_compatability"
        
        # check if manage mode
        if [[ $# -gt 0 ]]; then
            local first=$1
            if [[ "${first,,}" == "manage" ]]; then
                local fmng="$DIR_ORTHOLOGER/bin/manage_project.sh"
                [[ -x $fmng ]] || { error_msg "no manage script found ($fmng)"; return 255; }
                shift
                export CMD_MANAGE="$CMD_PIPELINE manage"
                info_msg "run manage: source $fmng"
                local nargs=$#
                local ec=0
                source $fmng "$@"
                ec=$?
                (( nargs==0 )) && return 0
                return $ec
            elif [[ "${first,,}" == "odbmap" ]]; then
                local fmap="$DIR_ORTHOLOGER/bin/orthomapper.sh"
                [[ -x $fmap ]] || {
                    error_msg "no orthomapper script found ($fmap)"
                    return 255
                }
                shift
                export CMD_ODBMAP=${CMD_ODBMAP:="$CMD_PIPELINE odbmap"}
                local nargs=$#
                local ec=0
                ###info_msg "run orthomapper: source $fmap \"$@\""
                source $fmap "$@"
                ec=$?
                (( nargs==0 )) && return 0
                return $ec
            fi
        fi

        # fill project specific argument hooks
        # used when processing the arguments
        debug_msg "define_args"
        define_args
        
        debug_msg "base_pipeline"
        base_pipeline "$@"

    }


    
    #===============================================================
    #                EXPORT VARIABLES AND FUNCTIONS
    #===============================================================
    #
    # Export functions
    #
    export -f set_python3
    export -f trim_name
    export -f step_status_msg
    export -f add_plugin
    export -f add_project_dir
    export -f check_proj_functions
    export -f dump_proj_functions
    export -f reload_usercommon
    export -f lock_dir_name
    export -f v_lock_dir_name
    export -f request_lock
    export -f remove_lock
    export -f is_locked
    export -f get_runmode
    export -f get_runmode_str
    export -f get_jobid_file_templ
    export -f v_get_jobid_file_templ
    export -f get_jobid_file
    export -f add_jobid_to_file
    export -f replace_jobid_file
    export -f update_jobid_file
    export -f clear_jobid_file
    export -f get_jobids
    export -f get_njobs
    export -f uses_scheduler
    export -f print_header
    export -f print_footer
    export -f get_footer_time
    export -f create_header
    export -f add_to_top
    export -f add_header
    export -f add_header_tmp
    export -f add_footer
    export -f add_header_footer
    export -f check_footer
    export -f pl_dump_base_conf_sh
    export -f pl_dump_conf_sh
    export -f pl_dump_pipeline_sh
    export -f script_name
    export -f find_script
    export -f get_todo_name
    export -f check_todo_file
    export -f get_todo_indexes
    ###export -f restore_todo_arrays
    export -f clear_todo_vars
    export -f set_todo_arrays
    export -f set_todo_vars
    export -f set_todo_vars_with_check
    ###export -f get_todo_lblind
    export -f update_todo
    export -f update_todo_pipeline
    export -f set_scheduler_njobmax

    export -f set_script_path
    export -f base_configure_env
    export -f check_requirement
    export -f check_requirement_list
    export -f check_directories
    export -f check_executables
    export -f get_run_script
    export -f get_plugin_script
    export -f find_plugin_scripts
    export -f parse_check_plugin_ec
    export -f check_plugin
    export -f get_joblog_dir
    export -f v_get_joblog_dir
    export -f create_joblog_dirs
    export -f create_project_dirs
    export -f check_project_dirs
    export -f has_project_dirs
    export -f usage_base
    export -f usage
    export -f extended_help
    export -f help_xxx
    export -f set_njobmax
    export -f set_multiprocessing_stepid
    
    export -f set_op_step_defaults

    export -f print_steps_table

    export -f dump_op_config_steps
    export -f dump_config_steps
    export -f get_step_depend
    export -f check_op_steps
    export -f verify_plugins
    export -f is_required_label
    export -f has_required_label

    export -f set_stepid
    export -f disable_step
    export -f enable_step
    export -f is_stepid_valid
    export -f is_runmode_batch
    export -f is_runmode_local
    export -f is_runmode_interactive
    export -f is_step_done
    export -f is_step_enabled
    export -f check_step_dependancy
    export -f get_njobmax_local
    export -f get_step_nparallel
    export -f get_step_nmerge
    export -f get_step_minjobs

    export -f check_step_reqtools
    export -f check_step_reqtools_all
    
    export -f check_scheduler_setup
    export -f check_scheduler_status
    export -f step_start
    export -f step_done
    export -f step_timing
    export -f step_timing_dt
    export -f step_timing_dispatch
    export -f init_step
    export -f save_jobspecs
    export -f restore_jobspecs
    export -f check_step_joblists
    export -f time_reset
    export -f time_set_minmax
    export -f time_set_from_footer
    export -f time_set_mean
    export -f time_step_per_todo_label
    export -f time_parse_log
    export -f time_steps
    export -f time_step_per_todo_label
    
    export -f add_jobargs
    export -f step_per_todo_label
    export -f step_per_label
    export -f get_done_file
    export -f get_done_code
    export -f get_done_code_str
    export -f is_valid_done
    export -f update_done_file
    export -f force_remake_missing
    export -f force_remake_all
    export -f force_remake_new
    export -f force_remake_none
    export -f dump_done_status
    export -f load_done_flag
    export -f check_done_flag
    export -f step_fatal_msg
    export -f step_error_msg
    export -f step_warn_msg
    export -f step_info_msg
    export -f cstep_error_msg
    export -f cstep_warn_msg
    export -f remake_target
    export -f check_sources
    export -f mask_ok_sources
    export -f run_cmd
    export -f dispatch_cmd
    export -f parse_xargs_ec
    export -f check_error_logs
    export -f check_error
    export -f reset_joblists
    export -f get_pid
    export -f pid_file
    export -f set_pid_file
    export -f chk_pid_file
    export -f rem_pid_file
    export -f cleanup_pid_file
    export -f split_step_data
    export -f get_step_id
    export -f get_step_val
    export -f expand_step
    export -f expand_step_list
    export -f expand_step_list_check
    export -f update_done_steps
    export -f update_done_from_step_list
    export -f check_done_pipeline
    export -f cleanup_scratch
    export -f clean_empty_logs
    export -f check_pending_jobs
    export -f kill_pipeline
    export -f kill_pending_jobs
    export -f trap_ctrlc
    export -f trap_exit
    export -f trap_exit_lite
    export -f trap_with_arg
    export -f trap_soft_abort
    export -f tmp_setup_trap
    export -f print_extended_exit_info
    export -f print_exit_info
    export -f load_check_pipeline
    export -f check_full_pipeline
    export -f check_full_setup
    export -f run_pipeline
    export -f define_args
    export -f base_pipeline
    export -f setup_environment
    export -f op_step_setup
    export -f verify_setup
    export -f pipeline
    export -f rmcmd

    export -f print_version
    
    export -f check_todo_special_labels

    export -f file_type

    export -f parse_input_file
    
    # flag that file is loaded
    LOADED_COMMON_PIPELINE_SH=0
    # the re-load protection
else
    echo "*************** SKIPPING RELOAD OF PIPELINE ****************************"
fi
