#!/usr/bin/env bash
#
# orthologer - Pipeline for deducing orthology from fasta files.
# Copyright (C) 2005-2023 E. Zdobnov lab: F. Tegenfeldt
# <fredrik.tegenfeldt@unige.ch> based on code by E. Kriventseva
#
# This file is part of orthologer.
#
# orthologer is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# orthologer is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with orthologer.  If not, see <http://www.gnu.org/licenses/>.
#
#--------------------------------------------------------------------------
# Check prerequisites (bash version)
source $DIR_ORTHOLOGER/bin/check_prerequisites.sh

# Check user id - possibly fail if root
source $DIR_ORTHOLOGER/bin/check_runas_root.sh
#--------------------------------------------------------------------------


# A long if statement that spans over the whole file - do not indent...
if [[ "${LOADED_COMMON_BASE_SH+1}" == "" ]]; then

    # define OS dependent resource functions (nr of threads)
    source $DIR_ORTHOLOGER/bin/get_os_resources.sh
    
    # Make sure system env TMPDIR points somewhere sensible
    # and if not, clear it!
    # An empty TMPDIR implies /tmp is used.
    TMPDIR=${TMPDIR:=""}
    [[ -d "$TMPDIR" ]] || TMPDIR=
    
    # for sorting and string comparison
    export _set_LANG=C
    LANG=${LANG:=$_set_LANG}
    
    # important if todo labels contain eg '_' chars
    export _saved_LANG=$LANG
    
    # check bash version for declare -g compatability
    export DECLARE_G='declare -g'
    if [[ ${BASH_VERSINFO[0]} -eq 4 && ${BASH_VERSINFO[1]} -lt 2 ]]; then
        DECLARE_G='declare'
    fi
    readonly DECLARE_G

    # global set
    # trap usage of unset variables
    set -u
###    set -e # if a function returns false, unchecked -> exit!
    
    # load error codes - TODO - not yet used
    [[ -f "${DIR_ORTHOLOGER}/bin/error_codes.sh" ]] && source "${DIR_ORTHOLOGER}/bin/error_codes.sh"

    # set original IFS
    export IFS_SAVE=$IFS
    export _ORG_IFS=$IFS

    # flags if common.sh has been loaded, if so it also gives the error code
    export LOADED_COMMON_BASE_SH=${LOADED_COMMON_BASE_SH=}

    # global verbose flag
    export VERBOSE=${VERBOSE:=0}
    export DRY_RUN=${DRY_RUN:=0}         # if set to 1 it will only print what it will do

    # user and machine string
    export SYS_UNAME="$(uname -a)"
    export SYS_WHOAMI="$(whoami 2>/dev/null)"
    [[ -n $SYS_WHOAMI ]] || SYS_WHOAMI="uid$(id -u)"

    #-----------------------------------------------
    # Global general variables
    #-----------------------------------------------
    # tmp dir location
    export TMP_DIR_BASE=${TMP_DIR_BASE:-/tmp}
    TMP_FILES=${TMP_FILES:-""} # NOTE THIS ONE SHOULD NOT BE EXPORTED

    # use as none character
    export NONE_VALUE="N"$'\000'"NE"
    readonly NONE_VALUE
    
    # attributes used in printing
    export PRETTY_PRINT=${PRETTY_PRINT:-1}

    export FACE_BOLD=""
    export FACE_NORMAL=""
    export ALWAYS_BOLD=""
    export ALWAYS_NORMAL=""
    export COLOR_RED=""
    export COLOR_GREEN=""
    export COLOR_ORANGE=""
    export COLOR_RESET=""
    export TERM_OK=0
    test -t 1 && TERM_OK=1

    # reset to normal print if no TERM is defined    
    reset_attributes() {
        if (( TERM_OK == 0 )); then
            PRETTY_PRINT=0
        fi

        if [[ $PRETTY_PRINT -eq 1 ]]; then
            FACE_BOLD=$(tput bold)
            FACE_NORMAL=$(tput sgr0)
            COLOR_RED=$(tput setaf 1)
            COLOR_GREEN=$(tput setaf 2)
            COLOR_ORANGE=$(tput setaf 3)
            COLOR_RESET=$(tput sgr0)
        else
            FACE_BOLD=""
            FACE_NORMAL=""
            COLOR_RED=""
            COLOR_GREEN=""
            COLOR_ORANGE=""
            COLOR_RESET=""
        fi
        if (( TERM_OK == 1 )); then
            ALWAYS_BOLD=$(tput bold)
            ALWAYS_NORMAL=$(tput sgr0)
        fi
    }
    # call it directly
    reset_attributes

    #-----------------------------------------------
    # a variant of getopts allowing for long opts
    # based on example in:
    # http://stackoverflow.com/questions/18414054/bash-getopts-reading-optarg-for-optional-flags
    #-----------------------------------------------
    ###declare -A OPTSLONG
    #getlongopts() {
    #    [[ "$1" =~ ^((-{1,2})([Hh]$|[Hh][Ee][Ll][Pp])|)$ ]] || return 1
    #
    #    while [[ $# -gt 0 ]]; do
    #        opt="$1"
    #        shift;
    #        current_arg="$1"
    #        if [[ "$current_arg" =~ ^-{1,2}.* ]]; then
    #            echo "WARNING: You may have left an argument blank. Double check your command." 
    #        fi
    #        case "$opt" in
    #            "-a"|"--apple"      ) APPLE="$1"; shift;;
    #            "-b"|"--banana"     ) BANANA="$1"; shift;;
    #            "-c"|"--cherry"     ) CHERRY="$1"; shift;;
    #            "-d"|"--dfruit"     ) DFRUIT="$1"; shift;;
    #            "-e"|"--eggplant"   ) EGGPLANT="$1"; shift;;
    #            "-f"|"--fig"        ) FIG="$1"; shift;;
    #            *                   ) echo "ERROR: Invalid option: \""$opt"\"" >&2
    #                exit 1;;
    #        esac
    #    done
    #}
    #-----------------------------------------------
    # check if pipe contains anything not in
    # A-Z,a-z,0-9 or _ and .
    #-----------------------------------------------
    good_label_chr() {
        LANG=C grep -P '[^\x41-\x5A,\x61-\x7A,\x30-\x39,\x2E,\x5F]'
    }

    #-----------------------------------------------
    # provide a generic way of setting a return variable
    # NOTE: if it is a variable that was exported
    # in a parent shell, the export will be reset
    # to be relative this shell - thus not set anymore in the parent
    #-----------------------------------------------
    set_var() {
        unset -v "$1" || return 255 #echoerr "invalid identifier: $1" >&2
        printf -v "$1" '%s' "$2"
    }
    
    #-----------------------------------------------
    # check if dry run
    #-----------------------------------------------
    set_dryrun() {
        DRY_RUN=1
    }
    unset_dryrun() {
        DRY_RUN=0
    }
    is_dryrun() {
        [[ $DRY_RUN -eq 0 ]] || DRY_RUN=1
        return $((1-DRY_RUN))
    }

    #-----------------------------------------------
    # return 0 if No and 1 if Yes
    #-----------------------------------------------
    ask_noyes() {
        [[ $# -ne 0 ]] || return 0
        local q="$*"
        local -u answer="N"
        read -r -p "$q (y/N)? " answer
        [[ "$answer" == "Y" ]] || return 0
        return 1
    }

    #-----------------------------------------------
    # return 1 if No and 0 if Yes
    #-----------------------------------------------
    ask_yesno() {
        [[ $# -ne 0 ]] || return 0
        local q="$*"
        local -u answer="Y"
        read -r -p "$q (Y/n)? " answer
        [[ "$answer" == "N" ]] || return 0
        return 1
    }

    #-----------------------------------------------
    # output YES or NO depending on arg.
    # If arg is anything but 1, it will echo YES.
    # Otherwise "NO".
    #-----------------------------------------------
    yesno_str() {
        [[ $# -eq 1 ]] || { echo "NO"; return 0; }
        [[ "$1" == "1" ]] && { echo "YES"; return 0; }
        echo "NO"
    }

    #-----------------------------------------------
    # replace character
    # args: <string> <substring> <replacement>
    #-----------------------------------------------
    replace_char() {
        [[ $# -eq 3 ]] || return
        local str="$1"
        echo "${str//$2/$3}"
    }

    #-----------------------------------------------
    # create an array from a delimited string
    # call it using
    #  eval $(create_array variable ':' "mydata:is:here now")
    # will create an array in variable with 3 elements
    #-----------------------------------------------
    create_array() {
        [[ $# -ge 3 ]] || return
        local var="$1"
        local delim="$2"
        shift
        shift
        local str="$*"
        echo "IFS='$delim' read -ra $var <<< \"$str\""
    }
    create_array_ws() {
        [[ $# -ge 2 ]] || return
        local var="$1"
        shift
        local str="$*"
        local delim=$'\x20'$'\x09'
        echo "IFS='$delim' read -ra $var <<< \"$str\""
    }
    create_array_eol() {
        [[ $# -ge 2 ]] || return
        local var="$1"
        shift
        local str="$*"
        local delim=$'\x0A'$'\x08'
        echo "IFS='$delim' read -ra $var <<< \"$str\""
    }
    #-----------------------------------------------
    # function to set various IFS
    #-----------------------------------------------
    set_ifs_default() {
        IFS=$_ORG_IFS
    }
    set_ifs() {
        [[ $# -gt 1 ]] || return 0
        IFS="$*"
    }
    set_ifs_ws() {
        IFS=$'\x20'$'\x09'
    }
    set_ifs_eol() {
        IFS=$'\x0A'$'\x08' #(echo -en "\n\b")
    }
    set_ifs_null() {
        IFS=$'\x00'
    }
    
    #-----------------------------------------------
    # tokenize a string
    # NOTE: dont use the set_ifs functions here!
    #-----------------------------------------------
    tokenize() {
        local ifs="$1"
        shift
	    echo "$@" | tr $ifs ' '
        #local v=$*
        #[[ ${#v} -gt 0 ]] || return 0
        #local saveifs=$IFS
        #IFS="$ifs"
        ## split up according to IFS -> do not do "$@"
        #local str=( $* )
        #IFS=$saveifs
        #
        #echo "${str[@]}"
    }
    # NOTE the following function requires 4.4 (?)
    v_tokenize() {
        local var="$1"
        local delim="$2"
        shift
        shift
        local data="$*"
        # declare -n is not valid in 4.2
        declare -n target="$var"
        target=()
        local saveifs=$IFS
        local x=""
        IFS="$delim"
        for x in $data; do
            target+=("$x")
        done
        IFS="$saveifs"
    }

    sort_csv() {
        local input="$1"
        local csv=""
        local A=( $(tokenize "," "$input") )
        readarray -t A < <(printf '%s\n' "${A[@]}" | sort -u)
        # merge sorted array separated by commas
        printf -v csv '%s,' "${A[@]}"
        # remove trailing comma
        echo ${csv:0:-1}
    }

    #-----------------------------------------------
    # check a line
    #-----------------------------------------------
    check_line() {
        [[ $# -ge 1 ]] || return 0
        local line="$*"
        [[ ${#line} -gt 0     ]] || return 1
        [[ ${line:0:1} == '#' ]] && return 1
        return 0
    }
    #-----------------------------------------------
    #
    #-----------------------------------------------
    get_lines() {
        [[ $# -eq 1 ]] || return 0
        local f="$1"
        [[ -f "$f" ]] || return 255 #TODO: add message
        local line=""
        while IFS= read -r line || [[ -n "$line" ]]; do
            check_line "$line" || continue
            echo "$line"
        done < "$f"
    }

    #-----------------------------------------------
    #
    #-----------------------------------------------
    get_lines_uc() {
        [[ $# -eq 1 ]] || return 0
        local f="$1"
        [[ -f "$f" ]] || return 255 #TODO: add message
        local line=""
        while IFS= read -r line || [[ -n "$line" ]]; do
            check_line "$line" || continue
            [[ "${line:0:1}" == '#' ]] && continue
            echo "${line^^}"
        done < "$f"
    }

    #-----------------------------------------------
    # remove many files
    #-----------------------------------------------
    rm_many() {
        local args="$*"
        while [[ true ]]; do
            ls -Uf "$args" | head -n 10000 | xargs rm -f --preserve_root 2>/dev/null
        done
    }

    #-----------------------------------------------
    # check if directory is writeable
    #-----------------------------------------------
    iswriteable() {
        [[ $# -eq 1 ]] || return 1
        local dir="$1"
        [[ -d "$dir" ]] || return 1

        local tstfile=""
        tstfile=$(mktemp "$dir/iswriteable_XXXXXX" 2>/dev/null)
        [[ -z $tstfile ]] && return 1
        rm -f "$tstfile"
        return 0
    }

    #-----------------------------------------------
    # cleanup a directory
    # args: <title> <force y/N> <file(s)>
    #-----------------------------------------------
    cleanup_dir() {
        [[ $# -eq 3 ]] || return 255

        local title="${1}"
        local -u force="$2"
        shift
        shift
        local filewc="${*}"

        echo "Cleanup : ${title}"

        if [[ "$force" != "Y" ]]; then
            local -u answer="N"
            read -r -p "Proceed (y/N)? " answer
            if [[ "$answer" != "Y" ]]; then
                echo "Aborting cleanup"
                exit 0
            fi
        fi
        echo "Cleaning ${title}"
        rm --preserve-root -f "$filewc"
    }

    #-----------------------------------------------
    # indirect assignment
    #-----------------------------------------------
    assign() {
        [[ $# -eq 2 ]] || return
        export -n "$1"="$2"
    }

    #-----------------------------------------------
    # evaluate a variable
    #-----------------------------------------------
    evalvar() {
        [[ $# -eq 1 ]] || return 0
        local var="$1"
        echo "${!var}"
    }

    OLDevalvar() {
        [[ $# -eq 1 ]] || return
        local var="$1"
        [[ -z $var ]] && return
        isset "$var" || { error_msg "failed evaluating variable $var"; return 255; }
        # dont use eval as it removes quotes in strings
        echo "${!var}"
    }


    #-----------------------------------------------
    # remove leading white space
    #-----------------------------------------------
    #-----------------------------------------------
    # remove leading white space
    #-----------------------------------------------
    rm_leading_ws() {
        [[ $# -gt 0 ]] || return 0
        local var="$*"
        echo "${var#"${var%%[![:space:]]*}"}"
    }

    rm_trailing_ws() {
        [[ $# -gt 0 ]] || return 0
        local var="$*"
        echo "${var%"${var##*[![:space:]]}"}"
    }

    trim() {
        [[ $# -gt 0 ]] || return 0
        local var="$*"
        var="${var#"${var%%[![:space:]]*}"}"
        var="${var%"${var##*[![:space:]]}"}"
        echo "$var"
    }
    
    #-----------------------------------------------
    # clear all keys in an associate array without 
    #-----------------------------------------------
    clear_assoc() {
        # NOTE this only works for bash >= 4.2 !!
        local var=$1
        unset $var
        $DECLARE_G -A $var
    }
    
    #-----------------------------------------------
    # check if given variable is set
    #-----------------------------------------------
    isset_assoc() {
        grep -q '^declare -A' <(declare -p "$1" 2>/dev/null)
    }

    isset_array() {
        grep -q '^declare -a' <(declare -p "$1" 2>/dev/null)
#        [[ $? -eq 0 ]] || {
#            echo "========================================="
#            echo "========================================="
#            echo "FAILED FOR $1"
#            declare -p "$1"
#            echo "========================================="
#            echo "========================================="
#        }
    }

    isset_var() {
        declare -p "$1" >& /dev/null && return 0
        return 1
        #grep -q '^declare '   <(declare -p "$1" 2>/dev/null) || return 1
        #grep -q '^declare -A' <(declare -p "$1" 2>/dev/null) && return 1
        #grep -q '^declare -a' <(declare -p "$1" 2>/dev/null) && return 1
        #grep -q '^declare -x' <(declare -p "$1" 2>/dev/null) && return 0
        #return 1
    }

    isset() {
        local var="$1"
        [[ -z "$var" ]]     && return 1 #{ echo "no varname given"; return 1; }
        declare -p "$1" 2>/dev/null | grep -q '^declare ' && return 0
        ####grep -q '^declare ' <(declare -p "$1" 2>/dev/null) && return 0
        return 1
    }

    haskey() {
        [[ $# -eq 2 ]] || return 255
        local var="$1"
        local key="$2"
        eval '[ ${'$var'[$key]+muahaha} ]'  
    }
    #-----------------------------------------------
    # check if variable is empty
    #-----------------------------------------------
    isempty() {
        [ $# -eq 1 ] || return 1
        # variable name
        local var="$1"
        local len=0
        # check array first as arrays will also test true as isset_var
        # but not the reverse
        if isset_array $var; then
            set +u
            eval len=$(echo \${#${var}[@]})
            set -u
        elif isset_assoc $var; then
            set +u
            eval len=$(echo \${#${var}[@]})
            set -u
        elif isset_var $var; then
            set +u
            value=$(evalvar $var)
            set -u
            len=${#value}
        fi
        (( len == 0 ))
    }

    #-----------------------------------------------
    # check if name is a bash function
    #-----------------------------------------------
    isfunc() {
        [[ $# -eq 1 ]] || return 1
        [[ $(declare -f $1) ]]
    }

    #-----------------------------------------------
    #
    #-----------------------------------------------
    check_function() {
        [[ $# -eq 2 ]] || return 0
        local f=$1
        local doerror=$2
        isfunc "$f" && return 0;
        (( doerror==1 )) && error_msg "$f is missing"
        (( doerror==0 )) && warn_msg "$f is missing"
        return 1
    }

    #-----------------------------------------------
    #
    #-----------------------------------------------
    check_function_status() {
        [[ $# -eq 2 ]] || return 0
        local f="$1"
        local doerror=$2
        isfunc "$f" && { ok_status_msg "$f"; return 0; }
        (( doerror==1 )) && xerror_status_msg "$f" "MISSING"
        (( doerror==0 )) && xwarn_status_msg  "$f" "MISSING"
        return 1
    }

    #-----------------------------------------------
    # check if file contains no data
    # it finds the first non-comment amd non-empty line
    # if none is found it returns 1, otherwise 0
    #-----------------------------------------------
    file_has_data() {
        local f=$1
        [[ -f "$f" ]] || return 1
        grep -q -m 1 -vE '(^#)|(^\s*[/*])|(^\s*$)' "$f"
    }
    
    #-----------------------------------------------
    # set a variable in a file
    # arguments: <file> <variable> <new value>
    # the variable definition in the file should be
    # of the bash format
    #  <variable>=<value>
    # note that there are no spaces around the '='
    #-----------------------------------------------
    file_setvar() {
        [[ $# -eq 3 ]] || return 255
        local fscript="$1"
        [[ -f $fscript ]] || return 255
        
        local var="$2"
        local value=""
        value=$(echo "$3" | sed 's/\//\\\//g') || return 255
        # ugly!!!!
        local ftmp=
        tmp_make ftmp file_setvar
        cp -f $fscript $ftmp || return 255
        cat $ftmp | sed 's/^\(\s*\)'"${var}="'.*/'"${var}=${value}"'/' > "$fscript"
        cat $ftmp | sed 's/^\(\s*\)'"export ${var}="'.*/'"export ${var}=${value}"'/' > "$fscript"
        #sed -i 's/^\(\s*\)'"${var}="'.*/'"${var}=${value}"'/' "$fscript"
        #sed -i 's/^\(\s*\)'"export ${var}="'.*/'"export ${var}=${value}"'/' "$fscript"
        return 0
    }

    file_add_setvar() {
        [[ $# -eq 3 ]] || return 255
        local fscript="$1"
        [[ -f $fscript ]] || return 255
        
        # if file is empty, add one line otherwise sed wont work
        [[ -s $fscript ]] || { echo "# new file" > "$fscript"; }
        local var="$2"
        local value=""
        value=$(echo "$3" | sed 's/\//\\\//g') || return 255


        if grep -q "^${var}=" "$fscript"; then
            local ftmp=
            tmp_make ftmp file_setvar
            cp -f $fscript $ftmp || return 255
            cat $ftmp | sed 's/^\(\s*\)'"${var}="'.*/'"${var}=${value}"'/' > "$fscript"
            cat $ftmp | sed 's/^\(\s*\)'"export ${var}="'.*/'"export ${var}=${value}"'/' > "$fscript"
            #sed -i 's/^\(\s*\)'"${var}="'.*/'"${var}=${value}"'/' "$fscript"
            #sed -i 's/^\(\s*\)'"export ${var}="'.*/'"export ${var}=${value}"'/' "$fscript"
        else
            # the initial match enforces a return code = 100 if no match is found
            ####sed -i '/^\(\s*\)'"${var}="'/!{q100}; s/^\(\s*\)'"${var}="'.*/'"${var}=${value}"'/' "$fscript" || {
            {
                echo "${var}=${value}      # added automatically"
            } >> "$fscript"
        fi
        return 0
    }
    #-----------------------------------------------
    # get a variable
    # it does NOT source the file (security...)
    #-----------------------------------------------
    file_getvar() {
        [[ $# -eq 2 ]] || return 255
        local fscript="$1"
        [[ -f $fscript ]] || return 255
        
        local var="$2"
        local value=""
        local line=""
        while read line; do
            value="$line"
            break # only read in the FIRST match
        done < <(grep '^'$var'=' "$fscript")
        if [[ -z $value ]]; then
            while read line; do
                value="$line"
                break # only read in the FIRST match
            done < <(grep '^export '$var'=' "$fscript")
        fi
        
        [[ -n $value ]] || return 1 # variable did not find any match
        local v=$(echo "$value" | cut -d '=' -f 2)
        sed -e 's/^"//' -e 's/"$//' <<<"$v"
        return 0
    }

    #-----------------------------------------------
    # check nr of arguments
    # arguments: <N args> <N requested> <label>
    #-----------------------------------------------
    check_nargs() {
        [[ $# -eq 3 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local name="$3"
        [[ $1 -eq $2 ]] || fatal_msg_wrong_nargs "$1" "$name"
    }

    #-----------------------------------------------
    # export dictionary as variables
    # they will be exported as
    #  <dictname>__<key>
    #-----------------------------------------------
    export_dict() {
        check_nargs $# 1 "$FUNCNAME"

        local varname="$1"
        isempty "$varname" && return 0
        local keys=""
        keys="$(eval echo \${!${varname}[@]})"
        local k=""
        for k in $keys; do
            local value="$(evalvar ${varname}[$k])"
            local newvar="${varname}__$k"
            assign "$newvar" "$value"
            export "$newvar"
        done
    }

    #-----------------------------------------------
    # export array as variables
    # they will be exported as
    #  <arrname>__<index>
    #-----------------------------------------------
    export_array() {
        check_nargs $# 1 "$FUNCNAME"

        local varname="$1"
        isempty "$varname" && return 0
        local n=""
        n=$(eval echo \${#${varname}[@]})
        local iv=
        for iv in $(seq 0 $((n-1))); do
            local value="$(evalvar ${varname}[$iv])"
            local newvar=${varname}__$iv
            assign "$newvar" "$value"
            export "$newvar"
        done
    }

    #-----------------------------------------------
    # export the declare statement for a given
    # variable
    # they will be exported as
    #  DECLARE__<varname>_
    # the variable contains the declare statement
    # required to create the varaible
    #-----------------------------------------------
    export_decl() {
        check_nargs $# 1 "$FUNCNAME"

        # get variable name
        local varname="$1"
        isempty "$varname" && return 0

        # get declaration string
        debug_msg "get declaration string for var $varname"
        local declstr="$(declare -p $varname 2>/dev/null)"
        [[ -n "$declstr" ]] || return 11
        
        # create declaration variable
        local newvar="DECLARE__${varname}"
        debug_msg "assign var DECLARE__$varname"
        assign "$newvar" "$declstr"
        export "$newvar"
        return 0
    }

    # string contains arrays and assoc arrays to be exported
    export EXPORT_DICTS_ARRAYS=${EXPORT_DICTS_ARRAYS=""}
    # contains only array env that should only be declared (not filled)
    export EXPORT_DICTS_ARRAYS_NOVAL=${EXPORT_DICTS_ARRAYS_NOVAL=""}
    # idem, only associative arrays
    export EXPORT_DICTS_ASSOC_NOVAL=${EXPORT_DICTS_ASSOC_NOVAL=""}
    export_decl_reset() {
        EXPORT_DICTS_ARRAYS=""
        EXPORT_DICTS_ARRAYS_NOVAL=""
        EXPORT_DICTS_ASSOC_NOVAL=""
    }
    
    export_decl_add() {
        [[ $# -eq 1 ]] || return 0
        local varname="$1"
        # check if it is in the string already
        # note all variables have a leading and trailing space
        [[ "$EXPORT_DICTS_ARRAYS"       =~ " $varname " ]] && return 0
        [[ "$EXPORT_DICTS_ARRAYS_NOVAL" =~ " $varname " ]] && return 0
        [[ "$EXPORT_DICTS_ASSOC_NOVAL"  =~ " $varname " ]] && return 0
        # note space around $varname...
        EXPORT_DICTS_ARRAYS="$EXPORT_DICTS_ARRAYS $varname "
    }

    export_decl_add_noval() {
        [[ $# -eq 1 ]] || return 0
        local varname="$1"
        # check if it is in the string already
        # note all variables have a leading and trailing space
        [[ "$EXPORT_DICTS_ARRAYS"       =~ " $varname " ]] && return 0
        [[ "$EXPORT_DICTS_ARRAYS_NOVAL" =~ " $varname " ]] && return 0
        [[ "$EXPORT_DICTS_ASSOC_NOVAL"  =~ " $varname " ]] && return 0
        # note space around $varname...
        EXPORT_DICTS_ARRAYS_NOVAL="$EXPORT_DICTS_ARRAYS_NOVAL $varname "
    }

    export_decl_add_assoc_noval() {
        [[ $# -eq 1 ]] || return 0
        local varname="$1"
        # check if it is in the string already
        # note all variables have a leading and trailing space
        [[ "$EXPORT_DICTS_ARRAYS"       =~ " $varname " ]] && return 0
        [[ "$EXPORT_DICTS_ARRAYS_NOVAL" =~ " $varname " ]] && return 0
        [[ "$EXPORT_DICTS_ASSOC_NOVAL"  =~ " $varname " ]] && return 0
        # note space around $varname...
        EXPORT_DICTS_ASSOC_NOVAL="$EXPORT_DICTS_ASSOC_NOVAL $varname "
    }

    #-----------------------------------------------
    # loops over all variables added and
    # creates the 'declare -p' strings
    #-----------------------------------------------
    export_decl_doall() {
        local var=""
        for var in $EXPORT_DICTS_ARRAYS; do
            debug_msg "exporting dict/array variable $var"
            export_decl $var
        done
    }

    export_decl_stdout() {
        local var=""
        for var in $EXPORT_DICTS_ARRAYS; do
            declare -p $var 2>/dev/null
        done
        for var in $EXPORT_DICTS_ARRAYS_NOVAL; do
            echo "declare -a $var"
        done
        for var in $EXPORT_DICTS_ASSOC_NOVAL; do
            echo "declare -A $var"
        done
    }
    
    #-----------------------------------------------
    # restore saved arrays/dicts
    #-----------------------------------------------
    _restore_decl() {
        local declvar="$1"
        set +u
        local cmd=${!declvar}
        set -u
        # make sure it is a proper declare string
        [[ "${cmd:0:7}" == "declare" ]] || return 10
        eval ${!declvar}
    }
    
    restore_decl() {
        check_nargs $# 1 "$FUNCNAME"

        local varname="$1"
        local declvar="DECLARE__${varname}"
        _restore_decl "$declvar"
    }
    #-----------------------------------------------
    # search for all DECLARE__ variables and restore
    #-----------------------------------------------
    restore_decl_doall() {
        local var=""
        for var in $(compgen -A variable "DECLARE__"); do
            _restore_decl $var
        done
        return 0
    }
    
    # get list of all DECLARE__XXX variables
    restore_decl_all_OLD() {
        local var=""
        local cmd=""
        for var in $(compgen -A variable DECLARE__); do
            set +u
            cmd=${!var}
            set -u
            # make sure it is a proper declare string
            [[ "${cmd:0:7}" == "declare" ]] || continue
            # skip if variable is already defined
            varname="${var:9}[@]"

            [[ "${!varname+1}" == "" ]] || { debug_msg "$FUNCNAME : array is already defined $varname -> skip";  continue; }
            # fatal if variable is already defined as general variable
            varname="${var:9}"
            [[ "${!varname+1}" == "" ]] || fatal_msg "$FUNCNAME : variable already defined: $varname"
            echo "${cmd};"
        done
    }
    restore_decl_all() {
        local var=""
        local cmd=""
        local varname=""
        local ndef=0
        while read var; do
            debug_msg "got variable $var"
            #for var in $(compgen -A variable | grep "^DECLARE__"); do
            set +u
            cmd=${!var}
            set -u
            debug_msg "got command ${cmd:0:50} ..."
            # make sure it is a proper declare string
            [[ "${cmd:0:7}" == "declare" ]] || continue
            # skip if variable is already defined
            varname="${var:9}[@]"

            [[ "${!varname+1}" == "" ]] || { debug_msg "$FUNCNAME : array is already defined $varname -> skip";  continue; }
            # fatal if variable is already defined as general variable
            varname="${var:9}"
            [[ "${!varname+1}" == "" ]] || fatal_msg "$FUNCNAME : variable already defined: $varname"
            debug_msg "ADDING command for var $var"
            echo "${cmd}"
            ndef=$((ndef+1))
        done < <(compgen -A variable "DECLARE__")
        (( ndef == 0 )) || return 0
        # nothing defined
        debug_msg "NOTHING defined"
        return 1
    }
    
    #-----------------------------------------------
    # check if variable is integer
    #-----------------------------------------------
    is_decimal() {
        # filter octal/hex/ord()
        local num=$(printf '%s' "$1" | sed "s/^0*\([1-9]\)/\1/; s/'/^/")
        test "$num" && printf '%f' "$num" >/dev/null 2>&1
    }
    is_integer() {
        # filter octal/hex/ord()
        local num=$(printf '%s' "$1" | sed "s/^0*\([1-9]\)/\1/; s/'/^/")
        test "$num" && printf '%d' "$num" >/dev/null 2>&1
    }
    is_float() {
        test "$1" && printf '%g' "$1" >/dev/null 2>&1
    }
    # 128 bit -> md5,...
    is_hash_128bit() {
        [[ "$1" =~ ^[a-f0-9]{32}$ ]] && return 0
        return 1
    }
    # 160 bit -> sha1,...
    is_hash_160bit() {
        [[ "$1" =~ ^[a-f0-9]{40}$ ]] && return 0
        return 1
    }
    is_hash() {
        is_hash_160bit "$1" && return 0
        is_hash_128bit "$1"
        return $?
    }
    #-----------------------------------------------
    # make temporary file under
    # /tmp/pipeline_<user>
    # NOTE that for the automatic tmp file cleaner
    # to work at a exit signal, there must be a trap
    # call *in the same shell*.
    # Assume function A() creates a tmp file and then
    # another function calls it:
    #   (A)
    # Then the tmp file created will NOT be deleted
    # as $() will start function A in a subshell where
    # the trap is not defined.
    #
    # If the function would be called just
    #   A
    # there is no subshell and hence it's deleted.
    #
    #-----------------------------------------------

    # base directory for tmp files
    tmp_base() {
	    local var="$1"
	    local tdir="${TMP_DIR_BASE}/pipeline_$SYS_WHOAMI"
        local "$var" && set_var "$var" "$tdir"
    }

    #-----------------------------------------------
    # Create tmp file.
    # Arguments:
    #  <var>    variable name that will contain the file name
    #  <label>  extra label
    #
    # File name is created as:
    #   $TMP_DIR_BASE/<label>_XXXXXXXX
    #
    # Note it cannot be called as in a sub-shell
    # like $(tmp_make) as that would exclude
    # the file created from the automatic delete.
    #-----------------------------------------------
    tmp_make() {
        [[ $# -ge 2 ]] || return 0
        local var="$1"
        local label="$2"
        local save=0
        [[ $# -eq 3 ]] && save=$3
        [[ "$save" == "0" ]] || save=1
        
        # create user temp dir
        local tmpdir=""
        tmpdir="${TMP_DIR_BASE}/pipeline_$SYS_WHOAMI"
        if [[ ! -d "$tmpdir" ]]; then
            mkdir -p "$tmpdir" >& /dev/null
            [[ $? -eq 0 ]] || fatal_msg "unable to create temporary directory $tmpdir"
        fi
        

        # ensure that time stamp of tmp dir is updated
        touch "$tmpdir" || return 255

        # create new tmp file
        local tf=
        local ec=1
        local ntries=0
        while (( ec > 0 )); do
            tf=$(mktemp "${tmpdir}/${label}_XXXXXXXX")
            ec=$?
            (( ec == 0 )) || tf=""
            (( ntries > 5 )) && break
            ntries=$((ntries+1))
            (( ec == 0 )) || sleep 1
        done

        [[ -f "$tf" ]] || fatal_msg "tmp_make: failed creating tmp file as ${tmpdir}/${label}_XXXXXXXX"
        
        # save it in list of tmp files
        # NOTE no check if it already exists...
        # if save=0 -> tmp file will be removed in tmp_cleanup_files
        # if save=1 -> tmp file will be kept
        [[ ${#TMP_FILES} -eq 0 ]] || TMP_FILES="${TMP_FILES},"
        TMP_FILES="${TMP_FILES}${tf}:${save}"
        
        debug_msg "created tmp file <$tf> with save code=$save"

        # return tmp file name through variable
        eval $var="$tf"

        return $ec
    }


    tmp_make_dir() {
        [[ $# -ge 2 ]] || return 0
        local var="$1"
        local label="$2"
        local save=0
        [[ $# -eq 3 ]] && save=$3
        [[ "$save" == "0" ]] || save=1
        
        # create user temp dir
        local _tmpdir=""
        _tmpdir="${TMP_DIR_BASE}/pipeline_$SYS_WHOAMI"
        if [[ ! -d "$_tmpdir" ]]; then
            mkdir -p "$_tmpdir" >& /dev/null
            [[ $? -eq 0 ]] || fatal_msg "unable to create temporary directory $_tmpdir"
        fi
        
        # ensure that time stamp of tmp dir is updated
        touch "$_tmpdir" || return 255

        # create new tmp file
        local tf=
        local ec=1
        local ntries=0
        while (( ec > 0 )); do
            tf=$(mktemp -d "${_tmpdir}/${label}_XXXXXXXX")
            ec=$?
            (( ec == 0 )) || tf=""
            (( ntries > 5 )) && break
            ntries=$((ntries+1))
            (( ec == 0 )) || sleep 1
        done
        [[ -d "$tf" ]] || fatal_msg "tmp_make: failed creating tmp dir as ${_tmpdir}/${label}_XXXXXXXX"
        
        # save it in list of tmp files
        # NOTE no check if it already exists...
        # if save=0 -> tmp file will be removed in tmp_cleanup_files
        # if save=1 -> tmp file will be kept
        [[ ${#TMP_FILES} -eq 0 ]] || TMP_FILES="${TMP_FILES},"
        TMP_FILES="${TMP_FILES}${tf}:${save}"
        
        debug_msg "created tmp file <$tf> with save code=$save"

        # return tmp file name through variable
        eval $var="$tf"

        return $ec
    }

    #-----------------------------------------------
    # delete tmp file
    #-----------------------------------------------
    tmp_delete() {
        [[ $# -eq 1 ]] || return 0
        [[ ${#TMP_FILES} -gt 0 ]] || return 0
        local f="$1"
        local files=()
        files=($(tokenize ',' "$TMP_FILES"))
        ###v_tokenize files ',' "$TMP_FILES"
        local save=0
        local x=""
        local newtmp=""
        for x in "${files[@]}"; do
            # remove anything after ':' in x
            if [[ "$f" == "${x%:*}" ]]; then
                save=${x##*:}
                [[ $save -ne 0 ]] || rm -f "$f"
            else
                [[ ${#newtmp} -eq 0 ]]  || newtmp="${newtmp},"
                newtmp="${newtmp}$x"
            fi
        done
        TMP_FILES="$newtmp"
        return 0
    }

    #-----------------------------------------------
    # mark tmp file to be kept (cleanup...)
    #-----------------------------------------------
    tmp_mark_keep() {
        [[ $# -eq 1 ]] || return 0
        [[ ${#TMP_FILES} -gt 0 ]] || return 0
        local f="$1"
        local files=()
        files=($(tokenize ',' "$TMP_FILES"))
        ###v_tokenize files ',' "$TMP_FILES"
        local x=""
        local newtmp=""
        for x in "${files[@]}"; do
            [[ ${#newtmp} -eq 0 ]]  || newtmp="${newtmp},"
            # remove anything after ':' in x
            if [[ "$f" == "${x%:*}" ]]; then
                newtmp="${newtmp}${f}:1"
            else
                newtmp="${newtmp}$x"
            fi
        done
        TMP_FILES="$newtmp"
        return 0
    }

    #-----------------------------------------------
    # mark tmp file to be deleted (cleanup...)
    #-----------------------------------------------
    tmp_mark_delete() {
        [[ $# -eq 1 ]] || return 0
        [[ ${#TMP_FILES} -gt 0 ]] || return 0
        local f="$1"
        local files=()
        files=($(tokenize ',' "$TMP_FILES"))
        ###v_tokenize files ',' "$TMP_FILES"
        local x=""
        local newtmp=""
        for x in "${files[@]}"; do
            [[ ${#newtmp} -eq 0 ]]  || newtmp="${newtmp},"
            # remove anything after ':' in x
            if [[ "$f" == "${x%:*}" ]]; then
                newtmp="${newtmp}${f}:0"
            else
                newtmp="${newtmp}$x"
            fi
        done
        TMP_FILES="$newtmp"
        return 0
    }

    #-----------------------------------------------
    # remove all files in TMP_FILES
    #-----------------------------------------------
    tmp_cleanup_files() {
        local tmpstr="$TMP_FILES"
        [[ $# -eq 1 ]] && tmpstr="$1"
        [[ ${#tmpstr} -gt 0 ]] || return 0
        verbose_info_msg "cleanup tmp files: $tmpstr"
        local f=""
        local files=()
        files=($(tokenize ',' "$tmpstr"))
        ###v_tokenize files ',' "$tmpstr"
        local x=""
        for x in "${files[@]}"; do
            debug_msg "checking tmp file $x"
            # remove anything after ':' in x
            f="${x%:*}"
            local save=${x##*:}
            [[ $save -ne 0 ]] || {
                [[ -f $f ]] && { debug_msg "removing tmp file $f"; rm -f "$f"; }
                [[ -d $f ]] && {
                    local ftmp=$(cd $f && pwd)
                    local usrdir=$(cd "${TMP_DIR_BASE}/pipeline_$SYS_WHOAMI" && pwd)
                    local fbase=${ftmp:0:${#usrdir}}
                    if [[ "$fbase" == "$usrdir" ]]; then
                        debug_msg "removing tmp dir $f"
                        rm -Rf "$f"
                    else
                        error_msg "corrupt entry in TMP_FILES: $f"
                        error_msg "will not remove"
                    fi
                }
                continue
            }
            debug_msg "keeping tmp file $f"
        done
        TMP_FILES=""
    }

    #-----------------------------------------------
    # clear TMP_FILES without removing any file
    #-----------------------------------------------
    tmp_clear_files() {
        TMP_FILES=""
    }

    #-----------------------------------------------
    # add to TMP_FILES variable
    #-----------------------------------------------
    tmp_add_file() {
        [[ $# -eq 1 ]] || return 0
        local f="$1"
        [[ ${#TMP_FILES} -eq 0 ]] || TMP_FILES="${TMP_FILES},"
        debug_msg "adding tmp file $f"
        TMP_FILES="${TMP_FILES}${f}:0"
    }

    #-----------------------------------------------
    # create a dir, print msg if DRY_RUN
    #-----------------------------------------------
    create_dir() {
        [[ $# -eq 1 ]] || return 0
        local dname="$1"
        is_dryrun && { dryrun_info_msg "create directory <$dname>"; return 0; }
        mkdir -p "$dname" || { error_msg "unable to create directory $dname"; return 255; }
        verbose_info_msg "created directory <$dname>"
        return 0
    }

    create_link() {
        [[ $# -eq 2 ]] || return 0
        local src="$1"
        local trg="$2"
        is_dryrun && { dryrun_info_msg "create soft link <$src -> $trg>"; return 0; }
        ln -sf "$src" "$trg" || { error_msg "unable to create soft link <$src -> $trg>"; return 255; }
        verbose_info_msg "created soft link <$trg>"
        return 0
    }

    #-----------------------------------------------
    # expand dir or file name
    #-----------------------------------------------
    expand_name() {
        [[ $# -lt 1 ]] && return 255
        local name="$*"
        local expname="$name"
        local edir=""
        local fdir=""
        if [[ -d "$name" ]]; then
            edir=$(cd "$name" && pwd)
            expname="$edir"
        elif [[ -f "$name" ]]; then
            fdir=$(dirname "$name")
            edir=$(cd "$fdir" && pwd)
            expname="${edir}"/$(basename "$name")
        else
	        # file does not exist
	        fdir=$(dirname "$name")
            if [[ -d $fdir ]]; then
	            expname=$(cd "$fdir" && pwd)/$(basename "$name")
            else
                expname="$name"
            fi
        fi
        echo "$expname"
    }

    reduce_name() {
        local name="$1"
        local remove="$DIR_ORTHOLOGER"
        [[ $# -eq 2 ]] && remove="$2"
        local r=${name//$remove/}
        [[ "$r" == "$name" ]] && { echo $r; return 0; }
        [[ ${r:0:1} == "/" ]] && { echo ${r:1}; return 0; }
        echo $r
        
    }
    #-----------------------------------------------
    # get file extension
    #-----------------------------------------------
    file_extension() {
        [[ $# -eq 1 ]] || return 1
        local name=""
        name=$(basename "$1")
        echo "${name##*.}"
    }

    #-----------------------------------------------
    # file base name excluding extension
    #-----------------------------------------------
    file_basename() {
        [[ $# -eq 1 ]] || return 1
        local name=""
        name=$(basename "$1")
        echo "${name%.*}"
    }

    #-----------------------------------------------
    # set extension
    #-----------------------------------------------
    set_extension() {
        [[ $# -eq 2 ]] || return 1
        local orgname="$1"
        local newext="$2"
        local fbase=""
        fbase=$(file_basename "$orgname")
        local fdir=""
        fdir=$(dirname "$orgname")
        echo "$fdir/${fbase}.${newext}"
    }

    #-----------------------------------------------
    # check if two files are the same
    # it does not follow links...
    #-----------------------------------------------
    are_same_files() {
        [[ $# -ne 2 ]] && return 4
        local fA="$1"
        local fB="$2"
        [[ -f "$fA" ]] || return 1
        [[ -f "$fB" ]] || return 2
        local eA=""
        local eB=""
        eA=$(expand_name "$fA")
        eB=$(expand_name "$fB")
        [[ "$eA" == "$eB" ]] && return 0
        return 3
    }

    #-----------------------------------------------
    # check if two files are the same
    # it does not follow links...
    #-----------------------------------------------
    are_same_dirs() {
        [[ $# -ne 2 ]] && return 4
        local fA=$(expand_name "$1")
        local fB=$(expand_name "$2")
        local dirA="$fA"
        local dirB="$fB"
        [[ -f "$fA" ]] && dirA=$(dirname "$fA")
        [[ -f "$fB" ]] && dirB=$(dirname "$fB")
        [[ -d "$dirA" ]] || return 1
        [[ -d "$dirB" ]] || return 2
        [[ "$dirA" == "$dirB" ]] && return 0
        return 3
    }

    #-----------------------------------------------
    # find the newest file
    # code grabbed from:
    # http://stackoverflow.com/questions/5885934/bash-function-to-find-newest-file-matching-pattern
    #-----------------------------------------------
    find_newest_file() {
        [[ $# -ge 1 ]] || return 0
        # get pattern
        local -r glob_pattern=$*

        # To avoid printing garbage if no files match the pattern, set
        # 'nullglob' if necessary
        local -i need_to_unset_nullglob=0
        if [[ ":$BASHOPTS:" != *:nullglob:* ]] ; then
            shopt -s nullglob
            need_to_unset_nullglob=1
        fi

        local newest_file=""
        local file=""
        for file in $glob_pattern ; do
            echo $file
            [[ -z $newest_file || $file -nt $newest_file ]] && newest_file=$file
        done

        # To avoid unexpected behaviour elsewhere, unset nullglob if it was
        # set by this function
        (( need_to_unset_nullglob )) && shopt -u nullglob

        # Use printf instead of echo in case the file name begins with '-'
        [[ -n $newest_file ]] && echo "$newest_file"

        return 0
    }

    select_newest_file() {
        [[ $# -eq 2 ]] || return 0
        local -r fileA="$1"
        local -r fileB="$2"

        # To avoid printing garbage if no files match the pattern, set
        # 'nullglob' if necessary
        local -i need_to_unset_nullglob=0
        if [[ ":$BASHOPTS:" != *:nullglob:* ]] ; then
            shopt -s nullglob
            need_to_unset_nullglob=1
        fi

        local newest_file="$fileA"
        [[ -z "$newest_file" || "$fileB" -nt "$newest_file" ]] && newest_file="$fileB"

        # To avoid unexpected behaviour elsewhere, unset nullglob if it was
        # set by this function
        (( need_to_unset_nullglob )) && shopt -u nullglob

        # Use printf instead of echo in case the file name begins with '-'
        [[ -f "$newest_file" ]] && echo "$newest_file"

        return 0
    }

    #-----------------------------------------------
    # cleanup path
    #-----------------------------------------------
    cleanup_path() {
        [[ $# -gt 0 ]] || return 0

        # split path into an array
        local path=()
        path=($(tokenize ':' "$@"))
        ###v_tokenize path ':' "$@"
        [[ ${#path[@]} -gt 0 ]] || return 0
        
        # loop over paths and remove those not available
        local newpath=
        local p=
        declare -A map
        for p in "${path[@]}"; do
            [[ -d $p ]] || continue
            haskey map "$p" && continue
            map["$p"]=0
            [[ -z $newpath ]] && { newpath="$p"; continue; }
            newpath="${newpath}:$p"
        done
        echo "$newpath"
    }

    add_to_path() {
        [[ $# -ge 2 ]] || fatal_msg "add_to_path: not enough args $#"
        local path="$1"
        local add="$2"
        [[ -n $add ]] || { echo "$path"; return 0; }
        path="${add}:$path"
        cleanup_path "$path"
    }
    
    #-----------------------------------------------
    # find files in a path
    # will return a ':' delimited string with
    # file names
    #-----------------------------------------------
    #find_files() {
    #    
    #}

    #-----------------------------------------------
    # find a file in given path
    # args:
    #   <name>    file name to be found
    #   <path>    directory path separated with ':'
    #   <match>   test option, eg "-f" for file, "-x" for executable
    #-----------------------------------------------
    find_in_path() {
        [[ $# -eq 3 ]] || return 0
        local name="$1"
        local match="$3"
        local path=()
        path=($(tokenize ':' "$2"))
        # split path and loop over dirs
        local p=
        for p in "${path[@]}"; do
            local d=
            eval d="$p/$name"
            if   [[ $match == "-f" ]]; then
                [[ -f "$d" ]] && { echo "$d"; return 0; }
            elif [[ $match == "-x" ]]; then
                [[ -x "$d" ]] && { echo "$d"; return 0; }
            else
                fatal_msg "find_in_path : got unknown match option <$match>"
            fi
        done
        return 1
    }
    #-----------------------------------------------
    # find executable
    # args: <name> [<path>]
    #
    # It searches
    # 1. using 'which' command (PATH etc)
    # 2. find exec in given path using 'find'
    #
    # In the second case it will thus do a deep
    # search for all matches.
    # If multiple matches are found in path, it will
    # take the one with the latest time stamp as
    # given by 'ls -t'.
    #-----------------------------------------------
    find_exe() {
        local path=
        local extra=
        [[ $# -ge 2 ]] && path="$2"
        [[ $# -ge 3 ]] && extra="$3"
        local name="$1"
        [[ -n $name ]] || return 1

        # find first by system path
        local exe=""
        # --skip-alias option is not available in Ubuntu
        ###exe=$(which --skip-alias "$name" 2>/dev/null)
        exe=$(/usr/bin/which "$name" 2>/dev/null)
        [[ $? -eq 0 ]] || { exe=""; }
        [[ -z $exe  ]] || { echo "$exe"; return 0; }

        # not found in system wide search and no path given -> return with error
        [[ -n "$path" ]] || return 1
        local saveifs=$IFS
        
        local patharr=()
        eval $(create_array patharr ':' "$path")
        # find the newest executable in the path
        local newest=""
        local ep=""

        # split path and loop over dirs
        local p=""
        local i=0
	local all=()
        for ((i=0; i<${#patharr[@]}; i++ )); do
            p="${patharr[$i]}"
            eval ep="$p" # really needed?
            # find all - need to reset IFS in order for the array to work with filenames
            # containing spaces
            local file=""
            while IFS= read -r -d '' file; do
                all+=("$file")
            done < <(find "$ep/" -name "$name" -executable -type f -print0 2>/dev/null)
	done
	
        # loop over all found
        for ((i=0; i<${#all[@]}; i++)); do
            local f="${all[$i]}"
            if [[ -n $extra ]]; then
                # check extra file
                local d=$(dirname $f)/$extra
                [[ -f $d ]] || continue
            fi
            
            # no $newest defined
            [[ -z $newest ]] && { newest="$f"; continue; }
            # check if $f is newer than $newest
            [[ "$f" -nt "$newest" ]] && {
                debug_msg "$f is newer than $newest"
                newest="$f"
            }
        done

        [[ -n "$newest" ]] || return 1

        # output result
        echo "$newest"

        return 0
    }

    #-----------------------------------------------
    # calls the above and return the directory
    #-----------------------------------------------
    find_exe_dir() {
        local path=
        local extra=
        [[ $# -ge 2 ]] && path="$2"
        [[ $# -ge 3 ]] && extra="$3"
        local exename=
        local ec=0
        exename=$(find_exe "$1" "$path" "$extra")
        ec=$?
        [[ $ec -ne 0 ]] && return 1
        [[ -z "$exename" ]] && return 1
        dirname "$exename"
        return $?
    }

    #--------------------------------------------------
    # retry a commands a fixed nr of times
    # and given sleep period
    #--------------------------------------------------
    retry_cmd() {
        local ntries=$1
        local tsleep=$2
        shift
        shift
        local cmd="$*"
        if ! is_integer "$ntries"; then
            return 1
        fi
        if ! is_float "$tsleep"; then
            return 2
        fi
        local ec=0
        while (( ntries>0 )); do
            $cmd && return 0;
            ec=$?
            warn_msg "failed with ec=$ec - retrying after $tsleep seconds" 1>&2
            warn_msg "failed command: $cmd" 1>&2
            sleep $tsleep
            ntries=$((ntries-1))
        done
        return $ec
    }
    
    # like the above but masks out stderr
    retry_cmd_no_stderr() {
        local ntries=$1
        local tsleep=$2
        shift
        shift
        local cmd="$*"
        if ! is_integer "$ntries"; then
            return 1
        fi
        if ! is_float "$tsleep"; then
            return 2
        fi
        local ec=0
        while (( ntries>0 )); do
            $cmd 2>/dev/null && return 0;
            ec=$?
            warn_msg "failed with ec=$ec - retrying after $tsleep seconds" 1>&2
            warn_msg "failed command: $cmd" 1>&2
            sleep $tsleep
            ntries=$((ntries-1))
        done
        return $ec
    }

    check_url() {
        [[ $# -eq 1 ]] || return 1
        local line=$(curl --head "$1" 2> /dev/null | head -n 1)
        [[ -n $line ]] || return 1
        local code=$(echo $line | cut -f 2 -d ' ')
        local ec=0
        if [[ "$code" == "404" ]]; then
            ec=1
        elif [[ "$code" == "405" ]]; then
            ec=0 # method not allowed
        elif [[ "${code:0:2}" == "30" ]]; then
            ec=0
        elif [[ "${code:0:2}" == "20" ]]; then
            ec=0
        else
            warn_msg "check_url() - url <$1> returned unclassified code $code"
            return 2
        fi
        if (( ec>0 )); then
            warn_msg "URL check failed with error code $code"
            warn_msg "URL: $1"
        fi
        return $ec
    }

    #--------------------------------------------------
    # copy or link file
    # arguments: <flag> <target> <file1> <file2> ...
    #  <flag>   : 0 -> make link
    #             1 -> copy
    #             +10 -> same as above but overwrite existing targets
    #  <target> : target directory or file
    #  <fileX>  : source file(s)
    #--------------------------------------------------
    copy_file() {
        [[ $# -eq 3 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"

        local what="$1"
	    local ftrg="$2"
        local fsrc="$3"

        # flag if target is dir or file
        local trgtype=0
        [[ -f "$ftrg" ]] && trgtype=1 #file
        [[ -d "$ftrg" ]] && trgtype=2 #directory

        if [[ $trgtype -eq 0 ]]; then
            local ndir=""
            ndir=$(dirname "$ftrg")
            [[ -d "$ndir" ]] && trgtype=1 #file
        fi

        # skip or not to skip
        local skip_check=0
        if [[ $what -ge 10 ]]; then
            what=$((what-10))
            skip_check=1
        fi

        # check file
        [[ -f "$fsrc" ]] || { error_msg "Missing source file <$fsrc>"; return 255; }
        local trg=""
        [[ $trgtype -eq 1 ]] && trg="$ftrg"
        [[ $trgtype -eq 2 ]] && trg="${ftrg}"/$(basename "$fsrc")

        if [[ $skip_check -eq 0 ]]; then
            check_file "$FUNCNAME" "$trg" 2>/dev/null && {
                verbose_warn_msg "target file already exists: $trg"
                return 1
            }
        fi

	    [[ $what -eq 1 ]] && cp -f  "$fsrc" "$trg"
	    [[ $what -eq 0 ]] && {
            # link - expand name to ensure it's validity
            local f=""
            verbose_info_msg "copy_file: $fsrc <-> $ftrg"
            if are_same_dirs "$fsrc" "$trg"; then
                f=$(basename "$fsrc")
            else
                f=$(expand_name "$fsrc")
            fi
            verbose_info_msg "copy_file: soft link $f with $trg"
            ln -sf "$f" "$trg"
        }
        return 0
    }

    #--------------------------------------------------
    # copy file and tries a maximum N times
    # args: <src> <trg> <N> <sleep>
    # where sleep is in seconds (float).
    # It will return the return code from the cp command.
    #--------------------------------------------------
    copy_with_retry() {
        [[ $# -eq 4 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local src="$1"
        local trg="$2"
        local N="$3"
        local T="$4"
        [[ $N -lt 1 ]] && N=1
        local ec=255
        local ntry=0
        while [[ $ntry -lt $N ]]; do
            cp -f "$src" "$trg"
            ec=$?
            [[ $ec -eq 0 ]] && break
            ntry=$((ntry+1))
            sleep $T
        done
        return $ec
    }

    #-----------------------------------------------
    # check the existance of a file
    #-----------------------------------------------
    check_file() {
        [[ $# -eq 2 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local label="$1"
        local file="$2"

        [[ -f "$file" ]] && return 0
        [[ -h "$file" ]] && return 0
        
        error_msg "$label : unable to access file <${file}>"

	    return 255
    }

    #-----------------------------------------------
    # check if a file is executable
    #-----------------------------------------------
    check_file_exe() {
        [[ $# -eq 2 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local label="$1"
        local file="$2"

        [[ -x "$file" ]] && return 0
	    error_msg "$label : file is not executable or missing <${file}>"
	    return 255
    }

    #-----------------------------------------------
    # check if argument is a directory
    #-----------------------------------------------
    check_dir() {
        [[ $# -eq 2 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local label="$1"
        local dir="$2"

        [[ -d "$dir" ]] && return 0
	    error_msg "$label : unable to access directory <${dir}>"
	    return 255
    }

    #-----------------------------------------------
    # check that input file exists and that
    # it is not equal to output
    #-----------------------------------------------
    check_inpout_files() {
        [[ $# -eq 3 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local label="$1"
        local inpfile="$2"
        local outfile="$3"

        check_file "$label" "$inpfile" || return 1
        are_same_files "$inpfile" "$outfile"
        if [[ $? -eq 0 ]]; then
            error_msg "$label : input and output files are the same!"
            error_msg "$label : input  -> $inpfile"
            error_msg "$label : output -> $outfile"
            return 1
        fi
        return 0
    }

    #-----------------------------------------------
    # Check if variable is properly set
    #-----------------------------------------------
    check_variable() {
        [[ $# -eq 2 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local label="$1"
        local var="$2"
        local expvar=""
        expvar="$(eval echo \$$var)"

        [[ -z $expvar ]] || return 0
        error_msg "$label : varible \$$var has zero length"
        return 255
    }

    #-----------------------------------------------
    # check if target is newer than source
    #-----------------------------------------------
    check_if_newer_source() {
        [[ $# -eq 2 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
        local source="$1"
        local target="$2"

        # source does not exist -> error
        [[ -f "$source" ]] || return 255

        # target does not exist -> remake
        [[ -f "$target" ]] || return 0

        # target is older than source -> remake
        [[ "$target" -nt "$source" ]] || return 0
        return 1
    }
    
    #    # if target is older than source -> remake
    #    local ttarget=0
    #    ttarget=$(stat -L -c %Y "$target")
    #    local tsource=0
    #    tsource=$(stat -L -c %Y "$source")
    #    [[ $tsource -gt $ttarget ]] && return 0

    #    # both target and source exist and target is the youngest -> NO remake
    #    return 1
    #}


    #-----------------------------------------------
    # get current time in epoch seconds
    # for other scales its prefixed by
    # * S -> seconds
    # * M -> milliseconds
    # * U -> microseconds
    # * N -> nanoseconds
    #-----------------------------------------------
    time_sec() {
        echo "S$(date '+%s')"
    }
    time_nsec() {
        echo "N$(date '+%s%N')"
    }
    time_usec() {
        local t=$(date '+%s%N')
        echo "U$((t/1000))"
    }
    time_msec() {
        local t=$(date '+%s%N')
        echo "M$((t/1000000))"
    }
    export _TIME_SCALE="1.0"
    export _TIME_PREFIX=""
    export _TIME_VALUE=0
    time_scale() {
        local T="$1"
        _TIME_SCALE="1.0"
        _TIME_PREFIX="${T:0:1}"
        _TIME_VALUE="${T:1}"
        if [[ "$_TIME_PREFIX" == "S" ]]; then
            _TIME_SCALE="1.0"
        elif [[ "$_TIME_PREFIX" == "M" ]]; then
            _TIME_SCALE="1.0e-3"
        elif [[ "$_TIME_PREFIX" == "U" ]]; then
            _TIME_SCALE="1.0e-6"
        elif [[ "$_TIME_PREFIX" == "N" ]]; then
            _TIME_SCALE="1.0e-9"
        else
            _TIME_SCALE="1.0"
            _TIME_VALUE="$T"
        fi
    }
    time_convert_ref() {
        local Tref=$1
        local T=$2
        time_scale "$Tref"
        local scaleRef=$_TIME_SCALE
        local prefixRef=$_TIME_PREFIX
        time_scale "$T"
        local scale=$_TIME_SCALE
        if [[ "$_TIME_PREFIX" == "$prefixRef" ]]; then
            echo "$_TIME_VALUE"
            return 0
        fi
        echo "${_TIME_VALUE}*$_TIME_SCALE/$scaleRef"
    }
    #-----------------------------------------------
    # return time difference in seconds
    #-----------------------------------------------
    delta_time() {
        [[ $# -ge 1 ]] || return 1
        local T0=$1
        local T1=0
        if [[ $# -eq 2 ]]; then
            T1="$2"
        else
            T1=$(time_nsec)
        fi
        # convert both times to seconds
        local T0r=$(time_convert_ref "S0" "$T0")
        local T1r=$(time_convert_ref "S0" "$T1")
        calc "(${T1r})-(${T0r})"
    }
    
    #-----------------------------------------------
    # return time sum in seconds
    #-----------------------------------------------
    sum_time() {
        [[ $# -ge 1 ]] || return 1
        local T0=$1
        local T1=0
        if [[ $# -eq 2 ]]; then
            T1="$2"
        else
            T1=$(time_nsec)
        fi
        # convert both times to seconds
        local T0r=$(time_convert_ref "S0" "$T0")
        local T1r=$(time_convert_ref "S0" "$T1")
        calc "(${T1r})+(${T0r})"
    }
    
    date_locale() {
        # just because LANG is set to C for the sorting
        LANG=$_saved_LANG
        date '+%x %X'
        LANG=$_set_LANG
    }


    #-----------------------------------------------
    # separator lines
    #-----------------------------------------------
    print_repeated() {
        [[ $# -eq 2 ]] || return 0
        printf "%0.s$1" $(seq 1 "$2")
    }

    print_separator() {
        local ch='-'
        local N=80
        [[ $# -ge 1 ]] && ch="$1"
        [[ $# -eq 2 ]] && N="$2"
        print_repeated "$ch" "$N"
        echo
    }

    #-----------------------------------------------
    # output to stderr
    #-----------------------------------------------
    echoerr() {
        echo "${@}" 1>&2
    }

    printferr() {
        printf "${@}" 1>&2
    }


    export QUIET_INFO_MSG=0
    export _SAVED_QUIET_INFO_MSG=0
    set_info_msg_quiet() {
        _SAVED_QUIET_INFO_MSG=$QUIET_INFO_MSG
        [[ $VERBOSE -gt 0 ]] || QUIET_INFO_MSG=1
    }
    reset_info_msg_quiet() {
        QUIET_INFO_MSG=$_SAVED_QUIET_INFO_MSG
    }
    
    export MSG_PREFIX=""
    msg_str() {
        local level=$1
        shift
        printf "%-25s%-12s%s" "$(date_locale)" "$level" "${MSG_PREFIX}$*"
    }
    msg_stepid_str() {
        local level=$1
        local stepid=$2
        shift
        shift
        printf "%-25s%-25s%-12s%s" "$(date_locale)" "$stepid" "$level" "${MSG_PREFIX}$*"
    }
    msg_sel_str() {
        local w=$1
        local level=$2
        local msg="$3"
        local mstr="$msg"
        is_dryrun && msg="<DRY RUN> $msg"
        if (( w == 0 )); then
            mstr=$(msg_str "$level" "$msg")
        elif (( w == 1 )); then
            mstr=$(msg_stepid_str "$level" "$STEP_FULL_ID" "$msg")
        fi
        echo "$mstr"
    }
#    v_msg_sel_str() {
#        local var="$1"
#        local w=$2
#        local level=$3
#        local msg="$4"
#        local saveIFS=$IFS
#        IFS=$'\n'
#        local lines=( $(echo -e "$msg") )
#        
#        is_dryrun && msg="<DRY RUN> $msg"
#        local mstr="$msg"
#        # make sure that w=0 if STEP_FULL_ID is not set...
#        if isset STEP_FULL_ID; then
#            [[ -n "$STEP_FULL_ID" ]] || w=0
#        fi
#
#        if (( w == 0 )); then
#            mstr=$(msg_str "$level" "$msg")
#        elif (( w == 1 )); then
#            mstr=$(msg_stepid_str "$level" "$STEP_FULL_ID" "$msg")
#        fi
#        local "$var" && set_var "$var" "$mstr"
#    }

    export _MSG_SEL_PREFIX=""
    export _MSG_SEL_STR=()
    v_msg_sel_str() {
        _MSG_SEL_STR=()
        local w=$1
        local level=$2
        local msg="$3"
        local saveIFS=$IFS
        IFS=$'\n'
        local lines=( $(echo -e "$msg") )
        local prefix="$_MSG_SEL_PREFIX"
        is_dryrun && prefix="${prefix}<DRY RUN>"
        if (( w == 0 )); then
            prefix=$(msg_str "$level" "${prefix}")
        elif (( w == 1 )); then
            prefix=$(msg_stepid_str "$level" "$STEP_FULL_ID" "${prefix}")
        fi
        for L in "${lines[@]}"; do
            _MSG_SEL_STR+=("$prefix $L")
        done
        IFS=$saveIFS
    }
    #-----------------------------------------------
    # main message functions
    # fatal_msg -> stderr -> exit 255
    # error_msg -> stderr
    # warn_msg  -> stdout
    # info_msg  -> stdout
    # info_msg_stderr  -> stderr
    # Those with argument 'w'
    # w=0 -> message without STEP_ID
    # w=1 -> message with STEP_ID
    #-----------------------------------------------
    export _LAST_MSG_ERROR=""
    export _LAST_MSG_WARN=""
    clear_last_msg() {
        _LAST_MSG_ERROR=""
        _LAST_MSG_WARN=""
    }
    fatal_raw_msg_no_exit() {
        local m=""
        [[ $# -eq 0 ]] || m="$1"
	    _LAST_MSG_ERROR="$m"
        echoerr "${COLOR_RED}${m}${COLOR_RESET}"
    }

    fatal_raw_msg() {
        local m=""
        [[ $# -eq 0 ]] || m="$1"
	    _LAST_MSG_ERROR="$m"
        fatal_raw_msg_no_exit "$m"
        exit 255
    }
    error_raw_msg() {
        local m=""
        [[ $# -eq 0 ]] || m="$1"
	    _LAST_MSG_ERROR="$m"
        echoerr "${COLOR_RED}${m}${COLOR_RESET}"
    }
    warn_raw_msg() {
        local m=""
        [[ $# -eq 0 ]] || m="$1"
	    _LAST_MSG_WARN="$m"
        echo "${COLOR_ORANGE}${m}${COLOR_RESET}"
    }
    info_raw_msg() {
        local m=""
        [[ $# -eq 0 ]] || m="$1"
        echo "${COLOR_GREEN}${m}${COLOR_RESET}"
    }

    fatal_msg() {
        local m=""
        local w=0
        [[ $# -gt 0 ]] && m="$1"
        [[ $# -eq 2 ]] && w=$2
        v_msg_sel_str $w "FATAL" "$m"
        local mstr=""
        for mstr in "${_MSG_SEL_STR[@]}"; do
            fatal_raw_msg_no_exit "$mstr"
        done
        exit 255
    }
    error_msg() {
        local m=""
        local w=0
        [[ $# -gt 0 ]] && m="$1"
        [[ $# -eq 2 ]] && w=$2
        v_msg_sel_str $w "ERROR" "$m"
        local mstr=""
        for mstr in "${_MSG_SEL_STR[@]}"; do
            error_raw_msg "$mstr"
        done
    }
    warn_msg() {
        local m=""
        local w=0
        [[ $# -gt 0 ]] && m="$1"
        [[ $# -eq 2 ]] && w=$2
        v_msg_sel_str $w "WARNING" "$m"
        local mstr=""
        for mstr in "${_MSG_SEL_STR[@]}"; do
            warn_raw_msg "$mstr"
        done
    }
    info_msg() {
        [[ $QUIET_INFO_MSG -eq 0 ]] || return
        local m=""
        local w=0
        [[ $# -gt 0 ]] && m="$1"
        [[ $# -eq 2 ]] && w=$2
        v_msg_sel_str $w "INFO" "$m"
        local mstr=""
        for mstr in "${_MSG_SEL_STR[@]}"; do
            info_raw_msg "$mstr"
        done
    }
    
    info_msg_stderr() {
        info_msg "$*" 1>&2
    }

    warn_msg_stderr() {
        warn_msg "$*" 1>&2
    }

    #-----------------------------------------------
    # special messages
    #-----------------------------------------------
    fatal_msg_wrong_nargs() {
        local lbl="?"
        local n="?"
        [[ $# -gt 0 ]] && n="$1"
        [[ $# -gt 1 ]] && lbl="$2"
        fatal_msg "wrong nr of arguments ($n) given to <$lbl>"
    }
    error_msg_wrong_nargs() {
        local lbl="?"
        local n="?"
        [[ $# -gt 0 ]] && n="$1"
        [[ $# -gt 1 ]] && lbl="$2"
        error_msg "wrong nr of arguments ($n) given to <$lbl>"
        return 255
    }

    #-----------------------------------------------
    # message only output in dry run mode
    #-----------------------------------------------
    dryrun_info_msg() {
        local m=""
        [[ $# -eq 0 ]] || m="$1"
        is_dryrun && info_msg "$m"
        return 0
    }
    #-----------------------------------------------
    # verbose messages
    # VERBOSE>0
    #-----------------------------------------------
    verbose_info_msg() {
        [[ $VERBOSE -ne 0 ]] || return
        local m=""
        [[ $# -eq 0 ]] || m="$1"
        info_msg_stderr "<VERBOSE> $m"
    }
    verbose_warn_msg() {
        [[ $VERBOSE -ne 0 ]] || return
        local m=""
        [[ $# -eq 0 ]] || m="$1"
        warn_msg_stderr "<VERBOSE> $m"
    }
    verbose_error_msg() {
        [[ $VERBOSE -ne 0 ]] || return
        local m=""
        [[ $# -eq 0 ]] || m="$1"
        error_msg "<VERBOSE> $m"
    }

    #-----------------------------------------------
    # debug messages
    # VERBOSE>1
    #-----------------------------------------------
    debug_msg() {
        [[ $VERBOSE -gt 1 ]] || return 0
        local m=""
        [[ $# -eq 0 ]] || m="$1"
        info_msg "<DEBUG> <$$> $m" 1>&2
        return 0
    }

    #-----------------------------------------------
    # counted messages with possibility to suppress
    # If MAX_MESSAGES==0 then all msgs are output.
    #-----------------------------------------------
    # counter used for reducing the nr of line output
    export CHECK_COUNTER=0
    export MAX_MESSAGES=${MAX_MESSAGES=0}
    cnt_warn_msg() {
        CHECK_COUNTER=$((CHECK_COUNTER+1))
        [[ $MAX_MESSAGES -eq 0 ]] && CHECK_COUNTER=0
        local text="$1"
        local w=0
        [[ $# -gt 1 ]] && w=$2
        
        [[ $CHECK_COUNTER -le $MAX_MESSAGES ]] && warn_msg "$text" $w
        [[ $CHECK_COUNTER -eq 0             ]] && return
        [[ $CHECK_COUNTER -eq $MAX_MESSAGES ]] && warn_msg "further error/warning messages will be suppressed" $w
    }

    cnt_error_msg() {
        CHECK_COUNTER=$((CHECK_COUNTER+1))
        [[ $MAX_MESSAGES -eq 0 ]] && CHECK_COUNTER=0
        local text="$1"
        local w=0
        [[ $# -gt 1 ]] && w=$2
        
        [[ $CHECK_COUNTER -le $MAX_MESSAGES ]] && error_msg "$text" $w
        [[ $CHECK_COUNTER -eq 0             ]] && return
        [[ $CHECK_COUNTER -eq $MAX_MESSAGES ]] && error_msg "further error/warning messages will be suppressed" $w
    }

    cnt_sum_msg() {
        if [[ $CHECK_COUNTER -gt 0 ]] && [[ $CHECK_COUNTER -ge $MAX_MESSAGES ]]; then
	        local nextras=$((CHECK_COUNTER-MAX_MESSAGES))
	        [[ $nextras -eq 0 ]] || warn_msg "suppressed an additional $nextras error/warning messages"
        fi
    }

    #-----------------------------------------------
    # status messages
    # format is:
    # <left aligned string len 60>       [<status>]
    #-----------------------------------------------
    header_status_msg() {
        printf "%-65sStatus\n" "$1"
    }
    ok_status_msg() {
        local m=""
        [[ $# -eq 0 ]] || m="$1"
        printf "%-65s[${COLOR_GREEN}OK${COLOR_RESET}]\n" "${m}"
    }
    error_status_msg() {
        local m=""
        [[ $# -eq 0 ]] || m="$1"
        printferr "%-65s[${COLOR_RED}ERROR${COLOR_RESET}]\n" "${m}"
    }
    warn_status_msg() {
        local m=""
        [[ $# -eq 0 ]] || m="$1"
        printf "%-65s[${COLOR_ORANGE}WARNING${COLOR_RESET}]\n" "${m}"
    }
    disabled_status_msg() {
        local m=""
        [[ $# -eq 0 ]] || m="$1"
        printf "%-65s[${COLOR_GREEN}DISABLED${COLOR_RESET}]\n" "${m}"
    }

    xok_status_msg() {
        local m=""
        local status="OK"
        [[ $# -gt 0 ]] && m="$1"
        [[ $# -gt 1 ]] && status="$2"
        printf "%-65s[${COLOR_GREEN}${status}${COLOR_RESET}]\n" "${m}"
    }
    xerror_status_msg() {
        local m=""
        local status="ERROR"
        [[ $# -gt 0 ]] && m="$1"
        [[ $# -gt 1 ]] && status="$2"
        printferr "%-65s[${COLOR_RED}${status}${COLOR_RESET}]\n" "${m}"
    }
    xwarn_status_msg() {
        local m=""
        local status="WARNING"
        [[ $# -gt 0 ]] && m="$1"
        [[ $# -gt 1 ]] && status="$2"
        printf "%-65s[${COLOR_ORANGE}${status}${COLOR_RESET}]\n" "${m}"
    }

    xec_status_msg() {
        [[ $# -ge 2 ]] || return
        local ec=$1
        local m="$2"
        if   (( ec == 0 )); then
            xok_status_msg "$m"
#        elif (( ec == 1 )); then
#            xwarn_status_msg "$m"
        else
            xerror_status_msg "$m"
        fi
    }
    
    num_status_msg() {
        [[ $# -ne 3 ]] && return
        local label=$1
        local ntot=$2
        local nok=$3
        ###    [[ $nok -eq 0 ]] && [[ $ntot -eq 0 ]] && return

        if [[ $ntot -eq $nok ]]; then
            # all are done -> OK
            printf "%-65s[${COLOR_GREEN}%s${COLOR_RESET}]\n" "${label}" "OK"
        else
            # print nr of jobs done and total nr of jobs
            printf "%-65s[${COLOR_RED}%s${COLOR_RESET}]\n" "${label}" "$nok/$ntot"
        fi
    }

    #-----------------------------------------------
    # provide numbered titles
    #-----------------------------------------------
    export _ntit_echo_title_=0
    echo_title_reset() {
        _ntit_echo_title_=1
        if [[ $# -eq 1 ]]; then
            local nr=$1
            # make sur it is numerical
            printf "%d\n" "$nr" >& /dev/null
            [[ $? -eq 0 ]] && _ntit_echo_title_=$nr
        fi
    }
    echo_title() {
        printf "#%02d.  %s\n" "$_ntit_echo_title_" "$1"
        _ntit_echo_title_=$((_ntit_echo_title_+1))
    }
    #-----------------------------------------------
    # remove comment lines, ie lines starting
    # with a '#'
    #-----------------------------------------------
    remove_comments() {
        local inp="$1"
        [[ -f "$inp" ]] || return 1
        #create tmp file
        local bname0=""
        local bname=""
        bname0=$(basename "$inp")
        bname=$(replace_char "$bname0" " " "_")
        local tmpfile=""
        tmp_make tmpfile "$bname"
        grep -v "^#" "$inp" > "$tmpfile"
        echo $tmpfile
        return 0
    }


    #-----------------------------------------------
    # match the head of two strings
    # the length is defined by the shortest string
    #-----------------------------------------------
    match_str_head() {
        [[ $# -ne 2 ]] && return
        local str="$1"
        local ref="$2"
        # get shortest length
        local nstr=${#str}
        local nref=${#ref}
        local ncmp=$nstr
        [[ $ncmp -gt $nref ]] && ncmp=$nref

        local cstr=${str:0:$ncmp}
        local rstr=${ref:0:$ncmp}

        # print ref string if match
        [[ "$rstr" == "$cstr" ]] && { echo "$ref"; return; }
        return
    }

    #-----------------------------------------------
    # check if given string is a certain type
    #-----------------------------------------------
    istype() {
        [[ $# -eq 0 ]] && return 1
        local var=$1
        [[ $# -eq 2 ]] && opt=$2
        printf "%$opt" "$var" &> /dev/null
        [[ $? -eq 0 ]] && return 0
        return 1
    }

    float_lt() {
        local A=$1
        local B=$2
        [[ $(awk "BEGIN{print $A < $B}") == "0" ]] || return 0
        return 1
    }
    float_gt() {
        float_lt $1 $2 || return 0
        return 1
    }
    #-----------------------------------------------
    # check range of float/int
    # if lower or upper is 'X', then no limit
    #-----------------------------------------------
    inrange_float() {
        [[ $# -eq 3 ]] || return 255
        local var=$1
        local low=$2
        local up=$3
        low=${low,,}
        up=${up,,}
        local lowok=""
        local upok=""
        [[ "$low" == "x" ]] || {
            ###lowok=$(echo $var $low | awk '{if ($1 < $2) print 0; else print 1}')
            lowok=$(awk 'BEGIN{if ('$var' < '$low') print 0; else print 1}')
	        [[ $lowok -eq 1 ]] || return 1
        }
        [[ "$up" == "x" ]] || {
            ###upok=$(echo $var $up   | awk '{if ($1 < $2) print 1; else print 0}')
            upok=$(awk 'BEGIN{if ('$var' < '$up') print 1; else print 0}')
	        [[ $upok -eq 1 ]] || return 1
        }
        return 0
    }

    inrange_int() {
        [[ $# -eq 3 ]] || return 255
        local var=$1
        local low=$2
        local up=$3
        low=${low,,}
        up=${up,,}
        [[ "$low" == "x" ]] || {
	        [[ $var -lt $low ]] && return 1
        }
        [[ "$up" == "x" ]] || {
	        [[ $var -le $up ]] || return 1
        }
        return 0
    }

    #-----------------------------------------------
    # Check if variable is in a given set.
    # The search is done by string.
    #-----------------------------------------------
    inrange_set() {
        [[ $# -ge 2 ]] || return 255
        local var=$1
        shift
        local saveifs=$IFS
        IFS=$_ORG_IFS
        local vset=( $* )
        IFS=$saveifs
        local rval=1
        local i=0
        for ((i=0; i<${#vset[@]}; i++)); do
            [[ "$var" == "${vset[$i]}" ]] && { rval=0; break; }
        done
        return $rval
    }
    inrange_set_delim() {
        [[ $# -ge 3 ]] || return 255
        local var=$1
        local delim=$2
        shift
        shift
        local vset="$*"
        local values=()
        eval $(create_array values "$delim" "$vset")
        local rval=1
        local i=0
        for ((i=0; i<${#values[@]}; i++)); do
            [[ "$var" == "${values[$i]}" ]] && { rval=0; break; }
        done
        return $rval
    }

    #-----------------------------------------------
    # simple calculator for floats
    #-----------------------------------------------
    calc() {
        awk -v OFMT='%f' "BEGIN{ print $* }"
    }

    calc_nbatch() {
        local A=$1
        local B=$2
        local R=$(calc "int($A/$B)" 2>/dev/null)
        [[ -n $R ]] || return 255
        [[ "$A" == "$B" ]] || R=$((R+1))
        echo $R
    }
    #-----------------------------------------------
    # calculate nr of pairs given index ranges
    # matrix type
    #  0 : full      -> use all in B
    #  1 : full-diag -> exclude B==A
    #  2 : half      -> use indB>indA
    #  3 : half+diag -> use indB>=indA
    #-----------------------------------------------
    calc_matrix_npairs() {
        [[ $# -eq 5 ]] || return 255
        local matrix=$1
        local indA0=$2
        local indA1=$3
        local indB0=$4
        local indB1=$5
        # if a single one is <0 then empty range
        [[ $indA0 -ge 0 ]] || { echo "0"; return 0; }
        [[ $indB0 -ge 0 ]] || { echo "0"; return 0; }
        [[ $indA1 -ge 0 ]] || { echo "0"; return 0; }
        [[ $indB1 -ge 0 ]] || { echo "0"; return 0; }
        #
        local npairs=0
        local ec=0
        local nA=$((1+indA1-indA0))
        local nB=$((1+indB1-indB0))
        [[ $nA -gt 0 ]] || return 255
        [[ $nB -gt 0 ]] || return 255
        if   [[ $matrix -eq 0 ]]; then
            # full matrix
            npairs=$((nA*nB))
        elif [[ $matrix -ge 1 ]]; then
            local up=0
            local low=0
            [[ $indB0 -gt $indA1 ]] && up=1
            [[ $indA0 -gt $indB1 ]] && low=1
            if [[ $up -eq 1 ]]; then
                # entirely in upper half
                npairs=$((nA*nB))
            elif [[ $low -eq 1 ]]; then
                # entirely in lower half - no pairs
                npairs=0
            else
                # in both parts
                # find overlap
                local start=indA0
                local end=indB1
                [[ $indA0 -gt $indB0 ]] || start=indB0
                [[ $indA1 -gt $indB1 ]] || end=indA1
                local overlap=$((end-start+1))

                # what matrix?
                if [[ $matrix -eq 1 ]]; then
                    # just remove the diagonal == overlap
                    npairs=$((nA*nB - overlap))
                else
                    # remove blocks from nA*nB:
                    # * the half square inc diagonal = overlap*(overlap+1)/2
                    # * A0>B0 -> remove square (A0-B0)*overlap
                    # * A1>B1 -> remove square (A1-B1)*(B1-B0)
                    local dB=$((indB1-indB0+1))
                    local dX=$((indA1-indB1))
                    [[ $dX -gt 0 ]] || dX=0
                    local dY=$((indA0-indB0))
                    [[ $dY -gt 0 ]] || dY=0
                    npairs=$((nA*nB-(overlap*(overlap+1)/2)-dY*overlap-dX*dB))
                    [[ $matrix -eq 3 ]] && npairs=$((npairs+overlap))
                fi
            fi
        else
            npairs=0
            ec=255
        fi
        echo $npairs
        return $ec
    }


    #---------------------------------------------------------------------------
    # NOT YET USED...
    export _MP_NCORES=0
    export _MP_NHYPER=0
    export _MP_NTHREADS=0
    get_multiprocessing_params() {
        _MP_NCORES=$(get_ncores)
        _MP_THREADS=$(get_nthreads_total)
        _MP_NHYPER=$(get_hyperthreading)
    }
    
    #===============================================================
    #                EXPORT VARIABLES AND FUNCTIONS
    #===============================================================
    #
    # Export functions
    #
    export -f reset_attributes
    export -f good_label_chr

    export -f set_var
    export -f set_dryrun
    export -f unset_dryrun
    export -f is_dryrun
    export -f ask_noyes
    export -f ask_yesno
    export -f yesno_str

    export -f create_array
    export -f create_array_ws
    export -f create_array_eol
    export -f set_ifs
    export -f set_ifs_default
    export -f set_ifs_ws
    export -f set_ifs_eol
    
    export -f tokenize
    export -f v_tokenize
    export -f sort_csv
    export -f check_line
    export -f get_lines
    export -f get_lines_uc
    export -f rm_many
    export -f iswriteable
    export -f cleanup_dir
    export -f assign
    export -f evalvar
    export -f rm_leading_ws
    export -f rm_trailing_ws
    export -f trim

    export -f clear_assoc
    
    export -f isset_assoc
    export -f isset_array
    export -f isset_var
    export -f isset
    export -f isempty
    export -f haskey
    
    export -f isfunc
    export -f check_function
    export -f check_function_status

    export -f check_nargs

    export -f file_has_data
    export -f file_setvar
    export -f file_add_setvar
    export -f file_getvar
    export -f export_dict
    export -f export_array
    export -f export_decl
    export -f export_decl_add
    export -f export_decl_add_noval
    export -f export_decl_add_assoc_noval
    export -f export_decl_reset
    export -f export_decl_doall
    export -f export_decl_stdout
#    export -f _restore_decl
#    export -f restore_decl
#    export -f restore_decl_doall
#    export -f restore_decl_all
    
    export -f is_integer
    export -f is_decimal
    export -f is_float
    export -f is_hash_128bit
    export -f is_hash_160bit
    export -f is_hash
    
    export -f tmp_base
    export -f tmp_make
    export -f tmp_make_dir
    export -f tmp_cleanup_files
    export -f tmp_clear_files
    export -f tmp_add_file
    export -f tmp_delete
    export -f tmp_mark_delete
    export -f tmp_mark_keep
    export -f create_dir
    export -f create_link
    export -f expand_name
    export -f reduce_name
    export -f file_extension
    export -f file_basename
    export -f set_extension
    export -f are_same_files
    export -f are_same_dirs
    export -f find_newest_file
    export -f select_newest_file
    export -f cleanup_path
    export -f find_in_path
    export -f find_exe
    export -f find_exe_dir
    export -f retry_cmd
    export -f retry_cmd_no_stderr
    export -f check_url
    export -f copy_file
    export -f copy_with_retry
    export -f check_file
    export -f check_file_exe
    export -f check_dir
    export -f check_inpout_files
    
    export -f check_variable
    export -f check_if_newer_source
    export -f time_sec
    export -f time_msec
    export -f time_usec
    export -f time_nsec
    export -f time_scale
    export -f time_convert_ref
    export -f delta_time
    export -f sum_time
    
    export -f date_locale

    export -f print_repeated
    export -f print_separator
    export -f echoerr
    export -f printferr
    export -f msg_str
    export -f msg_stepid_str
    export -f msg_sel_str
    export -f v_msg_sel_str
    export -f set_info_msg_quiet
    export -f reset_info_msg_quiet
    export -f fatal_raw_msg_no_exit
    export -f fatal_raw_msg
    export -f error_raw_msg
    export -f warn_raw_msg
    export -f info_raw_msg
    export -f fatal_msg
    export -f error_msg
    export -f warn_msg
    export -f info_msg
    export -f info_msg_stderr
    export -f warn_msg_stderr
    export -f fatal_msg_wrong_nargs
    export -f error_msg_wrong_nargs
    export -f clear_last_msg
    
    export -f dryrun_info_msg
    export -f verbose_info_msg
    export -f verbose_warn_msg
    export -f verbose_error_msg
    export -f debug_msg

    export -f cnt_warn_msg
    export -f cnt_error_msg
    export -f cnt_sum_msg
    export -f header_status_msg
    export -f ok_status_msg
    export -f error_status_msg
    export -f warn_status_msg
    export -f disabled_status_msg
    export -f xok_status_msg
    export -f xwarn_status_msg
    export -f xerror_status_msg
    export -f xec_status_msg
    export -f num_status_msg
    export -f echo_title_reset
    export -f echo_title

    export -f remove_comments
    export -f match_str_head
    export -f istype
    export -f float_lt
    export -f float_gt
    export -f inrange_float
    export -f inrange_int
    export -f inrange_set
    export -f inrange_set_delim

    export -f calc
    export -f calc_nbatch
    export -f calc_matrix_npairs
    export -f get_multiprocessing_params
    
    export -f replace_char

    LOADED_COMMON_BASE_SH=0
    # the re-load protection
fi
