#!/usr/bin/env bash
#
# orthologer - Pipeline for deducing orthology from fasta files.
# Copyright (C) 2005-2023 E. Zdobnov lab: F. Tegenfeldt
# <fredrik.tegenfeldt@unige.ch> based on code by E. Kriventseva
#
# This file is part of orthologer.
#
# orthologer is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# orthologer is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with orthologer.  If not, see <http://www.gnu.org/licenses/>.
#
if [[ "${LOADED_COMMON_CREATE_SH+1}" == "" ]]; then
    set -u
    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    # FUNCTIONS
    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    #--------------------------------------------------
    # parse csv string
    #--------------------------------------------------
    parse_csv_dirs() {
        local csv="$1"
        [[ ${#csv} -eq 0 ]] && return

        local dirstr=""
        local dmatch=""
        local ec=0
        local acsv=($(tokenize ',' $csv))
        local d=
        for d in ${acsv[@]}; do
            dmatch=$(match_str_head ${d^^} "FASTA")
            [[ ${#dmatch} -gt 0 ]] && { dirstr="$dirstr FASTA"; continue; }
            dmatch=$(match_str_head ${d^^} "SEQUENCES")
            [[ ${#dmatch} -gt 0 ]] && { dirstr="$dirstr Sequences"; continue; }
            dmatch=$(match_str_head ${d^^} "BRH")
            [[ ${#dmatch} -gt 0 ]] && { dirstr="$dirstr BRH"; continue; }
            dmatch=$(match_str_head ${d^^} "PWC")
            [[ ${#dmatch} -gt 0 ]] && { dirstr="$dirstr PWC"; continue; }
            dmatch=$(match_str_head ${d^^} "CLUSTER")
            [[ ${#dmatch} -gt 0 ]] && { dirstr="$dirstr Cluster"; continue; }
            [[ ${#d} -gt 0 ]] && ec=1
        done
        echo $dirstr
        return $ec
    }

    #--------------------------------------------------
    # Run a simple gene prediction using 'prodigal'.
    # Arguments: <source> <target>
    #  <source>   source nucleotide fasta
    #  <target>   prodigal output AA fasta
    #--------------------------------------------------
    export PRODIGAL_CMDLINE=
    simple_gene_prediction() {
        local fsrc="$1"
        local ftrg="$2"
        [[ -f $fsrc ]] || {
            error_msg "source nucleotide fasta file is not readable $fsrc"
            return 255
        }
        if [[ -f $EXE_PRODIGAL ]]; then
            # run simple gene prediction using prodigal
            PRODIGAL_CMDLINE="$EXE_PRODIGAL -i $fsrc -a $ftrg"
            $EXE_PRODIGAL -i "$fsrc" -a "$ftrg" >& "${ftrg}.prodigal.log" || {
                error_msg "prodigal run failed - see log at ${ftrg}.prodigal.log"
                return 255
            }
        else
            error_msg "----------------------------------------------------------"
            error_msg "Unable to run 'prodigal' since executable is not found."
            error_msg "If the location of 'prodigal' is known, set DIR_PRODIGAL"
            error_msg "in 'orthologer_conf.sh'."
            error_msg "Aborting gene prediction."
            error_msg "----------------------------------------------------------"
            return 255
        fi
        return 0
    }
    #--------------------------------------------------
    # copy or link fasta files into project
    # arguments: <flag> <src>
    #  <flag> : 0 -> make link
    #           1 -> copy
    #  <src>  : source directory
    #
    # files imported are defined by the used TODO file
    #--------------------------------------------------
    copy_fasta_files() {
        local what=$1
        local src="$2"
        [[ -d "$src" ]]  || return 1

        local nbad=0
        local i=0
        for ((i=0; i<${#PL_TODO_LABELS[@]}; i++ )); do
            local A="${PL_TODO_LABELS[$i]}"
	        local fs="${src}/${A}.${EXT_FASTA}"
	        [[ -f "$fs" ]] || { nbad=$((nbad+1)); continue; }
	        local fstarget="${DIR_PROJECT_SEQUENCES}/${A}.${EXT_FASTA}"
	        if [[ -f "$fstarget" ]]; then
	            warn_msg "Fasta file ${A}.${EXT_FASTA} already exists in dir $DIR_PROJECT_SEQUENCES"
	        else
                copy_file $what "$fstarget" "$fs"
	        fi
        done
        return 0
    }

    #--------------------------------------------------
    # import raw fasta file with a given label
    #
    # This will copy (link) the raw fasta file
    # to DIR_PROJECT_RAWDATA and relabel it using <label>.
    #
    # arguments: <flag> <raw fasta file> <label>
    #
    #    <flag>    : same as for copy_file
    #    <fasta>   : raw fasta file
    #    <label>   : unique species label for fasta file
    #
    # A link is then created in DIR_PROJECT_SEQUENCES .
    #
    # Return values
    #    0 - OK
    #    1 - skipping import for valid reason
    #  255 - failed import
    #
    #--------------------------------------------------
    # Add 'prodigal' call to translate nucleotide file.
    #
    # always the same: imported_<label>
    # 
    # if nucleotide:
    #   newfasta = RawData/${label}.${EXT_FASTA}.translated
    #   prodigal -i <fasta> -a $newfasta >& RawData/${label}-translated.log
    #   -> use new fasta
    # 
    import_fasta() {
        [[ $# -eq 3 ]] || return 1
        local flag=$1    # copy or softlink
        local srcfasta="$2" # source fasta file
        local label="$3" # label, possibly prefixed by '+' to force relabling of fasta records
        
        # check if fasta source file should be relabeled (ie, with a '+' as first chr in name)
        local fc="${label:0:1}"
        local do_relabel=0
        [[ "$fc" == "+" ]] && { do_relabel=1; label=${label:1}; }

        # target name
        local newfasta=$DIR_PROJECT_RAWDATA/${label}.${EXT_FASTA}
        verbose_info_msg "import fasta file ($label): $srcfasta"
        verbose_info_msg "new fasta file: $newfasta"

        # dry run
        is_dryrun && { dryrun_msg "import fasta file ($label): <$srcfasta> to <$newfasta>"; return 0; }

        # check source raw fasta
        [[ -f "$srcfasta" ]] || { error_msg "source fasta file is not readable : $srcfasta"; return 255; }

        # check if new file is the same as input file
        local samefiles=0
        are_same_files "$srcfasta" "$newfasta"
        if [[ $? -eq 0 ]]; then
            samefiles=1
            verbose_info_msg "new fasta file and original are the same files!"
            if [[ $do_relabel -eq 1 ]]; then
                error_msg "raw fasta to be relabeled has the same name as the target file!"
                return 255
            fi
        fi

        # do not allow overwrite if files are not already the same (RAWDATA)
        [[ -f "$newfasta" ]] && [[ $samefiles -eq 0 ]] && {
            warn_msg "skip - target relabeled fasta already exists $(trim_name $newfasta $DIR_PROJECT)"
            return 1
        }

        # check SEQUENCES name
        local seqfasta=$DIR_PROJECT_SEQUENCES/${label}.${EXT_FASTA}
        [[ -f "$seqfasta" ]] && {
            warn_msg "skip - sequence fasta already exists $(trim_name $seqfasta $DIR_PROJECT)"
            return 1
        }

        # which file to actually import - set below, depends on fasta type
        local impfasta=
        
        #---------------------------------------------------------------------
        # check if nucleotide
        # if so, make a prodigal call
        local fstype=$($DIR_ORTHOLOGER/bin/check_fasta.py -f $srcfasta --type)
        if [[ "$fstype" == "PROT" ]]; then
            verbose_info_msg "import protein fasta"
            impfasta="$srcfasta"
        elif [[ "$fstype" == "NUCL" ]]; then
            info_msg "import nucleotide fasta - convert the file to protein"
            local startdate=$(date)
            local T0=$(time_msec)
            impfasta="$DIR_PROJECT_RAWDATA/${label}.${EXT_FASTA}.prodigal"
            local nbytes=$(stat -L -c '%s' "$srcfasta")
            nbytes=$((nbytes/1024/1024))
            if (( nbytes > $GENEPRED_MAXSIZE )); then
                error_msg "Fasta file is too large (>$GENEPRED_MAXSIZE)."
                error_msg "The limit can be changed by setting GENEPRED_MAXSIZE=<max in MB>"
                return 255
            fi
            
            if [[ -f $impfasta ]]; then
                info_msg "gene prediction already exists at $impfasta"
            else
                info_msg "convert to protein sequences - attempt to make a simple gene prediction using prodigal"
                local fstmp=
                tmp_make fstmp "prodigal_XXXXX"
                simple_gene_prediction "$srcfasta" "$fstmp" || return 255
                local dT=$(delta_time $T0)
                add_header_footer prodigal "$fstmp" "$impfasta" "$startdate" "$dT" "$PRODIGAL_CMDLINE"
                mv -f "${fstmp}.prodigal.log" "${impfasta}.prodigal.log"
                info_msg "gene prediction in $(trim_name $impfasta)"
            fi
        else
            error_msg "requested import file is not a fasta file ($srcfasta)"
            return 255
        fi
        
        #---------------------------------------------------------------------
        # Now fasta files are defined as follows
        #   srcfasta = the original given source fasta
        #   impfasta = fasta to be relabeled (if AA the its == srcfasta)
        #---------------------------------------------------------------------
        # copy raw fasta file to DIR_PROJECT_RAWDATA
        local rawcp="$DIR_PROJECT_RAWDATA/imported_${label}"
        local newmap="${newfasta}.maptxt"
        if [[ $samefiles -eq 0 ]]; then
            verbose_info_msg "copy/link fasta file $srcfasta to $rawcp"
            copy_file "$flag" "$rawcp" "$srcfasta"
            [[ $? -le 1 ]] || return 255

            # relabel raw fasta
            if [[ $do_relabel -eq 1 ]]; then
                verbose_info_msg "relabel fasta file to label $label, file: $impfasta"
                $PYTHON3 $DIR_ORTHOLOGER_BIN/relabel_fasta.py -a -m "$newmap" -f "$impfasta" -l $label > "$newfasta"
                [[ $? -eq 0 ]] || {
                    error_msg "failed relabel of fasta $impfasta"
                    rm -f "$newmap"
                    return 255
                }                
            else
                # do not relabel
                # check that label in file actually matches the expectation
                local foundlabel="";
                local ec=0
                foundlabel=$($PYTHON3 $DIR_ORTHOLOGER_BIN/check_fasta.py --fasta "$impfasta" --label --spsep ":")
                ec=$?
                [[ $ec -eq 0 ]] || {
                    error_msg "fasta file seems to be bad: $impfasta"
                    error_msg "maybe the file needs relabling (add '+' before label in fasta definition file)"
                    rm -f "$rawcp" >& /dev/null # remove file imported_xxx
                    return 255
                }
                if [[ "$foundlabel" != "$label" ]]; then
                    foundlabel=${foundlabel^^}
                    if [[ "$foundlabel" == "$label" ]]; then
                        error_msg "found label ($foundlabel) matches expected ($label) only if it is converted to uppercase"
                        rm -f "$rawcp" >& /dev/null # remove file imported_xxx
                        return 255
                    else
                        error_msg "found label ($foundlabel) does not match expected ($label) - is it relabeled or is the given label wrong?"
                        rm -f "$rawcp" >& /dev/null # remove file imported_xxx
                        return 255
                    fi
                fi
                
                if [[ $flag -eq 0 ]]; then
                    #link to original since rawcp is linked to original
                    verbose_info_msg "soft link original fasta file $impfasta to $newfasta"
                    ln -sf "$newfasta" "$impfasta" || fatal_msg "failed soft link <$newfasta> to <$impfasta>"

                    # we dont need rawcp anymore
                    rm -f "$rawcp" >& /dev/null
                else
                    # link to copy - do using basename of rawcp as it is in the same directory
                    local brawcp=$(basename "$impfasta")
                    local bnewfasta=$(basename "$newfasta")
                    verbose_info_msg "soft link fasta file $brawcp to $bnewfasta"
                    (
                        cd $(dirname "$newfasta")
                        ln -sf "$bnewfasta" "$brawcp"
                    )
                    [[ $? -le 1 ]] || fatal_msg "failed soft link <$bnewfasta> to <$brawcp>"
                fi
            fi
        fi
        # copy without overwriting target
        local ec=0
        local newfasta_rel="../Rawdata/$(basename $newfasta)"
        (
            cd $DIR_PROJECT_SEQUENCES
            ln -sf  "$newfasta_rel" "$(basename $seqfasta)" || { ec=$?; error_msg "unable to link $seqfasta with $newfasta"; }
        )
        return 0
    }

    #--------------------------------------------------
    # import raw fasta files with a given label
    #
    # This will copy (link) the raw fasta file
    # to DIR_PROJECT_RAWDATA and relabel it using <label>.
    #
    # arguments: <flag> <raw fasta file> <label>
    #
    #    <flag>    : same as for copy_file
    #    <fasta>   : file with labels and raw fasta files
    #
    # A link is then created in DIR_PROJECT_SEQUENCES .
    #--------------------------------------------------
    export N_IMPORTED=0
    import_fasta_from_source() {
        N_IMPORTED=0
        [[ $# -eq 2 ]] || return 255
        local flag=$1
        
        # input file specs
        local inputFile="$2"
        parse_input_file "$inputFile" || { error_msg "error parsing input filename : $inputFile"; return 255; }

        local nfiles=${#FASTA_FILES[@]}
        local index=0
        local nok=0
        local nfail=0
        for (( index=0; index<nfiles; index++ )); do
            local f=${FASTA_FILES[$index]}
            local t=${FASTA_TAXIDS[$index]}
            local m=${FASTA_REMAP_IDS[$index]}
            
            # import fasta file
            # it will check if file already exists , import_fasta outputs a warning and returns 0
            # hence, the "imported raw fasta ..." msg will be displayed in such a case
            import_fasta "$m" "$f" "$t"
            local ec=$?
            debug_msg "import_fasta returned ec=$?"
            [[ $ec -eq 0 ]] && { info_msg "imported raw fasta with label $t and file $f"; nok=$((nok+1)); }
            [[ $ec -gt 1 ]] && nfail=$((nfail+1))
        done

        # output info
        info_msg "imported $nok files"
        
        local rval=0
        if [[ $nfail -ne 0 ]]; then
            warn_msg "failed importing $nfail files"
            warn_msg "you can re-load the files using:"
            warn_msg " $CMD_MANAGE -f $inputFile"
            echo
            rval=0
        fi

        N_IMPORTED=$nok
        
        return $rval
    }

    #--------------------------------------------------
    # parse a rawfasta file and write to stdout the
    # labels found (without leading '+')
    #--------------------------------------------------
    rawfasta2todo() {
        [[ $# -eq 1 ]] || { error_msg "rawfasta2todo: no rawfasta file is given"; return 255; }
        local fraw="$1"
        [[ -f "$fraw" ]] || { error_msg "raw fasta list file is not readable ($fraw)"; return 255; }
        [[ -s "$fraw" ]] || { error_msg "raw fasta list file is empty ($fraw)";        return 255; }

        local line=""
        local words=()
        # read per line
        while IFS= read -r line || [[ -n "$line" ]]; do
            check_line "$line" || continue

            # split in words according to ws
            eval $(create_array_ws words "$line")
            
            # first word is the label, remove any leading '+'
            label=${words[0]}
            label=${label##*+}
            
            # do not do uppercase label!!
            echo $label
        done<"$fraw" | sort -u
    }

    #--------------------------------------------------
    # create or update todo file
    #--------------------------------------------------
    create_todo() {
        [[ $# -eq 2 ]] || { error_msg "create_todo: missing argument(s)"; return 255; }
        local ftodo=$1    # todo file
        local fimport=$2  # fasta import file (e.g test.txt)
        local dirname_todo=$(dirname $ftodo)
        [[ -d $dirname_todo ]] || mkdir -p $dirname_todo
        local ftmptodo=""
        tmp_make ftmptodo "todo"
        # create a todo file
        rawfasta2todo "$fimport" > "$ftmptodo"
        
        # check for bad labels - will exit if bad is found
        check_todo_special_labels "$ftmptodo"  || fatal_msg "import file ($fimport) contains a bad label (TPA) - it has a special meaning for the blast tools and it will cause the pipeline to break - please change label!"
        
        # move to todo file
        mv -f "$ftmptodo" "$ftodo"
        chmod 644  "$ftodo"
    }
    
    #--------------------------------------------------
    # import (copy or link) sequence files from source
    # arguments: <flag> <src> <onlyfasta>
    #  <flag> : 0 -> make link
    #           1 -> copy
    #  <src>  : source project directory
    #
    # Files are imported from <src>/Sequences .
    # They are defined by the used TODO file.
    #
    # TODO: this does not handle filenames with spaces!
    #--------------------------------------------------
    import_sequences() {
        [[ $# -ne 3 ]] && return 1
        local flag=$1
        local src="$2"
        [[ -d "$src" ]] || { error_msg "given source does not exist"; return 2; }

        local onlyfasta=$3

        local nspec=${#PL_TODO_LABELS[@]}
        [[ $nspec -eq 0 ]] && { error_msg "empty species list in TODO file $PL_TODO"; return 3; }

        src=$(expand_name "$src")

        local fsrc=""
        local trg=""
        local nok=0

        # loop over all species and copy Sequences
        local A=
        for A in "${PL_TODO_LABELS[@]}"; do
            local seqf="${src}/${A}"
            # fasta files
	        fsrc="${seqf}.${EXT_FASTA}"
	        trg="${DIR_PROJECT_SEQUENCES}"
            copy_file "$flag" "$trg" "$fsrc"
            [[ $? -eq 0 ]] && nok=$((nok+1))

            # check if only fasta files should be considered
            [[ $onlyfasta -eq 0 ]] || continue

            # fasta cleaned files
	        fsrc="$(ls ${seqf}.$EXT_FASTA_CLEAN 2>/dev/null)"
	        trg="${DIR_PROJECT_SEQUENCES}"
            copy_file "$flag" "$trg" "$fsrc"
            [[ $? -eq 0 ]] && nok=$((nok+1))

            # fasta CDHIT files
	        trg="${DIR_PROJECT_SEQUENCES}"
            for fsrc in ${seqf}.${EXT_FASTA_SELECTED}*; do
                copy_file "$flag" "$trg" "$fsrc"
                [[ $? -eq 0 ]] && nok=$((nok+1))
            done

            # fasta_selected stats files
	        fsrc="$(ls ${seqf}.$EXT_FASTA_STATS 2>/dev/null)"
	        trg="${DIR_PROJECT_SEQUENCES}"
            copy_file "$flag" "$trg" "$fsrc"
            [[ $? -eq 0 ]] && nok=$((nok+1))


            # fasta_selected seqlen files
	        fsrc="$(ls ${seqf}.$EXT_FASTA_SEQLEN 2>/dev/null)"
	        trg="${DIR_PROJECT_SEQUENCES}"
            copy_file "$flag" "$trg" "$fsrc"
            [[ $? -eq 0 ]] && nok=$((nok+1))

            # masked + DB files
	        fsrc="$(ls ${seqf}.${EXT_FASTA_MASKED} 2>/dev/null)"
	        trg="${DIR_PROJECT_SEQUENCES}"
            copy_file "$flag" "$trg" "$fsrc"
            [[ $? -eq 0 ]] && nok=$((nok+1))

            # similarity inparalogs
	        fsrc="$(ls ${seqf}.${EXT_INPAR_SELECTED} 2>/dev/null)"
	        trg="${DIR_PROJECT_SEQUENCES}"
            copy_file "$flag" "$trg" "$fsrc"
            [[ $? -eq 0 ]] && nok=$((nok+1))

        done

        info_msg "imported $nok files"
    }

    #--------------------------------------------------
    # Import sequence data from source
    # arguments: <flag> <src> [ext1,ext2,...]
    #  <flag> : 0 -> make link
    #           1 -> copy
    #  <src>  : source project directory
    #  <ext>  " extensions to be used
    #
    # Files are imported from <src>/PWC .
    # They are defined by the used TODO file.
    #--------------------------------------------------
    import_pwc() {

        [[ $# -lt 2 ]] && return 1
        local flag=$1
        local src="$2"
        [[ -d "$src/PWC" ]] || { error_msg "given PWC source does not exist <$src/PWC>"; return 2; }
        local exts=""
        [[ $# -eq 3 ]] && exts="$3"
        
        local nspec=${#PL_TODO_LABELS[@]}
        [[ $nspec -eq 0 ]] && return 3
        info_msg "---------------------------------------------------------------------------------"
        info_msg " NOTE: The import_pwc only imports half of the alignment matrix."
        info_msg "---------------------------------------------------------------------------------"

        src=$(expand_name "$src")

        if [[ ${#exts} -eq 0 ]]; then
            exts="${EXT_ALIGN},${EXT_BRH},${EXT_INPAR}"
            ###        exts="${EXT_ALIGN},${EXT_ALIGNBEST},${EXT_BRH},${EXT_INPAR}"
        fi
        local extarr=( $(tokenize "," "$exts") )
        local nok=0

        CHECK_COUNTER=0
        MAX_MESSAGES=20
        # Do pairwise files
        local fsrc=
        local fileAB=
        local fileBA=

        # Loop over extensions
        for ext in ${extarr[@]}; do
            local ibofs=0
            [[ $ext == $EXT_ALIGN ]]     && ibofs=0
            [[ $ext == $EXT_BRH ]]       && ibofs=1
            [[ $ext == $EXT_INPAR ]]     && ibofs=1

            info_msg "Importing <$ext> from PWC"
            
            local nmiss=0
            local ec=0
            # Loop over A
            local indA=0
            for (( indA=0; indA<nspec; indA++ )); do
                local A=${PL_TODO_LABELS[$indA]}
                get_pwc_dir "$A"
                local trgAB="R_PWC_DIR"
                [[ -d $trgAB ]] || mkdir $trgAB

                local nd=$((indA%20))
                (( indA == 0 )) && nd=1
                if (( nd == 0 )); then
                    info_msg "processed $indA out of $nspec"
                fi
                # Loop over B
                local indB=0
                for (( indB=indA+ibofs; indB<nspec; indB++ )); do
                    local B=${PL_TODO_LABELS[$indB]}
                    get_pwc_dir "$B"
                    local trgBA="R_PWC_DIR"
                    [[ -d $trgBA ]] || mkdir $trgBA

                    # external alignment file [A,B]
                    fsrc="$(get_external_pwc_file ${A} ${B} ${ext} $src)"

                    if [[ -f $fsrc ]]; then
                        # AB is OK
                        copy_file $flag "$trgAB" "$fsrc"
                        ec=$?
                        [[ $ec -le 1 ]] || {
                            cnt_error_msg "failed copying file <$fsrc> to <$trgAB>"
                            continue
                        }
                        [[ $ec -eq 0 ]] && nok=$((nok+1))
                    else
                        error_msg "missing file for ${A}_${B}.$ext in dir $src/PWC"
                        nmiss=$((nmiss+1))
                        continue
                    fi
                done
            done
            [[ $nmiss -eq 0 ]] || warn_msg "for label $A - $nmiss missing files!"
        done
        info_msg "imported $nok files"
    }

    #--------------------------------------------------
    # Import cluster data
    #--------------------------------------------------
    import_cluster() {

        [[ $# -ne 2 ]] && return 1
        local flag=$1
        local src="$2"
        [[ -d "$src" ]] || { error_msg "given source does not exist"; return 2; }

        src=$(expand_name "$src")


        # loop over all extensions
        local nok=0
        local ext=
        local ec=0
        for ext in $EXT_CLUSTER $EXT_CLUSTER_RAW $EXT_CLUSTER_INPAR; do
            # get file source and target
            local f=$src/${PL_TODO_NAME}.$ext
            local trg=$DIR_PROJECT_CLUSTER

            copy_file "$flag" "$trg" "$f"
            ec=$?
            [[ $ec -le 1 ]] || {
                cnt_error_msg "failed copying file <$f> to <$trg>"
                continue
            }
            [[ $ec -eq 0 ]] && nok=$((nok+1))
        done
        info_msg "imported $nok files"
    }

    #--------------------------------------------------
    # Import data from source
    # arguments: <flag> <dirs> <src>
    #  <flag> : 0 -> make link
    #           1 -> copy
    #  <dirs> : csv string with containing
    #           FASTA,SEQUENCES,PWC or CLUSTER
    #           i.e it describes what to import
    #  <src>  : source project directory
    #
    # Files imported:
    #  FASTA     - fasta files from the given src dir
    #  SEQUENCES - files from src/Sequences
    #  PWC       - files from src/PWC
    #  CLUSTER   - files from src/Cluster
    #
    # They are defined by the used TODO file.
    #--------------------------------------------------
    import_data() {
        [[ $# -ne 3 ]] && { error_msg "import_data wrong nr of arguments $#"; return 1; }
        local flag=$1
        local csv="$2"
        local src="$3"
        [[ ${#csv} -eq 0 ]] && return 0
        [[ -d $src ]] || { error_msg "import source is not readable: $src"; return 2; }

        # translate flag
        fstr="copy"
        [[ $flag -eq 0 ]] && fstr="link"
        local ec=0

        local found_FASTA=0
        local found_SEQ=0
        local found_BRH=0
        local found_PWC=0
        local found_CLUS=0
        local found_ANY=0

        # parse csv string
        local all=$(parse_csv_dirs $csv)
        [[ ${#all} -eq 0 ]] && {
            error_msg "import_data source string contains unknown element(s)"
            error_msg "must be any of FASTA,SEQUENCES,PWC,BRH or CLUSTER (case insensitive, full string not required)"
            return 255
        }

        # loop over all elements
        local d=
        for d in $all; do
            # ignore strings previously found
            [[ "$d" == "FASTA"     ]] && [[ $found_FASTA -eq 1 ]] && continue
            [[ "$d" == "BRH"       ]] && [[ $found_BRH   -eq 1 ]] && continue
            [[ "$d" == "Sequences" ]] && [[ $found_SEQ   -eq 1 ]] && continue
            [[ "$d" == "PWC"       ]] && [[ $found_PWC   -eq 1 ]] && continue
            [[ "$d" == "Cluster"   ]] && [[ $found_CLUS  -eq 1 ]] && continue

            [[ "$d" == "FASTA"     ]] && [[ $found_SEQ   -eq 1 ]] && { warn_msg "ignored FASTA since SEQUENCES was already given"; continue; }
            [[ "$d" == "Sequences" ]] && [[ $found_FASTA -eq 1 ]] && { warn_msg "ignored SEQUENCES since FASTA was already given"; continue; }
            
            [[ "$d" == "BRH"   ]]     && [[ $found_PWC   -eq 1 ]] && { warn_msg "ignored BRH since PWC was already given"; continue; }
            [[ "$d" == "PWC"   ]]     && [[ $found_BRH   -eq 1 ]] && { warn_msg "ignored PWC since BRH was already given"; continue; }


            [[ "$d" == "FASTA" ]] && { f=1; found_FASTA=1; found_ANY=1; info_msg "importing ($fstr) fasta files from $src"; import_sequences $flag "${src}" 1; ec=$?; }
            [[ $ec -ne 0 ]] && { error_msg "import fasta returned error $ec"; break; }

            [[ "$d" == "Sequences" ]] && { f=1; found_SEQ=1; found_ANY=1; info_msg "importing ($fstr) from $src/Sequences"; import_sequences $flag "${src}/Sequences" 0; ec=$?; }
            [[ $ec -ne 0 ]] && { error_msg "import sequence returned error $ec"; break; }

            [[ "$d" == "PWC" ]]       && { f=1; found_PWC=1; found_ANY=1; info_msg "importing ($fstr) from $src/PWC"; import_pwc $flag "$src"; ec=$?; }
            [[ $ec -ne 0 ]] && { error_msg "import PWC returned error $ec"; break; }

            [[ "$d" == "BRH" ]]       && { f=1; found_BRH=1; found_ANY=1; info_msg "importing BRH's ($fstr) from $src/PWC"; import_pwc $flag "$src" "$EXT_BRH"; ec=$?; }
            [[ $ec -ne 0 ]] && { error_msg "import BRH returned error $ec"; break; }

            [[ "$d" == "Cluster" ]]   && { f=1; found_CLUS=1; found_ANY=1; info_msg "importing ($fstr) from $src/Cluster"; import_cluster $flag "$src/Cluster"; ec=$?; }
            [[ $ec -ne 0 ]] && { error_msg "import Cluster returned error $ec"; break; }
        done
        [[ $found_ANY -ne 0 ]] || {  error_msg "did not find any valid CSV string"; ec=255; }
        return $ec
    }

    # remove a single label from project
    # DANGEROUS FUNCTION - NOT YET CALLED ANYWHERE!
    remove_label() {
        fatal_msg "remove_label -> DO NOT USE FOR NOW!"
        
        [[ $# -ne 1 ]] && { error_msg_wrong_nargs "$#" "$FUNCNAME"; return 255; }
        local label="$1"
        local n=$(grep -c '^'$label'$' "$PL_TODO")
        [[ $n -eq 0 ]] && { info_msg "remove label: skip - label ($label) does not exist in given todo file ($PL_TODO)"; return 0; }
        [[ $n -ne 1 ]] && { info_msg "remove label: skip - label ($label) is ambiguous in given todo file ($PL_TODO)";   return 0; }
        # remove label
        # 1. from todo file
        # 2. Rawdata/${label}.fs*
        # 3. Rawdata/imported_${label}_*
        # 4. Sequences/${label}.*
        # 5. PWC/${label}_*.*
        # 6. PWC/*_${label}.*
        local ftmp=""
        tmp_make ftmp "rmlabel"
        
        # update current todo file
        grep -v '^'$label'$' "$PL_TODO" > "$ftmp"
        [[ $? -eq 0 ]] && {
            cp -f "$ftmp" "$PL_TODO"
            if [[ $? -eq 0 ]]; then
                info_msg "updated todo file $PL_TODO"
            else
                error_msg "failed updating todo file!"
            fi
            
        }
        # remove files from Rawdata
        rm -f $DIR_PROJECT_RAWDATA/${label}.fs*
        if [[ $? -eq 0 ]]; then
            info_msg "removed files Rawdata/${label}.fs*"
        else
            error_msg "failed removing files Rawdata/${label}.fs* !"
        fi

        # remove files from Sequences
        rm -f $DIR_PROJECT_SEQUENCES/${label}.*
        if [[ $? -eq 0 ]]; then
            info_msg "removed files Sequences/${label}.*"
        else
            error_msg "failed removing files Sequences/${label}.* !"
        fi

        # remove from PWC
        rm -f $DIR_PROJECT_PWC/${label}/*
        if [[ $? -eq 0 ]]; then
            info_msg "removed files PWC/${label}/*"
        else
            error_msg "failed removing files PWC/${label}/* !"
        fi
        rm -f $DIR_PROJECT_PWC/*/*_${label}.*
        if [[ $? -eq 0 ]]; then
            info_msg "removed files PWC/*/*_${label}.*"
        else
            error_msg "failed removing files PWC/*/*_${label}.*"
        fi
        info_msg "NOTE ---> files in Cluster are NOT deleted <---"
        info_msg "If it's required, remove files manually:"
        info_msg "> rm -f Cluster/${PL_TODO_NAME}.*"
    }
    export LOADED_COMMON_CREATE_SH=0
fi
