#!/usr/bin/env bash
#
# orthologer - Pipeline for deducing orthology from fasta files.
# Copyright (C) 2005-2023 E. Zdobnov lab: F. Tegenfeldt
# <fredrik.tegenfeldt@unige.ch> based on code by E. Kriventseva
#
# This file is part of orthologer.
#
# orthologer is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# orthologer is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with orthologer.  If not, see <http://www.gnu.org/licenses/>.
#

set -u


help_args() {
    echo "./archive.sh -s <step(s)> [-t <todo file> -f -D <trgdir> -z]"
    echo "  -d               : dry run - don't do anything"
    echo "  -R               : run the archive, otherwise it will just print the setup"
    echo "  -t <todo file>   : todo file to use"
    echo "  -D <trg dir>     : target dir for archive file"
    echo "  -s <steps>       : select which steps to archive"
    echo "  -x <step>        : archive by directory for given label (ignored if -s is set)"
    echo "                   : a label is either a step id or any of the below"
    echo "                   : - PWCSHORT  - archives only .brh and .inpar files, respects -N and -S"
    echo "                   : - PWCFAST   - will archive everything under PWC, one archive per label, ignores -N and -S"
    echo "  -N <nmax>        : max nr of files per tar cmd (supersedes -l)"
    echo "  -S <size>        : size limit per tar file (eg 1K, 56G, etc)"
    echo "  -a <label>       : end label A - supersedes OP_LABELA_FIRST"
    echo "  -A <label>       : end label A - supersedes OP_LABELA_LAST"
    echo "  -b <label>       : end label B - supersedes OP_LABELB_FIRST"
    echo "  -B <label>       : end label B - supersedes OP_LABELB_LAST"
    echo "  -l               : package per label (PWC)"
    echo ""
    echo "  -T <method>      : archive method: t(ar) or m(ksquash)"
    echo "  -n <nthreads>    : nr of threads (mksquash)"
    echo "  -L               : (tar) do not dereference links in the tar archive"
    echo "                     by default the tar command uses --dereference in order"
    echo "                     to save the actual file and not just the link"
    echo "  -z <level>       : gzip archive file at given gzip level"
    echo "  -f               : force creation of archive"
    echo ""
    echo "  -p               : print all steps"
    echo "  -h               : this message"
    echo ""
    echo "To run a faster pairwise archive, use option -l and make sure that OP_LABELB_FIRST and OP_LABELB_LAST are empty."
    echo "The archive will then just loop over the PWC directories and save everything with the correct extension."
    echo "If OP_LABELB_xxx are set, they archiver will loop over all possible pairwise files."
    echo ""
}

[[ $# -eq 0 ]] && { help_args; exit 0; }

# make sure local common.sh file exists
[[ -x orthologer_conf.sh ]] || { echo "error: no orthologer_conf.sh found - still using old common.sh?" 1>&2; exit 255; }
source orthologer_conf.sh

# pre 4.3 did not support gzip levels
OLD_SQUASHFS=1


ARCH_TARGET_DIR=
ARCH_TARGET_NAME=

ARCH_FORCE=0
ARCH_FORCE_STEP=1
ARCH_STEPS=""
ARCH_STEPDIR=""
ARCH_PWC=0

ARCH_TARGET_FILE_BASE=
ARCH_TARGET_FILE=""
ARCH_TARGET_FILE_FINAL=""
ARCH_GZIP_LEVEL=0
ARCH_EXT=".tar"
ARCH_FINAL_EXT=".tar"
ARCH_GZIP_EXT=".gz"
ARCH_NMAX=100
ARCH_SIZE_LIMIT=0
ARCH_SIZE_LIMIT_HR="<NA>"

ARCH_PERLBL_PWC=0
TAR_DEREF=1
DRY_RUN=0
ARCH_DORUN=0

ARCH_METHOD=tar
ARCH_TAR_BIN=
ARCH_MKSQUASHFS_BIN=
ARCH_NTHREADS=1

TARLBL=""
TARCNT=0

# extra files to be added to the tar ball
TAREXTRAS=""

process_args() {
    OPTIND=1
    local method="tar"
    local opts=":hfdz:N:s:S:x:t:D:a:A:b:B:T:LlVRn:"
    while getopts $opts OPT; do
        case "$OPT" in
            h)  help_args
                exit 0
                ;;
            p)  print_steps_table
                exit 0
                ;;
            R)  ARCH_DORUN=1
                ;;
            t)  PL_TODO="$OPTARG"
                ;;
            d)  DRY_RUN=1
                ;;
            a)  OP_LABELA_FIRST=$OPTARG
                ;;
            A)  OP_LABELA_LAST=$OPTARG
                ;;
            b)  OP_LABELB_FIRST=$OPTARG
                ;;
            B)  OP_LABELB_LAST=$OPTARG
                ;;
            z)  ARCH_GZIP_LEVEL="$OPTARG"
                ;;
            T)  method="$OPTARG"
                ;;
            n)  ARCH_NTHREADS="$OPTARG"
                ;;
            D)  ARCH_TARGET_DIR="$OPTARG"
                ;;
            s)  ARCH_STEPS="$OPTARG"
                ;;
            x)  ARCH_STEPDIR="$OPTARG"
                ;;
            f)  ARCH_FORCE=1
                ;;
            N)  ARCH_NMAX=$OPTARG
                ;;
            S)  ARCH_SIZE_LIMIT=$(numfmt --from 'iec' $OPTARG 2>/dev/null)
                [[ $? -eq 0 ]] ||  { error_msg "size limit given by -S is not recognized"; return 255; }
                ARCH_SIZE_LIMIT_HR="$OPTARG"
                ;;
            l)  ARCH_PERLBL_PWC=1
                ;;
            L)  TAR_DEREF=0
                ;;
            V)  VERBOSE=1
                ;;
            
            *)  error_msg "unknown option ($OPTARG)"
                return 255
                ;;
        esac
    done
    
    # if size limit is set, reset Nmax limit
    (( ARCH_SIZE_LIMIT == 0 )) || ARCH_NMAX=0
    
    # check method
    method=${method,,}
    [[ ${method:0:1} == "t" ]] && ARCH_METHOD="tar"
    [[ ${method:0:1} == "m" ]] && ARCH_METHOD="mksquashfs"
    [[ -n $ARCH_METHOD ]] || { error_msg "error: invalid archive method <$method> - should be t(ar) or m(ksquashfs)"; return 255; }

    if [[ $ARCH_METHOD == "tar" ]]; then
        info_msg "package using tar"
        ARCH_TAR_BIN=$(/usr/bin/which tar)
        [[ -x $ARCH_TAR_BIN ]] || { error_msg "missing archive binary for method <$ARCH_METHOD>"; return 255; }
        ARCH_EXT=".tar"
        ARCH_FINAL_EXT=".tar"
        [[ $ARCH_GZIP_LEVEL == "0" ]] || ARCH_FINAL_EXT=".tar.gz"
    else
        info_msg "package using mksquashfs"
        ARCH_MKSQUASHFS_BIN=$(/usr/bin/which mksquashfs)
        [[ -x $ARCH_MKSQUASHFS_BIN ]] || { error_msg "missing archive binary for method <$ARCH_METHOD>"; return 255; }
        ARCH_EXT=".sqfs"
        ARCH_FINAL_EXT=".sqfs"
        ####[[ $ARCH_GZIP_LEVEL == "0" ]] && ARCH_GZIP_LEVEL="1"
    fi

    # check -s and -x
    [[ -n $ARCH_STEPS ]] && [[ -n $ARCH_STEPDIR ]] && { warn_msg "ignoring option -x as -s is set"; ARCH_STEPDIR=""; }
    
    # steps is required
    [[ -z $ARCH_STEPS ]] && [[ -z $ARCH_STEPDIR ]] && { error_msg "missing required option -s <step(s)> or -x <dir>"; return 255; }

    # set nmax to 0 if per label
    ###[[ $ARCH_PERLBL_PWC -eq 1 ]] && ARCH_NMAX=0

    # make sure target dir is there
    [[ ${#ARCH_TARGET_DIR} -gt 0 ]] || ARCH_TARGET_DIR=/tmp

    # check options
    [[ -d $ARCH_TARGET_DIR ]] || { error_msg "unable to read target directory (-D) <$ARCH_TARGET_DIR>"; return 255; }

    # check target dir
    iswriteable "$ARCH_TARGET_DIR"  || { error_msg "unable to write in target directory (-D) <$ARCH_TARGET_DIR>"; return 255; }

    # check todo file
    [[ -f $PL_TODO ]] || { error_msg "todo file $PL_TODO not found"; return 255; }
    
    # setup environment...
    setup_environment || return 255

    # make target name
    ARCH_TARGET_NAME="archive_${PL_TODO_NAME}"

    # check if we should run or not
    [[ $ARCH_DORUN -eq 1 ]] || { print_setup; return 1; }

    # steps... or directory
    if [[ -n $ARCH_STEPS ]]; then
        expand_step_list $ARCH_STEPS
        ARCH_STEPS="${STEP_ID_LIST[@]}"
        [[ $? -eq 0 ]] || { error_msg "failed expanding step list $ARCH_STEPS"; return 255; }
    else
        local D=$ARCH_STEPDIR
        [[ ${ARCH_STEPDIR:0:3} == "PWC" ]] && D=${ARCH_STEPDIR:0:3}
        [[ -d $D ]] || { error_msg "directory given by -x is not readable (exists?) - $D"; return 255; }
    fi
    
    return 0
}

yesno() {
    [[ $# -eq 1 ]] || return 0
    local str="NO"
    [[ $1 -eq 0 ]] || str="YES"
    echo $str
}

print_setup() {
    local fmt="%-40s%-20s\n"

    echo   "========= SETUP ========="
    printf "$fmt" "TODO file" "$PL_TODO"
    printf "$fmt" "Nr of labels" "${#PL_TODO_LABELS[@]}"
    printf "$fmt" "Label A start" "$OP_LABELA_FIRST"
    printf "$fmt" "Label A end"   "$OP_LABELA_LAST"
    printf "$fmt" "Label B start" "$OP_LABELB_FIRST"
    printf "$fmt" "Label B end"   "$OP_LABELB_LAST"
    printf "$fmt" "Archive steps" "$ARCH_STEPS"
    printf "$fmt" "Archive directory" "$ARCH_STEPDIR"
    printf "$fmt" "Archive target directory" "$ARCH_TARGET_DIR"
    printf "$fmt" "Max nr of files per archive" "$ARCH_NMAX"
    printf "$fmt" "Max size per archive (pre gzip)" "$ARCH_SIZE_LIMIT_HR"
    printf "$fmt" "Archive per label (pairwise)" "$(yesno $ARCH_PERLBL_PWC)"
    printf "$fmt" "Dereference links" "$(yesno $TAR_DEREF)"
    printf "$fmt" "Force write of archive" "$(yesno $ARCH_FORCE)"
    printf "$fmt" "Gzip archive" "$(yesno $ARCH_GZIP_LEVEL)($ARCH_GZIP_LEVEL)"
    printf "$fmt" "Verbose output"    "$(yesno $VERBOSE)"
    echo
    echo   "Add option -R to actually run the archiving"
    echo

}

set_archive_name() {
    local stepid="$STEP_ID"
    [[ $# -eq 1 ]] && stepid="$1"
    local ext=".tar"
    local cnt=""
    local tarlbl=""
    [[ ${#TARLBL} -eq 0 ]] || tarlbl="_${TARLBL}"
    [[ $TARCNT -eq -1 ]] || cnt=$(printf "_%06d" $TARCNT)
    local base="${ARCH_TARGET_FILE_BASE}_${stepid}${tarlbl}${cnt}"
    ARCH_TARGET_FILE="$base$ARCH_EXT"
    ARCH_TARGET_FILE_FINAL="$base$ARCH_FINAL_EXT"
}

check_archive_files() {
    if [[ $ARCH_FORCE -gt 0 ]]; then
        [[ -f $ARCH_TARGET_FILE       ]] && rm -f $ARCH_TARGET_FILE
        [[ -f $ARCH_TARGET_FILE_FINAL ]] && rm -f $ARCH_TARGET_FILE_FINAL
        [[ -f ${ARCH_TARGET_FILE}.gz  ]] && rm -f ${ARCH_TARGET_FILE}.gz
    else
        [[ -f $ARCH_TARGET_FILE       ]] && { warn_msg "archive file exists already - use -f option to force overwrite <$ARCH_TARGET_FILE>";       return 1; }
        [[ -f $ARCH_TARGET_FILE_FINAL ]] && { warn_msg "archive file exists already - use -f option to force overwrite <$ARCH_TARGET_FILE_FINAL>"; return 1; }
        [[ -f ${ARCH_TARGET_FILE}.gz  ]] && { warn_msg "archive file exists already - use -f option to force overwrite <${ARCH_TARGET_FILE}.gz>";  return 1; }

    fi
    return 0
}

conv_size() {
    local s=$1
    numfmt --from 'iec' $s
}

file_size() {
    [[ $# -eq 1 ]] || { echo "0"; return 0; }
    [[ -f "$1" ]] || { echo "0"; return 0; }
    stat -L -c '%s' "$1"
}

do_save() {
    [[ $# -eq 2 ]] || return 0
    local nfiles="$1"
    local fsize="$2"
    
    # quit if ARCH_NMAX is zero -> do NOT save!
    ###[[ $ARCH_NMAX -eq 0 ]] && return 1

    # if nfiles >= ARCH_NMAX -> do save!
    [[ $ARCH_NMAX -gt 0 ]] && [[ $nfiles -ge $ARCH_NMAX ]] && return 0

    # if fsize large enough
    [[ $ARCH_SIZE_LIMIT -gt 0 ]] && [[ $fsize -ge $ARCH_SIZE_LIMIT ]] && return 0
    
    # do NOT save
    return 1
}

ARCH_NFILES=0
ARCH_NFILES_MSG=""
get_arch_nfiles() {
    ARCH_NFILES=0
    ARCH_NFILES_MSG=""
    [[ $# -eq 2 ]] || return 0
    local tot_nfiles="$1"
    local tot_fsize="$2"
    
    ARCH_NFILES=$tot_nfiles
    if   [[ $ARCH_NMAX -gt 0 ]]; then
        if (( tot_nfiles > $ARCH_NMAX )); then
            ARCH_NFILES=$ARCH_NMAX
            ARCH_NFILES_MSG="nr of files"
        fi
    elif [[ $ARCH_SIZE_LIMIT -gt 0 ]]; then
        if (( tot_fsize > 0 )); then
            # this assumes files sizes are distributed evenly...
            if (( tot_fsize > ARCH_SIZE_LIMIT )); then
                ARCH_NFILES=$(calc "int(1+${nfiles}.0*${ARCH_SIZE_LIMIT}.0/${tot_fsize}.0)")
                ARCH_NFILES_MSG="total size of files"
            fi
        fi
    fi
}

init_archive() {
    [[ ${#ARCH_TARGET_FILE_BASE} -gt 0 ]] || { error_msg "init_archive() missing tar file base name"; return 255; }

    # reset tar file counter
    TARCNT=0
    TARLBL=""
}

# tar the given
mksqfscmd(){
    [[ $# -gt 0 ]] || return 0
    local tfile="$1"
    local tmpfile=""
    
    # check if file exists and allows for append
    local targs="-no-xattrs -processors $ARCH_NTHREADS -Xcompression-level $ARCH_GZIP_LEVEL -comp gzip -no-progress -noappend"
    [[ $OLD_SQUASHFS == 1 ]] && targs="-no-xattrs -processors $ARCH_NTHREADS -comp gzip -no-progress -noappend"
    
    # if dry run, just output command
    [[ $DRY_RUN -eq 0 ]] || { echo "mksquashfs <files in $tfile> $ARCH_TARGET_FILE_FINAL $targs"; return 0; }

    tmp_make tmpfile "archive_squashfs"  || fatal_msg "failed creating tmp file at $tmpfile"
    
    local nfiles=$(wc -l $tfile | cut -f 1 -d " ")
    info_msg "adding $nfiles files to <$ARCH_TARGET_FILE_FINAL>"

    # actually run the archive
    local ec=0
    $ARCH_MKSQUASHFS_BIN $(cat $tfile) "$tmpfile" $targs
    ec=$?
    if [[ $ec -eq 0 ]]; then
        mv -f "$tmpfile" "$ARCH_TARGET_FILE_FINAL"
        info_msg "mksquashfs command ok : mksquashfs <files in $tfile> $ARCH_TARGET_FILE_FINAL $targs"
        chmod 644 "$ARCH_TARGET_FILE_FINAL"
        return 0
    fi
    rm -f "$tmpfile"
    error_msg "failed mksquashfs command : mksquashfs <files in $tfile> $ARCH_TARGET_FILE_FINAL $targs"
    return 255
}

mksqfscmd_dir() {
    [[ $# -gt 0 ]] || return 0
    local tdir="$1"
    [[ -d $tdir ]] || { error_msg "unable to read source directory <$tdir>"; return 255; }

    local targs="-no-xattrs -processors $ARCH_NTHREADS -comp gzip -Xcompression-level $ARCH_GZIP_LEVEL -no-progress -noappend"
    [[ $OLD_SQUASHFS == 1 ]] && targs="-no-xattrs -processors $ARCH_NTHREADS -comp gzip -no-progress -noappend"
    
    # if dry run, just output command
    [[ $DRY_RUN -eq 0 ]] || { echo "mksquashfs $targs" "$tdir"; return 0; }

    # create output file
    local tmpfile=""
    tmp_make tmpfile "archive_squashfs"

    # squash it
    local ec=0
    $ARCH_MKSQUASHFS_BIN $tdir $tmpfile $targs
    ec=$?
    if [[ $ec -eq 0 ]]; then
        mv -f "$tmpfile" "$ARCH_TARGET_FILE_FINAL"
        info_msg "mksquashfs command ok : mksquashfs $tdir $ARCH_TARGET_FILE_FINAL $targs"
        chmod 644 "$ARCH_TARGET_FILE_FINAL"
        return 0
    fi
    rm -f "$tmpfile"
    error_msg "failed mksquashfs_dir command : mksquashfs $tdir $archfile $targs"
    return 0
}

# tar the given
tarcmd(){
    [[ $# -gt 0 ]] || return 0
    local tfile="$1"

    # check if file exists and allows for append
    # -c -> create archive (no append)
    local targs="-cf"

    # dereference links??
    [[ $TAR_DEREF -eq 1 ]] && targs="--dereference $targs"

    
    # if dry run, just output command
    [[ $DRY_RUN -eq 0 ]] || { echo "tar $targs" "$ARCH_TARGET_FILE" "--files-from \"$tfile\""; return 0; }

    local nfiles=$(wc -l $tfile | cut -f 1 -d " ")
    info_msg "adding $nfiles files to <$ARCH_TARGET_FILE>"

    # actually run the archive
    local ec=0
    local tmpfile=""
    tmp_make tmpfile "archive_tar"
    if [[ $ARCH_GZIP_LEVEL -gt 0 ]]; then
        $ARCH_TAR_BIN $targs - --files-from "$tfile" | gzip -$ARCH_GZIP_LEVEL > "$tmpfile"
        ec=$?
    else
        $ARCH_TAR_BIN $targs - --files-from "$tfile" > "$tmpfile"
        ec=$?
    fi
    if [[ $ec -eq 0 ]]; then
        # move tmp file to target
        mv -f "$tmpfile" "$ARCH_TARGET_FILE_FINAL"
        info_msg "tar command ok : tar $targs $ARCH_TARGET_FILE_FINAL --files-from $tfile"
        chmod 644 "$ARCH_TARGET_FILE_FINAL"
        return 0
    fi
    error_msg "failed tar command : tar $targs $ARCH_TARGET_FILE_FINAL --files-from $tfile"
    return 255
}

#
# tars an entire directory given the extensions
# it does the gzipping in the tar cmd if requested
# it also opies the file to its final destination
#
tarcmd_dir() {
    [[ $# -gt 0 ]] || return 0
    local tdir="$1"
    [[ -d $tdir ]] || { error_msg "unable to read source directory <$tdir>"; return 255; }
    shift
    local extensions="$*"
    local wildcards=""
    for x in $extensions; do
        wildcards="$wildcards $tdir/"'*'".$x"
    done
    [[ ${#wildcards} -gt 0 ]] || wildcards=$tdir/'*.*'

    local targs="-cf"
    
    # dereference links??
    [[ $TAR_DEREF -eq 1 ]] && targs="--dereference $targs"
    
    info_msg "adding files $wildcards to <$ARCH_TARGET_FILE>"
    # if dry run, just output command
    [[ $DRY_RUN -eq 0 ]] || { echo "tar $targs" "$ARCH_TARGET_FILE" "$wildcards"; return 0; }

    # create output file
    local tmptar=""
    tmp_make tmptar "archive_tar"

    # create the tarball
    local ec=0
    if [[ $ARCH_GZIP_LEVEL -gt 0 ]]; then
        $ARCH_TAR_BIN $targs - $wildcards | gzip -$ARCH_GZIP_LEVEL > "$tmptar" || return 255
        ec=$?
    else
        $ARCH_TAR_BIN $targs - $wildcards > "$tmptar" || return 255
        ec=$?
    fi

    # copy file
    if [[ $ec -eq 0 ]]; then
        # move tmp file to target
        mv -f "$tmptar" "$ARCH_TARGET_FILE_FINAL" || return 255
        chmod 644 "$ARCH_TARGET_FILE_FINAL"
        return 0
    fi
    
    rm -f "$tmptar"
    error_msg "failed tar command : tar $targs $ARCH_TARGET_FILE_FINAL $wildcards"
    return 255
}

archivecmd() {
    [[ $# -eq 1 ]] || { error_msg "archivecmd: not enough args $#"; return 255; }
    local tfile="$1"
    [[ -f $tfile ]] || { error_msg "archivecmd: missing file $tfile"; return 255; }
    
    if [[ $ARCH_METHOD == "tar" ]]; then
        #tarcmd "$tmpfiles"
        tarcmd "$tfile"
        ec=$?
    else
        #mksqfscmd "$tmpfiles"
        mksqfscmd "$tfile"
        ec=$?
    fi
    return $ec
}

archivecmd_dir() {
    [[ $# -ge 1 ]] || { error_msg "archivecmd_dir: not enough args $#"; return 255; }
    local tdir="$1"
    shift
    local extensions="$*"
    if [[ $ARCH_METHOD == "tar" ]]; then
        tarcmd_dir "$tdir" $extensions
        ec=$?
    else
        mksqfscmd_dir "$tdir"
        ec=$?
    fi
    return $ec
}


# arguments: <file1|dir1> ...
archive_files() {
    [[ $# -ge 1 ]] || return 255
    local files="$*"
    [[ ${#files} -gt 0 ]] || return 255

    local nfiles=0
    local nmissing=0
    local fsize=0
    # tmp file with names of all files to be included
    local tmpfiles=""
    tmp_make tmpfiles "archive_files"
    local f=
    for f in $files; do
        local fsz=0
        if [[ -f "$f" ]]; then
            echo "$f"
            fsz=$(stat -L -c '%s' "$f")
        elif [[ -d "$f" ]]; then
            find -f "$f" -mindepth 1
            fsz=$(du -sB1 "$f" --apparent-size | awk '{print$1;}')
        else
            nmissing=$((nmissing+1))
            verbose_error_msg "skipping missing file <$f>"
            continue
        fi

        fsize=$((fsize+fsz))
        nfiles=$((nfiles+1))
    done | sort > "$tmpfiles"
    [[ $nfiles -gt 0 ]]   || { rm -f "$tmpfiles"; warn_msg "no files found for step $STEP_ID"; return 0; }
    [[ $nmissing -eq 0 ]] || warn_msg "missing $nmissing files in step $STEP_ID"

    local tmpprefix="$tmpfiles"

    # get nr of files per chunk
    get_arch_nfiles "$nfiles" "$fsize"
    info_msg "got $nfiles file(s) with total size of $((fsize/1024/1024))MB"

    # split files into chunks
    if (( ARCH_NFILES > 1 )); then
        info_msg "will split file list based on $ARCH_NFILES_MSG in $ARCH_NFILES per chunk"
        tmpprefix="${tmpfiles}."
        split -n "$ARCH_NFILES" "$tmpfiles" "$tmpprefix"
        info_msg "will split file list in $ARCH_NFILES per chunk"
    else
        info_msg "will create one archive file"
    fi
    
    
    # loop over file(s)
    local ec=0
    TARCNT=0
    for f in ${tmpprefix}*; do
        set_archive_name
        if check_archive_files; then
            info_msg "archiving to $ARCH_TARGET_FILE"
            archivecmd "$f"
            ec=$?
            [[ $ec -eq 0 ]] || { error_msg "failed archiving to $ARCH_TARGET_FILE"; break; }
            rm -f "$f"
        else
            info_msg "skipping label $TARLBL - archive exists"
        fi
        TARCNT=$((TARCNT+1))
    done

    return $ec
}

archive_dir() {
    [[ $# -ge 2 ]] || return 255
    local dir="$1"
    local maxdepth="$2"
    [[ -d $dir ]] || { error_msg "given directory is not readable: $dir"; return 255; }

    local extensions=""
    shift
    shift
    if [[ $# -ge 1 ]]; then
        extensions="$*"
    fi

    local basedir=$(basename "$dir")
    local nfiles=0
    local fsize=0
    local deptharg="-mindepth 1 -maxdepth $maxdepth"
    [[ "$maxdepth" == "0" ]] && deptharg="-mindepth 1"
    
    # tmp file with names of all files to be included
    local tmpfiles=""
    tmp_make tmpfiles "archive_dir"

    if [[ -z $extensions ]]; then
        # if extensions are NOT given
        find "$dir/" $deptharg -type f | sort > $tmpfiles || return 255
        fsize=$(du -sB1 "$dir/" --apparent-size | awk '{print$1;}')
    else
        # if extensions are given
        local arg=""
        set -o noglob
        for ext in $extensions; do
            [[ -z "$arg" ]] || arg="$arg -or "
            arg="$arg -name "'*.'"$ext"
        done
        find "$dir" $deptharg -type f \( $arg \) | sort > $tmpfiles
        set +o noglob
        
        if (( ARCH_SIZE_LIMIT > 0 )); then
            local allDone=0
            local line=
            local saveIFS=$IFS
            IFS=$'\n'
            until (( allDone == 1 )); do
                read line || allDone=1
                [[ -n $line ]] || continue
                fsz=$(stat -L -c '%s' "$line")
                fsize=$((fsize+fsz))
            done < "$tmpfiles"
            IFS=$saveIFS
        fi
    fi

    # check nr of files
    nfiles=$(cat $tmpfiles | wc -l)
    [[ $nfiles -gt 0 ]]   || { rm -f "$tmpfiles"; warn_msg "no files found in dir $dir"; return 0; }

    # decide if collection in tmpfiles should be split
    local tmpprefix="$tmpfiles"

    # get nr of chunks
    get_arch_nfiles "$nfiles" "$fsize"
    info_msg "got $nfiles file(s) with total size of $((fsize/1024/1024))MB"
    # split files into chunks
    if (( ARCH_NFILES > 1 )); then
        info_msg "will split file list based on $ARCH_NFILES_MSG in $ARCH_NFILES files per chunk"
        tmpprefix="${tmpfiles}."
        split -l "$ARCH_NFILES" "$tmpfiles" "$tmpprefix"
    else
        info_msg "will create one archive file"
    fi
    
    info_msg "process all chunks"
    # loop over file(s)
    local ec=0
    TARCNT=0
    for f in ${tmpprefix}*; do
        set_archive_name ${basedir^^}DIR
        if check_archive_files; then
            info_msg "archiving to $ARCH_TARGET_FILE"
            archivecmd "$f"
            ec=$?
            [[ $ec -eq 0 ]] || { error_msg "failed archiving to $ARCH_TARGET_FILE"; break; }
            rm -f "$f"
        else
            info_msg "skipping label $TARLBL - archive exists"
        fi
        TARCNT=$((TARCNT+1))
    done

    info_msg "done archiving directory $dir"
    return $ec
}






#SKIP_INIT=0
# arguments: <dir> <ext1> <ext2> <ext3> ...
archive_per_todo() {
    [[ $# -ge 1 ]] || return 255
    local bdir="$1"
    [[ -d "$bdir" ]] || return 255
    shift
    local extensions="$*"
    [[ ${#extensions} -gt 0 ]] || return 255
    
    # silly way to count nr of extensions
    local nexts=0
    for ext in $extensions; do
        nexts=$((nexts+1))
    done

    # tmp file with names of all files to be included
    local tmpfiles=""
    tmp_make tmpfiles "archive_per_todo"

    local nfiles=0
    local first=1

    local i=0
    local N=${#PL_TODO_LABELS[@]}
    local ilast=$((N-1))
    local ndone=0
    local nsaves=0
    local nmissing=0
    local fsize=0
    local fsz=0
    
    local nmaxfiles=$((N*nexts))
    for ((i=0; i<N; i++)); do
        local t="${PL_TODO_LABELS[$i]}"
        isok_label "A" $t || isok_label "B" $t || { debug_msg "skipping todo label $t"; continue; }
        for ext in $extensions; do
            local f="${bdir}/${t}.${ext}"
            [[ -f "$f" ]] || { nmissing=$((nmissing+1)); verbose_error_msg "skipping missing file <$f>"; continue; }
            echo "$f"
            nfiles=$((nfiles+1))
            fsz=$(file_size "$f")
            fsize=$((fsize+fsz))
        done  >> $tmpfiles
        # if no files found, skip
        [[ $nfiles -eq 0 ]] && continue
        ###[[ $i -lt $ilast ]] && [[ $nfiles -gt 0 ]] || continue

        # get tar label
        TARLBL="$t"

        local save=0
        do_save $nfiles $fsize && save=1
        [[ $i -eq $ilast ]] && save=1

        if [[ $save -eq 1 ]]; then
            # set tar file name
            set_archive_name
            info_msg "got nr of files = $nfiles - save flag = $save , file size = $fsize"
            
            # check tar files if they exist or are to be over written
            if check_archive_files; then
                
                # tar the files
                archivecmd $tmpfiles || return 255

                ndone=$((ndone+nfiles))
                nsaves=$((nsaves+1))
            else
                info_msg "skipping label $TARLBL - archive exists"
            fi
            

            # empty the file without removing it
            rm -f $tmpfiles && touch $tmpfiles
            
            nfiles=0
            fsize=0
            
            # move to next tar name, if not last
            ###[[ $i -eq $ilast ]] || TARCNT=$((TARCNT+1))
            TARCNT=$((TARCNT+1))
            
            # dry run...
            [[ $DRY_RUN -eq 0 ]] || continue

            # not first anymore...
            first=0
        fi
    done
    [[ $nsaves -gt 0   ]] || { warn_msg "nothing saved for step $STEP_ID"; return 0; }
    [[ $nmissing -eq 0 ]] || warn_msg "missing $nmissing files in step $STEP_ID";
    info_msg "created $nsaves tar files"

    return 0
}


         
#
# archive pairwise per label A
# save everything in PWC/A/A_x.* with the given extension(s)
#
archive_pairwise_fast() {
    [[ $# -ge 1 ]] || return 255
    local bdir="$1"
    [[ -d "$bdir" ]] || return 255
    shift
    # get extensions, if none -> flag all
    local extensions="$*"
    local doall=1
    [[ ${#extensions} -gt 0 ]] || doall=0

    # don't use the TARCNT in the file name
    TARCNT=-1

    local i=0
    local N=${#PL_TODO_LABELS[@]}
    local T=""
    local ndone=0

    # loop over all todo labels
    for T in "${PL_TODO_LABELS[@]}"; do
        # check label
        isok_label "A" $T || continue

        # PWC directory
        local pwcdir="$bdir/$T"
        [[ -d $pwcdir ]] || continue

        # set tar name
        TARLBL="$T"
        set_archive_name
        # check if tar file already exists
        check_archive_files || { info_msg "skipping label $TARLBL - archive exists"; continue; }

        # check if not empty
        local isempty=1
        if [[ ${#extensions} -gt 0 ]]; then
            # make sure there is at least one file
            for ext in $extensions; do
                # this avoids buffer overflow problems
                [[ "$(find $pwcdir/ -maxdepth 1 -name '*.'$ext -print -quit)" ]] && { isempty=0; break; }
            done
        else
            [[ "$(find $pwcdir/ -maxdepth 1 -name '*.*' -print -quit)" ]] && isempty=0
        fi

        # skip label if directory is empty
        [[ $isempty -eq 0 ]] || { info_msg "skipping label $TARLBL - no files with extentions ($extensions)"; continue; }

        # backup the whole directory
        # note gzip is done in archivecmd_dir
        archivecmd_dir $pwcdir $extensions || return 255
        ndone=$((ndone+1))
    done
    info_msg "created $ndone tar files"
    return 0
}

# arguments: <matrix> <dir> <ext1> <ext2> <ext3> ...
archive_pairwise() {
    [[ $# -ge 1 ]] || return 255
    local matrix=$1
    local bdir="$2"
    [[ -d "$bdir" ]] || return 255
    shift
    shift
    local extensions="$*"

    if [[ $ARCH_PERLBL_PWC -eq 1 && ${#OP_LABELB_FIRST} -eq 0 && ${#OP_LABELB_LAST} -eq 0 ]]; then
        info_msg "running per label PWC archiving"
        archive_pairwise_fast "$bdir" $extensions
        return $?
    fi

    [[ ${#extensions} -gt 0 ]] || { error_msg "no extensions given!"; return 255; }

    # silly way to count nr of extensions
    local nexts=0
    for ext in $extensions; do
        nexts=$((nexts+1))
    done

    local nfiles=0
    local fsize=0
    local first=1

    local i=0
    local N=${#PL_TODO_LABELS[@]}

    local ndone=0
    local nsaves=0
    local nmissing=0

    local Np1=$((N+1))
    local Nm1=$((N-1))
    local Nm2=$((N-2)) # used for last B if no diagonal
    local npairs=0

    local do_diag=0
    [[ $matrix -eq 0 ]] && do_diag=1

    # get nr of pairwise depending on matrix type
    case "$matrix" in
        "0") # full
            npairs=$((N*N))
            ;;
        "1") # full minus diagonal
            npairs=$((N*Nm1))
            ;;
        "2") # half matrix, with diagonal
            npairs=$((N*Np1/2))
            ;;
        "3") # half matrix, no diagonal
            npairs=$((N*Nm1/2))
            ;;
        *)
            fatal_msg "step_pairwise() matrix check illegal type ($matrix)"
            ;;
    esac

    # nr of files
    local nmaxfiles=$((npairs*nexts))

    # tmp file with names of all files to be included
    local tmpfiles=""
    tmp_make tmpfiles "archive_pairwise"

    # loops
#    [[ $DRY_RUN -eq 0 ]] && echo -n "   "
    local iA=0
    local lastA=0
    for ((iA=0; iA<N; iA++)); do
        local tA=${PL_TODO_LABELS[$iA]}
        local isokA=0
        isok_label "A" $tA && isokA=1
        local iB=0
        local iBstart=0
        [[ $matrix -eq 2 ]] && iBstart=$((iA+1))

        # check if last A
        [[ $iA -eq $Nm1 ]] && lastA=1

        # set TARLBL - used only if per lbl is set
        TARLBL="$tA"

        # flag last B
        local lastB=0

        # loop over B
        for ((iB=$iBstart; iB<N; iB++)); do
            local lastAB=0
            local tB=${PL_TODO_LABELS[$iB]}
            [[ $iB -eq $Nm1 ]] && lastB=1
            # check if the very last one
            # the logic varies depending on which matrix config
            # the idea is to always do the action within the B loop
            # and never do outside
            [[ $matrix -eq 0 ]] && [[ $lastB -eq 1    ]] && [[ $lastA -eq 1    ]] && lastAB=1
            [[ $matrix -eq 1 ]] && [[ $iB    -eq $Nm2 ]] && [[ $lastA -eq 1    ]] && lastAB=1
            [[ $matrix -eq 2 ]] && [[ $lastB -eq 1    ]] && [[ $iA    -eq $Nm2 ]] && lastAB=1

            # exclude diagonal when matrix is not 'full'
            [[ $lastAB -eq 0 ]] && [[ $matrix -ne 0 ]] && [[ "$tB" == "$tA" ]] && continue

            # loop over extensions and gather files
            if [[ $isokA -eq 1 ]]; then
                if isok_label "B" $tB; then
                    for ext in $extensions; do
                        local f="${bdir}/${tA}/${tA}_${tB}.${ext}"
                        [[ -f "$f" ]] || { nmissing=$((nmissing+1)); verbose_error_msg "skipping missing file <$f>"; continue; }
                        echo "$f"
                        fsz=$(file_size "$f")
                        fsize=$((fsize+fsz))
                        nfiles=$((nfiles+1))
                    done >> $tmpfiles
                else
                    warn_msg "pairwise: skipping todo label B $tB"
                fi
            fi
            # if no files found, skip
            [[ $nfiles -gt 0 ]] || continue


            # check if we should save things
            # save if
            # * PERLBL is set and last PWC in A (lastB)
            # * PERLBL is NOT set and nr of files or size is enough
            # * last AB is reached
            local save=0
            if [[ $ARCH_PERLBL_PWC -eq 1 ]]; then
                # save per label - last [A,B] reached for label A
                [[ $lastB -eq 1 ]] && save=1
            else
                # check nr of files OR size
                do_save $nfiles $fsize && save=1
            fi

            # in any case save if the very last [A,B] s reached
            [[ $lastAB -eq 1 ]] && save=1

            if [[ $save -eq 1 ]]; then
                set_archive_name
                info_msg "got nr of files = $nfiles - save flag = $save , file size = $fsize"
                if check_archive_files; then
                    # tar the files
                    archivecmd $tmpfiles || return 255

                    ndone=$((ndone+nfiles))
                    nsaves=$((nsaves+1))
                else
                    info_msg "skipping label $TARLBL - archive exists"
                fi

                # empty the file without removing it
                rm -f $tmpfiles
                touch $tmpfiles
                nfiles=0
                fsize=0
                
                # move to next tar name, if not last
                #[[ $lastAB -eq 1 ]] || TARCNT=$((TARCNT+1))
                TARCNT=$((TARCNT+1))
                
                [[ $DRY_RUN -eq 0 ]] || continue
#                local perc=$(calc "int(100*${ndone}/$nmaxfiles)")
                [[ $lastAB -eq 1 ]] && perc="100"
                
#                local pstr=$(printf "%3d" $perc)
#                [[ $first -eq 1 ]] || echo -en "\b\b\b"
#                echo -n "${perc}%"
                first=0
            fi
        done
    done
    [[ $nsaves -gt 0 ]] || { warn_msg "nothing saved for step $STEP_ID"; return 0; }
    [[ $nmissing -eq 0 ]] || warn_msg "missing $nmissing files in step $STEP_ID";
    info_msg "created $nsaves tar files"

    return 0
}

start_step() {
    info_msg "------ Archiving -> $STEP_ID"
    ARCH_FORCE_STEP=1   # should this be set here??
    TAREXTRAS=""        # extras must be set after this call
    init_archive
}

end_step() {
    info_msg "------ Done -> $STEP_ID"
}

archive_preproc() {
    start_step || return 255
    archive_per_todo Rawdata $EXT_FASTA ${EXT_FASTA}.maptxt || return 255
    archive_per_todo Sequences $EXT_FASTA_CLEAN || return 255
    end_step
}

archive_masker() {
    start_step || return 255
    archive_per_todo Sequences $EXT_FASTA_MASKED || return 255
    end_step
}

archive_select() {
    start_step || return 255
    archive_per_todo Sequences $EXT_FASTA_SELECTED ${EXT_FASTA_SELECTED}.clstr || return 255
    end_step
}

archive_stats() {
    start_step || return 255
    TAREXTRAS="Sequences/${PL_TODO_NAME}_fasta_totlen.dat"
    archive_per_todo Sequences $EXT_FASTA_STATS $EXT_FASTA_SEQLEN || return 255
    end_step
}

archive_makeinparsel() {
    start_step || return 255
    archive_per_todo Sequences $EXT_INPAR_SELECTED || return 255
    end_step
}

archive_formatdb() {
    start_step || return 255
    local exts="$(get_formatdb_exts $EXT_FASTA_SELECTED)"
    [[ ${#exts} -gt 0 ]] || { error_msgs "failed archiving formatdb - undefined DB extensions"; return 255; }
    archive_per_todo Sequences $exts || return 255
    end_step
}

archive_alignment() {
    start_step || return 255
    archive_pairwise 0 PWC $EXT_ALIGN || return 255
    end_step
}

archive_makebrh() {
    start_step || return 255
    archive_pairwise 2 PWC $EXT_BRH       || return 255
    end_step
}

archive_makeinpar() {
    start_step || return 255
    archive_pairwise 2 PWC $EXT_INPAR     || return 255
    end_step
}

# archive all in PWC directory - does not check for files
archive_pwc() {
    start_step || return 255
    archive_pairwise_fast PWC || return 255
    end_step
}



archive_files_nodes() {
    [[ $# -ge 1 ]] || fatal_msg_wrong_nargs $# "$FUNCNAME"
    local allexts=$*

    # get directory of todo file
    local dirtodo=$(dirname $PL_TODO)
    

    # get all leaves (script in brhclus get_runs...)
    local allfiles=""
    local ec=0

    if (( TREE_ENABLED == 0 )); then
        # no tree - a single node given by PL_TODO
        for ext in $allexts; do
            allfiles="$DIR_PROJECT_CLUSTER/${PL_TODO_NAME}.$ext"
        done
    else
        # tree - get all nodes
        # check tree file
        [[ -f $TREE_INPUT_FILE ]] || { step_error_msg "missing tree file $TREE_INPUT_FILE"; return 255; }

        # get a list of ALL internal nodes
        get_internal_nodes || {
            step_error_msg "archive_files_nodes: unable to get nodes from $TREE_INPUT_FILE"
            return 255
        }
        for node in ${_NODES[@]}; do
            for ext in $allexts; do
                allfiles="$allfiles $DIR_PROJECT_CLUSTER/${node}.$ext"
            done
        done
    fi
    [[ -n $allfiles ]] || { step_error_msg "nothing found to archive!"; return 255; }
    archive_files "$allfiles"
}


archive_setbrh() {
    start_step || return 255

    # set tar name
    set_archive_name
    check_archive_files || return 255

    archive_files_nodes $EXT_BRH || return 255

    end_step
}

archive_setinpar() {
    start_step || return 255

    # set tar name
    set_archive_name
    check_archive_files || return 255

    archive_files_nodes $EXT_INPAR || return 255

    end_step
}

archive_setinparsel() {
    start_step || return 255

    # set tar name
    set_archive_name
    check_archive_files || return 255

    archive_files_nodes $EXT_INPAR_SELECTED || return 255
    
    end_step
}

archive_cluster() {
    start_step || return 255

    # set tar name
    set_archive_name
    check_archive_files || return 255

    archive_files_nodes $EXT_CLUSTER_RAW ${EXT_CLUSTER_RAW}.log || return 255

    end_step
}

archive_mergeinpar() {
    start_step || return 255

    # set tar name
    set_archive_name
    check_archive_files || return 255

    archive_files_nodes $EXT_CLUSTER_INPAR ${EXT_CLUSTER_INPAR}.log ${EXT_INPAR}.unresolved || return 255

    end_step
}

archive_mergeinparsel() {
    start_step || return 255

    # set tar name
    set_archive_name
    check_archive_files || return 255

    archive_files_nodes $EXT_CLUSTER ${EXT_CLUSTER}.log ${EXT_INPAR_SELECTED}.unresolved || return 255

    end_step
}

archive_full_dir() {
    [[ $# -eq 2 ]] || return 255
    local adir="$1"
    local maxdepth="$2"
    [[ -d $adir ]] || { error_msg "missing source directory: $adir"; return 255; }
    start_step || return 255

    local bname=$(basename "$adir")
    
    archive_dir "$adir" "$maxdepth" || return 255

    end_step
}

archive_rawdata_dir() {
    archive_full_dir Rawdata 1 || return 255
}

archive_sequence_dir() {
    archive_full_dir Sequence 1 || return 255
}

archive_todo_dir() {
    archive_full_dir todo 1 || return 255
}

archive_cluster_dir() {
    archive_full_dir Cluster 1 || return 255
}

# fast archive PWC with all files - one archive per label containing everything
archive_pwcfast_dir() {
    start_step || return 255
    archive_pairwise_fast PWC || return 255
    end_step
}

# archive PWC but only brh and inpar files
archive_pwcshort_dir() {
    start_step || return 255
    archive_dir PWC 2 brh inpar || return 255
    end_step
}

# archive PWC with all files
archive_pwc_dir() {
    archive_full_dir PWC 2 || return 255
}

make_archive() {
    ARCH_TARGET_FILE_BASE="${ARCH_TARGET_DIR}/${ARCH_TARGET_NAME}"
    if [[ -n $ARCH_STEPDIR ]]; then
        local s=$(basename "$ARCH_STEPDIR")
        STEP_ID=${s///}
        #if [[ $STEP_ID == PWC ]]; then
        #    [[ $ARCH_PERLBL_PWC -eq 1 ]] || { error_msg "for 'PWC' requires option per label -l"; return 255; }
        #fi
        local func=archive_${STEP_ID,,}_dir
        isfunc $func || { warn_msg "missing archive function for $ARCH_STEPDIR"; return 255; }
        $func || { error_msg "failed archiving $ARCH_STEPDIR";   return 255; }
        return 0
    fi        

    for s in $ARCH_STEPS; do
        set_stepid $s
##        info_msg "creating archive(s) <${ARCH_TARGET_FILE_BASE}_${s}_XXXXXX$ARCH_FINAL_EXT>"
        local func=archive_${s,,}
        isfunc $func || { warn_msg "missing archive function for $s"; continue; }
        $func || { error_msg "failed archiving step $s";   return 255; }
    done
}

# process args
process_args "${@}" || exit 255

make_archive || exit 255

tmp_cleanup_files

