#!/usr/bin/env bash

MERGE_NERROR_MAX=1000
MERGE_NERROR=0

# merge by pairs
# returns:
# 0   : success
# 1   : too many missing source files
# 2   : failed grep on some source files
# 255 : fatal failure (args,...)
#
# MERGE_NERROR : nr of missing source files
merge_by_pairs() {
    MERGE_NERROR=0
    # arguments
    [[ $# -ge 3 ]] || { error_msg_wrong_nargs "$#" "merge_by_pairs"; return 255; }
    local fpairs="$1"
    local srcstep="$2"
    local ext="$3"
    # not that index 0 -> 'bash', 1 -> first arg, etc
    local kids=( ${@:4} )

    # set max nr of errors alowed
    local nerror_max=0
    (( IGNORE_MISSING == 1 )) && nerror_max=$MERGE_NERROR_MAX

    # check pairs file
    [[ -f $fpairs ]] || { error_msg "missing pairs file $fpairs"; return 255; }

    # loop over pairs file
    local srcfiles=()
    local AB=()
    local A=""
    local B=""
    local nfailed=0
    local nfiles_max=1000
    
    while read line; do
        [[ "${line:0:1}" == "#" ]] && continue
        AB=( $line )
        [[ ${#AB[@]} -eq 2 ]] || {
            step_error_msg "bad line in pairs file $fpairs";
            step_error_msg "line: $line"
            return 255
        }
        # get pairwise
        A=${AB[0]}
        B=${AB[1]}
        local fsource="$DIR_PROJECT_PWC/${A}/${A}_${B}.$ext"
        if [[ ! -f $fsource ]]; then
            if (( IGNORE_MISSING == 0 )); then
                step_error_msg "missing PWC file $fsource"
            fi
            MERGE_NERROR=$((MERGE_NERROR+1))
            # check nr of missing files
            if (( MERGE_NERROR > nerror_max )); then
                step_error_msg "found $MERGE_NERROR missing files (max allowed $nerror_max) - aborting"
	            return 1
            fi
            continue
        fi

        # append pair to srcfiles and JOBARGS (job args is the same as srcfiles in this case)
        srcfiles+=("$fsource")
        if [[ ${#srcfiles} -ge $nfiles_max ]]; then
            local f=""
            for f in ${srcfiles[@]}; do
                grep -h -v "^#" "$f"
                [[ $? -lt 2 ]] || nfailed=$((nfailed+1))
            done
            # reset array
            srcfiles=()
        fi
    done < $fpairs
    
    # check if any remaining
    if [[ ${#srcfiles} -gt 0 ]]; then
        local f=""
        for f in ${srcfiles[@]}; do
            grep -h -v "^#" "$f"
            [[ $? -lt 2 ]] || nfailed=$((nfailed+1))
        done
    fi
    (( nfailed==0 )) || { step_error_msg "failed processing $nfailed source files!"; return 2; }
    return 0
}

#==================================================================================================
# Merge sequence files using todo file
#--------------------------------------------------------------------------------------------------
merge_by_index_seq() {
    MERGE_NERROR=0
    # arguments
    [[ $# -eq 3 ]] || { error_msg_wrong_nargs "$#" "merge_by_index_seq"; return 255; }
    local ftodo="$1"
    local srcstep="$2"
    local ext="$3"

    # check todo file
    [[ -f $ftodo ]] || { error_msg "missing todo file $ftodo"; return 255; }

    # extract labels
    local todolabels=( $(sort -u --ignore-case "$ftodo" | grep -v "^#") )
    local nlabels=${#todolabels[@]}
        
    local ec=0
    # loop over first label
    local nerror_max=0
    (( IGNORE_MISSING == 1 )) && nerror_max=$MERGE_NERROR_MAX
    local nfailed=0
    
    local A=""
    for A in ${todolabels[@]}; do
	# check if label A is a required label (in PL_TODO_LABELS_REQ)
	# if no required labels, it always returns 'true' (error code=0)
	is_required_label $A || continue
	
        # get sequence source file - is empty if non-existing
        local fsource=$DIR_PROJECT_SEQUENCES/${A}.$ext
        if [[ ! -f $fsource ]]; then
            if (( IGNORE_MISSING == 0 )); then
                step_error_msg "run_merge_files.sh: missing file $fsource"
            fi
            MERGE_NERROR=$((MERGE_NERROR+1))
            # check nr of missing files
            if (( MERGE_NERROR > nerror_max )); then
                step_error_msg "found $MERGE_NERROR missing files (max allowed $nerror_max) - aborting"
	            return 1
            fi
            continue
        fi
        # append pair to srcfiles and JOBARGS (job args is the same as srcfiles in this case)
        srcfiles+=("$fsource")
        
    done
    
    # merge all sequence files
    if [[ ${#srcfiles[@]} -gt 0 ]]; then
        # merge files excluding '#' lines
        # use zgrep (==gunzip+grep), works like grep
        # note that if zgrep finds no match, it will return 1, hence the -lt 2
        local f=""
        for f in ${srcfiles[@]}; do
            grep -h -v "^#" "$f"
            [[ $? -lt 2 ]] || nfailed=$((nfailed+1))
        done
    fi
    
    (( nfailed==0 )) || { step_error_msg "failed processing $nfailed source files!"; return 2; }
    return 0
}

#==================================================================================================
# Main
#--------------------------------------------------------------------------------------------------
# Check arguments
#  source step id      - required in order to determine the source compression
#  todo/pairs file     - todo file (SETINPARSEL) or pair file for level
#  target file         - final output file
#  extension           - extension of source
[[ $# -eq 1 ]] || fatal_msg "usage: $0 <source step id,todo or pair file,target,ext>"

# process input csv string
input="$1"
inparr=( $(tokenize "," $input) )
nargs=${#inparr[@]}
[[ $nargs -ge 4 ]] || fatal_msg "run_merge_files: expected at least 4 arguments (${inparr[@]})"

# extract args
src_step="${inparr[0]}"
flabels="${inparr[1]}"
ftrg="${inparr[2]}"
ext="${inparr[3]}"

# tmp output file
tmp_make tmpfile ${STEP_ID,,}

# timer
startdate=$(date)
tstart=$(time_msec)

if [[ $STEP_ID == "SETINPARSEL" ]]; then
    # merge by index
    merge_by_index_seq "$flabels" "$src_step" "$ext" > "$tmpfile"
    ec=$?
else
    merge_by_pairs "$flabels" "$src_step" "$ext" > "$tmpfile"
    ec=$?
fi
(( ec==0 )) || exit 255

# check if any errors - ie any missing files
header_msg=""
if (( MERGE_NERROR > 0 )); then
    if (( IGNORE_MISSING == 0 )); then
        step_error_msg "failed due to $MERGE_NERROR missing file(s)"
        exit 255
    else
        header_msg=" --- merged OK but with $MERGE_NERROR missing file(s) ---"
        step_warn_msg "$header_msg"
    fi
fi

# to be added to the header
mergemsg="simple merge$header_msg"

dT=$(delta_time $tstart)
add_header_footer "$STEP_ID" "$tmpfile" "$ftrg" "$startdate" $dT "$mergemsg"
[ $? -lt 2 ] || step_fatal_msg "failed adding header/footer!"
chmod 644 "$ftrg"
