#!/bin/bash -u
#
# orthologer - Pipeline for deducing orthology from fasta files.
# Copyright (C) 2005-2023 E. Zdobnov lab: F. Tegenfeldt
# <fredrik.tegenfeldt@unige.ch> based on code by E. Kriventseva
#
# This file is part of orthologer.
#
# orthologer is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# orthologer is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with orthologer.  If not, see <http://www.gnu.org/licenses/>.
#
#
# Run merge of PWC files (SETBRH,SETINPAR).
# Note that this does NOT use OP_LABELX_START/END ranges.
#
#--- check if environment is ok -----------------
[ "${LOADED_COMMON_ORTHODB_SH+1}" == "" ] && echo "ERROR $0 : main config has not been loaded" 1>&2 && exit 255
#------------------------------------------------

fatal_msg "run_merge_pwc_files.sh : OBSOLETE"





# returns:
# 0 : file found and OK
# 1 : skip, no error
# 2 : skip, error
# PWC_SOURCE_FILE -> PWC file found
PWC_SOURCE_FILE=""
get_pwc() {
    PWC_SOURCE_FILE=""
    # get labels
    [[ $# -eq 4 ]] || return 255
    local A=$1
    local B=$2
    local ext=$3
    local cmpext=$4
    
    # check for required species
    has_required_label $A $B
    [[ $? -eq 0 ]] || return 1
    
    # get pair file name for [A,B] (try also [B,A] if prev does not exist)
    local ec=0
    local fAB=""
    local fBA=""
    
    v_get_pwc_file f ${A} ${B} ${ext}
    v_get_filename fAB "$f" "$cmpext"
    ec=$?
    if [[ $ec -eq 0 ]]; then
        # set source file, no compression
        PWC_SOURCE_FILE="$fAB"
    elif [[ $ec -eq 2 ]]; then
        # not found, try swap labels
        v_get_pwc_file f ${B} ${A} ${ext}
        v_get_filename fBA "$f" "$cmpext"
        ec=$?
        [[ $ec -eq 0 ]] && PWC_SOURCE_FILE="$fBA"
    fi
    # check if OK
    if [[ $ec -ne 0 ]]; then
        if (( IGNORE_MISSING == 0 )); then
            local msg=""
            v_parse_ec_get_filename msg $ec
            step_error_msg "run_merge_pwc_files.sh: $msg , file $fAB"
        fi
        PWC_SOURCE_FILE=""
    fi
    [[ ${#PWC_SOURCE_FILE} -gt 0 ]] || return 2
    return 0
}

MERGE_NERROR_MAX=1000
MERGE_NERROR=0
# merge by pairs
# returns:
# 0   : success
# 1   : too many missing source files
# 2   : failed grep on some source files
# 255 : fatal failure (args,...)
#
# MERGE_NERROR : nr of missing source files
merge_by_pairs() {
    MERGE_NERROR=0
    # arguments
    [[ $# -eq 3 ]] || { error_msg_wrong_nargs "$#" "merge_by_pairs"; return 255; }
    local src_id="$1"
    local ext="$2"
    local output="$3"
    
    # deduced from args
    # compression extensions
    local trgcmp=
    local srccmp=
    v_get_cmpext trgcmp "$STEP_ID"
    v_get_cmpext srccmp "$src_id"

    # grep cmd
    local grepcmd='grep'
    [[ -z $srccmp ]] || grepcmd='zgrep'

    # set max nr of errors alowed
    local nerror_max=0
    (( IGNORE_MISSING == 1 )) && nerror_max=$MERGE_NERROR_MAX

    # get pairs file
    local fpairs=${PL_TODO/.todo/.pairs}
    if [[ -f $fpairs ]]; then
        local ftree=${PL_TODO/.todo/.${EXT_TREE}}
        [[ -f $ftree ]] || create_tree
        [[ -f $ftree ]] || { error_msg "failure creating tree file $ftree"; return 255; }
        create_pairs || return 255;
    fi

    # loop over pairs file
    local srcfiles=()
    local AB=()
    local A=""
    local B=""
    
    local nfiles_max=1000
    
    while read line; do
        [[ "${line:0:1}" == "#" ]] && continue
        AB=( $line )
        [[ ${#AB[@]} -eq 2 ]] || {
            error_msg "bad line in pairs file $fpairs";
            error_msg "line: $line"
            return 255
        }
        A=${AB[0]}
        B=${AB[1]}
        get_pwc "$A" "$B" "$ext" "$srccmp"
            
        if [[ ${#PWC_SOURCE_FILE} -eq 0 ]]; then
            ###step_error_msg "missing pair-wise file for pair [${A},${B}]"
            MERGE_NERROR=$((MERGE_NERROR+1))
            continue
        fi

        # append pair to srcfiles and JOBARGS (job args is the same as srcfiles in this case)
        srcfiles+=("$PWC_SOURCE_FILE")
        if [[ ${#srcfiles} -ge $nfiles_max ]]; then
            local f=""
            for f in ${srcfiles[@]}; do
                $grepcmd -h -v "^#" "$f"
                [[ $? -lt 2 ]] || nfailed=$((nfailed+1))
            done >> $output
            
            # reset array
            srcfiles=()
        fi
    done < $fpairs
    
    # check if any remaining
    if [[ ${#srcfiles} -gt 0 ]]; then
        local i=0
        local f=""
        for f in ${srcfiles[@]}; do
            $grepcmd -h -v "^#" "$f"
            [[ $? -lt 2 ]] || nfailed=$((nfailed+1))
        done >> $output
    fi
    
    (( nfailed==0 )) || { step_error_msg "failed processing $nfailed source files!"; return 2; }
    return 0
}

# merge by indices
# returns:
# 0   : success
# 1   : too many missing source files
# 2   : failed grep on some source files
# 255 : fatal failure (args,...)
#
# MERGE_NERROR : nr of missing source files
merge_by_index() {
    MERGE_NERROR=0
    # arguments
    [[ $# -eq 3 ]] || { error_msg_wrong_nargs "$#" "merge_by_index"; return 255; }
    local src_id="$1"
    local ext="$2"
    local output="$3"
    
    # deduced from args
    # compression extensions
    local trgcmp=
    local srccmp=
    v_get_cmpext trgcmp "$STEP_ID"
    v_get_cmpext srccmp "$src_id"

    # get matrix type for step
    matrix=$(get_matrix_type $STEP_ID)
    [[ $matrix -ge  0 ]] && [[ $matrix -le 2 ]] || fatal_msg "step_mergefiles() got wrong matrix type ($matrix)"

    # check OP_LABELB_START/END wrt PL_TODO_LABELS_REQ
    # If OP_LABELx are set, make sure they are consistent with the required label.
    local vop="${OP_LABELA_START}${OP_LABELA_END}${OP_LABELB_START}${OP_LABELB_END}"
    if [[ -n $vop ]]; then
        # some OP_LABELx are set
        local isOK=1
        local lblreq="${PL_TODO_LABELS_REQ:1: -1}"
        if [[ -n $lblreq ]]; then
            # required label is set
            local vopA="${OP_LABELA_START}${OP_LABELA_END}"
            if [[ -z $vopA ]]; then
                # LABELA_x are not set
                # it is OK if both LABELB_x are set to the required label
                [[ $OP_LABELB_START == $lblreq ]] || isOK=0
                [[ $OP_LABELB_END   == $lblreq ]] || isOK=0
            else
                # LABELA_x are set -> incorrect
                isOK=0
            fi
        fi
        if (( isOK == 0 )); then
            warn_msg "===================================================================="
            warn_msg "---> OP_LABELX_START/END is set incorrectly wrt required labels <---"
            warn_msg "OP_LABELA_START    = $OP_LABELA_START"
            warn_msg "OP_LABELA_END      = $OP_LABELA_END"
            warn_msg "OP_LABELB_START    = $OP_LABELB_START"
            warn_msg "OP_LABELB_END      = $OP_LABELB_END"
            warn_msg "PL_TODO_LABELS_REQ = $lblreq"
            warn_msg "===================================================================="
        fi
    fi
    # get index ranges based on matrix type
    get_pairwise_matrix $matrix 1
    local beginA=${PAIRWISE_MATRIX[0]}
    local endA=${PAIRWISE_MATRIX[1]}
    local beginB=${PAIRWISE_MATRIX[2]}
    local endB=${PAIRWISE_MATRIX[3]}

    debug_msg "merge_pwc matrix: $matrix"
    
    # grep cmd
    local grepcmd='grep'
    [[ -z $srccmp ]] || grepcmd='zgrep'

    
    # loop over first label
    local indA=0
    local ec=0

    local nfailed=0
    debug_msg "merge_pwc range A [$beginA,$endA]"
    debug_msg "merge_pwc range B [$beginB,$endB]"

    local nerror_max=0
    (( IGNORE_MISSING == 1 )) && nerror_max=$MERGE_NERROR_MAX
    
    for (( indA=beginA; indA<endA+1; indA++ )); do
        # label of A
        local A=${PL_TODO_LABELS[$indA]}
        
        # loop over second label and fill up SRCFILES with all data for pairs [A,B]
        # do the upper half matrix fo all possible pairs (excluding [B,A] and [A,A])
        
        local iBstart=$beginB
        [[ $beginB -gt $indA ]] || iBstart=$((indA+1))
        
        local srcfiles=()
        local indB=0
        for (( indB=iBstart; indB<endB+1; indB++ )); do
            # label of B
            local B=${PL_TODO_LABELS[$indB]}

            get_pwc "$A" "$B" "$ext" "$srccmp"
            
            if [[ ${#PWC_SOURCE_FILE} -eq 0 ]]; then
                ###step_error_msg "missing pair-wise file for pair [${A},${B}]"
                MERGE_NERROR=$((MERGE_NERROR+1))
                continue
            fi

            # append pair to srcfiles and JOBARGS (job args is the same as srcfiles in this case)
            srcfiles+=("$PWC_SOURCE_FILE")
            
        done
        
        # check nr of missing files
        if (( MERGE_NERROR > nerror_max )); then
            step_error_msg "found $MERGE_NERROR missing files (max allowed $nerror_max) - aborting"
	        return 1
        fi
        
        # merge all BRH files for label A
        if [[ ${#srcfiles[@]} -gt 0 ]]; then
            # merge files excluding '#' lines
            # use zgrep (==gunzip+grep), works like grep
            # note that if zgrep finds no match, it will return 1, hence the -lt 2
            for f in ${srcfiles[@]}; do
                $grepcmd -h -v "^#" "$f"
                [[ $? -lt 2 ]] || nfailed=$((nfailed+1))
            done >> $output
        fi
    done
    
    (( nfailed==0 )) || { step_error_msg "failed processing $nfailed source files!"; return 2; }
    return 0
}


[ $# -eq 1 ] || fatal_msg "usage: $0 <source step id,ext>"

# restore todo arrays
restore_todo_arrays

# process input csv string
input="$1"
inparr=( $(tokenize "," $input) )
[ ${#inparr[@]} -eq 3 ] || fatal_msg "run_merge_pwc_files: expected 3 arguments (${inparr[@]})"
src_id="${inparr[0]}"
ext="${inparr[1]}"
dopairs="${inparr[2]}"

# output file
output="${DIR_PROJECT_CLUSTER}/${PL_TODO_NAME}.${ext}"

# tmp output file
tmp_make tmpfile ${STEP_ID,,}

# timer
startdate=$(date)
tstart=$(time_sec)

if [[ "$dopairs"=="0" ]]; then
    # merge by index
    merge_by_index "$src_id" "$ext" "$tmpfile"
    ec=$?
else
    # merge by pairs
    merge_by_pairs "$src_id" "$ext" "$tmpfile"
    ec=$?
fi
(( ec==0 )) || exit 255

# check if any errors - ie any missing files
header_msg=""
if (( MERGE_NERROR > 0 )); then
    if (( IGNORE_MISSING == 0 )); then
        step_error_msg "failed due to $MERGE_NERROR missing file(s)"
        exit 255
    else
        header_msg=" --- merged OK but with $MERGE_NERROR missing file(s) ---"
        step_warn_msg "$header_msg"
    fi
fi

# to be added to the header
# it may be modified below
mergemsg="simple merge$header_msg"

# check if SETINPAR and there are required labels
if [[ $STEP_ID == SETINPAR && -n $PL_TODO_LABELS_REQ ]]; then
    # check if there are any required labels
    tmp_make tmpinp ${STEP_ID,,}
    for A in $PL_TODO_LABELS_REQ; do
        $PYTHON3 $DIR_PIPELINE/bin/extract_single_inpar.py -s $A -f "$tmpfile" || { error_msg "failed extracting required labels from inparalogs"; exit 255; }
    done > $tmpinp
    cp -f $tmpinp $tmpfile ||  { error_msg "failed copying filtered inparalogs... out of storage??"; exit 255; }
    mergemsg="${mergemsg}, with labels ($PL_TODO_LABELS_REQ)"
fi

tmp_make tmpout "mergepwc"

dT=$(dtime $tstart)
add_header_footer "$STEP_ID" "$tmpfile" "$tmpout" "$startdate" $dT "$mergemsg"
[ $? -lt 2 ] || step_fatal_msg "failed adding header/footer!"

trgcmp=
v_get_cmpext trgcmp "$STEP_ID"
compress_sync "$tmpout" "$trgcmp" "$output" "run_merge_pwc_files.sh"
chmod 644 "$output"
