#!/bin/bash -u
#
# orthologer - Pipeline for deducing orthology from fasta files.
# Copyright (C) 2005-2023 E. Zdobnov lab: F. Tegenfeldt
# <fredrik.tegenfeldt@unige.ch> based on code by E. Kriventseva
#
# This file is part of orthologer.
#
# orthologer is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# orthologer is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with orthologer.  If not, see <http://www.gnu.org/licenses/>.
#
#
# Run merge of PWC files (SETBRH,SETINPAR).
# Note that this does NOT use OP_LABELX_START/END ranges.
#
#--- check if environment is ok -----------------
[ "${LOADED_COMMON_ORTHODB_SH+1}" == "" ] && echo "ERROR $0 : main config has not been loaded" 1>&2 && exit 255
#------------------------------------------------



fatal_msg "OBSOLETE - run_merge_files_tree.sh"




# returns:
# 0 : file found and OK
# 1 : skip, no error
# 2 : skip, error
# PWC_SOURCE_FILE -> PWC file found
PWC_SOURCE_FILE=""
get_pwc() {
    PWC_SOURCE_FILE=""
    # get labels
    [[ $# -eq 3 ]] || return 255
    local A=$1
    local B=$2
    local ext=$3
    
    # check for required species
    has_required_label $A $B
    [[ $? -eq 0 ]] || return 1
    
    # get pair file name for [A,B] (try also [B,A] if prev does not exist)
    local ec=0
    
    PWC_SOURCE_FILE="$DIR_PROJECT_PWC/${A}/${A}_${B}.$ext"
    if [[ ! -f $PWC_SOURCE_FILE ]]; then
        PWC_SOURCE_FILE="$DIR_PROJECT_PWC/${B}/${B}_${A}.$ext"
        [[ -f $PWC_SOURCE_FILE ]] || ec=1
    fi
    # check if OK
    if [[ $ec -ne 0 ]]; then
        if (( IGNORE_MISSING == 0 )); then
            step_error_msg "run_merge_pwc_files.sh: missing file $PWC_SOURCE_FILE"
        fi
        PWC_SOURCE_FILE=""
        return 2
    fi
    return 0
}

SEQ_SOURCE_FILE=""
get_seq() {
    SEQ_SOURCE_FILE=""
    # get labels
    [[ $# -eq 2 ]] || return 255
    local A=$1
    local ext=$2
    
    # check for required species
    has_required_label $A
    [[ $? -eq 0 ]] || return 1
    
    # get pair file name for [A,B] (try also [B,A] if prev does not exist)
    SEQ_SOURCE_FILE=$DIR_PROJECT_SEQUENCES/${A}.$ext
    if [[ ! -f $SEQ_SOURCE_FILE ]]; then
        if (( IGNORE_MISSING == 0 )); then
            step_error_msg "run_merge_pwc_trees.sh: missing file $SEQ_SOURCE_FILE"
        fi
        SEQ_SOURCE_FILE=""
        return 2
    fi
    return 0
}




MERGE_NERROR_MAX=1000
MERGE_NERROR=0
# merge by pairs
# returns:
# 0   : success
# 1   : too many missing source files
# 2   : failed grep on some source files
# 255 : fatal failure (args,...)
#
# MERGE_NERROR : nr of missing source files
merge_by_pairs() {
    MERGE_NERROR=0
    # arguments
    [[ $# -ge 3 ]] || { error_msg_wrong_nargs "$#" "merge_by_pairs"; return 255; }
    local ftodo="$1"
    local srcstep="$2"
    local ext="$3"
    # not that index 0 -> 'bash', 1 -> first arg, etc
    local kids=( ${@:4} )

    # set max nr of errors alowed
    local nerror_max=0
    (( IGNORE_MISSING == 1 )) && nerror_max=$MERGE_NERROR_MAX

    # get pairs file
    local fpairs=${ftodo/.todo/.$EXT_PAIRS}
    [[ -f $fpairs ]] || { error_msg "missing pairs file $fpairs"; return 255; }

    if (( SET_MERGE_TREE_INCLUDE_KIDS==1 )); then
        echo "# merge includes children nodes"
    else
        echo "# merge does NOT include children nodes"
    fi

    # loop over pairs file
    local srcfiles=()
    local AB=()
    local A=""
    local B=""
    
    local nfiles_max=1000
    
    while read line; do
        [[ "${line:0:1}" == "#" ]] && continue
        AB=( $line )
        [[ ${#AB[@]} -eq 2 ]] || {
            error_msg "bad line in pairs file $fpairs";
            error_msg "line: $line"
            return 255
        }
        A=${AB[0]}
        B=${AB[1]}
        get_pwc "$A" "$B" "$ext"
            
        if [[ ${#PWC_SOURCE_FILE} -eq 0 ]]; then
            step_error_msg "missing pair-wise file for pair [${A},${B}]"
            MERGE_NERROR=$((MERGE_NERROR+1))
            continue
        fi

        # append pair to srcfiles and JOBARGS (job args is the same as srcfiles in this case)
        srcfiles+=("$PWC_SOURCE_FILE")
        if [[ ${#srcfiles} -ge $nfiles_max ]]; then
            grep -h -v "^#" ${srcfiles[@]}
            [[ $? -lt 2 ]] || nfailed=$((nfailed+1))
            # reset array
            srcfiles=()
        fi
    done < $fpairs
    
    # check if any remaining
    if [[ ${#srcfiles} -gt 0 ]]; then
        grep -h -v "^#" ${srcfiles[@]}
        [[ $? -lt 2 ]] || nfailed=$((nfailed+1))
    fi

    # now get kids
    if (( SET_MERGE_TREE_INCLUDE_KIDS==1 )); then
        srcfiles=""
        local k=""
        for k in ${kids[@]}; do
            local fk=$DIR_PROJECT_CLUSTER/${k}.$ext
            [[ -f $fk ]] || {
                step_error_msg "missing seed BRH file at $fk"
                MERGE_NERROR=$((MERGE_NERROR+1))
                continue
            }
            srcfiles="$srcfiles $fk"
        done
        if [[ ${#srcfiles} -gt 0 ]]; then
            grep -h -v "^#" $srcfiles
            [[ $? -lt 2 ]] || nfailed=$((nfailed+1))
        fi
    fi
    (( nfailed==0 )) || { step_error_msg "failed processing $nfailed source files!"; return 2; }
    return 0
}


merge_by_pairs_nochecks() {
    MERGE_NERROR=0
    # arguments
    [[ $# -ge 3 ]] || { error_msg_wrong_nargs "$#" "merge_by_pairs_nochecks"; return 255; }
    local ftodo="$1"
    local srcstep="$2"
    local ext="$3"
    # not that index 0 -> 'bash', 1 -> first arg, etc
    local kids=( ${@:4} )
    
    # set max nr of errors alowed
    local nerror_max=0
    (( IGNORE_MISSING == 1 )) && nerror_max=$MERGE_NERROR_MAX

    # get pairs file
    local fpairs=${ftodo/.todo/.$EXT_PAIRS}
    [[ -f $fpairs ]] || { error_msg "missing pairs file $fpairs"; return 255; }

    if (( SET_MERGE_TREE_INCLUDE_KIDS==1 )); then
        echo "# merge includes children nodes"
    else
        echo "# merge does NOT include children nodes"
    fi
    if [[ ${#kids[@]} -gt 0 ]]; then
        echo "#CHILDREN: ${kids[@]}"
    else
        echo "#LEAFNODE"
    fi
    # loop over pairs file
    local srcfiles=()
    local AB=()
    local A=""
    local B=""
    
    local nfiles_max=1000
    
    while read line; do
        [[ "${line:0:1}" == "#" ]] && continue
        AB=( $line )
        [[ ${#AB[@]} -eq 2 ]] || {
            error_msg "bad line in pairs file $fpairs";
            error_msg "line: $line"
            return 255
        }
        A=${AB[0]}
        B=${AB[1]}
        if [[ $A < $B ]]; then
            PWC_SOURCE_FILE=PWC/$A/${A}_${B}.$ext
        else
            PWC_SOURCE_FILE=PWC/$B/${B}_${A}.$ext
        fi

        # append pair to srcfiles and JOBARGS (job args is the same as srcfiles in this case)
        srcfiles+=("$PWC_SOURCE_FILE")
        if [[ ${#srcfiles[@]} -ge $nfiles_max ]]; then
            # normal removal of just commented lines
            grep -h -v "^#" ${srcfiles[@]}

            # TEMP: removal of all lines ending with 1 or 2 (fuzzy BRHs)
            # TEMP: this is added for the 2020 bacteria run where part of the BRH's were
            # TEMP: erroneously run using fuzzy BRHs
            #sed '/ [1,2,3,4,5,6]$/d' ${srcfiles[@]}

            [[ $? -lt 2 ]] || nfailed=$((nfailed+1))
            # reset array
            srcfiles=()
        fi
    done < $fpairs
    
    # check if any remaining
    if [[ ${#srcfiles[@]} -gt 0 ]]; then
        # TEMP
        grep -h -v "^#" ${srcfiles[@]}
        #sed '/ [1,2,3,4,5,6]$/d' ${srcfiles[@]}
        [[ $? -lt 2 ]] || nfailed=$((nfailed+1))
    fi

    # now get kids
    if (( SET_MERGE_TREE_INCLUDE_KIDS==1 )); then
        srcfiles=""
        local k=""
        for k in ${kids[@]}; do
            local fk=$DIR_PROJECT_CLUSTER/${k}.$ext
            [[ -f $fk ]] || {
                step_error_msg "missing seed BRH file at $fk"
                MERGE_NERROR=$((MERGE_NERROR+1))
                continue
            }
            srcfiles="$srcfiles $fk"
        done
        if [[ ${#srcfiles} -gt 0 ]]; then
            # TEMP
            grep -h -v "^#" $srcfiles
            #sed '/ [1,2,3,4,5,6]$/d' ${srcfiles[@]}
            [[ $? -lt 2 ]] || nfailed=$((nfailed+1))
        fi
    fi
    (( nfailed==0 )) || { step_error_msg "failed processing $nfailed source files!"; return 2; }
    return 0
}



# merge by indices
# returns:
# 0   : success
# 1   : too many missing source files
# 2   : failed grep on some source files
# 255 : fatal failure (args,...)
#
# MERGE_NERROR : nr of missing source files
merge_by_index_pwc() {
    MERGE_NERROR=0
    # arguments
    [[ $# -eq 3 ]] || { error_msg_wrong_nargs "$#" "merge_by_index"; return 255; }
    local ftodo="$1"
    local srcstep="$2"
    local ext="$3"

    # check todo file
    [[ -f $ftodo ]] || { error_msg "missing todo file $ftodo"; return 255; }

    # extract labels
    local todolabels=( $(sort -u --ignore-case "$ftodo" | grep -v "^#") )
    local nlabels=${#todolabels[@]}
    
    # get matrix type for step
    matrix=$(get_matrix_type $STEP_ID)
    [[ $matrix -lt 0 ]] && fatal_msg "run_merge_pwc_trees() unexpected matrix type $matrix"
    [[ $matrix -gt 3 ]] && fatal_msg "run_merge_pwc_trees() unexpected matrix type $matrix"
    [[ $matrix -eq 1 ]] && fatal_msg "run_merge_pwc_trees() unexpected matrix type 1"
    debug_msg "merge_pwc matrix: $matrix"
    
    local ec=0
    # loop over first label
    local indA=0
    local nerror_max=0
    (( IGNORE_MISSING == 1 )) && nerror_max=$MERGE_NERROR_MAX
    
    for (( indA=0; indA<nlabels; indA++ )); do
        # label of A
        local A=${todolabels[$indA]}
        
        # loop over second label and fill up SRCFILES with all data for pairs [A,B]
        # do the upper half matrix fo all possible pairs (excluding [B,A] and [A,A])

        # set iBstart 
        local iBstart=0
        if [[ $matrix -eq 2 ]]; then
            iBstart=$((indA+1))
        elif [[ $matrix -eq 3 ]]; then
            iBstart=$indA
        fi
        local srcfiles=()
        local indB=0
        for (( indB=iBstart; indB<nlabels; indB++ )); do
            # label of B
            local B=${todolabels[$indB]}

            get_pwc "$A" "$B" "$ext"
            
            if [[ ${#PWC_SOURCE_FILE} -eq 0 ]]; then
                MERGE_NERROR=$((MERGE_NERROR+1))
                continue
            fi

            # append pair to srcfiles and JOBARGS (job args is the same as srcfiles in this case)
            srcfiles+=("$PWC_SOURCE_FILE")
            
        done
        
        # check nr of missing files
        if (( MERGE_NERROR > nerror_max )); then
            step_error_msg "found $MERGE_NERROR missing files (max allowed $nerror_max) - aborting"
	        return 1
        fi
        
        # merge all BRH files for label A
        if [[ ${#srcfiles[@]} -gt 0 ]]; then
            # merge files excluding '#' lines
            # use zgrep (==gunzip+grep), works like grep
            # note that if zgrep finds no match, it will return 1, hence the -lt 2
            grep -h -v "^#" "${srcfiles[@]}"
            [[ $? -lt 2 ]] || nfailed=$((nfailed+1))
        fi
    done
    
    (( nfailed==0 )) || { step_error_msg "failed processing $nfailed source files!"; return 2; }
    return 0
}

merge_by_index_pwc_nochecks() {
    MERGE_NERROR=0
    # arguments                                                                                                        
    [[ $# -eq 3 ]] || { error_msg_wrong_nargs "$#" "merge_by_index"; return 255; }
    local ftodo="$1"
    local srcstep="$2"
    local ext="$3"

    # check todo file                                                                                                  
    [[ -f $ftodo ]] || { error_msg "missing todo file $ftodo"; return 255; }

    # extract labels                                                                                                   
    local todolabels=( $(sort -u --ignore-case "$ftodo" | grep -v "^#") )
    local nlabels=${#todolabels[@]}

    # get matrix type for step                                                                                         
    matrix=$(get_matrix_type $STEP_ID)
    [[ $matrix -lt 0 ]] && fatal_msg "run_merge_pwc_trees() unexpected matrix type $matrix"
    [[ $matrix -gt 3 ]] && fatal_msg "run_merge_pwc_trees() unexpected matrix type $matrix"
    [[ $matrix -eq 1 ]] && fatal_msg "run_merge_pwc_trees() unexpected matrix type 1"
    debug_msg "merge_pwc matrix: $matrix"

    local ec=0
    # loop over first label                                                                                            
    local indA=0
    local nerror_max=0
    (( IGNORE_MISSING == 1 )) && nerror_max=$MERGE_NERROR_MAX

    for (( indA=0; indA<nlabels; indA++ )); do
        # label of A                                                                                                   
        local A=${todolabels[$indA]}

        # loop over second label and fill up SRCFILES with all data for pairs [A,B]                                    
        # do the upper half matrix fo all possible pairs (excluding [B,A] and [A,A])                                   

        # set iBstart                                                                                                  
        local iBstart=0
        if [[ $matrix -eq 2 ]]; then

            iBstart=$((indA+1))
        elif [[ $matrix -eq 3 ]]; then
            iBstart=$indA
        fi
        local srcfiles=()
	    local indB=0
	    for (( indB=iBstart; indB<nlabels; indB++ )); do
            # label of B                                                                                               
	        local B=${todolabels[$indB]}

	        PWC_SOURCE_FILE=PWC/$A/${A}_${B}.$ext

            if [[ ${#PWC_SOURCE_FILE} -eq 0 ]]; then
                MERGE_NERROR=$((MERGE_NERROR+1))
                continue
            fi

            # append pair to srcfiles and JOBARGS (job args is the same as srcfiles in this case)
	        srcfiles+=("$PWC_SOURCE_FILE")

        done

	    # check nr of missing files                                                                                    
        if (( MERGE_NERROR > nerror_max )); then
            step_error_msg "found $MERGE_NERROR missing files (max allowed $nerror_max) - aborting"
            return 1
        fi

        # merge all BRH files for label A                                                                              
        if [[ ${#srcfiles[@]} -gt 0 ]]; then
            # merge files excluding '#' lines                                                                          

            sed '/ [1,2]$/d; /^#/d;' ${srcfiles[@]}
            [[ $? -lt 1 ]] || nfailed=$((nfailed+1))
        fi
    done

    (( nfailed==0 )) || { step_error_msg "failed processing $nfailed source files!"; return 2; }
    return 0
}

merge_by_index_seq() {
    MERGE_NERROR=0
    # arguments
    [[ $# -eq 3 ]] || { error_msg_wrong_nargs "$#" "merge_by_index_seq"; return 255; }
    local ftodo="$1"
    local srcstep="$2"
    local ext="$3"

    # check todo file
    [[ -f $ftodo ]] || { error_msg "missing todo file $ftodo"; return 255; }

    # extract labels
    local todolabels=( $(sort -u --ignore-case "$ftodo" | grep -v "^#") )
    local nlabels=${#todolabels[@]}
        
    local ec=0
    # loop over first label
    local indA=0
    local nerror_max=0
    (( IGNORE_MISSING == 1 )) && nerror_max=$MERGE_NERROR_MAX
    
    for (( ind=0; ind<nlabels; ind++ )); do
        # label of A
        local A=${todolabels[$ind]}
        get_seq "$A" "$ext"
            
        if [[ ${#SEQ_SOURCE_FILE} -eq 0 ]]; then
            MERGE_NERROR=$((MERGE_NERROR+1))
            # check nr of missing files
            if (( MERGE_NERROR > nerror_max )); then
                step_error_msg "found $MERGE_NERROR missing files (max allowed $nerror_max) - aborting"
	            return 1
            fi
            continue
        fi

        # append pair to srcfiles and JOBARGS (job args is the same as srcfiles in this case)
        srcfiles+=("$SEQ_SOURCE_FILE")
        
    done
    # merge all BRH files for label A
    if [[ ${#srcfiles[@]} -gt 0 ]]; then
        # merge files excluding '#' lines
        # use zgrep (==gunzip+grep), works like grep
        # note that if zgrep finds no match, it will return 1, hence the -lt 2
        grep -h -v "^#" "${srcfiles[@]}"
        [[ $? -lt 2 ]] || nfailed=$((nfailed+1))
    fi
    
    (( nfailed==0 )) || { step_error_msg "failed processing $nfailed source files!"; return 2; }
    return 0
}

#==================================================================================================
# Check arguments
#  source step id      - required in order to determine the source compression
#  root todo file      - todo file for level
#  target file         - final output file (Cluster/<todo>.brh)
#  extension           - extension of source (EXT_BRH)
#  child1,child2,...   - labels for each child node (not present if root is a leaf node)
[[ $# -eq 1 ]] || fatal_msg "usage: $0 <source step id,root todo file,target file,ext,[child1,child2,...]>"

# process input csv string
input="$1"
inparr=( $(tokenize "," $input) )
nargs=${#inparr[@]}
[[ $nargs -ge 4 ]] || fatal_msg "run_merge_files_trees: expected at least 4 arguments (${inparr[@]})"

# extract args
src_step="${inparr[0]}"
ftodo="${inparr[1]}"
ftrg="${inparr[2]}"
ext="${inparr[3]}"

# kids are given as labels, not full filename
kids=( ${inparr[@]:4} )
nkids=${#kids[@]}

# tmp output file
tmp_make tmpfile ${STEP_ID,,}

# timer
startdate=$(date)
tstart=$(time_msec)

if [[ $STEP_ID == "SETINPARSEL" ]]; then
    # merge by index
    merge_by_index_seq "$ftodo" "$src_step" "$ext" > "$tmpfile"
    ec=$?
else
    if (( nkids == 0 )); then
        # merge by index
        merge_by_index_pwc "$ftodo" "$src_step" "$ext" > "$tmpfile"
        ec=$?
    else
        # merge by pairs
        if (( USE_NOCHECKS == 1 )); then
            merge_by_pairs_nochecks "$ftodo" "$src_step" "$ext" "${kids[@]}" > "$tmpfile"
            ec=$?
        else
            merge_by_pairs "$ftodo" "$src_step" "$ext" "${kids[@]}" > "$tmpfile"
            ec=$?
        fi
    fi
fi
(( ec==0 )) || exit 255

# check if any errors - ie any missing files
header_msg=""
if (( MERGE_NERROR > 0 )); then
    if (( IGNORE_MISSING == 0 )); then
        step_error_msg "failed due to $MERGE_NERROR missing file(s)"
        exit 255
    else
        header_msg=" --- merged OK but with $MERGE_NERROR missing file(s) ---"
        step_warn_msg "$header_msg"
    fi
fi

# to be added to the header
mergemsg="simple merge$header_msg"

dT=$(delta_time $tstart)
add_header_footer "$STEP_ID" "$tmpfile" "$ftrg" "$startdate" $dT "$mergemsg"
[ $? -lt 2 ] || step_fatal_msg "failed adding header/footer!"
chmod 644 "$ftrg"
