#!/usr/bin/env bash
#
# orthologer - Pipeline for deducing orthology from fasta files.
# Copyright (C) 2005-2023 E. Zdobnov lab: F. Tegenfeldt
# <fredrik.tegenfeldt@unige.ch> based on code by E. Kriventseva
#
# This file is part of orthologer.
#
# orthologer is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# orthologer is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with orthologer.  If not, see <http://www.gnu.org/licenses/>.
#
#
# This script contains the common parts for submitting a job
#
#--- check if environment is ok -----------------
[[ "${LOADED_COMMON_PIPELINE_SH+1}" == "" ]] && echo "ERROR $0 : main config has not been loaded" 1>&2 && exit 255
#------------------------------------------------
set -u

#---------------------
# variables
#---------------------
export SJ_CHECK_POLLTIME=${SJ_CHECK_POLLTIME=5}
#
export SJ_SUBLBL=
export SJ_OPTIND=1
export SJ_MEMUSAGE=0
export SJ_NCPUPROC=1
export SJ_JOBSTR=""
export SJ_JOBSTR_JOBID=""

export SJ_BATCH_JOBID_NAME="SHELL"
export SJ_BATCH_SCRIPT=""
export SJ_BATCH_JOBID=

# job and script
export SJ_SCRIPT=""
export SJ_SCRIPT_ARGS=""
export SJ_SCHEDOPTS=""

# job log file
export SJ_JOBLABEL=""
export SJ_JOBSTDOUT=""
export SJ_JOBSTDERR=""
export SJ_NOLOG=0

export SJ_JOBLOG_STDERR_EXT="stderr"
export SJ_JOBLOG_STDOUT_EXT="stdout"

# flag, if 1, just quit without error
export SJ_QUIT=0

#
sj_usage() {
    echo -e \
        "Usage: scheduler_common.sh [-hvj -n <cpu> -m <memory>] <script> <Job label> <main species> [script args]\n\n\
 -n <nr>        : nr of core's per process\n\
 -m <mem>       : estimated memory usage(NOT YET IMPLEMENTED)\n\
 -j             : only save job log in case of error\n\
 -v             : print current version and exit\n\
 -h             : print this message\n"
}

sj_process_args_common() {
    # better reset OPTIND in case getopts is called elsewhere
    OPTIND=1
    while getopts :jhvn:m:p:s: OPT; do
        case "$OPT" in
	        h)	sj_usage
                SJ_QUIT=1
		        return 0
		        ;;
	        v)	echo "$(basename $0) version 1.0"
                SJ_QUIT=1
		        return 0
		        ;;
            j)  SJ_NOLOG=1
                ;;
	        m)  SJ_MEMUSAGE="$OPTARG"
	            ;;
	        n)  SJ_NCPUPROC="$OPTARG"
	            ;;
	        \?)	# getopts issues an error message
	            error_msg "$0: argument parse error"
		        sj_usage >&2
		        return 2
		        ;;
        esac
    done
    shift $(expr $OPTIND - 1)
    # scheduler extra options
    schedoptsvar=${OP_STEP_SCHEDOPTS[$STEP_ID]}
    assign SJ_SCHEDOPTS "$schedoptsvar"

    # check if there is anything left, there should be...
    if [[ $# -lt 1 ]]; then
        error_msg "sj_process_args_common: after <$*>"
        sj_usage >&2
        fatal_msg "$0: no script provided - $#"
    fi

    # get job id string
    SJ_SCRIPT="$1"

    # these are set later
    SJ_JOBSTR=""
    SJ_SUBLBL=""
    SJ_JOBSTDOUT=""
    SJ_JOBSTDERR=""
    SJ_SCRIPT_ARGS=""

    # finally make sure some important vars are set
    [[ ${#PL_TODO_NAME} -gt 0 ]] || fatal_msg "PL_TODO_NAME is empty!"
    [[ ${#STEP_ID}      -gt 0 ]] || fatal_msg "STEP_ID is empty!"

    return 0
}


sj_process_args() {
    # takes the same arguments as sj_submit <JOBSTR> <SUBLBL> [ <SCRIPT ARGS> ]
    # check if there is anything left, there should be...
    if [[ $# -lt 2 ]]; then
        error_msg "sj_process_args: after <$*>"
        sj_usage >&2
        fatal_msg "$0: too few args - $#"
    fi

    # get job id string
    SJ_JOBSTR="$1"
    SJ_SUBLBL="$2"

    # make sure jobstr does not contain spaces
    #replace_char "$SJ_JOBSTR" " " "_")
    SJ_JOBSTR=${SJ_JOBSTR// /_}

    # job log file name is set by the specific scheduler scripts
    SJ_JOBSTDOUT=""
    SJ_JOBSTDERR=""

    # the rest of the args define the script command
    shift
    shift
    SJ_SCRIPT_ARGS="$*"

    # finally make sure some important vars are set
    [[ ${#SJ_JOBSTR} -gt 0 ]] || fatal_msg "SJ_JOBSTR is empty!"

    return 0
}

#
sj_save_jobid() {
    [[ $# -eq 1 ]] || return
    local jobid=$1
    add_jobid_to_file "$STEP_ID" "$jobid"
}

# get nr of running jobs for given step
sj_running_jobs() {
    [[ $# -eq 1 ]] || return 0
    local stepid=$1
    local nrem=$(sj_check $stepid)
    [[ ${#nrem} -eq 0 ]] && nrem=0
    echo $nrem
}

# return nr of jobs running/pending for user
sj_njobs_all() {
    sj_list_jobs $SYS_WHOAMI | wc -l
}

# wait for scheduler
sj_wait_scheduler() {
    [[ $# -eq 2 ]] || return
    local stepid=$1
    local waituntil=$2

    step_info_msg "WAITING FOR SCHEDULER"

    local njobs=$((waituntil+1))
    local nwait=0
    # wait until scheduler becomes sufficiently free
    while [[ $njobs -gt $waituntil ]]; do
        echo -n "."
        njobs=$(sj_njobs_all)
        [[ $njobs -gt 0 ]] && sleep $SJ_CHECK_POLLTIME
        if [[ $nwait -eq 79 ]]; then
            echo
            nwait=0
        else
            nwait=$((nwait+1))
        fi
    done
    echo
}

toHMS() {
    [[ $# -eq 1 ]] || return 0
    local H=$(calc "int($1)")
    local M=$(calc "int(60*($1 - $H))")
    local S=$(calc "int(3600*($1-$H-($M/60)))")
    printf "%02dh%02dm%02ds" $H $M $S
}

# wait function for jobs at given step
sj_wait() {
    step_info_msg "WAITING"

    # make sure poll time is a value
    is_decimal "$SJ_CHECK_POLLTIME" || SJ_CHECK_POLLTIME=5

    local nrem=0
    local nwait=0
    local nwaittot=0
    
    # get nr of remaining at start
    nrem=$(sj_check $STEP_ID)
    local nstart=$nrem

    debug_msg "sj_wait: $nrem jobs remaining!"
    
    # just sleep a bit before checking
    [[ $nrem -gt 0 ]] && sleep $SJ_CHECK_POLLTIME

    # summarize every twenty minutes
    local tline=$((20*60))
    
    # if tline is less than the polltime, summarize every 2*polltime
    [[ $tline -gt $SJ_CHECK_POLLTIME ]] || tline=$((2*SJ_CHECK_POLLTIME))
    
    # nr of waits per summary
    local nprog=$(calc "int(0.5+($tline/$SJ_CHECK_POLLTIME))")
    
    # loop until nrem is zero
    while [[ $nrem -gt 0 ]]; do
        echo -n "."
        nrem=$(sj_check $STEP_ID)
        local ec=$?
        # job id file missing -> nothing submitted, skip
        [[ $ec -eq 1 ]] && { nrem=0; break; }

        # some other error
        if [[ $ec -ne 0 ]]; then
            error_msg "sj_wait() failed with error code $ec"
            return 1
        fi
        debug_msg "sj_wait: still $nrem jobs remaining!"
        # sleep if jobs are remaining
        [[ $nrem -gt 0 ]] && sleep $SJ_CHECK_POLLTIME
        if [[ $nwait -eq $nprog ]]; then
            nwaittot=$((nwaittot+nwait))
            local T=0
            local iv=0
            local Test=0
            local Teststr="N/A"
	        local ndone=$((nstart-nrem))
            T=$(calc "$nwaittot*$SJ_CHECK_POLLTIME/3600.0")
            [[ $ndone -gt 0 ]] && iv=$(calc "$T/$ndone")
            local ivtst=0
            ivtst=$(calc "$iv==0")
            [[ $ivtst -eq 1 ]] || {
                Test=$(calc "$iv*$nrem")
                Teststr=$(toHMS $Test) #$(printf "%.2f" $Test)
            }
            printf "%s / %s [hrs] , %d/%d\n" $(toHMS $T) $Teststr $ndone $nrem
            nwait=0
        else
            nwait=$((nwait+1))
        fi
    done
    echo
    step_info_msg "DONE WAITING"
    
    # remove jobid file
    ###step_error_msg "NOTE jobid file is NOT removed! $fjid"
    clear_jobid_file

    return 0
}



SJ_R_JOBLOG_NAME=""
#
# get job log file base name given the args
# NOTE that pid, if given must be prefixed with a '_'
#
sj_joblog_base() {
    local label="${1-*}"
    local stepid="${2-*}"
    local jobstr="${3-*}"
    SJ_R_JOBLOG_NAME="${label}_${stepid}_${jobstr}_${PID_PIPELINE}"
}
sj_joblog_base_nopid() {
    local label="${1-*}"
    local stepid="${2-*}"
    local jobstr="${3-*}"
    SJ_R_JOBLOG_NAME="${label}_${stepid}_${jobstr}"
}

# split job log file name
# <dir>/<todo label>_<stepid>_<jobstr>_<pid pipeline>_<redirect>_<scheduler id>.log
# note that <jobstr> may contain '_'
SJ_R_JOBLOGDIR=""
SJ_R_TODO_LABEL=""
SJ_R_STEPID=""
SJ_R_JOBLABEL=""
SJ_R_PID=""
SJ_R_ISERROR=""
SJ_R_SCHEDID=""
sj_joblog_split() {
    local f="$1"
    SJ_R_TODO_LABEL=""
    SJ_R_STEPID=""
    SJ_R_JOBLABEL=""
    SJ_R_PID=""
    SJ_R_ISERROR=""
    SJ_R_SCHEDID=""
    SJ_R_JOBLOGDIR=$(dirname $f)
    f=$(basename $f .log)
    local saveIFS="$IFS"
    IFS="_"
    local elems=( $f )
    local nelems=${#elems[@]}
    local terminal=""
    if (( nelems < 6 )); then
        return 255
    elif (( nelems == 6 )); then
        SJ_R_TODO_LABEL="${elems[0]}"
        SJ_R_STEPID="${elems[1]}"
        SJ_R_JOBLABEL="${elems[2]}"
        SJ_R_PID="${elems[3]}"
        terminal="${elems[4]}"
        SJ_R_SCHEDID="${elems[5]}"
    else
        local nx=$((nelems-6))
        SJ_R_TODO_LABEL="${elems[0]}"
        SJ_R_STEPID="${elems[1]}"
        for (( i=0; i<nx+1; i++ )); do
            if (( i == 0 )); then
                SJ_R_JOBLABEL="${elems[2]}"
            else
                SJ_R_JOBLABEL="${SJ_R_JOBLABEL}_${elems[$((i+2))]}"
            fi
        done
        SJ_R_PID="${elems[$((nx+3))]}"
        terminal="${elems[$((nx+4))]}"
        SJ_R_SCHEDID="${elems[$((nx+5))]}"
    fi
    IFS="$saveIFS"
    SJ_R_ISERROR=0
    [[ "$terminal" == "${SJ_JOBLOG_STDERR_EXT}" ]] && SJ_R_ISERROR=1
}

# template - also used in kill_pending_jobs()
R_SJ_SUBMIT_TMP_FILE_TEMPL=
sj_submit_tmp_file_templ() {
    R_SJ_SUBMIT_TMP_FILE_TEMPL="$DIR_PROJECT/$BASE_SCRATCH/script_${PL_TODO_NAME}_${1-}"
}

# script to be submitted - full name created using mktemp
R_SJ_SUBMIT_TMP_FILE=
sj_submit_tmp_file() {
    R_SJ_SUBMIT_TMP_FILE=
    [[ $# -eq 1 ]] || return 255
    sj_submit_tmp_file_templ "$1"
    local tmpfile=$(mktemp "${R_SJ_SUBMIT_TMP_FILE_TEMPL}_XXXXXXXXX")
    chmod 755 "$tmpfile"
    R_SJ_SUBMIT_TMP_FILE="$tmpfile"
}

# create shell script to be submitted
sj_prepare_script() {
    SJ_BATCH_SCRIPT=""

    [[ -d "$DIR_PROJECT/$BASE_SCRATCH" ]] || { error_msg "project scratch dir does not exist <$DIR_PROJECT/$BASE_SCRATCH>"; return 255; }

    # check if run mode uses batch or not - used below in the script to reset the name
    local use_batch_id="$SJ_BATCH_JOBID_NAME"
    is_runmode_batch || use_batch_id="SHELL"
    #
    # Create script to be run
    #
    sj_submit_tmp_file "$STEP_ID" || return 255
    chmod 755 "$R_SJ_SUBMIT_TMP_FILE"
    {
        echo "#!/usr/bin/env bash"
        # add declare statements in order to restore all arrays/dictionnaries
        if [[ -f "$EXPORT_DECL_FILE" ]]; then
            # read declare statements from file
            cat "$EXPORT_DECL_FILE"
        else
            # save declare statements from local environment
            export_decl_stdout
        fi        
    } >> "$R_SJ_SUBMIT_TMP_FILE"
    # create main script
    cat >> "$R_SJ_SUBMIT_TMP_FILE" <<EOF
# make sure TMPDIR points somewhere sensible
TMPDIR=${TMPDIR:=""}
if [[ -n "$TMPDIR" ]]; then
    [[ -d "$TMPDIR" ]] || TMPDIR=
fi

# set arrays related to TODO file
set_todo_arrays

# move to workspace
cd "$DIR_PROJECT"

# find python3
set_python3

# restore all arrays/dictionnaries
#ftmp=""
#tmp_make ftmp declall
#restore_decl_all > \$ftmp
#if [[ \$? -eq 0 ]]; then
#  source \$ftmp
#fi
#tmp_delete \$ftmp

# make sure it is correct...
export SJ_BATCH_JOBID_NAME=$use_batch_id


echo "#PIPELINE VERSION  : $OP_VERSION"
echo "#PIPELINE DIR      : $DIR_ORTHOLOGER"
echo "#WORK DIR          : $DIR_PROJECT" 
echo "#TODO              : $PL_TODO"
echo "#TODO LABEL        : $PL_TODO_NAME"
echo "#MACHINE           : \$(uname -a)"
echo "#DATE              : \$(date_locale)"
echo "#COMMAND           : $SJ_SCRIPT $SJ_SCRIPT_ARGS"
echo "#JOB ID            : \${!SJ_BATCH_JOBID_NAME}"
echo "#JOB LABEL         : $SJ_JOBSTR"

__sj_tstart=\$(date +%s)

( tmp_setup_trap; source "$SJ_SCRIPT" "$SJ_SCRIPT_ARGS" )
__sj_err=\$?

__sj_tend=\$(date +%s)
__sj_dt=\$((__sj_tend-__sj_tstart))
echo "#TIME: \$__sj_dt"
if [[ \$__sj_err -eq 0 ]]; then
   echo "#JOBDONEOK"
else
   echo "#JOBERRCODE: \$__sj_err"
fi
debug_msg "Quitting script OK"
exit \$__sj_err
EOF
    SJ_BATCH_SCRIPT="$R_SJ_SUBMIT_TMP_FILE"
    ###cp $SJ_BATCH_SCRIPT ${SJ_BATCH_SCRIPT}.save
}


# create shell script to be submitted
sj_prepare_script_array() {
    #NOT DONE!!!!
    fatal_msg "CALLED sj_prepare_script_array!"

}

#-----------------------------------------------------
# reset everything for the sj_submit_job() function
#-----------------------------------------------------
sj_submit_precheck() {
    SJ_BATCH_JOBID=

    # the job log bases should be set in sj_submit()
    [[ ${#SJ_JOBLABEL}  -gt 0 ]] || { error_msg "SJ_JOBLABEL is empty!";  return 255; }
    [[ ${#SJ_JOBSTDOUT} -gt 0 ]] || { error_msg "SJ_JOBSTDOUT is empty!"; return 255; }
    [[ ${#SJ_JOBSTDERR} -gt 0 ]] || { error_msg "SJ_JOBSTDERR is empty!"; return 255; }


    # check batch script
    [[ -x "$SJ_BATCH_SCRIPT" ]] || { error_msg "batch script not executable by user <$SJ_BATCH_SCRIPT>"; return 255; }

    # make sure this one is not empty
    [[ ${#SJ_NCPUPROC} -gt 0 ]] || SJ_NCPUPROC=1
}

#-----------------------------------------------------
# submit the job using scheduler or local
# args:  <JOBSTR> <SUBLBL> [ <SCRIPT ARGS> ]
#-----------------------------------------------------
sj_submit() {
    local ec=0
    # check argument
    [[ $# -eq 0 ]] && return 1

    local args="${@}"

    # get all arrays
    #debug_msg "restore_decl_all for stepid $STEP_ID"
    #if is_runmode_interactive; then
    #tmp_make ftmp declall
    #restore_decl_all >& $ftmp
    #if [[ $? -eq 0 ]]; then
    #    source $ftmp

    # read declare statements from file
    # this is needed when jobs are run in local mode - xargs runs this in a sub-shell
    [[ -f "$EXPORT_DECL_FILE" ]] && {
        debug_msg "scheduler: re-declaring arrays/dicts from file $EXPORT_DECL_FILE"
        source "$EXPORT_DECL_FILE"
    }

    # get arguments
    debug_msg "scheduler: process args ${args}"
    sj_process_args $args || return 1

    # try to find a reasonable name for the job
    # get joblog dir
    get_joblog_dir "$SJ_SUBLBL"
    [[ -d "$R_JOBLOG_DIR/" ]] || fatal_msg "no job log directory: $R_JOBLOG_DIR"

    # get joblog names - these are actually base names - hence at least .log should be added
    sj_joblog_base "$PL_TODO_NAME" "$STEP_ID" "$SJ_JOBSTR"
    SJ_JOBLABEL="$SJ_R_JOBLOG_NAME"
    SJ_JOBSTDOUT="$R_JOBLOG_DIR/${SJ_JOBLABEL}_${SJ_JOBLOG_STDOUT_EXT}"
    SJ_JOBSTDERR="$R_JOBLOG_DIR/${SJ_JOBLABEL}_${SJ_JOBLOG_STDERR_EXT}"
    
    # prepare script, script name in SJ_BATCH_SCRIPT
    local save=$SJ_BATCH_JOBID_NAME
    is_runmode_local && SJ_BATCH_JOBID_NAME=SHELL

    debug_msg "scheduler: prepare script"
    sj_prepare_script
    [[ $? -eq 0 ]] ||  { SJ_BATCH_JOBID_NAME=$save; return 255; }

    # make sure all variables are proper
    debug_msg "scheduler: submit precheck"
    sj_submit_precheck || { SJ_BATCH_JOBID_NAME=$save; return 255; }

    # restore
    SJ_BATCH_JOBID_NAME=$save

    # some verbose info
    verbose_info_msg "job script ($SJ_BATCH_SCRIPT)"
    verbose_info_msg "-> stdout: ${SJ_JOBSTDOUT}_local.log"
    verbose_info_msg "-> stderr: ${SJ_JOBSTDERR}_local.log"

    #---------------------------------------------------------------------
    # submit the job locally
    if is_runmode_local; then
        verbose_info_msg "-> job is run locally"
        # run script locally, not in a sub shell (use 'source')
        ###        source "$SJ_BATCH_SCRIPT" 1>"${SJ_JOBSTDOUT}_local.log" 2>"${SJ_JOBSTDERR}_local.log"
        # run script locally, in a sub shell, like this tmp files will be removed earlier
        #tmp_add_file "$SJ_BATCH_SCRIPT"
        #echo "TMP_FILES before exec of script: $TMP_FILES"
        ( "$SJ_BATCH_SCRIPT" 1>"${SJ_JOBSTDOUT}_local.log" 2>"${SJ_JOBSTDERR}_local.log"; exit $? )
        ec=$?
        #echo "TMP_FILES after exec of script: $TMP_FILES"
        #echo "ERROR code $ec"
        [[ $ec -eq 0 ]] && rm -f "$SJ_BATCH_SCRIPT"
        return $ec
    fi

    #---------------------------------------------------------------------
    # submit through scheduler
    local isbatch=0
    if is_runmode_batch; then
        isbatch=1
        verbose_info_msg "-> non-interactive submit"
        sj_submit_job
        err=$?
    elif is_runmode_interactive; then
        verbose_info_msg "-> interactive submit"
        sj_interactive_job
        err=$?
    else
        fatal_msg "unknown run mode in sj_submit()"
    fi
    
    if [[ $err -ne 0 ]]; then
        step_error_msg "error submitting job"
    else
        if [[ $isbatch -eq 1 ]]; then
            # save jobid only if true batch job (not interactive)
            step_info_msg "submitted ($SJ_BATCH_JOBID) $SJ_JOBLABEL"
            sj_save_jobid $SJ_BATCH_JOBID
            err=$?
            [[ $err -eq 0 ]] || step_error_msg "failed saving jobid $SJ_BATCH_JOBID"
        fi
    fi
    return $err
}


# submit with retries
# NOT to be used for now! Needs work...
#submit_with_retry() {
#    fatal_msg "do not use submit_with_retry() yet!!!"
#
#    # run submit command
#    local cmdlist="${@}"
#    local counter=0
#    local maxcount=5
#    local ec=0;
#    while [[ $counter -lt $maxcount ]]; do
#        sj_submit "$cmdlist"
#        ec=$?
#        counter=$((counter+1))
#        [[ $ec -eq 0 ]] && break;
#        step_error_msg "submit failed with error $ec --- retry $((counter+1))/$maxcount"
#        local retrysleep=$(awk "BEGIN{ print 1.0+($RANDOM/32767.0) }")
#        sleep $retrysleep
#    done
#    return $ec
#}

# check for running jobs
# returns
#    0 - ok (even if no arg is given or no jobid file found)
#    1 - sj_list_jobs() failed
sj_check_jobids() {
    [[ $# -eq 1 ]] || return 0
    local stepid=$1
    ###local fjid="$(get_jobid_file $stepid)"
    local fjid=""
    v_get_jobid_file fjid "$stepid"
    debug_msg "check jobids in file $fjid"
    [[ -f "$fjid" ]] || { debug_msg "no jobid file is found!"; return 0; }

    #echoerr "DBG: sj_check_jobids content:"
    #cat $fjid 1>&2
    #echoerr "DBG: END sj_check_jobids content"
    local njid=0
    local njidrem=0
    local jids=""
    local jidremtot=""
    local ec=0

    # check if user has any jobs
    local uid=$SYS_WHOAMI
    local nmaxjobs=$(sj_list_jobs $uid | wc -l)
###    local nmaxjobs=$(squeue -u $uid --noheader -o "%.7i" 2>/dev/null | wc -l) # nr of jobs+1

    [[ $nmaxjobs -gt 0 ]] || return 0

    # loop over all job id's
    IFS=$_ORG_IFS
    local jobid=
    ###for jobid in $(cat "$fjid"); do
    while read jobid; do
        debug_msg "got jobid $jobid"
        if [[ $njid -eq 0 ]]; then
            jids="$jobid"
        else
            jids="${jids},$jobid"
        fi
        njid=$((njid+1))

        # check in blocks of 100
        if [[ $njid -ge 100 ]]; then

            # get nr of jobs still in queue (including pending jobs)
            local jidrem=
            jidrem=$(sj_list_jobs $uid "$jids")
            [[ $? -eq 0 ]] || ec=1
            jidremtot="$jidremtot $jidrem"
            njid=0
        fi
    done < "$fjid"
    # check any remaining
    if [[ $njid -gt 0 ]]; then

        local jidrem=
        jidrem=$(sj_list_jobs $uid "$jids")
        [[ $? -eq 0 ]] || ec=2
        jidremtot="$jidremtot $jidrem"
        njid=0
    fi

    # update jobid file
    rm -f "$fjid"
    touch "$fjid"
    local j=
    for j in $jidremtot; do
        echo $j >> "$fjid"
    done

    # echo the remaining job ids
    echo $jidremtot
    return $ec
}

sj_get_njobs() {
    [[ $# -eq 1 ]] || return 0
    local stepid=$1
    local fjid="$(get_jobid_file $stepid)"

    [[ -f "$fjid" ]] || return 0
    wc -l < "$fjid"
}

# check for running jobs
sj_check() {
    [[ $# -eq 1 ]] || return 0
    local stepid=$1
    local jobids=
    jobids=$(sj_check_jobids $stepid)
    [[ $? -eq 0 ]] || { echo 0; return 255; }

    debug_msg "sj_check: jobids = $jobids"
    
    local njidrem=0
    local j=
    for j in $jobids; do
        njidrem=$((njidrem+1))
    done
    echo $njidrem
    return 0
}


sj_cancel_jobs() {
    [[ $# -eq 1 ]] || return
    local stepid=$1
    local fjid="$(get_jobid_file $stepid)"
    [[ -f "$fjid" ]] || return 1
    local ncancelled=0
    local nfailed=0
    local jobid=
    for jobid in $(cat "$fjid"); do
        sj_cancel_jobid $jobid
        if [[ $? -ne 0 ]]; then
            nfailed=$((nfailed+1))
        else
            ncancelled=$((ncancelled+1))
        fi
    done
    info_msg "$stepid: cancelled $ncancelled jobs"
    [[ $nfailed -gt 0 ]] && error_msg "$stepid: failed cancelling $nfailed jobs!!"
}


# export functions
export -f sj_usage
export -f sj_process_args
export -f sj_save_jobid
export -f sj_running_jobs
export -f sj_wait_scheduler
export -f sj_wait

export -f sj_joblog_base
export -f sj_joblog_base_nopid
export -f sj_joblog_split

export -f sj_prepare_script
export -f sj_submit_tmp_file_templ
export -f sj_submit_tmp_file
export -f sj_submit
export -f sj_submit_precheck

export -f sj_check
export -f sj_get_njobs
