#!/Users/runner/bioconda/conda-bld/maker_1616698748543/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_place/bin/perl 

eval 'exec /Users/runner/bioconda/conda-bld/maker_1616698748543/_h_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_place/bin/perl  -S $0 ${1+"$@"}'
    if 0; # not running under some shell

use FindBin;
use lib "$FindBin::RealBin/../src/inc/perl/lib"; #micelaneous
use lib "$FindBin::RealBin/../lib";
use lib "$FindBin::RealBin/../perl/lib";

BEGIN {
    if($FindBin::RealBin =~ /\/src\/bin\/?$/){
	my $bad = "$FindBin::RealBin/maker";
	(my $good =  $bad) =~ s/\/src\/bin\/maker$/\/bin\/maker/;
	die "ERROR: You are running MAKER from the src directory. This is what you called:\n".
	    "BAD --> $bad\n".
	    "\n".
	    "When MAKER has been correcly installed it will be located here:\n".
	    "GOOD --> $good\n".
	    "\n".
	    "Make sure you followed the instructions in the INSTALL documentation,\n".
	    "succeffully completed the installation, and that you have set the location\n".
	    "for MAKER's executables correctly in your PATH environmental variable.\n\n";
    }
}

#always load forks before anything else
use forks;
use forks::shared;

#now load everyting else
use strict "vars";
use strict "refs";
use warnings;
use Carp;
use vars qw($LOG $VERSION $RANK $C);
use Cwd;
use List::Util;
use Storable qw(nfreeze thaw nstore retrieve);
use FileHandle;
use File::Path;
use Getopt::Long qw(:config no_ignore_case);
use File::Temp qw(tempfile tempdir);
use MAKER::ConfigData;
use Bio::DB::Fasta;
use GI;
use Dumper::GFF::GFFV3;
use Iterator::Any;
use Iterator::Fasta;
use Iterator::GFF3;
use Fasta;
use FastaChunker;
use maker::auto_annotator;
use cluster;
use repeat_mask_seq;
use runlog;
use ds_utility;
use GFFDB;
use Error qw(:try);
use Error::Simple;
use Perl::Unsafe::Signals;
use Process::MpiChunk;
use Process::MpiTiers;
use Parallel::Application::MPI qw(:all);
use Proc::Signal;
use File::NFSLock;
use POSIX;
use Sys::Hostname;

BEGIN{
   $VERSION = '3.01.03';

   #what to do on ^C
   $SIG{INT}  = sub {print STDERR "SIGINT received\n"; print STDERR Carp::longmess; exit_maker(SIGINT);};
   $SIG{QUIT} = sub {print STDERR "SIGQUIT received\n"; exit_maker(SIGQUIT);};
   $SIG{ABRT} = sub {print STDERR "SIGABRT received\n"; exit_maker(SIGABRT);};
   $SIG{KILL} = sub {print STDERR "SIGKILL received\n"; exit_maker(SIGKILL);};
   $SIG{TERM} = sub {print STDERR "SIGTERM received\n"; exit_maker(SIGTERM);};

   #output to log file of seq that caused rank to die
   $SIG{'__DIE__'} = sub {
       if($LOG && defined $_[0]){
	   $LOG->add_entry("DIED","COUNT",$LOG->get_die_count+1);
       }
             
       $RANK ||= 'NA';
       die $_[0]."--> rank=$RANK, hostname=".Sys::Hostname::hostname()."\n";
   };
   
   #supress warnings from storable module
   $SIG{'__WARN__'} = sub {
       warn $_[0] if($_[0] !~ /Not a CODE reference/ &&
		     $_[0] !~ /Can\'t store item/ &&
		     $_[0] !~ /Find\:\:skip_pattern/ &
		     $_[0] !~ /File\/Find\.pm/);
   };


   #fix issue with AnyDBM_File reodering
   require MAKER::ConfigData;
   my $any = MAKER::ConfigData->config('AnyDBM_ISA');
   if($any && @$any){
       for my $mod (@$any) {
	   if (eval "require $mod") {
	       @AnyDBM_File::ISA = ($mod);
	       last;
	   }
       }
   }
}

{
my $tid = threads->tid if exists $INC{'threads.pm'};
END {
    my $stat = $?; #help preserve exit status
    my $is_thread = 1 if(defined($tid) && $tid != threads->tid);
    cleanup($is_thread);
    $? = $stat if($stat);
}
}

#--keep STDOUT from buffering
select((select(STDOUT), $|=1)[0]); #make STDOUT buffer flush immediately

#--set object variables for serialization of data
$Storable::forgive_me = 1; #allows serializaion of objects with GLOBs
$Storable::Deparse = 1; #now serializes CODE refs
$Storable::Eval = 1; #now serializes CODE refs

#--usage statement
my $usage = "
MAKER version $VERSION

Usage:

     maker [options] <maker_opts> <maker_bopts> <maker_exe>


Description:

     MAKER is a program that produces gene annotations in GFF3 format using
     evidence such as EST alignments and protein homology. MAKER can be used to
     produce gene annotations for new genomes as well as update annotations
     from existing genome databases.

     The three input arguments are control files that specify how MAKER should
     behave. All options for MAKER should be set in the control files, but a
     few can also be set on the command line. Command line options provide a
     convenient machanism to override commonly altered control file values.
     MAKER will automatically search for the control files in the current
     working directory if they are not specified on the command line.

     Input files listed in the control options files must be in fasta format
     unless otherwise specified. Please see MAKER documentation to learn more
     about control file  configuration.  MAKER will automatically try and
     locate the user control files in the current working directory if these
     arguments are not supplied when initializing MAKER.

     It is important to note that MAKER does not try and recalculated data that
     it has already calculated.  For example, if you run an analysis twice on
     the same dataset you will notice that MAKER does not rerun any of the
     BLAST analyses, but instead uses the blast analyses stored from the
     previous run. To force MAKER to rerun all analyses, use the -f flag.

     MAKER also supports parallelization via MPI on computer clusters. Just
     launch MAKER via mpiexec (i.e. mpiexec -n 40 maker). MPI support must be
     configured during the MAKER installation process for this to work though
     

Options:

     -genome|g <file>    Overrides the genome file path in the control files

     -RM_off|R           Turns all repeat masking options off.

     -datastore/         Forcably turn on/off MAKER's two deep directory
      nodatastore        structure for output.  Always on by default.

     -old_struct         Use the old directory styles (MAKER 2.26 and lower)

     -base    <string>   Set the base name MAKER uses to save output files.
                         MAKER uses the input genome file name by default.

     -tries|t <integer>  Run contigs up to the specified number of tries.

     -cpus|c  <integer>  Tells how many cpus to use for BLAST analysis.
                         Note: this is for BLAST and not for MPI!

     -force|f            Forces MAKER to delete old files before running again.
			 This will require all blast analyses to be rerun.

     -again|a            recaculate all annotations and output files even if no
			 settings have changed. Does not delete old analyses.

     -quiet|q            Regular quiet. Only a handlful of status messages.

     -qq                 Even more quiet. There are no status messages.

     -dsindex            Quickly generate datastore index file. Note that this
                         will not check if run settings have changed on contigs

     -nolock             Turn off file locks. May be usful on some file systems,
                         but can cause race conditions if running in parallel.

     -TMP                Specify temporary directory to use.

     -CTL                Generate empty control files in the current directory.

     -OPTS               Generates just the maker_opts.ctl file.

     -BOPTS              Generates just the maker_bopts.ctl file.

     -EXE                Generates just the maker_exe.ctl file.

     -MWAS    <option>   Easy way to control mwas_server for web-based GUI

                              options:  STOP
                                        START
                                        RESTART

     -version            Prints the MAKER version.

     -help|?             Prints this usage statement.


";

#------------------------------------------------------------------------------
#----------------------------------- MAIN -------------------------------------
#------------------------------------------------------------------------------


#--process MPICC and MPIDIR from @ARGV first
my %OPT;
carp "Calling GetOptions pre-init" if($main::debug);
Getopt::Long::Configure(qw(pass_through));
GetOptions("MPICC=s" => \$OPT{MPICC}, #hidden
	   "MPIDIR=s" => \$OPT{MPIDIR},
	   "mpi"      => \$main::is_mpi); #hidden

#--set MPI configuration if specified on the command line
if($OPT{MPICC} || $OPT{MPIDIR}){
    print STDERR "Overriding default MPI settings\n";
    die "ERROR: You must specify both MPICC and MPIDIR\n"
	if(!$OPT{MPICC} && !$OPT{MPIDIR});
    die "ERROR:$OPT{MPICC} does not exist\n" if(! -f $OPT{MPICC});
    die "ERROR:$OPT{MPICC} isn't mpicc\n" if($OPT{MPICC} !~ /^(.*\/)?mpicc$/);
    die "ERROR:$OPT{MPIDIR} does not exist\n" if(! -d $OPT{MPIDIR});
    die "ERROR:Cannot find mpi.h in $OPT{MPIDIR}\n"
	if(! -f "$OPT{MPIDIR}/mpi.h");

    #paths must be absolute
    $OPT{MPICC} = Cwd::abs_path($OPT{MPICC});
    $OPT{MPIDIR} = Cwd::abs_path($OPT{MPIDIR});

    #configure MPI but do not write to disk (in memory only)
    MAKER::ConfigData->set_feature(mpi_support => 1);
    MAKER::ConfigData->set_feature(mpi_override => 1);
    MAKER::ConfigData->set_config(MPICC => $OPT{MPICC});
    MAKER::ConfigData->set_config(MPIDIR => $OPT{MPIDIR});
}

#--Start MPI
carp "Calling MPI_Init" if($main::debug);
MPI_Init();
carp "Calling MPI_Comm_rank" if($main::debug);
my $rank = $RANK = MPI_Comm_rank(); #my proccess number
GI::RANK($rank);
carp "Calling MPI_Comm_size" if($main::debug);
my $size = MPI_Comm_size(); #how many proccesses
my $root = 0; #define root node (only changed for debugging)
my $start_time = time();

#---Process options on the command line
carp "Calling GetOptions" if($main::debug);
Getopt::Long::Configure(qw(no_pass_through));
GetOptions("RM_off|R" => \$OPT{R},
	   "force|f" => \$OPT{force},
	   "genome|g=s" => \$OPT{genome},
	   "est=s" => \$OPT{est},
	   "protein=s" => \$OPT{protein},
	   "altest=s" => \$OPT{altest},
	   "est2genome" => \$OPT{est2genome},
	   "altest2genome" => \$OPT{altest2genome},
	   "protein2genome" => \$OPT{protein2genome},
	   "est_forward" => \$OPT{est_forward},
	   "cpus|c=i" => \$OPT{cpus},
	   "tries=i" =>\$OPT{tries},
	   "dsindex" =>\$OPT{dsindex},
	   "again|a" =>\$OPT{again},
	   "nolock|nl" =>\$OPT{_no_lock},
	   "nolocal" =>\$main::nolocal,
	   "quiet|q" =>\$main::quiet,
	   "qq"    =>\$main::qq,
           "check" =>\$OPT{check},
	   "base=s" =>\$OPT{out_name},
	   "datastore!" =>\$OPT{datastore},
	   "old_struct" =>\$main::old_struct,
	   "TMP=s" => \$OPT{TMP},
	   "mpi_blastdb=s" => \$OPT{mpi_blastdb},
	   "debug" => \$main::debug,
	   "debugmpi" => \$main::debugmpi,
	   "ignore_nfs_tmp" => \$OPT{ignore_nfs_tmp},
	   'fix_nucleotides' => \$main::fix_nucleotides,
	   "help|?" => sub {print $usage if($rank == $root); exit_maker(0)},
	   "version" => sub{print "$VERSION\n" if($rank == $root);
			    exit_maker(0)},
	   "MWAS=s" =>sub {exec("$FindBin::RealBin/../MWAS/bin/mwas_server", $_[1])
			       if($rank == $root);
			   exit_maker(0);},
	   "CTL" => sub {GI::generate_control_files()
			     if($rank == $root);
			 exit_maker(0);},
	   "OPTS" => sub {GI::generate_control_files('','opts')
			      if($rank == $root);
			  exit_maker(0);},
	   "BOPTS" => sub {GI::generate_control_files('','bopts')
			       if($rank == $root);
			   exit_maker(0);},
	   "EXE" => sub {GI::generate_control_files('','exe')
			     if($rank == $root);
			 exit_maker(0);},
	   "EVM" => \$OPT{EVM},
    );
$main::quiet = 1 if($main::qq);
$C = 0;

#--verbose version info
if($main::debug){
    print STDERR "****MODULE VERSION INFO\n";
    print "perl version $^V\n";
    foreach my $key (sort keys %INC){
	next unless($key =~ /\.pm$/);
	my $name = $key;
	$name =~ s/\.pm$//;
	$name =~ s/\//::/g;

	my $ver;
	eval "\$ver = \$${name}::VERSION";
	$ver = 'UNKNOWN' if(!defined($ver));
	$ver = 'UNKNOWN' if($ver =~ /^\-1\,/);

	print STDERR "\t$ver\t$name\t$INC{$key}\n"
    }
    print "\n\n";
}

#--get files off the command line
my @ctlfiles = @ARGV;
if (! @ctlfiles) {
    if (-e "maker_opts.ctl" && -e "maker_bopts.ctl" && -e "maker_exe.ctl" && -e "maker_evm.ctl") {
	@ctlfiles = qw(maker_opts.ctl maker_bopts.ctl maker_exe.ctl maker_evm.ctl);
    }
    elsif($rank == $root) {
	print STDERR  "ERROR: Control files not found\n";
	print $usage;
	exit_maker(0);
    }
    else{
	exit_maker(0);
    }
}

#------INITIATE MPI VARIABLES AND SIGNAL CODES------
#--mpi message tags
my $who_I_am       = 1111;
my $what_I_want    = 2222;
my $result_status  = 3333;
my $request_status = 4444;
my $c_res_status   = 5555;
my $c_req_status   = 6666;
my $chunk_status   = 7777;
my $work_order     = 8888; #generic data tag
my $mpi_data       = 9999;
my $mpi_list       = 1212;
my $message_length = 1313;
my $message_stat   = 1414;

#--message_stat type signals
my $message_ok = 0;

#--what_I_want type signals
my $need_tier   = 1;
my $need_helper = 2;
my $have_c_res  = 3;
my $need_c_res  = 4;

#--request_status signals
my $wait_as_helper = 5;
my $yes_work       = 6;
my $yes_helper     = 7;
my $no_helper      = 8;
my $reset          = 9;
my $wait_ask_again = 10;
my $made_note      = 11;
my $terminate      = 12;

#--c_req_status signals
my $go_chunk       = 13;
my $c_reset        = 14;
my $c_terminate    = 15;

#--results_status signals
my $yes_result = 16;
my $no_result  = 17;

#--c_res_status signal
my $yes_c_res = 18;
my $no_c_res  = 19;

#--chunk_status signals
my $yes_chunk = 20;
my $no_chunk  = 21;

#---variables for thread and the root node
my @c_results;
my @res_loc;
my @limbo_stack;
my @active;
my @chunks :shared;
my @returned_chunks :shared;
my $t_need_flag :shared;
my $t_tier :shared;
my $t_tier_result :shared;
my $t_chunk :shared;
my $t_chunk_result :shared;
my $t_terminate :shared;
my $empty;

#---Process the control files
#varibles that are persistent outside of try blocks
my %CTL_OPT;
my $iter;
my $DS_CTL;
my $GFF_DB;
my $build;
my $g_index;
my $gdbfile;
my @failed;
my @inter;

#--parse options from control files
if($rank == $root){
    print STDERR "STATUS: Parsing control files...\n" unless($main::quiet);
    carp "Calling GI::load_control_files" if($main::debug);
    %CTL_OPT = GI::load_control_files(\@ctlfiles, \%OPT, $size);
    $CTL_OPT{_is_root} = 1;
}

#--syncronize TMP for all nodes
if($rank == $root) {
    for(my $i = 0; $i < $size; $i++){
	next if $i == $root;
	MPI_Send(\$File::NFSLock::TYPE, $i, $mpi_data); #sync lock type
	MPI_Send(\$CTL_OPT{TMP}, $i, $mpi_data);
    }
    carp "Calling GI::new_instance_temp" if($main::debug);
    my $tmp = GI::new_instance_temp($CTL_OPT{TMP});
    carp "Calling GI::mount_check" if($main::debug);
    my $mount = GI::mount_check($tmp =~ /(.*\/)[^\/]*$/);

    my %temp_list;
    my @index;
    ($index[$rank]) = $mount;
    $temp_list{$index[$rank]} = $tmp;

    for(my $i = 0; $i < $size; $i++){
	next if($i == $root);
	my $list;
	MPI_Recv(\$list,  $i, $mpi_list);
	my ($mount, $tmp) = @$list; #mount is ip:directory
	$index[$i] = $mount;
	$temp_list{$index[$i]} = $tmp;
    }
    
    #send final tmp to all nodes
    carp "Calling GI::set_global_temp" if($main::debug);
    GI::set_global_temp($temp_list{$index[$root]}); #for root    
    for(my $i = 0; $i < $size; $i++){
	next if($i == $root);
	MPI_Send(\$temp_list{$index[$i]}, $i, $mpi_data);
    }
}
else{
    my $tmp;
    my $lock_type;
    MPI_Recv(\$lock_type, $root, $mpi_data); #fix lock type
    $File::NFSLock::TYPE = $lock_type; #fix lock type
    MPI_Recv(\$tmp, $root, $mpi_data);
    carp "Calling GI::new_instance_temp" if($main::debug);
    $tmp = GI::new_instance_temp($tmp);
    carp "Calling GI::mount_check" if($main::debug);
    my ($mount) = GI::mount_check($tmp =~ /(.*\/)[^\/]*$/);
    my $list = [$mount, $tmp];
    MPI_Send(\$list, $root, $mpi_list);
    MPI_Recv(\$tmp, $root, $mpi_data);
    carp "Calling GI::set_global_temp" if($main::debug);
    GI::set_global_temp($tmp);
}

#--process and index BLAST databases
if($rank == $root){
    #---set up blast databases and indexes for analyisis
    print STDERR "STATUS: Processing and indexing input FASTA files...\n"
	unless($main::quiet);
    if ($CTL_OPT{force} && !$CTL_OPT{_multi_chpc}){
	carp "Calling File::Path::rmtree" if($main::debug);
	File::Path::rmtree($CTL_OPT{mpi_blastdb});
    }

    my @to_do;
    my @ins = qw(genome protein est altest repeat_protein rmlib);
    foreach my $in (@ins){
	my @files = split(/\,/, $CTL_OPT{$in});
	my %uniq = map {/^([^\:]+)\:?(.*)?/} @files;
	carp "Calling GI::s_abs_path" if($main::debug);
	@files = map {($uniq{$_}) ? GI::s_abs_path($_).":$_" : $_} keys %uniq;

	my $key  = ($in =~ /protein/) ? 'protein' : 'nucleotide';
	my $bins = ($in eq 'genome' || $in eq 'rmlib') ? 1 : 10;
	my $bdir = $CTL_OPT{mpi_blastdb};
	my $alt  = $CTL_OPT{alt_peptide};
	
	push(@to_do, map {['split_db', $_, $key, $bins, $bdir, $alt]} @files);
    }
    carp "Calling List::Util::shuffle" if($main::debug);
    @to_do = List::Util::shuffle(@to_do); #shuffle the order (efficiency)

    my $split_count = @to_do;
    while((my $args = shift @to_do) || $split_count > 0){
	if($size == 1){
	    shift @$args;
	    carp "Calling GI::split_db" if($main::debug);
	    my $split = GI::split_db(@$args);
	    carp "Calling GI::build_fasta_index" if($main::debug);
	    GI::build_fasta_index($split);
	    $split_count--;
	    next;
	}

	my $who;
	my $res_stat;
	MPI_Recv(\$who, MPI_ANY_SOURCE, $who_I_am);
	MPI_Recv(\$res_stat, $who, $result_status);
	if($res_stat == $yes_result){
	    my $data;
	    MPI_Recv(\$data, $who, $mpi_data);
	    push(@to_do, @$data);
	    carp "Calling List::Util::shuffle" if($main::debug);
	    @to_do = List::Util::shuffle(@to_do);  #shuffle the order
	    $split_count--;
	}

	if($args){
	    MPI_Send(\$yes_work, $who, $request_status);
	    MPI_Send(\$args, $who, $mpi_data);
	}
	else{
	    MPI_Send(\$wait_ask_again, $who, $request_status);
	}
    }

    for(my $i = 0; $i < $size; $i++){
	next if($i == $root);
	MPI_Recv(\$empty, $i, $who_I_am);
	MPI_Recv(\$empty, $i, $result_status);
	MPI_Send(\$reset, $i, $request_status);
    }

    #separte the final send from the others to create a barrier
    carp "Calling GI::create_blastdb" if($main::debug);
    GI::create_blastdb(\%CTL_OPT);
    $gdbfile = $CTL_OPT{_g_db}[0];
    for(my $i = 0; $i < $size; $i++){
        next if($i == $root);
	MPI_Send(\$gdbfile, $i, $mpi_data);
    }
}
else{
    my $split;
    while(1){
	my $res_stat = ($split) ? $yes_result: $no_result;
	MPI_Send(\$rank, $root, $who_I_am);
	MPI_Send(\$res_stat, $root, $result_status);
	if($split){
	    MPI_Send(\$split, $root, $mpi_data);
	    undef $split;
	}

	my $req_stat;
	MPI_Recv(\$req_stat, $root, $request_status);
	
	if($req_stat == $reset){
	    MPI_Recv(\$gdbfile, $root, $mpi_data);
	    last;
	}

	if($req_stat == $wait_ask_again){
	    sleep 0.1;
	    next;
	}

	my $args;
	MPI_Recv(\$args, $root, $mpi_data);
	my $type = shift @$args;
	if($type eq 'split_db'){
	    carp "Calling GI::split_db" if($main::debug);
	    $split = GI::split_db(@$args);
	    foreach my $e (@$split){
		$e = ['index', $e];
	    }
	}
	else{
	    carp "Calling GI::build_fasta_index" if($main::debug);
	    GI::build_fasta_index($args)
	}
    }
}
carp "Calling GI::build_fasta_index" if($main::debug);
$g_index = GI::build_fasta_index([$gdbfile]);

if($rank == $root){
#--set up GFF3 DB
    print STDERR "STATUS: Setting up database for any GFF3 input...\n"
	unless($main::quiet);
    carp "Calling GFFDB::new" if($main::debug);
    $GFF_DB = new GFFDB(\%CTL_OPT);
    carp "Calling GFFDB::next_build" if($main::debug);
    $build = $GFF_DB->next_build;

    #--open datastructure controller
    carp "Calling ds_utility::new" if($main::debug);
    $DS_CTL = ds_utility->new(\%CTL_OPT);

    #---load genome file into iterator
    carp "Calling Iterator::Fasta::new" if($main::debug);
    $iter = new Iterator::Fasta($CTL_OPT{_g_db}->[0]);
    carp "Calling Iterator::Fasta::skip_file" if($main::debug);
    $iter->skip_file($DS_CTL->{log});
    carp "Calling Iterator::Fasta::step" if($main::debug);
    $iter->step($CTL_OPT{'_step'});

    if($OPT{dsindex}){
	$iter->skip_file($DS_CTL->{log});
	while(my $q_def = $iter->nextDef()){
	    my $id = Fasta::def2SeqID($q_def);
	    my $safe_id = Fasta::seqID2SafeID($id);
	    my $dir = $DS_CTL->id_to_dir($id);
	    my $message = (-f "$dir/$safe_id.gff") ? 'FINISHED' : 'STARTED';
	    $DS_CTL->add_entry($id, $dir, $message);
	}
	exit_maker(0);
    }
}

#---------------------------
#------RUN WITHOUT MPI------
#---------------------------
if($size == 1){
    print STDERR "STATUS: Now running MAKER...\n" unless($main::quiet);

    my $tier;
    my $f_count = @failed;
    my $i_count = @inter;
    while(my $q_def = $iter->nextDef() || shift @failed || shift @inter){
	$tier = Process::MpiTiers->new({q_def   => $q_def,
                                    g_index => $g_index,
                                    CTL_OPT => \%CTL_OPT,
                                    DS_CTL  => $DS_CTL,
                                    dbfile  => $GFF_DB->dbfile,
				    build   => $build},
                                   '0',
                                   'Process::MpiChunk'
	    );

	next if($tier->terminated);

	#take a short break before processing previously failed contigs
	#this handles heavy processor usage when failure is related
	#to maker process overlap
	sleep 1 if($f_count != @failed);

	$tier->run_all(0);

	push(@failed, $tier->q_def) if ($tier->failed);
	push(@inter, $tier->q_def)
	    if($tier->interrupt && $i_count == @inter);

	$f_count = @failed; #reset failure count
	$i_count = @inter; #reset interrupt count
    }

    print STDERR "\n\nMaker is now finished!!!\n\n" unless($main::qq);
    my $end_time = time;
    my $elapsed= $end_time-$start_time;
    print "\n\n";
    print "Start_time: $start_time\n";
    print "End_time:   $end_time\n";
    print "Elapsed:    $elapsed\n";

    exit_maker(0);
}

#--------------------------------------
#---------PRIMARY MPI PROCCESS---------
#--------------------------------------

#--check if root node
if ($rank == $root) {
    print STDERR "STATUS: Now running MAKER...\n" unless($main::quiet);

    while(1){
	#see if all contigs are finished
	if($iter->finished && @failed == 0 && @inter == 0){
	    my $go = 0;
	    foreach my $n (@active){
		if((defined($n) && $n != 0)){
		    $go = 1;
		    last;
		}
	    }
	    last if(! $go)
	}
	
	#see who asks for a file
	my $who;
	print "COMM INITIALIZATION\t|  RECV\t|  who_I_am\t\t\t|  $rank\t<--\tANY\t|  ".$C++."\n" if($main::debugmpi);
	MPI_Recv(\$who, MPI_ANY_SOURCE, $who_I_am);
	
	#see what the mpi node wants
	my $what;
	print "COMM INITIALIZATION\t|  RECV\t|  what_I_want\t\t\t|  $rank\t<--\t$who\t|  ".$C++."\n" if($main::debugmpi);
	MPI_Recv(\$what, $who, $what_I_want);
	
	#if the node wants a tier to process, do this
	if($what == $need_tier){
	    #receive result status
	    my $rs_type;
	    print "COMM TIER REQUESTED\t|  RECV\t|  result_status (is result?)\t|  $rank\t<--\t$who\t|  ".$C++."\n" if($main::debugmpi);
	    MPI_Recv(\$rs_type, $who, $result_status);
	    
	    #get result if available
	    if($rs_type == $yes_result){
		my $result;
		print "COMM TIER REQUESTED\t|  RCV2\t|  mpi_data (tier_result)\t|  $rank\t<--\t$who\t|  ".$C++."\n" if($main::debugmpi);
		MPI_Recv(\$result, $who, $mpi_data);
		push(@failed, $result->{-q_def}) if ($result->{-failed});
		push(@inter, $result->{-q_def}) if ($result->{-interrupt});
	    }
	    
	    #if a contig is available send tier
	    my $tier;
	    my $f_count = @failed;
	    my $i_count = @inter;
	    while(my $q_def = $iter->nextDef || shift @failed || shift @inter){
		$tier = Process::MpiTiers->new({q_def => $q_def,
						g_index => $g_index,
						CTL_OPT => \%CTL_OPT,
						DS_CTL  => $DS_CTL,
						dbfile  => $GFF_DB->dbfile,
						build   => $build},
					       $who,
					       'Process::MpiChunk',
					       0
		    );
		
		last if(! $tier->terminated);
	    }
	    
	    #tag tier as being seen in two conflicting maker instances
	    $tier->{_seen}++ if(defined $tier && $i_count != @inter);
	    
	    #take a short break before processing failed contigs
	    #this handles heavy processor usage when failure is related
	    #to maker process overlap
	    sleep 1 if($f_count != @failed);
	    
	    if(defined $tier && ! $tier->terminated){
		#say tier is available and send it
		print "COMM TIER REQUESTED\t|  SEND\t|  req_stat (yes_work)\t\t|  $rank\t-->\t$who\t|  ".$C++."\n" if($main::debugmpi);
		MPI_Send(\$yes_work, $who, $request_status);
		print "COMM TIER REQUESTED\t|  SND2\t|  mpi_data (tier)\t\t|  $rank\t-->\t$who\t|  ".$C++."\n" if($main::debugmpi);
		MPI_Send(\$tier, $who, $mpi_data);
		$tier = undef;
		@limbo_stack = grep {$_ != $who} @limbo_stack if(defined($active[$who]) && $active[$who] == 0);
		$active[$who] = 1;
	    }
	    else{
		print "COMM TIER REQUESTED\t|  SEND\t|  req_stat (wait_ask_again)\t|  $rank\t-->\t$who\t|  ".$C++."\n" if($main::debugmpi);
		MPI_Send(\$wait_ask_again, $who, $request_status);
		push(@limbo_stack, $who) if(!defined($active[$who]) || $active[$who]);
		$active[$who] = 0;
	    }

	    next;
	}
	#if the node wants a helper or needs a chunk result, do this
	elsif($what == $need_helper || $what == $need_c_res){
	    #--first send c_res_status
	    # send ids of nodes with chunk results
	    if(defined ($res_loc[$who])){
		print "COMM HAVE C_RESULT\t|  SEND\t|  c_res_status (yes_c_res)\t|  $rank\t-->\t$who\t|  ".$C++."\n" if($main::debugmpi);
		MPI_Send(\$yes_c_res, $who, $c_res_status);
		print "COMM HAVE C_RESULT\t|  SND2\t|  mpi_list (result_loc_list)\t|  $rank\t-->\t$who\t|  ".$C++."\n" if($main::debugmpi);
		MPI_Send(\$res_loc[$who], $who, $mpi_list);
		$res_loc[$who] = undef;
	    }
	    #no one has anything yet
	    else{
		print "COMM HAVE C_RESULT\t|  SEND\t|  c_res_status (no_c_res)\t|  $rank\t-->\t$who\t|  ".$C++."\n" if($main::debugmpi);
		MPI_Send(\$no_c_res, $who, $c_res_status);
	    }
	    
	    #continue the rest if the node needs a helper
	    if($what == $need_helper){
		#find the number of helpers required
		my $num_helpers_req;
		print "HELPER/RESULT REQUESTED\t|  RECV\t|  work_order (num_helpers_req)\t|  $rank\t<--\t$who\t|  ".$C++."\n" if($main::debugmpi);
		MPI_Recv(\$num_helpers_req, $who, $work_order);
		
		#number of helpers available
		my $sec_node_avail = @limbo_stack;
		
		#signal that no helpers are available
		if($sec_node_avail == 0){
			print "HELPER/RESULT REQUESTED\t|  SEND\t|  req_stat (no_helpers_avail)\t|  $rank\t-->\t$who\t|  ".$C++."\n" if($main::debugmpi);
			MPI_Send(\$no_helper, $who, $request_status);
		}
		else{ #if node helpers are available
		    #helpers to send
		    my $helpers = [];
		    
		    #secondary node helpers
		    while(@{$helpers} < $num_helpers_req && @limbo_stack > 0){
			my $helper = shift @limbo_stack;
			$active[$helper] = 1; #indicates who they work for
			push(@{$helpers}, $helper);
		    }
		    
		    #say help is available and send ids of the helpers
		    print "HELPER/RESULT REQUESTED\t|  SEND\t|  req_stat (yes_helpers_avail)\t|  $rank\t-->\t$who\t|  ".$C++."\n" if($main::debugmpi);
		    MPI_Send(\$yes_helper, $who, $request_status);
		    print "HELPER/RESULT REQUESTED\t|  SND2\t|  mpi_list (helper_loc_list)\t|  $rank\t-->\t$who\t|  ".$C++."\n" if($main::debugmpi);
		    MPI_Send(\$helpers, $who, $mpi_list);
		    
		    #signal to limbo nodes to become a helper
		    foreach my $helper (@$helpers){
			#turn node into helper node
			print "HELPER/RESULT REQUESTED\t|  RECV\t|  who_I_am (clr nd)\t\t|  $rank\t<--\t$helper\t|  ".$C++."\n" if($main::debugmpi);
			MPI_Recv(\$empty, $helper, $who_I_am); #clear
			print "HELPER/RESULT REQUESTED\t|  RECV\t|  what_I_want (clr nd)\t\t|  $rank\t<--\t$helper\t|  ".$C++."\n" if($main::debugmpi);
			MPI_Recv(\$empty, $helper, $what_I_want); #clear
			print "HELPER/RESULT REQUESTED\t|  RECV\t|  result_status (clr nd)\t|  $rank\t<--\t$helper\t|  ".$C++."\n" if($main::debugmpi);
			MPI_Recv(\$empty, $helper, $result_status); #clear
			print "HELPER/RESULT REQUESTED\t|  SEND\t|  req_stat (wait_as_helper nd)\t|  $rank\t-->\t$helper\t|  ".$C++."\n" if($main::debugmpi);
			MPI_Send(\$wait_as_helper, $helper, $request_status);
		    }
		}
	    }

	    next;
	}
	#if the node has a chunk result, do this
	elsif($what == $have_c_res){
	    #extra handshaking to avoid non-blocking send in hydra MVAPICH2
	    print "COMM HAVE C_RESULT\t|  SEND\t|  req_stat (arrived?)\t\t|  $root\t-->\t$who\t|  ".$C++."\n" if($main::debugmpi);
	    MPI_Send(\$made_note, $who, $request_status);
	    
	    #get the owner of the result
	    my $owner;
	    print "COMM HAVE C_RESULT\t|  RECV\t|  work_order (res own fr root)\t|  $rank\t<--\t$who\t|  ".$C++."\n" if($main::debugmpi);
	    MPI_Recv(\$owner, $who, $work_order);

	    #take note of owner to tell him he has a result waiting
	    push(@{$res_loc[$owner]}, $who);

	    next;
	}
	#if what the node wants is something else
	else{
	    confess "ERROR: Invalid request type\n";
	}
    }
    
    #---tell mpi nodes to terminate
    for(my $i = 1; $i < $size; $i++){
	print "TERMINATION\t|  RECV\t|  who_I_am (clr)\t\t|  $rank\t<--\t$i\t|  ".$C++."\n" if($main::debugmpi);
	MPI_Recv(\$empty, $i, $who_I_am); #clear who I am response
	print "TERMINATION\t|  RECV\t|  what_I_want (clr)\t|  $rank\t<--\t$i\t|  ".$C++."\n" if($main::debugmpi);
	MPI_Recv(\$empty, $i, $what_I_want); #clear what I want response
	print "TERMINATION\t|  RECV\t|  result_status (clr)\t|  $rank\t<--\t$i\t|  ".$C++."\n" if($main::debugmpi);
	MPI_Recv(\$empty, $i, $result_status); #clear result status response
	print "TERMINATION\t|  SEND\t|  req_stat (terminate)\t|  $rank\t-->\t$i\t|  ".$C++."\n" if($main::debugmpi);
	MPI_Send(\$terminate, $i, $request_status); #signal to terminate
    }

    print STDERR "\n\nMaker is now finished!!!\n\n" unless($main::qq);
    my $end_time = time;
    my $elapsed = $end_time-$start_time;
    print "\n\n";
    print "Start_time: $start_time\n";
    print "End_time:   $end_time\n";
    print "Elapsed:    $elapsed\n";
}
#------SECONDARY MPI PROCESSES------
else {
    my $tier_result;
    my @tiers; #tier buffer (multiple level tiers)
    my $chunk_result;
    my $be_helper;

    #create threads
    $t_need_flag = 1;
    my $thr = threads->new(\&node_thread, $gdbfile);

    while (1) {
	#====INTERNAL CHUNK THREAD
	die "FATAL: Thread terminated, causing all processes to fail\n"
	    if(!$thr || !$thr->is_running);

	#check on results from internal thread
	if (defined($t_chunk_result)){
	    my $c_res =  ${retrieve($t_chunk_result)};
	    unlink($t_chunk_result);
	    $t_chunk_result = undef;
	    update_chunk(\@tiers, $c_res);
	    run(\@tiers, $rank);
	    next;
	}
	#====END THREAD

	#decide what this node needs
	my $what;
	my $chunk;
	if(defined $chunk_result){
	    #NOTE: $be_helper is set to true here
	    $what = $have_c_res;
	}
	elsif($be_helper && (my $ftier = terminated(\@tiers))){
	    if(update_chunk(\@tiers, $ftier)){
		run(\@tiers, $rank);
		next;
	    }
	    else{
		#NOTE: $be_helper is set to true here
		$what = $have_c_res;
		$chunk_result = $ftier;
	    }
	}
	elsif($be_helper && !@tiers){
	    #see who needs help
	    my $who;
	    print "HELPER/RESULT REQUESTED\t|  RECV\t|  who_I_am (for no-root)\t|  $rank\t<--\tANY\t|  ".$C++."\n" if($main::debugmpi);
	    MPI_Recv(\$who, MPI_ANY_SOURCE, $who_I_am);

	    #get request_status for chunk (was chunk available?)
	    my $chunk_status;
	    print "HELPER/RESULT REQUESTED\t|  RECV\t|  c_req_stat (is_chunk? no-root)\t|  $rank\t<--\t$who\t|  ".$C++."\n" if($main::debugmpi);
	    MPI_Recv(\$chunk_status, $who, $c_req_status );

	    #if there is a chunk do this
	    if($chunk_status == $go_chunk){
		#get chunk to process
		my $r_chunk;
		print "HELPER/RESULT REQUESTED\t|  RCV2\t|  mpi_data (chunk for no-root)\t|  $rank\t<--\t$who\t|  ".$C++."\n" if($main::debugmpi);
		MPI_Recv(\$r_chunk, $who, $mpi_data);

		#if chunk is tier, treat as such
		if(ref($r_chunk) eq 'Process::MpiTiers'){
		   push(@tiers, $r_chunk);
		   run(\@tiers, $rank);
		   next;
		}

		#run chunk
		$r_chunk->run($rank);
		$chunk_result = $r_chunk;
		next;
	    }
	    #if the reset signal is received do this
	    elsif($chunk_status == $c_reset){
		$be_helper = 0;
		next;
	    }
	    else{
		confess "ERROR: Invalid chunk status signal\n;";
	    }
	    next;
	}
	elsif(!$be_helper && !@tiers){
	    $what = $need_tier;
	}
	elsif(!$be_helper && (my $tier = terminated(\@tiers))){
	    if(@tiers){
		update_chunk(\@tiers, $tier);
		run(\@tiers, $rank);
		next;
	    }
	    else{
		$what = $need_tier;

		#collect errors and failures if any
		$tier_result->{-error} = $tier->error;
		$tier_result->{-failed} = $tier->failed;
		$tier_result->{-interrupt} = $tier->interrupt
		    if(!$tier->{_seen});
		$tier_result->{-q_def} = $tier->q_def
		    if($tier->failed || $tier_result->{-interrupt});
		$tier = undef;
	    }
	}
	elsif((chunk_total_count(\@tiers) == 1) && (num_chunks(\@tiers) == 1)){
	    #run lonesome chunk (should never happen)
	    my $r_chunk = next_chunk(\@tiers); #running chunk
	
	    #if chunk is tier, treat as such
	    if(ref($r_chunk) eq 'Process::MpiTiers'){
	       push(@tiers, $r_chunk);
	       run(\@tiers, $rank);
	       next;
	    }
	
	    $r_chunk->run($rank);
	    update_chunk(\@tiers, $r_chunk);
	    run(\@tiers, $rank);
	    next;
	}
	elsif((result_count(\@tiers) == chunk_total_count(\@tiers)-1) && (num_chunks(\@tiers) == 1)){
	    #run last chunk outside of thread
	    my $r_chunk = next_chunk(\@tiers); #running chunk
	    die "ERROR: Logic error\n" if(! $r_chunk);

	    #if chunk is tier, treat as such
	    if(ref($r_chunk) eq 'Process::MpiTiers'){
	       push(@tiers, $r_chunk);
	       run(\@tiers, $rank);
	       next;
	    }

	    $r_chunk->run($rank);
	    update_chunk(\@tiers, $r_chunk);
	    run(\@tiers, $rank);
	    next;
	}
	elsif(!$t_need_flag && (result_count(\@tiers) == chunk_total_count(\@tiers) - 1)){
	    #waiting on thread
	    sleep 0.1;
	    next;
	}
	elsif((num_chunks(\@tiers) > 1) || (!$t_need_flag && num_chunks(\@tiers) > 0)){
	    #gets chunk or tier
	    my $r_chunk;
	    if($t_need_flag){
		$r_chunk = next_chunk(\@tiers);
	    }

	    #if chunk is tier, treat as such
	    if(ref($r_chunk) eq 'Process::MpiTiers'){
	    	push(@tiers, $r_chunk);
	    	run(\@tiers, $rank);
	    	next;
	    }
	    
	    $chunk = $r_chunk;
	    $what = $need_helper;
	}
	elsif(result_count(\@tiers) < chunk_total_count(\@tiers)){
	    my $r_chunk = next_chunk(\@tiers) if($t_need_flag);
	    
	    #if chunk is tier, treat as such
	    if(ref($r_chunk) eq 'Process::MpiTiers'){
	       push(@tiers, $r_chunk);
	       run(\@tiers, $rank);
	       next;
	    }

	    $chunk = $r_chunk;
	    $what = $need_c_res;
	}
	else{
	    run(\@tiers, $rank);
	    next;
	}

	#tell the  primary process what node it is speaking to
	print "COMM INITIALIZATION\t|  SEND\t|  who_I_am\t\t\t|  $rank\t-->\t$root\t|  ".$C++."\n" if($main::debugmpi);
	MPI_Send(\$rank, $root, $who_I_am );
		
	#--tell primary node what this node needs
	print "COMM INITIALIZATION\t|  SEND\t|  what_I_want\t\t\t|  $rank\t-->\t$root\t|  ".$C++."\n" if($main::debugmpi);
	MPI_Send(\$what, $root, $what_I_want );
	
	#if what I want is a tier do this
	if($what == $need_tier){
	    #Send result status
	    my $rs_type = (defined($tier_result)) ? $yes_result: $no_result;
	    my $stat = (defined($tier_result)) ? "yes_result": "no_result" if($main::debugmpi);
	    print "COMM TIER REQUESTED\t|  SEND\t|  result_status ($stat)\t|  $rank\t-->\t$root\t|  ".$C++."\n" if($main::debugmpi);
	    MPI_Send(\$rs_type, $root, $result_status );
	    
	    #Send result if available
	    if($rs_type == $yes_result){
		print "COMM TIER REQUESTED\t|  SND2\t|  mpi_data (tier_result)\t|  $rank\t-->\t$root\t|  ".$C++."\n" if($main::debugmpi);
		MPI_Send(\$tier_result, $root, $mpi_data);
		$tier_result = undef;
	    }
	    
	    #get request_status for the tier
	    my $req_status;
	    print "COMM TIER REQUESTED\t|  RECV\t|  req_stat (is tier?)\t\t|  $rank\t<--\t$root\t|  ".$C++."\n" if($main::debugmpi);
	    MPI_Recv(\$req_status, $root, $request_status);
	    
	    #get tier and run if it if there is one
	    if($req_status == $yes_work){
	       my $tier;
	       print "COMM TIER REQUESTED\t|  RCV2\t|  mpi_data (tier)\t\t|  $rank\t<--\t$root\t|  ".$C++."\n" if($main::debugmpi);
	       MPI_Recv(\$tier, $root, $mpi_data);
	       push(@tiers, $tier);
	       run(\@tiers, $rank);
	       next;
	    }
	    #just wait and then try again later
	    elsif($req_status == $wait_ask_again){
		sleep 0.1;
		next;
	    }
	    #reset signal received
	    elsif($req_status == $reset){
		$be_helper = 0;
		next;
	    }
	    #wait as helper if asked to (blocks with MPI_Recv)
	    elsif($req_status == $wait_as_helper){
		$be_helper = 1;
		next
	    }
	    #termination signal received
	    elsif($req_status == $terminate){
                last; #exits MPI loop
	    }
	    else{
		confess "ERROR: Invalid request status type\n";
	    }

	    next;
	} #if what I want is help or the result from a helper do this
	elsif ($what == $need_helper || $what == $need_c_res){
	    #give chunk to thread
	    if(defined($chunk) && chunk_total_count(\@tiers) > 1){
		$t_need_flag = 0;
		my $file = (tempfile("mpiXXXXX", UNLINK => 0, TMPDIR => 1))[1];
		nstore(\$chunk, $file);
		$t_chunk = $file;
		$chunk = undef;
	    }

	    #check c_result_status
	    my $c_res_stat;
	    print "COMM HAVE C_RESULT\t|  RECV\t|  c_res_status (is c_res?)\t|  $rank\t<--\t$root\t|  ".$C++."\n" if($main::debugmpi);
	    MPI_Recv(\$c_res_stat, $root, $c_res_status);

	    #if there are chunk results, do this
	    my $locs = [];
	    if($c_res_stat == $yes_c_res){
		#get ids of nodes with chunk result
		print "HELPER/RESULT REQUESTED\t|  RCV2\t|  mpi_list (result_loc_list)\t|  $rank\t<--\t$root\t|  ".$C++."\n" if($main::debugmpi);
		MPI_Recv(\$locs, $root, $mpi_list);
	    }

	    #send off chunks if the node needs a helper
	    my $helpers = [];
	    if ($what == $need_helper){
		#send the number of helpers required
		my $num_helpers_req = (num_chunks(\@tiers) - @$locs > 0) ?
		    num_chunks(\@tiers) - @$locs : 0;

		print "HELPER/RESULT REQUESTED\t|  SEND\t|  work_order (num_helpers_req)\t|  $rank\t-->\t$root\t|  ".$C++."\n" if($main::debugmpi);
		MPI_Send(\$num_helpers_req, $root, $work_order);
		
		#see if helper is available
		my $help_stat;
		print "HELPER/RESULT REQUESTED\t|  RECV\t|  req_stat (is helper avail?)\t|  $rank\t<--\t$root\t|  ".$C++."\n" if($main::debugmpi);
		MPI_Recv(\$help_stat, $root, $request_status);
		
		if($help_stat == $yes_helper){
		    print "HELPER/RESULT REQUESTED\t|  RCV2\t|  mpi_list (helper_loc_list)\t|  $rank\t<--\t$root\t|  ".$C++."\n" if($main::debugmpi);
		    MPI_Recv(\$helpers, $root, $mpi_list); #get helper ids
		}
	    }

	    #send chunk to helper
	    foreach my $helper (sort {$a <=> $b} @{$helpers}){
		print "HELPER/RESULT REQUESTED\t|  SEND\t|  who_I_am (for *helpers)\t|  $rank\t-->\t$helper\t|  ".$C++."\n" if($main::debugmpi);
		MPI_Send(\$rank, $helper, $who_I_am); #say I need help

		#send chunk if available
		my $r_chunk = next_chunk(\@tiers, 'chunk') || next_chunk(\@tiers, 'tier');
		while(ref($r_chunk) eq 'Process::MpiTiers' && @tiers < 3){
		    push(@tiers, $r_chunk);
		    actualize(\@tiers, $rank);
		    $r_chunk = next_chunk(\@tiers, 'chunk') || next_chunk(\@tiers, 'tier');
		}

		if($r_chunk){
		    print "HELPER/RESULT REQUESTED\t|  SEND\t|  c_req_stat (go_chunk *helpers)\t|  $rank\t-->\t$helper\t|  ".$C++."\n" if($main::debugmpi);
		    MPI_Send(\$go_chunk, $helper, $c_req_status); #is available
		    print "HELPER/RESULT REQUESTED\t|  SND2\t|  mpi_data (chunk *helpers)\t|  $rank\t-->\t$helper\t|  ".$C++."\n" if($main::debugmpi);
		    MPI_Send(\$r_chunk, $helper, $mpi_data); #send chunk
		}
		else{
		    print "HELPER/RESULT REQUESTED\t|  SEND\t|  c_req_stat (reset *helpers)\t|  $rank\t-->\t$helper\t|  ".$C++."\n" if($main::debugmpi);
		    MPI_Send(\$c_reset, $helper, $c_req_status); #send reset
		}
	    }

	    #get results from nodes and send them something or release them
	    foreach my $loc (@{$locs}){
		my $c_res;
		print "COMM HAVE C_RESULT\t|  RCV2\t|  mpi_data (c_res no-root)\t|  $rank\t<--\t$loc\t|  ".$C++."\n" if($main::debugmpi);
		MPI_Recv(\$c_res, $loc, $mpi_data);
		update_chunk(\@tiers, $c_res);
		actualize(\@tiers, $rank);

		print "HELPER/RESULT REQUESTED\t|  SEND\t|  who_I_am (for xhelpers)\t|  $rank\t-->\t$loc\t|  ".$C++."\n" if($main::debugmpi);
		MPI_Send(\$rank,  $loc, $who_I_am); #restablish relationship

		#send something else or release helper
		my $r_chunk = next_chunk(\@tiers, 'chunk') || next_chunk(\@tiers, 'tier');
		while(ref($r_chunk) eq 'Process::MpiTiers' && @tiers < 3){
		    push(@tiers, $r_chunk);
		    actualize(\@tiers, $rank);
		    $r_chunk = next_chunk(\@tiers, 'chunk') || next_chunk(\@tiers, 'tier');
		}

		if($r_chunk){
		    print "HELPER/RESULT REQUESTED\t|  SEND\t|  c_req_stat (go_chunk xhelpers)\t|  $rank\t-->\t$loc\t|  ".$C++."\n" if($main::debugmpi);
                    MPI_Send(\$go_chunk, $loc, $c_req_status); #say chunk is available
		    print "HELPER/RESULT REQUESTED\t|  SND2\t|  mpi_data (chunk xhelpers)\t|  $rank\t-->\t$loc\t|  ".$C++."\n" if($main::debugmpi);
                    MPI_Send(\$r_chunk, $loc, $mpi_data); #send chunk
		}
		else{
		    print "HELPER/RESULT REQUESTED\t|  SEND\t|  c_req_stat (reset xhelpers)\t|  $rank\t-->\t$loc\t|  ".$C++."\n" if($main::debugmpi);
                    MPI_Send(\$c_reset, $loc, $c_req_status); #send reset signal
		}
	    }

	    #nothing provided and nothing to advance to
	    if(!@{$helpers} && !@{$locs} && !$chunk && !$t_need_flag && !$t_tier_result){
		sleep 0.1; #don't sleep too log or you'll be sorry
	    }

	    #finally run local chunk if there is one
	    if($chunk){
		my $r_chunk = $chunk; #running chunk
		undef $chunk;
		$r_chunk->run($rank);
		update_chunk(\@tiers,$r_chunk);
	    }

	    run(\@tiers, $rank);
	    next;
	}
	#if just finished a helper chunk, inform that it is finished
	elsif($what == $have_c_res){
	    #extra handshaking to handle non-blocking send in hydra MPIAVCH2
	    my $req_status;
	    print "COMM HAVE C_RESULT\t|  RECV\t|  req_stat (arrived?)\t\t|  $rank\t<--\t$root\t|  ".$C++."\n" if($main::debugmpi);
            MPI_Recv(\$req_status, $root, $request_status);

	    if($req_status ne $made_note){
		confess "ERROR: Invalid request_status type\n";
	    }

	    #send the owner id of the result
	    my $owner = $chunk_result->id();
	    ($owner) = split(":", $owner);
	    print "COMM HAVE C_RESULT\t|  SEND\t|  work_order (res own to root)\t|  $rank\t-->\t$root\t|  ".$C++."\n" if($main::debugmpi);
	    MPI_Send(\$owner, $root, $work_order);

	    #send the result
	    print "COMM HAVE C_RESULT\t|  SND2\t|  mpi_data (c_res frm no-root)\t|  $rank\t-->\t$owner\t|  ".$C++."\n" if($main::debugmpi);
	    MPI_Send(\$chunk_result, $owner, $mpi_data);
	    $chunk_result = undef;

	    next;
	}
    }
}

#---------ALL NODES----------
exit_maker(0); #calls MPI_Finalize

#-----------------------------------------------------------------------------
#----------------------------------- SUBS ------------------------------------
#-----------------------------------------------------------------------------
{
my $clean = 0;
sub cleanup {
    my $is_thread = shift;

    #set alarm so it can just die if frozen
    local $SIG{ALRM} = sub { exit(@_) };
    alarm 300;

    return if($clean++); #clean runs only once

    File::Temp::cleanup() unless($is_thread); #I may not want to delete these right away
    GI::LOCK()->unlock() if(GI::LOCK()); #remove global lock

    #kill and wait for response
    my $time = time;
    while(abs($time - time) < 10){
	my ($found, $signaled) = Proc::Signal::reap_children_by_name(SIGUSR1, 'maintain.pl');
	if($found){
	    sleep 1;
	}
	else{
	    last;
	}
    }

    Proc::Signal::reap_children_by_name(SIGKILL, 'maintain.pl'); #clean up any stubborn maintainers

    if(!$is_thread){
	$t_terminate = 1;
	foreach(threads->list){
	    next if($_->is_detached);
	    $time = 0;
	    sleep 0.1 while(!$_->is_joinable && $time++ < 50); #wait on join up to five seconds
	    if($_->is_joinable){ #join to clear
	    	$_->join();
	    }
	    else{ #frozen? just kill it
	    	$_->kill('KILL');
		$_->detach();
	    }
	}
    }

    return;
}
}

sub exit_maker {
    cleanup();
    local $SIG{ALRM} = sub { exit(@_) };
    alarm 30;
    MPI_Finalize() if(!@_ || $_[0] == 0);
    exit(@_);
}

#other things for root node to do
#(thread allows root to process tiers like a secondary node)
sub node_thread {
   my $gdbfile = shift @_;
   my $tier;
   my $chunk;

   #set up thread signal to exit gracefully
   #don't override other signals within threads because
   #unlike ithreads forks uses real system signals
   $SIG{USR1} = sub{threads->exit};
   $SIG{INT}  = sub{print STDERR "SIGINT thread\n";threads->exit};
   $SIG{QUIT} = sub{print STDERR "SIGQUIT thread\n";threads->exit};
   $SIG{ABRT} = sub{print STDERR "SIGABRT thread\n";threads->exit};
   $SIG{KILL} = sub{print STDERR "SIGKILL thread\n";threads->exit};
   $SIG{TERM} = sub{print STDERR "SIGTERM thread\n";threads->exit};

   #setup global index because of weird NFS failure in thread
   my $g_index = GI::build_fasta_index([$gdbfile]);

   #thread initialized and waiting
   $t_need_flag = 1;

   while(! $t_terminate){
      #load serialized tier into tier
      if(! defined ($tier) && defined ($t_tier)){
	 $t_need_flag = 0;
	 $tier = ${retrieve($t_tier)};
	 $tier->rank($rank); #set rank
	 unlink $t_tier;
	 $t_tier = undef;
	 next;
      }#process tier
      elsif(defined($tier)){
	 #get chunk results from other nodes
	 while(my $file = shift @returned_chunks){
	    my $res = ${retrieve($file)};
	    unlink($file);
	    $tier->update_chunk($res);
	 }

	 #run the tier as far as possible
	 $tier->run($rank);

	 #get all chunks available
	 my $chnk = $tier->next_chunk;
	 while(my $o_chnk = $tier->next_chunk){
	    my $file = (tempfile("mpiXXXXX", UNLINK => 0, TMPDIR => 1))[1];
	    nstore(\$o_chnk, $file);
	    push (@chunks, $file);
	 }

	 #run chunks one at a time
	 $chnk->run($rank) if ($chnk);
	 $tier->update_chunk($chnk) if ($chnk);
	 while(my $file = shift @chunks){
	    $chnk = ${retrieve($file)};
	    unlink($file);
	    if($tier->failed){ #skip chunks after failure
		$tier->update_chunk($chnk);
		next;
	    }

	    $chnk->run($rank);
	    $tier->update_chunk($chnk);
	 }

	 #let tier advance if possible
	 $tier->run($rank);

	 #terminate tier, wait, or continue
	 if($tier->terminated){
	    my $tier_result;
	    $tier_result->{-error} = $tier->error;
	    $tier_result->{-failed} = $tier->failed;
	    $tier_result->{-interrupt} = $tier->interrupt if(!$tier->{_seen});
	    $tier_result->{-q_def} = $tier->q_def
		if($tier->failed || $tier_result->{-interrupt});
	    $tier = undef;

	    sleep 0.1 while (defined ($t_tier_result) && !$t_terminate); #pause
	    my $file = (tempfile("mpiXXXXX", UNLINK => 0, TMPDIR => 1))[1];
	    nstore(\$tier_result, $file);
	    $t_tier_result = $file;
	    $tier_result = undef;
	    $t_need_flag = 1;
	 }#take a break
	 elsif($tier->num_chunks == 0){
	    #keeps thread from hogging resources
	    sleep 0.1;
	 }

	 next;
      }#load serialized chunk into chunk
      elsif(! defined ($chunk) && defined ($t_chunk)){
	 $t_need_flag = 0;
	 $chunk = ${retrieve($t_chunk)};
	 unlink($t_chunk);
	 $t_chunk = undef;
	 next;
      }#process chunk
      elsif(defined($chunk)){
	 $chunk->run($rank);
	 sleep 0.1 while (defined ($t_chunk_result) && !$t_terminate); #pause
	 my $file = (tempfile("mpiXXXXX", UNLINK => 0, TMPDIR => 1))[1];
	 nstore(\$chunk, $file);
	 $t_chunk_result = $file;
	 $chunk = undef;
	 $t_need_flag = 1;
	 next;
      }#take a break
      else{
	 #keeps thread form hogging resources when there is nothing to do
	 sleep 0.1;
      }
   }

   return;
}
#----------------------------------------------------------------------------
#easy dump of string to a tempfile
sub totemp{
   my $data = shift @_;

   my ($fh, $name) = tempfile();
   print $fh $data;
   close ($fh);

   return $name;
}
#----------------------------------------------------------------------------
#function to itterate through chunk buffer and return the first MpiChunk found
sub next_chunk {
   my $buffer = shift;
   my $what = shift;

   #return outer chunks first
   if(!$what || $what eq 'chunk'){
      for(my $i = @$buffer - 1; $i >= 0; $i--){
	 my $chunk = $buffer->[$i]->next_chunk($what);
	 return $chunk if($chunk);
      }
   }
   else{ #return inner chunks first
      for(my $i = 0; $i < @$buffer; $i++){
         my $chunk = $buffer->[$i]->next_chunk($what);
         return $chunk if($chunk);
      }
   }

   return undef;
}

#----------------------------------------------------------------------------
#function to itterate through chunk buffer and returns first finished tier
sub terminated {
   my $buffer = shift;

   my @keep;
   while (my $t = shift @$buffer){
       if($t->terminated){
	   unshift(@$buffer, @keep);
	   return $t;
       }
       else{
	   push(@keep, $t);
       }
   }

   @$buffer = @keep;
   return;
}

#----------------------------------------------------------------------------
#function to itterate through chunk buffer and return the first MpiChunk found
sub update_chunk {
    my $buffer = shift;
    my $chunk = shift;;
    
    my $stat;
    foreach my $t (@$buffer){
	if($t->id eq $chunk->parent){
	    $t->update_chunk($chunk);
	    $stat = 1;
	    last;
	}
    }

    return $stat;
}
#----------------------------------------------------------------------------
#function to itterate through chunk buffer and count all total chunkse
sub chunk_total_count {
   my $buffer = shift;

   actualize($buffer);

   my $sum = 0;
   foreach my $t (@$buffer){
      $sum += $t->chunk_total_count();
   }

   return $sum;
}
#----------------------------------------------------------------------------
#function to itterate through chunk buffer andcount results
sub result_count {
   my $buffer = shift;

   my $sum = 0;
   foreach my $t (@$buffer){
      $sum += $t->result_count();
   }

   return $sum;
}
#----------------------------------------------------------------------------
#function to itterate through chunk buffer and update the correct one
sub num_chunks {
   my $buffer = shift;

   actualize($buffer);

   my $sum = 0;
   foreach my $t (@$buffer){
      $sum += $t->num_chunks();
   }

   return $sum;
}
#----------------------------------------------------------------------------
#function to itterate through chunk buffer to adavance without run
sub actualize {
   my $buffer = shift;
   my $rank = shift;

   foreach my $t (@$buffer){
       $t->actualize($rank);
   }
}
#----------------------------------------------------------------------------
#function to itterate through buffer to run
sub run {
   my $buffer = shift;
   my $rank = shift;

   actualize($buffer, $rank);   

   if(chunk_total_count($buffer) > 1){
       return;
   }

   while((chunk_total_count($buffer) == 1) && (my $chunk = next_chunk($buffer))){
       if(ref($chunk) eq 'Process::MpiTiers'){
	   push(@$buffer, $chunk);
	   actualize($buffer, $rank);
	   next;
       }
       else{
	   $chunk->run($rank);
	   update_chunk($buffer, $chunk);
	   actualize($buffer, $rank);
       }
   }

   return;
}
