package pacbio;

import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.concurrent.ArrayBlockingQueue;

import dna.AminoAcid;
import dna.Data;
import fileIO.ByteFile;
import fileIO.FileFormat;
import fileIO.ReadWrite;
import shared.Parser;
import shared.PreParser;
import shared.ReadStats;
import shared.Shared;
import shared.Timer;
import shared.Tools;
import stream.ConcurrentReadInputStream;
import stream.ConcurrentReadOutputStream;
import stream.FASTQ;
import stream.FastaReadInputStream;
import stream.Read;
import stream.SamLine;
import structures.IntList;
import structures.ListNum;

/**
 * Detects inverted repeats in PacBio reads.
 * Attempts to determine whether reads are chimeric
 * due to a missing adapter.
 * 
 * @author Brian Bushnell
 * @date June 5, 2019
 *
 */
public final class IceCreamFinder {
	
	/*--------------------------------------------------------------*/
	/*----------------        Initialization        ----------------*/
	/*--------------------------------------------------------------*/
	
	/**
	 * Code entrance from the command line.
	 * @param args Command line arguments
	 */
	public static void main(String[] args){
		//Start a timer immediately upon code entrance.
		Timer t=new Timer();
		
		//Create an instance of this class
		IceCreamFinder x=new IceCreamFinder(args);
		
		//Run the object
		x.process(t);
		
		//Close the print stream if it was redirected
		Shared.closeStream(x.outstream);
	}
	
	/**
	 * Constructor.
	 * @param args Command line arguments
	 */
	public IceCreamFinder(String[] args){
		
		{//Preparse block for help, config files, and outstream
			PreParser pp=new PreParser(args, getClass(), false);
			args=pp.args;
			outstream=pp.outstream;
		}
		
		//Set shared static variables prior to parsing
		ReadWrite.USE_PIGZ=ReadWrite.USE_UNPIGZ=true;
		ReadWrite.USE_BGZIP=ReadWrite.USE_UNBGZIP=ReadWrite.PREFER_BGZIP=true;
		ReadWrite.MAX_ZIP_THREADS=Shared.threads();
		FASTQ.TEST_INTERLEAVED=FASTQ.FORCE_INTERLEAVED=false;
		SamLine.SET_FROM_OK=true;
		Shared.setBufferData(1000000);
		Shared.FASTA_WRAP=511;
		Data.USE_SAMBAMBA=false;//Sambamba changes PacBio headers.
		Read.CHANGE_QUALITY=false;
		
		{//Parse the arguments
			final Parser parser=parse(args);
			Parser.processQuality();
			
			maxReads=parser.maxReads;
			overwrite=ReadStats.overwrite=parser.overwrite;
			append=ReadStats.append=parser.append;
			
			in1=parser.in1;
			extin=parser.extin;

			out1=parser.out1;
			extout=parser.extout;
		}	
		
		//Determine how many threads may be used
		threads=Shared.threads();
		
		//Ensure there is an input file
		if(in1==null){throw new RuntimeException("Error - at least one input file is required.");}
		fixExtensions(); //Add or remove .gz or .bz2 as needed
		checkFileExistence(); //Ensure files can be read and written
		checkStatics(); //Adjust file-related static fields as needed for this program 
		
		//Create output FileFormat objects
		ffout1=FileFormat.testOutput(out1, FileFormat.FASTQ, extout, true, overwrite, append, false);
		ffoutb=FileFormat.testOutput(outb, FileFormat.FASTQ, extout, true, overwrite, append, false);
		ffoutj=FileFormat.testOutput(outj, FileFormat.FASTA, extout, true, overwrite, append, false);

		//Create input FileFormat objects
		ffin1=FileFormat.testInput(in1, FileFormat.FASTQ, extin, true, true);

		queue=new ArrayBlockingQueue<ArrayList<Read>>(queuelen);
		
		final int alen=(adapter==null ? 20 : adapter.length);
		stride=(int)(alen*1.9f);
		window=(int)(alen*3.8f+10);
		ALIGN_ROWS=alen+1;
		ALIGN_COLUMNS=window+2;
		
		maxSwScore=MultiStateAligner9PacBioAdapter.maxQuality(alen);
		minSwScore=(int)(maxSwScore*MINIMUM_ALIGNMENT_SCORE_RATIO);
		minSwScoreSuspect=(int)(maxSwScore*Tools.min(MINIMUM_ALIGNMENT_SCORE_RATIO*SUSPECT_RATIO, MINIMUM_ALIGNMENT_SCORE_RATIO-((1-SUSPECT_RATIO)*.2f)));
		maxImperfectSwScore=MultiStateAligner9PacBioAdapter.maxImperfectScore(alen);
		suspectMidpoint=(minSwScoreSuspect+minSwScore)/2;
		if(adapter==null){alignAdapter=false;}
	}
	
	/*--------------------------------------------------------------*/
	/*----------------    Initialization Helpers    ----------------*/
	/*--------------------------------------------------------------*/
	
	/** Parse arguments from the command line */
	private Parser parse(String[] args){
		
		//Create a parser object
		Parser parser=new Parser();
		
		//Set any necessary Parser defaults here
		//parser.foo=bar;
		
		//Parse each argument
		for(int i=0; i<args.length; i++){
			String arg=args[i];
			
			//Break arguments into their constituent parts, in the form of "a=b"
			String[] split=arg.split("=");
			String a=split[0].toLowerCase();
			String b=split.length>1 ? split[1] : null;
			if(b!=null && b.equalsIgnoreCase("null")){b=null;}
			
			if(a.equals("verbose")){
				verbose=Tools.parseBoolean(b);
			}else if(a.equals("icecreamonly") || a.equals("ico")){
				filterIceCreamOnly=Tools.parseBoolean(b);
			}else if(a.equals("keepshortreads")){
				keepShortReads=Tools.parseBoolean(b);
			}else if(a.equals("modifyheader") || a.equals("modifyheaders") || a.equals("changeheader") || a.equals("changeheaders")){
				modifyHeader=Tools.parseBoolean(b);
			}else if(a.equals("ccs")){
				CCS=Tools.parseBoolean(b);
			}else if(a.equals("npad")){
				npad=Integer.parseInt(b);
			}else if(a.equals("realign")){
				realign=Tools.parseBoolean(b);
			}else if(a.equals("aligninverse") || a.equals("alignrc")){
				alignRC=Tools.parseBoolean(b);
			}else if(a.equals("alignadapter")){
				alignAdapter=Tools.parseBoolean(b);
			}else if(a.equals("flaglongreads")){
				flagLongReads=Tools.parseBoolean(b);
			}else if(a.equals("adapter")){
				adapter=b==null ? null : b.getBytes();
			}else if(a.equals("targetqlen") || a.equals("qlen")){
				targetQlen=Integer.parseInt(b);
			}else if(a.equals("maxqlenfraction") || a.equals("maxfraction") || a.equals("qlenfraction")){
				maxQlenFraction=Float.parseFloat(b);
			}else if(a.equals("junctionfraction") || a.equals("shortfraction")){
				minJunctionFraction=Float.parseFloat(b);
			}else if(a.equals("minratio1") || a.equals("ratio1") || a.equals("id1")){
				minRatio1=Float.parseFloat(b);
			}else if(a.equals("minratio2") || a.equals("ratio2") || a.equals("id2")){
				minRatio2=Float.parseFloat(b);
			}else if(a.equals("minratio") || a.equals("ratio") || a.equals("id")){
				minRatio1=minRatio2=Float.parseFloat(b);
			}else if(a.equals("adapterratio") || a.equals("ratior")){
				adapterRatio=Float.parseFloat(b);
			}else if(a.equals("minqlen")){
				minQlen=Integer.parseInt(b);
			}else if(a.equals("minscore")){
				minScore=Integer.parseInt(b);
			}else if(a.equals("parsecustom")){
				parseCustom=Tools.parseBoolean(b);
			}else if(a.equals("printtiming") || a.equals("extended")){
				printTiming=Tools.parseBoolean(b);
			}else if(a.equals("queuelen") || a.equals("qlen")){
				queuelen=Integer.parseInt(b);
			}else if(a.equals("outb") || a.equals("outbad")){
				outb=b;
			}else if(a.equals("outj") || a.equals("outjunctions") || a.equals("junctions")){
				outj=b;
			}else if(a.equals("parse_flag_goes_here")){
				long fake_variable=Tools.parseKMG(b);
				//Set a variable here
			}else if(parser.parse(arg, a, b)){//Parse standard flags in the parser
				//do nothing
			}else{
				outstream.println("Unknown parameter "+args[i]);
				assert(false) : "Unknown parameter "+args[i];
			}
		}
		
		return parser;
	}
	
	/** Add or remove .gz or .bz2 as needed */
	private void fixExtensions(){
		in1=Tools.fixExtension(in1);
	}
	
	/** Ensure files can be read and written */
	private void checkFileExistence(){
		//Ensure output files can be written
		if(!Tools.testOutputFiles(overwrite, append, false, out1, outb, outj)){
			outstream.println((out1==null)+", "+(outb==null)+", "+out1+", "+outb+", "+outj);
			throw new RuntimeException("\n\noverwrite="+overwrite+"; Can't write to output files "+out1+", "+outb+"\n");
		}
		
		//Ensure input files can be read
		if(!Tools.testInputFiles(false, true, in1)){
			throw new RuntimeException("\nCan't read some input files.\n");  
		}
		
		//Ensure that no file was specified multiple times
		if(!Tools.testForDuplicateFiles(true, in1, out1, outb, outj)){
			throw new RuntimeException("\nSome file names were specified multiple times.\n");
		}
	}
	
	/** Adjust file-related static fields as needed for this program */
	private static void checkStatics(){
		//Adjust the number of threads for input file reading
		if(!ByteFile.FORCE_MODE_BF1 && !ByteFile.FORCE_MODE_BF2 && Shared.threads()>2){
			ByteFile.FORCE_MODE_BF2=true;
		}
		
		assert(FastaReadInputStream.settingsOK());
	}
	
	/*--------------------------------------------------------------*/
	/*----------------         Outer Methods        ----------------*/
	/*--------------------------------------------------------------*/

	/** Create read streams and process all data */
	void process(Timer t){
		
		//Turn off read validation in the input threads to increase speed
		final boolean vic=Read.VALIDATE_IN_CONSTRUCTOR;
		Read.VALIDATE_IN_CONSTRUCTOR=Shared.threads()<4;
		
		//Create a read input stream
		final ConcurrentReadInputStream cris=makeCris();
		
		//Optionally create read output streams
		final ConcurrentReadOutputStream ros=makeCros(ffout1);
		final ConcurrentReadOutputStream rosb=makeCros(ffoutb);
		final ConcurrentReadOutputStream rosj=makeCros(ffoutj);
		
		//Reset counters
		readsProcessed=readsOut=0;
		basesProcessed=basesOut=0;
		junctionsOut=0;
		
		//Process the reads in separate threads
		spawnThreads(cris, ros, rosb, rosj);
		
		if(verbose){outstream.println("Finished; closing streams.");}
		
		//Write anything that was accumulated by ReadStats
		errorState|=ReadStats.writeAll();
		//Close the read streams
		errorState|=ReadWrite.closeStreams(cris, ros, rosb, rosj);
		
		//Reset read validation
		Read.VALIDATE_IN_CONSTRUCTOR=vic;
		
		//Report timing and results
		t.stop();
		outstream.println(Tools.timeReadsBasesProcessed(t, readsProcessed, basesProcessed, 8));
		outstream.println(Tools.readsBasesOut(readsProcessed, basesProcessed, readsOut, basesOut, 8, false));
		
		long readsFiltered=readsProcessed-readsOut;
		outstream.println(Tools.numberPercent("Reads Filtered:", readsFiltered, readsFiltered*100.0/(readsProcessed), 3, 8));
		outstream.println(Tools.number("Total ZMWs:", ZMWs, 8));
		outstream.println(Tools.numberPercent("Bad ZMWs:", iceCreamZMWs, iceCreamZMWs*100.0/(ZMWs), 3, 8));
		outstream.println(Tools.numberPercent("Missing Adapter:", missingAdapterZMWs, missingAdapterZMWs*100.0/(ZMWs), 3, 8));
		outstream.println(Tools.numberPercent("Hidden Adapter:", hiddenAdapterZMWs, hiddenAdapterZMWs*100.0/(ZMWs), 3, 8));
		outstream.println(Tools.number("Avg Score Ratio:", iceCreamRatio/missingAdapterZMWs, 3, 8));
		outstream.println(Tools.number("Score Cutoff:", minRatio2, 3, 8));
		
		if(printTiming){
			outstream.println("Iterations:         "+alignmentIters/1000000+"m");
			outstream.println("Short Iterations:   "+alignmentItersShort/1000000+"m");
			outstream.println("Elapsed:            "+elapsed/1000000+"ms");
			outstream.println("Elapsed Short:      "+elapsedShort/1000000+"ms");
		}
		
		if(parseCustom){
			outstream.println();
			outstream.println(Tools.numberPercent("True Positive:", truePositiveReads, truePositiveReads*100.0/(readsProcessed), 2, 8));
			outstream.println(Tools.numberPercent("True Negative:", trueNegativeReads, trueNegativeReads*100.0/(readsProcessed), 2, 8));
			outstream.println(Tools.numberPercent("False Positive:", falsePositiveReads, falsePositiveReads*100.0/(readsProcessed), 2, 8));
			outstream.println(Tools.numberPercent("False Negative:", falseNegativeReads, falseNegativeReads*100.0/(readsProcessed), 2, 8));
			
			double snr=(truePositiveReads+trueNegativeReads)/Tools.max(1, falsePositiveReads+falseNegativeReads);
			snr=10*Math.log10(snr);
			outstream.println(Tools.number("SNR:", snr, 2, 8));
		}
		
		//Throw an exception of there was an error in a thread
		if(errorState){
			throw new RuntimeException(getClass().getName()+" terminated in an error state; the output may be corrupt.");
		}
	}
	
	private ConcurrentReadInputStream makeCris(){
		ConcurrentReadInputStream cris=ConcurrentReadInputStream.getReadInputStream(maxReads, true, ffin1, null, null, null);
		cris.start(); //Start the stream
		if(verbose){outstream.println("Started cris");}
//		boolean paired=cris.paired();
//		if(!ffin1.samOrBam()){outstream.println("Input is being processed as "+(paired ? "paired" : "unpaired"));}
		return cris;
	}
	
	private ConcurrentReadOutputStream makeCros(FileFormat ff){
		if(ff==null){return null;}

		//Select output buffer size based on whether it needs to be ordered
		final int buff=16;

		final ConcurrentReadOutputStream ros=ConcurrentReadOutputStream.getStream(ff, null, buff, null, ff.samOrBam() && ffin1.samOrBam());
		ros.start(); //Start the stream
		return ros;
	}
	
	/** Spawn process threads */
	private void spawnThreads(final ConcurrentReadInputStream cris, 
			final ConcurrentReadOutputStream ros, final ConcurrentReadOutputStream rosb,
			final ConcurrentReadOutputStream rosj){
		
		//Do anything necessary prior to processing
		
		//Fill a list with ProcessThreads
		ArrayList<ProcessThread> alpt=new ArrayList<ProcessThread>(threads);
		for(int i=0; i<threads; i++){
			alpt.add(new ProcessThread(ros, rosb, rosj, i));
		}
		
		//Start the threads
		for(ProcessThread pt : alpt){
			pt.start();
		}
		
		handleCris(cris);
		
		//Wait for threads to finish
		waitForThreads(alpt);
		
		//Do anything necessary after processing
		
	}
	
	/** 
	 * Pull reads from the cris;
	 * organize them into lists of subreads from the same ZMW;
	 * put those lists into the shared queue.
	 */
	private void handleCris(final ConcurrentReadInputStream cris){
		//Grab the first ListNum of reads
		ListNum<Read> ln=cris.nextList();

		ArrayList<Read> buffer=new ArrayList<Read>();
		long prevZmw=-1;
		
		long added=0;
		
		//As long as there is a nonempty read list...
		while(ln!=null && ln.size()>0){

			for(Read r : ln) {
				long zmw;
				try {
					zmw=Tools.parseZmw(r.id);
				} catch (Exception e) {
					zmw=r.numericID;//For testing only; disable for production
				}
				if(verbose){System.err.println("Fetched read "+r.id+"; "+(zmw!=prevZmw)+", "+buffer.isEmpty()+", "+zmw+", "+prevZmw);}
				if(zmw!=prevZmw && !buffer.isEmpty()){
					ZMWs++;
					addToQueue(buffer);
					added+=buffer.size();
					buffer=new ArrayList<Read>();
				}
				buffer.add(r);
				prevZmw=zmw;
			}
			
			cris.returnList(ln);
			
			//Fetch a new list
			ln=cris.nextList();
		}

		if(!buffer.isEmpty()){
			ZMWs++;
			added+=buffer.size();
			addToQueue(buffer);
		}
		
		//Notify the input stream that the final list was used
		if(ln!=null){
			cris.returnList(ln.id, ln.list==null || ln.list.isEmpty());
		}
		
		//Notify worker threads that there is no more data
		for(int i=0; i<threads; i++){
			addToQueue(poison);
		}
	}
	
	private void waitForThreads(ArrayList<ProcessThread> alpt){
		
		//Wait for completion of all threads
		boolean success=true;
		for(ProcessThread pt : alpt){
			
			//Wait until this thread has terminated
			while(pt.getState()!=Thread.State.TERMINATED){
				try {
					//Attempt a join operation
					pt.join();
				} catch (InterruptedException e) {
					//Potentially handle this, if it is expected to occur
					e.printStackTrace();
				}
			}
			
			//Accumulate per-thread statistics
			readsProcessed+=pt.readsProcessedT;
			basesProcessed+=pt.basesProcessedT;

			truePositiveReads+=pt.truePositiveReadsT;
			trueNegativeReads+=pt.trueNegativeReadsT;
			falsePositiveReads+=pt.falsePositiveReadsT;
			falseNegativeReads+=pt.falseNegativeReadsT;
			
			readsOut+=pt.readsOutT;
			basesOut+=pt.basesOutT;
			
			alignmentIters+=pt.ica.iters();
			alignmentItersShort+=pt.ica.itersShort();
			elapsed+=pt.elapsedT;
			elapsedShort+=pt.elapsedShortT;
			
			iceCreamReads+=pt.iceCreamReadsT;
			iceCreamBases+=pt.iceCreamBasesT;
			iceCreamZMWs+=pt.iceCreamZMWsT;
			iceCreamRatio+=pt.iceCreamRatioT;
			missingAdapterZMWs+=pt.missingAdapterZMWsT;
			hiddenAdapterZMWs+=pt.hiddenAdapterZMWsT;
			
			success&=pt.success;
		}
		
		//Track whether any threads failed
		if(!success){errorState=true;}
	}
	
	/*--------------------------------------------------------------*/
	/*----------------         Inner Methods        ----------------*/
	/*--------------------------------------------------------------*/
	
	private void addToQueue(ArrayList<Read> buffer){
		if(verbose) {System.err.println("Adding to queue "+(buffer==poison ? "poison" : buffer.get(0).id));}
		while(buffer!=null) {
			try {
				queue.put(buffer);
				buffer=null;
			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
	}
	
	private ArrayList<Read> pullFromQueue(){
		ArrayList<Read> buffer=null;
		while(buffer==null) {
			try {
				buffer=queue.take();
			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		if(verbose){System.err.println("Pulled from queue "+(buffer==poison ? "poison" : buffer.get(0).id));}
		return buffer==poison ? null : buffer;
	}
	
	/*--------------------------------------------------------------*/
	/*----------------         Inner Classes        ----------------*/
	/*--------------------------------------------------------------*/
	
	/** This class is static to prevent accidental writing to shared variables.
	 * It is safe to remove the static modifier. */
	private class ProcessThread extends Thread {
		
		//Constructor
		ProcessThread(final ConcurrentReadOutputStream ros_, 
				final ConcurrentReadOutputStream rosb_, final ConcurrentReadOutputStream rosj_, final int tid_){
			ros=ros_;
			rosb=rosb_;
			rosj=rosj_;
			tid=tid_;
		}
		
		//Called by start()
		@Override
		public void run(){
			//Do anything necessary prior to processing
			
			//Process the reads
			processInner();
			
			//Do anything necessary after processing
			
			//Indicate successful exit status
			success=true;
		}
		
		/** Iterate through the reads */
		void processInner(){

			//As long as there is a nonempty read list...
			for(ArrayList<Read> reads=pullFromQueue(); reads!=null; reads=pullFromQueue()){
//				if(verbose){outstream.println("Fetched "+reads.size()+" reads.");} //Disabled due to non-static access
				
				processList(reads);
			}
		}
		
		private int medianLength(ArrayList<Read> list){
			if(list.size()<3){return -1;}
			IntList lengths=new IntList(list.size()-2);
			
			for(int i=1; i<list.size()-1; i++){
				lengths.add(list.get(i).length());
			}
			lengths.sort();
			int median=lengths.get((lengths.size-1)/2);
			return median;
		}
		
		boolean flagAdapters(final ArrayList<Read> reads, int median){
			boolean found=false;
			for(Read r : reads){
				if(r.length()>1.5f*median){
					r.setDiscarded(true);
					found=true;
				}
			}
			return found;
		}
		
		void processList(final ArrayList<Read> reads){
			
			long numBases=0;
			
			//Loop through each read in the list
			for(int idx=0; idx<reads.size(); idx++){
				final Read r1=reads.get(idx);
				
				//Validate reads in worker threads
				if(!r1.validated()){r1.validate(true);}

				//Track the initial length for statistics
				final int initialLength1=r1.length();

				//Increment counters
				readsProcessedT++;
				basesProcessedT+=initialLength1;
				numBases+=initialLength1;
			}

			final ArrayList<Read> good=new ArrayList<Read>();
			final ArrayList<Read> bad=new ArrayList<Read>();
			final boolean fullPass=CCS || reads.size()>=3;

			Read sample=null;
			final int medianLength=medianLength(reads);
			boolean foundLongRead=false;
			if(reads.size()>=3){
				if(flagLongReads){foundLongRead=flagAdapters(reads, medianLength);}
				for(int i=1; i<reads.size()-1; i++){
					sample=reads.get(i);
					if(sample.length()==medianLength){break;}
				}
			}else{
				for(Read r : reads){
					if(sample==null || sample.length()<r.length()){sample=r;}
				}
			}
			assert(sample!=null);
			final AlignmentResult ar=align(sample, fullPass);
//			AlignmentResult aa=null;
			
			boolean foundBadRead=false;
			int adaptersCounted=0;
			if(ar!=null && (ar.icecream || !filterIceCreamOnly)){
				foundBadRead=true;
			}
			if(alignAdapter && sample.length()>=adapter.length){
				AlignmentResult aa=(adapterRatio<=0 ? null : fa.alignForwardShort(adapter, sample.bases, 0, sample.bases.length-1, adapterRatio));
//				AlignmentResult aa=(adapterRatio<=0 ? null : ica.alignForwardShort(adapter, sample.bases, 0, sample.bases.length-1, -99999, adapterRatio));
				if(aa!=null || ar!=null || adapterRatio<=0){
					adaptersCounted=lookForAdapter(sample, false /*aa==null*/);
					foundBadRead|=adaptersCounted>0;
				}
//				aa=fa.alignForward(adapter, sample.bases, 0, sample.bases.length-1, -999999999, adapterRatio);
//				if(aa!=null){foundBadRead=true;}
			}
			

//			System.err.println("\n"+iceCreamZMWs+", "+missingAdapterZMWs+", "+hiddenAdapterZMWs+", "+foundBadRead+", "+foundLongRead+", "+(ar!=null));
			if(foundBadRead || foundLongRead){iceCreamZMWs++;}
			
			if(ar!=null && ar.icecream){
				iceCreamRatio+=ar.ratio;
				if(adaptersCounted==0 && !foundLongRead){missingAdapterZMWs++;}
			}
			if(adaptersCounted>0 || foundLongRead){
				hiddenAdapterZMWs++;
			}
//			System.err.println(iceCreamZMWs+", "+missingAdapterZMWs+", "+hiddenAdapterZMWs+", "+foundBadRead+", "+foundLongRead+", "+(ar!=null));
			
			if(foundBadRead || (foundLongRead && !keepShortReads)){
				if(modifyHeader && ar!=null){
					sample.id=sample.id+"\tratio="+ar.ratio+"\tjunction="+ar.junctionLoc;
				}
				if(!keepShortReads || adaptersCounted>1){
					for(Read r : reads){
						r.setDiscarded(true);
					}
				}else if(reads.size()>2){
					for(int i=1; i<reads.size()-1; i++){
						reads.get(i).setDiscarded(true);
					}
					Read r=reads.get(0);
					if(r.length()>=0.35f*medianLength){r.setDiscarded(true);}
					r=reads.get(reads.size()-1);
					if(r.length()>=0.35f*medianLength){r.setDiscarded(true);}
				}else{
					for(Read r : reads){
						if(r.length()>=0.35f*medianLength){r.setDiscarded(true);}
					}
				}
			}
			
			ArrayList<Read> junctions=null;
			if(ar!=null && rosj!=null){
				Read r=ar.alignedRead;
				int width=Tools.min(200, ar.junctionLoc, r.length()-ar.junctionLoc);
				int a=ar.junctionLoc-width, b=ar.junctionLoc+width;
				byte[] bases=Arrays.copyOfRange(r.bases, a, b);
				Read junction=new Read(bases, null, r.id+"\tjunction:"+a+"-"+b, r.numericID);
//				assert(false) : junction;
				junctions=new ArrayList<Read>(1);
				junctions.add(junction);
			}
//			assert(false) : ros+", "+rosb+", "+rosj+", "+junctions;
			
			for(Read r : reads){
				boolean trueIceCream=(parseCustom ? ReadBuilder.isIceCream(r.id) : false);
				if(r.discarded()){
					bad.add(r);
					if(trueIceCream){truePositiveReadsT++;}
					else{falsePositiveReadsT++;}
				}else{
					good.add(r);
					readsOutT++;
					basesOutT+=r.length();
					if(trueIceCream){falseNegativeReadsT++;}
					else{trueNegativeReadsT++;}
				}
			}

			if(ros!=null && good!=null && !good.isEmpty()){ros.add(good, 0);}
			if(rosb!=null && bad!=null && !bad.isEmpty()){rosb.add(bad, 0);}
			if(rosj!=null && junctions!=null && !junctions.isEmpty()){rosj.add(junctions, 0);}
		}
		
		/**
		 * Process a read.
		 * @param r The read
		 * @return True if the read should be kept, false if it should be discarded.
		 */
		AlignmentResult align(final Read r, boolean fullPass){
			if(!alignRC){return null;}
			final byte[] bases=r.bases;
			
			int qlen=(int)Tools.max(minQlen, Tools.min(targetQlen, bases.length*maxQlenFraction));
			if(qlen>0.45f*bases.length){return null;}//Ignore short stuff
			
			//Perform an initial scan using the tips of the reads to look for an inverted repeat
			boolean tipOnly=filterIceCreamOnly && fullPass;
//			System.err.println(filterIceCreamOnly+", "+fullPass+", "+tipOnly);
			AlignmentResult a=alignLeft(bases, qlen, minRatio1, true, tipOnly);
			AlignmentResult b=(fullPass && false ? null : alignRight(bases, qlen, minRatio1, true, tipOnly));//A second alignment is not needed for a full pass.
			AlignmentResult ar=(a==null ? b : b==null ? a : a.maxScore>=b.maxScore ? a : b);
			
			//If nothing was detected, return
			if(ar==null){return null;}
			ar.alignedRead=r;
			
			//At this point, the read contains an inverted repeat of length qlen.
			final int expectedJunction=ar.rLen/2;
			
			if(ar.left){
				ar.junctionLoc=ar.maxRpos/2;
			}else{
				int innerLeft=ar.maxRpos;
				int innerRight=ar.rLen-ar.qLen;
				ar.junctionLoc=(innerLeft+innerRight)/2;
			}
			
			if(fullPass){
				int dif=Tools.absdif(expectedJunction, ar.junctionLoc);
				if(dif>expectedJunction*0.1) {
					if(filterIceCreamOnly){return ar;}
				}
			}
			
			if(realign){
				if(ar.junctionLoc<expectedJunction){
					int qlen2=(int)(ar.junctionLoc*0.9);
					if(qlen2>=qlen){
						ar=alignLeft(bases, qlen2, minRatio2, false, false);
					}
				}else{
					int qlen2=(int)((ar.rLen-ar.junctionLoc)*0.9);
					if(qlen2>=qlen){
						ar=alignRight(bases, qlen2, minRatio2, false, false);
					}
				}
				if(ar==null){return null;}
				ar.alignedRead=r;
			}
			
			//At this point, the read contains an inverted repeat mirrored across a junction.
			final float junctionFraction;
			if(ar.left){
				ar.junctionLoc=ar.maxRpos/2;
				junctionFraction=ar.junctionLoc/(float)ar.rLen;
			}else{
				int innerLeft=ar.maxRpos;
				int innerRight=ar.rLen-ar.qLen;
				ar.junctionLoc=(innerLeft+innerRight)/2;
				junctionFraction=(ar.rLen-ar.junctionLoc)/(float)ar.rLen;
			}
			if(fullPass){
				int dif=Tools.absdif(expectedJunction, ar.junctionLoc);
				if(dif>expectedJunction*0.1) {
					ar.icecream=false;
					if(filterIceCreamOnly){return ar;}
				}
			}
			
			if(filterIceCreamOnly){
				if(junctionFraction<minJunctionFraction){return ar;}
			}
			
			ar.icecream=true;
			return ar;
		}
		
		private AlignmentResult alignLeft(final byte[] bases, final int qlen, final float minRatio, boolean v2, boolean tipOnly){
			final byte[] query=Arrays.copyOfRange(bases, 0, qlen);
			AminoAcid.reverseComplementBasesInPlace(query);
			final AlignmentResult ar;
			final int rstop=bases.length-1;
			final int rstart=(tipOnly ? Tools.max(qlen, rstop-(int)(tipRatio*qlen)) : qlen);
			if(v2){
				elapsedShortT-=System.nanoTime();
				ar=ica.alignForwardShort(query, bases, rstart, rstop, minScore, minRatio);
				elapsedShortT+=System.nanoTime();
			}else{
				elapsedT-=System.nanoTime();
				ar=ica.alignForward(query, bases, rstart, rstop, minScore, minRatio);
				elapsedT+=System.nanoTime();
			}
			if(ar!=null){ar.left=true;}
			return ar;
		}
		
		private AlignmentResult alignRight(final byte[] bases, final int qlen, final float minRatio, boolean v2, boolean tipOnly){
			final byte[] query=Arrays.copyOfRange(bases, bases.length-qlen, bases.length);
			AminoAcid.reverseComplementBasesInPlace(query);
			final AlignmentResult ar;
			final int rstop=(tipOnly ? Tools.min((int)(tipRatio*qlen), bases.length-qlen-1) : bases.length-qlen-1);
			final int rstart=0;
			if(v2){
				elapsedShortT-=System.nanoTime();
				ar=ica.alignForwardShort(query, bases, rstart, rstop, minScore, minRatio);
				elapsedShortT+=System.nanoTime();
			}else{
				elapsedT-=System.nanoTime();
				ar=ica.alignForward(query, bases, rstart, rstop, minScore, minRatio);
				elapsedT+=System.nanoTime();
			}
			if(ar!=null){ar.left=false;}
			return ar;
		}

		/**
		 * @param r
		 */
		private int lookForAdapter(Read r, boolean print) {
			
			int begin=0;
			while(begin<r.length() && r.bases[begin]=='N'){begin++;} //Skip reads made of 'N'
			if(begin>=r.length()){return 0;}

//			final byte[] array=npad(r.bases, pad ? npad : 0);
			final byte[] array=npad(r.bases, npad);
			
			int lim=array.length-npad-stride;
			
			int plusFound=0;
			int minusFound=0;

			int lastSuspect=-1;
			int lastConfirmed=-1;
			
//			final int revisedStride=(r.length()>1800 ? stride : (int)(stride*0.6f));
			for(int i=begin; i<lim; i+=stride){
				int j=Tools.min(i+window, array.length-1);
				if(j-i<window){
					lim=0; //Last loop cycle
//					i=Tools.max(0, array.length-2*query1.length);
				}
				
				{
					
					int[] rvec=msa.fillAndScoreLimited(adapter, array, i, j, minSwScoreSuspect);
					if(rvec!=null && rvec[0]>=minSwScoreSuspect){
						int score=rvec[0];
						int start=rvec[1];
						int stop=rvec[2];
						assert(score>=minSwScoreSuspect);
						if((i==0 || start>i) && (j==array.length-1 || stop<j)){
							boolean kill=(score>=minSwScore ||
									(score>=suspectMidpoint && lastSuspect>0 && start>=lastSuspect && start-lastSuspect<suspectDistance) ||
									(lastConfirmed>0 && start>=lastConfirmed && start-lastConfirmed<suspectDistance));
							
							if(!kill && USE_LOCALITY && array.length-stop>window){//Look ahead
								rvec=msa.fillAndScoreLimited(adapter, array, stop, stop+window, minSwScoreSuspect);
								if(rvec!=null){
									if(score>=suspectMidpoint && rvec[0]>=minSwScoreSuspect && rvec[1]-stop<suspectDistance){kill=true;}
									else if(score>=minSwScoreSuspect && rvec[0]>=minSwScore && rvec[1]-stop<suspectDistance){kill=true;}
								}
							}
							
							if(!kill && USE_ALT_MSA){//Try alternate msa
								rvec=msa2.fillAndScoreLimited(adapter, array, Tools.max(0, start-4), Tools.min(stop+4, array.length-1), minSwScoreSuspect);
								if(rvec!=null && rvec[0]>=(minSwScore)){kill=true;}
							}
							
							if(kill){
								if(print) {System.err.println("Found adapter at distance "+Tools.min(start, array.length-stop));}
//								System.out.println("-:"+score+", "+minSwScore+", "+minSwScoreSuspect+"\t"+lastSuspect+", "+start+", "+stop);
								plusFound++;
								for(int x=Tools.max(0, start); x<=stop; x++){array[x]='X';}
								if(USE_LOCALITY && score>=minSwScore){lastConfirmed=Tools.max(lastConfirmed, stop);}
							}
						}
//						System.out.println("Set lastSuspect="+stop+" on score "+score);
						if(USE_LOCALITY){lastSuspect=Tools.max(lastSuspect, stop);}
					}
				}
			}

			int found=plusFound+minusFound;
			
			r.setHasAdapter(found>0);
			r.setDiscarded(found>0);
			
//			if(!pad && found<1) {
//				return lookForAdapter(r, true);
//			}
			
			return found;
		}
		
		private byte[] npad(final byte[] array, final int pad){
			if(pad<0){return array;}
			final int len=array.length+2*pad;
			byte[] r=new byte[len];
			for(int i=0; i<pad; i++){r[i]='N';}
			for(int i=pad, j=0; j<array.length; i++, j++){r[i]=array[j];}
			for(int i=array.length+pad, limit=Tools.min(r.length, len+50); i<limit; i++){r[i]='N';}
			return r;
		}

		/** Number of reads processed by this thread */
		protected long readsProcessedT=0;
		/** Number of bases processed by this thread */
		protected long basesProcessedT=0;
		

		protected long truePositiveReadsT=0;
		protected long falsePositiveReadsT=0;
		protected long trueNegativeReadsT=0;
		protected long falseNegativeReadsT=0;
		protected long elapsedT=0;
		protected long elapsedShortT=0;
		
		protected long iceCreamReadsT=0;
		protected long iceCreamBasesT=0;
		protected long iceCreamZMWsT=0;
		protected double iceCreamRatioT=0;
		protected long missingAdapterZMWsT=0;
		protected long hiddenAdapterZMWsT=0;
		
		/** Number of reads retained by this thread */
		protected long readsOutT=0;
		/** Number of bases retained by this thread */
		protected long basesOutT=0;
		
		/** True only if this thread has completed successfully */
		boolean success=false;
		
		/** Shared output stream */
		private final ConcurrentReadOutputStream ros;
		/** Shared output stream for bad reads */
		private final ConcurrentReadOutputStream rosb;
		/** Shared output stream for junctions */
		private final ConcurrentReadOutputStream rosj;
		/** Thread ID */
		final int tid;
		
		/** Aligner for this thread */
		final IceCreamAligner ica=IceCreamAligner.makeAligner(32);
		final FlatAligner fa=new FlatAligner();
		final MultiStateAligner9PacBioAdapter msa=alignAdapter ? new MultiStateAligner9PacBioAdapter(ALIGN_ROWS, ALIGN_COLUMNS) : null;
		final MultiStateAligner9PacBioAdapter2 msa2=alignAdapter ? new MultiStateAligner9PacBioAdapter2(ALIGN_ROWS, ALIGN_COLUMNS) : null;
	}
	
	/*--------------------------------------------------------------*/
	/*----------------            Fields            ----------------*/
	/*--------------------------------------------------------------*/

	/** Primary input file path */
	private String in1=null;

	/** Primary output file path */
	private String out1=null;
	private String outb=null;
	private String outj=null;
	
	/** Override input file extension */
	private String extin=null;
	/** Override output file extension */
	private String extout=null;

	private int targetQlen=352;
	private int minQlen=100;
	
	/** Make a query at most this fraction of read length */
	private float maxQlenFraction=0.15f;
	
	/** Exit alignment early if score drops below this.
	 * An aggressive optimization that may miss some low-quality inverted repeats.
	 * -700 seems safe. */
	private int minScore=-800;
	
	/** Fraction of maximum alignment score to consider as matching for initial alignment */ 
	private float minRatio1=0.60f;
	
	/** Fraction of maximum alignment score to consider as matching for realignment */ 
	private float minRatio2=0.64f;
	
	private float adapterRatio=0.18f; //.18 for fa, or 0.57/0.6 for ica
	
	private float tipRatio=1.5f;
	
	/** Short half of inverted repeat must be at least this fraction of read length to be classed as a triangle */
	private float minJunctionFraction=0.4f;
	
	/** Only filter triangle reads, not all inverted repeats */
	private boolean filterIceCreamOnly=true;
	
	/** Align again once the junction position is provisionally identified */
	private boolean realign=true;
	
	/** For internal read array transfers */ 
	private int queuelen=80;
	
	/** For grading synthetic data */
	private boolean parseCustom;
	
	/** Input reads are CCS (full pass) */
	private boolean CCS;
	
	private boolean keepShortReads=true;
	
	private boolean modifyHeader=false;
	
	/*--------------------------------------------------------------*/

	protected long alignmentIters=0;
	protected long alignmentItersShort=0;
	protected long elapsed=0;
	protected long elapsedShort=0;
	protected boolean printTiming=false;
	
	/** Number of reads processed */
	protected long readsProcessed=0;
	/** Number of bases processed */
	protected long basesProcessed=0;

	/** Number of reads retained */
	protected long readsOut=0;
	/** Number of bases retained */
	protected long basesOut=0;
	
	/** Quit after processing this many input reads; -1 means no limit */
	private long maxReads=-1;

	protected long iceCreamReads=0;
	protected long iceCreamBases=0;
	protected long iceCreamZMWs=0;
	protected double iceCreamRatio=0;
	protected long missingAdapterZMWs=0;
	protected long hiddenAdapterZMWs=0;
	protected long ZMWs=0;
	
	protected long junctionsOut=0;
	
	protected long truePositiveReads=0;
	protected long falsePositiveReads=0;
	protected long trueNegativeReads=0;
	protected long falseNegativeReads=0;
	
	protected final int stride;
	protected final int window;
	protected final int ALIGN_ROWS;
	protected final int ALIGN_COLUMNS;

	protected final int maxSwScore;
	protected final int minSwScore;
	protected final int minSwScoreSuspect;
	protected final int maxImperfectSwScore;
	protected final int suspectMidpoint;
	protected final int suspectDistance=100;
	protected int npad=0;  //This is for catching adapters on the tips, which are not very relevant to ice cream.  Set to 35 for tip apdapters.
	
	private static float MINIMUM_ALIGNMENT_SCORE_RATIO=0.31f; //0.31f: At 250bp reads, approx 0.01% false-positive and 94% true-positive.
	private static float SUSPECT_RATIO=0.85F;
	public static boolean USE_LOCALITY=true;
	public static boolean USE_ALT_MSA=true;
	
	private byte[] adapter="ATCTCTCTCAACAACAACAACGGAGGAGGAGGAAAAGAGAGAGAT".getBytes(); //This one seems to be correct
//	private byte[] adapter="ATCTCTCTCTTTTCCTCCTCCTCCGTTGTTGTTGTTGAGAGAGAT".getBytes();
	private boolean alignAdapter=true;
	private boolean alignRC=true;
	private boolean flagLongReads=true;
	
	/*--------------------------------------------------------------*/
	/*----------------         Final Fields         ----------------*/
	/*--------------------------------------------------------------*/

	/** Primary input file */
	private final FileFormat ffin1;
	
	/** Primary output file */
	private final FileFormat ffout1;
	
	/** Bad output file */
	private final FileFormat ffoutb;
	
	/** Junction output file */
	private final FileFormat ffoutj;
	
	private final int threads;
	private final ArrayBlockingQueue<ArrayList<Read>> queue;
	private final ArrayList<Read> poison=new ArrayList<Read>(0);
	
	/*--------------------------------------------------------------*/
	/*----------------        Common Fields         ----------------*/
	/*--------------------------------------------------------------*/
	
	/** Print status messages to this output stream */
	private PrintStream outstream=System.err;
	/** Print verbose messages */
	public static boolean verbose=false;
	/** True if an error was encountered */
	public boolean errorState=false;
	/** Overwrite existing output files */
	private boolean overwrite=false;
	/** Append to existing output files */
	private boolean append=false;
	
}
