/**
    libmaus2
    Copyright (C) 2009-2021 German Tischler-Höhle
    Copyright (C) 2011-2014 Genome Research Limited

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
**/
#if ! defined(LIBMAUS2_SUFFIXSORT_BWTB3M_BWTMERGECOMPUTESAMPLEDSA_HPP)
#define LIBMAUS2_SUFFIXSORT_BWTB3M_BWTMERGECOMPUTESAMPLEDSA_HPP

#include <libmaus2/suffixsort/bwtb3m/BWTB3MBase.hpp>
#include <libmaus2/suffixsort/bwtb3m/BwtMergeSortOptions.hpp>
#include <libmaus2/lf/ImpCompactHuffmanWaveletLF.hpp>
#include <libmaus2/util/TempFileNameGenerator.hpp>
#include <libmaus2/util/OutputFileNameTools.hpp>
#include <libmaus2/sorting/PairFileSorting.hpp>

#include <libmaus2/suffixsort/ByteInputTypes.hpp>
#include <libmaus2/suffixsort/CompactInputTypes.hpp>
#include <libmaus2/suffixsort/PacInputTypes.hpp>
#include <libmaus2/suffixsort/PacTermInputTypes.hpp>
#include <libmaus2/suffixsort/Lz4InputTypes.hpp>
#include <libmaus2/suffixsort/Utf8InputTypes.hpp>

namespace libmaus2
{
	namespace suffixsort
	{
		namespace bwtb3m
		{
			template<typename input_types_type>
			struct BwtMergeComputeSampledSA : public libmaus2::suffixsort::bwtb3m::BWTB3MBase
			{
				static void checkBwtBlockDecode(
					std::pair<uint64_t,uint64_t> const & isai,
					std::pair<uint64_t,uint64_t> const & isapre,
					std::string const & fn,
					uint64_t const fs,
					::libmaus2::lf::ImpCompactHuffmanWaveletLF const & IHWT,
					::libmaus2::aio::SynchronousGenericOutput<uint64_t> & SGO,
					::libmaus2::aio::SynchronousGenericOutput<uint64_t> & ISGO,
					uint64_t const sasamplingrate,
					uint64_t const isasamplingrate,
					int64_t const ibs = -1
				)
				{
					assert ( ::libmaus2::rank::PopCnt8<sizeof(unsigned long)>::popcnt8(sasamplingrate) == 1 );
					assert ( ::libmaus2::rank::PopCnt8<sizeof(unsigned long)>::popcnt8(isasamplingrate) == 1 );

					uint64_t r = isai.first;
					uint64_t p = isai.second;
					uint64_t const sasamplingmask = sasamplingrate-1;
					uint64_t const isasamplingmask = isasamplingrate-1;
					// size of block
					uint64_t const bs = ibs >= 0 ? ibs : ((p >  isapre.second) ? (p-isapre.second) : (fs - isapre.second));

					// (*logstr) << "bs=" << bs << std::endl;

					typename input_types_type::circular_reverse_wrapper CRWR(fn,p);

					if ( ! p )
					{
						for ( uint64_t j = 0; j < bs; ++j )
						{
							if ( !(r & sasamplingmask) )
							{
								SGO.put(r);
								SGO.put(p);
							}
							if ( !(p & isasamplingmask) )
							{
								ISGO.put(p);
								ISGO.put(r);
							}

							#if ! defined(NDEBUG)
							int64_t syma =
							#endif
								CRWR.get();
							#if ! defined(NDEBUG)
							int64_t symb =
							#endif
								IHWT[r];
							#if ! defined(NDEBUG)
							assert ( syma == symb );
							#endif
							// (*logstr) << "(" << syma << "," << symb << ")";

							r = IHWT(r);

							if ( ! p )
								p = fs;
							--p;
						}
					}
					else
					{
						for ( uint64_t j = 0; j < bs; ++j )
						{
							if ( !(r & sasamplingmask) )
							{
								SGO.put(r);
								SGO.put(p);
							}
							if ( !(p & isasamplingmask) )
							{
								ISGO.put(p);
								ISGO.put(r);
							}

							#if ! defined(NDEBUG)
							int64_t syma =
							#endif
								CRWR.get();
							#if ! defined(NDEBUG)
							int64_t symb =
							#endif
								IHWT[r];
							#if ! defined(NDEBUG)
							assert ( syma == symb );
							#endif
							// (*logstr) << "(" << syma << "," << symb << ")";

							r = IHWT(r);

							--p;
						}
					}

					if ( r != isapre.first ) {
						libmaus2::exception::LibMausException lme;
						lme.getStream() << "checkBwtBlockDecode(): r=" << r << " != " << isapre.first << "=isapre.first" << std::endl;
						lme.finish();
						throw lme;
					}
				}

				static void checkSampledSA(
					std::string const & fn,
					uint64_t const fs,
					std::string const mergedsaname,
					uint64_t const sasamplingrate,
					uint64_t const numthreads,
					uint64_t const lfblockmult,
					std::ostream * logstr
				)
				{
					// number of sampled suffix array elements
					uint64_t const nsa = (fs + sasamplingrate - 1) / sasamplingrate;

					// check that this matches what we have in the file
					assert ( ::libmaus2::util::GetFileSize::getFileSize(mergedsaname) / (sizeof(uint64_t)) ==  nsa + 2 );

					if ( nsa && nsa-1 )
					{
						uint64_t const checkpos = nsa-1;
						uint64_t const satcheckpacks = numthreads * lfblockmult;
						uint64_t const sacheckblocksize = (checkpos + satcheckpacks-1) / satcheckpacks;
						uint64_t const sacheckpacks = ( checkpos + sacheckblocksize - 1 ) / sacheckblocksize;

						if ( logstr )
							(*logstr) << "[V] checking suffix array on text...";
						::libmaus2::parallel::SynchronousCounter<uint64_t> SC;
						int64_t lastperc = -1;
						#if defined(_OPENMP)
						#pragma omp parallel for schedule(dynamic,1) num_threads(numthreads)
						#endif
						for ( int64_t t = 0; t < static_cast<int64_t>(sacheckpacks); ++t )
						{
							uint64_t const low = t * sacheckblocksize;
							uint64_t const high = std::min(low+sacheckblocksize,checkpos);
							uint64_t const cnt = high-low;

							// std::cerr << "low=" << low << " high=" << high << " nsa=" << nsa << " cnt=" << cnt << std::endl;

							::libmaus2::aio::SynchronousGenericInput<uint64_t> SGIsa(mergedsaname,16*1024,low+2,cnt+1);

							typename input_types_type::circular_suffix_comparator CSC(fn,fs);

							int64_t const fp = SGIsa.get();
							assert ( fp >= 0 );

							uint64_t prev = fp;

							while ( SGIsa.peek() >= 0 )
							{
								int64_t const p = SGIsa.get();
								assert ( p >= 0 );

								// std::cerr << "pre " << prev.first << " SA[" << r << "]=" << p << std::endl;

								bool const ok = CSC (prev,p);
								assert ( ok );

								prev = p;
							}

							uint64_t const sc = ++SC;
							int64_t const newperc = (sc*100) / sacheckpacks;

							{
								libmaus2::aio::StreamLock::lock_type::scope_lock_type slock(libmaus2::aio::StreamLock::cerrlock);
								if ( newperc != lastperc )
								{
									if ( logstr )
										(*logstr) << "(" << newperc << ")";
									lastperc = newperc;
								}
							}
						}
						if ( logstr )
							(*logstr) << "done." << std::endl;
					}

				}

				static uint64_t readBlockRanksSize(std::string const & mergedisaname)
				{
					return ::libmaus2::util::GetFileSize::getFileSize(mergedisaname)/(2*sizeof(uint64_t));
				}

				static void computeSampledSA(
					libmaus2::util::TempFileNameGenerator & gtmpgen,
					std::string const & fn,
					uint64_t const fs,
					::libmaus2::lf::ImpCompactHuffmanWaveletLF const & IHWT,
					std::string const & mergedisaname,
					std::string const & outfn,
					// std::string const & tmpfilenamebase,
					uint64_t const numthreads,
					uint64_t const lfblockmult,
					uint64_t const sasamplingrate,
					uint64_t const isasamplingrate,
					uint64_t const blockmem,
					uint64_t const fanin,
					std::ostream * logstr
				)
				{
					// read size of sampled isa
					uint64_t const nsisa = readBlockRanksSize(mergedisaname);

					::libmaus2::aio::SynchronousGenericInput<uint64_t>::unique_ptr_type SGIsisasa(new ::libmaus2::aio::SynchronousGenericInput<uint64_t>(mergedisaname,16*1024));
					int64_t const fr = SGIsisasa->get(); assert ( fr != -1 );
					int64_t const fp = SGIsisasa->get(); assert ( fp != -1 );

					if ( logstr )
						(*logstr) << "[V] computing sampled suffix array parts...";

					std::pair<uint64_t,uint64_t> const isa0(fr,fp);
					std::pair<uint64_t,uint64_t> isapre(isa0);

					::std::vector< std::string > satempfilenames(numthreads);
					::std::vector< std::string > isatempfilenames(numthreads);
					::libmaus2::autoarray::AutoArray < ::libmaus2::aio::SynchronousGenericOutput<uint64_t>::unique_ptr_type > SAF(numthreads);
					::libmaus2::autoarray::AutoArray < ::libmaus2::aio::SynchronousGenericOutput<uint64_t>::unique_ptr_type > ISAF(numthreads);
					for ( uint64_t i = 0; i < numthreads; ++i )
					{
						satempfilenames[i] = ( gtmpgen.getFileName() + ".sampledsa_" + ::libmaus2::util::NumberSerialisation::formatNumber(i,6) );
						::libmaus2::util::TempFileRemovalContainer::addTempFile(satempfilenames[i]);
						::libmaus2::aio::SynchronousGenericOutput<uint64_t>::unique_ptr_type tSAFi(
							new ::libmaus2::aio::SynchronousGenericOutput<uint64_t>(satempfilenames[i],8*1024)
						);
						SAF[i] = std::move(tSAFi);

						isatempfilenames[i] = ( gtmpgen.getFileName() + ".sampledisa_" + ::libmaus2::util::NumberSerialisation::formatNumber(i,6) );
						::libmaus2::util::TempFileRemovalContainer::addTempFile(isatempfilenames[i]);
						::libmaus2::aio::SynchronousGenericOutput<uint64_t>::unique_ptr_type tISAFi(
							new ::libmaus2::aio::SynchronousGenericOutput<uint64_t>(isatempfilenames[i],8*1024)
						);
						ISAF[i] = std::move(tISAFi);
					}

					std::vector < std::pair< std::pair<uint64_t,uint64_t>, std::pair<uint64_t,uint64_t> > > WV;
					int64_t lastperc = -1;

					if ( nsisa > 1 )
					{
						for ( int64_t i = 1; i <= static_cast<int64_t>(nsisa); ++i )
						{
							int64_t const nr = (i == static_cast<int64_t>(nsisa)) ? isa0.first : SGIsisasa->get();
							int64_t const np = (i == static_cast<int64_t>(nsisa)) ? isa0.second : ((nr != -1) ? SGIsisasa->get() : -1);
							assert ( np >= 0 );

							std::pair<uint64_t,uint64_t> isai(nr,np);

							WV.push_back(std::pair< std::pair<uint64_t,uint64_t>, std::pair<uint64_t,uint64_t> >(isai,isapre));

							isapre.first = nr;
							isapre.second = np;

							if ( ((WV.size() % (lfblockmult*numthreads)) == 0) || i == static_cast<int64_t>(nsisa) )
							{
								#if defined(_OPENMP)
								#pragma omp parallel for num_threads(numthreads)
								#endif
								for ( int64_t j = 0; j < static_cast<int64_t>(WV.size()); ++j )
								{
									#if defined(_OPENMP)
									uint64_t const tid = omp_get_thread_num();
									#else
									uint64_t const tid = 0;
									#endif
									checkBwtBlockDecode(WV[j].first,WV[j].second,fn,fs,IHWT,*SAF[tid],*ISAF[tid],sasamplingrate,isasamplingrate);
								}

								WV.resize(0);
							}

							int64_t const newperc = ((i)*100) / (nsisa);
							if ( newperc != lastperc )
							{
								if ( logstr )
									(*logstr) << "(" << newperc << ")";
								lastperc = newperc;
							}
						}
					}
					else
					{
						assert ( fp == 0 );
						checkBwtBlockDecode(
							std::pair<uint64_t,uint64_t>(fr,0),
							std::pair<uint64_t,uint64_t>(fr,0),
							fn,fs,IHWT,*SAF[0],*ISAF[0],sasamplingrate,isasamplingrate,
							fs);
					}

					SGIsisasa.reset();

					uint64_t sawritten = 0;
					for ( uint64_t i = 0; i < SAF.size(); ++i )
					{
						SAF[i]->flush();
						sawritten += SAF[i]->getWrittenWords();
						SAF[i].reset();
					}
					uint64_t isawritten = 0;
					for ( uint64_t i = 0; i < ISAF.size(); ++i )
					{
						ISAF[i]->flush();
						isawritten += ISAF[i]->getWrittenWords();
						ISAF[i].reset();
					}

					if ( logstr )
						(*logstr) << "done." << std::endl;

					uint64_t const saexpected = 2* ( (fs + sasamplingrate - 1)/sasamplingrate );
					uint64_t const isaexpected = 2* ( (fs + isasamplingrate - 1)/isasamplingrate );

					if ( logstr )
					{
						(*logstr) << "[V] sawritten=" << sawritten << " saexpected=" << saexpected << std::endl;
						(*logstr) << "[V] isawritten=" << isawritten << " isaexpected=" << isaexpected << std::endl;
						assert ( sawritten == saexpected );
						assert ( isawritten == isaexpected );
					}

					if ( logstr )
						(*logstr) << "[V] sorting and merging sampled suffix array parts...";
					std::string const mergedsaname = ::libmaus2::util::OutputFileNameTools::clipOff(outfn,".bwt") + ".sa";
					{
					::libmaus2::aio::OutputStreamInstance::unique_ptr_type pmergedsa(new ::libmaus2::aio::OutputStreamInstance(mergedsaname));
					// write sampling rate
					::libmaus2::serialize::Serialize<uint64_t>::serialize(*pmergedsa,sasamplingrate);
					::libmaus2::serialize::Serialize<uint64_t>::serialize(*pmergedsa,(fs + sasamplingrate-1)/sasamplingrate);
					std::string const mergesatmp = mergedsaname + ".tmp";
					::libmaus2::util::TempFileRemovalContainer::addTempFile(mergesatmp);
					::libmaus2::sorting::PairFileSorting::sortPairFile(
						satempfilenames,mergesatmp,
						false /* second comp */,
						false /* keep first */,
						true /* keep second */,
						*pmergedsa /* output stream */,
						blockmem/* /2 par in place now*/,
						numthreads /* parallel */,
						true /* delete input */,
						fanin,
						logstr
					);
					pmergedsa->flush();
					pmergedsa.reset();
					libmaus2::aio::FileRemoval::removeFile(mergesatmp.c_str());
					}
					if ( logstr )
						(*logstr) << "done." << std::endl;

					if ( logstr )
						(*logstr) << "[V] sorting and merging sampled inverse suffix array parts...";
					std::string const mergedisaoutname = ::libmaus2::util::OutputFileNameTools::clipOff(outfn,".bwt") + ".isa";
					::libmaus2::aio::OutputStreamInstance::unique_ptr_type pmergedisa(new ::libmaus2::aio::OutputStreamInstance(mergedisaoutname));
					// write sampling rate
					::libmaus2::serialize::Serialize<uint64_t>::serialize(*pmergedisa,isasamplingrate);
					::libmaus2::serialize::Serialize<uint64_t>::serialize(*pmergedisa,(fs+isasamplingrate-1)/isasamplingrate);
					std::string const mergeisatmp = mergedisaoutname + ".tmp";
					::libmaus2::util::TempFileRemovalContainer::addTempFile(mergeisatmp);
					::libmaus2::sorting::PairFileSorting::sortPairFile(
						isatempfilenames,mergeisatmp,
						false /* second comp */,
						false /* keep first */,
						true /* keep second */,
						*pmergedisa /* output stream */,
						blockmem/2/*par*/,
						numthreads /* parallel */,
						true /* delete input */,
						fanin,
						logstr
					);
					if ( logstr )
						(*logstr) << "done." << std::endl;
					libmaus2::aio::FileRemoval::removeFile(mergeisatmp.c_str());

					#if 0
					// check sampled suffix array by pairwise comparison on text
					// this can take quadratic time in fs
					checkSampledSA(fn,fs,mergedsaname,sasamplingrate,numthreads,lfblockmult);
					#endif
				}
			};

			extern template struct BwtMergeComputeSampledSA< libmaus2::suffixsort::ByteInputTypes<32> >;
			extern template struct BwtMergeComputeSampledSA< libmaus2::suffixsort::ByteInputTypes<64> >;
			extern template struct BwtMergeComputeSampledSA< libmaus2::suffixsort::CompactInputTypes<32> >;
			extern template struct BwtMergeComputeSampledSA< libmaus2::suffixsort::CompactInputTypes<64> >;
			extern template struct BwtMergeComputeSampledSA< libmaus2::suffixsort::PacInputTypes<32> >;
			extern template struct BwtMergeComputeSampledSA< libmaus2::suffixsort::PacInputTypes<64> >;
			extern template struct BwtMergeComputeSampledSA< libmaus2::suffixsort::PacTermInputTypes<32> >;
			extern template struct BwtMergeComputeSampledSA< libmaus2::suffixsort::PacTermInputTypes<64> >;
			extern template struct BwtMergeComputeSampledSA< libmaus2::suffixsort::Lz4InputTypes<32> >;
			extern template struct BwtMergeComputeSampledSA< libmaus2::suffixsort::Lz4InputTypes<64> >;
			extern template struct BwtMergeComputeSampledSA< libmaus2::suffixsort::Utf8InputTypes<32> >;
			extern template struct BwtMergeComputeSampledSA< libmaus2::suffixsort::Utf8InputTypes<64> >;
		}
	}
}
#endif
