/**
    libmaus2
    Copyright (C) 2009-2022 German Tischler-Höhle
    Copyright (C) 2011-2014 Genome Research Limited

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
**/
#if ! defined(LIBMAUS2_SUFFIXSORT_BWTB3M_BWTMERGESORTTEMPLATE_HPP)
#define LIBMAUS2_SUFFIXSORT_BWTB3M_BWTMERGESORTTEMPLATE_HPP

#include <libmaus2/suffixsort/bwtb3m/BwtMergeComputeSampledSA.hpp>
#include <libmaus2/suffixsort/bwtb3m/BwtMergeParallelGapFragMerge.hpp>
#include <libmaus2/suffixsort/bwtb3m/BwtMergeIsaParallelBase.hpp>
#include <libmaus2/suffixsort/bwtb3m/GapArrayWrapper.hpp>
#include <libmaus2/util/AtomicArray.hpp>
#include <libmaus2/suffixsort/bwtb3m/PreIsaAdapter.hpp>
#include <libmaus2/suffixsort/bwtb3m/BwtMergeIsaBase.hpp>
#include <libmaus2/suffixsort/bwtb3m/BWTB3MBase.hpp>
#include <libmaus2/lf/ImpCompactHuffmanWaveletLF.hpp>
#include <libmaus2/suffixsort/GapArrayByte.hpp>
#include <libmaus2/suffixsort/bwtb3m/BwtMergeSortOptions.hpp>
#include <libmaus2/suffixsort/BwtMergeBlockSortResult.hpp>
#include <libmaus2/lf/DArray.hpp>
#include <libmaus2/suffixsort/bwtb3m/MergeStrategyMergeGapRequest.hpp>
#include <libmaus2/suffixsort/bwtb3m/MergeStrategyMergeInternalBlock.hpp>
#include <libmaus2/suffixsort/bwtb3m/MergeStrategyMergeInternalSmallBlock.hpp>
#include <libmaus2/suffixsort/bwtb3m/MergeStrategyMergeExternalBlock.hpp>
#include <libmaus2/util/SimpleCountingHash.hpp>
#include <libmaus2/util/UnsignedCharVariant.hpp>
#include <libmaus2/suffixsort/GapMergePacket.hpp>
#include <libmaus2/bitio/BitVectorOutput.hpp>
#include <libmaus2/bitio/BitVectorInput.hpp>
#include <libmaus2/wavelet/RlToHwtTermRequest.hpp>
#include <libmaus2/gamma/SparseGammaGapMultiFileLevelSet.hpp>
#include <libmaus2/parallel/LockedBool.hpp>
#include <libmaus2/sorting/PairFileSorting.hpp>
#include <libmaus2/suffixsort/BwtMergeBlockSortRequestBase.hpp>
#include <libmaus2/suffixsort/BwtMergeTempFileNameSetVector.hpp>
#include <libmaus2/util/NumberMapSerialisation.hpp>
#include <libmaus2/suffixsort/bwtb3m/MergeStrategyBaseBlock.hpp>
#include <libmaus2/suffixsort/bwtb3m/MergeStrategyConstruction.hpp>
#include <libmaus2/suffixsort/bwtb3m/BaseBlockSorting.hpp>
#include <libmaus2/suffixsort/bwtb3m/BwtMergeSortResult.hpp>
#include <libmaus2/aio/ArrayFile.hpp>
#include <libmaus2/math/ilog.hpp>
#include <libmaus2/util/PrefixSums.hpp>
#include <libmaus2/util/BorderArray.hpp>
#include <libmaus2/util/AtomicArray.hpp>

#include <libmaus2/suffixsort/bwtb3m/InputTypesForward.hpp>


namespace libmaus2
{
	namespace suffixsort
	{
		namespace bwtb3m
		{
			struct BwtMergeSortState
			{
				private:
				// options
				BwtMergeSortOptions options;

				// file name of serialised character histogram (table with terminator symbol)
				std::string chistfilename;
				// file name of serialised huffman tree
				std::string huftreefilename;

				// file size
				uint64_t fs;

				// target block size
				uint64_t tblocksize;

				// number of blocks
				uint64_t numblocks;
				// final block size
				uint64_t blocksize;
				// full block product
				uint64_t fullblockprod;
				// extraneous
				uint64_t extrasyms;
				// full blocks
				uint64_t fullblocks;
				// reduced blocks
				uint64_t redblocks;

				// next power of two for block size
				uint64_t blocksizenexttwo;
				// prev power of two
				uint64_t blocksizeprevtwo;

				// ISA sampling rate during block merging
				uint64_t preisasamplingrate;

				// array of computed LCP values between block and start of next block
				std::shared_ptr< libmaus2::util::AtomicArray<uint64_t> > V_boundedlcpblockvalues;

				// base tmp directory name
				std::string tmpdirname;
				// tmpdirname as path object
				std::experimental::filesystem::path tmppath;
				std::experimental::filesystem::path base_tmp_path;
				std::experimental::filesystem::path ds_tmp_path_base_ds_tmp;
				std::experimental::filesystem::path merge_tmp_path;
				std::experimental::filesystem::path ds_tmp_path_merge_ds_tmp;

				std::unique_ptr<libmaus2::util::DirectoryStructure> DSbase;
				std::unique_ptr<libmaus2::util::DirectoryStructure> DSmerge;
				std::map < MergeStrategyBlock *, std::string > M_merge_dirs;
				std::map < MergeStrategyBlock *, std::size_t> M_merge_gt_expected;

				std::unique_ptr< ::libmaus2::suffixsort::BwtMergeTempFileNameSetVector > blocktmpnames;

				std::map<int64_t,uint64_t> chistnoterm;
				std::map<int64_t,uint64_t> chist;
				int64_t bwtterm;
				uint64_t maxsym;

				libmaus2::huffman::HuffmanTree::unique_ptr_type uhnode;

				std::shared_ptr<libmaus2::suffixsort::bwtb3m::MergeTree> merge_tree;

				static void serialiseNumber(std::ostream & ostr, uint64_t const n) {
					libmaus2::util::NumberSerialisation::serialiseNumber(ostr,n);
				}
				static uint64_t deserialiseNumber(std::istream & istr) {
					return libmaus2::util::NumberSerialisation::deserialiseNumber(istr);
				}

				void serialise(std::ostream & ostr) const
				{
					options.serialise(ostr);

					libmaus2::util::StringSerialisation::serialiseString(ostr,chistfilename);
					libmaus2::util::StringSerialisation::serialiseString(ostr,huftreefilename);

					libmaus2::util::NumberSerialisation::serialiseNumber(ostr,fs);
					libmaus2::util::NumberSerialisation::serialiseNumber(ostr,tblocksize);
					libmaus2::util::NumberSerialisation::serialiseNumber(ostr,numblocks);
					libmaus2::util::NumberSerialisation::serialiseNumber(ostr,fullblockprod);
					libmaus2::util::NumberSerialisation::serialiseNumber(ostr,extrasyms);
					libmaus2::util::NumberSerialisation::serialiseNumber(ostr,fullblocks);
					libmaus2::util::NumberSerialisation::serialiseNumber(ostr,redblocks);
					libmaus2::util::NumberSerialisation::serialiseNumber(ostr,blocksizenexttwo);
					libmaus2::util::NumberSerialisation::serialiseNumber(ostr,blocksizeprevtwo);
					libmaus2::util::NumberSerialisation::serialiseNumber(ostr,preisasamplingrate);

					V_boundedlcpblockvalues->serialise(ostr,serialiseNumber);

					libmaus2::util::StringSerialisation::serialiseString(ostr,tmpdirname);
					libmaus2::util::StringSerialisation::serialiseString(ostr,tmppath.string());

					libmaus2::util::StringSerialisation::serialiseString(ostr,base_tmp_path.string());
					libmaus2::util::StringSerialisation::serialiseString(ostr,ds_tmp_path_base_ds_tmp.string());

					libmaus2::util::StringSerialisation::serialiseString(ostr,merge_tmp_path.string());
					libmaus2::util::StringSerialisation::serialiseString(ostr,ds_tmp_path_merge_ds_tmp.string());

					libmaus2::util::NumberMapSerialisation::serialiseMap<std::ostream,int64_t,uint64_t>(ostr,chistnoterm);
					libmaus2::util::NumberMapSerialisation::serialiseMap<std::ostream,int64_t,uint64_t>(ostr,chist);
					libmaus2::util::NumberSerialisation::serialiseSignedNumber(ostr,bwtterm);
					libmaus2::util::NumberSerialisation::serialiseNumber(ostr,maxsym);

					if ( merge_tree )
					{
						libmaus2::util::NumberSerialisation::serialiseNumber(ostr,1);
						merge_tree->serialise(ostr);
					}
				}

				void deserialise(std::istream & istr)
				{
					options.deserialise(istr);

					chistfilename = libmaus2::util::StringSerialisation::deserialiseString(istr);
					huftreefilename = libmaus2::util::StringSerialisation::deserialiseString(istr);

					fs = libmaus2::util::NumberSerialisation::deserialiseNumber(istr);
					tblocksize = libmaus2::util::NumberSerialisation::deserialiseNumber(istr);
					numblocks = libmaus2::util::NumberSerialisation::deserialiseNumber(istr);
					fullblockprod = libmaus2::util::NumberSerialisation::deserialiseNumber(istr);
					extrasyms = libmaus2::util::NumberSerialisation::deserialiseNumber(istr);
					fullblocks = libmaus2::util::NumberSerialisation::deserialiseNumber(istr);
					redblocks = libmaus2::util::NumberSerialisation::deserialiseNumber(istr);
					blocksizenexttwo = libmaus2::util::NumberSerialisation::deserialiseNumber(istr);
					blocksizeprevtwo = libmaus2::util::NumberSerialisation::deserialiseNumber(istr);
					preisasamplingrate = libmaus2::util::NumberSerialisation::deserialiseNumber(istr);

					std::shared_ptr< libmaus2::util::AtomicArray<uint64_t> > t_V_boundedlcpblockvalues(
						new libmaus2::util::AtomicArray<uint64_t>(
							istr,
							deserialiseNumber
						)
					);

					tmpdirname = libmaus2::util::StringSerialisation::deserialiseString(istr);
					tmppath = libmaus2::util::StringSerialisation::deserialiseString(istr);

					base_tmp_path = libmaus2::util::StringSerialisation::deserialiseString(istr);
					ds_tmp_path_base_ds_tmp = libmaus2::util::StringSerialisation::deserialiseString(istr);

					merge_tmp_path = libmaus2::util::StringSerialisation::deserialiseString(istr);
					ds_tmp_path_merge_ds_tmp = libmaus2::util::StringSerialisation::deserialiseString(istr);

					std::unique_ptr<libmaus2::util::DirectoryStructure> t_DSbase(
						BwtMergeTempFileNameSetVector::getDirectoryStructure(
							ds_tmp_path_base_ds_tmp.string(),
							64 /* mod */,
							base_tmp_path.string(),
							numblocks,
							options.numthreads /* bwt */,
							options.numthreads /* gt */
						)
					);

					DSbase = std::move(t_DSbase);

					std::unique_ptr< ::libmaus2::suffixsort::BwtMergeTempFileNameSetVector > t_blocktmpnames(
						new ::libmaus2::suffixsort::BwtMergeTempFileNameSetVector(*DSbase, getNumBlocks(), options.numthreads /* bwt */, options.numthreads /* gt */)
					);

					blocktmpnames = std::move(t_blocktmpnames);

					chistnoterm = libmaus2::util::NumberMapSerialisation::deserialiseMap<std::istream,int64_t,uint64_t>(istr);
					chist       = libmaus2::util::NumberMapSerialisation::deserialiseMap<std::istream,int64_t,uint64_t>(istr);
					bwtterm = libmaus2::util::NumberSerialisation::deserialiseSignedNumber(istr);
					maxsym = libmaus2::util::NumberSerialisation::deserialiseNumber(istr);

					if ( chist.size() )
						computeHuffmanTree();

					bool const have_merge_tree = libmaus2::util::NumberSerialisation::deserialiseNumber(istr) != 0;

					if ( have_merge_tree )
					{
						std::shared_ptr<libmaus2::suffixsort::bwtb3m::MergeTree> t_merge_tree(
							new libmaus2::suffixsort::bwtb3m::MergeTree(istr)
						);
						merge_tree = t_merge_tree;

						setupMergeDirectoryStructure();
					}
				}

				static uint64_t getFileSize(
					BwtMergeSortOptions const & options
				)
				{
					// check whether file exists
					if ( ! ::libmaus2::util::GetFileSize::fileExists(options.fn) )
					{
						::libmaus2::exception::LibMausException se;
						se.getStream() << "File " << options.fn << " does not exist or cannot be opened." << std::endl;
						se.finish();
						throw se;
					}

					/* get file size */
					uint64_t const fs = InputTypesForward::getFileSize(options.fn,options.inputtype,options.wordsize);

					/* check that file is not empty */
					if ( ! fs )
					{
						::libmaus2::exception::LibMausException se;
						se.getStream() << "File " << options.fn << " is empty." << std::endl;
						se.finish();
						throw se;
					}

					return fs;
				}

				static uint64_t getDefaultBlockSize(uint64_t const mem, uint64_t const threads, uint64_t const fs)
				{
					uint64_t const memblocksize = std::max(static_cast<uint64_t>(0.95 * mem / ( 5 * threads )),static_cast<uint64_t>(1));
					uint64_t const fsblocksize = (fs + threads - 1)/threads;
					return std::min(memblocksize,fsblocksize);
				}


				static std::map<int64_t,uint64_t> mergeMaps(
					std::map<int64_t,uint64_t> const & A,
					std::map<int64_t,uint64_t> const & B)
				{
					std::map<int64_t,uint64_t>::const_iterator aita = A.begin(), aite = A.end();
					std::map<int64_t,uint64_t>::const_iterator bita = B.begin(), bite = B.end();
					std::map<int64_t,uint64_t> C;

					while ( aita != aite && bita != bite )
					{
						if ( aita->first < bita->first )
						{
							C[aita->first] = aita->second;
							++aita;
						}
						else if ( bita->first < aita->first )
						{
							C[bita->first] = bita->second;
							++bita;
						}
						else
						{
							C[aita->first] = aita->second + bita->second;
							++aita; ++bita;
						}
					}

					while ( aita != aite )
					{
						C[aita->first] = aita->second;
						++aita;
					}

					while ( bita != bite )
					{
						C[bita->first] = bita->second;
						++bita;
					}

					return C;
				}

				static std::experimental::filesystem::path ensureDirectory(std::experimental::filesystem::path path)
				{
					libmaus2::aio::OutputStreamFactoryContainer::mkdirp(path.string(),0700);
					return path;
				}

				BwtMergeSortState() = delete;
				BwtMergeSortState(BwtMergeSortOptions const & r_options)
				:
					options(r_options),
					chistfilename(options.outfn + ".chist"),
					huftreefilename(options.outfn + ".huftree"),
					fs(getFileSize(options)),
					tblocksize(std::max(static_cast<uint64_t>(1),std::min(options.maxblocksize,getDefaultBlockSize(options.mem,options.numthreads,fs)))),
					numblocks((fs + tblocksize - 1) / tblocksize),
					blocksize((fs + numblocks - 1)/ numblocks),
					fullblockprod(numblocks * blocksize),
					extrasyms(fullblockprod - fs),
					fullblocks(numblocks - extrasyms),
					redblocks(numblocks - fullblocks),
					blocksizenexttwo(::libmaus2::math::nextTwoPow(blocksize)),
					blocksizeprevtwo((blocksize == blocksizenexttwo) ? blocksize : (blocksizenexttwo / 2)),
					preisasamplingrate(std::min(options.maxpreisasamplingrate,blocksizeprevtwo)),
					V_boundedlcpblockvalues(new libmaus2::util::AtomicArray<uint64_t>(numblocks,0)),
					// tmp directory name
					tmpdirname(ensureDirectory(options.tmpfilenamebase + "_tmpdir")),
					// path object for tmp directory name
					tmppath(tmpdirname),
					base_tmp_path(ensureDirectory(tmppath / "base_tmp")),
					ds_tmp_path_base_ds_tmp(tmppath / "base_ds.tmp"),
					merge_tmp_path(ensureDirectory(tmppath / "merge_tmp")),
					ds_tmp_path_merge_ds_tmp(tmppath / "merge_ds.tmp"),
					DSbase(
						BwtMergeTempFileNameSetVector::getDirectoryStructure(
							ds_tmp_path_base_ds_tmp.string(),
							64 /* mod */,
							base_tmp_path.string(),
							numblocks,
							options.numthreads /* bwt */,
							options.numthreads /* gt */
						)
					),
					DSmerge(),
					blocktmpnames(
						new ::libmaus2::suffixsort::BwtMergeTempFileNameSetVector(*DSbase, getNumBlocks(), options.numthreads /* bwt */, options.numthreads /* gt */)
					),
					chistnoterm(),
					chist(),
					bwtterm(0),
					maxsym(0),
					uhnode()
				{
					// check
					assert ( extrasyms < numblocks );
					// check
					assert ( fullblocks * blocksize + redblocks * (blocksize-1) == fs );
					// there should be at least one block as input size is not zero
					assert ( numblocks );

					DSbase->doGenerate();

					// std::cerr << "blocktmpnames=\n" << blocktmpnames->toString();
				}

				public:
				static std::unique_ptr<BwtMergeSortState> construct(BwtMergeSortOptions const & options)
				{
					std::unique_ptr<BwtMergeSortState> tptr(new BwtMergeSortState(options));
					return tptr;
				}

				void cleanup()
				{
					if ( DSmerge ) {
						for ( auto const & P : M_merge_dirs )
							libmaus2::aio::OutputStreamFactoryContainer::rmdir(P.second);
						DSmerge->doRemove();
					}
					else
						libmaus2::aio::OutputStreamFactoryContainer::rmdir(merge_tmp_path);

					DSbase->doRemove();
					libmaus2::aio::FileRemoval::removeFile(ds_tmp_path_base_ds_tmp.string());
					libmaus2::aio::FileRemoval::removeFile(ds_tmp_path_merge_ds_tmp.string());
					libmaus2::aio::OutputStreamFactoryContainer::rmdir(tmpdirname);
				}

				uint64_t getFS() const
				{
					return fs;
				}

				uint64_t getBlockSize() const
				{
					return blocksize;
				}

				uint64_t getNumBlocks() const
				{
					return numblocks;
				}

				uint64_t getFullBlocks() const
				{
					return fullblocks;
				}

				uint64_t getReducedBlocks() const
				{
					return redblocks;
				}

				uint64_t getPreISASamplingRate() const
				{
					return preisasamplingrate;
				}

				libmaus2::util::AtomicArray<uint64_t> & getBlockLCPArray()
				{
					return *V_boundedlcpblockvalues;
				}

				libmaus2::util::AtomicArray<uint64_t> const & getBlockLCPArray() const
				{
					return *V_boundedlcpblockvalues;
				}

				::libmaus2::suffixsort::BwtMergeTempFileNameSetVector const & getBlockTmpNames() const
				{
					return *blocktmpnames;
				}

				uint64_t getBlockStart(uint64_t const b) const
				{
					return (b < fullblocks) ? (b*blocksize) : (fullblocks * blocksize + (b-fullblocks) * (blocksize-1));
				}

				uint64_t getBlockSize(uint64_t const b) const
				{
					return (b < fullblocks) ? blocksize : (blocksize-1);
				}

				std::map<int64_t,uint64_t> const & getCharacterHistogram() const
				{
					return chist;
				}

				std::map<int64_t,uint64_t> const & getCharacterHistogramNoTerm() const
				{
					return chistnoterm;
				}

				int64_t getBWTTerm() const
				{
					return bwtterm;
				}

				uint64_t getMaxSym() const
				{
					return maxsym;
				}

				std::string getCHistFileName() const
				{
					return chistfilename;
				}

				std::string getHufTreeFileName() const
				{
					return huftreefilename;
				}

				void computeCharacterHistogram()
				{
					// combine frequences for blocks to frequences of complete input text
					chistnoterm.clear();
					for ( uint64_t b = 0; b < getNumBlocks(); ++b )
					{
						std::string const freqstmpfilename = (*blocktmpnames)[b].getHistFreq();
						libmaus2::aio::InputStreamInstance ISI(freqstmpfilename);
						std::map<int64_t,uint64_t> const blockfreqs(libmaus2::util::NumberMapSerialisation::deserialiseMap<std::istream,int64_t,uint64_t>(ISI));
						chistnoterm = mergeMaps(chistnoterm,blockfreqs);
					}

					if ( chistnoterm.begin() == chistnoterm.end() )
					{
						libmaus2::exception::LibMausException lme;
						lme.getStream() << "[E] character histogram is empty" << std::endl;
						lme.finish();
						throw lme;
					}

					/* compute bwt terminator symbol (maximum symbol plus 1) */
					bwtterm = chistnoterm.rbegin()->first+1;
					chist = chistnoterm;
					chist[bwtterm] = 1;
					maxsym = chist.size() ? chist.rbegin()->first : 0;
				}

				void serialiseCharacterHistogram() const
				{
					libmaus2::aio::OutputStreamInstance::unique_ptr_type chistCOS(new libmaus2::aio::OutputStreamInstance(chistfilename));
					(*chistCOS) << ::libmaus2::util::NumberMapSerialisation::serialiseMap(chist);
					chistCOS->flush();
					chistCOS.reset();
				}

				void computeHuffmanTree()
				{
					libmaus2::huffman::HuffmanTree::unique_ptr_type t_uhnode(
						new libmaus2::huffman::HuffmanTree(
							chist.begin(),
							chist.size(),
							false /* sort by depth */,
							true /* set code */,
							true /* reorder by depth first */
						)
					);

					uhnode = std::move(t_uhnode);
				}

				libmaus2::huffman::HuffmanTree const & getHuffmanTree() const
				{
					assert ( uhnode.get() );
					return *uhnode;
				}

				void serialiseHuffmanTree() const
				{
					libmaus2::aio::OutputStreamInstance::unique_ptr_type huftreeCOS(new libmaus2::aio::OutputStreamInstance(huftreefilename));
					uhnode->serialise(*huftreeCOS);
					huftreeCOS->flush();
					huftreeCOS.reset();
				}

				std::size_t getMergeGTExpected(MergeStrategyBlock * p) const
				{
					auto it = M_merge_gt_expected.find(p);

					if ( it == M_merge_gt_expected.end() ) {
						libmaus2::exception::LibMausException lme;
						lme.getStream() << "BwtMergeState::getMergeGTExpected: unable to find node" << std::endl;
						lme.finish();
						throw lme;
					}

					return it->second;
				}

				std::string getDirectoryForMerge(MergeStrategyBlock * p) const
				{
					auto it = M_merge_dirs.find(p);

					if ( it == M_merge_dirs.end() ) {
						libmaus2::exception::LibMausException lme;
						lme.getStream() << "BwtMergeState::getDirectoryForMerge: unable to find node" << std::endl;
						lme.finish();
						throw lme;
					}

					return it->second;
				}

				void fillGTExpected()
				{
					std::vector < MergeStrategyMergeInternalBlock * > V_internal;
					std::vector < MergeStrategyMergeInternalSmallBlock * > V_internal_small;
					std::vector < MergeStrategyMergeExternalBlock * > V_external;

					merge_tree->getMergeBlocks(V_internal,V_internal_small,V_external);

					for ( auto * p : V_internal )
						M_merge_gt_expected[p] = p->getNumGtTempFilesRequired();
					for ( auto * p : V_internal_small )
						M_merge_gt_expected[p] = p->getNumGtTempFilesRequired();
					for ( auto * p : V_external )
						M_merge_gt_expected[p] = p->getNumGtTempFilesRequired();
				}

				void setupMergeDirectoryStructure()
				{
					std::vector < MergeStrategyMergeInternalBlock * > V_internal;
					std::vector < MergeStrategyMergeInternalSmallBlock * > V_internal_small;
					std::vector < MergeStrategyMergeExternalBlock * > V_external;

					merge_tree->getMergeBlocks(V_internal,V_internal_small,V_external);

					std::size_t const num_merge_nodes = V_internal.size() + V_internal_small.size() + V_external.size();

					std::unique_ptr<libmaus2::util::DirectoryStructure> tDS(
						new libmaus2::util::DirectoryStructure(
							ds_tmp_path_merge_ds_tmp.string(),
							64 /* mod */,
							num_merge_nodes,
							merge_tmp_path.string()
						)
					);

					DSmerge = std::move(tDS);
					DSmerge->setAsciiFlag(true);

					std::size_t z = 0;
					for ( auto * p : V_internal )
						M_merge_dirs[p] = (*DSmerge)[z++];
					for ( auto * p : V_internal_small )
						M_merge_dirs[p] = (*DSmerge)[z++];
					for ( auto * p : V_external )
						M_merge_dirs[p] = (*DSmerge)[z++];
				}

				void setMergeTree(std::shared_ptr<libmaus2::suffixsort::bwtb3m::MergeTree> r_merge_tree)
				{
					merge_tree = r_merge_tree;

					std::vector < MergeStrategyMergeInternalBlock * > V_internal;
					std::vector < MergeStrategyMergeInternalSmallBlock * > V_internal_small;
					std::vector < MergeStrategyMergeExternalBlock * > V_external;

					merge_tree->getMergeBlocks(V_internal,V_internal_small,V_external);

					setupMergeDirectoryStructure();
					DSmerge->doGenerate();
					for ( auto & P : M_merge_dirs ) {
						libmaus2::aio::OutputStreamFactoryContainer::mkdir(P.second,0700);
						P.first->sortresult.setTempPrefixSingle(P.second);
					}
				}
			};
		}
	}
}

namespace libmaus2
{
	namespace suffixsort
	{
		namespace bwtb3m
		{
			struct GapArrayComputationResult
			{
				GapArrayWrapper G;
				uint64_t Gsize;
				std::vector < std::string > gtpartnames;
				uint64_t zactive;
				::libmaus2::autoarray::AutoArray<uint64_t> zabsblockpos;

				GapArrayComputationResult()
				: zactive(0)
				{

				}

				GapArrayComputationResult(
					std::shared_ptr<libmaus2::util::AtomicArray<uint32_t> > rG,
					uint64_t const rGsize,
					std::vector < std::string > const & rgtpartnames,
					uint64_t const rzactive,
					::libmaus2::autoarray::AutoArray<uint64_t> & rzabsblockpos
				)
				: G(rG), Gsize(rGsize), gtpartnames(rgtpartnames), zactive(rzactive), zabsblockpos(rzabsblockpos) {}
			};
		}
	}
}
namespace libmaus2
{
	namespace suffixsort
	{
		namespace bwtb3m
		{

			struct GapArrayByteComputationResult
			{
				libmaus2::suffixsort::GapArrayByte::shared_ptr_type G;
				uint64_t Gsize;
				std::vector < std::string > gtpartnames;
				uint64_t zactive;
				::libmaus2::autoarray::AutoArray<uint64_t> zabsblockpos;

				GapArrayByteComputationResult()
				: zactive(0)
				{

				}

				GapArrayByteComputationResult(
					libmaus2::suffixsort::GapArrayByte::shared_ptr_type rG,
					uint64_t const rGsize,
					std::vector < std::string > const & rgtpartnames,
					uint64_t const rzactive,
					::libmaus2::autoarray::AutoArray<uint64_t> & rzabsblockpos
				)
				: G(rG), Gsize(rGsize), gtpartnames(rgtpartnames), zactive(rzactive), zabsblockpos(rzabsblockpos) {}
			};
		}
	}
}
namespace libmaus2
{
	namespace suffixsort
	{
		namespace bwtb3m
		{
			struct SparseGapArrayComputationResult
			{
				// name of gap file
				std::vector<std::string> fn;
				std::vector < std::string > gtpartnames;
				uint64_t zactive;
				::libmaus2::autoarray::AutoArray<uint64_t> zabsblockpos;

				SparseGapArrayComputationResult()
				: zactive(0)
				{

				}

				SparseGapArrayComputationResult(
					std::vector < std::string > const & rfn,
					std::vector < std::string > const & rgtpartnames,
					uint64_t const rzactive,
					::libmaus2::autoarray::AutoArray<uint64_t> & rzabsblockpos
				)
				: fn(rfn), gtpartnames(rgtpartnames), zactive(rzactive), zabsblockpos(rzabsblockpos) {}
			};
		}
	}
}

namespace libmaus2
{
	namespace suffixsort
	{
		namespace bwtb3m
		{
		}
	}
}



namespace libmaus2
{
	namespace suffixsort
	{
		namespace bwtb3m
		{
			template<typename input_types_type>
			struct BwtMergeSortTemplate : public libmaus2::suffixsort::bwtb3m::BWTB3MBase
			{
				static std::vector< std::vector<std::string> > stringVectorPack(std::vector<std::string> const & Vin)
				{
					std::vector< std::vector<std::string> > Vout;
					for ( uint64_t i = 0; i < Vin.size(); ++i )
						Vout.push_back(std::vector<std::string>(1,Vin[i]));
					return Vout;
				}

				static void concatenateGT(
					std::vector<std::string> const & gtpartnames,
					std::string const & newgtpart, // blockresults.files.gt
					std::string const & newmergedgtname,
					std::ostream * logstr
				)
				{
					::libmaus2::timing::RealTimeClock rtc;
					rtc.start();

					std::vector< std::string > allfiles(gtpartnames.begin(),gtpartnames.end());
					allfiles.push_back(newgtpart);

					#if 0
					// encoder for new gt stream
					libmaus2::bitio::BitVectorOutput GTHEFref(newmergedgtname);
					libmaus2::bitio::BitVectorInput BVI(allfiles);
					uint64_t const tn = libmaus2::bitio::BitVectorInput::getLength(allfiles);
					for ( uint64_t i = 0; i < tn; ++i )
						GTHEFref.writeBit(BVI.readBit());
					GTHEFref.flush();
					#endif

					libmaus2::bitio::BitVectorOutput GTHEF(newmergedgtname);

					unsigned int prevbits = 0;
					uint64_t prev = 0;

					// append part streams
					for ( uint64_t z = 0; z < allfiles.size(); ++z )
					{
						uint64_t const n = libmaus2::bitio::BitVectorInput::getLength(allfiles[z]);
						uint64_t const fullwords = n / 64;
						uint64_t const restbits = n-fullwords*64;
						libmaus2::aio::SynchronousGenericInput<uint64_t> SGI(allfiles[z],8192);
						uint64_t v = 0;

						if ( !prevbits )
						{

							// copy full words
							for ( uint64_t i = 0; i < fullwords; ++i )
							{
								bool const ok = SGI.getNext(v);
								assert ( ok );
								GTHEF.SGO.put(v);
							}


							// get next word
							if ( restbits )
							{
								SGI.getNext(prev);
								// move bits to top of word
								prev <<= (64-restbits);
								// number of bits left
								prevbits = restbits;
							}
							else
							{
								prevbits = 0;
								prev = 0;
							}
						}
						else
						{
							// process full words
							for ( uint64_t i = 0; i < fullwords; ++i )
							{
								bool const ok = SGI.getNext(v);
								assert ( ok );

								GTHEF.SGO.put(prev | (v >> prevbits));
								prev = v << (64-prevbits);
							}

							// if there are bits left in this stream
							if ( restbits )
							{
								// get next word
								SGI.getNext(v);
								// move to top
								v <<= (64 - restbits);

								// rest with previous rest fill more than a word
								if ( restbits + prevbits >= 64 )
								{
									GTHEF.SGO.put(prev | (v >> prevbits));
									prev = v << (64-prevbits);
									prevbits = restbits + prevbits - 64;
								}
								else
								{
									prev = prev | (v >> prevbits);
									prevbits = prevbits + restbits;
								}
							}
							// leave as is
							else
							{

							}
						}

						libmaus2::aio::FileRemoval::removeFile(gtpartnames[z].c_str());
					}

					for ( uint64_t i = 0; i < prevbits; ++i )
						GTHEF.writeBit((prev >> (63-i)) & 1);

					// flush gt stream
					GTHEF.flush();

					if ( logstr )
						(*logstr) << "[V] concatenated bit vectors in time " << rtc.getElapsedSeconds() << std::endl;
				}


				static libmaus2::wavelet::ImpCompactHuffmanWaveletTree::unique_ptr_type ensureWaveletTreeGenerated(
					::libmaus2::suffixsort::BwtMergeBlockSortResult const & blockresults,
					std::ostream * logstr
				)
				{
					// generate wavelet tree from request if necessary
					if ( ! libmaus2::util::GetFileSize::fileExists(blockresults.getFiles().getHWT()) )
					{
						libmaus2::timing::RealTimeClock rtc; rtc.start();
						if ( logstr )
							(*logstr) << "[V] Generating HWT for gap file computation...";
						assert ( libmaus2::util::GetFileSize::fileExists(blockresults.getFiles().getHWTReq() ) );
						libmaus2::wavelet::RlToHwtTermRequest::unique_ptr_type ureq(libmaus2::wavelet::RlToHwtTermRequest::load(blockresults.getFiles().getHWTReq()));
						libmaus2::wavelet::ImpCompactHuffmanWaveletTree::unique_ptr_type ptr(ureq->dispatch<rl_decoder>());
						libmaus2::aio::FileRemoval::removeFile ( blockresults.getFiles().getHWTReq().c_str() );
						if ( logstr )
							(*logstr) << "done, time " << rtc.getElapsedSeconds() << std::endl;
						return ptr;
					}
					else
					{
						libmaus2::aio::InputStreamInstance CIS(blockresults.getFiles().getHWT());
						libmaus2::wavelet::ImpCompactHuffmanWaveletTree::unique_ptr_type ptr(new libmaus2::wavelet::ImpCompactHuffmanWaveletTree(CIS));
						return ptr;
					}
				}

				static GapArrayComputationResult computeGapArray(
					libmaus2::util::TempFileNameGenerator & gtmpgen,
					std::string const & fn, // name of text file
					uint64_t const fs, // length of text file in symbols
					uint64_t const blockstart, // start offset
					uint64_t const cblocksize, // block size
					uint64_t const nextblockstart, // start of next block (mod fs)
					uint64_t const mergeprocrightend, // right end of merged area
					::libmaus2::suffixsort::BwtMergeBlockSortResult const & blockresults, // information on block
					std::vector<std::string> const & mergedgtname, // previous gt file name
					// std::string const & newmergedgtname, // new gt file name
					::libmaus2::lf::DArray * const accD, // accumulated symbol freqs for block
					std::vector < ::libmaus2::suffixsort::BwtMergeZBlock > const & zblocks, // lf starting points
					uint64_t const numthreads,
					std::ostream * logstr,
					int const verbose
				)
				{
					std::vector < std::string > gtpartnames(zblocks.size());
					for ( std::size_t z = 0; z < gtpartnames.size(); ++z ) {
						std::string const gtpartname = gtmpgen.getFileName() + "_" + ::libmaus2::util::NumberSerialisation::formatNumber(z,4) + ".gt";
						::libmaus2::util::TempFileRemovalContainer::addTempFile(gtpartname);
						gtpartnames[z] = gtpartname;
					}

					// gap array
					uint64_t const Gsize = cblocksize+1;

					std::shared_ptr<libmaus2::util::AtomicArray<uint32_t> > pG(
						new libmaus2::util::AtomicArray<uint32_t>(Gsize,0,numthreads)
					);
					libmaus2::util::AtomicArray<uint32_t> & G(*pG);

					std::string const histfn = blockresults.getFiles().getHist();

					if ( verbose >= 5 && logstr )
					{
						(*logstr) << "[V] loading histogram " << histfn << std::endl;
					}

					// set up lf mapping
					::libmaus2::lf::DArray D(histfn);

					if ( verbose >= 5 && logstr )
					{
						(*logstr) << "[V] loading histogram done" << std::endl;
					}

					accD->merge(D);
					#if 0
					bool const hwtdelayed = ensureWaveletTreeGenerated(blockresults);
					#endif
					if ( verbose >= 5 && logstr )
					{
						(*logstr) << "[V] loading HWT" << std::endl;
					}

					::libmaus2::wavelet::ImpCompactHuffmanWaveletTree::unique_ptr_type ICHWL(ensureWaveletTreeGenerated(blockresults,logstr));

					if ( verbose >= 5 && logstr )
					{
						(*logstr) << "[V] loading HWT done" << std::endl;
					}

					if ( verbose >= 5 && logstr )
					{
						(*logstr) << "[V] setting up LF" << std::endl;
					}

					::libmaus2::lf::ImpCompactHuffmanWaveletLF IHWL(ICHWL);

					if ( verbose >= 5 && logstr )
					{
						(*logstr) << "[V] setting up LF done" << std::endl;
					}

					IHWL.D = D.D;
					assert ( cblocksize == IHWL.n );

					// rank of position 0 in this block (for computing new gt array/stream)
					uint64_t const lp0 = blockresults.getBlockP0Rank();

					if ( verbose >= 5 && logstr )
					{
						(*logstr) << "[V] calling getSymbolAtPosition for last symbol of first/left block" << std::endl;
					}

					// last symbol in this block
					int64_t const firstblocklast = input_types_type::linear_wrapper::getSymbolAtPosition(fn,(nextblockstart+fs-1)%fs);

					if ( verbose >= 5 && logstr )
					{
						(*logstr) << "[V] calling getSymbolAtPosition for last symbol of first/left block done" << std::endl;
					}

					/**
					 * array of absolute positions
					 **/
					uint64_t const zactive = zblocks.size();
					::libmaus2::autoarray::AutoArray<uint64_t> zabsblockpos(zactive+1,false);
					for ( uint64_t z = 0; z < zactive; ++z )
						zabsblockpos[z] = zblocks[z].getZAbsPos();
					zabsblockpos [ zactive ] = blockstart + cblocksize;


					if ( verbose >= 5 && logstr )
					{
						(*logstr) << "[V] entering gap array loop" << std::endl;
					}

					::libmaus2::timing::RealTimeClock rtc;
					rtc.start();
					#if defined(_OPENMP)
					#pragma omp parallel for schedule(dynamic,1) num_threads(numthreads)
					#endif
					for ( int64_t z = 0; z < static_cast<int64_t>(zactive); ++z )
					{
						::libmaus2::timing::RealTimeClock subsubrtc; subsubrtc.start();

						::libmaus2::suffixsort::BwtMergeZBlock const & zblock = zblocks[z];

						std::string const gtpartname = gtpartnames.at(z);

						#if 0
						::libmaus2::huffman::HuffmanEncoderFileStd GTHEF(gtpartname);
						#endif
						libmaus2::bitio::BitVectorOutput GTHEF(gtpartname);

						#if 0
						::libmaus2::bitio::BitStreamFileDecoder gtfile(mergedgtname, (mergeprocrightend - zblock.getZAbsPos()) );
						#endif
						libmaus2::bitio::BitVectorInput gtfile(mergedgtname, (mergeprocrightend - zblock.getZAbsPos()) );

						typename input_types_type::circular_reverse_wrapper CRWR(fn,zblock.getZAbsPos() % fs);
						uint64_t r = zblock.getZRank();

						uint64_t const zlen = zabsblockpos [ z ] - zabsblockpos [z+1];

						for ( uint64_t i = 0; i < zlen; ++i )
						{
							GTHEF.writeBit(r > lp0);

							int64_t const sym = CRWR.get();
							bool const gtf = gtfile.readBit();

							r = IHWL.step(sym,r) + ((sym == firstblocklast)?gtf:0);

							G[r]++;
						}

						GTHEF.flush();
					}
					if ( logstr )
						(*logstr) << "[V] computed gap array in time " << rtc.getElapsedSeconds() << std::endl;

					uint64_t const cperblock = (Gsize + numthreads - 1)/numthreads;
					uint64_t const cblocks = (Gsize + cperblock - 1)/cperblock;
					std::atomic<uint64_t> gs(0);
					libmaus2::parallel::StdSpinLock gslock;

					#if defined(_OPENMP)
					#pragma omp parallel for num_threads(numthreads)
					#endif
					for ( uint64_t t = 0; t < cblocks; ++t )
					{
						uint64_t const low = t * cperblock;
						uint64_t const high = std::min(low+cperblock,Gsize);
						uint64_t s = 0;
						std::atomic<uint32_t> const * g = G.get() + low;
						std::atomic<uint32_t> const * const ge = G.get() + high;
						while ( g != ge )
							s += *(g++);
						libmaus2::parallel::ScopeStdSpinLock slock(gslock);
						gs += s;
					}

					uint64_t es = 0;
					for ( int64_t z = 0; z < static_cast<int64_t>(zactive); ++z )
					{
						uint64_t const zlen = zabsblockpos [ z ] - zabsblockpos [z+1];
						es += zlen;
					}

					if ( logstr )
					{
						(*logstr) << "[V] gs=" << gs << " es=" << es << std::endl;
					}

					assert ( es == gs );

					return GapArrayComputationResult(pG,Gsize,gtpartnames,zactive,zabsblockpos);
				}

				static GapArrayComputationResult computeGapArray(
					libmaus2::util::TempFileNameGenerator & gtmpgen,
					std::string const & fn, // name of text file
					uint64_t const fs, // length of text file in symbols
					libmaus2::suffixsort::bwtb3m::MergeStrategyMergeGapRequest const & msmgr, // merge request
					std::vector<std::string> const & mergedgtname, // previous gt file name
					::libmaus2::lf::DArray * const accD, // accumulated symbol freqs for block
					uint64_t const numthreads,
					std::ostream * logstr,
					int const verbose
				)
				{
					uint64_t const into = msmgr.into;

					std::vector<libmaus2::suffixsort::bwtb3m::MergeStrategyBlock::shared_ptr_type> const & children =
						*(msmgr.pchildren);

					::libmaus2::suffixsort::BwtMergeBlockSortResult const & blockresults = children[into]->sortresult;

					uint64_t const blockstart = blockresults.getBlockStart();
					uint64_t const cblocksize = blockresults.getCBlockSize();
					uint64_t const nextblockstart = (blockstart+cblocksize)%fs;
					uint64_t const mergeprocrightend =
						children.at(children.size()-1)->sortresult.getBlockStart() +
						children.at(children.size()-1)->sortresult.getCBlockSize();
					// use gap object's zblocks vector
					std::vector < ::libmaus2::suffixsort::BwtMergeZBlock > const & zblocks = msmgr.zblocks;

					return computeGapArray(gtmpgen,fn,fs,blockstart,cblocksize,nextblockstart,mergeprocrightend,
						blockresults,mergedgtname,accD,zblocks,numthreads,logstr,verbose);
				}

				static GapArrayByteComputationResult computeGapArrayByte(
					libmaus2::util::TempFileNameGenerator & gtmpgen,
					std::string const & fn, // name of text file
					uint64_t const fs, // length of text file in symbols
					uint64_t const blockstart, // start offset
					uint64_t const cblocksize, // block size
					uint64_t const nextblockstart, // start of next block (mod fs)
					uint64_t const mergeprocrightend, // right end of merged area
					::libmaus2::suffixsort::BwtMergeBlockSortResult const & blockresults, // information on block
					std::vector<std::string> const & mergedgtname, // previous gt file name
					#if 0
					std::string const & newmergedgtname, // new gt file name
					std::string const & gapoverflowtmpfilename, // gap overflow tmp file
					#endif
					::libmaus2::lf::DArray * const accD, // accumulated symbol freqs for block
					std::vector < ::libmaus2::suffixsort::BwtMergeZBlock > const & zblocks, // lf starting points
					uint64_t const numthreads,
					std::ostream * logstr
				)
				{
					std::string const gapoverflowtmpfilename = gtmpgen.getFileName() + "_gapoverflow";

					// gap array
					uint64_t const Gsize = cblocksize+1;
					::libmaus2::suffixsort::GapArrayByte::shared_ptr_type pG(
						new ::libmaus2::suffixsort::GapArrayByte(
							Gsize,
							512, /* number of overflow words per thread */
							numthreads,
							gapoverflowtmpfilename
						)
					);
					::libmaus2::suffixsort::GapArrayByte & G = *pG;

					// set up lf mapping
					::libmaus2::lf::DArray D(static_cast<std::string const &>(blockresults.getFiles().getHist()));
					accD->merge(D);
					#if 0
					bool const hwtdelayed = ensureWaveletTreeGenerated(blockresults);
					#endif
					::libmaus2::wavelet::ImpCompactHuffmanWaveletTree::unique_ptr_type ICHWL(ensureWaveletTreeGenerated(blockresults,logstr));
					::libmaus2::lf::ImpCompactHuffmanWaveletLF IHWL(ICHWL);
					IHWL.D = D.D;
					assert ( cblocksize == IHWL.n );

					// rank of position 0 in this block (for computing new gt array/stream)
					uint64_t const lp0 = blockresults.getBlockP0Rank();

					// last symbol in this block
					int64_t const firstblocklast = input_types_type::linear_wrapper::getSymbolAtPosition(fn,(nextblockstart+fs-1)%fs);

					/**
					 * array of absolute positions
					 **/
					uint64_t const zactive = zblocks.size();
					::libmaus2::autoarray::AutoArray<uint64_t> zabsblockpos(zactive+1,false);
					for ( uint64_t z = 0; z < zactive; ++z )
						zabsblockpos[z] = zblocks[z].getZAbsPos();
					zabsblockpos [ zactive ] = blockstart + cblocksize;

					std::vector < std::string > gtpartnames(zactive);

					::libmaus2::timing::RealTimeClock rtc;
					rtc.start();
					#if defined(_OPENMP)
					#pragma omp parallel for schedule(dynamic,1) num_threads(numthreads)
					#endif
					for ( int64_t z = 0; z < static_cast<int64_t>(zactive); ++z )
					{
						::libmaus2::timing::RealTimeClock subsubrtc; subsubrtc.start();

						::libmaus2::suffixsort::BwtMergeZBlock const & zblock = zblocks[z];

						std::string const gtpartname = gtmpgen.getFileName() + "_" + ::libmaus2::util::NumberSerialisation::formatNumber(z,4) + ".gt";
						::libmaus2::util::TempFileRemovalContainer::addTempFile(gtpartname);
						gtpartnames[z] = gtpartname;
						#if 0
						::libmaus2::huffman::HuffmanEncoderFileStd GTHEF(gtpartname);
						#endif
						libmaus2::bitio::BitVectorOutput GTHEF(gtpartname);

						#if 0
						::libmaus2::bitio::BitStreamFileDecoder gtfile(mergedgtname, (mergeprocrightend - zblock.getZAbsPos()) );
						#endif
						libmaus2::bitio::BitVectorInput gtfile(mergedgtname, (mergeprocrightend - zblock.getZAbsPos()) );

						typename input_types_type::circular_reverse_wrapper CRWR(fn,zblock.getZAbsPos() % fs);
						uint64_t r = zblock.getZRank();

						uint64_t const zlen = zabsblockpos [ z ] - zabsblockpos [z+1];

						#if defined(_OPENMP)
						uint64_t const tid = omp_get_thread_num();
						#else
						uint64_t const tid = 0;
						#endif

						for ( uint64_t i = 0; i < zlen; ++i )
						{
							GTHEF.writeBit(r > lp0);

							int64_t const sym = CRWR.get();
							bool const gtf = gtfile.readBit();

							r = IHWL.step(sym,r) + ((sym == firstblocklast)?gtf:0);

							if ( G(r) )
								G(r,tid);
						}

						GTHEF.flush();
					}
					if ( logstr )
						(*logstr) << "[V] computed gap array in time " << rtc.getElapsedSeconds() << std::endl;

					G.flush();

					return GapArrayByteComputationResult(pG,Gsize,gtpartnames,zactive,zabsblockpos);
				}

				static GapArrayByteComputationResult computeGapArrayByte(
					libmaus2::util::TempFileNameGenerator & gtmpgen,
					std::string const & fn, // name of text file
					uint64_t const fs, // length of text file in symbols
					libmaus2::suffixsort::bwtb3m::MergeStrategyMergeGapRequest const & msmgr, // merge request
					std::vector<std::string> const & mergedgtname, // previous gt file name
					#if 0
					std::string const & newmergedgtname, // new gt file name
					std::string const & gapoverflowtmpfilename, // gap overflow tmp file
					#endif
					::libmaus2::lf::DArray * const accD, // accumulated symbol freqs for block
					uint64_t const numthreads,
					std::ostream * logstr
				)
				{
					uint64_t const into = msmgr.into;

					std::vector<libmaus2::suffixsort::bwtb3m::MergeStrategyBlock::shared_ptr_type> const & children =
						*(msmgr.pchildren);

					::libmaus2::suffixsort::BwtMergeBlockSortResult const & blockresults = children[into]->sortresult;

					uint64_t const blockstart = blockresults.getBlockStart();
					uint64_t const cblocksize = blockresults.getCBlockSize();
					uint64_t const nextblockstart = (blockstart+cblocksize)%fs;
					uint64_t const mergeprocrightend =
						children.at(children.size()-1)->sortresult.getBlockStart() +
						children.at(children.size()-1)->sortresult.getCBlockSize();
					// use gap object's zblocks vector
					std::vector < ::libmaus2::suffixsort::BwtMergeZBlock > const & zblocks = msmgr.zblocks;

					return computeGapArrayByte(gtmpgen,fn,fs,blockstart,cblocksize,nextblockstart,mergeprocrightend,
						blockresults,mergedgtname/*,newmergedgtname,gapoverflowtmpfilename*/,accD,zblocks,numthreads,logstr);
				}

				struct ZNext
				{
					uint64_t znext;
					uint64_t znextcount;
					libmaus2::parallel::StdMutex lock;

					ZNext(uint64_t const rznextcount) : znext(0), znextcount(rznextcount) {}

					bool getNext(uint64_t & next)
					{
						libmaus2::parallel::StdMutex::scope_lock_type slock(lock);

						if ( znext == znextcount )
							return false;
						else
						{
							next = znext++;
							return true;
						}
					}
				};

				static SparseGapArrayComputationResult computeSparseGapArray(
					libmaus2::util::TempFileNameGenerator & gtmpgen,
					std::string const & fn,
					uint64_t const fs,
					uint64_t const blockstart,
					uint64_t const cblocksize,
					uint64_t const nextblockstart,
					uint64_t const mergeprocrightend,
					::libmaus2::suffixsort::BwtMergeBlockSortResult const & blockresults,
					std::vector<std::string> const & mergedgtname,
					// std::string const & newmergedgtname,
					::libmaus2::lf::DArray * const accD,
					//
					// std::string const & outputgapfilename,
					std::string const & tmpfileprefix,
					uint64_t const maxmem,
					std::vector < ::libmaus2::suffixsort::BwtMergeZBlock > const & zblocks,
					uint64_t const numthreads,
					std::ostream * logstr
				)
				{
					uint64_t const memperthread = (maxmem + numthreads-1)/numthreads;
					uint64_t const wordsperthread = ( memperthread + sizeof(uint64_t) - 1 ) / sizeof(uint64_t);
					// uint64_t const wordsperthread = 600;
					uint64_t const parcheck = 64*1024;

					libmaus2::autoarray::AutoArray< libmaus2::autoarray::AutoArray<uint64_t> > GG(numthreads,false);
					for ( uint64_t i = 0; i < numthreads; ++i )
						GG[i] = libmaus2::autoarray::AutoArray<uint64_t>(wordsperthread,false);

					libmaus2::util::TempFileNameGenerator tmpgen(tmpfileprefix,3);
					libmaus2::gamma::SparseGammaGapMultiFileLevelSet SGGFS(tmpgen,numthreads);

					// set up lf mapping
					::libmaus2::lf::DArray D(static_cast<std::string const &>(blockresults.getFiles().getHist()));
					accD->merge(D);
					#if 0
					bool const hwtdelayed =
						ensureWaveletTreeGenerated(blockresults,logstr);
					#endif
					::libmaus2::wavelet::ImpCompactHuffmanWaveletTree::unique_ptr_type ICHWL(ensureWaveletTreeGenerated(blockresults,logstr));
					::libmaus2::lf::ImpCompactHuffmanWaveletLF IHWL(ICHWL);
					IHWL.D = D.D;
					assert ( cblocksize == IHWL.n );

					// rank of position 0 in this block (for computing new gt array/stream)
					uint64_t const lp0 = blockresults.getBlockP0Rank();

					// last symbol in this block
					int64_t const firstblocklast = input_types_type::linear_wrapper::getSymbolAtPosition(fn,(nextblockstart+fs-1)%fs);

					/**
					 * array of absolute positions
					 **/
					uint64_t const zactive = zblocks.size();
					::libmaus2::autoarray::AutoArray<uint64_t> zabsblockpos(zactive+1,false);
					for ( uint64_t z = 0; z < zactive; ++z )
						zabsblockpos[z] = zblocks[z].getZAbsPos();
					zabsblockpos [ zactive ] = blockstart + cblocksize;
					uint64_t gs = 0;
					for ( uint64_t z = 0; z < zactive; ++z )
						gs += zabsblockpos[z]-zabsblockpos[z+1];

					std::vector < std::string > gtpartnames(zactive);

					::libmaus2::timing::RealTimeClock rtc;
					rtc.start();

					ZNext znext(zactive);

					uint64_t termcnt = 0;
					libmaus2::parallel::StdMutex termcntlock;

					libmaus2::parallel::StdSemaphore qsem; // queue semaphore
					libmaus2::parallel::StdSemaphore tsem; // term semaphore
					libmaus2::parallel::StdSemaphore globsem; // meta semaphore for both above
					libmaus2::parallel::LockedBool termflag(false);
					libmaus2::parallel::LockedBool qterm(false);
					std::atomic<int> parfailed(0);

					SGGFS.registerMergePackSemaphore(&qsem);
					SGGFS.registerMergePackSemaphore(&globsem);
					SGGFS.registerTermSemaphore(&tsem);
					SGGFS.registerTermSemaphore(&globsem);
					SGGFS.setTermSemCnt(numthreads);

					#if defined(_OPENMP)
					#pragma omp parallel num_threads(numthreads)
					#endif
					{
						try
						{
							uint64_t z = 0;

							while ( (!parfailed.load()) && znext.getNext(z) )
							{
								#if defined(_OPENMP)
								uint64_t const tid = omp_get_thread_num();
								#else
								uint64_t const tid = 0;
								#endif

								uint64_t * const Ga = GG[tid].begin();
								uint64_t *       Gc = Ga;
								uint64_t * const Ge = GG[tid].end();

								::libmaus2::timing::RealTimeClock subsubrtc; subsubrtc.start();

								::libmaus2::suffixsort::BwtMergeZBlock const & zblock = zblocks[z];

								std::string const gtpartname = gtmpgen.getFileName() + "_" + ::libmaus2::util::NumberSerialisation::formatNumber(z,4) + ".gt";
								::libmaus2::util::TempFileRemovalContainer::addTempFile(gtpartname);
								gtpartnames[z] = gtpartname;
								libmaus2::bitio::BitVectorOutput GTHEF(gtpartname);

								libmaus2::bitio::BitVectorInput gtfile(mergedgtname, (mergeprocrightend - zblock.getZAbsPos()) );

								typename input_types_type::circular_reverse_wrapper CRWR(fn,zblock.getZAbsPos() % fs);
								uint64_t r = zblock.getZRank();

								uint64_t const zlen = zabsblockpos [ z ] - zabsblockpos [z+1];
								uint64_t const fullblocks = zlen/(Ge-Ga);
								uint64_t const rest = zlen - fullblocks * (Ge-Ga);

								for ( uint64_t b = 0; b < fullblocks; ++b )
								{
									Gc = Ga;

									while ( Gc != Ge )
									{
										uint64_t * const Te = Gc + std::min(parcheck,static_cast<uint64_t>(Ge-Gc));

										for ( ; Gc != Te; ++Gc )
										{
											GTHEF.writeBit(r > lp0);
											int64_t const sym = CRWR.get();
											bool const gtf = gtfile.readBit();
											r = IHWL.step(sym,r) + ((sym == firstblocklast)?gtf:0);
											*Gc = r;
										}

										while ( globsem.trywait() )
										{
											qsem.wait();
											SGGFS.checkMergeSingle();
										}
									}

									std::string const tfn = tmpgen.getFileName();
									libmaus2::gamma::SparseGammaGapBlockEncoder::encodeArray(Ga,Gc,tfn);
									SGGFS.putFile(std::vector<std::string>(1,tfn));

									while ( globsem.trywait() )
									{
										qsem.wait();
										SGGFS.checkMergeSingle();
									}
								}

								if ( rest )
								{
									Gc = Ga;

									while ( Gc != Ga + rest )
									{
										uint64_t * const Te = Gc + std::min(parcheck,static_cast<uint64_t>((Ga+rest)-Gc));

										for ( ; Gc != Te; ++Gc )
										{
											GTHEF.writeBit(r > lp0);
											int64_t const sym = CRWR.get();
											bool const gtf = gtfile.readBit();
											r = IHWL.step(sym,r) + ((sym == firstblocklast)?gtf:0);
											*Gc = r;
										}

										while ( globsem.trywait() )
										{
											qsem.wait();
											SGGFS.checkMergeSingle();
										}
									}

									std::string const tfn = tmpgen.getFileName();
									libmaus2::gamma::SparseGammaGapBlockEncoder::encodeArray(Ga,Gc,tfn);
									SGGFS.putFile(std::vector<std::string>(1,tfn));

									while ( globsem.trywait() )
									{
										qsem.wait();
										SGGFS.checkMergeSingle();
									}
								}

								GTHEF.flush();

								while ( globsem.trywait() )
								{
									qsem.wait();
									SGGFS.checkMergeSingle();
								}
							}

							{
								libmaus2::parallel::StdMutex::scope_lock_type slock(termcntlock);
								if ( ++termcnt == numthreads )
									termflag.set(true);
							}

							bool running = true;
							while ( (!(parfailed.load())) && running )
							{
								if ( termflag.get() && (!qterm.get()) && SGGFS.isMergingQueueEmpty() )
								{
									for ( uint64_t i = 0; i < numthreads; ++i )
									{
										tsem.post();
										globsem.post();
									}

									qterm.set(true);
								}

								// std::cerr << "waiting for glob...";
								bool const globok = globsem.timedWait();
								// std::cerr << "done." << std::endl;

								if ( globok )
								{
									if ( qsem.trywait() )
										SGGFS.checkMergeSingle();
									else
									{
										#if ! defined(NDEBUG)
										bool const tsemok =
										#endif
											tsem.trywait();
										#if ! defined(NDEBUG)
										assert ( tsemok );
										#endif
										running = false;
									}
								}
							}
						}
						catch(std::exception const & ex)
						{
							libmaus2::aio::StreamLock::lock_type::scope_lock_type slock(
								libmaus2::aio::StreamLock::cerrlock
							);
							std::cerr << "[E] computeSparseGapArray caught exception: " << ex.what() << std::endl;
							parfailed.store(1);
						}
					}

					if ( parfailed.load() )
						throw libmaus2::exception::LibMausException("[E] parallel loop failed in computeSparseGapArray");

					//std::vector<std::string> const outputgapfilenames =
					libmaus2::gamma::SparseGammaGapMultiFileLevelSet::MergeDenseResult const MDR =
						SGGFS.mergeToDense(gtmpgen,cblocksize+1,numthreads);

					if ( logstr )
						(*logstr) << "[V] computed gap array in time " << rtc.getElapsedSeconds() << " gs=" << gs << " MDR.s=" << MDR.s << std::endl;

					assert ( gs == MDR.s );

					return SparseGapArrayComputationResult(MDR.V,gtpartnames,zactive,zabsblockpos);
				}

				static SparseGapArrayComputationResult computeSparseGapArray(
					libmaus2::util::TempFileNameGenerator & gtmpgen,
					std::string const & fn, // name of text file
					uint64_t const fs, // length of text file in symbols
					libmaus2::suffixsort::bwtb3m::MergeStrategyMergeGapRequest const & msmgr, // merge request
					uint64_t const ihwtspace,
					std::vector<std::string> const & mergedgtname, // previous gt file name
					// std::string const & newmergedgtname, // new gt file name
					::libmaus2::lf::DArray * const accD, // accumulated symbol freqs for block
					// std::string const & outputgapfilename,
					std::string const & tmpfileprefix,
					uint64_t const maxmem,
					uint64_t const numthreads,
					std::ostream * logstr
				)
				{
					uint64_t const into = msmgr.into;

					std::vector<libmaus2::suffixsort::bwtb3m::MergeStrategyBlock::shared_ptr_type> const & children =
						*(msmgr.pchildren);

					::libmaus2::suffixsort::BwtMergeBlockSortResult const & blockresults = children[into]->sortresult;

					uint64_t const blockstart = blockresults.getBlockStart();
					uint64_t const cblocksize = blockresults.getCBlockSize();
					uint64_t const nextblockstart = (blockstart+cblocksize)%fs;
					uint64_t const mergeprocrightend =
						children.at(children.size()-1)->sortresult.getBlockStart() +
						children.at(children.size()-1)->sortresult.getCBlockSize();
					std::vector < ::libmaus2::suffixsort::BwtMergeZBlock > const & zblocks = msmgr.zblocks;

					return computeSparseGapArray(gtmpgen,fn,fs,blockstart,cblocksize,nextblockstart,mergeprocrightend,
						blockresults,mergedgtname,
						// newmergedgtname,
						accD,
						// outputgapfilename,
						tmpfileprefix,
						maxmem-ihwtspace,
						zblocks,
						numthreads,
						logstr
					);
				}

				static std::vector<std::string> stringVectorAppend(std::vector<std::string> V, std::vector<std::string> const & W)
				{
					for ( uint64_t i = 0; i < W.size(); ++i )
						V.push_back(W[i]);
					return V;
				}

				template<typename gap_array>
				static void splitGapArray(
					gap_array & G,
					uint64_t const Gsize,
					uint64_t const numthreads,
					std::vector < std::pair<uint64_t,uint64_t> > & wpacks,
					std::vector < uint64_t > & P,
					std::ostream * logstr,
					int const verbose
				)
				{
					typedef typename gap_array::sequence_type sequence_type;

					if ( verbose >= 5 && logstr )
					{
						(*logstr) << "[V] computing work packets" << std::endl;
					}

					uint64_t const logG = std::max(libmaus2::math::ilog(Gsize),static_cast<unsigned int>(1));
					uint64_t const logG2 = logG*logG;
					// target number of G samples
					uint64_t const tnumGsamp = std::max(Gsize / logG2,static_cast<uint64_t>(256*numthreads));
					uint64_t const Gsampleblocksize = (Gsize + tnumGsamp - 1) / tnumGsamp;
					// number of G samples
					uint64_t const numGsamp = (Gsize + Gsampleblocksize - 1) / Gsampleblocksize;

					libmaus2::autoarray::AutoArray < uint64_t > Gsamples(numGsamp,false);

					uint64_t const samplesPerPackage = (numGsamp + numthreads - 1)/numthreads;
					uint64_t const numSamplePackages = (numGsamp + samplesPerPackage - 1)/samplesPerPackage;

					#if defined(_OPENMP)
					#pragma omp parallel for num_threads(numthreads)
					#endif
					for ( uint64_t tid = 0; tid < numSamplePackages; ++tid )
					{
						uint64_t const tlow  = tid * samplesPerPackage;
						uint64_t const thigh = std::min(tlow + samplesPerPackage, numGsamp);
						assert ( thigh >= tlow );

						sequence_type gp = G.getOffsetSequence(tlow * Gsampleblocksize);

						for ( uint64_t ti = tlow; ti < thigh; ++ti )
						{
							uint64_t s = 0;
							uint64_t const low = ti * Gsampleblocksize;
							uint64_t const high = std::min(low + Gsampleblocksize, Gsize);

							for ( uint64_t i = low; i < high; ++i )
								s += gp.get();
							s += (high-low);

							if ( high == Gsize && high != low )
								s -= 1;
							Gsamples[ti] = s;
						}
					}

					#if 0
					#if defined(_OPENMP)
					#pragma omp parallel for num_threads(numthreads)
					#endif
					for ( uint64_t t = 0; t < numGsamp; ++t )
					{
						uint64_t const low = t * Gsampleblocksize;
						uint64_t const high = std::min(low + Gsampleblocksize, Gsize);
						assert ( high >= low );
						uint64_t s = 0;
						sequence_type gp = G.getOffsetSequence(low);
						for ( uint64_t i = low; i < high; ++i )
							s += gp.get();
						s += (high-low);
						if ( high == Gsize && high != low )
							s -= 1;
						Gsamples[t] = s;
					}
					#endif

					#if 0
					std::vector<uint64_t> G_A(Gsamples.begin(),Gsamples.end());
					std::vector<uint64_t> G_B(Gsamples.begin(),Gsamples.end());

					libmaus2::util::PrefixSums::prefixSums(G_A.begin(),G_A.end());
					libmaus2::util::PrefixSums::parallelPrefixSums(G_A.begin(),G_A.end(),numthreads);
					#endif

					uint64_t const Gsum = libmaus2::util::PrefixSums::parallelPrefixSums(Gsamples.begin(),Gsamples.end(),numthreads);

					if ( verbose >= 5 && logstr )
					{
						(*logstr) << "[V] G size " << Gsize << " number of G samples " << numGsamp << std::endl;
					}

					uint64_t const Gsumperthread = (Gsum + numthreads-1)/numthreads;
					wpacks = std::vector < std::pair<uint64_t,uint64_t> >(numthreads);
					#if defined(_OPENMP)
					#pragma omp parallel for num_threads(numthreads)
					#endif
					for ( uint64_t i = 0; i < numthreads; ++i )
					{
						uint64_t const target = i * Gsumperthread;
						uint64_t const * p = ::std::lower_bound(Gsamples.begin(),Gsamples.end(),target);

						if ( p == Gsamples.end() )
							--p;
						while ( *p > target )
							--p;

						assert ( *p <= target );

						uint64_t iv = (p - Gsamples.begin()) * Gsampleblocksize;
						uint64_t s = *p;

						sequence_type gp = G.getOffsetSequence(iv);
						while ( s < target && iv < Gsize )
						{
							s += (gp.get())+1;
							iv++;
						}
						if ( iv == Gsize )
							s -= 1;

						wpacks[i].first = iv;
						if ( i )
							wpacks[i-1].second = iv;
						// std::cerr << "i=" << i << " iv=" << iv << " Gsize=" << Gsize << std::endl;
					}
					wpacks.back().second = Gsize;

					// remove empty packages
					{
						uint64_t o = 0;
						for ( uint64_t i = 0; i < wpacks.size(); ++i )
							if ( wpacks[i].first != wpacks[i].second )
								wpacks[o++] = wpacks[i];
						wpacks.resize(o);
					}

					P.resize(wpacks.size()+1);
					#if defined(_OPENMP)
					#pragma omp parallel for num_threads(numthreads)
					#endif
					for ( uint64_t i = 0; i < wpacks.size(); ++i )
					{
						uint64_t const low = wpacks[i].first;
						uint64_t const high = wpacks[i].second;

						sequence_type gp = G.getOffsetSequence(low);
						uint64_t s = 0;
						for ( uint64_t i = low; i < high; ++i )
							s += gp.get();

						P[i] = s;

					}
					libmaus2::util::PrefixSums::prefixSums(P.begin(),P.end());
				}

				static void setupSampledISA(
					libmaus2::util::TempFileNameGenerator & gtmpgen,
					::libmaus2::suffixsort::BwtMergeBlockSortResult & result,
					uint64_t const numisa
				)
				{
					std::vector < std::string > Vsampledisa(numisa);
					for ( uint64_t i = 0; i < Vsampledisa.size(); ++i )
					{
						Vsampledisa[i] = (
							gtmpgen.getFileName()
							// result.getFiles().getBWT()
							+ "_"
							+ ::libmaus2::util::NumberSerialisation::formatNumber(Vsampledisa.size(),6)
							+ ".sampledisa"
						);
						::libmaus2::util::TempFileRemovalContainer::addTempFile(Vsampledisa[i]);

					}
					result.setSampledISA(Vsampledisa);
				}

				static bool compareFiles(std::vector<std::string> const & A, std::vector<std::string> const & B)
				{
					libmaus2::aio::ConcatInputStream CA(A);
					libmaus2::aio::ConcatInputStream CB(B);

					libmaus2::autoarray::AutoArray<char> BA(16*1024);
					libmaus2::autoarray::AutoArray<char> BB(BA.size());

					while ( CA && CB )
					{
						CA.read(BA.begin(),BA.size());
						CB.read(BB.begin(),BB.size());
						assert ( CA.gcount() == CB.gcount() );

						uint64_t const ca = CA.gcount();

						if (
							! std::equal(BA.begin(),BA.begin()+ca,BB.begin())
						)
							return false;
					}

					bool const oka = static_cast<bool>(CA);
					bool const okb = static_cast<bool>(CB);

					return oka == okb;
				}

				static void mergeBlocks(
					libmaus2::util::TempFileNameGenerator & gtmpgen,
					libmaus2::suffixsort::bwtb3m::MergeStrategyMergeInternalBlock & mergereq,
					std::string const fn,
					uint64_t const fs,
					// std::string const tmpfilenamebase,
					uint64_t const rlencoderblocksize,
					uint64_t const lfblockmult,
					uint64_t const numthreads,
					::std::map<int64_t,uint64_t> const & /* chist */,
					uint64_t const bwtterm,
					std::string const & huftreefilename,
					std::ostream * logstr,
					int const verbose
				)
				{
					if ( logstr )
						(*logstr) << "[V] Merging BWT blocks MergeStrategyMergeInternalBlock." << std::endl;

					assert ( mergereq.children.size() > 1 );
					assert ( mergereq.children.size() == mergereq.gaprequests.size()+1 );

					// std::cerr << "[V] Merging BWT blocks with gapmembound=" << gapmembound << std::endl;

					/*
					 * remove unused file
					 */
					if ( verbose >= 5 && logstr )
					{
						(*logstr) << "[V] removing unused file " << mergereq.children[mergereq.children.size()-1]->sortresult.getFiles().getHWT() << std::endl;
					}
					libmaus2::aio::FileRemoval::removeFile ( mergereq.children[mergereq.children.size()-1]->sortresult.getFiles().getHWT().c_str() );

					if ( verbose >= 5 && logstr )
					{
						(*logstr) << "[V] setting meta info on BwtMergeBlockSortResult" << std::endl;
					}

					// get result object
					::libmaus2::suffixsort::BwtMergeBlockSortResult & result = mergereq.sortresult;
					// fill result structure
					result.setBlockStart( mergereq.children[0]->sortresult.getBlockStart() );
					result.setCBlockSize( 0 );
					for ( uint64_t i = 0; i < mergereq.children.size(); ++i )
						result.setCBlockSize( result.getCBlockSize() + mergereq.children[i]->sortresult.getCBlockSize() );
					// set up temp file names
					// of output bwt,
					// sampled inverse suffix array filename,
					// gt bit array,
					// huffman shaped wavelet tree and
					// histogram
					// result.setTempPrefixSingleAndRegisterAsTemp(gtmpgen);

					if ( verbose >= 5 && logstr )
					{
						(*logstr) << "[V] handling " << mergereq.children.size() << " child nodes" << std::endl;
					}

					// if we merge only two blocks together, then we do not need to write the gap array to disk
					if ( mergereq.children.size() == 2 )
					{
						// std::cerr << "** WHITEBOX INTERNAL 1 **" << std::endl;

						if ( verbose >= 5 && logstr )
						{
							(*logstr) << "[V] loading block character histogram" << std::endl;
						}

						std::string const & sblockhist = mergereq.children[1]->sortresult.getFiles().getHist();
						// load char histogram for last/second block (for merging)
						::libmaus2::lf::DArray::unique_ptr_type accD(new ::libmaus2::lf::DArray(
							sblockhist
							)
						);
						// first block
						::libmaus2::suffixsort::BwtMergeBlockSortResult const & blockresults =
							mergereq.children[0]->sortresult;

						// start of first block
						uint64_t const blockstart = blockresults.getBlockStart();
						// size of first block
						uint64_t const cblocksize = blockresults.getCBlockSize();

						if ( verbose >= 5 && logstr )
						{
							(*logstr) << "[V] calling computeGapArray" << std::endl;
						}

						// compute gap array
						GapArrayComputationResult const GACR = computeGapArray(
							gtmpgen,
							fn,fs,*(mergereq.gaprequests[0]),
							mergereq.children[1]->sortresult.getFiles().getGT(), // previous gt files
							// tmpfilenamebase + "_gparts", // new gt files
							accD.get(),
							numthreads,
							logstr,
							verbose
						);

						uint64_t const Gsize = (cblocksize+1);

						if ( verbose >= 5 && logstr )
						{
							(*logstr) << "[V] call to computeGapArray finished" << std::endl;
						}

						#if 0
						// concatenate gt vectors
						concatenateGT(
							GACR.gtpartnames,
							blockresults.getFiles().getGT(),
							result.getFiles().getGT()
						);
						#endif

						if ( verbose >= 5 && logstr )
						{
							(*logstr) << "[V] renaming gt files" << std::endl;
						}

						// move gt files for left block to output by renaming them
						std::vector<std::string> oldgtnames;
						for ( uint64_t i = 0; i < blockresults.getFiles().getGT().size(); ++i )
						{
							std::ostringstream ostr;
							ostr << gtmpgen.getFileName() << "_renamed_" << std::setw(6) << std::setfill('0') << i << std::setw(0) << ".gt";
							std::string const renamed = ostr.str();
							oldgtnames.push_back(ostr.str());
							::libmaus2::util::TempFileRemovalContainer::addTempFile(renamed);
							libmaus2::aio::OutputStreamFactoryContainer::rename(blockresults.getFiles().getGT()[i].c_str(), renamed.c_str());
						}

						// concatenate new gt files and old ones to obtain output gt files
						result.setGT(stringVectorAppend(GACR.gtpartnames,oldgtnames));

						::libmaus2::timing::RealTimeClock rtc; rtc.start();

						if ( logstr )
							(*logstr) << "[V] computing gap array split...";

						std::vector < std::pair<uint64_t,uint64_t> > wpacks;
						std::vector < uint64_t > P;
						splitGapArray(GACR.G,Gsize,numthreads,wpacks,P,logstr,verbose);

						if ( logstr )
							(*logstr) << "done, time " << rtc.getElapsedSeconds() << std::endl;

						if ( verbose >= 5 && logstr )
						{
							(*logstr) << "[V] calling mergeIsa" << std::endl;
						}

						// merge sampled inverse suffix arrays, returns rank of position 0 (relative to block start)
						std::pair < uint64_t, std::vector<std::string> > const PPP = BwtMergeIsaParallelBase<GapArrayWrapper>::mergeIsaParallel(
							gtmpgen,wpacks,P,
							mergereq.children[1]->sortresult.getFiles().getSampledISAVector(),
							blockresults.getFiles().getSampledISAVector(),
							blockstart,
							GACR.G,
							GACR.Gsize,
							numthreads,
							logstr
						);

						result.setBlockP0Rank(PPP.first);
						result.setSampledISA(PPP.second);

						#define LIBMAUS2_SUFFIXSORT_BWTB3M_PARALLEL_ISA_MERGE_DEBUGa

						#if defined(LIBMAUS2_SUFFIXSORT_BWTB3M_PARALLEL_ISA_MERGE_DEBUG)
						{
							libmaus2::util::GetCObject<uint32_t const *> mergeGO(GACR.G.begin());
							std::string const isadebugtmp = gtmpgen.getFileName(true);
							uint64_t const serialblockp0rank = BwtMergeIsaBase< libmaus2::util::GetCObject<uint32_t const *> >::mergeIsa(
								mergereq.children[1]->sortresult.getFiles().getSampledISAVector(), // old sampled isa
								blockresults.getFiles().getSampledISAVector(), // new sampled isa
								isadebugtmp,blockstart,
								mergeGO/*GACR.G.begin()*/,
								cblocksize+1 /* Gsize */,logstr
							);

							assert ( PPP.first == serialblockp0rank );
							assert ( compareFiles(PPP.second,std::vector<std::string>(1,isadebugtmp)) );

							libmaus2::aio::FileRemoval::removeFile(isadebugtmp);
						}
						#endif

						if ( verbose >= 5 && logstr )
						{
							(*logstr) << "[V] call to mergeIsa finished" << std::endl;
						}

						if ( logstr )
							(*logstr) << "[V] merging BWTs...";

						rtc.start();

						std::vector < std::string > encfilenames(wpacks.size());
						for ( uint64_t i = 0; i < wpacks.size(); ++i )
						{
							encfilenames[i] = (
								gtmpgen.getFileName()
								// result.getFiles().getBWT()
								+ "_"
								+ ::libmaus2::util::NumberSerialisation::formatNumber(encfilenames.size(),6)
								+ ".bwt"
							);
							::libmaus2::util::TempFileRemovalContainer::addTempFile(encfilenames[i]);
						}

						encfilenames.resize(wpacks.size());

						assert ( wpacks.size() <= numthreads );
						// std::cerr << "done,time=" << wprtc.getElapsedSeconds() << ")";

						if ( verbose >= 5 && logstr )
						{
							(*logstr) << "[V] generated " << wpacks.size() << " work packages" << std::endl;
						}

						// std::cerr << "(setting up IDDs...";
						::libmaus2::timing::RealTimeClock wprtc; wprtc.start();
						wprtc.start();

						unsigned int const albits = rl_decoder::haveAlphabetBits() ? rl_decoder::getAlBits(mergereq.children[0]->sortresult.getFiles().getBWT()) : 0;

						if ( verbose >= 5 && logstr )
						{
							(*logstr) << "[V] setting up IDD" << std::endl;
						}

						::libmaus2::huffman::IndexDecoderDataArray IDD0(
							mergereq.children[0]->sortresult.getFiles().getBWT(),numthreads);
						::libmaus2::huffman::IndexDecoderDataArray IDD1(
							mergereq.children[1]->sortresult.getFiles().getBWT(),numthreads);

						if ( verbose >= 5 && logstr )
						{
							(*logstr) << "[V] setting up IDD done" << std::endl;
						}

						if ( verbose >= 5 && logstr )
						{
							(*logstr) << "[V] setting up IECV" << std::endl;
						}

						::libmaus2::huffman::IndexEntryContainerVector::unique_ptr_type IECV0 = ::libmaus2::huffman::IndexLoader::loadAccIndex(
							mergereq.children[0]->sortresult.getFiles().getBWT()
						);
						::libmaus2::huffman::IndexEntryContainerVector::unique_ptr_type IECV1 = ::libmaus2::huffman::IndexLoader::loadAccIndex(
							mergereq.children[1]->sortresult.getFiles().getBWT()
						);

						if ( verbose >= 5 && logstr )
						{
							(*logstr) << "[V] setting up IECV done" << std::endl;
						}

						if ( verbose >= 5 && logstr )
						{
							(*logstr) << "[V] performing merge" << std::endl;
						}

						#if defined(_OPENMP)
						#pragma omp parallel for schedule(dynamic,1) num_threads(numthreads)
						#endif
						for ( int64_t b = 0; b < static_cast<int64_t>(wpacks.size()); ++b )
						{
							uint64_t const ilow = wpacks[b].first;
							uint64_t const ihigh = wpacks[b].second;

							if ( ilow != ihigh )
							{
								bool const islast = (ihigh == Gsize);
								std::string const encfilename = encfilenames[b];

								if ( verbose >= 5 && logstr )
								{
									libmaus2::aio::StreamLock::lock_type::scope_lock_type slock(libmaus2::aio::StreamLock::cerrlock);
									(*logstr) << "[V] setting up decoders for left and right block for merge package " << b << std::endl;
								}

								rl_decoder leftrlin(IDD0,IECV0.get(),ilow);
								rl_decoder rightrlin(IDD1,IECV1.get(),P[b]);

								if ( verbose >= 5 && logstr )
								{
									libmaus2::aio::StreamLock::lock_type::scope_lock_type slock(libmaus2::aio::StreamLock::cerrlock);
									(*logstr) << "[V] setting up decoders for left and right block for merge package " << b << " done." << std::endl;
								}

								uint64_t const outsuf = (ihigh-ilow)-(islast?1:0) + (P[b+1]-P[b]);

								if ( verbose >= 5 && logstr )
								{
									libmaus2::aio::StreamLock::lock_type::scope_lock_type slock(libmaus2::aio::StreamLock::cerrlock);
									(*logstr) << "[V] setting up encoder for merge package " << b << std::endl;
								}

								rl_encoder bwtenc(encfilename,albits,outsuf,rlencoderblocksize);

								if ( verbose >= 5 && logstr )
								{
									libmaus2::aio::StreamLock::lock_type::scope_lock_type slock(libmaus2::aio::StreamLock::cerrlock);
									(*logstr) << "[V] setting up encoder for merge package " << b << " done" << std::endl;
								}

								if ( verbose >= 5 && logstr )
								{
									libmaus2::aio::StreamLock::lock_type::scope_lock_type slock(libmaus2::aio::StreamLock::cerrlock);
									(*logstr) << "[V] entering merge loop for merge package " << b << std::endl;
								}

								if ( islast )
								{
									for ( uint64_t j = ilow; j < ihigh-1; ++j )
									{
										for ( uint64_t i = 0; i < GACR.G[j]; ++i )
											bwtenc.encode(rightrlin.decode());

										bwtenc.encode(leftrlin.decode());
									}

									for ( uint64_t i = 0; i < GACR.G[cblocksize]; ++i )
										bwtenc.encode(rightrlin.decode());
								}
								else
								{
									for ( uint64_t j = ilow; j < ihigh; ++j )
									{
										for ( uint64_t i = 0; i < GACR.G[j]; ++i )
											bwtenc.encode(rightrlin.decode());

										bwtenc.encode(leftrlin.decode());
									}
								}

								if ( verbose >= 5 && logstr )
								{
									libmaus2::aio::StreamLock::lock_type::scope_lock_type slock(libmaus2::aio::StreamLock::cerrlock);
									(*logstr) << "[V] left merge loop for merge package " << b << std::endl;
								}

								bwtenc.flush();

								if ( verbose >= 5 && logstr )
								{
									libmaus2::aio::StreamLock::lock_type::scope_lock_type slock(libmaus2::aio::StreamLock::cerrlock);
									(*logstr) << "[V] flushed encoder for merge package " << b << std::endl;
								}
							}
						}
						if ( logstr )
							(*logstr) << "done, time " << rtc.getElapsedSeconds() << std::endl;

						#if 0
						std::cerr << "[V] concatenating bwt parts...";
						rtc.start();
						rl_encoder::concatenate(encfilenames,result.getFiles().getBWT());
						std::cerr << "done, time " << rtc.getElapsedSeconds() << std::endl;
						#endif

						result.setBWT(encfilenames);

						#if 0
						std::cerr << "[V] removing tmp files...";
						rtc.start();
						for ( uint64_t i = 0; i < encfilenames.size(); ++i )
							libmaus2::aio::FileRemoval::removeFile ( encfilenames[i].c_str() );
						std::cerr << "done, time " << rtc.getElapsedSeconds() << std::endl;
						#endif

						// save histogram
						// std::cerr << "[V] saving histogram...";

						if ( verbose >= 5 && logstr )
						{
							(*logstr) << "[V] serialising accD" << std::endl;
						}

						rtc.start();
						accD->serialise(static_cast<std::string const & >(result.getFiles().getHist()));
						// std::cerr << "done, time " << rtc.getElapsedSeconds() << std::endl;

						if ( verbose >= 5 && logstr )
						{
							(*logstr) << "[V] serialising accD done" << std::endl;
						}
					}
					else
					{
						// std::cerr << "** WHITEBOX INTERNAL 2 **" << std::endl;

						std::vector < std::string > gapfilenames;
						std::vector < std::vector<std::string> > bwtfilenames;
						for ( uint64_t bb = 0; bb < mergereq.children.size(); ++bb )
						{
							// gap file name
							if ( bb+1 < mergereq.children.size() )
							{
								std::string const newgapname = gtmpgen.getFileName() + "_merging_" + ::libmaus2::util::NumberSerialisation::formatNumber(bb,4) + ".gap";
								::libmaus2::util::TempFileRemovalContainer::addTempFile(newgapname);
								gapfilenames.push_back(newgapname);
							}

							// create new names for the input bwt names
							std::vector<std::string> newbwtnames;
							for ( uint64_t i = 0; i < mergereq.children[bb]->sortresult.getFiles().getBWT().size(); ++i )
							{
								std::string const newbwtname = gtmpgen.getFileName() + "_merging_"
									+ ::libmaus2::util::NumberSerialisation::formatNumber(bb,4)
									+ "_"
									+ ::libmaus2::util::NumberSerialisation::formatNumber(i,4)
									+ ".bwt";
								::libmaus2::util::TempFileRemovalContainer::addTempFile(newbwtname);
								newbwtnames.push_back(newbwtname);
							}
							bwtfilenames.push_back(newbwtnames);
						}

						// rename last bwt file set
						for ( uint64_t i = 0; i < mergereq.children.back()->sortresult.getFiles().getBWT().size(); ++i )
						{
							libmaus2::aio::OutputStreamFactoryContainer::rename (
								mergereq.children.back()->sortresult.getFiles().getBWT()[i].c_str(),
								bwtfilenames.back()[i].c_str()
							);
						}

						std::vector<std::string> mergedgtname  = mergereq.children.back()->sortresult.getFiles().getGT();
						std::vector<std::string> mergedisaname = mergereq.children.back()->sortresult.getFiles().getSampledISAVector();

						// load char histogram for last block
						std::string const & lblockhist = mergereq.children.back()->sortresult.getFiles().getHist();
						::libmaus2::lf::DArray::unique_ptr_type accD(new ::libmaus2::lf::DArray(lblockhist));

						/**
						 * iteratively merge blocks together
						 **/
						for ( uint64_t bb = 0; bb+1 < mergereq.children.size(); ++bb )
						{
							// block we merge into
							uint64_t const bx = mergereq.children.size()-bb-2;
							if ( logstr )
								(*logstr) << "[V] merging blocks " << bx+1 << " to end into " << bx << std::endl;
							::libmaus2::suffixsort::BwtMergeBlockSortResult const & blockresults =
								mergereq.children[bx]->sortresult;

							// output files for this iteration
							std::string const newmergedgtname = gtmpgen.getFileName() + "_merged_" + ::libmaus2::util::NumberSerialisation::formatNumber(bx,4) + ".gt";
							::libmaus2::util::TempFileRemovalContainer::addTempFile(newmergedgtname);
							std::string const newmergedisaname = gtmpgen.getFileName() + "_merged_" + ::libmaus2::util::NumberSerialisation::formatNumber(bx,4) + ".sampledisa";
							::libmaus2::util::TempFileRemovalContainer::addTempFile(newmergedisaname);
							// gap file
							std::string const gapfile = gapfilenames[bx];

							// start of this block
							uint64_t const blockstart = blockresults.getBlockStart();
							// size of this block
							uint64_t const cblocksize = blockresults.getCBlockSize();

							// compute gap array
							GapArrayComputationResult const GACR = computeGapArray(
								gtmpgen,fn,fs,*(mergereq.gaprequests[bx]),
								mergedgtname,
								accD.get(),
								numthreads,
								logstr,
								verbose
							);

							// save the gap file
							GACR.G.saveGapFile(GACR.Gsize,gapfile,logstr);

							if ( logstr )
								(*logstr) << "[V] computing gap array split...";

							libmaus2::timing::RealTimeClock splitrtc; splitrtc.start();
							std::vector < std::pair<uint64_t,uint64_t> > wpacks;
							std::vector < uint64_t > P;
							splitGapArray(GACR.G,(cblocksize+1) /* Gsize */,numthreads,wpacks,P,logstr,verbose);

							if ( logstr )
								(*logstr) << "done, time " << splitrtc.getElapsedSeconds() << std::endl;

							std::pair < uint64_t, std::vector<std::string> > const PPP = BwtMergeIsaParallelBase<GapArrayWrapper>::mergeIsaParallel(
								gtmpgen,wpacks,P,
								mergedisaname,
								blockresults.getFiles().getSampledISAVector(),
								blockstart,
								GACR.G,
								GACR.Gsize,
								numthreads,
								logstr
							);

							result.setBlockP0Rank(PPP.first);

							#if defined(LIBMAUS2_SUFFIXSORT_BWTB3M_PARALLEL_ISA_MERGE_DEBUG)
							{
								// merge sampled inverse suffix arrays, returns rank of position 0 (relative to block start)
								std::string const isadebugtmp = gtmpgen.getFileName(true);
								libmaus2::util::GetCObject<uint32_t const *> mergeGO(GACR.G.begin());
								uint64_t const serialblockp0rank = BwtMergeIsaBase< libmaus2::util::GetCObject<uint32_t const *> >::mergeIsa(mergedisaname,blockresults.getFiles().getSampledISAVector(),isadebugtmp,blockstart,mergeGO /*GACR.G.begin()*/,cblocksize+1 /*Gsize*/,logstr);

								assert ( PPP.first == serialblockp0rank );
								assert ( compareFiles(PPP.second,std::vector<std::string>(1,isadebugtmp)) );

								libmaus2::aio::FileRemoval::removeFile(isadebugtmp);
							}
							#endif

							#if 0
							// concatenate gt vectors
							concatenateGT(GACR.gtpartnames,blockresults.getFiles().getGT(),newmergedgtname);
							#endif

							// rename files
							std::vector<std::string> oldgtnames;
							for ( uint64_t i = 0; i < blockresults.getFiles().getGT().size(); ++i )
							{
								std::ostringstream ostr;
								ostr << gtmpgen.getFileName()
									<< "_renamed_"
									<< std::setw(6) << std::setfill('0') << bx << std::setw(0)
									<< "_"
									<< std::setw(6) << std::setfill('0') << i << std::setw(0)
									<< ".gt";
								std::string const renamed = ostr.str();
								oldgtnames.push_back(ostr.str());
								::libmaus2::util::TempFileRemovalContainer::addTempFile(renamed);
								libmaus2::aio::OutputStreamFactoryContainer::rename(blockresults.getFiles().getGT()[i].c_str(), renamed.c_str());
							}

							// result.setGT(stringVectorAppend(GACR.gtpartnames,blockresults.getFiles().getGT()));

							/*
							 * remove files we no longer need
							 */
							// files local to this block
							for ( uint64_t i = 0; i < blockresults.getFiles().getBWT().size(); ++i )
								libmaus2::aio::OutputStreamFactoryContainer::rename ( blockresults.getFiles().getBWT()[i].c_str(), bwtfilenames[bx][i].c_str() );
							blockresults.removeFilesButBwt();
							// previous stage gt bit vector
							for ( uint64_t i = 0; i < mergedgtname.size(); ++i )
								libmaus2::aio::FileRemoval::removeFile ( mergedgtname[i].c_str() );

							// update current file names
							mergedgtname = stringVectorAppend(GACR.gtpartnames,oldgtnames);
							mergedisaname = PPP.second; // std::vector<std::string>(1,newmergedisaname);
						}

						// renamed sampled inverse suffix array
						result.setSampledISA(mergedisaname);
						// libmaus2::aio::OutputStreamFactoryContainer::rename ( mergedisaname.c_str(), result.getFiles().getSampledISA().c_str() );
						// rename gt bit array filename
						// libmaus2::aio::OutputStreamFactoryContainer::rename ( mergedgtname.c_str(), result.getFiles().getGT().c_str() );
						result.setGT(mergedgtname);
						// save histogram
						accD->serialise(static_cast<std::string const & >(result.getFiles().getHist()));

						if ( logstr )
							(*logstr) << "[V] merging parts...";
						::libmaus2::timing::RealTimeClock mprtc;
						mprtc.start();
						result.setBWT(BwtMergeParallelGapFragMerge::parallelGapFragMerge(
							gtmpgen,
							bwtfilenames,
							stringVectorPack(gapfilenames),
							// result.getFiles().getBWT(),
							//gtmpgen.getFileName()+"_gpart",
							numthreads,
							lfblockmult,rlencoderblocksize,logstr,verbose));
						if ( logstr )
							(*logstr) << "done, time " << mprtc.getElapsedSeconds() << std::endl;

						for ( uint64_t i = 0; i < gapfilenames.size(); ++i )
							libmaus2::aio::FileRemoval::removeFile ( gapfilenames[i].c_str() );
						for ( uint64_t i = 0; i < bwtfilenames.size(); ++i )
							for ( uint64_t j = 0; j < bwtfilenames[i].size(); ++j )
								libmaus2::aio::FileRemoval::removeFile ( bwtfilenames[i][j].c_str() );
					}

					#if 0
					if ( logstr )
						(*logstr) << "[V] computing term symbol hwt...";
					::libmaus2::timing::RealTimeClock mprtc;
					mprtc.start();
					if ( input_types_type::utf8Wavelet() )
						libmaus2::wavelet::RlToHwtBase<true,rl_decoder>::rlToHwtTerm(result.getFiles().getBWT(),result.getFiles().getHWT(),tmpfilenamebase + "_wt",chist,bwtterm,result.getBlockP0Rank());
					else
						libmaus2::wavelet::RlToHwtBase<false,rl_decoder>::rlToHwtTerm(result.getFiles().getBWT(),result.getFiles().getHWT(),tmpfilenamebase + "_wt",chist,bwtterm,result.getBlockP0Rank());
					if ( logstr )
						(*logstr) << "done, time " << mprtc.getElapsedSeconds() << std::endl;
					#endif

					libmaus2::util::TempFileRemovalContainer::addTempFile(result.getFiles().getHWTReq());
					{
					libmaus2::aio::OutputStreamInstance hwtreqCOS(result.getFiles().getHWTReq());
					libmaus2::wavelet::RlToHwtTermRequest::serialise(
						hwtreqCOS,
						result.getFiles().getBWT(),
						result.getFiles().getHWT(),
						gtmpgen.getFileName() + "_wt", // XXX replace
						huftreefilename,
						bwtterm,
						result.getBlockP0Rank(),
						input_types_type::utf8Wavelet(),
						numthreads
					);
					hwtreqCOS.flush();
					// hwtreqCOS.close();
					}

					// remove obsolete files
					for ( uint64_t b = 0; b < mergereq.children.size(); ++b )
						mergereq.children[b]->sortresult.removeFiles();

					mergereq.releaseChildren();
				}

				static void mergeBlocks(
					libmaus2::util::TempFileNameGenerator & gtmpgen,
					libmaus2::suffixsort::bwtb3m::MergeStrategyMergeInternalSmallBlock & mergereq,
					std::string const fn,
					uint64_t const fs,
					// std::string const tmpfilenamebase,
					uint64_t const rlencoderblocksize,
					uint64_t const lfblockmult,
					uint64_t const numthreads,
					::std::map<int64_t,uint64_t> const & /* chist */,
					uint64_t const bwtterm,
					std::string const & huftreefilename,
					std::ostream * logstr,
					int const verbose
				)
				{
					assert ( mergereq.children.size() > 1 );
					assert ( mergereq.children.size() == mergereq.gaprequests.size()+1 );

					if ( logstr )
						(*logstr) << "[V] Merging BWT blocks MergeStrategyMergeInternalSmallBlock." << std::endl;

					/*
					 * remove unused file
					 */
					libmaus2::aio::FileRemoval::removeFile ( mergereq.children[mergereq.children.size()-1]->sortresult.getFiles().getHWT().c_str() );

					// get result object
					::libmaus2::suffixsort::BwtMergeBlockSortResult & result = mergereq.sortresult;
					// fill result structure
					result.setBlockStart( mergereq.children[0]->sortresult.getBlockStart() );
					result.setCBlockSize( 0 );
					for ( uint64_t i = 0; i < mergereq.children.size(); ++i )
						result.setCBlockSize( result.getCBlockSize() + mergereq.children[i]->sortresult.getCBlockSize() );
					// set up temp file names
					// of output bwt,
					// sampled inverse suffix array filename,
					// gt bit array,
					// huffman shaped wavelet tree and
					// histogram
					// result.setTempPrefixSingleAndRegisterAsTemp(gtmpgen);

					// if we merge only two blocks together, then we do not need to write the gap array to disk
					if ( mergereq.children.size() == 2 )
					{
						// std::cerr << "** WHITEBOX INTERNAL SMALL 1 **" << std::endl;

						std::string const & sblockhist = mergereq.children[1]->sortresult.getFiles().getHist();
						// load char histogram for last/second block (for merging)
						::libmaus2::lf::DArray::unique_ptr_type accD(new ::libmaus2::lf::DArray(
							sblockhist
							)
						);
						// first block
						::libmaus2::suffixsort::BwtMergeBlockSortResult const & blockresults =
							mergereq.children[0]->sortresult;

						// start of first block
						uint64_t const blockstart = blockresults.getBlockStart();
						// size of first block
						uint64_t const cblocksize = blockresults.getCBlockSize();

						// compute gap array
						GapArrayByteComputationResult const GACR = computeGapArrayByte(
							gtmpgen,
							fn,fs,*(mergereq.gaprequests[0]),
							mergereq.children[1]->sortresult.getFiles().getGT(), // previous gt files
							#if 0
							tmpfilenamebase + "_gparts", // new gt files
							tmpfilenamebase + "_gapoverflow",
							#endif
							accD.get(),
							numthreads,
							logstr
						);

						#if 0
						// concatenate gt vectors
						concatenateGT(
							GACR.gtpartnames,
							blockresults.getFiles().getGT(),
							result.getFiles().getGT()
						);
						#endif

						std::vector<std::string> oldgtnames;
						for ( uint64_t i = 0; i < blockresults.getFiles().getGT().size(); ++i )
						{
							std::ostringstream ostr;
							ostr << gtmpgen.getFileName() << "_renamed_" << std::setw(6) << std::setfill('0') << i << std::setw(0) << ".gt";
							std::string const renamed = ostr.str();
							oldgtnames.push_back(ostr.str());
							::libmaus2::util::TempFileRemovalContainer::addTempFile(renamed);
							libmaus2::aio::OutputStreamFactoryContainer::rename(blockresults.getFiles().getGT()[i].c_str(), renamed.c_str());
						}

						result.setGT(stringVectorAppend(GACR.gtpartnames,oldgtnames));

						::libmaus2::timing::RealTimeClock rtc; rtc.start();

						if ( logstr )
							(*logstr) << "[V] splitting gap array...";

						std::vector < std::pair<uint64_t,uint64_t> > wpacks;
						std::vector < uint64_t > P;
						splitGapArray(
							*(GACR.G),
							cblocksize+1,
							numthreads,
							wpacks,
							P,
							logstr,
							verbose
						);

						if ( logstr )
							(*logstr) << "done, time " << rtc.getElapsedSeconds() << std::endl;

						// merge sampled inverse suffix arrays, returns rank of position 0 (relative to block start)
						std::pair < uint64_t, std::vector<std::string> > const PPP = BwtMergeIsaParallelBase<GapArrayByte>::mergeIsaParallel(
							gtmpgen,wpacks,P,
							mergereq.children[1]->sortresult.getFiles().getSampledISAVector(),
							blockresults.getFiles().getSampledISAVector(),
							blockstart,
							*(GACR.G),
							GACR.Gsize,
							numthreads,
							logstr
						);

						result.setBlockP0Rank(PPP.first);
						result.setSampledISA(PPP.second);

						#if defined(LIBMAUS2_SUFFIXSORT_BWTB3M_PARALLEL_ISA_MERGE_DEBUG)
						{
							std::string const isadebugtmp = gtmpgen.getFileName(true);

							// merge sampled inverse suffix arrays, returns rank of position 0 (relative to block start)
							libmaus2::suffixsort::GapArrayByteDecoder::unique_ptr_type pgap0dec(GACR.G->getDecoder());
							libmaus2::suffixsort::GapArrayByteDecoderBuffer::unique_ptr_type pgap0decbuf(new libmaus2::suffixsort::GapArrayByteDecoderBuffer(*pgap0dec,8192));
							// libmaus2::suffixsort::GapArrayByteDecoderBuffer::iterator gap0decbufit = pgap0decbuf->begin();
							uint64_t const serialblockp0rank = BwtMergeIsaBase<libmaus2::suffixsort::GapArrayByteDecoderBuffer>::mergeIsa(
								mergereq.children[1]->sortresult.getFiles().getSampledISAVector(), // old sampled isa
								blockresults.getFiles().getSampledISAVector(), // new sampled isa
								isadebugtmp,blockstart,*pgap0decbuf/*gap0decbufit*//*GACR.G.begin()*/,cblocksize+1 /* GACR.G.size() */,logstr
							);
							pgap0decbuf.reset();
							pgap0dec.reset();

							assert ( PPP.first == serialblockp0rank );
							assert ( compareFiles(PPP.second,std::vector<std::string>(1,isadebugtmp)) );

							libmaus2::aio::FileRemoval::removeFile(isadebugtmp);
						}
						#endif

						rtc.start();
						if ( logstr )
							(*logstr) << "[V] merging BWTs...";

						#if 0
						//
						uint64_t const totalsuf = result.getCBlockSize();
						// number of packets
						uint64_t const numpack = numthreads;
						// suffixes per thread
						uint64_t const tpacksize = (totalsuf + numpack-1)/numpack;
						//
						uint64_t ilow = 0;
						// intervals on G
						std::vector < std::pair<uint64_t,uint64_t> > wpacks;
						// prefix sums over G
						std::vector < uint64_t > P;
						P.push_back(0);

						// std::cerr << "(computing work packets...";
						libmaus2::suffixsort::GapArrayByteDecoder::unique_ptr_type pgap1dec(GACR.G->getDecoder());
						libmaus2::suffixsort::GapArrayByteDecoderBuffer::unique_ptr_type pgap1decbuf(new libmaus2::suffixsort::GapArrayByteDecoderBuffer(*pgap1dec,8192));
						libmaus2::suffixsort::GapArrayByteDecoderBuffer::iterator gap1decbufit = pgap1decbuf->begin();
						while ( ilow != (cblocksize+1) )
						{
							uint64_t s = 0;
							uint64_t ihigh = ilow;

							while ( ihigh != (cblocksize+1) && s < tpacksize )
							{
								s += *(gap1decbufit++) + 1; // (GACR.G[ihigh++]+1);
								ihigh += 1;
							}

							uint64_t const p = s-(ihigh-ilow);

							if ( ihigh+1 == (cblocksize+1) && (*gap1decbufit == 0) /* GACR.G[ihigh] == 0 */ )
							{
								ihigh++;
								gap1decbufit++;
							}

							// std::cerr << "[" << ilow << "," << ihigh << ")" << std::endl;

							#if defined(GAP_ARRAY_BYTE_DEBUG)
							{
								// check obtained prefix sum
								libmaus2::suffixsort::GapArrayByteDecoder::unique_ptr_type pgap2dec(GACR.G->getDecoder(ilow));
								libmaus2::suffixsort::GapArrayByteDecoderBuffer::unique_ptr_type pgap2decbuf(new libmaus2::suffixsort::GapArrayByteDecoderBuffer(*pgap2dec,8192));
								libmaus2::suffixsort::GapArrayByteDecoderBuffer::iterator gap2decbufit = pgap2decbuf->begin();

								uint64_t a = 0;
								for ( uint64_t ia = ilow; ia < ihigh; ++ia )
									a += *(gap2decbufit++);

								assert ( p == a );
							}
							#endif

							P.push_back(P.back() + p);
							wpacks.push_back(std::pair<uint64_t,uint64_t>(ilow,ihigh));
							ilow = ihigh;
						}
						assert ( wpacks.size() <= numthreads );
						// std::cerr << "done,time=" << wprtc.getElapsedSeconds() << ")";
						pgap1decbuf.reset();
						pgap1dec.reset();
						#endif

						std::vector < std::string > encfilenames(wpacks.size());
						for ( uint64_t i = 0; i < wpacks.size(); ++i )
						{
							encfilenames[i] = (
								gtmpgen.getFileName()
								// result.getFiles().getBWT()
								+ "_"
								+ ::libmaus2::util::NumberSerialisation::formatNumber(encfilenames.size(),6)
								+ ".bwt"
							);
							::libmaus2::util::TempFileRemovalContainer::addTempFile(encfilenames[i]);
						}

						// std::cerr << "(setting up IDDs...";
						::libmaus2::timing::RealTimeClock wprtc; wprtc.start();
						wprtc.start();
						unsigned int const albits = rl_decoder::haveAlphabetBits() ? rl_decoder::getAlBits(mergereq.children[0]->sortresult.getFiles().getBWT()) : 0;
						::libmaus2::huffman::IndexDecoderDataArray IDD0(
							mergereq.children[0]->sortresult.getFiles().getBWT(),numthreads);
						::libmaus2::huffman::IndexDecoderDataArray IDD1(
							mergereq.children[1]->sortresult.getFiles().getBWT(),numthreads);

						::libmaus2::huffman::IndexEntryContainerVector::unique_ptr_type IECV0 = ::libmaus2::huffman::IndexLoader::loadAccIndex(
							mergereq.children[0]->sortresult.getFiles().getBWT()
						);
						::libmaus2::huffman::IndexEntryContainerVector::unique_ptr_type IECV1 = ::libmaus2::huffman::IndexLoader::loadAccIndex(
							mergereq.children[1]->sortresult.getFiles().getBWT()
						);

						#if defined(_OPENMP)
						#pragma omp parallel for schedule(dynamic,1) num_threads(numthreads)
						#endif
						for ( int64_t b = 0; b < static_cast<int64_t>(wpacks.size()); ++b )
						{
							uint64_t const ilow = wpacks[b].first;
							uint64_t const ihigh = wpacks[b].second;

							if ( ilow != ihigh )
							{
								bool const islast = (ihigh == (cblocksize+1));
								std::string const encfilename = encfilenames[b];

								rl_decoder leftrlin(IDD0,IECV0.get(),ilow);
								rl_decoder rightrlin(IDD1,IECV1.get(),P[b]);

								uint64_t const outsuf = (ihigh-ilow)-(islast?1:0) + (P[b+1]-P[b]);

								rl_encoder bwtenc(encfilename,albits,outsuf,rlencoderblocksize);

								libmaus2::suffixsort::GapArrayByteDecoder::unique_ptr_type pgap3dec(GACR.G->getDecoder(ilow));
								libmaus2::suffixsort::GapArrayByteDecoderBuffer::unique_ptr_type pgap3decbuf(new libmaus2::suffixsort::GapArrayByteDecoderBuffer(*pgap3dec,8192));
								libmaus2::suffixsort::GapArrayByteDecoderBuffer::iterator gap3decbufit = pgap3decbuf->begin();

								if ( islast )
								{
									for ( uint64_t j = ilow; j < ihigh-1; ++j )
									{
										uint64_t const GACRGj = *(gap3decbufit++);

										for ( uint64_t i = 0; i < GACRGj; ++i )
											bwtenc.encode(rightrlin.decode());

										bwtenc.encode(leftrlin.decode());
									}

									assert ( ihigh == cblocksize+1 );

									uint64_t const GACRGcblocksize = *(gap3decbufit++);
									for ( uint64_t i = 0; i < GACRGcblocksize; ++i )
										bwtenc.encode(rightrlin.decode());
								}
								else
								{
									for ( uint64_t j = ilow; j < ihigh; ++j )
									{
										uint64_t const GACRGj = *(gap3decbufit++);

										for ( uint64_t i = 0; i < GACRGj; ++i )
											bwtenc.encode(rightrlin.decode());

										bwtenc.encode(leftrlin.decode());
									}
								}

								bwtenc.flush();
							}
						}
						if ( logstr )
							(*logstr) << "done, time " << rtc.getElapsedSeconds() << std::endl;

						#if 0
						std::cerr << "[V] concatenating bwt parts...";
						rtc.start();
						rl_encoder::concatenate(encfilenames,result.getFiles().getBWT());
						std::cerr << "done, time " << rtc.getElapsedSeconds() << std::endl;
						#endif

						result.setBWT(encfilenames);

						#if 0
						std::cerr << "[V] removing tmp files...";
						rtc.start();
						for ( uint64_t i = 0; i < encfilenames.size(); ++i )
							libmaus2::aio::FileRemoval::removeFile ( encfilenames[i].c_str() );
						std::cerr << "done, time " << rtc.getElapsedSeconds() << std::endl;
						#endif

						// save histogram
						if ( logstr )
							(*logstr) << "[V] saving histogram...";
						rtc.start();
						accD->serialise(static_cast<std::string const & >(result.getFiles().getHist()));
						if ( logstr )
							(*logstr) << "done, time " << rtc.getElapsedSeconds() << std::endl;
					}
					else
					{
						// std::cerr << "** WHITEBOX INTERNAL 2 **" << std::endl;

						std::vector < std::string > gapfilenames;
						std::vector < std::vector<std::string> > bwtfilenames;
						for ( uint64_t bb = 0; bb < mergereq.children.size(); ++bb )
						{
							// gap file name
							if ( bb+1 < mergereq.children.size() )
							{
								std::string const newgapname = gtmpgen.getFileName() + "_merging_" + ::libmaus2::util::NumberSerialisation::formatNumber(bb,4) + ".gap";
								::libmaus2::util::TempFileRemovalContainer::addTempFile(newgapname);
								gapfilenames.push_back(newgapname);
							}

							// bwt name
							std::vector<std::string> newbwtnames;
							for ( uint64_t i = 0; i < mergereq.children[bb]->sortresult.getFiles().getBWT().size(); ++i )
							{
								std::string const newbwtname = gtmpgen.getFileName() + "_merging_"
									+ ::libmaus2::util::NumberSerialisation::formatNumber(bb,4)
									+ "_"
									+ ::libmaus2::util::NumberSerialisation::formatNumber(i,4)
									+ ".bwt";
								::libmaus2::util::TempFileRemovalContainer::addTempFile(newbwtname);
								newbwtnames.push_back(newbwtname);
							}
							bwtfilenames.push_back(newbwtnames);
						}

						// rename last bwt file set
						for ( uint64_t i = 0; i < mergereq.children.back()->sortresult.getFiles().getBWT().size(); ++i )
						{
							libmaus2::aio::OutputStreamFactoryContainer::rename (
								mergereq.children.back()->sortresult.getFiles().getBWT()[i].c_str(),
								bwtfilenames.back()[i].c_str()
							);
						}

						std::vector<std::string> mergedgtname  = mergereq.children.back()->sortresult.getFiles().getGT();
						std::vector<std::string> mergedisaname = mergereq.children.back()->sortresult.getFiles().getSampledISAVector();

						// load char histogram for last block
						std::string const & lblockhist = mergereq.children.back()->sortresult.getFiles().getHist();
						::libmaus2::lf::DArray::unique_ptr_type accD(new ::libmaus2::lf::DArray(lblockhist));

						/**
						 * iteratively merge blocks together
						 **/
						for ( uint64_t bb = 0; bb+1 < mergereq.children.size(); ++bb )
						{
							// block we merge into
							uint64_t const bx = mergereq.children.size()-bb-2;
							if ( logstr )
								(*logstr) << "[V] merging blocks " << bx+1 << " to end into " << bx << std::endl;
							::libmaus2::suffixsort::BwtMergeBlockSortResult const & blockresults =
								mergereq.children[bx]->sortresult;

							// output files for this iteration
							std::string const newmergedisaname = gtmpgen.getFileName() + "_merged_" + ::libmaus2::util::NumberSerialisation::formatNumber(bx,4) + ".sampledisa";
							::libmaus2::util::TempFileRemovalContainer::addTempFile(newmergedisaname);

							// gap file
							std::string const gapfile = gapfilenames[bx];

							// start of this block
							uint64_t const blockstart = blockresults.getBlockStart();
							// size of this block
							uint64_t const cblocksize = blockresults.getCBlockSize();

							// compute gap array
							GapArrayByteComputationResult const GACR = computeGapArrayByte(
								gtmpgen,
								fn,fs,*(mergereq.gaprequests[bx]),
								mergedgtname,
								#if 0
								newmergedgtname,
								newmergedgapoverflow,
								#endif
								accD.get(),
								numthreads,
								logstr
							);

							// save the gap file
							#if defined(LIBMAUS2_SUFFIXSORT_BWTB3M_HUFGAP)
							GACR.G->saveHufGapArray(gapfile);
							#else
							GACR.G->saveGammaGapArray(gapfile);
							#endif

							::libmaus2::timing::RealTimeClock rtc; rtc.start();

							if ( logstr )
								(*logstr) << "[V] splitting gap array...";

							std::vector < std::pair<uint64_t,uint64_t> > wpacks;
							std::vector < uint64_t > P;
							splitGapArray(
								*(GACR.G),
								cblocksize+1,
								numthreads,
								wpacks,
								P,
								logstr,
								verbose
							);

							if ( logstr )
								(*logstr) << "done, time " << rtc.getElapsedSeconds() << std::endl;

							// merge sampled inverse suffix arrays, returns rank of position 0 (relative to block start)
							std::pair < uint64_t, std::vector<std::string> > const PPP = BwtMergeIsaParallelBase<GapArrayByte>::mergeIsaParallel(
								gtmpgen,wpacks,P,
								mergedisaname,
								blockresults.getFiles().getSampledISAVector(),
								blockstart,
								*(GACR.G),
								GACR.Gsize,
								numthreads,
								logstr
							);

							result.setBlockP0Rank(PPP.first);

							#if defined(LIBMAUS2_SUFFIXSORT_BWTB3M_PARALLEL_ISA_MERGE_DEBUG)
							{
								std::string const isadebugtmp = gtmpgen.getFileName(true);

								// merge sampled inverse suffix arrays, returns rank of position 0 (relative to block start)
								// libmaus2::util::GetCObject<uint32_t const *> mergeGO(GACR.G.begin());
								libmaus2::suffixsort::GapArrayByteDecoder::unique_ptr_type pgap0dec(GACR.G->getDecoder());
								libmaus2::suffixsort::GapArrayByteDecoderBuffer::unique_ptr_type pgap0decbuf(new libmaus2::suffixsort::GapArrayByteDecoderBuffer(*pgap0dec,8192));
								uint64_t const serialblockp0rank = BwtMergeIsaBase<libmaus2::suffixsort::GapArrayByteDecoderBuffer>::mergeIsa(mergedisaname,blockresults.getFiles().getSampledISAVector(),isadebugtmp,blockstart,*pgap0decbuf/*mergeGO*/ /*GACR.G.begin()*/,cblocksize+1 /*GACR.G.size()*/,logstr);

								assert ( PPP.first == serialblockp0rank );
								assert ( compareFiles(PPP.second,std::vector<std::string>(1,isadebugtmp)) );

								libmaus2::aio::FileRemoval::removeFile(isadebugtmp);
							}
							#endif

							#if 0
							// concatenate gt vectors
							concatenateGT(GACR.gtpartnames,blockresults.getFiles().getGT(),newmergedgtname);
							#endif

							// rename files
							std::vector<std::string> oldgtnames;
							for ( uint64_t i = 0; i < blockresults.getFiles().getGT().size(); ++i )
							{
								std::ostringstream ostr;
								ostr << gtmpgen.getFileName()
									<< "_renamed_"
									<< std::setw(6) << std::setfill('0') << bx << std::setw(0)
									<< "_"
									<< std::setw(6) << std::setfill('0') << i << std::setw(0)
									<< ".gt";
								std::string const renamed = ostr.str();
								oldgtnames.push_back(ostr.str());
								::libmaus2::util::TempFileRemovalContainer::addTempFile(renamed);
								libmaus2::aio::OutputStreamFactoryContainer::rename(blockresults.getFiles().getGT()[i].c_str(), renamed.c_str());
							}

							// result.setGT(stringVectorAppend(GACR.gtpartnames,blockresults.getFiles().getGT()));

							/*
							 * remove files we no longer need
							 */
							// files local to this block
							for ( uint64_t i = 0; i < blockresults.getFiles().getBWT().size(); ++i )
								libmaus2::aio::OutputStreamFactoryContainer::rename ( blockresults.getFiles().getBWT()[i].c_str(), bwtfilenames[bx][i].c_str() );
							blockresults.removeFilesButBwt();
							// previous stage gt bit vector
							for ( uint64_t i = 0; i < mergedgtname.size(); ++i )
								libmaus2::aio::FileRemoval::removeFile ( mergedgtname[i].c_str() );

							// update current file names
							mergedgtname = stringVectorAppend(GACR.gtpartnames,oldgtnames);
							mergedisaname = PPP.second;
						}


						// renamed sampled inverse suffix array
						result.setSampledISA(mergedisaname);
						// libmaus2::aio::OutputStreamFactoryContainer::rename ( mergedisaname.c_str(), result.getFiles().getSampledISA().c_str() );
						// rename gt bit array filename
						// libmaus2::aio::OutputStreamFactoryContainer::rename ( mergedgtname.c_str(), result.getFiles().getGT().c_str() );
						result.setGT(mergedgtname);
						// save histogram
						accD->serialise(static_cast<std::string const & >(result.getFiles().getHist()));

						if ( logstr )
							(*logstr) << "[V] merging parts...";
						::libmaus2::timing::RealTimeClock mprtc;
						mprtc.start();
						result.setBWT(BwtMergeParallelGapFragMerge::parallelGapFragMerge(
							gtmpgen,
							bwtfilenames,
							stringVectorPack(gapfilenames),
							// result.getFiles().getBWT(),
							// tmpfilenamebase+"_gpart",
							numthreads,
							lfblockmult,rlencoderblocksize,logstr,verbose));
						if ( logstr )
							(*logstr) << "done, time " << mprtc.getElapsedSeconds() << std::endl;

						for ( uint64_t i = 0; i < gapfilenames.size(); ++i )
							libmaus2::aio::FileRemoval::removeFile ( gapfilenames[i].c_str() );
						for ( uint64_t i = 0; i < bwtfilenames.size(); ++i )
							for ( uint64_t j = 0; j < bwtfilenames[i].size(); ++j )
								libmaus2::aio::FileRemoval::removeFile ( bwtfilenames[i][j].c_str() );
					}

					#if 0
					std::cerr << "[V] computing term symbol hwt...";
					::libmaus2::timing::RealTimeClock mprtc;
					mprtc.start();
					if ( input_types_type::utf8Wavelet() )
						libmaus2::wavelet::RlToHwtBase<true,rl_decoder>::rlToHwtTerm(result.getFiles().getBWT(),result.getFiles().getHWT(),tmpfilenamebase + "_wt",chist,bwtterm,result.getBlockP0Rank());
					else
						libmaus2::wavelet::RlToHwtBase<false,rl_decoder>::rlToHwtTerm(result.getFiles().getBWT(),result.getFiles().getHWT(),tmpfilenamebase + "_wt",chist,bwtterm,result.getBlockP0Rank());
					std::cerr << "done, time " << mprtc.getElapsedSeconds() << std::endl;
					#endif

					libmaus2::util::TempFileRemovalContainer::addTempFile(result.getFiles().getHWTReq());
					{
					libmaus2::aio::OutputStreamInstance hwtreqCOS(result.getFiles().getHWTReq());
					libmaus2::wavelet::RlToHwtTermRequest::serialise(
						hwtreqCOS,
						result.getFiles().getBWT(),
						result.getFiles().getHWT(),
						gtmpgen.getFileName() + "_wt", // XXX replace
						huftreefilename,
						bwtterm,
						result.getBlockP0Rank(),
						input_types_type::utf8Wavelet(),
						numthreads
					);
					hwtreqCOS.flush();
					//hwtreqCOS.close();
					}

					// remove obsolete files
					for ( uint64_t b = 0; b < mergereq.children.size(); ++b )
						mergereq.children[b]->sortresult.removeFiles();

					mergereq.releaseChildren();
				}

				static void mergeBlocks(
					libmaus2::util::TempFileNameGenerator & gtmpgen,
					libmaus2::suffixsort::bwtb3m::MergeStrategyMergeExternalBlock & mergereq,
					std::string const fn,
					uint64_t const fs,
					// std::string const tmpfilenamebase,
					std::string const sparsetmpfilenamebase,
					uint64_t const rlencoderblocksize,
					uint64_t const lfblockmult,
					uint64_t const numthreads,
					::std::map<int64_t,uint64_t> const & /* chist */,
					uint64_t const bwtterm,
					uint64_t const mem,
					std::string const & huftreefilename,
					std::ostream * logstr,
					int const verbose
				)
				{
					if ( logstr )
						(*logstr) << "[V] Merging BWT blocks MergeStrategyMergeExternalBlock." << std::endl;

					assert ( mergereq.children.size() > 1 );
					assert ( mergereq.children.size() == mergereq.gaprequests.size()+1 );

					/*
					 * remove unused file
					 */
					libmaus2::aio::FileRemoval::removeFile ( mergereq.children[mergereq.children.size()-1]->sortresult.getFiles().getHWT().c_str() );

					// get result object
					::libmaus2::suffixsort::BwtMergeBlockSortResult & result = mergereq.sortresult;
					// fill result structure
					result.setBlockStart( mergereq.children[0]->sortresult.getBlockStart() );
					result.setCBlockSize ( 0 );
					for ( uint64_t i = 0; i < mergereq.children.size(); ++i )
						result.setCBlockSize( result.getCBlockSize() + mergereq.children[i]->sortresult.getCBlockSize() );
					// set up file names for
					// huffman shaped wavelet tree and
					// histogram
					// result.setTempPrefixSingleAndRegisterAsTemp(gtmpgen);

					{
						std::vector < std::vector < std::string > > gapfilenames;
						std::vector < std::vector < std::string > > bwtfilenames;
						for ( uint64_t bb = 0; bb < mergereq.children.size(); ++bb )
						{
							// gap file name
							if ( bb+1 < mergereq.children.size() )
								gapfilenames.push_back(std::vector<std::string>());

							// bwt name
							std::vector<std::string> newbwtnames;
							for ( uint64_t i = 0; i < mergereq.children[bb]->sortresult.getFiles().getBWT().size(); ++i )
							{
								std::string const newbwtname = gtmpgen.getFileName() + "_merging_"
									+ ::libmaus2::util::NumberSerialisation::formatNumber(bb,4)
									+ "_"
									+ ::libmaus2::util::NumberSerialisation::formatNumber(i,4)
									+ ".bwt";
								::libmaus2::util::TempFileRemovalContainer::addTempFile(newbwtname);
								newbwtnames.push_back(newbwtname);
							}
							bwtfilenames.push_back(newbwtnames);
						}

						// rename last bwt file set
						for ( uint64_t i = 0; i < mergereq.children.back()->sortresult.getFiles().getBWT().size(); ++i )
						{
							libmaus2::aio::OutputStreamFactoryContainer::rename (
								mergereq.children.back()->sortresult.getFiles().getBWT()[i].c_str(),
								bwtfilenames.back()[i].c_str()
							);
						}


						std::vector<std::string> mergedgtname  = mergereq.children.back()->sortresult.getFiles().getGT();
						std::vector<std::string> mergedisaname = mergereq.children.back()->sortresult.getFiles().getSampledISAVector();

						// load char histogram for last block
						std::string const & lblockhist = mergereq.children.back()->sortresult.getFiles().getHist();
						::libmaus2::lf::DArray::unique_ptr_type accD(new ::libmaus2::lf::DArray(lblockhist));

						/**
						 * iteratively merge blocks together
						 **/
						for ( uint64_t bb = 0; bb+1 < mergereq.children.size(); ++bb )
						{
							// std::cerr << "** WHITEBOX EXTERNAL **" << std::endl;

							// block we merge into
							uint64_t const bx = mergereq.children.size()-bb-2;
							if ( logstr )
								(*logstr) << "[V] merging blocks " << bx+1 << " to end into " << bx << std::endl;
							::libmaus2::suffixsort::BwtMergeBlockSortResult const & blockresults =
								mergereq.children[bx]->sortresult;

							// output files for this iteration
							std::string const newmergedisaname = gtmpgen.getFileName() + "_merged_" + ::libmaus2::util::NumberSerialisation::formatNumber(bx,4) + ".sampledisa";
							::libmaus2::util::TempFileRemovalContainer::addTempFile(newmergedisaname);

							// start of this block
							uint64_t const blockstart = blockresults.getBlockStart();
							// size of this block
							uint64_t const cblocksize = blockresults.getCBlockSize();

							// std::cerr << "*** compute sparse ***" << std::endl;

							SparseGapArrayComputationResult const GACR = computeSparseGapArray(
								gtmpgen,
								fn,fs,*(mergereq.gaprequests[bx]),
								mergereq.children[mergereq.gaprequests[bx]->into]->getIHWTSpaceBytes(),
								mergedgtname,
								accD.get(),
								sparsetmpfilenamebase+"_sparsegap",
								mem,
								numthreads,
								logstr
							);
							gapfilenames[bx] = GACR.fn;

							// merge sampled inverse suffix arrays, returns rank of position 0 (relative to block start)
							libmaus2::gamma::GammaGapDecoder GGD(gapfilenames[bx],0/* offset */,0 /* psymoff */,numthreads);
							result.setBlockP0Rank( BwtMergeIsaBase<libmaus2::gamma::GammaGapDecoder>::mergeIsa(mergedisaname,blockresults.getFiles().getSampledISAVector(),newmergedisaname,blockstart,GGD,cblocksize+1 /*GACR.G.size()*/,logstr) );

							// concatenate gt vectors
							//concatenateGT(GACR.gtpartnames,blockresults.getFiles().getGT(),newmergedgtname);

							// rename files
							std::vector<std::string> oldgtnames;
							for ( uint64_t i = 0; i < blockresults.getFiles().getGT().size(); ++i )
							{
								std::ostringstream ostr;
								ostr << gtmpgen.getFileName()
									<< "_renamed_"
									<< std::setw(6) << std::setfill('0') << bx << std::setw(0)
									<< "_"
									<< std::setw(6) << std::setfill('0') << i << std::setw(0)
									<< ".gt";
								std::string const renamed = ostr.str();
								oldgtnames.push_back(ostr.str());
								::libmaus2::util::TempFileRemovalContainer::addTempFile(renamed);
								libmaus2::aio::OutputStreamFactoryContainer::rename(blockresults.getFiles().getGT()[i].c_str(), renamed.c_str());
							}

							/*
							 * remove files we no longer need
							 */
							// files local to this block
							for ( uint64_t i = 0; i < blockresults.getFiles().getBWT().size(); ++i )
								libmaus2::aio::OutputStreamFactoryContainer::rename ( blockresults.getFiles().getBWT()[i].c_str(), bwtfilenames[bx][i].c_str() );
							blockresults.removeFilesButBwt();
							// previous stage gt bit vector
							for ( uint64_t i = 0; i < mergedgtname.size(); ++i )
								libmaus2::aio::FileRemoval::removeFile ( mergedgtname[i].c_str() );

							// update current file names
							mergedgtname = stringVectorAppend(GACR.gtpartnames,oldgtnames);
							mergedisaname = std::vector<std::string>(1,newmergedisaname);
						}

						// renamed sampled inverse suffix array
						// libmaus2::aio::OutputStreamFactoryContainer::rename ( mergedisaname.c_str(), result.getFiles().getSampledISA().c_str() );
						result.setSampledISA(mergedisaname);
						// rename gt bit array filename
						// libmaus2::aio::OutputStreamFactoryContainer::rename ( mergedgtname.c_str(), result.getFiles().getGT().c_str() );
						result.setGT(mergedgtname);
						// save histogram
						accD->serialise(static_cast<std::string const & >(result.getFiles().getHist()));

						if ( logstr )
							(*logstr) << "[V] merging parts...";
						::libmaus2::timing::RealTimeClock mprtc;
						mprtc.start();
						result.setBWT(
							BwtMergeParallelGapFragMerge::parallelGapFragMerge(gtmpgen,bwtfilenames,gapfilenames/* tmpfilenamebase+"_gpart" */,numthreads,
								lfblockmult,rlencoderblocksize,logstr,verbose)
						);
						if ( logstr )
							(*logstr) << "done, time " << mprtc.getElapsedSeconds() << std::endl;

						for ( uint64_t i = 0; i < gapfilenames.size(); ++i )
							for ( uint64_t j = 0; j < gapfilenames[i].size(); ++j )
								libmaus2::aio::FileRemoval::removeFile ( gapfilenames[i][j].c_str() );
						for ( uint64_t i = 0; i < bwtfilenames.size(); ++i )
							for ( uint64_t j = 0; j < bwtfilenames[i].size(); ++j )
								libmaus2::aio::FileRemoval::removeFile ( bwtfilenames[i][j].c_str() );
					}

					#if 0
					std::cerr << "[V] computing term symbol hwt...";
					::libmaus2::timing::RealTimeClock mprtc;
					mprtc.start();
					if ( input_types_type::utf8Wavelet() )
						libmaus2::wavelet::RlToHwtBase<true,rl_decoder>::rlToHwtTerm(result.getFiles().getBWT(),result.getFiles().getHWT(),tmpfilenamebase + "_wt",chist,bwtterm,result.getBlockP0Rank());
					else
						libmaus2::wavelet::RlToHwtBase<false,rl_decoder>::rlToHwtTerm(result.getFiles().getBWT(),result.getFiles().getHWT(),tmpfilenamebase + "_wt",chist,bwtterm,result.getBlockP0Rank());
					std::cerr << "done, time " << mprtc.getElapsedSeconds() << std::endl;
					#endif

					libmaus2::util::TempFileRemovalContainer::addTempFile(result.getFiles().getHWTReq());
					{
					libmaus2::aio::OutputStreamInstance hwtreqCOS(result.getFiles().getHWTReq());
					libmaus2::wavelet::RlToHwtTermRequest::serialise(hwtreqCOS,
						result.getFiles().getBWT(),
						result.getFiles().getHWT(),
						gtmpgen.getFileName() + "_wt", // XXX replace
						huftreefilename,
						bwtterm,
						result.getBlockP0Rank(),
						input_types_type::utf8Wavelet(),
						numthreads
					);
					hwtreqCOS.flush();
					//hwtreqCOS.close();
					}

					// remove obsolete files
					for ( uint64_t b = 0; b < mergereq.children.size(); ++b )
						mergereq.children[b]->sortresult.removeFiles();

					mergereq.releaseChildren();
				}

				static std::string sortIsaFile(
					libmaus2::util::TempFileNameGenerator & gtmpgen,
					std::vector<std::string> const & mergedisaname,
					uint64_t const blockmem,
					uint64_t const numthreads,
					uint64_t const fanin,
					std::ostream * logstr
				)
				{
					// sort sampled inverse suffix array file
					std::string const mergeisatmp = gtmpgen.getFileName(true);
					std::string const mergeisatmpout = gtmpgen.getFileName(true);
					::libmaus2::sorting::PairFileSorting::sortPairFile(
						mergedisaname,mergeisatmp,true /* second comp */,
						true,true,mergeisatmpout,blockmem/2/*par*/,numthreads /* parallel */,false /* delete input */,fanin,logstr);
					libmaus2::aio::FileRemoval::removeFile (mergeisatmp);
					return mergeisatmpout;
				}

				static ::libmaus2::autoarray::AutoArray< std::pair<uint64_t,uint64_t> > readBlockRanks(std::string const & mergedisaname)
				{
					// read sampled isa
					uint64_t const nsisa = BwtMergeComputeSampledSA<input_types_type>::readBlockRanksSize(mergedisaname);
					::libmaus2::autoarray::AutoArray< std::pair<uint64_t,uint64_t> > blockranks(nsisa,false);
					::libmaus2::aio::SynchronousGenericInput<uint64_t>::unique_ptr_type SGIsisa(new ::libmaus2::aio::SynchronousGenericInput<uint64_t>(mergedisaname,16*1024));
					for ( uint64_t i = 0; i < nsisa; ++i )
					{
						int64_t const r = SGIsisa->get();
						assert ( r >= 0 );
						int64_t const p = SGIsisa->get();
						assert ( p >= 0 );
						blockranks [ i ] = std::pair<uint64_t,uint64_t>(r,p);
					}
					SGIsisa.reset();

					return blockranks;
				}



				static uint64_t getDefaultBlockSize(uint64_t const mem, uint64_t const threads, uint64_t const fs)
				{
					uint64_t const memblocksize = std::max(static_cast<uint64_t>(0.95 * mem / ( 5 * threads )),static_cast<uint64_t>(1));
					uint64_t const fsblocksize = (fs + threads - 1)/threads;
					return std::min(memblocksize,fsblocksize);
				}

				static std::map<int64_t,uint64_t> getBlockSymFreqs(std::string const fn, uint64_t const glow, uint64_t ghigh, uint64_t const numthreads)
				{
					typedef typename input_types_type::linear_wrapper stream_type;
					typedef typename input_types_type::base_input_stream::char_type char_type;
					typedef typename ::libmaus2::util::UnsignedCharVariant<char_type>::type unsigned_char_type;

					uint64_t const fs = ghigh-glow;

					uint64_t const symsperfrag = (fs + numthreads - 1)/numthreads;
					uint64_t const numfrags = (fs + symsperfrag - 1)/symsperfrag;

					libmaus2::util::HistogramSet HS(numfrags,256);

					#if defined(_OPENMP)
					#pragma omp parallel for num_threads(numthreads)
					#endif
					for ( int64_t t = 0; t < static_cast<int64_t>(numfrags); ++t )
					{
						uint64_t const low = glow + t * symsperfrag;
						uint64_t const high = std::min(low+symsperfrag,ghigh);
						uint64_t const size = high-low;
						uint64_t todo = size;

						stream_type CIS(fn);
						CIS.seekg(low);

						::libmaus2::autoarray::AutoArray<unsigned_char_type> B(16*1024,false);
						libmaus2::util::Histogram & H = HS[t];

						while ( todo )
						{
							uint64_t const toread = std::min(todo,B.size());
							CIS.read(reinterpret_cast<char_type *>(B.begin()),toread);
							assert ( CIS.gcount() == static_cast<int64_t>(toread) );
							for ( uint64_t i = 0; i < toread; ++i )
								H (B[i]);
							todo -= toread;
						}
					}

					::libmaus2::util::Histogram::unique_ptr_type PH(HS.merge());
					::libmaus2::util::Histogram & H(*PH);

					return H.getByType<int64_t>();
				}


				struct CopyToMemoryResult
				{
					libmaus2::aio::ArrayFile<char const *>::shared_ptr_type COAF;
					libmaus2::autoarray::AutoArray<char>::shared_ptr_type COA;

					CopyToMemoryResult()
					{}
					CopyToMemoryResult(
						libmaus2::aio::ArrayFile<char const *>::shared_ptr_type r_COAF,
						libmaus2::autoarray::AutoArray<char>::shared_ptr_type r_COA
					) : COAF(r_COAF), COA(r_COA)
					{}
				};

				static CopyToMemoryResult copyInputToMemory(
					BwtMergeSortOptions const & options, std::ostream * logstr,
					std::string & fn
				)
				{
					libmaus2::aio::ArrayFile<char const *>::shared_ptr_type COAF;
					libmaus2::autoarray::AutoArray<char>::shared_ptr_type COA;


					if ( options.copyinputtomemory )
					{
						if ( input_types_type::utf8Wavelet() )
						{
							std::string const nfn = "mem://copied_" + fn;
							libmaus2::aio::InputStreamInstance ininst(fn);
							libmaus2::aio::OutputStreamInstance outinst(nfn);
							uint64_t const fs = libmaus2::util::GetFileSize::getFileSize(ininst);
							libmaus2::util::GetFileSize::copy(ininst,outinst,fs);
							if ( logstr )
								(*logstr) << "[V] copied " << fn << " to " << nfn << std::endl;

							{
								libmaus2::aio::InputStreamInstance ininst(fn + ".idx");
								libmaus2::aio::OutputStreamInstance outinst(nfn + ".idx");
								uint64_t const fs = libmaus2::util::GetFileSize::getFileSize(ininst);
								libmaus2::util::GetFileSize::copy(ininst,outinst,fs);
								if ( logstr )
									*logstr << "[V] copied " << fn+".idx" << " to " << nfn+".idx" << std::endl;
							}

							fn = nfn;
						}
						else
						{
							uint64_t const fs = libmaus2::util::GetFileSize::getFileSize(fn);
							libmaus2::autoarray::AutoArray<char>::shared_ptr_type tCOA(
								new libmaus2::autoarray::AutoArray<char>(fs,false)
							);
							COA = tCOA;

							#if defined(_OPENMP)
							uint64_t const bs = 1024ull * 1024ull;
							uint64_t const numthreads = options.numthreads;
							uint64_t const numblocks = (fs + bs - 1)/bs;

							if ( logstr )
								(*logstr) << "[V] copying " << fn << " using " << numthreads << " threads, stripe size " << bs << std::endl;

							#pragma omp parallel num_threads(numthreads)
							{
								uint64_t const tid = omp_get_thread_num();
								libmaus2::aio::InputStreamInstance ininst(fn);

								uint64_t bid = tid;

								while ( bid < numblocks )
								{
									uint64_t const low = bid * bs;
									uint64_t const high = std::min(low+bs,fs);
									assert ( high > low );

									ininst.clear();
									ininst.seekg(low);

									ininst.read(COA->begin() + low,high-low);

									assert ( ininst && (ininst.gcount() == static_cast<int64_t>(high-low)) );

									bid += numthreads;
								}
							}
							#else
							{
								libmaus2::aio::InputStreamInstance ininst(fn);
								ininst.read(COA->begin(),fs);
								assert ( ininst && (ininst.gcount() == static_cast<int64_t>(fs)) );
							}
							#endif

							libmaus2::aio::ArrayFile<char const *>::shared_ptr_type tCOAF(
								new libmaus2::aio::ArrayFile<char const *>(COA->begin(),COA->end())
							);
							COAF = tCOAF;
							std::string const nfn = COAF->getURL();
							if ( logstr )
								*logstr << "[V] copied " << fn << " to " << nfn << std::endl;

							#if 0
							{
								libmaus2::aio::InputStreamInstance ISI0(fn);
								libmaus2::aio::InputStreamInstance ISI1(nfn);
								for ( uint64_t i = 0; i < fs; ++i )
								{
									int const c0 = ISI0.get();
									int const c1 = ISI1.get();
									assert ( c0 != std::istream::traits_type::eof() );
									assert ( c0 == c1 );
								}
							}
							#endif

							fn = nfn;
						}
					}

					return CopyToMemoryResult(COAF,COA);
				}

				static void findPeriods(
					BwtMergeSortOptions const & options,
					BwtMergeSortState & state,
					std::ostream * logstr,
					std::size_t const numblocks,
					std::string const & fn
				)
				{
					std::vector<int64_t> minblockperiods(numblocks);
					// compute periods
					libmaus2::parallel::StdMutex block;
					#if defined(_OPENMP)
					#pragma omp parallel for schedule(dynamic,1) num_threads(options.numthreads)
					#endif
					for ( uint64_t bb = 0; bb < state.getNumBlocks(); ++bb )
					{
						// block id
						uint64_t const b = numblocks-bb-1;
						// start of block in file
						uint64_t const blockstart = state.getBlockStart(b);

						typedef typename input_types_type::base_input_stream base_input_stream;
						typedef typename base_input_stream::char_type char_type;
						typedef typename input_types_type::circular_wrapper circular_wrapper;

						uint64_t const readlen = 3 * state.getBlockSize();
						circular_wrapper textstr(fn,blockstart);
						libmaus2::autoarray::AutoArray<char_type> A(readlen,false);
						textstr.read(&A[0],A.size());

						{
						libmaus2::parallel::StdMutex::scope_lock_type slock(block);
						if ( logstr )
							*logstr << "\r" << std::string(80,' ') << "\r[Checking " << (bb+1) << "/" << numblocks << "]\r";
						}

						libmaus2::util::BorderArray<uint32_t> SBA(A.begin(),A.size());

						uint64_t minper = std::numeric_limits<uint64_t>::max();
						for ( uint64_t i = (state.getBlockSize()-1); i < A.size(); ++i )
						{
							uint64_t const period = (i+1)-SBA[i];

							// if period divides length of the prefix
							if (
								( ( (i+1) % period ) == 0 ) && ((i+1) / period > 1)
								&&
								(period < minper)
							)
							{
								{
								libmaus2::parallel::StdMutex::scope_lock_type slock(block);
								if ( logstr )
									*logstr << "\nlen " << (i+1) << " block " << b << " period " << period << std::endl;
								}
								minper = period;
							}
						}

						if ( minper == std::numeric_limits<uint64_t>::max() )
							minblockperiods[b] = -1;
						else
							minblockperiods[b] = minper;

					}
					if ( logstr )
						*logstr << std::endl;
				}

				static uint64_t computeLCPOfBlockWithNextBlock(
					BwtMergeSortOptions const & options,
					BwtMergeSortState & state,
					uint64_t const b,
					std::string const & fn
				)
				{
					// start of block in file
					uint64_t const blockstart = state.getBlockStart(b);
					// size of this block
					uint64_t const cblocksize = state.getBlockSize(b);

					// start of next block
					uint64_t const nextblockstart = (blockstart + cblocksize) % state.getFS();

					// find bounded lcp between this block and start of next
					uint64_t const blcp = libmaus2::suffixsort::BwtMergeBlockSortRequestBase::findSplitCommonBounded<input_types_type>(fn,blockstart,cblocksize,nextblockstart,state.getFS(),options.largelcpthres);

					return blcp;
				}


				static void computeBlockLCPValuesWithStartOfNext(
					BwtMergeSortOptions const & options,
					BwtMergeSortState & state,
					std::string const & fn,
					std::ostream * logstr
				)
				{
					libmaus2::util::AtomicArray<uint64_t> & V_boundedlcpblockvalues = state.getBlockLCPArray();

					std::atomic<uint64_t> lcpblockscomputed(0);
					#if defined(_OPENMP)
					#pragma omp parallel for schedule(dynamic,1) num_threads(options.numthreads)
					#endif
					for ( uint64_t bb = 0; bb < state.getNumBlocks(); ++bb )
					{
						// block id
						uint64_t const b = state.getNumBlocks()-bb-1;

						// find bounded lcp between this block and start of next
						uint64_t const blcp = computeLCPOfBlockWithNextBlock(
							options,state,b,fn
						);

						uint64_t const l_finished = ++lcpblockscomputed;

						if ( logstr )
						{
							libmaus2::aio::StreamLock::lock_type::scope_lock_type slock(
								libmaus2::aio::StreamLock::cerrlock
							);
							*logstr << "(" << static_cast<double>(l_finished)/state.getNumBlocks() << ")";
						}

						V_boundedlcpblockvalues.at(b).store(blcp);
					}
					if ( logstr )
						*logstr << "done." << std::endl;

					for ( uint64_t b = 0; b < state.getNumBlocks(); ++b )
						if ( V_boundedlcpblockvalues.at(b).load() >= options.largelcpthres )
						{
							if ( logstr )
								*logstr << "[V] Recomputing lcp value for block " << b << "...";

							// start of block in file
							uint64_t const blockstart = state.getBlockStart(b);
							// size of this block
							uint64_t const cblocksize = state.getBlockSize(b);

							// start of next block
							uint64_t const nextblockstart = (blockstart + cblocksize) % state.getFS();

							// find bounded lcp between this block and start of next
							uint64_t const blcp = libmaus2::suffixsort::BwtMergeBlockSortRequestBase::findSplitCommon<input_types_type>(fn,blockstart,cblocksize,nextblockstart,state.getFS());

							V_boundedlcpblockvalues.at(b).store(blcp);

							if ( logstr )
								*logstr << blcp << std::endl;
						}
				}

				static void computeBlockSymFreqs(
					BwtMergeSortOptions const & options,
					BwtMergeSortState & state,
					uint64_t const b,
					std::string const & fn,
					std::string const & freqstmpfilename
				)
				{
					// start of block in file
					uint64_t const blockstart = state.getBlockStart(b);
					// size of this block
					uint64_t const cblocksize = state.getBlockSize(b);

					std::map<int64_t,uint64_t> const blockfreqs =
						getBlockSymFreqs(fn,blockstart,blockstart+cblocksize,options.numthreads);

					libmaus2::aio::OutputStreamInstance freqCOS(freqstmpfilename);
					libmaus2::util::NumberMapSerialisation::serialiseMap(freqCOS,blockfreqs);
					freqCOS.flush();
				}

				static void computeBlockSymFreqs(
					BwtMergeSortOptions const & options,
					BwtMergeSortState & state,
					std::string const & fn
				)
				{
					::libmaus2::suffixsort::BwtMergeTempFileNameSetVector const & blocktmpnames(state.getBlockTmpNames());

					// iterate over blocks (in reverse order)
					for ( uint64_t bb = 0; bb < state.getNumBlocks(); ++bb )
					{
						// block id
						uint64_t const b = state.getNumBlocks()-bb-1;
						std::string const freqstmpfilename = blocktmpnames[b].getHistFreq();
						computeBlockSymFreqs(options,state,b,fn,freqstmpfilename);
					}

				}

				static std::shared_ptr<libmaus2::suffixsort::bwtb3m::MergeTree> constructMergeTree(
					std::ostream * logstr,
					BwtMergeSortOptions const & options,
					BwtMergeSortState & state,
					std::string const & fn,
					uint64_t const rlencoderblocksize
				)
				{
					::libmaus2::suffixsort::BwtMergeTempFileNameSetVector const & blocktmpnames(state.getBlockTmpNames());
					libmaus2::huffman::HuffmanTree const & hnode = state.getHuffmanTree();

					// file name of serialised character histogram (table with terminator symbol)
					std::string const chistfilename = state.getCHistFileName();

					// file name of serialised huffman tree
					std::string const huftreefilename = state.getHufTreeFileName();

					::libmaus2::huffman::HuffmanTree::EncodeTable::unique_ptr_type EC(
						new ::libmaus2::huffman::HuffmanTree::EncodeTable(hnode));

					int64_t const bwtterm = state.getBWTTerm();
					uint64_t const maxsym = state.getMaxSym();

					libmaus2::util::AtomicArray<uint64_t> const & V_boundedlcpblockvalues = state.getBlockLCPArray();

					#if defined(BWTB3M_MEMORY_DEBUG)
					if ( logstr )
						*logstr << "[M"<< (mcnt++) << "] " << libmaus2::util::MemUsage() << " " << libmaus2::autoarray::AutoArrayMemUsage() << std::endl;
					#endif

					// merge tree leafs
					std::vector < libmaus2::suffixsort::bwtb3m::MergeStrategyBlock::shared_ptr_type >
						stratleafs(state.getNumBlocks());

					for ( uint64_t bb = 0; bb < state.getNumBlocks(); ++bb )
					{
						uint64_t const b = state.getNumBlocks()-bb-1;

						// start of block in file
						uint64_t const blockstart = state.getBlockStart(b);
						// size of this block
						uint64_t const cblocksize = state.getBlockSize(b);

						// symbol frequency map
						// std::map<int64_t,uint64_t> const & blockfreqs = blockfreqvec[b];
						std::string const freqstmpfilename = blocktmpnames[b].getHistFreq();
						libmaus2::aio::InputStreamInstance freqCIS(freqstmpfilename);
						std::map<int64_t,uint64_t> const blockfreqs =
							libmaus2::util::NumberMapSerialisation::deserialiseMap<libmaus2::aio::InputStreamInstance,int64_t,uint64_t>(freqCIS);

						libmaus2::aio::FileRemoval::removeFile(freqstmpfilename.c_str());
						//
						uint64_t const sourcelengthbits = input_types_type::getSourceLengthBits(fn,blockstart,blockstart+cblocksize,blockfreqs);
						//
						uint64_t const sourcelengthbytes = input_types_type::getSourceLengthBytes(fn,blockstart,blockstart+cblocksize,blockfreqs);
						//
						uint64_t const sourcetextindexbits = input_types_type::getSourceTextIndexBits(fn,blockstart,blockstart+cblocksize,blockfreqs);

						libmaus2::suffixsort::bwtb3m::MergeStrategyBlock::shared_ptr_type PMSB = libmaus2::suffixsort::bwtb3m::MergeStrategyBaseBlock::construct(
							blockstart,blockstart+cblocksize,
							*EC,
							blockfreqs,
							sourcelengthbits,sourcelengthbytes,
							sourcetextindexbits,
							input_types_type::getWordSize()
						);

						/* set up and register sort request */
						::libmaus2::suffixsort::BwtMergeZBlockRequestVector zreqvec;
						dynamic_cast<libmaus2::suffixsort::bwtb3m::MergeStrategyBaseBlock *>(PMSB.get())->sortreq =
							libmaus2::suffixsort::BwtMergeBlockSortRequest(
								input_types_type::getType(),
								input_types_type::getWordSize(),
								fn,state.getFS(),
								chistfilename,
								huftreefilename,
								bwtterm,
								maxsym,
								blocktmpnames[b].serialise(),
								options.tmpfilenamebase /* only used if computeTermSymbolHwt == true */,
								rlencoderblocksize,
								state.getPreISASamplingRate(),
								blockstart,cblocksize,
								zreqvec,
								options.computeTermSymbolHwt,
								V_boundedlcpblockvalues.at(b).load(),
								options.numthreads, /* down stream threads */
								options.verbose
							);

						// if ( logstr ) { *logstr << *PMSB; }

						stratleafs[b] = PMSB;

						#if defined(BWTB3M_MEMORY_DEBUG)
						if ( logstr )
							*logstr << "[M"<< (mcnt++) << "] " << libmaus2::util::MemUsage() << " " << libmaus2::autoarray::AutoArrayMemUsage() << std::endl;
						#endif
					}

					EC.reset();

					#if defined(BWTB3M_MEMORY_DEBUG)
					if ( logstr )
						*logstr << "[M"<< (mcnt++) << "] " << libmaus2::util::MemUsage() << " " << libmaus2::autoarray::AutoArrayMemUsage() << std::endl;
					#endif

					if ( logstr )
						*logstr << "[V] constructing merge tree" << std::endl;

					// construct merge tree and register z requests
					std::shared_ptr<libmaus2::suffixsort::bwtb3m::MergeTree> mergetree =
						libmaus2::suffixsort::bwtb3m::MergeStrategyConstruction::constructMergeTree(
							stratleafs,options.mem,options.numthreads,options.wordsperthread,logstr
						);

					// run tree serialisation/deserialisation loop
					assert ( mergetree->checkSerialisation() );

					if ( options.verbose >= 5 && logstr )
						*logstr << "[V] checked serialisation" << std::endl;

					#if defined(BWTB3M_MEMORY_DEBUG)
					if ( logstr )
						*logstr << "[M"<< (mcnt++) << "] " << libmaus2::util::MemUsage() << " " << libmaus2::autoarray::AutoArrayMemUsage() << std::endl;
					#endif

					return mergetree;
				}

				static BwtMergeSortResult computeBwt(
					BwtMergeSortOptions const & options,
					std::ostream * logstr
				)
				{
					// clock
					libmaus2::timing::RealTimeClock bwtclock;
					bwtclock.start();

					std::unique_ptr<BwtMergeSortState> pstate(BwtMergeSortState::construct(options));
					BwtMergeSortState & state(*pstate);

					// input file name
					std::string fn = options.fn;
					/* copy input data to memory if requested */
					CopyToMemoryResult const copyResult = copyInputToMemory(options,logstr,fn);


					// RL encoder block size
					uint64_t const rlencoderblocksize = 16*1024;

					if ( logstr )
					{
						*logstr << "[V] sorting file " << fn << " of size " << state.getFS() << " with block size " << state.getBlockSize() << " (" << state.getNumBlocks() << " blocks)" << " and " << options.numthreads << " threads" << std::endl;
						*logstr << "[V] full blocks " << state.getFullBlocks() << " reduced blocks " << state.getReducedBlocks() << std::endl;
					}

					#if defined(BWTB3M_MEMORY_DEBUG)
					uint64_t mcnt = 0;
					#endif

					// file name of serialised character histogram (table with terminator symbol)
					std::string const chistfilename = state.getCHistFileName();

					// file name of serialised huffman tree
					std::string const huftreefilename = state.getHufTreeFileName();

					// set up temp file removal container
					::libmaus2::util::TempFileRemovalContainer::setup();

					#if defined(BWTB3M_MEMORY_DEBUG)
					if ( logstr )
						*logstr << "[M"<< (mcnt++) << "] " << libmaus2::util::MemUsage() << " " << libmaus2::autoarray::AutoArrayMemUsage() << std::endl;
					#endif

					if ( logstr )
						*logstr << "[V] computing LCP between block suffixes and the following block start: ";

					// compute LCP values of in block suffixes with start of next block
					computeBlockLCPValuesWithStartOfNext(options,state,fn,logstr);

					#if defined(BWTB3M_MEMORY_DEBUG)
					if ( logstr )
						*logstr << "[M"<< (mcnt++) << "] " << libmaus2::util::MemUsage() << " " << libmaus2::autoarray::AutoArrayMemUsage() << std::endl;
					#endif

					if ( logstr )
						*logstr << "[V] computing symbol frequences" << std::endl;

					// compute character frequences inside each block
					computeBlockSymFreqs(options,state,fn);

					// combine frequences for blocks to frequences of complete input text
					state.computeCharacterHistogram();

					auto const & chistnoterm = state.getCharacterHistogramNoTerm();
					auto const & chist       = state.getCharacterHistogram();

					/* compute bwt terminator symbol (maximum symbol plus 1) */
					int64_t const bwtterm = state.getBWTTerm();

					#if defined(BWTB3M_MEMORY_DEBUG)
					if ( logstr )
						*logstr << "[M"<< (mcnt++) << "] " << libmaus2::util::MemUsage() << " " << libmaus2::autoarray::AutoArrayMemUsage() << std::endl;
					#endif

					state.serialiseCharacterHistogram();

					if ( logstr )
					{
						*logstr << "[V] computed symbol frequences, input alphabet size is " << chistnoterm.size() << std::endl;
						*logstr << "[V] bwtterm=" << bwtterm << std::endl;
					}

					#if defined(BWTB3M_MEMORY_DEBUG)
					if ( logstr )
						*logstr << "[M"<< (mcnt++) << "] " << libmaus2::util::MemUsage() << " " << libmaus2::autoarray::AutoArrayMemUsage() << std::endl;
					#endif

					state.computeHuffmanTree();
					state.serialiseHuffmanTree();

					std::shared_ptr<libmaus2::suffixsort::bwtb3m::MergeTree> mergetree(
						constructMergeTree(logstr,options,state,fn,rlencoderblocksize)
					);

					state.setMergeTree(mergetree);

					// inner node queue
					std::deque<uint64_t> itodo;

					if ( logstr )
						(*logstr) << "[V] sorting blocks" << std::endl;

					#if defined(BWTB3M_MEMORY_DEBUG)
					if ( logstr )
						(*logstr) << "[M"<< (mcnt++) << "] " << libmaus2::util::MemUsage() << " " << libmaus2::autoarray::AutoArrayMemUsage() << std::endl;
					#endif

					// sort single blocks
					libmaus2::suffixsort::bwtb3m::BaseBlockSorting::unique_ptr_type BBS(
						new libmaus2::suffixsort::bwtb3m::BaseBlockSorting(
							*mergetree,options.mem,options.numthreads,
							itodo,logstr,options.verbose
						)
					);
					BBS->start();
					BBS->join();
					BBS.reset();

					if ( logstr )
						(*logstr) << "[V] sorted blocks" << std::endl;

					assert ( mergetree->checkSerialisation() );

					if ( options.verbose >= 5 && logstr )
						*logstr << "[V] checked serialisation after block sorting" << std::endl;

					#if defined(BWTB3M_MEMORY_DEBUG)
					if ( logstr )
						(*logstr) << "[M"<< (mcnt++) << "] " << libmaus2::util::MemUsage() << " " << libmaus2::autoarray::AutoArrayMemUsage() << std::endl;
					#endif

					#if 0
					uint64_t maxhwtsize = 0;
					for ( uint64_t i = 0; i < stratleafs.size(); ++i )
						maxhwtsize = std::max(maxhwtsize,::libmaus2::util::GetFileSize::getFileSize(stratleafs[i]->sortresult.getFiles().getHWT()));
					#endif

					if ( logstr )
						(*logstr) << "[V] filling gap request objects" << std::endl;
					mergetree->fillGapRequestObjects(options.numthreads);

					assert ( mergetree->checkSerialisation() );

					state.fillGTExpected();

					if ( options.verbose >= 5 && logstr )
						*logstr << "[V] checked serialisation after filling gap request objects" << std::endl;

					#if defined(BWTB3M_MEMORY_DEBUG)
					if ( logstr )
						(*logstr) << "[M"<< (mcnt++) << "] " << libmaus2::util::MemUsage() << " " << libmaus2::autoarray::AutoArrayMemUsage() << std::endl;
					#endif

					if ( logstr )
					{
						(*logstr) << "[V]" << std::string(80,'-') << std::endl;
						(*logstr) << *mergetree;
						(*logstr) << "[V]" << std::string(80,'-') << std::endl;
					}

					uint64_t mtmpid = 0;
					uint64_t const lfblockmult = 1;

					// construct temp file name generator object
					std::unique_ptr<libmaus2::util::TempFileNameGenerator> gtmpgen(
						new libmaus2::util::TempFileNameGenerator(options.tmpfilenamebase+"_gtmpdir",5)
					);

					/**
					 * process merge tree
					 **/
					while ( itodo.size() )
					{
						uint64_t const ifront = itodo.front();
						libmaus2::suffixsort::bwtb3m::MergeStrategyBlock * p = &((*mergetree)[ifront]);
						itodo.pop_front();

						if ( logstr )
						{
							(*logstr) << "[V] processing ";
							p->printBase((*logstr));
							(*logstr) << std::endl;
						}

						std::string const l_merge_tmp_dir = state.getDirectoryForMerge(p);

						// p->sortresult.setTempPrefixSingle(l_merge_tmp_dir);

						#if 0
						std::ostringstream tmpstr;
						tmpstr << options.tmpfilenamebase << "_" << std::setfill('0') << std::setw(6) << (mtmpid++);
						#endif

						std::ostringstream sparsetmpstr;
						sparsetmpstr << options.sparsetmpfilenamebase << "_" << std::setfill('0') << std::setw(6) << (mtmpid++);

						// std::size_t const gt_expected = state.getMergeGTExpected(p);

						if ( dynamic_cast<libmaus2::suffixsort::bwtb3m::MergeStrategyMergeInternalBlock *>(p) )
						{
							mergeBlocks(
								*gtmpgen,
								*(dynamic_cast<libmaus2::suffixsort::bwtb3m::MergeStrategyMergeInternalBlock *>(p)),
								fn,state.getFS(),
								rlencoderblocksize,
								lfblockmult,
								options.numthreads,
								chist,bwtterm,
								huftreefilename,
								logstr,
								options.verbose
							);
						}
						else if ( dynamic_cast<libmaus2::suffixsort::bwtb3m::MergeStrategyMergeInternalSmallBlock *>(p) )
						{
							mergeBlocks(
								*gtmpgen,
								*(dynamic_cast<libmaus2::suffixsort::bwtb3m::MergeStrategyMergeInternalSmallBlock *>(p)),
								fn,state.getFS(),
								rlencoderblocksize,
								lfblockmult,
								options.numthreads,
								chist,bwtterm,
								huftreefilename,
								logstr,
								options.verbose
							);
						}
						else if ( dynamic_cast<libmaus2::suffixsort::bwtb3m::MergeStrategyMergeExternalBlock *>(p) )
						{
							mergeBlocks(
								*gtmpgen,
								*(dynamic_cast<libmaus2::suffixsort::bwtb3m::MergeStrategyMergeExternalBlock *>(p)),
								fn,state.getFS(),
								sparsetmpstr.str(),
								rlencoderblocksize,
								lfblockmult,
								options.numthreads,
								chist,bwtterm,
								options.mem,
								huftreefilename,
								logstr,
								options.verbose
							);
						}

						bool const pfinished = p->hasParent() &&
							(*mergetree)[p->parentid].childFinished();

						assert ( mergetree->checkSerialisation() );

						if ( pfinished )
							itodo.push_back(p->parentid);

						#if defined(BWTB3M_MEMORY_DEBUG)
						if ( logstr )
							(*logstr) << "[M"<< (mcnt++) << "] " << libmaus2::util::MemUsage() << " " << libmaus2::autoarray::AutoArrayMemUsage() << std::endl;
						#endif

						// std::size_t const gt_produced = p->sortresult.getFiles().getGT().size();

						// std::cerr << "gt_expected=" << gt_expected << " gt_produced=" << gt_produced << std::endl;
					}

					uint64_t const memperthread = (options.mem + options.numthreads-1)/options.numthreads;

					::libmaus2::suffixsort::BwtMergeBlockSortResult const & mergeresult = mergetree->getSortResult();

					rl_encoder::concatenate(mergeresult.getFiles().getBWT(),options.outfn,true /* removeinput */);

					if ( logstr )
						(*logstr) << "[V] BWT computed in time " << bwtclock.formatTime(bwtclock.getElapsedSeconds()) << std::endl;

					// serialise character histogram
					std::string const outhist = ::libmaus2::util::OutputFileNameTools::clipOff(options.outfn,".bwt") + ".hist";
					::libmaus2::aio::OutputStreamInstance::unique_ptr_type Phistout(new ::libmaus2::aio::OutputStreamInstance(outhist));
					::libmaus2::util::NumberMapSerialisation::serialiseMap(*Phistout,chistnoterm);
					Phistout->flush();
					Phistout.reset();

					// remove hwt request for term symbol hwt
					libmaus2::aio::FileRemoval::removeFile ( mergeresult.getFiles().getHWTReq().c_str() );

					// remove hwt (null op)
					std::string const debhwt = ::libmaus2::util::OutputFileNameTools::clipOff(options.outfn,".bwt") + ".hwt" + ".deb";
					if ( libmaus2::aio::InputStreamFactoryContainer::tryOpen(mergeresult.getFiles().getHWT()) )
					{
						libmaus2::aio::OutputStreamFactoryContainer::rename ( mergeresult.getFiles().getHWT().c_str(), debhwt.c_str() );
						libmaus2::aio::FileRemoval::removeFile ( debhwt.c_str() );
					}

					// remove gt files
					for ( uint64_t i = 0; i < mergeresult.getFiles().getGT().size(); ++i )
						libmaus2::aio::FileRemoval::removeFile ( mergeresult.getFiles().getGT()[i].c_str() );

					BwtMergeSortResult result;

					if ( options.bwtonly )
					{
						std::vector<std::string> const Vmergedisaname = mergeresult.getFiles().getSampledISAVector();

						std::string const outisa = ::libmaus2::util::OutputFileNameTools::clipOff(options.outfn,".bwt") + ".preisa";

						if ( Vmergedisaname.size() == 1)
						{
							libmaus2::aio::OutputStreamFactoryContainer::rename(Vmergedisaname[0].c_str(),outisa.c_str());
						}
						else
						{
							libmaus2::aio::OutputStreamInstance OSI(outisa);

							for ( uint64_t i = 0; i < Vmergedisaname.size(); ++i )
							{
								libmaus2::aio::InputStreamInstance::unique_ptr_type ISI(new libmaus2::aio::InputStreamInstance(Vmergedisaname[i]));
								uint64_t const s = libmaus2::util::GetFileSize::getFileSize(*ISI);
								libmaus2::util::GetFileSize::copy(*ISI,OSI,s);
								ISI.reset();
								libmaus2::aio::FileRemoval::removeFile(Vmergedisaname[i]);
							}
						}

						std::string const outisameta = outisa + ".meta";
						libmaus2::aio::OutputStreamInstance outisametaOSI(outisameta);
						libmaus2::util::NumberSerialisation::serialiseNumber(outisametaOSI,state.getPreISASamplingRate());

						result = BwtMergeSortResult::setupBwtOnly(
							fn,
							options.outfn,
							outhist,
							outisameta,
							outisa
						);
					}
					else
					{
						if ( logstr )
							(*logstr) << "[V] computing Huffman shaped wavelet tree of final BWT...";
						std::string const outhwt = ::libmaus2::util::OutputFileNameTools::clipOff(options.outfn,".bwt") + ".hwt";
						libmaus2::wavelet::ImpCompactHuffmanWaveletTree::unique_ptr_type pICHWT;
						if ( input_types_type::utf8Wavelet() )
						{
							libmaus2::wavelet::ImpCompactHuffmanWaveletTree::unique_ptr_type tICHWT(
								libmaus2::wavelet::RlToHwtBase<true,rl_decoder>::rlToHwt(options.outfn, outhwt, options.tmpfilenamebase+"_finalhwttmp",options.numthreads)
							);
							pICHWT = std::move(tICHWT);
						}
						else
						{
							libmaus2::wavelet::ImpCompactHuffmanWaveletTree::unique_ptr_type tICHWT(
								libmaus2::wavelet::RlToHwtBase<false,rl_decoder>::rlToHwt(options.outfn, outhwt, options.tmpfilenamebase+"_finalhwttmp",options.numthreads)
							);
							pICHWT = std::move(tICHWT);
						}
						if ( logstr )
							(*logstr) << "done, " << std::endl;

						if ( logstr )
							(*logstr) << "[V] loading Huffman shaped wavelet tree of final BWT...";
						::libmaus2::lf::ImpCompactHuffmanWaveletLF IHWT(pICHWT);
						if ( logstr )
							(*logstr) << "done." << std::endl;

						// sort the sampled isa file
						uint64_t const blockmem = memperthread; // memory per thread
						// sort pre isa files by second component (position)
						std::string const mergedisaname = sortIsaFile(*gtmpgen,mergeresult.getFiles().getSampledISAVector(),blockmem,options.numthreads,options.fanin,logstr);

						// compute sampled suffix array and sampled inverse suffix array
						BwtMergeComputeSampledSA<input_types_type>::computeSampledSA(
							*gtmpgen,fn,state.getFS(),IHWT,mergedisaname,options.outfn,
							options.numthreads,lfblockmult,options.sasamplingrate,options.isasamplingrate,
							options.mem,
							options.fanin,
							logstr
						);

						// std::cerr << "[V] mergeresult.blockp0rank=" << mergeresult.blockp0rank << std::endl;

						result = BwtMergeSortResult::setupBwtSa(
							fn,
							options.outfn,
							outhist,
							outhwt,
							::libmaus2::util::OutputFileNameTools::clipOff(options.outfn,".bwt") + ".sa",
							::libmaus2::util::OutputFileNameTools::clipOff(options.outfn,".bwt") + ".isa"
						);

						libmaus2::aio::FileRemoval::removeFile(mergedisaname);

						for ( uint64_t i = 0; i < mergeresult.getFiles().getSampledISAVector().size(); ++i )
							libmaus2::aio::FileRemoval::removeFile(mergeresult.getFiles().getSampledISAVector()[i]);
					}

					#if 0
					::libmaus2::aio::FileRemoval::removeFile(chistfilename);
					::libmaus2::aio::FileRemoval::removeFile(huftreefilename);
					#endif
					libmaus2::aio::FileRemoval::removeFile(mergeresult.getFiles().getHist());

					#if 0
					std::ostringstream comstr;
					comstr << "ls -lR " << (options.tmpfilenamebase+"_tmpdir");
					system(comstr.str().c_str());
					#endif

					gtmpgen.reset();
					state.cleanup();

					return result;
				}
			};

			extern template struct BwtMergeSortTemplate< libmaus2::suffixsort::ByteInputTypes<32> >;
			extern template struct BwtMergeSortTemplate< libmaus2::suffixsort::ByteInputTypes<64> >;
			extern template struct BwtMergeSortTemplate< libmaus2::suffixsort::CompactInputTypes<32> >;
			extern template struct BwtMergeSortTemplate< libmaus2::suffixsort::CompactInputTypes<64> >;
			extern template struct BwtMergeSortTemplate< libmaus2::suffixsort::PacInputTypes<32> >;
			extern template struct BwtMergeSortTemplate< libmaus2::suffixsort::PacInputTypes<64> >;
			extern template struct BwtMergeSortTemplate< libmaus2::suffixsort::PacTermInputTypes<32> >;
			extern template struct BwtMergeSortTemplate< libmaus2::suffixsort::PacTermInputTypes<64> >;
			extern template struct BwtMergeSortTemplate< libmaus2::suffixsort::Lz4InputTypes<32> >;
			extern template struct BwtMergeSortTemplate< libmaus2::suffixsort::Lz4InputTypes<64> >;
			extern template struct BwtMergeSortTemplate< libmaus2::suffixsort::Utf8InputTypes<32> >;
			extern template struct BwtMergeSortTemplate< libmaus2::suffixsort::Utf8InputTypes<64> >;
		}
	}
}
#endif
