/*
    libmaus2
    Copyright (C) 2009-2015 German Tischler
    Copyright (C) 2011-2015 Genome Research Limited

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
#if ! defined(LIBMAUS2_INDEX_EXTERNALMEMORYINDEXGENERATOR_HPP)
#define LIBMAUS2_INDEX_EXTERNALMEMORYINDEXGENERATOR_HPP

#include <libmaus2/util/NumberSerialisation.hpp>
#include <libmaus2/util/PrefixSums.hpp>
#include <libmaus2/util/Concat.hpp>
#include <libmaus2/util/TempFileRemovalContainer.hpp>
#include <libmaus2/autoarray/AutoArray.hpp>
#include <libmaus2/index/ExternalMemoryIndexRecord.hpp>
#include <libmaus2/aio/OutputStreamInstance.hpp>
#include <iomanip>

namespace libmaus2
{
	namespace index
	{
		template<typename _data_type, unsigned int _base_level_log, unsigned int _inner_level_log>
		struct ExternalMemoryIndexGenerator
		{
			typedef _data_type data_type;
			static unsigned int const base_level_log = _base_level_log;
			static unsigned int const inner_level_log = _inner_level_log;
			typedef ExternalMemoryIndexGenerator<data_type,base_level_log,inner_level_log> this_type;
			typedef std::unique_ptr<this_type> unique_ptr_type;
			typedef std::shared_ptr<this_type> shared_ptr_type;

			static uint64_t const base_index_step = 1ull << base_level_log;
			static uint64_t const base_index_mask = (base_index_step-1);

			static uint64_t const inner_index_step = 1ull << inner_level_log;
			static uint64_t const inner_index_mask = (inner_index_step-1);

			std::string tmpprefix;
			libmaus2::aio::OutputStreamInstance::unique_ptr_type tmp0;
			std::vector<uint64_t> tmpcnt;

			libmaus2::aio::OutputStreamInstance::unique_ptr_type Pstream;
			std::ostream & stream;
			bool flushed;

			typedef ExternalMemoryIndexRecord<data_type> record_type;

			size_t byteSize() const
			{
				return
					sizeof(Pstream) +
					sizeof(flushed);
			}

			std::string getTmpFileName(uint64_t const index)
			{
				std::ostringstream ostr;
				assert ( tmpprefix.size() );
				ostr << tmpprefix << "_" << index << ".ext";
				std::string const fn = ostr.str();
				libmaus2::util::TempFileRemovalContainer::addTempFile(fn);
				return fn;
			}

			libmaus2::aio::OutputStreamInstance::unique_ptr_type openTmpFileForOutput(
				uint64_t const index
			)
			{
				std::string const fn = getTmpFileName(index);
				libmaus2::aio::OutputStreamInstance::unique_ptr_type ptr(new libmaus2::aio::OutputStreamInstance(fn));
				return ptr;
			}

			libmaus2::aio::InputStreamInstance::unique_ptr_type openTmpFileForInput(
				uint64_t const index
			)
			{
				std::string const fn = getTmpFileName(index);
				libmaus2::aio::InputStreamInstance::unique_ptr_type ptr(new libmaus2::aio::InputStreamInstance(fn));
				return ptr;
			}

			ExternalMemoryIndexGenerator(std::string const & filename, std::string const & rtmpprefix)
			:
			  tmpprefix(rtmpprefix),
			  tmp0(openTmpFileForOutput(0)),
			  tmpcnt(1,0),
			  Pstream(new libmaus2::aio::OutputStreamInstance(filename)), stream(*Pstream),
			  flushed(false)
			{

			}

			ExternalMemoryIndexGenerator(std::ostream & rstream, std::string const & rtmpprefix)
			:
			  tmpprefix(rtmpprefix),
			  tmp0(openTmpFileForOutput(0)),
			  tmpcnt(1,0),
			  Pstream(), stream(rstream), flushed(false)
			{

			}

			uint64_t setup()
			{
				uint64_t const curpos = stream.tellp();

				flushed = false;

				auto rtmp0(openTmpFileForOutput(0));
				tmp0 = std::move(rtmp0);
				tmpcnt = std::vector<uint64_t>(1,0);
				assert ( tmpcnt.size() == 1 );
				assert ( tmpcnt[0] == 0 );

				return curpos;
			}

			uint64_t flush()
			{
				if ( ! flushed )
				{
					uint64_t const object_size = data_type::getSerialisedObjectSize();
					uint64_t const record_size = 2*sizeof(uint64_t)+object_size;

					assert ( tmp0 );

					tmp0->flush();
					tmp0.reset();

					for ( uint64_t level = 0; tmpcnt[level] > inner_index_step; ++level )
					{
						libmaus2::aio::InputStreamInstance::unique_ptr_type ISI(openTmpFileForInput(level+0));
						libmaus2::aio::OutputStreamInstance::unique_ptr_type OSI(openTmpFileForOutput(level+1));

						uint64_t const incnt = tmpcnt[level];
						uint64_t outcnt = 0;
						uint64_t const expoutcnt = (incnt + inner_index_step-1)/inner_index_step;

						data_type D;
						for ( uint64_t j = 0; j < incnt; ++j )
						{
							uint64_t pfirst = libmaus2::util::NumberSerialisation::deserialiseNumber(*ISI);
							uint64_t psecond = libmaus2::util::NumberSerialisation::deserialiseNumber(*ISI);
							D.deserialise(*ISI);

							if ( (j & inner_index_mask) == 0 )
							{
								libmaus2::util::NumberSerialisation::serialiseNumber(*OSI,pfirst);
								libmaus2::util::NumberSerialisation::serialiseNumber(*OSI,psecond);
								D.serialise(*OSI);
								++outcnt;
							}
						}

						assert( ISI->peek() == std::istream::traits_type::eof());
						assert( outcnt == expoutcnt );

						std::streampos const OSIpos = OSI->tellp();
						std::streampos const exppos = outcnt * record_size;

						if ( OSIpos != exppos )
						{
							libmaus2::exception::LibMausException lme;
							lme.getStream() << "[E] ExternalMemoryIndexGenerator::flush: writing layer files OSIpos=" << OSIpos << " exppos=" << exppos << std::endl;
							lme.finish();
							throw lme;
						}

						OSI->flush();
						OSI.reset();
						ISI.reset();

						tmpcnt.push_back(outcnt);
					}

					uint64_t const numlevels = tmpcnt.size();

					std::vector<uint64_t> tmpoff(tmpcnt);
					// add value for holding the total sum
					tmpoff.push_back(0);
					libmaus2::util::PrefixSums::prefixSums(tmpoff.begin(),tmpoff.end());

					for ( uint64_t i = 0; i < tmpoff.size(); ++i )
						tmpoff[i] *= record_size;

					uint64_t const indexsize =
						sizeof(uint64_t) + // pointer to end
						tmpoff.back() + // sum of level lengths
						numlevels * sizeof(uint64_t) * 2 + // start,cnt pairs
						sizeof(uint64_t); // number of levels

					/**
					 * file format
					 *
					 * - pointer to end of index
					 * - level 0
					 * - level 1
					 * - ...
					 * - (levelstart,levelcnt) for level 0
					 * - (levelstart,levelcnt) for level 1
					 * - ...
					 * - number of levels
					 **/

					uint64_t const curp = stream.tellp();

					libmaus2::util::NumberSerialisation::serialiseNumber(stream,curp + indexsize);

					for ( uint64_t level = 0; level < numlevels; ++level )
					{
						libmaus2::aio::InputStreamInstance::unique_ptr_type ISI(openTmpFileForInput(level));
						libmaus2::util::Concat::concat(*ISI,stream);
						ISI.reset();
						libmaus2::aio::FileRemoval::removeFile(getTmpFileName(level));
					}
					for ( uint64_t level = 0; level < numlevels; ++level )
					{
						libmaus2::util::NumberSerialisation::serialiseNumber(stream,curp + tmpoff[level] + sizeof(uint64_t));
						libmaus2::util::NumberSerialisation::serialiseNumber(stream,tmpcnt[level]);
					}
					libmaus2::util::NumberSerialisation::serialiseNumber(stream,numlevels);

					std::streampos const OSIpos = stream.tellp();
					std::streampos const exppos = curp + indexsize;

					if ( OSIpos != exppos )
					{
						libmaus2::exception::LibMausException lme;
						lme.getStream() << "[E] ExternalMemoryIndexGenerator::flush: output stream OSIpos=" << OSIpos << " exppos=" << exppos << "curp=" << curp << " numlevels=" << numlevels << std::endl;
						lme.finish();
						throw lme;
					}

					// stream.flush();

					flushed = true;

					return stream.tellp();
				}
				else
				{
					libmaus2::exception::LibMausException lme;
					lme.getStream() << "ExternalMemoryIndexGenerator::flush(): generator is already flushed" << std::endl;
					lme.finish();
					throw lme;
				}
			}

			void put(data_type const & E, std::pair<uint64_t,uint64_t> const & P)
			{
				assert ( tmp0 );

				libmaus2::util::NumberSerialisation::serialiseNumber(*tmp0,P.first);
				libmaus2::util::NumberSerialisation::serialiseNumber(*tmp0,P.second);
				E.serialise(*tmp0);
				tmpcnt[0]++;
			}
		};
	}
}
#endif
