/*
    libmaus2
    Copyright (C) 2009-2013 German Tischler
    Copyright (C) 2011-2013 Genome Research Limited

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#if ! defined(BUFFEREDOUTPUT_HPP)
#define BUFFEREDOUTPUT_HPP

#include <libmaus2/autoarray/AutoArray.hpp>
#include <libmaus2/sorting/InPlaceParallelSort.hpp>
#include <libmaus2/util/PutObject.hpp>
#include <libmaus2/util/CountPutObject.hpp>
#include <libmaus2/util/GetObject.hpp>
#include <libmaus2/util/StreamGetObject.hpp>
#include <libmaus2/aio/CountOutputStream.hpp>
#include <libmaus2/aio/CharArrayInputStream.hpp>
#include <libmaus2/aio/CharArrayOutputStream.hpp>
#include <libmaus2/util/utf8.hpp>

namespace libmaus2
{
	namespace aio
	{
		/**
		 * base class for block buffer output
		 **/
		template<typename _data_type>
		struct BufferedOutputBase
		{
			//! data type
			typedef _data_type data_type;
			//! this type
			typedef BufferedOutputBase<data_type> this_type;
			//! unique pointer type
			typedef std::unique_ptr<this_type> unique_ptr_type;

			//! block buffer
			::libmaus2::autoarray::AutoArray<data_type> B;
			//! start of buffer pointer
			data_type * const pa;
			//! current pointer
			data_type * pc;
			//! end of buffer pointer
			data_type * const pe;

			//! total bytes written
			uint64_t totalwrittenbytes;
			//! total words written
			uint64_t totalwrittenwords;

			/**
			 * construct buffer of size bufsize
			 *
			 * @param bufsize buffer size (in elements)
			 **/
			BufferedOutputBase(uint64_t const bufsize)
			: B(bufsize,false), pa(B.begin()), pc(pa), pe(B.end()), totalwrittenbytes(0), totalwrittenwords(0)
			{

			}
			/**
			 * destructor
			 **/
			virtual ~BufferedOutputBase() {}

			/**
			 * flush buffer
			 **/
			void flush()
			{
				writeBuffer();
			}

			/**
			 * pure virtual buffer writing function
			 **/
			virtual void writeBuffer() = 0;

			/**
			 * put one element
			 *
			 * @param v element to be put in buffer
			 * @return true if buffer is full after putting the element
			 **/
			bool put(data_type const & v)
			{
				*(pc++) = v;
				if ( pc == pe )
				{
					flush();
					return true;
				}
				else
				{
					return false;
				}
			}

			/**
			 * put a list of elements
			 *
			 * @param A list start pointer
			 * @param n number of elements to be put in buffer
			 **/
			void put(data_type const * A, uint64_t n)
			{
				while ( n )
				{
					uint64_t const towrite = std::min(n,static_cast<uint64_t>(pe-pc));
					std::copy(A,A+towrite,pc);
					pc += towrite;
					A += towrite;
					n -= towrite;
					if ( pc == pe )
						flush();
				}
			}

			/**
			 * put an element without checking whether buffer fills up
			 *
			 * @param v element to be put in buffer
			 **/
			void uncheckedPut(data_type const & v)
			{
				*(pc++) = v;
			}

			/**
			 * @return space available in buffer (in elements)
			 **/
			uint64_t spaceLeft() const
			{
				return pe-pc;
			}

			/**
			 * @return number of words written (including those still in the buffer)
			 **/
			uint64_t getWrittenWords() const
			{
				return totalwrittenwords + (pc-pa);
			}

			/**
			 * @return number of bytes written (including those still in the buffer)
			 **/
			uint64_t getWrittenBytes() const
			{
				return getWrittenWords() * sizeof(data_type);
			}

			/**
			 * increase written accus by number of elements curently in buffer
			 **/
			void increaseWritten()
			{
				totalwrittenwords += (pc-pa);
				totalwrittenbytes += (pc-pa)*sizeof(data_type);
			}

			/**
			 * @return number of elements currently in buffer
			 **/
			uint64_t fill()
			{
				return pc-pa;
			}

			/**
			 * reset buffer
			 **/
			void reset()
			{
				increaseWritten();
				pc = pa;
			}
		};

		/**
		 * class for buffer file output
		 **/
		template<typename _data_type>
		struct BufferedOutput : public BufferedOutputBase<_data_type>
		{
			//! data type
			typedef _data_type data_type;
			//! this type
			typedef BufferedOutput<data_type> this_type;
			//! base class type
			typedef BufferedOutputBase<data_type> base_type;
			//! unique pointer type
			typedef std::unique_ptr<this_type> unique_ptr_type;

			private:
			std::ostream & out;

			public:
			/**
			 * constructor
			 *
			 * @param rout output stream
			 * @param bufsize size of buffer in elements
			 **/
			BufferedOutput(std::ostream & rout, uint64_t const bufsize)
			: BufferedOutputBase<_data_type>(bufsize), out(rout)
			{

			}
			/**
			 * destructor
			 **/
			~BufferedOutput()
			{
				base_type::flush();
			}

			/**
			 * write buffer to output stream
			 **/
			virtual void writeBuffer()
			{
				if ( base_type::fill() )
				{
					out.write ( reinterpret_cast<char const *>(base_type::pa), (base_type::pc-base_type::pa)*sizeof(data_type) );

					if ( ! out )
					{
						::libmaus2::exception::LibMausException se;
						se.getStream() << "Failed to write " << (base_type::pc-base_type::pa)*sizeof(data_type) << " bytes." << std::endl;
						se.finish();
						throw se;
					}

					base_type::reset();
				}
			}
		};

		/**
		 * null write output buffer. data put in buffer is written nowhere.
		 * this is useful for code directly using the buffer base.
		 **/
		template<typename _data_type>
		struct BufferedOutputNull : public BufferedOutputBase<_data_type>
		{
			//! data type
			typedef _data_type data_type;
			//! this type
			typedef BufferedOutputNull<data_type> this_type;
			//! base class type
			typedef BufferedOutputBase<data_type> base_type;
			//! unique pointer type
			typedef std::unique_ptr<this_type> unique_ptr_type;

			/**
			 * constructor
			 *
			 * @param bufsize size of buffer
			 **/
			BufferedOutputNull(uint64_t const bufsize)
			: BufferedOutputBase<_data_type>(bufsize)
			{

			}
			/**
			 * destructor
			 **/
			~BufferedOutputNull()
			{
				base_type::flush();
			}
			/**
			 * null writeBuffer function (does nothing)
			 **/
			virtual void writeBuffer()
			{
			}
		};

		/**
		 * class for sorting buffer file output
		 **/
		template<typename _data_type, typename _order_type = std::less<_data_type> >
		struct SortingBufferedOutput : public BufferedOutputBase<_data_type>
		{
			//! data type
			typedef _data_type data_type;
			//! order type
			typedef _order_type order_type;
			//! this type
			typedef SortingBufferedOutput<data_type,order_type> this_type;
			//! base class type
			typedef BufferedOutputBase<data_type> base_type;
			//! unique pointer type
			typedef std::unique_ptr<this_type> unique_ptr_type;
			//! order pointer type
			typedef std::unique_ptr<order_type> order_ptr_type;

			private:
			std::ostream & out;
			order_ptr_type Porder;
			order_type & order;
			std::vector<uint64_t> blocksizes;

			public:
			/**
			 * constructor
			 *
			 * @param rout output stream
			 * @param bufsize size of buffer in elements
			 **/
			SortingBufferedOutput(std::ostream & rout, uint64_t const bufsize)
			: BufferedOutputBase<_data_type>(bufsize), out(rout), Porder(new order_type), order(*Porder), blocksizes()
			{

			}

			/**
			 * constructor
			 *
			 * @param rout output stream
			 * @param bufsize size of buffer in elements
			 * @param rorder sort order
			 **/
			SortingBufferedOutput(std::ostream & rout, uint64_t const bufsize, order_type & rorder)
			: BufferedOutputBase<_data_type>(bufsize), out(rout), Porder(), order(rorder), blocksizes()
			{

			}

			/**
			 * destructor
			 **/
			~SortingBufferedOutput()
			{
				flush();
			}

			/**
			 * flush the buffer
			 **/
			void flush()
			{
				base_type::flush();
				out.flush();
			}

			/**
			 * write buffer to output stream
			 **/
			virtual void writeBuffer()
			{
				if ( base_type::fill() )
				{
					std::sort(base_type::pa,base_type::pc,order);

					out.write ( reinterpret_cast<char const *>(base_type::pa), base_type::fill()*sizeof(data_type) );

					if ( ! out )
					{
						::libmaus2::exception::LibMausException se;
						se.getStream() << "Failed to write " << base_type::fill()*sizeof(data_type) << " bytes." << std::endl;
						se.finish();
						throw se;
					}

					blocksizes.push_back(base_type::fill());

					base_type::reset();
				}
			}

			/**
			 * get block sizes (number of elements written in each block)
			 *
			 * @return block size vector
			 **/
			std::vector<uint64_t> const & getBlockSizes() const
			{
				return blocksizes;
			}
		};

		struct QuickSerialisedOrder {};
		struct MemorySerialisedOrder {};

		template<typename _data_type, typename _order_type = std::less<_data_type> >
		struct SerialisingBuffer
		{
			//! data type
			typedef _data_type data_type;
			//! order type
			typedef _order_type order_type;
			//!
			typedef SerialisingBuffer<data_type,order_type> this_type;

			std::ostream & out;
			libmaus2::autoarray::AutoArray<char *> B;
			order_type & order;

			// data pointers, filled from back
			char ** pe;
			char ** pc;

			// data, filled from front
			char * ca;
			char * cc;

			// number of data bytes written so far
			uint64_t w;

			// number of threads used for sorting
			uint64_t threads;

			/**
			 * comparator class
			 **/
			struct Comparator
			{
				order_type * order;

				Comparator(order_type & rorder)
				: order(&rorder)
				{

				}

				template
				<
					typename order_type,
					typename std::enable_if
						<
							std::is_base_of<QuickSerialisedOrder,order_type>::value,
							int
						>::type
						=
						0
				>
				bool compare(
					std::istream & GA,
					std::istream & GB,
					order_type & order
				) const
				{
					return order(GA,GB);
				}

				template
				<
					typename order_type,
					typename std::enable_if
						<
							!std::is_base_of<QuickSerialisedOrder,order_type>::value,
							int
						>::type
						=
						0
				>
				bool compare(
					std::istream & GA,
					std::istream & GB,
					order_type & order
				) const
				{
					data_type A;
					data_type B;
					A.deserialise(GA);
					B.deserialise(GB);
					return order(A,B);
				}

				template
				<
					typename order_type,
					typename std::enable_if
						<
							!std::is_base_of<MemorySerialisedOrder,order_type>::value,
							int
						>::type
						=
						0
				>
				bool compare(char * a, char * b) const
				{

					libmaus2::util::StreamGetObject<char const *> LA(a);
					libmaus2::util::StreamGetObject<char const *> LB(b);

					// get length of serialised objects
					uint64_t const utfa = libmaus2::util::UTF8::decodeUTF8Unchecked(LA);
					uint64_t const utfb = libmaus2::util::UTF8::decodeUTF8Unchecked(LB);

					// get size of length codes
					uint64_t const lutfa = libmaus2::util::UTF8::lengthUTF8(utfa);
					uint64_t const lutfb = libmaus2::util::UTF8::lengthUTF8(utfb);

					libmaus2::aio::CharArrayInputStream GA(a+lutfa,a+lutfa+utfa);
					libmaus2::aio::CharArrayInputStream GB(b+lutfb,b+lutfb+utfb);

					return compare<order_type>(GA,GB,*order);
				}

				template
				<
					typename order_type,
					typename std::enable_if
						<
							std::is_base_of<MemorySerialisedOrder,order_type>::value,
							int
						>::type
						=
						0
				>
				bool compare(char * a, char * b) const
				{

					libmaus2::util::StreamGetObject<char const *> LA(a);
					libmaus2::util::StreamGetObject<char const *> LB(b);

					// get length of serialised objects
					uint64_t const utfa = libmaus2::util::UTF8::decodeUTF8Unchecked(LA);
					uint64_t const utfb = libmaus2::util::UTF8::decodeUTF8Unchecked(LB);

					// get size of length codes
					uint64_t const lutfa = libmaus2::util::UTF8::lengthUTF8(utfa);
					uint64_t const lutfb = libmaus2::util::UTF8::lengthUTF8(utfb);

					return (*order)(a+lutfa,a+lutfa+utfa,b+lutfb,b+lutfb+utfb);
				}

				bool operator()(char * a, char * b) const
				{
					return compare<order_type>(a,b);
				}
			};

			Comparator comp;

			/**
			 * set up buffer for size
			 **/
			void setupBuffer(uint64_t const nbufsize)
			{
				uint64_t const abufsize = (nbufsize + sizeof(char *) - 1) / sizeof(char *);
				B = libmaus2::autoarray::AutoArray<char *>();
				B = libmaus2::autoarray::AutoArray<char *>(abufsize,false);

				pe = B.end(),
				pc = pe;

				ca = reinterpret_cast<char *>(B.begin());
				cc = ca;
			}

			/**
			 * increase buffer size
			 **/
			void bumpBuffer()
			{
				uint64_t obufsize = B.size() * sizeof(char *);
				setupBuffer(obufsize ? (2*obufsize) : 1);
			}

			SerialisingBuffer(
				std::ostream & rout,
				uint64_t const rbufsize,
				order_type & rorder,
				uint64_t const rthreads
			) : out(rout), order(rorder), w(0), threads(rthreads), comp(order)
			{
				setupBuffer(rbufsize);
			}

			/**
			 * flush buffer, return number of elements flushed and position of flushed block in output stream
			 **/
			std::pair<uint64_t,uint64_t> flush()
			{
				// number of bytes written before
				uint64_t const pre_w = w;
				// number of elements in buffer
				uint64_t const n = pe-pc;

				if ( threads <= 1 )
					std::sort(pc,pc+n,comp);
				else
					libmaus2::sorting::InPlaceParallelSort::inplacesort2<
						char **,Comparator
					>(
						pc,pc+n,
						threads,
						comp
					);

				// write sorted elements
				data_type D;
				for ( uint64_t i = 0; i < n; ++i )
				{
					// decode length of entry
					libmaus2::util::StreamGetObject<char const *> LA(pc[i]);

					uint64_t const utf = libmaus2::util::UTF8::decodeUTF8Unchecked(LA);
					// get length of utf-8 code
					uint64_t const lutf = libmaus2::util::UTF8::lengthUTF8(utf);

					// set up stream for decoding object
					libmaus2::aio::CharArrayInputStream GA(pc[i]+lutf,pc[i]+lutf+utf);

					// decode object
					D.deserialise(GA);
					assert ( GA );

					// encode object
					D.serialise(out);

					// add size of object
					w += utf;
				}

				// reset buffer pointers
				pc = pe;
				cc = ca;

				return std::pair<uint64_t,uint64_t>(n,pre_w);
			}

			/**
			 * put one object, return number of flushed elements and file position of flush block
			 **/
			std::pair<uint64_t,uint64_t> put(data_type const & D)
			{
				// check size of object
				libmaus2::aio::CountOutputStream CPO;
				D.serialise(CPO);
				CPO.flush();

				// get length of object size in utf8
				uint64_t const utfsize = libmaus2::util::UTF8::lengthUTF8(CPO.c);

				// available space
				uint64_t spaceav = reinterpret_cast<char const *>(pc) - cc;
				// required space
				uint64_t const spacereq = CPO.c + utfsize + sizeof(char *);

				// empty flush info
				std::pair<uint64_t,uint64_t> flushInfo(0,w);

				// if space is insufficient
				if ( spacereq > spaceav )
				{
					// flush buffer
					flushInfo = flush();
					// recompute free space
					spaceav = reinterpret_cast<char const *>(pc) - cc;

					// while buffer is too small to hold element
					while ( spacereq > spaceav )
					{
						assert ( pc == pe );
						assert ( cc == ca );
						bumpBuffer();
						spaceav = reinterpret_cast<char const *>(pc) - cc;
					}
				}

				// output stream
				libmaus2::aio::CharArrayOutputStream PO(cc,cc+utfsize+CPO.c);

				// encode length as UTF-8
				libmaus2::util::UTF8::encodeUTF8(CPO.c,PO);
				// encode object
				D.serialise(PO);

				assert ( PO );

				// store pointer
				*--pc = cc;
				// move data pointer
				cc += CPO.c + utfsize;

				return flushInfo;
			}
		};

		/**
		 * class for sorting buffer file output
		 **/
		template<typename _data_type, typename _order_type = std::less<_data_type> >
		struct SerialisingSortingBufferedOutput
		{
			//! data type
			typedef _data_type data_type;
			//! order type
			typedef _order_type order_type;
			//! this type
			typedef SerialisingSortingBufferedOutput<data_type,order_type> this_type;
			//! base class type
			typedef BufferedOutputBase<data_type> base_type;
			//! unique pointer type
			typedef std::unique_ptr<this_type> unique_ptr_type;
			//! order pointer type
			typedef std::unique_ptr<order_type> order_ptr_type;

			struct BlockDescriptor
			{
				uint64_t numel;
				uint64_t fileptr;

				BlockDescriptor() {}
				BlockDescriptor(uint64_t const rnumel, uint64_t const rfileptr) : numel(rnumel), fileptr(rfileptr) {}
			};

			private:
			std::ostream & out;
			order_ptr_type Porder;
			order_type & order;
			std::vector<BlockDescriptor> blocksizes;
			uint64_t const numthreads;
			SerialisingBuffer<data_type,order_type> B;

			public:
			/**
			 * constructor
			 *
			 * @param rout output stream
			 * @param bufsize size of buffer in elements
			 **/
			SerialisingSortingBufferedOutput(std::ostream & rout, uint64_t const bufsize, uint64_t const rnumthreads = 1)
			: out(rout), Porder(new order_type), order(*Porder), blocksizes(), numthreads(rnumthreads), B(out,bufsize,order,numthreads)
			{

			}

			/**
			 * constructor
			 *
			 * @param rout output stream
			 * @param bufsize size of buffer in elements
			 * @param rorder sort order
			 **/
			SerialisingSortingBufferedOutput(std::ostream & rout, uint64_t const bufsize, order_type & rorder, uint64_t const rnumthreads = 1)
			: out(rout), Porder(), order(rorder), blocksizes(), numthreads(rnumthreads), B(out,bufsize,order,numthreads)
			{

			}

			/**
			 * destructor
			 **/
			~SerialisingSortingBufferedOutput()
			{
				flush();
			}

			/**
			 * flush the buffer
			 **/
			void flush()
			{
				std::pair<uint64_t,uint64_t> const flushInfo = B.flush();

				if ( flushInfo.first )
					blocksizes.push_back(BlockDescriptor(flushInfo.first,flushInfo.second));
			}

			void put(data_type const & D)
			{
				std::pair<uint64_t,uint64_t> const flushInfo = B.put(D);

				if ( flushInfo.first )
					blocksizes.push_back(BlockDescriptor(flushInfo.first,flushInfo.second));
			}

			/**
			 * get block sizes (number of elements written in each block)
			 *
			 * @return block size vector
			 **/
			std::vector<BlockDescriptor> const & getBlockSizes() const
			{
				return blocksizes;
			}
		};
	}
}
#endif
