/*
    libmaus2
    Copyright (C) 2020 German Tischler

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
#if ! defined(LIBMAUS2_PARALLEL_THREADPOOL_INPUT_THREADPOOLINPUT_HPP)
#define LIBMAUS2_PARALLEL_THREADPOOL_INPUT_THREADPOOLINPUT_HPP

#include <libmaus2/aio/InputStreamInstance.hpp>
#include <libmaus2/autoarray/AutoArray.hpp>
#include <libmaus2/parallel/threadpool/ThreadPool.hpp>
#include <libmaus2/timing/RealTimeClock.hpp>
#include <cstdlib>

namespace libmaus2
{
	namespace parallel
	{
		namespace threadpool
		{
			namespace input
			{
				struct ThreadPoolInputBlock
				{
					typedef ThreadPoolInputBlock this_type;
					typedef std::unique_ptr<this_type> unique_ptr_type;
					typedef std::shared_ptr<this_type> shared_ptr_type;

					// block size
					std::atomic<uint64_t> blocksize;
					// size of putback area
					std::atomic<uint64_t> putback;
					// block memory (size putback + blocksize)
					libmaus2::autoarray::AutoArray<char> A;
					std::atomic<char *> pa;
					std::atomic<char *> pe;
					// start of memory block used
					std::atomic<char *> ca;
					// end of memory block used
					std::atomic<char *> ce;
					// id of block
					std::atomic<uint64_t> blockid;
					// true if this is the last input block
					std::atomic<bool> eof;

					ThreadPoolInputBlock(uint64_t const rblocksize, uint64_t const rputback)
					: blocksize(rblocksize), putback(rputback), A(blocksize+putback,false), pa(A.begin()), pe(A.end()), ca(A.end()), ce(A.end()), blockid(0), eof(false)
					{

					}

					// increase putback buffer size
					void bumpPutBack()
					{
						// pointer offsets
						off_t const off_ca = pe-ca;
						off_t const off_ce = pe-ce;
						uint64_t const one = 1;
						uint64_t const oldsize = blocksize+putback;
						putback = std::max(one,putback<<1);

						// allocate new buffer
						libmaus2::autoarray::AutoArray<char> B(blocksize+putback,false);
						// copy data
						std::copy(pa.load(),pe.load(),B.end()-oldsize);

						// put new values
						A = B;
						pa = A.begin();
						pe = A.end();
						ca = A.end()-off_ca;
						ce = A.end()-off_ce;
					}
				};
			}
		}
	}
}

namespace libmaus2
{
	namespace parallel
	{
		namespace threadpool
		{
			namespace input
			{
				struct ThreadPoolInputInfo
				{
					typedef ThreadPoolInputInfo this_type;
					typedef std::unique_ptr<this_type> unique_ptr_type;
					typedef std::shared_ptr<this_type> shared_ptr_type;

					// input stream pointer
					libmaus2::aio::InputStreamInstance::shared_ptr_type pISI;
					// input stream
					std::istream & in;
					libmaus2::parallel::threadpool::ThreadPoolStack<ThreadPoolInputBlock::shared_ptr_type> freeBlocks;
					libmaus2::parallel::threadpool::ThreadPoolStack<ThreadPoolInputBlock::shared_ptr_type> returnedBlocks;
					// stream id
					uint64_t const streamid;
					// total number of blocks
					uint64_t const numblocks;
					// minimum number of free blocks before reading starts
					uint64_t const blocksMinFree;
					std::atomic<uint64_t> blocksRead;
					std::atomic<uint64_t> blocksFree;
					std::atomic<bool> eof;
					std::mutex lock;
					libmaus2::parallel::threadpool::ThreadPoolQueue<ThreadPoolInputBlock::shared_ptr_type> outQueue;
					std::mutex outQueueReadLock;
					std::atomic<uint64_t> bytesRead;
					libmaus2::timing::RealTimeClock rtc;
					std::atomic<bool> rtcSet;

					void setup(uint64_t const numblocks, uint64_t const blocksize, uint64_t const putbacksize)
					{
						for ( uint64_t i = 0; i < numblocks; ++i )
						{
							ThreadPoolInputBlock::shared_ptr_type ptr(new ThreadPoolInputBlock(blocksize,putbacksize));
							freeBlocks.push(ptr);
							++blocksFree;
						}
					}

					double getSpeed() const
					{
						double const sec = rtc.getElapsedSeconds();
						uint64_t const bytes = bytesRead.load();
						return bytes / sec;
					}

					static uint64_t getDiv()
					{
						return 8;
					}

					ThreadPoolInputInfo(std::istream & rin, uint64_t const rstreamid, uint64_t const rnumblocks, uint64_t const blocksize, uint64_t const putbacksize)
					: pISI(), in(rin), freeBlocks(), returnedBlocks(),  streamid(rstreamid), numblocks(rnumblocks), blocksMinFree(std::max(static_cast<uint64_t>(1),numblocks/getDiv())), blocksRead(0), blocksFree(0), eof(false), lock(), outQueue(), outQueueReadLock(), bytesRead(0), rtc(), rtcSet(false)
					{
						setup(numblocks,blocksize,putbacksize);
					}

					ThreadPoolInputInfo(std::string const & rfn, uint64_t const rstreamid, uint64_t const rnumblocks, uint64_t const blocksize, uint64_t const putbacksize)
					: pISI(new libmaus2::aio::InputStreamInstance(rfn)), in(*pISI), freeBlocks(), returnedBlocks(),  streamid(rstreamid), numblocks(rnumblocks), blocksMinFree(std::max(static_cast<uint64_t>(1),numblocks/getDiv())), blocksRead(0), blocksFree(0), eof(false), lock(), outQueue(), outQueueReadLock(), bytesRead(0), rtc(), rtcSet(false)
					{
						setup(numblocks,blocksize,putbacksize);
					}
				};
			}
		}
	}
}

namespace libmaus2
{
	namespace parallel
	{
		namespace threadpool
		{
			namespace input
			{

				struct InputThreadCallbackInterface
				{
					virtual ~InputThreadCallbackInterface() {}
					virtual void inputThreadCallbackInterfaceBlockRead(ThreadPoolInputInfo * inputinfo) = 0;
				};
			}
		}
	}
}

namespace libmaus2
{
	namespace parallel
	{
		namespace threadpool
		{
			namespace input
			{

				struct InputThreadPackageData : public libmaus2::parallel::threadpool::ThreadWorkPackageData
				{
					typedef InputThreadPackageData this_type;
					typedef std::unique_ptr<this_type> unique_ptr_type;
					typedef std::shared_ptr<this_type> shared_ptr_type;

					ThreadPoolInputInfo * inputinfo;
					InputThreadCallbackInterface * callback;

					InputThreadPackageData() : libmaus2::parallel::threadpool::ThreadWorkPackageData(), inputinfo(nullptr), callback(nullptr)
					{

					}
				};

			}
		}
	}
}

namespace libmaus2
{
	namespace parallel
	{
		namespace threadpool
		{
			namespace input
			{
				struct InputThreadPackageDispatcher : public libmaus2::parallel::threadpool::ThreadPoolDispatcher
				{
					virtual void dispatch(libmaus2::parallel::threadpool::ThreadWorkPackage & package, libmaus2::parallel::threadpool::ThreadPool * /* pool */)
					{
						InputThreadPackageData & HP = dynamic_cast<InputThreadPackageData &>(*(package.data));
						ThreadPoolInputInfo & inputinfo = *(HP.inputinfo);
						InputThreadCallbackInterface & callback = *(HP.callback);

						uint64_t pushed = 0;

						// if we can get the lock
						if ( inputinfo.lock.try_lock() )
						{
							// adopt the lock in unique_lock
							std::unique_lock<std::mutex> slock(inputinfo.lock,std::adopt_lock_t());

							// put returned blocks into free list
							ThreadPoolInputBlock::shared_ptr_type ptr;
							while ( inputinfo.returnedBlocks.pop(ptr) )
							{
								inputinfo.freeBlocks.push(ptr);
								++inputinfo.blocksFree;
							}

							// get number of free blocks
							uint64_t const lnumfree = inputinfo.blocksFree.load();

							// if number of free blocks is at least blocksMinFree
							if ( lnumfree >= inputinfo.blocksMinFree )
							{
								// set time before getting first block
								if ( ! inputinfo.rtcSet.load() )
								{
									inputinfo.rtc.start();
									inputinfo.rtcSet.store(true);
								}

								// read while we have free blocks and not received end of file
								while ( inputinfo.freeBlocks.pop(ptr) && !inputinfo.eof.load() )
								{
									// decrement number of free blocks
									--inputinfo.blocksFree;

									// get pointer to start of data
									char * const ca = ptr->A.begin()+ptr->putback;
									// read block
									inputinfo.in.read(ca,ptr->blocksize);
									// get number of bytes we read
									std::size_t const n = inputinfo.in.gcount();
									// set end of data pointer
									char * const ce = ca + n;

									// set data in block structure
									ptr->ca = ca;
									ptr->ce = ce;
									ptr->blockid = inputinfo.blocksRead++;

									// increase total number of bytes read so far
									inputinfo.bytesRead += n;

									// check whether we reached the end of the file
									bool const eof = inputinfo.in.peek() == std::istream::traits_type::eof();

									// store eof if we have reached it
									if ( eof )
										inputinfo.eof.store(true);
									ptr->eof.store(eof);

									assert ( HP.callback );

									// put block in output queue
									inputinfo.outQueue.push(ptr);

									// increment number of blocks read
									++pushed;
								}
							}
						}

						// invoke callback if we have read any blocks
						if ( pushed )
							callback.inputThreadCallbackInterfaceBlockRead(HP.inputinfo);
					}
				};
			}
		}
	}
}

namespace libmaus2
{
	namespace parallel
	{
		namespace threadpool
		{
			namespace input
			{
				struct InputThreadControl
				{
					libmaus2::parallel::threadpool::ThreadPool & TP;
					std::vector< std::shared_ptr<std::istream> > Vstr;
					std::vector<ThreadPoolInputInfo::shared_ptr_type> Vin;
					uint64_t const numstreams;
					std::vector< std::shared_ptr< std::atomic<int> > > VstreamFinished;
					std::atomic<uint64_t> streamsFinished;
					std::vector<InputThreadCallbackInterface *> Viface;
					uint64_t const basepriority;
					uint64_t const modpriority;

					void registerDispatcher()
					{
						TP.registerDispatcher<libmaus2::parallel::threadpool::input::InputThreadPackageData>(
							std::shared_ptr<libmaus2::parallel::threadpool::ThreadPoolDispatcher>(
								new libmaus2::parallel::threadpool::input::InputThreadPackageDispatcher
							)
						);
					}


					InputThreadControl(
						libmaus2::parallel::threadpool::ThreadPool & rTP,
						std::vector< std::shared_ptr<std::istream> > rVstr,
						uint64_t const numblocks,
						uint64_t const blocksize,
						uint64_t const putbacksize,
						uint64_t const rbasepriority,
						uint64_t const rmodpriority
					)
					: TP(rTP), Vstr(rVstr), Vin(Vstr.size()), numstreams(Vin.size()), VstreamFinished(Vstr.size()), streamsFinished(0), Viface(), basepriority(rbasepriority), modpriority(rmodpriority)
					{
						registerDispatcher();

						for ( uint64_t i = 0; i < Vstr.size(); ++i )
						{
							ThreadPoolInputInfo::shared_ptr_type ptr(new ThreadPoolInputInfo(*Vstr[i],i,numblocks,blocksize,putbacksize));
							Vin[i] = ptr;
							VstreamFinished[i] = std::shared_ptr< std::atomic<int> >(new std::atomic<int>(0));
						}
					}

					InputThreadControl(
						libmaus2::parallel::threadpool::ThreadPool & rTP,
						std::vector<std::string> Vfn,
						uint64_t const numblocks,
						uint64_t const blocksize,
						uint64_t const putbacksize,
						uint64_t const rbasepriority,
						uint64_t const rmodpriority
					)
					: TP(rTP), Vstr(), Vin(), numstreams(Vfn.size()), VstreamFinished(Vfn.size()), streamsFinished(0), Viface(), basepriority(rbasepriority), modpriority(rmodpriority)
					{
						registerDispatcher();

						for ( uint64_t i = 0; i < Vfn.size(); ++i )
						{
							ThreadPoolInputInfo::shared_ptr_type ptr(new ThreadPoolInputInfo(Vfn[i],i,numblocks,blocksize,putbacksize));
							Vin.push_back(ptr);
							VstreamFinished[i] = std::shared_ptr< std::atomic<int> >(new std::atomic<int>(0));
						}
					}

					void setCallback(
						std::vector<InputThreadCallbackInterface *> rViface
					)
					{
						Viface = rViface;
					}

					bool readFinished()
					{
						return streamsFinished.load() == numstreams;
					}

					bool readFinished(uint64_t const streamid)
					{
						return VstreamFinished[streamid]->load();
					}

					void start()
					{
						for ( uint64_t i = 0; i < Vin.size(); ++i )
						{
							libmaus2::parallel::threadpool::ThreadWorkPackage::shared_ptr_type pack(TP.getPackage<InputThreadPackageData>());
							InputThreadPackageData & data = dynamic_cast<InputThreadPackageData &>(*(pack->data));
							data.inputinfo = Vin[i].get();
							data.callback = Viface[i];
							pack->prio = basepriority + ::std::rand()%modpriority;
							TP.enqueue(pack);
						}
					}

					bool getBlocks(
						std::vector<ThreadPoolInputBlock::shared_ptr_type> & V,
						ThreadPoolInputInfo * inputinfo
					)
					{
						std::lock_guard<std::mutex> slock(inputinfo->outQueueReadLock);
						ThreadPoolInputBlock::shared_ptr_type block;
						bool eof = false;
						while ( inputinfo->outQueue.pop(block) )
						{
							assert ( block );
							eof = eof || block->eof;
							V.push_back(block);
						}

						return eof;
					}

					void returnBlock(ThreadPoolInputInfo * inputinfo, ThreadPoolInputBlock::shared_ptr_type block)
					{
						bool const eof = block->eof.load();
						inputinfo->returnedBlocks.push(block);

						if ( eof )
						{
							++streamsFinished;
							VstreamFinished[inputinfo->streamid]->store(1);
						}
						else
						{
							libmaus2::parallel::threadpool::ThreadWorkPackage::shared_ptr_type pack(TP.getPackage<InputThreadPackageData>());
							InputThreadPackageData & data = dynamic_cast<InputThreadPackageData &>(*(pack->data));
							data.inputinfo = inputinfo;
							data.callback = Viface[inputinfo->streamid];
							pack->prio = basepriority + ::std::rand()%modpriority;
							TP.enqueue(pack);
						}
					}
				};

			}
		}
	}
}

namespace libmaus2
{
	namespace parallel
	{
		namespace threadpool
		{
			namespace input
			{
				template<typename _lock_type>
				struct InputThreadTrivialReturnInterface : public InputThreadCallbackInterface
				{
					typedef _lock_type lock_type;
					libmaus2::parallel::threadpool::ThreadPool & TP;
					InputThreadControl & IC;
					lock_type & lock;
					std::ostream & errstr;

					InputThreadTrivialReturnInterface(
						libmaus2::parallel::threadpool::ThreadPool & rTP,
						InputThreadControl & rIC,
						lock_type & rlock,
						std::ostream & rerrstr
					)
					: TP(rTP), IC(rIC), lock(rlock), errstr(rerrstr)
					{
					}

					virtual void inputThreadCallbackInterfaceBlockRead(ThreadPoolInputInfo * inputinfo)
					{
						std::vector<ThreadPoolInputBlock::shared_ptr_type> V;
						/* bool const eof = */ IC.getBlocks(V,inputinfo);

						uint64_t batchsize = V.size();

						{
							typename lock_type::scope_lock_type slock(lock);
							errstr << "[V] got batch of size " << batchsize << " for stream " << inputinfo->streamid
								<< " bytes read " << inputinfo->bytesRead.load()
								<< " speed " << inputinfo->getSpeed()/(1024.0*1024.0)
								<< std::endl;
						}

						for ( uint64_t i = 0; i < V.size(); ++i )
							IC.returnBlock(inputinfo,V[i]);

						if ( IC.readFinished() )
							TP.terminate();
					}
				};
			}
		}
	}
}
#endif
