// ===========================================================================
//
//                            PUBLIC DOMAIN NOTICE
//            National Center for Biotechnology Information (NCBI)
//
//  This software/database is a "United States Government Work" under the
//  terms of the United States Copyright Act. It was written as part of
//  the author's official duties as a United States Government employee and
//  thus cannot be copyrighted. This software/database is freely available
//  to the public for use. The National Library of Medicine and the U.S.
//  Government do not place any restriction on its use or reproduction.
//  We would, however, appreciate having the NCBI and the author cited in
//  any work or product based on this material.
//
//  Although all reasonable efforts have been taken to ensure the accuracy
//  and reliability of the software and data, the NLM and the U.S.
//  Government do not and cannot warrant the performance or results that
//  may be obtained by using this software or data. The NLM and the U.S.
//  Government disclaim all warranties, express or implied, including
//  warranties of performance, merchantability or fitness for any particular
//  purpose.
//
// ===========================================================================
//
// File Name:  rchive.go
//
// Author:  Jonathan Kans
//
// ==========================================================================

package main

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"container/heap"
	"encoding/binary"
	"fmt"
	"github.com/fiam/gounidecode/unidecode"
	"github.com/surgebase/porter2"
	"hash/crc32"
	"html"
	"io"
	"io/ioutil"
	"os"
	"os/user"
	"path"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"runtime/pprof"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
	"unicode"
)

// RCHIVE VERSION AND HELP MESSAGE TEXT

const rchiveVersion = "7.70"

const rchiveHelp = `
Processing Flags

  -strict     Remove HTML highlight tags
  -mixed      Allow PubMed mixed content

Data Source

  -input      Read XML from file instead of stdin

Local Record Indexing

  -archive    Base path for individual XML files
  -index      Use [parent/element@attribute^version] for identifier

  -flag       [strict|mixed|none]
  -gzip       Use compression for local XML files
  -hash       Print UIDs and checksum values to stdout

  -trie       Print archive trie

Documentation

  -help       Print this document
  -version    Print version number

Sample File Download

  ftp-cp ftp.ncbi.nlm.nih.gov /entrez/entrezdirect/samples carotene.xml.zip
  unzip carotene.xml.zip
  rm carotene.xml.zip

Mammalian Sequence Download

  download-sequence gbmam gbpri gbrod

Human Subset Extraction

  #!/bin/sh

  for fl in gbpri?.aso.gz gbpri??.aso.gz
  do
    run-ncbi-converter asn2all -i "$fl" -a t -b -c -O 9606 -f s > ${fl%.aso.gz}.xml
  done

PubMed Archive Preparation

  prepare-pubmed "$ARCHIVE"

PubMed Base Download

  cd "$PUBMED"
  download-pubmed baseline

PubMed Update Download

  cd "$PUBMED"
  download-pubmed updatefiles

Populate and Maintain Archive

  cd "$PUBMED"
  stash-pubmed mixed "$ARCHIVE"

Retrieve from PubMed Archive

  cat subset.uid |
  fetch-pubmed mixed "$ARCHIVE" > subset.xml

Get Archive UID List

  get-stash-uids "$ARCHIVE" > complete.uid

Reconstruct List of Versioned PMIDs

  cd "$PUBMED"
  rm -f "$ARCHIVE/versioned.uid"
  gunzip -c *.xml.gz |
  xtract -pattern PubmedArticle -if MedlineCitation/PMID@Version -gt 1 \
    -element MedlineCitation/PMID > "$ARCHIVE/versioned.uid"

DISABLE ANTI-VIRUS FILE SCANNING FOR LOCAL ARCHIVES OR DESIGNATE AS TRUSTED FILES

DISABLE SPOTLIGHT INDEXING FOR EXTERNAL DISKS CONTAINING LOCAL ARCHIVES
`

const rchiveExtras = `
Processing Commands (Internal)

  -prepare     [release|report] Compare daily update to archive
  -ignore      Ignore contents of object in -prepare comparisons
  -missing     Print list of missing identifiers
  -unique      File of UIDs for skipping all but last version

Miscellaneous

  -head        Print before everything else
  -tail        Print after everything else
  -hd          Print before each record
  -tl          Print after each record

PubMed Unique Unpacking

  for fl in *.xml.gz
  do
    echo "$fl"
    base=${fl%.xml.gz}
    gunzip -c "$fl" > "$base.tmp"
    xtract -mixed -input "$base.tmp" -pattern PubmedArticle -element MedlineCitation/PMID > "$base.uid"
    rchive -mixed -input "$base.tmp" -unique "$base.uid" -index MedlineCitation/PMID \
      -head "<PubmedArticleSet>" -tail "</PubmedArticleSet>" -pattern PubmedArticle |
    xtract -mixed -format indent -xml '<?xml version="1.0" encoding="utf-8"?>' \
      -doctype '<!DOCTYPE PubmedArticleSet SYSTEM "http://dtd.nlm.nih.gov/ncbi/pubmed/out/pubmed_180101.dtd">' > "$base.xml"
    rm "$base.tmp"
    rm "$base.uid"
  done
  rm *.xml.gz

Update Candidate Report

  cd "$PUBMED"
  gunzip -c *.xml.gz | xtract -strict -compress -format flush |
  rchive -prepare report -ignore DateRevised -archive "$ARCHIVE" \
    -index MedlineCitation/PMID -pattern PubmedArticle

Unnecessary Update Removal

  cd "$PUBMED"
  gunzip -c *.xml.gz | xtract -strict -compress -format flush |
  rchive -prepare release -ignore DateRevised -archive "$ARCHIVE" -index MedlineCitation/PMID \
    -head "<PubmedArticleSet>" -tail "</PubmedArticleSet>" -pattern PubmedArticle |
  xtract -format indent -xml '<?xml version="1.0" encoding="utf-8"?>' \
    -doctype '<!DOCTYPE PubmedArticleSet SYSTEM "http://dtd.nlm.nih.gov/ncbi/pubmed/out/pubmed_180101.dtd">' |
  gzip > newupdate.xml.gz

RELEASE FILES RECONSTRUCTION REQUIRES ARCHIVE TO ENSURE THAT RECORDS ARE PROCESSED IN SORTED ORDER

Reconstruct Release Files

  split -a 3 -l 30000 release.uid uids-
  n=1
  for x in uids-???
  do
    xmlfile=$(printf "pubmed18n%04d.xml.gz" "$n")
    n=$((n+1))
    echo "$xmlfile"
    cat "$x" |
    rchive -archive "$ARCHIVE" -head "<PubmedArticleSet>" -tail "</PubmedArticleSet>" |
    xtract -format indent -xml '<?xml version="1.0" encoding="utf-8"?>' \
      -doctype '<!DOCTYPE PubmedArticleSet SYSTEM "http://dtd.nlm.nih.gov/ncbi/pubmed/out/pubmed_180101.dtd">' |
    gzip > "$xmlfile"
  done
  rm -rf uids-???

DISABLE ANTI-VIRUS FILE SCANNING FOR LOCAL ARCHIVES OR DESIGNATE AS TRUSTED FILES

DISABLE SPOTLIGHT INDEXING FOR EXTERNAL DISKS CONTAINING LOCAL ARCHIVES
`

const rchiveAdvanced = `
Local Information Retrieval

  -e2index    Create Entrez index XML (in xtract)
  -invert     Generate inverted index on specified field
  -merge      Combine inverted indices, grouping by term
  -promote    Create postings file for each indexed term

  -phrase     Search postings for overlapping word pairs
  -search     Query on individual words (* truncation supported)

  -stems      Apply Porter2 stemming to queries
  -stops      Retain stop words in queries

  -counts     Print terms and counts

EXPERIMENTAL INDEXING CAN USE ARCHIVE TO ENSURE A NON-REDUNDANT SET OF RECORDS

Entrez Indexing

  cat carotene.xml | xtract -strict -repair -stems -e2index PubmedArticle \
    MedlineCitation/PMID ArticleTitle,Abstract/AbstractText > carotene.e2x

Index Inversion

  rchive -inverted PAIR carotene.e2x > carotene.inv

Merge Indices

  rchive -merge "$MERGED" PAIR carotene.inv

Create Postings

  rchive -promote "$POSTINGS" PAIR carotene.mrg

Reconstruct Term List Keys

  for fld in PAIR NORM STEM GRFT
  do
    rm -f "$POSTINGS/$fld/sections.txt"
    find "$POSTINGS/$fld" -name "*.mst" |
    sed -e 's,.*/\(.*\)\.mst,\1,' |
    sort | uniq > "$POSTINGS/$fld/sections.txt"
  done

Generate Term List Paths

  for fld in PAIR NORM STEM GRFT
  do
    find "$POSTINGS/$fld" -name "*.trm" |
    sed -e 's,\(.*/\)\(.*\.trm\),\1 \2,' |
    sort -k 2 | uniq | tr -d ' '
  done

Word Searching

  WordSearch() {
    rchive -search "$POSTINGS" "$*" |
    fetch-pubmed strict "$ARCHIVE"
  }

  WordSearch "catabolite repress*" |
  xtract -pattern PubmedArticle -element MedlineCitation/PMID LastName

Phrase Searching

  PhraseSearch() {
    rchive -phrase "$POSTINGS" "$*" |
    fetch-pubmed strict "$ARCHIVE" |
    xtract -pattern PubmedArticle -phrase "$*"
  }

  PhraseSearch selective serotonin reuptake inhibitor + monoamine oxidase inhibitor |
  xtract -pattern PubmedArticle -element MedlineCitation/PMID \
    -block MeshHeading -pfc "\n  " -element DescriptorName

Term List and Record Counts

  rchive -counts "$POSTINGS" NORM "transpos*"

DISABLE ANTI-VIRUS FILE SCANNING FOR LOCAL ARCHIVES OR DESIGNATE AS TRUSTED FILES

DISABLE SPOTLIGHT INDEXING FOR EXTERNAL DISKS CONTAINING LOCAL ARCHIVES
`

const rchiveInternal = `
Performance Default Overrides

  -proc     Number of CPU processors used
  -cons     Ratio of parsers to processors
  -serv     Concurrent parser instances
  -chan     Communication channel depth
  -heap     Order restoration heap size
  -farm     Node allocation buffer length
  -gogc     Garbage collection tuning knob

Debugging

  -timer    Report processing duration and rate

Execution Profiling

  ./rchive -profile -invert PAIR carotene.e2x  > /dev/null
  go tool pprof --pdf ./rchive ./cpu.pprof > ./callgraph.pdf
`

// DATA OBJECTS

type Find struct {
	Index  string
	Parent string
	Match  string
	Attrib string
	Versn  string
}

type Post struct {
	UID  string
	Next *Post
}

type Chain struct {
	First *Post
	Last  *Post
	Count int
}

type Complex struct {
	Index int
	Key   string
	Data  *Chain
}

type Master struct {
	TermOffset int32
	PostOffset int32
}

// UTILITIES

func ParseIndex(indx string) *Find {

	if indx == "" {
		return &Find{}
	}

	// parse parent/element@attribute^version index
	prnt, match := SplitInTwoAt(indx, "/", RIGHT)
	match, versn := SplitInTwoAt(match, "^", LEFT)
	match, attrib := SplitInTwoAt(match, "@", LEFT)

	return &Find{Index: indx, Parent: prnt, Match: match, Attrib: attrib, Versn: versn}
}

// XML TOKEN SEARCH FUNCTIONS

// ProcessXMLTokens looks for specific tokens on a partitioned string
func ProcessXMLTokens(Text, parent string, index int, tbls *Tables, find *Find, stream func(string, string)) string {

	if Text == "" || tbls == nil {
		return ""
	}

	// token parser variables
	Txtlen := len(Text)
	Idx := 0

	plainText := (!tbls.DoStrict && !tbls.DoMixed)

	// get next XML token
	nextToken := func(idx int) (TagType, string, string, int) {

		// lookup table array pointers
		inBlank := &tbls.InBlank
		inFirst := &tbls.InFirst
		inElement := &tbls.InElement

		text := Text[:]
		txtlen := Txtlen

		// XML string ends with > character, acts as sentinel to check if past end of text
		if idx >= txtlen {
			// signal end of XML string
			return ISCLOSED, "", "", 0
		}

		// skip past leading blanks
		ch := text[idx]
		for inBlank[ch] {
			idx++
			ch = text[idx]
		}

		start := idx

		if ch == '<' && (plainText || HTMLAhead(text, idx) == 0) {

			// at start of element
			idx++
			ch = text[idx]

			// check for legal first character of element
			if inFirst[ch] {

				// read element name
				start = idx
				idx++

				ch = text[idx]
				for inElement[ch] {
					idx++
					ch = text[idx]
				}

				str := text[start:idx]

				switch ch {
				case '>':
					// end of element
					idx++

					return STARTTAG, str[:], "", idx
				case '/':
					// self-closing element without attributes
					idx++
					ch = text[idx]
					if ch != '>' {
						fmt.Fprintf(os.Stderr, "\nSelf-closing element missing right angle bracket\n")
					}
					idx++

					return SELFTAG, str[:], "", idx
				case ' ', '\t', '\n', '\r', '\f':
					// attributes
					idx++
					start = idx
					ch = text[idx]
					for ch != '<' && ch != '>' {
						idx++
						ch = text[idx]
					}
					if ch != '>' {
						fmt.Fprintf(os.Stderr, "\nAttributes not followed by right angle bracket\n")
					}
					if text[idx-1] == '/' {
						// self-closing
						atr := text[start : idx-1]
						idx++
						return SELFTAG, str[:], atr[:], idx
					}
					atr := text[start:idx]
					idx++
					return STARTTAG, str[:], atr[:], idx
				default:
					fmt.Fprintf(os.Stderr, "\nUnexpected punctuation '%c' in XML element\n", ch)
					return STARTTAG, str[:], "", idx
				}

			} else {

				// punctuation character immediately after first angle bracket
				switch ch {
				case '/':
					// at start of end tag
					idx++
					start = idx
					ch = text[idx]
					// expect legal first character of element
					if inFirst[ch] {
						idx++
						ch = text[idx]
						for inElement[ch] {
							idx++
							ch = text[idx]
						}
						str := text[start:idx]
						if ch != '>' {
							fmt.Fprintf(os.Stderr, "\nUnexpected characters after end element name\n")
						}
						idx++

						return STOPTAG, str[:], "", idx
					}
					fmt.Fprintf(os.Stderr, "\nUnexpected punctuation '%c' in XML element\n", ch)
				case '?':
					// skip ?xml and ?processing instructions
					idx++
					ch = text[idx]
					for ch != '>' {
						idx++
						ch = text[idx]
					}
					idx++
				case '!':
					// skip !DOCTYPE, !comment, and ![CDATA[
					idx++
					start = idx
					ch = text[idx]
					which := NOTAG
					skipTo := ""
					if ch == '[' && strings.HasPrefix(text[idx:], "[CDATA[") {
						which = CDATATAG
						skipTo = "]]>"
						start += 7
					} else if ch == '-' && strings.HasPrefix(text[idx:], "--") {
						which = COMMENTTAG
						skipTo = "-->"
						start += 2
					}
					if which != NOTAG && skipTo != "" {
						// CDATA or comment block may contain internal angle brackets
						found := strings.Index(text[idx:], skipTo)
						if found < 0 {
							// string stops in middle of CDATA or comment
							return ISCLOSED, "", "", idx
						}
						// adjust position past end of CDATA or comment
						idx += found + len(skipTo)
					} else {
						// otherwise just skip to next right angle bracket
						for ch != '>' {
							idx++
							ch = text[idx]
						}
						idx++
					}
				default:
					fmt.Fprintf(os.Stderr, "\nUnexpected punctuation '%c' in XML element\n", ch)
				}
			}

		} else if ch != '>' {

			// at start of contents
			start = idx

			// find end of contents
			for {
				for ch != '<' && ch != '>' {
					idx++
					ch = text[idx]
				}
				if ch == '<' && !plainText {
					// optionally allow HTML text formatting elements and super/subscripts
					advance := HTMLAhead(text, idx)
					if advance > 0 {
						idx += advance
						ch = text[idx]
						continue
					}
				}
				break
			}

			// trim back past trailing blanks
			lst := idx - 1
			ch = text[lst]
			for inBlank[ch] && lst > start {
				lst--
				ch = text[lst]
			}

			str := text[start : lst+1]

			return CONTENTTAG, str[:], "", idx
		}

		return NOTAG, "", "", idx
	}

	// Stream tokens to obtain value of single index element

	// parseIndex recursive definition
	var parseIndex func(string, string, string) string

	// parse XML tags looking for trie index element
	parseIndex = func(strt, attr, prnt string) string {

		versn := ""

		// check for version attribute match
		if attr != "" && find.Versn != "" && strings.Contains(attr, find.Versn) {
			if strt == find.Match || find.Match == "" {
				if find.Parent == "" || prnt == find.Parent {
					attribs := ParseAttributes(attr)
					for i := 0; i < len(attribs)-1; i += 2 {
						if attribs[i] == find.Versn {
							versn = attribs[i+1]
						}
					}
				}
			}
		}

		// check for attribute index match
		if attr != "" && find.Attrib != "" && strings.Contains(attr, find.Attrib) {
			if strt == find.Match || find.Match == "" {
				if find.Parent == "" || prnt == find.Parent {
					attribs := ParseAttributes(attr)
					for i := 0; i < len(attribs)-1; i += 2 {
						if attribs[i] == find.Attrib {
							return attribs[i+1]
						}
					}
				}
			}
		}

		for {
			tag, name, attr, idx := nextToken(Idx)
			if tag == ISCLOSED {
				break
			}
			Idx = idx

			switch tag {
			case STARTTAG:
				id := parseIndex(name, attr, strt)
				if id != "" {
					return id
				}
			case SELFTAG:
			case STOPTAG:
				// break recursion
				return ""
			case CONTENTTAG:
				// check for content index match
				if strt == find.Match || find.Match == "" {
					if find.Parent == "" || prnt == find.Parent {
						// append version if specified as parent/element@attribute^version
						if versn != "" {
							name += "."
							name += versn
						}
						return name
					}
				}
			default:
			}
		}

		return ""
	}

	// ProcessXMLTokens

	if stream != nil {

		// stream content tokens to callback

		tag, name, _, idx := nextToken(Idx)

		elementName := ""

		// loop until closed
		for {
			if tag == ISCLOSED {
				break
			}

			Idx = idx

			if tag == STARTTAG {
				elementName = name
			} else if tag == CONTENTTAG {
				// send element name and content to callback
				stream(elementName, name)
			}

			tag, name, _, idx = nextToken(Idx)
		}

		return ""
	}

	if find != nil && find.Index != "" {

		// return indexed identifier

		tag, name, attr, idx := nextToken(Idx)

		// loop until start tag
		for {
			if tag == ISCLOSED {
				break
			}

			Idx = idx

			if tag == STARTTAG {
				break
			}

			tag, name, attr, idx = nextToken(Idx)
		}

		return parseIndex(name, attr, parent)
	}

	return ""
}

// DIRECTORY PATH UTILITIES

// MakeArchiveTrie allows a short prefix of letters with an optional underscore, and splits the remainder into character pairs
func MakeArchiveTrie(str string, arry [132]rune) string {

	if len(str) > 64 {
		return ""
	}

	if IsAllDigitsOrPeriod(str) {

		// limit trie to first 6 characters
		if len(str) > 6 {
			str = str[:6]
		}
	}

	max := 4
	k := 0
	for _, ch := range str {
		if unicode.IsLetter(ch) {
			k++
			continue
		}
		if ch == '_' {
			k++
			max = 6
		}
		break
	}

	// prefix is up to three letters if followed by digits, or up to four letters if followed by an underscore
	pfx := str[:k]
	if len(pfx) < max {
		str = str[k:]
	} else {
		pfx = ""
	}

	i := 0

	if pfx != "" {
		for _, ch := range pfx {
			arry[i] = ch
			i++
		}
		arry[i] = '/'
		i++
	}

	between := 0
	doSlash := false

	// remainder is divided in character pairs, e.g., NP_/06/00/51 for NP_060051.2
	for _, ch := range str {
		// break at period separating accession from version
		if ch == '.' {
			break
		}
		if doSlash {
			arry[i] = '/'
			i++
			doSlash = false
		}
		if ch == ' ' {
			ch = '_'
		}
		if !unicode.IsLetter(ch) && !unicode.IsDigit(ch) {
			ch = '_'
		}
		arry[i] = ch
		i++
		between++
		if between > 1 {
			doSlash = true
			between = 0
		}
	}

	res := string(arry[:i])

	if !strings.HasSuffix(res, "/") {
		arry[i] = '/'
		i++
		res = string(arry[:i])
	}

	return strings.ToUpper(res)
}

// MakePostingsTrie splits a string into characters, separated by path delimiting slashes
func MakePostingsTrie(str string, arry [516]rune) string {

	if len(str) > 256 {
		return ""
	}

	// expand greek letters, anglicize characters in other alphabets
	if IsNotASCII(str) {
		if HasGreek(str) {
			str = SpellGreek(str)
			str = CompressRunsOfSpaces(str)
		}
		str = unidecode.Unidecode(str)
		str = strings.TrimSpace(str)
	}

	i := 0
	doSlash := false

	for _, ch := range str {
		if doSlash {
			arry[i] = '/'
			i++
		}
		if ch == ' ' {
			ch = '_'
		}
		if !unicode.IsLetter(ch) && !unicode.IsDigit(ch) {
			ch = '_'
		}
		arry[i] = ch
		i++
		doSlash = true
	}

	return strings.ToLower(string(arry[:i]))
}

// POSTINGS FILE UTILITIES

// trieLen directory depth parameters are based on the observed size distribution of PubMed PAIR indices
var trieLen = map[string]int{
	"ac": 4,
	"af": 4,
	"an": 4,
	"ca": 4,
	"ce": 4,
	"cl": 4,
	"co": 4,
	"di": 4,
	"ex": 4,
	"ge": 4,
	"gr": 4,
	"he": 4,
	"hi": 4,
	"in": 4,
	"me": 4,
	"mo": 4,
	"no": 4,
	"pa": 4,
	"pe": 4,
	"pl": 4,
	"po": 4,
	"pr": 4,
	"re": 4,
	"si": 4,
	"sp": 4,
	"st": 4,
	"su": 4,
	"tr": 4,
	"tw": 4,
	"un": 3,
	"va": 3,
	"ve": 3,
	"vi": 3,
	"wh": 3,
}

func PostingDir(term string) string {

	if len(term) < 3 {
		return term
	}

	key := term[:2]

	num, ok := trieLen[key]
	if ok && len(term) >= num {
		return term[:num]
	}

	switch term[0] {
	case 'u', 'v', 'w', 'x', 'y', 'z':
		return term[:2]
	}

	return term[:3]
}

func IdentifierKey(term string) string {

	// remove punctuation from term
	key := strings.Map(func(c rune) rune {
		if !unicode.IsLetter(c) && !unicode.IsDigit(c) && c != ' ' && c != '-' && c != '_' {
			return -1
		}
		return c
	}, term)

	key = strings.Replace(key, " ", "_", -1)
	key = strings.Replace(key, "-", "_", -1)

	// use first 2, 3, or 4 characters of identifier for directory
	key = PostingDir(key)

	return key
}

func PostingPath(prom, field, term string, arry [516]rune) (string, string) {

	// use first few characters of identifier for directory
	dir := IdentifierKey(term)

	trie := MakePostingsTrie(dir, arry)
	if trie == "" {
		return "", ""
	}

	dpath := path.Join(prom, field, trie)

	return dpath, dir
}

func CommonOpenFile(dpath, fname string) (*os.File, int64) {

	fpath := path.Join(dpath, fname)
	if fpath == "" {
		return nil, 0
	}

	inFile, err := os.Open(fpath)
	if err != nil && os.IsNotExist(err) {
		return nil, 0
	}
	if err != nil {
		fmt.Println(err.Error())
		return nil, 0
	}

	fi, err := inFile.Stat()
	if err != nil {
		fmt.Println(err.Error())
		return nil, 0
	}

	size := fi.Size()

	return inFile, size
}

func ReadMasterIndex(dpath, key string) []Master {

	inFile, size := CommonOpenFile(dpath, key+".mst")
	if inFile == nil {
		return nil
	}

	defer inFile.Close()

	data := make([]Master, size/8)
	if data == nil {
		return nil
	}

	err := binary.Read(inFile, binary.LittleEndian, &data)
	if err != nil {
		fmt.Println(err.Error())
		return nil
	}

	return data
}

func ReadTermList(dpath, key string) []byte {

	inFile, size := CommonOpenFile(dpath, key+".trm")
	if inFile == nil {
		return nil
	}

	defer inFile.Close()

	data := make([]byte, size)
	if data == nil {
		return nil
	}

	err := binary.Read(inFile, binary.LittleEndian, &data)
	if err != nil {
		fmt.Println(err.Error())
		return nil
	}

	return data
}

func ReadPostingData(dpath, key string, offset int32, size int32) []int32 {

	inFile, _ := CommonOpenFile(dpath, key+".pst")
	if inFile == nil {
		return nil
	}

	defer inFile.Close()

	data := make([]int32, size/4)
	if data == nil {
		return nil
	}

	_, err := inFile.Seek(int64(offset), io.SeekStart)
	if err != nil {
		fmt.Println(err.Error())
		return nil
	}

	err = binary.Read(inFile, binary.LittleEndian, data)
	if err != nil {
		fmt.Println(err.Error())
		return nil
	}

	return data
}

func GetPostingIDs(prom, field, term string) []int32 {

	var arry [516]rune
	dpath, key := PostingPath(prom, field, term, arry)
	if dpath == "" {
		return nil
	}

	indx := ReadMasterIndex(dpath, key)
	if indx == nil {
		return nil
	}

	trms := ReadTermList(dpath, key)
	if trms == nil {
		return nil
	}

	strs := make([]string, len(indx)-1)
	if strs == nil {
		return nil
	}

	// populate array of strings from term list
	for i, j := 0, 1; i < len(indx)-1; i++ {
		from := indx[i].TermOffset
		to := indx[j].TermOffset - 1
		j++
		strs[i] = string(trms[from:to])
	}

	isWildCard := false
	if strings.HasSuffix(term, "*") {
		tlen := len(term)
		if tlen > 1 {
			isWildCard = true
			term = term[:tlen-1]
		}
		if tlen < 4 {
			fmt.Fprintf(os.Stderr, "Wildcard term '%s' must be at least 4 characters long - ignoring this word\n", term)
			return nil
		}
	}

	// binary search in term list
	num := len(strs)
	L, R := 0, num-1
	for L < R {
		mid := (L + R) / 2
		if strs[mid] < term {
			L = mid + 1
		} else {
			R = mid
		}
	}

	// wild card search scans term lists, fuses adjacent postings lists
	if isWildCard {
		if R < num && strings.HasPrefix(strs[R], term) {
			offset := indx[R].PostOffset
			for R < num && strings.HasPrefix(strs[R], term) {
				R++
			}
			size := indx[R].PostOffset - offset

			// term match, read relevant postings list section
			data := ReadPostingData(dpath, key, offset, size)
			if data == nil {
				return nil
			}

			merged := make(map[int32]bool)

			// combine all postings in term range
			for _, val := range data {
				merged[val] = true
			}

			fused := make([]int32, len(merged))

			// convert map to slice
			i := 0
			for num := range merged {
				fused[i] = num
				i++
			}

			sort.Slice(fused, func(i, j int) bool { return fused[i] < fused[j] })

			return fused
		}
		return nil
	}

	// regular search requires exact match from binary search
	if R < num && strs[R] == term {

		offset := indx[R].PostOffset
		size := indx[R+1].PostOffset - offset

		// term match, read relevant postings list section
		data := ReadPostingData(dpath, key, offset, size)
		if data == nil {
			return nil
		}
		return data
	}

	return nil
}

func IntersectIDs(prev, curr []int32) []int32 {

	if len(prev) < len(curr) {
		// swap to populate map with larger slice, test each value of smaller slice
		prev, curr = curr, prev
	}

	orig := make(map[int32]bool)

	for _, uid := range prev {
		orig[uid] = true
	}

	combo := make(map[int32]bool)

	// save postings from smaller list that also appear in larger list
	for _, uid := range curr {
		if orig[uid] {
			combo[uid] = true
		}
	}

	fused := make([]int32, len(combo))

	// convert map to slice
	i := 0
	for num := range combo {
		fused[i] = num
		i++
	}

	return fused
}

// SIMULATE PHRASE SEARCH BY LOOKING AT OVERLAPPING WORD PAIRS IN LOCAL POSTINGS FILES

type Arrays struct {
	Data []int32
}

func CleanupQueryString(str string) string {

	if str == "" {
		return ""
	}

	// cleanup string
	if IsNotASCII(str) {

		if HasGreek(str) {
			str = SpellGreek(str)
		}
		str = DoAccentTransform(str)
		if HasMarkup(str) {
			str = RemoveUnicodeMarkup(str)
		}
	}

	str = strings.ToLower(str)

	if HasAmpOrNotASCII(str) {
		str = html.UnescapeString(str)
	}

	if HasBadSpace(str) {
		str = CleanupBadSpaces(str)
	}
	if HasAngleBracket(str) {
		str = DoHTMLReplace(str)
	}

	str = strings.Replace(str, "--", "-", -1)

	str = CompressRunsOfSpaces(str)
	str = strings.TrimSpace(str)

	return str
}

func SearchForPhrase(base, phrase string, doStem, doStop bool) int {

	if phrase == "" {
		return 0
	}

	makeWordPairs := func(str string) []string {

		if str == "" {
			return nil
		}

		str = CleanupQueryString(str)

		var arry []string

		// break terms at punctuation, and at non-ASCII characters, also allow plus sign or ampersand to separate clauses
		words := strings.FieldsFunc(str, func(c rune) bool {
			return (!unicode.IsLetter(c) && !unicode.IsDigit(c) && c != '+' && c != '&') || c > 127
		})

		if len(words) < 2 {
			return nil
		}

		past := ""
		for _, item := range words {

			// skip a single character or anything starting with a digit
			if len(item) < 2 || unicode.IsDigit(rune(item[0])) {
				past = ""
				continue
			}

			if item == "+" || item == "&" {
				// plus sign or ampersand in query phrase separates clauses, breaks word pair chain
				past = ""
				continue
			}
			if doStop && IsStopWord(item) {
				// skip if stop word, also interrupts overlapping word pair chain
				past = ""
				continue
			}
			if doStem {
				// apply stemming algorithm
				item = porter2.Stem(item)
				item = strings.TrimSpace(item)
			}

			if past != "" {
				// index informative adjacent word pair
				adjacent := past + " " + item
				adjacent = CompressRunsOfSpaces(adjacent)
				adjacent = strings.TrimSpace(adjacent)
				arry = append(arry, adjacent)
			}
			past = item
		}

		return arry
	}

	var intersect []Arrays

	pairs := makeWordPairs(phrase)

	field := "PAIR"
	if doStem {
		field = "GRFT"
	}

	for _, term := range pairs {

		data := GetPostingIDs(base, field, term)
		if data == nil {
			// skip pairs that are not present in the index
			continue
		}

		one := Arrays{Data: data}
		intersect = append(intersect, one)
	}

	if len(intersect) < 1 {
		return 0
	}

	result := intersect[0].Data

	for i := 1; i < len(intersect); i++ {

		curr := intersect[i].Data

		result = IntersectIDs(result, curr)
	}

	sort.Slice(result, func(i, j int) bool { return result[i] < result[j] })

	for _, pmid := range result {
		fmt.Fprintf(os.Stdout, "%d\n", pmid)
	}

	return len(result)
}

func SearchForTerms(base, terms string, doStem, doStop bool) int {

	if terms == "" {
		return 0
	}

	makeTermList := func(str string) []string {

		if str == "" {
			return nil
		}

		str = CleanupQueryString(str)

		var arry []string

		// break terms at punctuation, and at non-ASCII characters, allow asterisk to indicate truncation wildcard
		terms := strings.FieldsFunc(str, func(c rune) bool {
			return (!unicode.IsLetter(c) && !unicode.IsDigit(c) && c != '*') || c > 127
		})

		for _, item := range terms {

			// skip a single character or anything starting with a digit
			if len(item) < 2 || unicode.IsDigit(rune(item[0])) {
				continue
			}

			// optional stop word removal
			if doStop && IsStopWord(item) {
				continue
			}
			if doStem {
				// apply stemming algorithm
				item = porter2.Stem(item)
				item = strings.TrimSpace(item)
			}

			// index single term
			arry = append(arry, item)
		}

		return arry
	}

	var intersect []Arrays

	words := makeTermList(terms)

	field := "NORM"
	if doStem {
		field = "STEM"
	}

	for _, term := range words {

		data := GetPostingIDs(base, field, term)
		if data == nil {
			// skip words that are not present in the index
			continue
		}

		one := Arrays{Data: data}
		intersect = append(intersect, one)
	}

	if len(intersect) < 1 {
		return 0
	}

	result := intersect[0].Data

	for i := 1; i < len(intersect); i++ {

		curr := intersect[i].Data

		result = IntersectIDs(result, curr)
	}

	sort.Slice(result, func(i, j int) bool { return result[i] < result[j] })

	for _, pmid := range result {
		fmt.Fprintf(os.Stdout, "%d\n", pmid)
	}

	return len(result)
}

func PrintTermCounts(base, field, term string) int {

	var arry [516]rune
	dpath, key := PostingPath(base, field, term, arry)
	if dpath == "" {
		return 0
	}

	indx := ReadMasterIndex(dpath, key)
	if indx == nil {
		return 0
	}

	trms := ReadTermList(dpath, key)
	if trms == nil {
		return 0
	}

	strs := make([]string, len(indx)-1)
	if strs == nil {
		return 0
	}

	// populate array of strings from term list
	for i, j := 0, 1; i < len(indx)-1; i++ {
		from := indx[i].TermOffset
		to := indx[j].TermOffset - 1
		j++
		strs[i] = string(trms[from:to])
	}

	isWildCard := false
	if strings.HasSuffix(term, "*") {
		tlen := len(term)
		if tlen > 1 {
			isWildCard = true
			term = term[:tlen-1]
		}
		if tlen < 4 {
			fmt.Fprintf(os.Stderr, "Wildcard term '%s' must be at least 4 characters long - ignoring this word\n", term)
			return 0
		}
	}

	// binary search in term list
	num := len(strs)
	L, R := 0, num-1
	for L < R {
		mid := (L + R) / 2
		if strs[mid] < term {
			L = mid + 1
		} else {
			R = mid
		}
	}

	count := 0

	// wild card search scans term lists
	if isWildCard {
		for R < num && strings.HasPrefix(strs[R], term) {
			offset := indx[R].PostOffset
			size := indx[R+1].PostOffset - offset
			fmt.Fprintf(os.Stdout, "%d\t%s\n", size/4, strs[R])
			count++
			R++
		}
	} else {
		// regular search requires exact match from binary search
		if R < num && strs[R] == term {
			offset := indx[R].PostOffset
			size := indx[R+1].PostOffset - offset
			fmt.Fprintf(os.Stdout, "%d\t%s\n", size/4, strs[R])
			count++
		}
	}

	return count
}

// FIND APPROPRIATE VERSION OF RECORD IN ARCHIVE

// GetAppropriateVersion returns the most recent version of a record in the archive
func GetAppropriateVersion(stash, id string, arry [132]rune) (string, string) {

	if id == "" {
		return "", ""
	}

	trie := MakeArchiveTrie(id, arry)
	if trie == "" {
		return "", ""
	}

	// finished if version (including default) is already present
	if strings.Contains(id, ".") {
		return id, trie
	}

	// otherwise examine directory for highest version of the record
	base := stash + "/" + trie + id
	blen := len(base)

	// look for all files with given ID, a dot, a version number, and an .xml or .xml.gz suffix
	files, err := filepath.Glob(base + ".*.xm*")

	if err != nil {
		fmt.Println(err.Error())
		return "", ""
	}

	var versions []string

	// extract versions from file path
	for _, str := range files {
		str = str[blen:]
		dot := strings.Index(str, ".")
		if dot >= 0 {
			str = str[:dot]
		}
		versions = append(versions, str)
	}

	if len(versions) < 1 {
		return "", ""
	}

	if len(versions) > 1 {
		// sort highest to lowest numerically
		sort.Slice(versions, func(i, j int) bool {
			// numeric sort on strings checks lengths first
			lni := len(versions[i])
			lnj := len(versions[j])
			// shorter string is numerically less, assuming no leading zeros
			if lni > lnj {
				return true
			}
			if lni < lnj {
				return false
			}
			// same length, can now do string comparison on contents
			return versions[i] > versions[j]
		})
	}

	// first in slice is highest numerical version
	ver := versions[0]

	return id + "." + ver, trie
}

// READ FILE IN ARCHIVE THAT STORES UIDS WITH NON-DEFAULT VERSIONS

// ReadVersionedList returns a map of identifiers that cannot be given the default ".1" version
func ReadVersionedList(stash string) map[string]bool {

	// create map to store identifiers
	versionedUids := make(map[string]bool)

	vpath := path.Join(stash, "versioned.uid")
	if vpath == "" {
		return nil
	}

	// open file of (non-versioned) identifiers for records with versions greater than 1
	fl, err := os.Open(vpath)
	if err != nil && os.IsNotExist(err) {
		return nil
	}
	if err != nil {
		fmt.Println(err.Error())
		fmt.Fprintf(os.Stderr, "\nWARNING: Unable to open versioned identifier file '%s'\n", vpath)
		return nil
	}

	defer fl.Close()

	scanr := bufio.NewScanner(fl)

	for scanr.Scan() {

		// read UID
		id := scanr.Text()

		// remove version if present
		pos := strings.Index(id, ".")
		if pos >= 0 {
			id = id[:pos]
		}

		// save in map
		versionedUids[id] = true
	}

	return versionedUids
}

// CONCURRENT GOROUTINE SERVERS

// processes with single goroutine call defer close(out) so consumer(s) can range over channel
// processes with multiple instances call defer wg.Done(), separate goroutine uses wg.Wait() to delay close(out)

func CreateUIDReader(in io.Reader, stash string, tbls *Tables) <-chan Extract {

	if in == nil || tbls == nil {
		return nil
	}

	out := make(chan Extract, tbls.ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create uid reader channel\n")
		os.Exit(1)
	}

	versioned := ReadVersionedList(stash)

	// uidReader reads uids from input stream and sends through channel
	uidReader := func(in io.Reader, out chan<- Extract) {

		// close channel when all records have been processed
		defer close(out)

		scanr := bufio.NewScanner(in)

		idx := 0
		for scanr.Scan() {

			// read lines of identifiers
			file := scanr.Text()
			idx++

			if !strings.Contains(file, ".") && (versioned == nil || !versioned[file]) {
				// add default version
				file += ".1"
			}

			out <- Extract{idx, "", file}
		}
	}

	// launch single uid reader goroutine
	go uidReader(in, out)

	return out
}

func CreateStashers(tbls *Tables, stash, parent, indx string, hash, zipp bool, inp <-chan Extract) <-chan string {

	if tbls == nil || inp == nil {
		return nil
	}

	out := make(chan string, tbls.ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create stasher channel\n")
		os.Exit(1)
	}

	find := ParseIndex(indx)

	// map to report UIDs with version greater than 1
	hasVersion := make(map[string]bool)

	// mutex to protect access to hasVersion map
	var vlock sync.Mutex

	sfx := ".xml"
	if zipp {
		sfx = ".xml.gz"
	}

	type StasherType int

	const (
		OKAY StasherType = iota
		WAIT
		BAIL
	)

	// mutex to protect access to inUse map
	var wlock sync.Mutex

	// map to track files currently being written
	inUse := make(map[string]int)

	// lockFile function prevents colliding writes
	lockFile := func(id string, index int) StasherType {
		// map is non-reentrant, protect with mutex
		wlock.Lock()
		// multiple return paths, schedule the unlock command up front
		defer wlock.Unlock()

		idx, ok := inUse[id]

		if ok {
			if idx < index {
				// later version is being written by another goroutine, skip this
				return BAIL
			}
			// earlier version is being written by another goroutine, wait
			return WAIT
		}

		// okay to write file, mark in use to prevent collision
		inUse[id] = index
		return OKAY
	}

	// freeFile function removes entry from inUse map
	freeFile := func(id string) {
		wlock.Lock()
		// free entry in map, later versions of same record can now be written
		delete(inUse, id)
		wlock.Unlock()
	}

	// trimLeft function reformats output, efficiently skipping leading spaces on each line
	trimLeft := func(text string) string {

		if text == "" {
			return ""
		}

		var buffer bytes.Buffer

		max := len(text)
		idx := 0
		inBlank := &tbls.InBlank

		for idx < max {

			// skip past leading blanks and empty lines
			for idx < max {
				ch := text[idx]
				if !inBlank[ch] {
					break
				}
				idx++
			}

			start := idx

			// skip to next newline
			for idx < max {
				if text[idx] == '\n' {
					break
				}
				idx++
			}

			str := text[start:idx]

			if str == "" {
				continue
			}

			// skip processing instruction
			if strings.HasPrefix(str, "<?") && strings.HasSuffix(str, "?>") {
				continue
			}

			// trim spaces next to angle bracket
			str = strings.Replace(str, "> ", ">", -1)
			str = strings.Replace(str, " <", "<", -1)

			buffer.WriteString(str[:])
			buffer.WriteString("\n")
		}

		return buffer.String()
	}

	// stashRecord saves individual XML record to archive file accessed by trie
	stashRecord := func(text, id string, index int) string {

		pos := strings.Index(id, ".")
		if pos >= 0 {
			ver := id[pos:]
			if ver != ".1" {
				// save high-versioned UID bases to map
				vlock.Lock()
				justID := id[:pos]
				hasVersion[justID] = true
				vlock.Unlock()
			}
		}

		var arry [132]rune
		trie := MakeArchiveTrie(id, arry)
		if trie == "" {
			return ""
		}

		attempts := 5
		keepChecking := true

		for keepChecking {
			// check if file is not being written by another goroutine
			switch lockFile(id, index) {
			case OKAY:
				// okay to save this record now
				keepChecking = false
			case WAIT:
				// earlier version is being saved, wait one second and try again
				time.Sleep(time.Second)
				attempts--
				if attempts < 1 {
					// cannot get lock after several attempts
					fmt.Fprintf(os.Stderr, "\nERROR: Unable to save '%s'\n", id)
					return ""
				}
			case BAIL:
				// later version is being saved, skip this one
				return ""
			default:
			}
		}

		// delete lock after writing file
		defer freeFile(id)

		dpath := path.Join(stash, trie)
		if dpath == "" {
			return ""
		}
		_, err := os.Stat(dpath)
		if err != nil && os.IsNotExist(err) {
			err = os.MkdirAll(dpath, os.ModePerm)
		}
		if err != nil {
			fmt.Println(err.Error())
			return ""
		}
		fpath := path.Join(dpath, id+sfx)
		if fpath == "" {
			return ""
		}

		// overwrites and truncates existing file
		fl, err := os.Create(fpath)
		if err != nil {
			fmt.Println(err.Error())
			return ""
		}

		defer fl.Close()

		defer fl.Sync()

		// remove leading spaces on each line
		str := trimLeft(text)

		res := ""

		if hash {
			// calculate hash code for verification table
			hsh := crc32.NewIEEE()
			hsh.Write([]byte(str))
			val := hsh.Sum32()
			res = strconv.FormatUint(uint64(val), 10)
		}

		if zipp {

			zpr, err := gzip.NewWriterLevel(fl, gzip.DefaultCompression)
			if err != nil {
				fmt.Println(err.Error())
				return ""
			}

			defer zpr.Close()

			wrtr := bufio.NewWriter(zpr)

			defer wrtr.Flush()

			// compress and copy record to file
			wrtr.WriteString(str)
			if !strings.HasSuffix(str, "\n") {
				wrtr.WriteString("\n")
			}

		} else {

			// copy record to file
			fl.WriteString(str)
			if !strings.HasSuffix(str, "\n") {
				fl.WriteString("\n")
			}
		}

		return res
	}

	// xmlStasher reads from channel and calls stashRecord
	xmlStasher := func(wg *sync.WaitGroup, inp <-chan Extract, out chan<- string) {

		defer wg.Done()

		for ext := range inp {

			ext.Ident = ProcessXMLTokens(ext.Text, parent, 0, tbls, find, nil)

			hsh := stashRecord(ext.Text, ext.Ident, ext.Index)

			res := ext.Ident
			if hash {
				res += "\t" + hsh
			}
			res += "\n"

			out <- res
		}
	}

	var wg sync.WaitGroup

	// launch multiple stasher goroutines
	for i := 0; i < tbls.NumServe; i++ {
		wg.Add(1)
		go xmlStasher(&wg, inp, out)
	}

	// launch separate anonymous goroutine to wait until all stashers are done
	go func() {
		wg.Wait()
		close(out)

		// now see if any versions > 1 were processed
		if len(hasVersion) < 1 {
			return
		}

		// use stash/versioned.uid file to speed up bulk fetching
		fpath := path.Join(stash, "versioned.uid")
		if fpath == "" {
			return
		}

		// appends if file exists, otherwise creates
		fl, err := os.OpenFile(fpath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
		if err != nil {
			fmt.Println(err.Error())
			return
		}

		vlock.Lock()

		// print UIDs with version > 1
		for uid := range hasVersion {
			fl.WriteString(uid)
			fl.WriteString("\n")
		}

		err = fl.Sync()
		if err != nil {
			fmt.Println(err.Error())
		}

		vlock.Unlock()

		// close file of versioned UIDs
		fl.Close()
	}()

	return out
}

func CreateFetchers(tbls *Tables, stash string, zipp bool, inp <-chan Extract) <-chan Extract {

	if tbls == nil || inp == nil {
		return nil
	}

	out := make(chan Extract, tbls.ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create fetcher channel\n")
		os.Exit(1)
	}

	sfx := ".xml"
	if zipp {
		sfx = ".xml.gz"
	}

	fetchRecord := func(file string, buf bytes.Buffer) string {

		var arry [132]rune
		file, trie := GetAppropriateVersion(stash, file, arry)

		if file == "" || trie == "" {
			return ""
		}

		fpath := path.Join(stash, trie, file+sfx)
		if fpath == "" {
			return ""
		}

		iszip := zipp

		inFile, err := os.Open(fpath)

		// if failed to find ".xml" file, try ".xml.gz" without requiring -gzip
		if err != nil && os.IsNotExist(err) && !zipp {
			iszip = true
			fpath := path.Join(stash, trie, file+".xml.gz")
			if fpath == "" {
				return ""
			}
			inFile, err = os.Open(fpath)
		}
		if err != nil {
			fmt.Println(err.Error())
			return ""
		}

		defer inFile.Close()

		brd := bufio.NewReader(inFile)

		if iszip {

			zpr, err := gzip.NewReader(brd)

			defer zpr.Close()

			if err == nil {
				// copy and decompress cached file contents
				buf.ReadFrom(zpr)
			}

		} else {

			// copy cached file contents
			buf.ReadFrom(brd)
		}

		str := buf.String()

		return str
	}

	// xmlFetcher reads XML from file
	xmlFetcher := func(wg *sync.WaitGroup, inp <-chan Extract, out chan<- Extract) {

		// report when more records to process
		defer wg.Done()

		var buf bytes.Buffer

		for ext := range inp {

			buf.Reset()

			str := fetchRecord(ext.Text, buf)

			out <- Extract{ext.Index, "", str}
		}
	}

	var wg sync.WaitGroup

	// launch multiple fetcher goroutines
	for i := 0; i < tbls.NumServe; i++ {
		wg.Add(1)
		go xmlFetcher(&wg, inp, out)
	}

	// launch separate anonymous goroutine to wait until all fetchers are done
	go func() {
		wg.Wait()
		close(out)
	}()

	return out
}

func CreateInverters(args []string, nvrt string, tbls *Tables) <-chan Complex {

	if args == nil || tbls == nil {
		return nil
	}

	out := make(chan Complex, tbls.ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create inverter channel array\n")
		os.Exit(1)
	}

	// map for inverted index
	inverted := make(map[string]*Chain)

	// mutex for inverted index
	var ilock sync.Mutex

	// post farm variables
	FarmPos := 0
	FarmMax := tbls.FarmSize
	FarmItems := make([]Post, FarmMax)

	// allocate multiple posts in a large array for memory management efficiency
	nextPost := func(uid string) *Post {
		// if farm array slots used up, allocate new array
		if FarmPos >= FarmMax {
			FarmItems = make([]Post, FarmMax)
			FarmPos = 0
		}

		if FarmItems == nil {
			return nil
		}

		// take node from next available slot in farm array
		post := &FarmItems[FarmPos]

		post.UID = uid[:]

		FarmPos++

		return post
	}

	// add single posting to farm
	addPost := func(term, uid string) {

		// protect with mutex
		ilock.Lock()

		defer ilock.Unlock()

		chn, ok := inverted[term]
		if !ok {
			chn = &Chain{}
			inverted[term] = chn
		}
		if chn != nil {
			pst := nextPost(uid)
			if chn.First == nil {
				chn.First = pst
			}
			if chn.Last != nil {
				prv := chn.Last
				prv.Next = pst
			}
			chn.Last = pst
			chn.Count++
		}
	}

	// xmlInverter sends UID and term strings to a callback for inversion
	xmlInverter := func(wg *sync.WaitGroup, fileName string) {

		defer wg.Done()

		f, err := os.Open(fileName)
		if err != nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to open input file '%s'\n", fileName)
			os.Exit(1)
		}

		// close input file when all records have been processed
		defer f.Close()

		var in io.Reader

		in = f

		// if suffix is ".gz", use decompressor
		iszip := false
		if strings.HasSuffix(fileName, ".gz") {
			iszip = true
		}

		if iszip {
			brd := bufio.NewReader(f)
			if brd == nil {
				fmt.Fprintf(os.Stderr, "\nERROR: Unable to create buffered reader on '%s'\n", fileName)
				os.Exit(1)
			}
			zpr, err := gzip.NewReader(brd)
			if err != nil {
				fmt.Fprintf(os.Stderr, "\nERROR: Unable to create decompressor on '%s'\n", fileName)
				os.Exit(1)
			}

			// close decompressor when all records have been processed
			defer zpr.Close()

			// use decompressor for reading file
			in = zpr
		}

		rdr := NewXMLReader(in, false, false, false)

		if rdr == nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to create XML Block Reader\n")
			os.Exit(1)
		}

		currUID := ""

		doInvert := func(tag, content string) {

			if tag == "IdxUid" {
				currUID = content
			} else if tag == nvrt {

				// expand greek letters, anglicize characters in other alphabets
				if IsNotASCII(content) {

					if HasGreek(content) {
						content = SpellGreek(content)
						content = CompressRunsOfSpaces(content)
					}
					content = unidecode.Unidecode(content)

					content = DoAccentTransform(content)
					content = UnicodeToASCII(content)

					content = strings.TrimSpace(content)
				}

				// remove punctuation from term
				content := strings.Map(func(c rune) rune {
					if !unicode.IsLetter(c) && !unicode.IsDigit(c) && c != ' ' && c != '-' && c != '_' {
						return -1
					}
					return c
				}, content)

				content = strings.Replace(content, "_", " ", -1)
				content = strings.Replace(content, "-", " ", -1)

				content = CompressRunsOfSpaces(content)
				content = strings.TrimSpace(content)

				if content != "" && currUID != "" {
					addPost(content, currUID)
				}
			}
		}

		// partition all input by pattern and send XML substring through channel
		PartitionPattern("IdxDocument", "", rdr,
			func(rec int, ofs int64, str string) {
				ProcessXMLTokens(str[:], "IdxDocument", 0, tbls, nil, doInvert)
			})
	}

	var wg sync.WaitGroup

	// launch multiple inverter goroutines
	for _, str := range args {
		wg.Add(1)
		go xmlInverter(&wg, str)
	}

	// launch separate anonymous goroutine to wait until all inverters are done
	go func() {
		wg.Wait()

		// send results to sorters
		for key, chn := range inverted {
			out <- Complex{0, key, chn}
		}

		close(out)
	}()

	return out
}

func CreateSorters(tbls *Tables, nvrt string, inp <-chan Complex) <-chan Extract {

	if tbls == nil || inp == nil {
		return nil
	}

	out := make(chan Extract, tbls.ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create sorter channel\n")
		os.Exit(1)
	}

	// xmlSorter reads from channel and prints one sorted posting list
	xmlSorter := func(wg *sync.WaitGroup, nvrt string, inp <-chan Complex, out chan<- Extract) {

		defer wg.Done()

		var buffer bytes.Buffer

		printChain := func(key string, chn *Chain) string {

			// sort each posting list in numeric order
			sorted := make([]string, chn.Count)
			i := 0
			for pst := chn.First; pst != nil; pst = pst.Next {
				sorted[i] = pst.UID
				i++
			}
			if chn.Count > 1 {
				sort.Slice(sorted, func(i, j int) bool {
					// numeric sort on strings checks lengths first
					lni := len(sorted[i])
					lnj := len(sorted[j])
					// shorter string is numerically less, assuming no leading zeros
					if lni < lnj {
						return true
					}
					if lni > lnj {
						return false
					}
					// same length, can now do string comparison on contents
					return sorted[i] < sorted[j]
				})
			}

			buffer.Reset()

			buffer.WriteString("  <InvDocument>\n")
			buffer.WriteString("    <InvKey>")
			buffer.WriteString(key)
			buffer.WriteString("</InvKey>\n")

			// print list of UIDs, skipping duplicates
			buffer.WriteString("    <InvIDs>\n")
			prev := ""
			for _, uid := range sorted {
				// detect duplicate UIDs, now in same list after conversion of one term entry from foreign alphabet
				if uid != prev {
					buffer.WriteString("      <")
					buffer.WriteString(nvrt)
					buffer.WriteString(">")
					buffer.WriteString(uid)
					buffer.WriteString("</")
					buffer.WriteString(nvrt)
					buffer.WriteString(">\n")
				}
				prev = uid
			}
			buffer.WriteString("    </InvIDs>\n")
			buffer.WriteString("  </InvDocument>\n")

			str := buffer.String()

			return str
		}

		for cplx := range inp {

			str := printChain(cplx.Key, cplx.Data)

			out <- Extract{0, cplx.Key, str}
		}
	}

	var wg sync.WaitGroup

	// launch multiple sorter goroutines
	for i := 0; i < tbls.NumServe; i++ {
		wg.Add(1)
		go xmlSorter(&wg, nvrt, inp, out)
	}

	// launch separate anonymous goroutine to wait until all sorters are done
	go func() {
		wg.Wait()
		close(out)
	}()

	return out
}

func CreateResolver(tbls *Tables, inp <-chan Extract) <-chan string {

	if tbls == nil || inp == nil {
		return nil
	}

	out := make(chan string, tbls.ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create resolver channel\n")
		os.Exit(1)
	}

	// xmlResolver distributes adjacent records with the same identifier prefix
	xmlResolver := func(inp <-chan Extract, out chan<- string) {

		// close channel when all records have been processed
		defer close(out)

		// map for inverted index
		inverted := make(map[string]string)

		// drain channel, populate map for alphabetizing
		for curr := range inp {

			inverted[curr.Ident] = curr.Text
		}

		// sort map keys in alphabetical order
		ordered := make([]Extract, len(inverted))

		i := 0
		for key, str := range inverted {
			ordered[i].Ident = key
			ordered[i].Text = str
			i++
		}
		sort.Slice(ordered, func(i, j int) bool { return ordered[i].Ident < ordered[j].Ident })

		// iterate through alphabetized results
		for _, curr := range ordered {

			// send result to output
			out <- curr.Text

			runtime.Gosched()
		}
	}

	// launch single resolver goroutine
	go xmlResolver(inp, out)

	return out
}

type Plex struct {
	Which int
	Ident string
	Text  string
}

type PlexHeap []Plex

// methods that satisfy heap.Interface
func (h PlexHeap) Len() int {
	return len(h)
}
func (h PlexHeap) Less(i, j int) bool {
	return h[i].Ident < h[j].Ident
}
func (h PlexHeap) Swap(i, j int) {
	h[i], h[j] = h[j], h[i]
}
func (h *PlexHeap) Push(x interface{}) {
	*h = append(*h, x.(Plex))
}
func (h *PlexHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

func CreatePresenters(args []string, tbls *Tables) []<-chan Plex {

	if args == nil || tbls == nil {
		return nil
	}

	numFiles := len(args)
	if numFiles < 1 {
		fmt.Fprintf(os.Stderr, "\nERROR: Not enough inverted files to merge\n")
		os.Exit(1)
	}

	chns := make([]<-chan Plex, numFiles)
	if chns == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create presenter channel array\n")
		os.Exit(1)
	}

	// xmlPresenter sends partitioned XML strings through channel
	xmlPresenter := func(fileNum int, fileName string, out chan<- Plex) {

		// close channel when all records have been processed
		defer close(out)

		f, err := os.Open(fileName)
		if err != nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to open input file '%s'\n", fileName)
			os.Exit(1)
		}

		// close input file when all records have been processed
		defer f.Close()

		var in io.Reader

		in = f

		// if suffix is ".gz", use decompressor
		iszip := false
		if strings.HasSuffix(fileName, ".gz") {
			iszip = true
		}

		if iszip {
			brd := bufio.NewReader(f)
			if brd == nil {
				fmt.Fprintf(os.Stderr, "\nERROR: Unable to create buffered reader on '%s'\n", fileName)
				os.Exit(1)
			}
			zpr, err := gzip.NewReader(brd)
			if err != nil {
				fmt.Fprintf(os.Stderr, "\nERROR: Unable to create decompressor on '%s'\n", fileName)
				os.Exit(1)
			}

			// close decompressor when all records have been processed
			defer zpr.Close()

			// use decompressor for reading file
			in = zpr
		}

		rdr := NewXMLReader(in, false, false, false)

		if rdr == nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to create XML Block Reader\n")
			os.Exit(1)
		}

		find := ParseIndex("InvKey")

		// partition all input by pattern and send XML substring through channel
		PartitionPattern("InvDocument", "", rdr,
			func(rec int, ofs int64, str string) {
				id := ProcessXMLTokens(str[:], "InvDocument", 0, tbls, find, nil)

				out <- Plex{fileNum, id, str}
			})
	}

	// launch multiple presenter goroutines
	for i, str := range args {

		chn := make(chan Plex, tbls.ChanDepth)
		if chn == nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to create presenter channel\n")
			os.Exit(1)
		}

		go xmlPresenter(i, str, chn)

		chns[i] = chn
	}

	// no need for separate anonymous goroutine to wait until all presenters are done

	return chns
}

func CreateManifold(tbls *Tables, inp []<-chan Plex) <-chan Plex {

	if tbls == nil || inp == nil {
		return nil
	}

	out := make(chan Plex, tbls.ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create manifold channel\n")
		os.Exit(1)
	}

	// xmlManifold restores original order with heap
	xmlManifold := func(inp []<-chan Plex, out chan<- Plex) {

		// close channel when all records have been processed
		defer close(out)

		// initialize empty heap
		hp := &PlexHeap{}
		heap.Init(hp)

		// read first object from all input channels in turn
		for _, chn := range inp {
			plx, ok := <-chn
			if ok {
				heap.Push(hp, plx)
			}
		}

		// reading from heap returns objects in alphabetical order
		for hp.Len() > 0 {

			// remove lowest item from heap, use interface type assertion
			curr := heap.Pop(hp).(Plex)

			out <- Plex{curr.Which, curr.Ident, curr.Text}

			// read next object from channel that just supplied lowest item
			chn := inp[curr.Which]
			plx, ok := <-chn
			if ok {
				heap.Push(hp, plx)
			}
		}
	}

	// launch single manifold goroutine
	go xmlManifold(inp, out)

	return out
}

func CreateMerger(tbls *Tables, field string, inp <-chan Plex) <-chan Plex {

	if tbls == nil || inp == nil {
		return nil
	}

	out := make(chan Plex, tbls.ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create merger channel\n")
		os.Exit(1)
	}

	fuseInverts := func(key string, arry []string) string {

		chn := &Chain{}

		// merge farm variables
		FarmPos := 0
		FarmMax := tbls.FarmSize
		FarmItems := make([]Post, FarmMax)

		// allocate multiple posts in a large array for memory management efficiency
		nextPost := func(uid string) *Post {
			// if farm array slots used up, allocate new array
			if FarmPos >= FarmMax {
				FarmItems = make([]Post, FarmMax)
				FarmPos = 0
			}

			if FarmItems == nil {
				return nil
			}

			// take node from next available slot in farm array
			post := &FarmItems[FarmPos]

			post.UID = uid[:]

			FarmPos++

			return post
		}

		// add single posting to farm
		addPost := func(uid string) {

			if chn != nil {
				pst := nextPost(uid)
				if chn.First == nil {
					chn.First = pst
				}
				if chn.Last != nil {
					prv := chn.Last
					prv.Next = pst
				}
				chn.Last = pst
				chn.Count++
			}
		}

		addUID := func(tag, content string) {

			if tag == field {

				addPost(content)
			}
		}

		writeChain := func(chn *Chain) string {

			// sort each posting list in numeric order
			sorted := make([]string, chn.Count)
			i := 0
			for pst := chn.First; pst != nil; pst = pst.Next {
				sorted[i] = pst.UID
				i++
			}
			sort.Slice(sorted, func(i, j int) bool {
				// numeric sort on strings checks lengths first
				lni := len(sorted[i])
				lnj := len(sorted[j])
				// shorter string is numerically less, assuming no leading zeros
				if lni < lnj {
					return true
				}
				if lni > lnj {
					return false
				}
				// same length, can now do string comparison on contents
				return sorted[i] < sorted[j]
			})

			var buffer bytes.Buffer

			buffer.WriteString("  <InvDocument>\n")
			buffer.WriteString("    <InvKey>")
			buffer.WriteString(key)
			buffer.WriteString("</InvKey>\n")

			// print list of UIDs, skipping duplicates
			buffer.WriteString("    <InvIDs>\n")
			last := ""
			for _, uid := range sorted {
				// detect duplicate UIDs, now in same list after conversion of one term entry from foreign alphabet
				if uid != last {
					buffer.WriteString("      <")
					buffer.WriteString(field)
					buffer.WriteString(">")
					buffer.WriteString(uid)
					buffer.WriteString("</")
					buffer.WriteString(field)
					buffer.WriteString(">\n")
				}
				last = uid
			}
			buffer.WriteString("    </InvIDs>\n")
			buffer.WriteString("  </InvDocument>\n")

			txt := buffer.String()

			return txt
		}

		for _, str := range arry {
			ProcessXMLTokens(str[:], "InvDocument", 0, tbls, nil, addUID)
		}

		return writeChain(chn)
	}

	// xmlMerger fuses adjacent inverted records with the same identifier
	xmlMerger := func(inp <-chan Plex, out chan<- Plex) {

		// close channel when all records have been processed
		defer close(out)

		// remember previous record
		prev := Plex{}

		// array to collect strings with same identifier
		var arry []string

		for curr := range inp {

			if curr.Text == "" {
				continue
			}

			// compare adjacent record identifiers
			if prev.Ident == curr.Ident {

				// save next string in slice
				arry = append(arry, curr.Text)

			} else {

				if len(arry) > 0 {

					// if next identifier is different, fuse all saved components
					prev.Text = fuseInverts(prev.Ident, arry)

					// and send combined results to output channel
					out <- prev

					// empty the slice
					arry = nil
				}

				// now remember this record
				prev = curr

				// and save its string as the first entry in the slice
				arry = append(arry, curr.Text)
			}
		}

		if len(arry) > 0 {

			// fuse remaining saved components
			prev.Text = fuseInverts(prev.Ident, arry)

			// send last record
			out <- prev

			arry = nil
		}
	}

	// launch single merger goroutine
	go xmlMerger(inp, out)

	return out
}

func CreateSplitter(tbls *Tables, inp <-chan Plex) <-chan Plex {

	if tbls == nil || inp == nil {
		return nil
	}

	out := make(chan Plex, tbls.ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create splitter channel\n")
		os.Exit(1)
	}

	// xmlSplitter distributes adjacent records with the same identifier prefix
	xmlSplitter := func(inp <-chan Plex, out chan<- Plex) {

		// close channel when all records have been processed
		defer close(out)

		var buffer bytes.Buffer

		currTag := ""
		prevTag := ""

		// remember previous record
		prev := Plex{}

		buffer.WriteString("<InvDocumentSet>\n")

		for curr := range inp {

			// use first few characters of identifier
			currTag = IdentifierKey(curr.Ident)

			// then truncate to 2 character prefix
			if len(currTag) > 2 {
				currTag = currTag[:2]
			}

			// compare keys from adjacent term lists
			if prev.Text != "" && prevTag != currTag {

				// after IdentifierKey converts space to underscore,
				// okay that x_ and x0 will be out of alphabetical order

				buffer.WriteString("</InvDocumentSet>\n")

				// if prefix keys are different, send previous grouped records
				txt := buffer.String()

				if txt != "" {
					out <- Plex{0, prevTag, txt}
				}

				buffer.Reset()

				buffer.WriteString("<InvDocumentSet>\n")
			}

			str := strings.TrimSpace(curr.Text)

			// add result to growing slice
			buffer.WriteString("  ")
			buffer.WriteString(str)
			if !strings.HasSuffix(str, "\n") {
				buffer.WriteString("\n")
			}

			// now remember this record
			prev = curr

			prevTag = currTag
		}

		if prev.Text != "" {

			// send last record
			buffer.WriteString("</InvDocumentSet>\n\n")

			txt := buffer.String()

			if txt != "" {
				out <- Plex{0, prevTag, txt}
			}
		}

		buffer.Reset()
	}

	// launch single splitter goroutine
	go xmlSplitter(inp, out)

	return out
}

func CreateDividers(tbls *Tables, merg, field string, zipp bool, inp <-chan Plex) <-chan string {

	if tbls == nil || inp == nil {
		return nil
	}

	out := make(chan string, tbls.ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create divider channel\n")
		os.Exit(1)
	}

	var dlock sync.Mutex

	sfx := ".mrg"
	if zipp {
		sfx = ".mrg.gz"
	}

	// saveMerged writes merged inverted index objects to file accessed by term prefix
	saveMerged := func(key, text string) string {

		if key == "" {
			return ""
		}

		dlock.Lock()

		defer dlock.Unlock()

		fpath := path.Join(merg, field, key+sfx)
		if fpath == "" {
			return ""
		}

		// appends if file exists, otherwise creates
		fl, err := os.OpenFile(fpath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
		if err != nil {
			fmt.Println(err.Error())
			return ""
		}

		defer fl.Close()

		defer fl.Sync()

		var out io.Writer

		out = fl

		if zipp {

			zpr, err := gzip.NewWriterLevel(fl, gzip.BestSpeed)
			if err != nil {
				fmt.Println(err.Error())
				return ""
			}

			defer zpr.Close()

			out = zpr
		}

		// create buffered writer layer
		wrtr := bufio.NewWriter(out)

		defer wrtr.Flush()

		// copy record to file
		wrtr.WriteString(text)
		if !strings.HasSuffix(text, "\n") {
			wrtr.WriteString("\n")
		}

		return key
	}

	// xmlDivider reads from channel and calls saveMerged
	xmlDivider := func(wg *sync.WaitGroup, inp <-chan Plex, out chan<- string) {

		defer wg.Done()

		for plx := range inp {

			key := saveMerged(plx.Ident, plx.Text)

			out <- key
		}
	}

	var wg sync.WaitGroup

	// launch multiple divider goroutines
	for i := 0; i < tbls.NumServe; i++ {
		wg.Add(1)
		go xmlDivider(&wg, inp, out)
	}

	// launch separate anonymous goroutine to wait until all dividers are done
	go func() {
		wg.Wait()
		close(out)
	}()

	return out
}

func CreatePromoters(args []string, prom, field string, tbls *Tables) <-chan string {

	if args == nil || tbls == nil {
		return nil
	}

	out := make(chan string, tbls.ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create promoter channel\n")
		os.Exit(1)
	}

	// xmlPromoter splits inverted index groups into subdirectories for term lists and postings files
	xmlPromoter := func(wg *sync.WaitGroup, fileName string, out chan<- string) {

		defer wg.Done()

		f, err := os.Open(fileName)
		if err != nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to open input file '%s'\n", fileName)
			os.Exit(1)
		}

		// close input file when all records have been processed
		defer f.Close()

		var in io.Reader

		in = f

		// if suffix is ".gz", use decompressor
		iszip := false
		if strings.HasSuffix(fileName, ".gz") {
			iszip = true
		}

		if iszip {
			brd := bufio.NewReader(f)
			if brd == nil {
				fmt.Fprintf(os.Stderr, "\nERROR: Unable to create buffered reader on '%s'\n", fileName)
				os.Exit(1)
			}
			zpr, err := gzip.NewReader(brd)
			if err != nil {
				fmt.Fprintf(os.Stderr, "\nERROR: Unable to create decompressor on '%s'\n", fileName)
				os.Exit(1)
			}

			// close decompressor when all records have been processed
			defer zpr.Close()

			// use decompressor for reading file
			in = zpr
		}

		rdr := NewXMLReader(in, false, false, false)

		if rdr == nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to create XML Block Reader\n")
			os.Exit(1)
		}

		getOnePosting := func(text string) (string, []int32) {

			chn := &Chain{}

			// promoter farm variables
			FarmPos := 0
			FarmMax := tbls.FarmSize
			FarmItems := make([]Post, FarmMax)

			// allocate multiple posts in a large array for memory management efficiency
			nextPost := func(uid string) *Post {
				// if farm array slots used up, allocate new array
				if FarmPos >= FarmMax {
					FarmItems = make([]Post, FarmMax)
					FarmPos = 0
				}

				if FarmItems == nil {
					return nil
				}

				// take node from next available slot in farm array
				post := &FarmItems[FarmPos]

				post.UID = uid[:]

				FarmPos++

				return post
			}

			// add single posting to farm
			addPost := func(uid string) {

				if chn != nil {
					pst := nextPost(uid)
					if chn.First == nil {
						chn.First = pst
					}
					if chn.Last != nil {
						prv := chn.Last
						prv.Next = pst
					}
					chn.Last = pst
					chn.Count++
				}
			}

			term := ""

			doPromote := func(tag, content string) {

				if tag == "InvKey" {

					// term used for postings file name
					term = content

					term = strings.ToLower(term)

				} else if tag == field {

					addPost(content)
				}
			}

			// explore data fields
			ProcessXMLTokens(text[:], "InvDocument", 0, tbls, nil, doPromote)

			if term == "" || chn.Count < 1 {
				return "", nil
			}

			// populate binary array
			data := make([]int32, chn.Count)
			i := 0
			for pst := chn.First; pst != nil; pst = pst.Next {
				str := pst.UID
				value, err := strconv.ParseInt(str, 10, 32)
				if err != nil {
					fmt.Println(err.Error())
					return "", nil
				}
				data[i] = int32(value)
				i++
			}

			return term, data
		}

		var termPos int32
		var postPos int32

		var indxList bytes.Buffer
		var termList bytes.Buffer
		var postList bytes.Buffer

		retlength := len("\n")

		addOnePosting := func(term string, data []int32) {

			dlength := len(data)
			tlength := len(term)

			// write to term list buffer
			termList.WriteString(term[:])
			termList.WriteString("\n")

			// write to postings buffer
			binary.Write(&postList, binary.LittleEndian, data)

			// write to master index buffer
			binary.Write(&indxList, binary.LittleEndian, termPos)
			binary.Write(&indxList, binary.LittleEndian, postPos)

			postPos += int32(dlength * 4)
			termPos += int32(tlength + retlength)
		}

		topOffMaster := func() {

			// phantom term and postings positions eliminates special case calculation at end
			binary.Write(&indxList, binary.LittleEndian, termPos)
			binary.Write(&indxList, binary.LittleEndian, postPos)
		}

		writeFile := func(dpath, fname string, bfr bytes.Buffer) {

			fpath := path.Join(dpath, fname)
			if fpath == "" {
				return
			}

			/*
				ex, er := os.Open(fpath)
				if er == nil {
					ex.Close()
					fmt.Fprintf(os.Stderr, "File '%s' already exists\n", fpath)
				}
			*/

			fl, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0600)
			if err != nil {
				fmt.Println(err.Error())
				return
			}

			defer fl.Close()

			defer fl.Sync()

			data := bfr.Bytes()

			wrtr := bufio.NewWriter(fl)

			_, err = wrtr.Write(data)
			if err != nil {
				fmt.Println(err.Error())
			}

			wrtr.Flush()
		}

		writeThreeFiles := func(key string) {

			var arry [516]rune
			dpath, key := PostingPath(prom, field, key, arry)
			if dpath == "" {
				return
			}

			// make subdirectories, if necessary
			_, err := os.Stat(dpath)
			if err != nil && os.IsNotExist(err) {
				err = os.MkdirAll(dpath, os.ModePerm)
			}
			if err != nil {
				fmt.Println(err.Error())
				return
			}

			writeFile(dpath, key+".trm", termList)

			writeFile(dpath, key+".pst", postList)

			writeFile(dpath, key+".mst", indxList)
		}

		currTag := ""
		prevTag := ""

		ok := false

		PartitionPattern("InvDocument", "", rdr,
			func(rec int, ofs int64, str string) {

				term, data := getOnePosting(str)

				if term == "" || data == nil {
					return
				}

				ok = true

				// use first few characters of identifier
				currTag = IdentifierKey(term)

				if prevTag != currTag {

					// after IdentifierKey converts space to underscore,
					// okay that xxx_ and xxx0 will be out of alphabetical order

					// directory prefix changed from last posting
					if prevTag != "" {

						topOffMaster()
						writeThreeFiles(prevTag)
						out <- prevTag
					}

					// reset buffers and position counters
					termPos = 0
					postPos = 0

					indxList.Reset()
					termList.Reset()
					postList.Reset()

				}

				addOnePosting(term, data)

				prevTag = currTag
			})

		if ok {

			// write last set of files
			topOffMaster()
			writeThreeFiles(prevTag)
			out <- prevTag
		}
	}

	var wg sync.WaitGroup

	// launch multiple promoter goroutines
	for _, str := range args {
		wg.Add(1)
		go xmlPromoter(&wg, str, out)
	}

	// launch separate anonymous goroutine to wait until all promoters are done
	go func() {
		wg.Wait()
		close(out)
	}()

	return out
}

// MAIN FUNCTION

func main() {

	// skip past executable name
	args := os.Args[1:]

	if len(args) < 1 {
		fmt.Fprintf(os.Stderr, "\nERROR: No command-line arguments supplied to rchive\n")
		os.Exit(1)
	}

	// CONCURRENCY, CLEANUP, AND DEBUGGING FLAGS

	// do these first because -defcpu and -maxcpu can be sent from wrapper before other arguments

	ncpu := runtime.NumCPU()
	if ncpu < 1 {
		ncpu = 1
	}

	// wrapper can limit maximum number of processors to use (undocumented)
	maxProcs := ncpu
	defProcs := 0

	// concurrent performance tuning parameters, can be overridden by -proc and -cons
	numProcs := 0
	serverRatio := 4

	// number of servers usually calculated by -cons server ratio, but can be overridden by -serv
	numServers := 0

	// number of channels usually equals number of servers, but can be overridden by -chan
	chanDepth := 0

	// miscellaneous tuning parameters
	heapSize := 16
	farmSize := 64

	// garbage collector control can be set by environment variable or default value with -gogc 0
	goGc := 600

	// XML data cleanup
	doCompress := false
	doCleanup := false
	doStrict := false
	doMixed := false
	deAccent := false
	doASCII := false
	doRepair := false
	doStem := false
	doStop := true
	doSound := false

	// -flag sets -strict or -mixed cleanup flags from argument
	flgs := ""

	// read data from file instead of stdin
	fileName := ""

	// debugging
	timr := false

	// profiling
	prfl := false

	// element to use as local data index
	indx := ""

	// file of index values for removing duplicates
	unqe := ""

	// path for local data indexed as trie
	stsh := ""

	// field for inverted index
	nvrt := ""

	// field for merging index files
	merg := ""

	// base for promoting inverted index to retrieval indices
	prom := ""

	// field for promoting inverted index files
	fild := ""

	// base for queries
	base := ""

	// query by phrase, normalized terms (with truncation wildcarding), optionally using stemmed terms
	phrs := ""
	srch := ""

	// print term list with counts
	trms := ""

	// use gzip compression on local data files
	zipp := false

	// print UIDs and hash values
	hshv := false

	// convert UIDs to archive trie
	trei := false

	// compare input record against stash
	cmpr := false
	cmprType := ""
	ignr := ""

	// flag missing identifiers
	msng := false

	// kludge to use non-threaded fetching for windows
	windows := false

	// get numeric value
	getNumericArg := func(name string, zer, min, max int) int {

		if len(args) < 2 {
			fmt.Fprintf(os.Stderr, "\nERROR: %s is missing\n", name)
			os.Exit(1)
		}
		value, err := strconv.Atoi(args[1])
		if err != nil {
			fmt.Fprintf(os.Stderr, "\nERROR: %s (%s) is not an integer\n", name, args[1])
			os.Exit(1)
		}
		// skip past first of two arguments
		args = args[1:]

		// special case for argument value of 0
		if value < 1 {
			return zer
		}
		// limit value to between specified minimum and maximum
		if value < min {
			return min
		}
		if value > max {
			return max
		}
		return value
	}

	inSwitch := true

	// get concurrency, cleanup, and debugging flags in any order
	for {

		inSwitch = true

		switch args[0] {

		// concurrency override arguments can be passed in by local wrapper script (undocumented)
		case "-maxcpu":
			maxProcs = getNumericArg("Maximum number of processors", 1, 1, ncpu)
		case "-defcpu":
			defProcs = getNumericArg("Default number of processors", ncpu, 1, ncpu)
		// performance tuning flags
		case "-proc":
			numProcs = getNumericArg("Number of processors", ncpu, 1, ncpu)
		case "-cons":
			serverRatio = getNumericArg("Parser to processor ratio", 4, 1, 32)
		case "-serv":
			numServers = getNumericArg("Concurrent parser count", 0, ncpu, 128)
		case "-chan":
			chanDepth = getNumericArg("Communication channel depth", 0, ncpu, 128)
		case "-heap":
			heapSize = getNumericArg("Unshuffler heap size", 8, 8, 64)
		case "-farm":
			farmSize = getNumericArg("Node buffer length", 4, 4, 2048)
		case "-gogc":
			goGc = getNumericArg("Garbage collection percentage", 0, 100, 1000)

		// read data from file
		case "-input":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: Input file name is missing\n")
				os.Exit(1)
			}
			fileName = args[1]
			// skip past first of two arguments
			args = args[1:]

		// file with selected indexes for removing duplicates
		case "-unique":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: Unique identifier file is missing\n")
				os.Exit(1)
			}
			unqe = args[1]
			// skip past first of two arguments
			args = args[1:]

		// local directory path for indexing
		case "-archive", "-stash":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: Archive path is missing\n")
				os.Exit(1)
			}
			stsh = args[1]
			if stsh != "" && !strings.HasSuffix(stsh, "/") {
				stsh += "/"
			}
			// skip past first of two arguments
			args = args[1:]

		// data element for indexing
		case "-index":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: Index element is missing\n")
				os.Exit(1)
			}
			indx = args[1]
			// skip past first of two arguments
			args = args[1:]

		// build inverted index
		case "-invert":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: Invert field is missing\n")
				os.Exit(1)
			}
			nvrt = args[1]
			// skip past first of two arguments
			args = args[1:]

		// merge inverted index files, distribute by prefix
		case "-merge":
			if len(args) < 3 {
				fmt.Fprintf(os.Stderr, "\nERROR: Merge field is missing\n")
				os.Exit(1)
			}
			merg = args[1]
			fild = args[2]
			// skip past first and second arguments
			args = args[2:]

		// promote inverted index to term-specific postings files
		case "-promote":
			if len(args) < 3 {
				fmt.Fprintf(os.Stderr, "\nERROR: Promote path is missing\n")
				os.Exit(1)
			}
			prom = args[1]
			fild = args[2]
			// skip past first and second arguments
			args = args[2:]

		case "-phrase":
			if len(args) < 3 {
				fmt.Fprintf(os.Stderr, "\nERROR: Postings path is missing\n")
				os.Exit(1)
			}
			base = args[1]
			phrs = args[2]
			// skip past first and second arguments
			args = args[2:]

		case "-search":
			if len(args) < 3 {
				fmt.Fprintf(os.Stderr, "\nERROR: Postings path is missing\n")
				os.Exit(1)
			}
			base = args[1]
			srch = args[2]
			// skip past first and second arguments
			args = args[2:]

		case "-counts":
			if len(args) < 4 {
				fmt.Fprintf(os.Stderr, "\nERROR: Postings path is missing\n")
				os.Exit(1)
			}
			base = args[1]
			fild = args[2]
			trms = args[3]
			// skip past first three arguments
			args = args[3:]

		case "-gzip":
			zipp = true
		case "-hash":
			hshv = true
		case "-trie":
			trei = true
		// check for missing records
		case "-missing":
			msng = true

		// use non-threaded fetch function for windows (undocumented)
		case "-windows":
			windows = true

		// data cleanup flags
		case "-compress":
			doCompress = true
		case "-spaces", "-cleanup":
			doCleanup = true
		case "-strict":
			doStrict = true
		case "-mixed", "-relaxed":
			doMixed = true
		case "-accent", "-plain":
			deAccent = true
		case "-ascii":
			doASCII = true
		case "-repair":
			doRepair = true
		case "-stems":
			doStem = true
		case "-stops":
			doStop = false

		case "-flag", "-flags":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: Flags argument is missing\n")
				os.Exit(1)
			}
			flgs = args[1]
			// skip past first of two arguments
			args = args[1:]

		// debugging flags
		case "-prepare":
			cmpr = true
			if len(args) > 1 {
				next := args[1]
				// if next argument is not another flag
				if next != "" && next[0] != '-' {
					// get optional data source specifier
					cmprType = next
					// skip past first of two arguments
					args = args[1:]
				}
			}
		case "-ignore":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: -ignore value is missing\n")
				os.Exit(1)
			}
			ignr = args[1]
			// skip past first of two arguments
			args = args[1:]

		case "-timer":
			timr = true
		case "-profile":
			prfl = true

		default:
			// if not any of the controls, set flag to break out of for loop
			inSwitch = false
		}

		if !inSwitch {
			break
		}

		// skip past argument
		args = args[1:]

		if len(args) < 1 {
			break
		}
	}

	// -flag allows script to set -strict or -mixed from argument
	switch flgs {
	case "strict":
		doStrict = true
	case "mixed":
		doMixed = true
	case "none", "default":
	default:
		if flgs != "" {
			fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized -flag value '%s'\n", flgs)
			os.Exit(1)
		}
	}

	// reality checks on number of processors to use
	// performance degrades if capacity is above maximum number of partitions per second (context switching?)
	if numProcs == 0 {
		if defProcs > 0 {
			numProcs = defProcs
		} else {
			// best performance measurement with current code is obtained when 4 to 6 processors are assigned,
			// varying slightly among queries on PubmedArticle, gene DocumentSummary, and INSDSeq sequence records
			numProcs = 4
		}
	}
	if numProcs > ncpu {
		numProcs = ncpu
	}
	if numProcs > maxProcs {
		numProcs = maxProcs
	}

	// allow simultaneous threads for multiplexed goroutines
	runtime.GOMAXPROCS(numProcs)

	// adjust garbage collection target percentage
	if goGc >= 100 {
		debug.SetGCPercent(goGc)
	}

	// explicit -serv argument overrides -cons ratio
	if numServers > 0 {
		serverRatio = numServers / numProcs
		// if numServers / numProcs is not a whole number, do not print serverRatio in -stats
		if numServers != numProcs*serverRatio {
			serverRatio = 0
		}
	} else {
		numServers = numProcs * serverRatio
	}
	// server limits
	if numServers > 128 {
		numServers = 128
	} else if numServers < 1 {
		numServers = numProcs
	}

	// explicit -chan argument overrides default to number of servers
	if chanDepth == 0 {
		chanDepth = numServers
	}

	// if copying from local files accessed by identifier, add dummy argument to bypass length tests
	if stsh != "" && indx == "" {
		args = append(args, "-dummy")
	} else if phrs != "" || srch != "" || trms != "" {
		args = append(args, "-dummy")
	} else if trei || cmpr {
		args = append(args, "-dummy")
	}

	// expand -archive ~/ to home directory path
	if stsh != "" {

		if stsh[:2] == "~/" {
			cur, err := user.Current()
			if err == nil {
				hom := cur.HomeDir
				stsh = strings.Replace(stsh, "~/", hom+"/", 1)
			}
		}
	}

	// expand -promote ~/ to home directory path
	if prom != "" {

		if prom[:2] == "~/" {
			cur, err := user.Current()
			if err == nil {
				hom := cur.HomeDir
				prom = strings.Replace(prom, "~/", hom+"/", 1)
			}
		}
	}

	if len(args) < 1 {
		fmt.Fprintf(os.Stderr, "\nERROR: Insufficient command-line arguments supplied to rchive\n")
		os.Exit(1)
	}

	// DOCUMENTATION COMMANDS

	inSwitch = true

	switch args[0] {
	case "-version":
		fmt.Printf("%s\n", rchiveVersion)
	case "-help":
		fmt.Printf("rchive %s\n%s\n", rchiveVersion, rchiveHelp)
	case "-extras", "-extra":
		fmt.Printf("rchive %s\n%s\n", rchiveVersion, rchiveExtras)
	case "-advanced":
		fmt.Printf("rchive %s\n%s\n", rchiveVersion, rchiveAdvanced)
	case "-internal":
		fmt.Printf("rchive %s\n%s\n", rchiveVersion, rchiveInternal)
	default:
		// if not any of the documentation commands, keep going
		inSwitch = false
	}

	if inSwitch {
		return
	}

	// INITIALIZE TABLES

	tbls := InitTables()
	if tbls == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Problem creating token streamer lookup tables\n")
		os.Exit(1)
	}

	// additional fields passed in master table
	tbls.ChanDepth = chanDepth
	tbls.FarmSize = farmSize
	tbls.HeapSize = heapSize
	tbls.NumServe = numServers

	// transformation properties also saved in table
	tbls.DoStrict = doStrict
	tbls.DoMixed = doMixed
	tbls.DoRepair = doRepair
	tbls.DeAccent = deAccent
	tbls.DoASCII = doASCII
	tbls.DoStem = doStem
	tbls.DoStop = doStop
	tbls.DoSound = doSound

	// FILE NAME CAN BE SUPPLIED WITH -input COMMAND

	in := os.Stdin

	// check for data being piped into stdin
	isPipe := false
	fi, err := os.Stdin.Stat()
	if err == nil {
		isPipe = bool((fi.Mode() & os.ModeNamedPipe) != 0)
	}

	usingFile := false

	if fileName != "" {

		inFile, err := os.Open(fileName)
		if err != nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to open input file '%s'\n", fileName)
			os.Exit(1)
		}

		defer inFile.Close()

		// use indicated file instead of stdin
		in = inFile
		usingFile = true

		if isPipe && runtime.GOOS != "windows" {
			mode := fi.Mode().String()
			fmt.Fprintf(os.Stderr, "\nERROR: Input data from both stdin and file '%s', mode is '%s'\n", fileName, mode)
			os.Exit(1)
		}
	}

	// check for -input command after extraction arguments
	for _, str := range args {
		if str == "-input" {
			fmt.Fprintf(os.Stderr, "\nERROR: Misplaced -input command\n")
			os.Exit(1)
		}
	}

	// START PROFILING IF REQUESTED

	if prfl {

		f, err := os.Create("cpu.pprof")
		if err != nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to create profile output file\n")
			os.Exit(1)
		}

		pprof.StartCPUProfile(f)

		defer pprof.StopCPUProfile()
	}

	// INITIALIZE PROCESS TIMER AND RECORD COUNT

	startTime := time.Now()
	recordCount := 0
	byteCount := 0

	// print processing rate and program duration
	printDuration := func(name string) {

		stopTime := time.Now()
		duration := stopTime.Sub(startTime)
		seconds := float64(duration.Nanoseconds()) / 1e9

		if recordCount >= 1000000 {
			throughput := float64(recordCount/100000) / 10.0
			fmt.Fprintf(os.Stderr, "\nRchive processed %.1f million %s in %.3f seconds", throughput, name, seconds)
		} else {
			fmt.Fprintf(os.Stderr, "\nRchive processed %d %s in %.3f seconds", recordCount, name, seconds)
		}

		if seconds >= 0.001 && recordCount > 0 {
			rate := int(float64(recordCount) / seconds)
			if rate >= 1000000 {
				fmt.Fprintf(os.Stderr, " (%d mega%s/second", rate/1000000, name)
			} else {
				fmt.Fprintf(os.Stderr, " (%d %s/second", rate, name)
			}
			if byteCount > 0 {
				rate := int(float64(byteCount) / seconds)
				if rate >= 1000000 {
					fmt.Fprintf(os.Stderr, ", %d megabytes/second", rate/1000000)
				} else if rate >= 1000 {
					fmt.Fprintf(os.Stderr, ", %d kilobytes/second", rate/1000)
				} else {
					fmt.Fprintf(os.Stderr, ", %d bytes/second", rate)
				}
			}
			fmt.Fprintf(os.Stderr, ")")
		}

		fmt.Fprintf(os.Stderr, "\n\n")
	}

	// ENTREZ INDEX INVERSION

	// -invert PAIR *.e2x.gz reads IdxDocumentSet XML and creates an inverted index
	if nvrt != "" {

		invq := CreateInverters(args, nvrt, tbls)
		srtq := CreateSorters(tbls, nvrt, invq)
		rslq := CreateResolver(tbls, srtq)

		if invq == nil || srtq == nil || rslq == nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to create inverter\n")
			os.Exit(1)
		}

		var out io.Writer

		out = os.Stdout

		if zipp {

			zpr, err := gzip.NewWriterLevel(out, gzip.BestSpeed)
			if err != nil {
				fmt.Fprintf(os.Stderr, "\nERROR: Unable to create compressor\n")
				os.Exit(1)
			}

			// close decompressor when all records have been processed
			defer zpr.Close()

			// use compressor for writing file
			out = zpr
		}

		// create buffered writer layer
		wrtr := bufio.NewWriter(out)

		wrtr.WriteString("<InvDocumentSet>\n")

		// drain channel of alphabetized results
		for str := range rslq {

			// send result to output
			wrtr.WriteString(str)

			recordCount++
			runtime.Gosched()
		}

		wrtr.WriteString("</InvDocumentSet>\n\n")

		wrtr.Flush()

		debug.FreeOSMemory()

		if timr {
			printDuration("terms")
		}

		return
	}

	// MERGE INVERTED INDEX FILES AND GROUP BY TERM

	// -merge combines inverted files, distributes by prefix
	if merg != "" && fild != "" {

		chns := CreatePresenters(args, tbls)
		mfld := CreateManifold(tbls, chns)
		mrgr := CreateMerger(tbls, fild, mfld)
		sptr := CreateSplitter(tbls, mrgr)
		astr := CreateDividers(tbls, merg, fild, zipp, sptr)

		if chns == nil || mfld == nil || mrgr == nil || sptr == nil || astr == nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to create inverted index merger\n")
			os.Exit(1)
		}

		// drain channel, print two-character index name
		for str := range astr {

			fmt.Fprintf(os.Stdout, "%s\n", str)

			recordCount++
			runtime.Gosched()
		}

		debug.FreeOSMemory()

		if timr {
			printDuration("terms")
		}

		return
	}

	// PROMOTE MERGED INVERTED INDEX TO TERM LIST AND POSTINGS FILES

	if prom != "" && fild != "" {

		prmq := CreatePromoters(args, prom, fild, tbls)

		if prmq == nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to create new postings file generator\n")
			os.Exit(1)
		}

		// drain channel, print 2-4 character file prefix
		for str := range prmq {

			fmt.Fprintf(os.Stdout, "%s\n", str)

			recordCount++
			runtime.Gosched()
		}

		debug.FreeOSMemory()

		if timr {
			printDuration("terms")
		}

		return
	}

	// TEST POSTINGS FILES

	if base != "" && phrs != "" {

		// doStop set to true, to match default value used in building the PAIR or GRFT index
		recordCount = SearchForPhrase(base, phrs, doStem, doStop)

		debug.FreeOSMemory()

		if timr {
			printDuration("records")
		}

		return
	}

	if base != "" && srch != "" {

		// doStop set to true, to match default value used in building the NORM or STEM index
		recordCount = SearchForTerms(base, srch, doStem, doStop)

		debug.FreeOSMemory()

		if timr {
			printDuration("records")
		}

		return
	}

	if base != "" && trms != "" {

		recordCount = PrintTermCounts(base, fild, trms)

		debug.FreeOSMemory()

		if timr {
			printDuration("terms")
		}

		return
	}

	// CREATE XML BLOCK READER FROM STDIN OR FILE

	rdr := NewXMLReader(in, doCompress, doCleanup, doStrict || doMixed)
	if rdr == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create XML Block Reader\n")
		os.Exit(1)
	}

	// CONFIRM INPUT DATA AVAILABILITY AFTER RUNNING COMMAND GENERATORS

	if fileName == "" && runtime.GOOS != "windows" {

		fromStdin := bool((fi.Mode() & os.ModeCharDevice) == 0)
		if !isPipe || !fromStdin {
			mode := fi.Mode().String()
			fmt.Fprintf(os.Stderr, "\nERROR: No data supplied to rchive from stdin or file, mode is '%s'\n", mode)
			os.Exit(1)
		}
	}

	if !usingFile && !isPipe {

		fmt.Fprintf(os.Stderr, "\nERROR: No XML input data supplied to rchive\n")
		os.Exit(1)
	}

	// SPECIFY STRINGS TO GO BEFORE AND AFTER ENTIRE OUTPUT OR EACH RECORD

	head := ""
	tail := ""

	hd := ""
	tl := ""

	for {

		inSwitch = true

		switch args[0] {
		case "-head":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: Pattern missing after -head command\n")
				os.Exit(1)
			}
			head = ConvertSlash(args[1])
		case "-tail":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: Pattern missing after -tail command\n")
				os.Exit(1)
			}
			tail = ConvertSlash(args[1])
		case "-hd":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: Pattern missing after -hd command\n")
				os.Exit(1)
			}
			hd = ConvertSlash(args[1])
		case "-tl":
			if len(args) < 2 {
				fmt.Fprintf(os.Stderr, "\nERROR: Pattern missing after -tl command\n")
				os.Exit(1)
			}
			tl = ConvertSlash(args[1])
		default:
			// if not any of the controls, set flag to break out of for loop
			inSwitch = false
		}

		if !inSwitch {
			break
		}

		// skip past arguments
		args = args[2:]
	}

	// PRODUCE ARCHIVE SUBPATH FROM IDENTIFIER

	// -trie converts identifier to directory subpath plus file name (undocumented)
	if trei {

		scanr := bufio.NewScanner(rdr.Reader)

		sfx := ".xml"
		if zipp {
			sfx = ".xml.gz"
		}

		// read lines of identifiers
		for scanr.Scan() {

			file := scanr.Text()

			var arry [132]rune
			trie := MakeArchiveTrie(file, arry)
			if trie == "" || file == "" {
				continue
			}

			fpath := path.Join(trie, file+sfx)
			if fpath == "" {
				continue
			}

			os.Stdout.WriteString(fpath)
			os.Stdout.WriteString("\n")
		}

		return
	}

	// CHECK FOR MISSING RECORDS IN LOCAL DIRECTORY INDEXED BY TRIE ON IDENTIFIER

	// -archive plus -missing checks for missing records
	if stsh != "" && msng {

		versioned := ReadVersionedList(stsh)

		scanr := bufio.NewScanner(rdr.Reader)

		sfx := ".xml"
		if zipp {
			sfx = ".xml.gz"
		}

		// read lines of identifiers
		for scanr.Scan() {

			file := scanr.Text()

			if !strings.Contains(file, ".") && (versioned == nil || !versioned[file]) {
				// add default version
				file += ".1"
			}

			var arry [132]rune
			file, trie := GetAppropriateVersion(stsh, file, arry)

			if file == "" || trie == "" {
				continue
			}

			fpath := path.Join(stsh, trie, file+sfx)
			if fpath == "" {
				continue
			}

			_, err := os.Stat(fpath)

			// if failed to find ".xml" file, try ".xml.gz" without requiring -gzip
			if err != nil && os.IsNotExist(err) && !zipp {
				fpath := path.Join(stsh, trie, file+".xml.gz")
				if fpath == "" {
					continue
				}
				_, err = os.Stat(fpath)
			}
			if err != nil && os.IsNotExist(err) {
				// record is missing from local file cache
				os.Stdout.WriteString(file)
				os.Stdout.WriteString("\n")
			}
		}

		return
	}

	// RETRIEVE XML COMPONENT RECORDS FROM LOCAL DIRECTORY INDEXED BY TRIE ON IDENTIFIER

	// alternative windows version limits memory by not using goroutines
	if stsh != "" && indx == "" && runtime.GOOS == "windows" && windows {

		versioned := ReadVersionedList(stsh)

		scanr := bufio.NewScanner(rdr.Reader)
		if scanr == nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to create UID scanner\n")
			os.Exit(1)
		}

		sfx := ".xml"
		if zipp {
			sfx = ".xml.gz"
		}

		if head != "" {
			os.Stdout.WriteString(head)
			os.Stdout.WriteString("\n")
		}

		var buf bytes.Buffer

		for scanr.Scan() {

			// read next identifier
			file := scanr.Text()

			if !strings.Contains(file, ".") && (versioned == nil || !versioned[file]) {
				// add default version
				file += ".1"
			}

			var arry [132]rune
			file, trie := GetAppropriateVersion(stsh, file, arry)

			if file == "" || trie == "" {
				continue
			}

			fpath := path.Join(stsh, trie, file+sfx)
			if fpath == "" {
				continue
			}

			iszip := zipp

			inFile, err := os.Open(fpath)

			// if failed to find ".xml" file, try ".xml.gz" without requiring -gzip
			if err != nil && os.IsNotExist(err) && !zipp {
				iszip = true
				fpath := path.Join(stsh, trie, file+".xml.gz")
				if fpath == "" {
					continue
				}
				inFile, err = os.Open(fpath)
			}
			if err != nil {
				continue
			}

			buf.Reset()

			brd := bufio.NewReader(inFile)

			if iszip {

				zpr, err := gzip.NewReader(brd)

				if err == nil {
					// copy and decompress cached file contents
					buf.ReadFrom(zpr)
				}

				zpr.Close()

			} else {

				// copy cached file contents
				buf.ReadFrom(brd)
			}

			inFile.Close()

			str := buf.String()

			if str == "" {
				continue
			}

			recordCount++

			if hd != "" {
				os.Stdout.WriteString(hd)
				os.Stdout.WriteString("\n")
			}

			if hshv {
				// calculate hash code for verification table
				hsh := crc32.NewIEEE()
				hsh.Write([]byte(str))
				val := hsh.Sum32()
				res := strconv.FormatUint(uint64(val), 10)
				txt := file + "\t" + res + "\n"
				os.Stdout.WriteString(txt)
			} else {
				// send result to output
				os.Stdout.WriteString(str)
				if !strings.HasSuffix(str, "\n") {
					os.Stdout.WriteString("\n")
				}
			}

			if tl != "" {
				os.Stdout.WriteString(tl)
				os.Stdout.WriteString("\n")
			}

			debug.FreeOSMemory()
		}

		if tail != "" {
			os.Stdout.WriteString(tail)
			os.Stdout.WriteString("\n")
		}

		debug.FreeOSMemory()

		if timr {
			printDuration("records")
		}

		return
	}

	// -archive without -index retrieves XML files in trie-based directory structure
	if stsh != "" && indx == "" {

		uidq := CreateUIDReader(rdr.Reader, stsh, tbls)
		strq := CreateFetchers(tbls, stsh, zipp, uidq)
		unsq := CreateUnshuffler(tbls, strq)

		if uidq == nil || strq == nil || unsq == nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to create stash reader\n")
			os.Exit(1)
		}

		if head != "" {
			os.Stdout.WriteString(head)
			os.Stdout.WriteString("\n")
		}

		// drain output channel
		for curr := range unsq {

			str := curr.Text

			if str == "" {
				continue
			}

			recordCount++

			if hd != "" {
				os.Stdout.WriteString(hd)
				os.Stdout.WriteString("\n")
			}

			if hshv {
				// calculate hash code for verification table
				hsh := crc32.NewIEEE()
				hsh.Write([]byte(str))
				val := hsh.Sum32()
				res := strconv.FormatUint(uint64(val), 10)
				txt := curr.Ident + "\t" + res + "\n"
				os.Stdout.WriteString(txt)
			} else {
				// send result to output
				os.Stdout.WriteString(str)
				if !strings.HasSuffix(str, "\n") {
					os.Stdout.WriteString("\n")
				}
			}

			if tl != "" {
				os.Stdout.WriteString(tl)
				os.Stdout.WriteString("\n")
			}
		}

		if tail != "" {
			os.Stdout.WriteString(tail)
			os.Stdout.WriteString("\n")
		}

		debug.FreeOSMemory()

		if timr {
			printDuration("records")
		}

		return
	}

	// ENSURE PRESENCE OF PATTERN ARGUMENT

	if len(args) < 1 {
		fmt.Fprintf(os.Stderr, "\nERROR: Insufficient command-line arguments supplied to rchive\n")
		os.Exit(1)
	}

	// allow -record as synonym of -pattern (undocumented)
	if args[0] == "-record" || args[0] == "-Record" {
		args[0] = "-pattern"
	}

	// make sure top-level -pattern command is next
	if args[0] != "-pattern" && args[0] != "-Pattern" {
		fmt.Fprintf(os.Stderr, "\nERROR: No -pattern in command-line arguments\n")
		os.Exit(1)
	}
	if len(args) < 2 {
		fmt.Fprintf(os.Stderr, "\nERROR: Item missing after -pattern command\n")
		os.Exit(1)
	}

	topPat := args[1]
	if topPat == "" {
		fmt.Fprintf(os.Stderr, "\nERROR: Item missing after -pattern command\n")
		os.Exit(1)
	}
	if strings.HasPrefix(topPat, "-") {
		fmt.Fprintf(os.Stderr, "\nERROR: Misplaced %s command\n", topPat)
		os.Exit(1)
	}

	// look for -pattern Parent/* construct for heterogeneous data, e.g., -pattern PubmedArticleSet/*
	topPattern, star := SplitInTwoAt(topPat, "/", LEFT)
	if topPattern == "" {
		return
	}

	parent := ""
	if star == "*" {
		parent = topPattern
	} else if star != "" {
		fmt.Fprintf(os.Stderr, "\nERROR: -pattern Parent/Child construct is not supported\n")
		os.Exit(1)
	}

	// COMPARE XML UPDATES TO LOCAL DIRECTORY, RETAIN NEW OR SUBSTANTIVELY CHANGED RECORDS

	// -prepare plus -archive plus -index plus -pattern compares XML files against stash
	if stsh != "" && indx != "" && cmpr {

		doReport := false
		if cmprType == "" || cmprType == "report" {
			doReport = true
		} else if cmprType != "release" {
			fmt.Fprintf(os.Stderr, "\nERROR: -prepare argument must be release or report\n")
			os.Exit(1)
		}

		versioned := ReadVersionedList(stsh)

		find := ParseIndex(indx)

		if head != "" {
			os.Stdout.WriteString(head)
			os.Stdout.WriteString("\n")
		}

		PartitionPattern(topPattern, star, rdr,
			func(rec int, ofs int64, str string) {
				recordCount++

				id := ProcessXMLTokens(str[:], parent, rec, tbls, find, nil)
				if id == "" {
					return
				}

				if !strings.Contains(id, ".") && (versioned == nil || !versioned[id]) {
					// add default version
					id += ".1"
				}

				var arry [132]rune
				id, trie := GetAppropriateVersion(stsh, id, arry)

				if id == "" || trie == "" {
					return
				}

				fpath := path.Join(stsh, trie, id+".xml")
				if fpath == "" {
					return
				}

				// print new or updated XML record
				printRecord := func(stn string, isNew bool) {

					if stn == "" {
						return
					}

					if doReport {
						if isNew {
							os.Stdout.WriteString("NW ")
							os.Stdout.WriteString(id)
							os.Stdout.WriteString("\n")
						} else {
							os.Stdout.WriteString("UP ")
							os.Stdout.WriteString(id)
							os.Stdout.WriteString("\n")
						}
						return
					}

					if hd != "" {
						os.Stdout.WriteString(hd)
						os.Stdout.WriteString("\n")
					}

					os.Stdout.WriteString(stn)
					os.Stdout.WriteString("\n")

					if tl != "" {
						os.Stdout.WriteString(tl)
						os.Stdout.WriteString("\n")
					}
				}

				_, err := os.Stat(fpath)
				if err != nil && os.IsNotExist(err) {
					// new record
					printRecord(str, true)
					return
				}
				if err != nil {
					return
				}

				buf, err := ioutil.ReadFile(fpath)
				if err != nil {
					return
				}

				txt := string(buf[:])
				if strings.HasSuffix(txt, "\n") {
					tlen := len(txt)
					txt = txt[:tlen-1]
				}

				// check for optional -ignore argument
				if ignr != "" {

					// ignore differences inside specified object
					ltag := "<" + ignr + ">"
					sleft, _ := SplitInTwoAt(str, ltag, LEFT)
					tleft, _ := SplitInTwoAt(txt, ltag, LEFT)

					rtag := "</" + ignr + ">"
					_, srght := SplitInTwoAt(str, rtag, RIGHT)
					_, trght := SplitInTwoAt(txt, rtag, RIGHT)

					if sleft == tleft && srght == trght {
						if doReport {
							os.Stdout.WriteString("NO ")
							os.Stdout.WriteString(id)
							os.Stdout.WriteString("\n")
						}
						return
					}

				} else {

					// compare entirety of objects
					if str == txt {
						if doReport {
							os.Stdout.WriteString("NO ")
							os.Stdout.WriteString(id)
							os.Stdout.WriteString("\n")
						}
						return
					}
				}

				// substantively modified record
				printRecord(str, false)
			})

		if tail != "" {
			os.Stdout.WriteString(tail)
			os.Stdout.WriteString("\n")
		}

		if timr {
			printDuration("records")
		}

		return
	}

	// SAVE XML COMPONENT RECORDS TO LOCAL DIRECTORY INDEXED BY TRIE ON IDENTIFIER

	// -archive plus -index plus -pattern saves XML files in trie-based directory structure
	if stsh != "" && indx != "" {

		xmlq := CreateProducer(topPattern, star, rdr, tbls)
		stsq := CreateStashers(tbls, stsh, parent, indx, hshv, zipp, xmlq)

		if xmlq == nil || stsq == nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to create stash generator\n")
			os.Exit(1)
		}

		// drain output channel
		for str := range stsq {

			if hshv {
				// print table of UIDs and hash values
				os.Stdout.WriteString(str)
			}

			recordCount++
			runtime.Gosched()
		}

		debug.FreeOSMemory()

		if timr {
			printDuration("records")
		}

		return
	}

	// READ FILE OF IDENTIFIERS AND EXTRACT SELECTED RECORDS FROM XML INPUT FILE

	// -index plus -unique [plus -head/-tail/-hd/-tl] plus -pattern with no other extraction arguments
	// takes an XML input file and a file of its UIDs and keeps only the last version of each record
	if indx != "" && unqe != "" && len(args) == 2 {

		// read file of identifiers to use for filtering
		fl, err := os.Open(unqe)
		if err != nil {
			fmt.Fprintf(os.Stderr, "\nERROR: Unable to open identifier file '%s'\n", unqe)
			os.Exit(1)
		}

		// create map that counts instances of each UID
		order := make(map[string]int)

		scanr := bufio.NewScanner(fl)

		// read lines of identifiers
		for scanr.Scan() {

			id := scanr.Text()

			// map records count for given identifier
			val := order[id]
			val++
			order[id] = val
		}

		fl.Close()

		find := ParseIndex(indx)

		if head != "" {
			os.Stdout.WriteString(head)
			os.Stdout.WriteString("\n")
		}

		PartitionPattern(topPattern, star, rdr,
			func(rec int, ofs int64, str string) {
				recordCount++

				id := ProcessXMLTokens(str[:], parent, rec, tbls, find, nil)
				if id == "" {
					return
				}

				val, ok := order[id]
				if !ok {
					// not in identifier list, skip
					return
				}
				// decrement count in map
				val--
				order[id] = val
				if val > 0 {
					// only write last record with a given identifier
					return
				}

				if hd != "" {
					os.Stdout.WriteString(hd)
					os.Stdout.WriteString("\n")
				}

				// write selected record
				os.Stdout.WriteString(str[:])
				os.Stdout.WriteString("\n")

				if tl != "" {
					os.Stdout.WriteString(tl)
					os.Stdout.WriteString("\n")
				}
			})

		if tail != "" {
			os.Stdout.WriteString(tail)
			os.Stdout.WriteString("\n")
		}

		if timr {
			printDuration("records")
		}

		return
	}

	// GENERATE RECORD INDEX ON XML INPUT FILE

	// -index plus -pattern prints record identifier, file offset, and XML size
	if indx != "" {

		lbl := ""
		// check for optional filename label after -pattern argument (undocumented)
		if len(args) > 3 && args[2] == "-lbl" {
			lbl = args[3]

			lbl = strings.TrimSpace(lbl)
			if strings.HasPrefix(lbl, "pubmed") {
				lbl = lbl[7:]
			}
			if strings.HasSuffix(lbl, ".xml.gz") {
				xlen := len(lbl)
				lbl = lbl[:xlen-7]
			}
			lbl = strings.TrimSpace(lbl)
		}

		// legend := "ID\tREC\tOFST\tSIZE"

		find := ParseIndex(indx)

		PartitionPattern(topPattern, star, rdr,
			func(rec int, ofs int64, str string) {
				recordCount++

				id := ProcessXMLTokens(str[:], parent, rec, tbls, find, nil)
				if id == "" {
					return
				}
				if lbl != "" {
					fmt.Printf("%s\t%d\t%d\t%d\t%s\n", id, rec, ofs, len(str), lbl)
				} else {
					fmt.Printf("%s\t%d\t%d\t%d\n", id, rec, ofs, len(str))
				}
			})

		if timr {
			printDuration("records")
		}

		return
	}

	// REPORT UNRECOGNIZED COMMAND

	fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized rchive command\n")
	os.Exit(1)
}
