// ===========================================================================
//
//                            PUBLIC DOMAIN NOTICE
//            National Center for Biotechnology Information (NCBI)
//
//  This software/database is a "United States Government Work" under the
//  terms of the United States Copyright Act. It was written as part of
//  the author's official duties as a United States Government employee and
//  thus cannot be copyrighted. This software/database is freely available
//  to the public for use. The National Library of Medicine and the U.S.
//  Government do not place any restriction on its use or reproduction.
//  We would, however, appreciate having the NCBI and the author cited in
//  any work or product based on this material.
//
//  Although all reasonable efforts have been taken to ensure the accuracy
//  and reliability of the software and data, the NLM and the U.S.
//  Government do not and cannot warrant the performance or results that
//  may be obtained by using this software or data. The NLM and the U.S.
//  Government disclaim all warranties, express or implied, including
//  warranties of performance, merchantability or fitness for any particular
//  purpose.
//
// ===========================================================================
//
// File Name:  common.go
//
// Author:  Jonathan Kans
//
// ==========================================================================

/*
  Download external GO libraries by running:

  cd "$GOPATH"
  go get -u github.com/antzucaro/matchr
  go get -u github.com/fiam/gounidecode/unidecode
  go get -u github.com/surgebase/porter2
  go get -u golang.org/x/text/runes
  go get -u golang.org/x/text/transform
  go get -u golang.org/x/text/unicode/norm

  Test for presence of go compiler, and cross-compile xtract and rchive executables, by running:

  if hash go 2>/dev/null
  then
    osname=`uname -s`
    osname="${osname:0:3}"
    case "$osname" in
      Dar )
        env GOOS=darwin GOARCH=amd64 go build -o xtract.Darwin xtract.go common.go
        env GOOS=darwin GOARCH=amd64 go build -o rchive.Darwin rchive.go common.go
      ;;
      Lin )
        env GOOS=linux GOARCH=amd64 go build -o xtract.Linux xtract.go common.go
        env GOOS=linux GOARCH=amd64 go build -o rchive.Linux rchive.go common.go
      ;;
      CYG | MIN )
        env GOOS=windows GOARCH=386 go build -o xtract.CYGWIN_NT xtract.go common.go
        env GOOS=windows GOARCH=386 go build -o rchive.CYGWIN_NT rchive.go common.go
      ;;
    esac
  fi
*/

package main

import (
	"bytes"
	"container/heap"
	"fmt"
	"golang.org/x/text/runes"
	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
	"io"
	"os"
	"strconv"
	"strings"
	"sync"
	"unicode"
)

// TYPED CONSTANTS

type SideType int

const (
	_ SideType = iota
	LEFT
	RIGHT
)

type TagType int

const (
	NOTAG TagType = iota
	STARTTAG
	SELFTAG
	STOPTAG
	ATTRIBTAG
	CONTENTTAG
	CDATATAG
	COMMENTTAG
	DOCTYPETAG
	OBJECTTAG
	CONTAINERTAG
	ISCLOSED
)

// ARGUMENT MAPS

var markupRunes = map[rune]rune{
	'\u00B2': '2',
	'\u00B3': '3',
	'\u00B9': '1',
	'\u2070': '0',
	'\u2071': '1',
	'\u2074': '4',
	'\u2075': '5',
	'\u2076': '6',
	'\u2077': '7',
	'\u2078': '8',
	'\u2079': '9',
	'\u207A': '+',
	'\u207B': '-',
	'\u207C': '=',
	'\u207D': '(',
	'\u207E': ')',
	'\u207F': 'n',
	'\u2080': '0',
	'\u2081': '1',
	'\u2082': '2',
	'\u2083': '3',
	'\u2084': '4',
	'\u2085': '5',
	'\u2086': '6',
	'\u2087': '7',
	'\u2088': '8',
	'\u2089': '9',
	'\u208A': '+',
	'\u208B': '-',
	'\u208C': '=',
	'\u208D': '(',
	'\u208E': ')',
}

var accentRunes = map[rune]rune{
	'\u00D8': 'O',
	'\u00F0': 'd',
	'\u00F8': 'o',
	'\u0111': 'd',
	'\u0131': 'i',
	'\u0141': 'L',
	'\u0142': 'l',
	'\u02BC': '\'',
}

var ligatureRunes = map[rune]string{
	'\u00DF': "ss",
	'\u00E6': "ae",
	'\uFB00': "ff",
	'\uFB01': "fi",
	'\uFB02': "fl",
	'\uFB03': "ffi",
	'\uFB04': "ffl",
	'\uFB05': "ft",
	'\uFB06': "st",
}

var greekRunes = map[rune]string{
	'\u0190': "epsilon",
	'\u025B': "epsilon",
	'\u03B1': "alpha",
	'\u03B2': "beta",
	'\u03B3': "gamma",
	'\u03B4': "delta",
	'\u03B5': "epsilon",
	'\u03B6': "zeta",
	'\u03B7': "eta",
	'\u03B8': "theta",
	'\u03B9': "iota",
	'\u03BA': "kappa",
	'\u03BB': "lambda",
	'\u03BC': "mu",
	'\u03BD': "nu",
	'\u03BE': "xi",
	'\u03BF': "omicron",
	'\u03C0': "pi",
	'\u03C1': "rho",
	'\u03C3': "sigma",
	'\u03C4': "tau",
	'\u03C5': "upsilon",
	'\u03C6': "phi",
	'\u03C7': "chi",
	'\u03C8': "psi",
	'\u03C9': "omega",
	'\u0391': "alpha",
	'\u0392': "beta",
	'\u0393': "gamma",
	'\u0394': "delta",
	'\u0395': "epsilon",
	'\u0396': "zeta",
	'\u0397': "eta",
	'\u0398': "theta",
	'\u0399': "iota",
	'\u039A': "kappa",
	'\u039B': "lambda",
	'\u039C': "mu",
	'\u039D': "nu",
	'\u039E': "xi",
	'\u039F': "omicron",
	'\u03A0': "pi",
	'\u03A1': "rho",
	'\u03A3': "sigma",
	'\u03A4': "tau",
	'\u03A5': "upsilon",
	'\u03A6': "phi",
	'\u03A7': "chi",
	'\u03A8': "psi",
	'\u03A9': "omega",
}

var isStopWord = map[string]bool{
	"!":             true,
	"\"":            true,
	"#":             true,
	"$":             true,
	"%":             true,
	"&":             true,
	"'":             true,
	"(":             true,
	")":             true,
	"*":             true,
	"+":             true,
	",":             true,
	"-":             true,
	".":             true,
	"/":             true,
	":":             true,
	";":             true,
	"<":             true,
	"=":             true,
	">":             true,
	"?":             true,
	"@":             true,
	"[":             true,
	"\\":            true,
	"]":             true,
	"^":             true,
	"_":             true,
	"`":             true,
	"{":             true,
	"|":             true,
	"}":             true,
	"~":             true,
	"a":             true,
	"about":         true,
	"again":         true,
	"all":           true,
	"almost":        true,
	"also":          true,
	"although":      true,
	"always":        true,
	"among":         true,
	"an":            true,
	"and":           true,
	"another":       true,
	"any":           true,
	"are":           true,
	"as":            true,
	"at":            true,
	"be":            true,
	"because":       true,
	"been":          true,
	"before":        true,
	"being":         true,
	"between":       true,
	"both":          true,
	"but":           true,
	"by":            true,
	"can":           true,
	"could":         true,
	"did":           true,
	"do":            true,
	"does":          true,
	"done":          true,
	"due":           true,
	"during":        true,
	"each":          true,
	"either":        true,
	"enough":        true,
	"especially":    true,
	"etc":           true,
	"for":           true,
	"found":         true,
	"from":          true,
	"further":       true,
	"had":           true,
	"has":           true,
	"have":          true,
	"having":        true,
	"here":          true,
	"how":           true,
	"however":       true,
	"i":             true,
	"if":            true,
	"in":            true,
	"into":          true,
	"is":            true,
	"it":            true,
	"its":           true,
	"itself":        true,
	"just":          true,
	"kg":            true,
	"km":            true,
	"made":          true,
	"mainly":        true,
	"make":          true,
	"may":           true,
	"mg":            true,
	"might":         true,
	"ml":            true,
	"mm":            true,
	"most":          true,
	"mostly":        true,
	"must":          true,
	"nearly":        true,
	"neither":       true,
	"no":            true,
	"nor":           true,
	"obtained":      true,
	"of":            true,
	"often":         true,
	"on":            true,
	"our":           true,
	"overall":       true,
	"perhaps":       true,
	"pmid":          true,
	"quite":         true,
	"rather":        true,
	"really":        true,
	"regarding":     true,
	"seem":          true,
	"seen":          true,
	"several":       true,
	"should":        true,
	"show":          true,
	"showed":        true,
	"shown":         true,
	"shows":         true,
	"significantly": true,
	"since":         true,
	"so":            true,
	"some":          true,
	"such":          true,
	"than":          true,
	"that":          true,
	"the":           true,
	"their":         true,
	"theirs":        true,
	"them":          true,
	"then":          true,
	"there":         true,
	"therefore":     true,
	"these":         true,
	"they":          true,
	"this":          true,
	"those":         true,
	"through":       true,
	"thus":          true,
	"to":            true,
	"upon":          true,
	"use":           true,
	"used":          true,
	"using":         true,
	"various":       true,
	"very":          true,
	"was":           true,
	"we":            true,
	"were":          true,
	"what":          true,
	"when":          true,
	"which":         true,
	"while":         true,
	"with":          true,
	"within":        true,
	"without":       true,
	"would":         true,
}

// DATA OBJECTS

type Tables struct {
	InBlank   [256]bool
	AltBlank  [256]bool
	InFirst   [256]bool
	InElement [256]bool
	ChanDepth int
	FarmSize  int
	HeapSize  int
	NumServe  int
	DoStrict  bool
	DoMixed   bool
	DoRepair  bool
	DeAccent  bool
	DoASCII   bool
	DoStem    bool
	DoStop    bool
	DoSound   bool
}

// UTILITIES

func IsNotJustWhitespace(str string) bool {

	for _, ch := range str {
		if ch != ' ' && ch != '\t' && ch != '\n' && ch != '\r' && ch != '\f' {
			return true
		}
	}

	return false
}

func IsNotASCII(str string) bool {

	for _, ch := range str {
		if ch > 127 {
			return true
		}
	}

	return false
}

func HasAmpOrNotASCII(str string) bool {

	for _, ch := range str {
		if ch == '&' || ch > 127 {
			return true
		}
	}

	return false
}

func IsAllCapsOrDigits(str string) bool {

	for _, ch := range str {
		if !unicode.IsUpper(ch) && !unicode.IsDigit(ch) {
			return false
		}
	}

	return true
}

func IsAllDigitsOrPeriod(str string) bool {

	for _, ch := range str {
		if !unicode.IsDigit(ch) && ch != '.' {
			return false
		}
	}

	return true
}

func IsAllNumeric(str string) bool {

	for _, ch := range str {
		if !unicode.IsDigit(ch) &&
			ch != '.' &&
			ch != '+' &&
			ch != '-' &&
			ch != '*' &&
			ch != '/' &&
			ch != ',' &&
			ch != '$' &&
			ch != '#' &&
			ch != '%' &&
			ch != '(' &&
			ch != ')' {
			return false
		}
	}

	return true
}

func HasAngleBracket(str string) bool {

	hasAmp := false
	hasSemi := false

	for _, ch := range str {
		if ch == '<' || ch == '>' {
			return true
		} else if ch == '&' {
			hasAmp = true
		} else if ch == ';' {
			hasSemi = true
		}
	}

	if hasAmp && hasSemi {
		if strings.Contains(str, "&lt;") || strings.Contains(str, "&gt;") || strings.Contains(str, "&amp;") {
			return true
		}
	}

	return false
}

func CompressRunsOfSpaces(str string) string {

	whiteSpace := false
	var buffer bytes.Buffer

	for _, ch := range str {
		if unicode.IsSpace(ch) {
			if !whiteSpace {
				buffer.WriteRune(' ')
			}
			whiteSpace = true
		} else {
			buffer.WriteRune(ch)
			whiteSpace = false
		}
	}

	return buffer.String()
}

func HasFlankingSpace(str string) bool {

	if str == "" {
		return false
	}

	ch := str[0]
	if ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' || ch == '\f' {
		return true
	}

	strlen := len(str)
	ch = str[strlen-1]
	if ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' || ch == '\f' {
		return true
	}

	return false
}

func HasBadSpace(str string) bool {

	for _, ch := range str {
		if unicode.IsSpace(ch) && ch != ' ' {
			return true
		}
	}

	return false
}

func CleanupBadSpaces(str string) string {

	var buffer bytes.Buffer

	for _, ch := range str {
		if unicode.IsSpace(ch) {
			buffer.WriteRune(' ')
		} else {
			buffer.WriteRune(ch)
		}
	}

	return buffer.String()
}

func HTMLAhead(text string, pos int) int {

	max := len(text) - pos

	if max > 2 && text[pos+2] == '>' {
		ch := text[pos+1]
		if ch == 'i' || ch == 'b' || ch == 'u' {
			return 3
		}
	} else if max > 3 && text[pos+3] == '>' {
		if text[pos+1] == '/' {
			ch := text[pos+2]
			if ch == 'i' || ch == 'b' || ch == 'u' {
				return 4
			}
		}
		if text[pos+2] == '/' {
			ch := text[pos+1]
			if ch == 'i' || ch == 'b' || ch == 'u' {
				return 4
			}
		}
	} else if max > 4 && text[pos+4] == '>' {
		if text[pos+1] == 's' && text[pos+2] == 'u' {
			ch := text[pos+3]
			if ch == 'p' || ch == 'b' {
				return 5
			}
		}
		/*
			if text[pos+3] == '/' && text[pos+2] == ' ' {
				ch := text[pos+1]
				if ch == 'i' || ch == 'b' || ch == 'u' {
					return 5
				}
			}
		*/
	} else if max > 5 && text[pos+5] == '>' {
		if text[pos+1] == '/' && text[pos+2] == 's' && text[pos+3] == 'u' {
			ch := text[pos+4]
			if ch == 'p' || ch == 'b' {
				return 6
			}
		}
		if text[pos+4] == '/' && text[pos+1] == 's' && text[pos+2] == 'u' {
			ch := text[pos+3]
			if ch == 'p' || ch == 'b' {
				return 6
			}
		}
		/*
			} else if max > 6 && text[pos+6] == '>' {
				if text[pos+5] == '/' && text[pos+4] == ' ' && text[pos+1] == 's' && text[pos+2] == 'u' {
					ch := text[pos+3]
					if ch == 'p' || ch == 'b' {
						return 7
					}
				}
		*/
	}

	return 0
}

func HTMLBehind(bufr []byte, pos int) bool {

	if pos > 1 && bufr[pos-2] == '<' {
		ch := bufr[pos-1]
		if ch == 'i' || ch == 'b' || ch == 'u' {
			return true
		}
	} else if pos > 2 && bufr[pos-3] == '<' {
		if bufr[pos-2] == '/' {
			ch := bufr[pos-1]
			if ch == 'i' || ch == 'b' || ch == 'u' {
				return true
			}
		}
		if bufr[pos-1] == '/' {
			ch := bufr[pos-2]
			if ch == 'i' || ch == 'b' || ch == 'u' {
				return true
			}
		}
	} else if pos > 3 && bufr[pos-4] == '<' {
		if bufr[pos-3] == 's' && bufr[pos-2] == 'u' {
			ch := bufr[pos-1]
			if ch == 'p' || ch == 'b' {
				return true
			}
		}
		/*
			if bufr[pos-1] == '/' && bufr[pos-2] == ' ' {
				ch := bufr[pos-3]
				if ch == 'i' || ch == 'b' || ch == 'u' {
					return true
				}
			}
		*/
	} else if pos > 4 && bufr[pos-5] == '<' {
		if bufr[pos-4] == '/' && bufr[pos-3] == 's' && bufr[pos-2] == 'u' {
			ch := bufr[pos-1]
			if ch == 'p' || ch == 'b' {
				return true
			}
		}
		if bufr[pos-1] == '/' && bufr[pos-4] == 's' && bufr[pos-3] == 'u' {
			ch := bufr[pos-2]
			if ch == 'p' || ch == 'b' {
				return true
			}
		}
		/*
			} else if pos > 5 && bufr[pos-6] == '<' {
				if bufr[pos-1] == '/' && bufr[pos-2] == ' ' && bufr[pos-5] == 's' && bufr[pos-4] == 'u' {
					ch := bufr[pos-3]
					if ch == 'p' || ch == 'b' {
						return true
					}
				}
		*/
	}

	return false
}

func HasMarkup(str string) bool {

	for _, ch := range str {
		if ch <= 127 {
			continue
		}
		// quick min-to-max check for Unicode superscript or subscript characters
		if (ch >= '\u00B2' && ch <= '\u00B9') || (ch >= '\u2070' && ch <= '\u208E') {
			return true
		}
	}

	return false
}

func RemoveUnicodeMarkup(str string) string {

	var buffer bytes.Buffer

	for _, ch := range str {
		if ch > 127 {
			if (ch >= '\u00B2' && ch <= '\u00B9') || (ch >= '\u2070' && ch <= '\u208E') {
				rn, ok := markupRunes[ch]
				if ok {
					ch = rn
				}
			}
		}
		buffer.WriteRune(ch)
	}

	return buffer.String()
}

func SimulateUnicodeMarkup(str string) string {

	var buffer bytes.Buffer

	for _, ch := range str {
		if ch > 127 {
			if (ch >= '\u00B2' && ch <= '\u00B9') || (ch >= '\u2070' && ch <= '\u207F') {
				rn, ok := markupRunes[ch]
				if ok {
					buffer.WriteString("<sup>")
					buffer.WriteRune(rn)
					buffer.WriteString("</sup>")
					continue
				}
			} else if ch >= '\u2080' && ch <= '\u208E' {
				rn, ok := markupRunes[ch]
				if ok {
					buffer.WriteString("<sub>")
					buffer.WriteRune(rn)
					buffer.WriteString("</sub>")
					continue
				}
			}
		}
		buffer.WriteRune(ch)
	}

	return buffer.String()
}

func SplitInTwoAt(str, chr string, side SideType) (string, string) {

	slash := strings.SplitN(str, chr, 2)
	if len(slash) > 1 {
		return slash[0], slash[1]
	}

	if side == LEFT {
		return str, ""
	}

	return "", str
}

func ConvertSlash(str string) string {

	if str == "" {
		return str
	}

	length := len(str)
	res := make([]byte, length+1, length+1)

	isSlash := false
	idx := 0
	for _, ch := range str {
		if isSlash {
			switch ch {
			case 'n':
				// line feed
				res[idx] = '\n'
			case 'r':
				// carriage return
				res[idx] = '\r'
			case 't':
				// horizontal tab
				res[idx] = '\t'
			case 'f':
				// form feed
				res[idx] = '\f'
			case 'a':
				// audible bell from terminal (undocumented)
				res[idx] = '\x07'
			default:
				res[idx] = byte(ch)
			}
			idx++
			isSlash = false
		} else if ch == '\\' {
			isSlash = true
		} else {
			res[idx] = byte(ch)
			idx++
		}
	}

	res = res[0:idx]

	return string(res)
}

var (
	rlock sync.Mutex
	replr *strings.Replacer
	rpair *strings.Replacer
)

func DoHTMLReplace(str string) string {

	// replacer/repairer not reentrant, protected by mutex
	rlock.Lock()

	if replr == nil {
		// handles mixed-content tags, with zero, one, or two levels of encoding
		replr = strings.NewReplacer(
			"<i>", "",
			"</i>", "",
			"<i/>", "",
			"<i />", "",
			"<b>", "",
			"</b>", "",
			"<b/>", "",
			"<b />", "",
			"<u>", "",
			"</u>", "",
			"<u/>", "",
			"<u />", "",
			"<sub>", "",
			"</sub>", "",
			"<sub/>", "",
			"<sub />", "",
			"<sup>", "",
			"</sup>", "",
			"<sup/>", "",
			"<sup />", "",
			"&lt;i&gt;", "",
			"&lt;/i&gt;", "",
			"&lt;i/&gt;", "",
			"&lt;i /&gt;", "",
			"&lt;b&gt;", "",
			"&lt;/b&gt;", "",
			"&lt;b/&gt;", "",
			"&lt;b /&gt;", "",
			"&lt;u&gt;", "",
			"&lt;/u&gt;", "",
			"&lt;u/&gt;", "",
			"&lt;u /&gt;", "",
			"&lt;sub&gt;", "",
			"&lt;/sub&gt;", "",
			"&lt;sub/&gt;", "",
			"&lt;sub /&gt;", "",
			"&lt;sup&gt;", "",
			"&lt;/sup&gt;", "",
			"&lt;sup/&gt;", "",
			"&lt;sup /&gt;", "",
			"&amp;lt;i&amp;gt;", "",
			"&amp;lt;/i&amp;gt;", "",
			"&amp;lt;i/&amp;gt;", "",
			"&amp;lt;i /&amp;gt;", "",
			"&amp;lt;b&amp;gt;", "",
			"&amp;lt;/b&amp;gt;", "",
			"&amp;lt;b/&amp;gt;", "",
			"&amp;lt;b /&amp;gt;", "",
			"&amp;lt;u&amp;gt;", "",
			"&amp;lt;/u&amp;gt;", "",
			"&amp;lt;u/&amp;gt;", "",
			"&amp;lt;u /&amp;gt;", "",
			"&amp;lt;sub&amp;gt;", "",
			"&amp;lt;/sub&amp;gt;", "",
			"&amp;lt;sub/&amp;gt;", "",
			"&amp;lt;sub /&amp;gt;", "",
			"&amp;lt;sup&amp;gt;", "",
			"&amp;lt;/sup&amp;gt;", "",
			"&amp;lt;sup/&amp;gt;", "",
			"&amp;lt;sup /&amp;gt;", "",
			"&amp;amp;", "&amp;",
		)
	}

	if replr != nil {
		str = replr.Replace(str)
	}

	rlock.Unlock()

	return str
}

func DoHTMLRepair(str string) string {

	// replacer/repairer not reentrant, protected by mutex
	rlock.Lock()

	if rpair == nil {
		// handles mixed-content tags, with zero, one, or two levels of encoding
		rpair = strings.NewReplacer(
			"&lt;i&gt;", "<i>",
			"&lt;/i&gt;", "</i>",
			"&lt;i/&gt;", "<i/>",
			"&lt;i /&gt;", "<i/>",
			"&lt;b&gt;", "<b>",
			"&lt;/b&gt;", "</b>",
			"&lt;b/&gt;", "<b/>",
			"&lt;b /&gt;", "<b/>",
			"&lt;u&gt;", "<u>",
			"&lt;/u&gt;", "</u>",
			"&lt;u/&gt;", "<u/>",
			"&lt;u /&gt;", "<u/>",
			"&lt;sub&gt;", "<sub>",
			"&lt;/sub&gt;", "</sub>",
			"&lt;sub/&gt;", "<sub/>",
			"&lt;sub /&gt;", "<sub/>",
			"&lt;sup&gt;", "<sup>",
			"&lt;/sup&gt;", "</sup>",
			"&lt;sup/&gt;", "<sup/>",
			"&lt;sup /&gt;", "<sup/>",
			"&amp;lt;i&amp;gt;", "<i>",
			"&amp;lt;/i&amp;gt;", "</i>",
			"&amp;lt;i/&amp;gt;", "<i/>",
			"&amp;lt;i /&amp;gt;", "<i/>",
			"&amp;lt;b&amp;gt;", "<b>",
			"&amp;lt;/b&amp;gt;", "</b>",
			"&amp;lt;b/&amp;gt;", "<b/>",
			"&amp;lt;b /&amp;gt;", "<b/>",
			"&amp;lt;u&amp;gt;", "<u>",
			"&amp;lt;/u&amp;gt;", "</u>",
			"&amp;lt;u/&amp;gt;", "<u/>",
			"&amp;lt;u /&amp;gt;", "<u/>",
			"&amp;lt;sub&amp;gt;", "<sub>",
			"&amp;lt;/sub&amp;gt;", "</sub>",
			"&amp;lt;sub/&amp;gt;", "<sub/>",
			"&amp;lt;sub /&amp;gt;", "<sub/>",
			"&amp;lt;sup&amp;gt;", "<sup>",
			"&amp;lt;/sup&amp;gt;", "</sup>",
			"&amp;lt;sup/&amp;gt;", "<sup/>",
			"&amp;lt;sup /&amp;gt;", "<sup/>",
			"&amp;amp;", "&amp;",
		)
	}

	if rpair != nil {
		str = rpair.Replace(str)
	}

	rlock.Unlock()

	return str
}

func DoTrimFlankingHTML(str string) string {

	badPrefix := [10]string{
		"<i></i>",
		"<b></b>",
		"<u></u>",
		"<sup></sup>",
		"<sub></sub>",
		"</i>",
		"</b>",
		"</u>",
		"</sup>",
		"</sub>",
	}

	badSuffix := [10]string{
		"<i></i>",
		"<b></b>",
		"<u></u>",
		"<sup></sup>",
		"<sub></sub>",
		"<i>",
		"<b>",
		"<u>",
		"<sup>",
		"<sub>",
	}

	if strings.Contains(str, "<") {
		goOn := true
		for goOn {
			goOn = false
			for _, tag := range badPrefix {
				if strings.HasPrefix(str, tag) {
					str = str[len(tag):]
					goOn = true
				}
			}
			for _, tag := range badSuffix {
				if strings.HasSuffix(str, tag) {
					str = str[:len(str)-len(tag)]
					goOn = true
				}
			}
		}
	}

	return str
}

func HasBadAccent(str string) bool {

	for _, ch := range str {
		if ch <= 127 {
			continue
		}
		// quick min-to-max check for additional characters to treat as accents
		if ch >= '\u00D8' && ch <= '\u02BC' {
			return true
		} else if ch >= '\uFB00' && ch <= '\uFB06' {
			return true
		}
	}

	return false
}

func FixBadAccent(str string) string {

	var buffer bytes.Buffer

	for _, ch := range str {
		if ch > 127 {
			if ch >= '\u00D8' && ch <= '\u02BC' {
				rn, ok := accentRunes[ch]
				if ok {
					buffer.WriteRune(rn)
					continue
				}
				st, ok := ligatureRunes[ch]
				if ok {
					buffer.WriteString(st)
					continue
				}
			}
			if ch >= '\uFB00' && ch <= '\uFB06' {
				st, ok := ligatureRunes[ch]
				if ok {
					buffer.WriteString(st)
					continue
				}
			}
		}
		buffer.WriteRune(ch)
	}

	return buffer.String()
}

var (
	tlock sync.Mutex
	tform transform.Transformer
)

func DoAccentTransform(str string) string {

	// transformer not reentrant, protected by mutex
	tlock.Lock()

	if tform == nil {
		tform = transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)
	}

	if tform != nil {
		str, _, _ = transform.String(tform, str)
	}

	tlock.Unlock()

	// look for characters not in current external runes conversion table
	if HasBadAccent(str) {
		str = FixBadAccent(str)
	}

	return str
}

func UnicodeToASCII(str string) string {

	var buffer bytes.Buffer

	for _, ch := range str {
		if ch > 127 {
			s := strconv.QuoteToASCII(string(ch))
			s = strings.ToUpper(s[3:7])
			for {
				if !strings.HasPrefix(s, "0") {
					break
				}
				s = s[1:]
			}
			buffer.WriteString("&#x")
			buffer.WriteString(s)
			buffer.WriteRune(';')
			continue
		}
		buffer.WriteRune(ch)
	}

	return buffer.String()
}

func HasGreek(str string) bool {

	for _, ch := range str {
		if ch <= 127 {
			continue
		}
		// quick min-to-max check for greek characters to convert to english words
		if ch >= '\u03B1' && ch <= '\u03C9' {
			return true
		} else if ch >= '\u0391' && ch <= '\u03A9' {
			return true
		} else if ch == '\u0190' || ch == '\u025B' {
			return true
		}
	}

	return false
}

func SpellGreek(str string) string {

	var buffer bytes.Buffer

	for _, ch := range str {
		if ch > 127 {
			if ch >= '\u03B1' && ch <= '\u03C9' {
				st, ok := greekRunes[ch]
				if ok {
					buffer.WriteString(" ")
					buffer.WriteString(st)
					buffer.WriteString(" ")
					continue
				}
			}
			if ch >= '\u0391' && ch <= '\u03A9' {
				st, ok := greekRunes[ch]
				if ok {
					buffer.WriteString(" ")
					buffer.WriteString(st)
					buffer.WriteString(" ")
					continue
				}
			}
			// treat latin letter open E as epsilon
			if ch == '\u0190' || ch == '\u025B' {
				st, ok := greekRunes[ch]
				if ok {
					buffer.WriteString(" ")
					buffer.WriteString(st)
					buffer.WriteString(" ")
					continue
				}
			}
		}
		buffer.WriteRune(ch)
	}

	return buffer.String()
}

var plock sync.RWMutex

func IsStopWord(str string) bool {

	plock.RLock()
	isSW := isStopWord[str]
	plock.RUnlock()

	return isSW
}

// CREATE COMMON DRIVER TABLES

// InitTables creates lookup tables to simplify the tokenizer
func InitTables() *Tables {

	tbls := &Tables{}

	for i := range tbls.InBlank {
		tbls.InBlank[i] = false
	}
	tbls.InBlank[' '] = true
	tbls.InBlank['\t'] = true
	tbls.InBlank['\n'] = true
	tbls.InBlank['\r'] = true
	tbls.InBlank['\f'] = true

	// alternative version of InBlank allows newlines to be counted
	for i := range tbls.AltBlank {
		tbls.AltBlank[i] = false
	}
	tbls.AltBlank[' '] = true
	tbls.AltBlank['\t'] = true
	tbls.AltBlank['\r'] = true
	tbls.AltBlank['\f'] = true

	// first character of element cannot be a digit, dash, or period
	for i := range tbls.InFirst {
		tbls.InFirst[i] = false
	}
	for ch := 'A'; ch <= 'Z'; ch++ {
		tbls.InFirst[ch] = true
	}
	for ch := 'a'; ch <= 'z'; ch++ {
		tbls.InFirst[ch] = true
	}
	tbls.InFirst['_'] = true

	// remaining characters also includes colon for namespace
	for i := range tbls.InElement {
		tbls.InElement[i] = false
	}
	for ch := 'A'; ch <= 'Z'; ch++ {
		tbls.InElement[ch] = true
	}
	for ch := 'a'; ch <= 'z'; ch++ {
		tbls.InElement[ch] = true
	}
	for ch := '0'; ch <= '9'; ch++ {
		tbls.InElement[ch] = true
	}
	tbls.InElement['_'] = true
	tbls.InElement['-'] = true
	tbls.InElement['.'] = true
	tbls.InElement[':'] = true

	return tbls
}

// READ XML INPUT FILE INTO SET OF BLOCKS

type XMLReader struct {
	Reader     io.Reader
	Buffer     []byte
	Remainder  string
	Position   int64
	Delta      int
	Closed     bool
	Docompress bool
	Docleanup  bool
	LeaveHTML  bool
}

func NewXMLReader(in io.Reader, doCompress, doCleanup, leaveHTML bool) *XMLReader {

	if in == nil {
		return nil
	}

	rdr := &XMLReader{Reader: in, Docompress: doCompress, Docleanup: doCleanup, LeaveHTML: leaveHTML}

	// 65536 appears to be the maximum number of characters presented to io.Reader when input is piped from stdin
	// increasing size of buffer when input is from a file does not improve program performance
	// additional 16384 bytes are reserved for copying previous remainder to start of buffer before next read
	const XMLBUFSIZE = 65536 + 16384

	rdr.Buffer = make([]byte, XMLBUFSIZE)

	return rdr
}

// NextBlock reads buffer, concatenates if necessary to place long element content into a single string
// all result strings end in > character that is used as a sentinel in subsequent code
func (rdr *XMLReader) NextBlock() string {

	if rdr == nil {
		return ""
	}

	// read one buffer, trim at last > and retain remainder for next call, signal if no > character
	nextBuffer := func() (string, bool, bool) {

		if rdr.Closed {
			return "", false, true
		}

		// prepend previous remainder to beginning of buffer
		m := copy(rdr.Buffer, rdr.Remainder)
		rdr.Remainder = ""
		if m > 16384 {
			// previous remainder is larger than reserved section, write and signal need to continue reading
			return string(rdr.Buffer[:m]), true, false
		}

		// read next block, append behind copied remainder from previous read
		n, err := rdr.Reader.Read(rdr.Buffer[m:])
		// with data piped through stdin, read function may not always return the same number of bytes each time
		if err != nil {
			if err != io.EOF {
				// real error
				fmt.Fprintf(os.Stderr, "\nERROR: %s\n", err.Error())
				// Ignore bytes - non-conforming implementations of io.Reader may returned mangled data on non-EOF errors
				rdr.Closed = true
				return "", false, true
			}
			// end of file
			rdr.Closed = true
			if n == 0 {
				// if EOF and no more data, do not send final remainder (not terminated by right angle bracket that is used as a sentinel)
				return "", false, true
			}
		}
		if n < 0 {
			// Reality check - non-conforming implementations of io.Reader may return -1
			fmt.Fprintf(os.Stderr, "\nERROR: io.Reader returned negative count %d\n", n)
			// treat as n == 0 in order to update file offset and avoid losing previous remainder
			n = 0
		}

		// keep track of file offset
		rdr.Position += int64(rdr.Delta)
		rdr.Delta = n

		// slice of actual characters read
		bufr := rdr.Buffer[:n+m]

		// look for last > character
		// safe to back up on UTF-8 rune array when looking for 7-bit ASCII character
		pos := -1
		for pos = len(bufr) - 1; pos >= 0; pos-- {
			if bufr[pos] == '>' {
				if rdr.LeaveHTML {
					// optionally skip backwards past embedded i, b, u, sub, and sup HTML open, close, and empty tags
					if HTMLBehind(bufr, pos) {
						continue
					}
				}
				// found end of XML tag, break
				break
			}
		}

		// trim back to last > character, save remainder for next buffer
		if pos > -1 {
			pos++
			rdr.Remainder = string(bufr[pos:])
			return string(bufr[:pos]), false, false
		}

		// no > found, signal need to continue reading long content
		return string(bufr[:]), true, false
	}

	// read next buffer
	line, cont, closed := nextBuffer()

	if closed {
		// no sentinel in remainder at end of file
		return ""
	}

	// if buffer does not end with > character
	if cont {
		var buff bytes.Buffer

		// keep reading long content blocks
		for {
			if line != "" {
				buff.WriteString(line)
			}
			if !cont {
				// last buffer ended with sentinel
				break
			}
			line, cont, closed = nextBuffer()
			if closed {
				// no sentinel in multi-block buffer at end of file
				return ""
			}
		}

		// concatenate blocks
		line = buff.String()
	}

	// trimming spaces here would throw off line tracking

	// optionally compress/cleanup tags/attributes and contents
	if rdr.Docompress {
		line = CompressRunsOfSpaces(line)
	}
	if rdr.Docleanup {
		if HasBadSpace(line) {
			line = CleanupBadSpaces(line)
		}
	}

	return line
}

// PARSE XML BLOCK STREAM INTO STRINGS FROM <PATTERN> TO </PATTERN>

// PartitionPattern splits XML input by pattern and sends individual records to a callback
func PartitionPattern(pat, star string, rdr *XMLReader, proc func(int, int64, string)) {

	if pat == "" || rdr == nil || proc == nil {
		return
	}

	type Scanner struct {
		Pattern   string
		PatLength int
		CharSkip  [256]int
	}

	// initialize <pattern> to </pattern> scanner
	newScanner := func(pattern string) *Scanner {

		if pattern == "" {
			return nil
		}

		scr := &Scanner{Pattern: pattern}

		patlen := len(pattern)
		scr.PatLength = patlen

		// position of last character in pattern
		last := patlen - 1

		// initialize bad character displacement table
		for i := range scr.CharSkip {
			scr.CharSkip[i] = patlen
		}
		for i := 0; i < last; i++ {
			ch := pattern[i]
			scr.CharSkip[ch] = last - i
		}

		return scr
	}

	// check surroundings of match candidate
	isAnElement := func(text string, lf, rt, mx int) bool {

		if (lf >= 0 && text[lf] == '<') || (lf > 0 && text[lf] == '/' && text[lf-1] == '<') {
			if (rt < mx && (text[rt] == '>' || text[rt] == ' ')) || (rt+1 < mx && text[rt] == '/' && text[rt+1] == '>') {
				return true
			}
		}

		return false
	}

	// modified Boyer-Moore-Horspool search function
	findNextMatch := func(scr *Scanner, text string, offset int) (int, int, int) {

		if scr == nil || text == "" {
			return -1, -1, -1
		}

		// copy values into local variables for speed
		txtlen := len(text)
		pattern := scr.Pattern[:]
		patlen := scr.PatLength
		max := txtlen - patlen
		last := patlen - 1
		skip := scr.CharSkip[:]

		i := offset

		for i <= max {
			j := last
			k := i + last
			for j >= 0 && text[k] == pattern[j] {
				j--
				k--
			}
			// require match candidate to be element name, i.e., <pattern ... >, </pattern ... >, or <pattern ... />
			if j < 0 && isAnElement(text, i-1, i+patlen, txtlen) {
				// find positions of flanking brackets
				lf := i - 1
				for lf > 0 && text[lf] != '<' {
					lf--
				}
				rt := i + patlen
				for rt < txtlen && text[rt] != '>' {
					rt++
				}
				return i + 1, lf, rt + 1
			}
			// find character in text above last character in pattern
			ch := text[i+last]
			// displacement table can shift pattern by one or more positions
			i += skip[ch]
		}

		return -1, -1, -1
	}

	type PatternType int

	const (
		NOPATTERN PatternType = iota
		STARTPATTERN
		SELFPATTERN
		STOPPATTERN
	)

	// find next element with pattern name
	nextPattern := func(scr *Scanner, text string, pos int) (PatternType, int, int) {

		if scr == nil || text == "" {
			return NOPATTERN, 0, 0
		}

		prev := pos

		for {
			next, start, stop := findNextMatch(scr, text, prev)
			if next < 0 {
				return NOPATTERN, 0, 0
			}

			prev = next + 1

			if text[start+1] == '/' {
				return STOPPATTERN, stop, prev
			} else if text[stop-2] == '/' {
				return SELFPATTERN, start, prev
			} else {
				return STARTPATTERN, start, prev
			}
		}
	}

	// -pattern Object construct

	doNormal := func() {

		// current depth of -pattern objects
		level := 0

		begin := 0
		inPattern := false

		line := ""
		var accumulator bytes.Buffer

		match := NOPATTERN
		pos := 0
		next := 0

		offset := int64(0)

		rec := 0

		scr := newScanner(pat)
		if scr == nil {
			return
		}

		for {

			begin = 0
			next = 0

			line = rdr.NextBlock()
			if line == "" {
				return
			}

			for {
				match, pos, next = nextPattern(scr, line, next)
				if match == STARTPATTERN {
					if level == 0 {
						inPattern = true
						begin = pos
						offset = rdr.Position + int64(pos)
					}
					level++
				} else if match == STOPPATTERN {
					level--
					if level == 0 {
						inPattern = false
						accumulator.WriteString(line[begin:pos])
						// read and process one -pattern object at a time
						str := accumulator.String()
						if str != "" {
							rec++
							proc(rec, offset, str[:])
						}
						// reset accumulator
						accumulator.Reset()
					}
				} else if match == SELFPATTERN {
				} else {
					if inPattern {
						accumulator.WriteString(line[begin:])
					}
					break
				}
			}
		}
	}

	// -pattern Parent/* construct now works with catenated files, but not if components
	// are recursive or self-closing objects, process those through -format first

	doStar := func() {

		// current depth of -pattern objects
		level := 0

		begin := 0
		inPattern := false

		line := ""
		var accumulator bytes.Buffer

		match := NOPATTERN
		pos := 0
		next := 0

		offset := int64(0)

		rec := 0

		scr := newScanner(pat)
		if scr == nil {
			return
		}

		last := pat

		// read to first <pattern> element
		for {

			next = 0

			line = rdr.NextBlock()
			if line == "" {
				break
			}

			match, pos, next = nextPattern(scr, line, next)
			if match == STARTPATTERN {
				break
			}
		}

		if match != STARTPATTERN {
			return
		}

		// find next element in XML
		nextElement := func(text string, pos int) string {

			txtlen := len(text)

			tag := ""
			for i := pos; i < txtlen; i++ {
				if text[i] == '<' {
					tag = text[i+1:]
					break
				}
			}
			if tag == "" {
				return ""
			}
			if tag[0] == '/' {
				if strings.HasPrefix(tag[1:], pat) {
					//should be </pattern> at end, want to continue if catenated files
					return "/"
				}
				return ""
			}
			for i, ch := range tag {
				if ch == '>' || ch == ' ' || ch == '/' {
					return tag[0:i]
				}
			}

			return ""
		}

		// read and process heterogeneous objects immediately below <pattern> parent
		for {
			tag := nextElement(line, next)
			if tag == "" {

				begin = 0
				next = 0

				line = rdr.NextBlock()
				if line == "" {
					break
				}

				tag = nextElement(line, next)
			}
			if tag == "" {
				return
			}

			// check for catenated parent set files
			if tag[0] == '/' {
				scr = newScanner(pat)
				if scr == nil {
					return
				}
				last = pat
				// confirm end </pattern> just found
				match, pos, next = nextPattern(scr, line, next)
				if match != STOPPATTERN {
					return
				}
				// now look for a new start <pattern> tag
				for {
					match, pos, next = nextPattern(scr, line, next)
					if match == STARTPATTERN {
						break
					}
					next = 0
					line = rdr.NextBlock()
					if line == "" {
						break
					}
				}
				if match != STARTPATTERN {
					return
				}
				// continue with processing loop
				continue
			}

			if tag != last {
				scr = newScanner(tag)
				if scr == nil {
					return
				}
				last = tag
			}

			for {
				match, pos, next = nextPattern(scr, line, next)
				if match == STARTPATTERN {
					if level == 0 {
						inPattern = true
						begin = pos
						offset = rdr.Position + int64(pos)
					}
					level++
				} else if match == STOPPATTERN {
					level--
					if level == 0 {
						inPattern = false
						accumulator.WriteString(line[begin:pos])
						// read and process one -pattern/* object at a time
						str := accumulator.String()
						if str != "" {
							rec++
							proc(rec, offset, str[:])
						}
						// reset accumulator
						accumulator.Reset()
						break
					}
				} else {
					if inPattern {
						accumulator.WriteString(line[begin:])
					}

					begin = 0
					next = 0

					line = rdr.NextBlock()
					if line == "" {
						break
					}
				}
			}
		}
	}

	// call appropriate handler
	if star == "" {
		doNormal()
	} else if star == "*" {
		doStar()
	}
}

// COLLECT AND FORMAT REQUESTED XML VALUES

// ParseAttributes is only run if attribute values are requested in element statements
func ParseAttributes(attrb string) []string {

	if attrb == "" {
		return nil
	}

	attlen := len(attrb)

	// count equal signs
	num := 0
	for i := 0; i < attlen; i++ {
		if attrb[i] == '=' {
			num += 2
		}
	}
	if num < 1 {
		return nil
	}

	// allocate array of proper size
	arry := make([]string, num)
	if arry == nil {
		return nil
	}

	start := 0
	idx := 0
	itm := 0

	// place tag and value in successive array slots
	for idx < attlen && itm < num {
		ch := attrb[idx]
		if ch == '=' {
			// skip past possible leading blanks
			for start < attlen {
				ch = attrb[start]
				if ch == ' ' || ch == '\n' || ch == '\t' || ch == '\r' || ch == '\f' {
					start++
				} else {
					break
				}
			}
			// =
			arry[itm] = attrb[start:idx]
			itm++
			// skip past equal sign and leading double quote
			idx += 2
			start = idx
		} else if ch == '"' {
			// "
			arry[itm] = attrb[start:idx]
			itm++
			// skip past trailing double quote and (possible) space
			idx += 2
			start = idx
		} else {
			idx++
		}
	}

	return arry
}

// UNSHUFFLER USES HEAP TO RESTORE OUTPUT OF MULTIPLE CONSUMERS TO ORIGINAL RECORD ORDER

type Extract struct {
	Index int
	Ident string
	Text  string
}

type ExtractHeap []Extract

// methods that satisfy heap.Interface
func (h ExtractHeap) Len() int {
	return len(h)
}
func (h ExtractHeap) Less(i, j int) bool {
	return h[i].Index < h[j].Index
}
func (h ExtractHeap) Swap(i, j int) {
	h[i], h[j] = h[j], h[i]
}
func (h *ExtractHeap) Push(x interface{}) {
	*h = append(*h, x.(Extract))
}
func (h *ExtractHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// CONCURRENT CONSUMER GOROUTINES PARSE AND PROCESS PARTITIONED XML OBJECTS

// ReadBlocks -> SplitPattern => StreamTokens => ParseXML => ProcessQuery -> MergeResults

// processes with single goroutine call defer close(out) so consumer(s) can range over channel
// processes with multiple instances call defer wg.Done(), separate goroutine uses wg.Wait() to delay close(out)

func CreateProducer(pat, star string, rdr *XMLReader, tbls *Tables) <-chan Extract {

	if rdr == nil || tbls == nil {
		return nil
	}

	out := make(chan Extract, tbls.ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create producer channel\n")
		os.Exit(1)
	}

	// xmlProducer sends partitioned XML strings through channel
	xmlProducer := func(pat, star string, rdr *XMLReader, out chan<- Extract) {

		// close channel when all records have been processed
		defer close(out)

		// partition all input by pattern and send XML substring to available consumer through channel
		PartitionPattern(pat, star, rdr,
			func(rec int, ofs int64, str string) {
				out <- Extract{rec, "", str}
			})
	}

	// launch single producer goroutine
	go xmlProducer(pat, star, rdr, out)

	return out
}

func CreateUnshuffler(tbls *Tables, inp <-chan Extract) <-chan Extract {

	if tbls == nil || inp == nil {
		return nil
	}

	out := make(chan Extract, tbls.ChanDepth)
	if out == nil {
		fmt.Fprintf(os.Stderr, "\nERROR: Unable to create unshuffler channel\n")
		os.Exit(1)
	}

	// xmlUnshuffler restores original order with heap
	xmlUnshuffler := func(inp <-chan Extract, out chan<- Extract) {

		// close channel when all records have been processed
		defer close(out)

		// initialize empty heap
		hp := &ExtractHeap{}
		heap.Init(hp)

		// index of next desired result
		next := 1

		delay := 0

		for ext := range inp {

			// push result onto heap
			heap.Push(hp, ext)

			// read several values before checking to see if next record to print has been processed
			if delay < tbls.HeapSize {
				delay++
				continue
			}

			delay = 0

			for hp.Len() > 0 {

				// remove lowest item from heap, use interface type assertion
				curr := heap.Pop(hp).(Extract)

				if curr.Index > next {

					// record should be printed later, push back onto heap
					heap.Push(hp, curr)
					// and go back to waiting on input channel
					break
				}

				// send even if empty to get all record counts for reordering
				out <- Extract{curr.Index, curr.Ident, curr.Text}

				// prevent ambiguous -limit filter from clogging heap (deprecated)
				if curr.Index == next {
					// increment index for next expected match
					next++
				}

				// keep checking heap to see if next result is already available
			}
		}

		// send remainder of heap to output
		for hp.Len() > 0 {
			curr := heap.Pop(hp).(Extract)

			out <- Extract{curr.Index, curr.Ident, curr.Text}
		}
	}

	// launch single unshuffler goroutine
	go xmlUnshuffler(inp, out)

	return out
}
