#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
#
# Copyright (C) 2010-2012 Marnix H. Medema
# University of Groningen
# Department of Microbial Physiology / Groningen Bioinformatics Centre
#
# Copyright (C) 2011,2012 Kai Blin
# University of Tuebingen
# Interfaculty Institute of Microbiology and Infection Medicine
# Div. of Microbiology/Biotechnology
#
# License: GNU Affero General Public License v3 or later
# A copy of GNU AGPL v3 should have been included in this software package in LICENSE.txt.
"""Run the antiSMASH pipeline"""

import sys
import os
import random

import logging
import argparse
from os import path
import multiprocessing
from helperlibs.bio import seqio
from antismash.config import load_config, set_config
from antismash import utils
from antismash.generic_modules import check_prereqs as generic_check_prereqs
from antismash.generic_modules import (
    hmm_detection,
    cassis,
    genefinding,
    fullhmmer,
    clusterfinder,
    smcogs,
    clusterblast,
    subclusterblast,
    knownclusterblast,
    active_site_finder,
    tta,
    gff_parser
)
from antismash.specific_modules import (
    lantipeptides,
    lassopeptides,
    nrpspks,
    thiopeptides,
    terpenes,
    sactipeptides,
)
from antismash.output_modules import (
    biosynml,
    embl,
    genbank,
    html,
    svg,
    txt,
    xls,
)

import numpy as np
from collections import defaultdict
from Bio.Alphabet import generic_dna
from Bio.Seq import Seq, UnknownSeq
from Bio.Alphabet import NucleotideAlphabet
from Bio.SeqRecord import SeqRecord
from Bio.SeqFeature import SeqFeature, FeatureLocation
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO
from datetime import datetime

# Always raise an exception when numpy runs into a floating point exception
np.seterr(all='raise')

ALL_CLUSTER_SPECIFIC_MODULES = [
    lantipeptides,
    lassopeptides,
    nrpspks,
    thiopeptides,
    terpenes,
    sactipeptides
]

def ValidateClusterTypes(clustertypes):
    class Validator(argparse.Action):
        def __call__(self, parser, args, values, option_string = None):
            values = values.replace(";",",").split(",")
            try:
                for value in values:
                    if value not in clustertypes:
                        raise ValueError('invalid clustertype {s!r}'.format(s = value))
            except ValueError as e:
                print "\nInput error:", e, "\n"
                print "Please choose from the following list:\n", "\n".join(clustertypes), "\n\nExample: --enable t1pks,nrps,other"
                sys.exit(1)
            setattr(args, self.dest, values)
    return Validator


def main():
    "Actually run the pipeline"
    multiprocessing.freeze_support()

 # We print the logging debug message as soon as the logging object is initialized after parsing the cmdline
    start_time = datetime.now()

    # First load the output plugins so we can present appropriate options
    output_plugins = load_output_plugins()

    clustertypes = hmm_detection.get_supported_cluster_types()

    parser = utils.getArgParser()

    #positional arguments
    parser.add_argument('sequences',
                        metavar='sequence',
                        nargs="*",
                        help="GenBank/EMBL/FASTA file(s) containing DNA.")

    #optional non-grouped arguments
    parser.add_argument('-h', '--help',
                        dest='help',
                        action='store_true',
                        default=False,
                        help="Show this help text.")
    parser.add_argument('--help-showall',
                        dest='help_showall',
                        action='store_true',
                        default=False,
                        help="Show full lists of arguments on this help text.")
    parser.add_argument('-c', '--cpus',
                        dest='cpus',
                        type=int,
                        default=multiprocessing.cpu_count(),
                        help="How many CPUs to use in parallel. (default: %(default)s)")

    ## grouped arguments

    group = parser.add_argument_group('Basic analysis options', '', basic=True)

    group.add_argument('--taxon',
                       dest='taxon',
                       default='bacteria',
                       choices=['bacteria', 'fungi'],
                       help="Taxonomic classifcation of input sequence. (default: %(default)s)")

    group.add_argument('--input-type',
                       dest='input_type',
                       default='nucl',
                       choices=['nucl', 'prot'],
                       help="Determine input type: amino acid sequence(s) or nucleotide sequence(s). (default: %(default)s)")

    group = parser.add_argument_group('Additional analysis',
                                      'Use -h along with the corresponding options to show '
                                      'additional analysis specific parameters. (e.g.: '
                                      '"{prog} -h --clusterblast --smcogs")'.format(prog=parser.prog),
                                      basic=True)


    group.add_argument('--transatpks_da',
                       dest='transatpks_da',
                       action='store_true',
                       default=False,
                       help="Find the most similar transatpks BGC based on distance score calculated by domain composition and domain sequence similarity.")

    group.add_argument('--transatpks_da_cutoff',
                       dest="transatpks_da_cutoff",
                       type=int,
                       default=10,
                       help="How many transATPKS assembly line are retured based on calculated domain alignment distance.")


    group.add_argument('--clusterblast',
                       dest='clusterblast',
                       action='store_true',
                       default=False,
                       help="Compare identified clusters against a database of antiSMASH-predicted clusters.")
    group.add_argument('--subclusterblast',
                       dest='subclusterblast',
                       action='store_true',
                       default=False,
                       help="Compare identified clusters against known subclusters responsible for synthesising precursors.")
    group.add_argument('--knownclusterblast',
                       dest='knownclusterblast',
                       action='store_true',
                       default=False,
                       help="Compare identified clusters against known gene clusters from the MIBiG database.")
    group.add_argument('--smcogs',
                       dest='smcogs',
                       action='store_true',
                       default=False,
                       help="Look for sec. met. clusters of orthologous groups.")
    group.add_argument('--inclusive',
                       dest='inclusive',
                       action='store_true',
                       default=False,
                       help="Use inclusive ClusterFinder algorithm for additional cluster detection.")
    group.add_argument('--cassis',
                       dest='cassis',
                       action='store_true',
                       default=False,
                       help="Use CASSIS algorithm for cluster border prediction (fungal seqs only).")
    group.add_argument('--borderpredict',
                       dest='borderpredict',
                       action='store_true',
                       default=False,
                       help="Use ClusterFinder algorithm to predict gene cluster borders.")
    group.add_argument('--full-hmmer',
                       dest='full_hmmer',
                       action='store_true',
                       default=False,
                       help="Run a whole-genome HMMer analysis.")
    group.add_argument('--asf',
                       dest='run_asf',
                       action='store_true',
                       default=False,
                       help='Run active site finder module.')
    group.add_argument('--tta',
                       dest='tta',
                       action='store_true',
                       default=False,
                       help="Run TTA codon detection module.")
    group = parser.add_argument_group('Advanced options')
    group.add_argument('--limit',
                       dest="limit",
                       type=int,
                       default=-1,
                       help="Only process the first <limit> records (default: %(default)s). -1 to disable")
    group.add_argument('--minlength',
                       dest="minlength",
                       type=int,
                       default=1000,
                       help="Only process sequences larger than <minlength> (default: 1000).")
    group.add_argument('--from',
                       dest='start',
                       type=int,
                       default=-1,
                       help="Start analysis at nucleotide specified.")
    group.add_argument('--to',
                       dest='end',
                       type=int,
                       default=-1,
                       help="End analysis at nucleotide specified")
    group.add_argument('--pfamdir',
                       dest='pfamdir',
                       default=argparse.SUPPRESS,
                       help="Directory the Pfam-A.hmm file is located in.")
    group.add_argument('--enable',
                       metavar="TYPES",
                       dest='enabled_cluster_types',
                       action=ValidateClusterTypes(clustertypes),
                       default=clustertypes,
                       help="Select sec. met. cluster types to search for.")
    group.add_argument('--fix-id-line',
                       dest='fix_id_line',
                       action='store_true',
                       default=False,
                       help="Try to fix invalid sequence file ID lines.")
    group.add_argument('--skip-cleanup',
                       dest='skip_cleanup',
                       action='store_true',
                       default=False,
                       help="Don't clean up temporary result files")
    group.add_argument('--dbgsandpuma',
                       dest="dbgsandpuma",
                       default='',
                       help="Reuse results from previous SANDPUMA run in specified directory")

    group = parser.add_argument_group('Gene finding options (ignored when ORFs are annotated)')
    group.add_argument('--genefinding',
                       dest='genefinding',
                       default='prodigal',
                       choices=['glimmer', 'prodigal', 'prodigal-m', 'none'],
                       help="Specify algorithm used for gene finding: Glimmer, "
                            "Prodigal, Prodigal Metagenomic/Anonymous mode or none. (default: %(default)s).")
    group.add_argument('--all-orfs',
                       dest='all_orfs',
                       action='store_true',
                       default=False,
                       help="Use all ORFs > 60 nt instead of running gene finding.")
    group.add_argument('--gff3',
                       dest='gff3',
                       default=False,
                       help="Specify GFF3 file to extract features from.")


    group = parser.add_argument_group('ClusterBlast specific options', '',
                                      param=["--clusterblast", "--subclusterblast", "--knownclusterblast"])
    group.add_argument('--nclusters',
                       dest='nclusters',
                       type=int,
                       default=10,
                       help="Number of clusters from ClusterBlast to display.")
    group.add_argument('--seed',
                       dest='seed',
                       type=int,
                       default=0,
                       help="Random number seed for ClusterBlast coloring.")
    group.add_argument('--homologyscalelimit',
                       dest='homologyscalelimit',
                       type=float,
                       default=0.0,
                       help="If positive float number greater than 1, limit horizontal shrinkage "
                            "of the graphical display of the query BGC in ClusterBlast results to "
                            "this ratio. Warning: some homologous genes may no longer be visible!")
    group.add_argument('--dbgclusterblast',
                       dest='dbgclusterblast',
                       default="",
                       help='Retrieve the resuts from previous antiSMASH run stored in the specified directory.')

    group = parser.add_argument_group('ClusterFinder specific options', '',
                                      param=["--inclusive", "--borderpredict"])
    group.add_argument('--cf_cdsnr',
                       dest='cdsnr',
                       type=int,
                       default=5,
                       help="Minimum size of a ClusterFinder cluster, in number of CDS.")
    group.add_argument('--cf_threshold',
                       dest='cf_prob_thres',
                       type=float,
                       default=0.6,
                       help="ClusterFinder mean probability threshold.")
    group.add_argument('--cf_npfams',
                       dest='cf_npfams',
                       type=int,
                       default=5,
                       help="Minimum number of biosynthetic Pfam domains in a ClusterFinder cluster.")

    group = parser.add_argument_group('Output options', '', basic=True)
    group.add_argument('--outputfolder',
                       dest='outputfoldername',
                       default=argparse.SUPPRESS,
                       help="Directory to write results to.")
    for plugin in output_plugins:
        if plugin.enabled:
            group.add_argument('--disable-%s' % plugin.name,
                               dest=plugin.name,
                               action='store_false',
                               default=argparse.SUPPRESS,
                               help="Disable %s" % plugin.short_description)
        else:
            group.add_argument('--enable-%s' % plugin.name,
                               dest=plugin.name,
                               action='store_true',
                               default=argparse.SUPPRESS,
                               help="Enable %s" % plugin.short_description)

    group = parser.add_argument_group('Debugging options for cluster-specific analyses', '', basic=False)
    group.add_argument('--minimal',
                       dest='minimal',
                       action='store_true',
                       default=False,
                       help="Only run minimal analysis, no cluster_specific modules unless explicitly enabled")
    for plugin in ALL_CLUSTER_SPECIFIC_MODULES:
        group.add_argument('--enable-%s' % plugin.name,
                           dest='enabled_specific_plugins',
                           action='append_const',
                           const=plugin.name,
                           help="Enable %s (default: enabled, unless --minimal is specified)" % plugin.short_description)
    group.add_argument('--without-fimo',
                       dest='without_fimo',
                       action='store_true',
                       default=False,
                       help="Don't require the non-free FIMO tool")

    group = parser.add_argument_group("Debugging & Logging options", '', basic=True)
    group.add_argument('-v', '--verbose',
                       dest='verbose',
                       action='store_true',
                       default=False,
                       help="Print verbose status information to stderr.")
    group.add_argument('-d', '--debug',
                       dest='debug',
                       action='store_true',
                       default=False,
                       help="Print debugging information to stderr.")
    group.add_argument('--logfile',
                       dest='logfile',
                       default=argparse.SUPPRESS,
                       help="Also write logging output to a file.")
    group.add_argument('--statusfile',
                       dest='statusfile',
                       default=argparse.SUPPRESS,
                       help="Write the current status to a file.")
    group.add_argument('--list-plugins',
                       dest='list_plugins',
                       action='store_true',
                       default=False,
                       help="List all available sec. met. detection modules.")
    group.add_argument('--check-prereqs',
                       dest='check_prereqs_only',
                       action='store_true',
                       default=False,
                       help="Just check if all prerequisites are met.")
    group.add_argument('--limit-to-record',
                       dest='limit_to_record',
                       default=argparse.SUPPRESS,
                       metavar="record_id",
                       help="Limit analysis to the record with ID record_id")
    group.add_argument('-V', '--version',
                       dest='version',
                       action='store_true',
                       default=False,
                       help="Display the version number and exit.")
    group.add_argument('--profiling',
                       dest='profile',
                       action='store_true',
                       default=False,
                       help="Generate a profiling report, disables multiprocess python.")

    ## endof grouped arguments

    #if --help, show help texts and exit
    if (list(set(["-h", "--help", "--help-showall"]) & set(sys.argv))):
        parser.print_help(None, "--help-showall" in sys.argv)
        sys.exit(0)

    #Parse arguments, removing hyphens from the beginning of file names to avoid conflicts with argparse
    infile_extensions = ('.fasta', '.fas', '.fa', '.gb', '.gbk', '.emb', '.embl')
    sys.argv = [arg.replace("-","< > HYPHEN < >") if (arg.endswith(infile_extensions) and arg[0] == "-") else arg for arg in sys.argv]
    options = parser.parse_args()
    options.sequences = [filename.replace("< > HYPHEN < >","-") for filename in options.sequences]

    options.borderpredict_only = False
    # If borderpredict is set, turn on ClusterFinder
    # if ClusterFinder was not selected, switch borderpredict-only to remove putative BGCs
    if options.borderpredict and not options.inclusive:
        options.borderpredict_only = True
        options.inclusive = True

    #Remove ClusterFinder cluster types when ClusterFinder is not turned on
    if not options.inclusive or options.borderpredict_only:
        options.enabled_cluster_types = [cl_type for cl_type in options.enabled_cluster_types if not cl_type.startswith("cf_")]

    # Logging is useful for all the following code, so make sure that is set up
    # right after parsing the arguments.
    setup_logging(options)

    if options.profile:
        pr = cProfile.Profile()
        pr.enable()

    #if -V, show version texts and exit
    if options.version:
        print "antiSMASH %s" % utils.get_version()
        sys.exit(0)

    logging.info("starting antiSMASH {version}, called with {cmdline}".format(version=utils.get_version(), cmdline=" ".join(sys.argv)))
    logging.debug("antismash analysis started at %s", str(start_time))
    logging.info("OPTIONS: " + str(options))
    logging.info("MINLENGTH: " + str(options.minlength))


    if options.nclusters > 50:
        logging.info("Number of clusters (%d) is too large. Reducing to 50.", options.nclusters)
        options.nclusters = 50
    logging.info("Number of clusters = %d", options.nclusters)
    if options.seed != 0:
        random.seed(options.seed)

    if options.input_type == 'prot' and (
            options.clusterblast or
            options.knownclusterblast or
            options.subclusterblast or
            options.tta or
            options.inclusive or
            options.full_hmmer):
        logging.error("Protein input option is not compatible with --clusterblast, --subclusterblast, " \
                      "--knownclusterblast, --inclusive, --tta, and --full-hmmer")
        sys.exit(2)
    if options.input_type == 'prot' and (options.start != -1 or options.end != -1):
        logging.error("Protein input option is not compatible with --start and --end.")
        sys.exit(2)

    if options.cassis and options.taxon != 'fungi':
        logging.error("CASSIS cluster border prediction only works for fungal sequences.")
        sys.exit(2)

    if options.cassis and options.without_fimo:
        logging.error("CASSIS depends on MEME/FIMO, cannot disable.")
        sys.exit(2)

    #Load configuration data from config file
    load_config(options)
    set_config(options)


    #Load and filter plugins
    utils.log_status("Loading detection plugins")
    plugins = load_cluster_specific_plugins(options)
    if options.list_plugins:
        list_available_plugins(options.enabled_cluster_types, plugins, output_plugins)
        sys.exit(0)

    output_plugins = filter_outputs(output_plugins, options)

    #Check prerequisites
    if check_prereqs(plugins, options) > 0:
        logging.error("Not all prerequisites met")
        sys.exit(1)
    if options.check_prereqs_only:
        logging.info("All prerequisites are met")
        sys.exit(0)
    if not options.sequences:
        parser.error("Please specify at least one sequence file")
    if options.gff3 and not os.path.exists(options.gff3):
        logging.error('No file found at %r', options.gff3)
        sys.exit(1)

    if 'outputfoldername' not in options:
        options.outputfoldername = path.splitext(path.basename(options.sequences[0]))[0]
    if not os.path.exists(options.outputfoldername):
        os.mkdir(options.outputfoldername)
    options.full_outputfolder_path = path.abspath(options.outputfoldername)

    if options.debug and os.path.exists(options.dbgclusterblast):
        logging.debug("Using %s instead of computing Clusterblasts and variantes!", options.dbgclusterblast)

    # Structure for options.extrarecord
    #
    # options.extrarecord[seq_record_id]=Namespace()
    # options.extrarecord[seq_record_id].extradata[extradataID] can contain the different (deserialized) objects,
    # e.g. ...extradata['SubClusterBlastData'] will contain the storage object for subclsuterblast results
    options.extrarecord = {}

    #Parse input sequence
    try:
        utils.log_status("Parsing the input sequence(s)")
        seq_records = parse_input_sequences(options)
        if options.input_type == 'nucl':
            #Check if input contains records > minimal length
            if len(seq_records) == 0:
                logging.error("Input does not contain contigs larger than minimum size of %d bp." % options.minlength)
                sys.exit(1)
    except IOError as e:
        logging.error(str(e))
        sys.exit(1)

    if len(seq_records) < 1:
        logging.error("Sequence file is incorrectly formatted or contains no sequences of sufficient quality.")
        sys.exit(1)

    options.record_idx = 1
    options.orig_record_idx =1


    temp_seq_records = []
    for seq_record in seq_records:
        utils.log_status("Analyzing record %d" % options.record_idx)
        logging.info("Analyzing record %d (%s)", options.record_idx, seq_record.id)
        utils.sort_features(seq_record)
        strip_record(seq_record)
        utils.fix_record_name_id(seq_record, options)

        run_analyses(seq_record, options, plugins)
        utils.sort_features(seq_record)
        temp_seq_records.append(seq_record)
        options.record_idx += 1
        options.orig_record_idx += 1

    #overwrite seq_records with newly assembled temp_seq_records
    seq_records = temp_seq_records
    del temp_seq_records

    #Write results
    options.plugins = plugins
    utils.log_status("Writing the output files")
    logging.debug("Writing output for %s sequence records", len(seq_records))
    write_results(output_plugins, seq_records, options)
    if not options.skip_cleanup:
        zip_results(seq_records, options)
    end_time = datetime.now()
    running_time = end_time-start_time

    if options.profile:
        pr.disable()
        write_profiling_results(pr, os.path.join(options.outputfoldername, "profiling_results"))


    logging.debug("antiSMASH calculation finished at %s; runtime: %s", str(end_time), str(running_time))
    utils.log_status("antiSMASH status: SUCCESS")
    logging.debug("antiSMASH status: SUCCESS")

def write_profiling_results(pr, target):
    s = StringIO()
    sortby = 'tottime'
    ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
    ps.dump_stats(target + ".bin")
    ps.print_stats(.25) # limit to the more meaningful first 25%
    ps.print_callers(.25)
    try:
        path_to_remove = os.path.dirname(os.path.realpath(__file__)) + os.path.sep
        open(target, "w").write(s.getvalue().replace(path_to_remove, ""))
        logging.info("Profiling report written to %s", target)
    except IOError:
        #if can't save to file, print to terminal, but only the head
        logging.debug("Couldn't open file to store profiling output")
        s.truncate(0)
        ps.print_stats(20) #first 20 lines only
        print s.getvalue()

def strip_record(seq_record):

    new_features = []

    for feature in seq_record.features:

        # Discard features added by antiSMASH
        if feature.type in ('cluster', 'cluster_border', 'CDS_motif', 'aSDomain'):
            continue

        # clean up antiSMASH annotations in CDS features
        if feature.type == 'CDS':
            if 'sec_met' in feature.qualifiers:
                del feature.qualifiers['sec_met']

        new_features.append(feature)

    seq_record.features = new_features


def run_analyses(seq_record, options, plugins):
    "Run antiSMASH analyses for a single SeqRecord"

    if 'next_clusternr' not in options:
        options.next_clusternr = 1

    options.clusternr_offset = options.next_clusternr

    #Detect gene clusters
    detect_geneclusters(seq_record, options)

    for f in utils.get_cluster_features(seq_record):
        logging.debug(f)

    #Do specific analyses
    # TODO: Run this in parallel, perhaps?
    if len(utils.get_cluster_features(seq_record)) > 0:
        cluster_specific_analysis(plugins, seq_record, options)
    unspecific_analysis(seq_record, options)

    if len(utils.get_cluster_features(seq_record)) > 0:

        utils.log_status("Detecting TTA codons")

        # Run TTA codon detection
        if options.tta:
            tta.detect(seq_record, options)

        #Run smCOG analysis
        if options.smcogs:
            utils.log_status("Detecting smCOGs for contig #%d" % options.record_idx)
            smcogs.run_smcog_analysis(seq_record, options)

        #Run ClusterBlast
        if options.clusterblast:
            utils.log_status("ClusterBlast analysis for contig #%d" % options.record_idx)
            clusterblast.run_clusterblast(seq_record, options)
            #clusterblastvars info could also be pickled are transferred some other way if we want it to be possible to reconstruct complete output from files

        #Run SubClusterBlast
        if options.subclusterblast:
            utils.log_status("SubclusterBlast analysis for contig #%d" % options.record_idx)
            subclusterblast.run_subclusterblast(seq_record, options)

        #Run KnownClusterBlast
        if options.knownclusterblast:
            utils.log_status("KnownclusterBlast analysis for contig #%d" % options.record_idx)
            knownclusterblast.run_knownclusterblast(seq_record, options)

        # run active site finder
        if options.run_asf:
            ASFObj = active_site_finder.active_site_finder(seq_record, options)
            status = ASFObj.execute()
            if status:
                logging.debug("Active site finder execution successful")
            else:
                logging.error("Error in active site finder module!")


def list_available_plugins(enabled_cluster_types, plugins, output_plugins):
    print("\nDetecting the following secondary metabolites:")
    for cluster_type in enabled_cluster_types:
        print(" * %s" % cluster_type)

    print("\nSupport for detailed predictions for the following secondary metabolites:")
    for plugin in plugins:
        print(" * %s" % plugin.short_description)

    print("\nSupport for the following output formats:")
    for plugin in output_plugins:
        print(" * %s" % plugin.short_description)


def filter_outputs(plugins, options):
    remaining = []
    for plugin in plugins:
        logging.debug(plugin.name)
        if plugin.name in options and plugin.enabled:
            logging.debug("Removing deactivated plugin %r", plugin.name)
            continue
        if not plugin.enabled and plugin.name not in options:
            logging.debug("Not activating disabled plugin %r", plugin.name)
            continue
        remaining.append(plugin)

    if not remaining:
        print("No plugins enabled, use --list-plugins to show available plugins")
        sys.exit(1)
    return remaining


def write_results(plugins, seq_records, options):
    for plugin in plugins:
        plugin.write(seq_records, options)


def zip_results(seq_records, options):
    "Create a zip archive with all the results generated so far"
    zip_name = '%s.zip' % seq_records[0].id
    utils.zip_path(path.abspath(options.outputfoldername), zip_name)


def setup_logging(options):
    "Set up the logging output"
    if options.debug:
        log_level = logging.DEBUG
    elif options.verbose:
        log_level = logging.INFO
    else:
        log_level = logging.WARNING

    logging.basicConfig(format='%(levelname)-7s %(asctime)s   %(message)s',
                        level=log_level, datefmt="%d/%m %H:%M:%S")
    if 'logfile' in options:
        if not (path.dirname(options.logfile) == "" or os.path.exists(path.dirname(options.logfile))):
            os.mkdir(path.dirname(options.logfile))
        fh = logging.FileHandler(options.logfile)
        if options.debug:
            log_level = logging.DEBUG
        else:
            log_level = logging.INFO
        fh.setLevel(log_level)
        fh.setFormatter(logging.Formatter(fmt='%(levelname)-7s %(asctime)s   %(message)s', datefmt="%d/%m %H:%M:%S"))
        logging.getLogger('').addHandler(fh)


def load_cluster_specific_plugins(options):
    """Load available secondary metabolite cluster specific prediction modules"""
    if options.minimal:
        logging.info('Minimal run, not loading all cluster specific modules')
        loaded_modules = []

        # Argparse gives None for a append_const argument that had nothing appended. Go figure.
        if options.enabled_specific_plugins is None:
            return loaded_modules

        for plugin in ALL_CLUSTER_SPECIFIC_MODULES:
            if plugin.name in options.enabled_specific_plugins:
                logging.info('Loading %s module', plugin.short_description)
                loaded_modules.append(plugin)

        return loaded_modules

    logging.info('Loading cluster specific modules')
    plugins = ALL_CLUSTER_SPECIFIC_MODULES

    return plugins


def load_output_plugins():
    """Load available output formats"""
    plugins = [biosynml, embl, genbank, html, svg, txt, xls]
    plugins.sort(cmp=lambda x, y: cmp(x.priority, y.priority))
    return plugins


def is_nucl_seq(sequence):
    if len(str(sequence).lower().replace("a","").replace("c","").replace("g","").replace("t","").replace("n","")) < 0.2 * len(sequence):
        return True
    else:
        return False


def generate_nucl_seq_record(sequences):
    "Generate nucleotide seq_record"
    if len(sequences) == 0:
        return []
    seq_record = SeqRecord(Seq(""),id="Protein_Input", name="ProteinInput",
                   description="antiSMASH protein input")
    position = 0
    cds_features = []
    cdsnames = []
    for sequence in sequences:
        startpos = position
        endpos = position + len(sequence) * 3
        position += len(sequence) * 3 + 1000
        location = FeatureLocation(startpos, endpos)
        cdsfeature = SeqFeature(location, type="CDS")
        cdsfeature.strand = 1
        sequence_id = sequence.id[:15].replace(" ","_")
        if sequence_id not in cdsnames:
            cdsfeature.qualifiers['product'] = [sequence_id]
            cdsfeature.qualifiers['locus_tag'] = [sequence_id]
            cdsnames.append(sequence_id)
        else:
            x = 1
            while sequence_id[:8] + "_" + str(x) in cdsnames:
                x += 1
            cdsfeature.qualifiers['product'] = [sequence_id[:8] + "_" + str(x)]
            cdsfeature.qualifiers['locus_tag'] = [sequence_id[:8] + "_" + str(x)]
            cdsnames.append(sequence_id[:8] + "_" + str(x))
        cdsfeature.qualifiers['translation'] = [str(sequence.seq).replace('.', 'X')]
        cds_features.append(cdsfeature)
    seq_record.features.extend(cds_features)
    return [seq_record]


def add_translations(seq_records):
    "Add a translation qualifier to all CDS features"
    for seq_record in seq_records:
        cdsfeatures = utils.get_cds_features(seq_record)
        for cdsfeature in cdsfeatures:
            if 'translation' not in cdsfeature.qualifiers or len(cdsfeature.qualifiers['translation']) == 0:
                if len(seq_record.seq) == 0:
                    logging.error('No amino acid sequence in input entry for CDS %r, ' \
                            'and no nucleotide sequence provided to translate it from.', cdsfeature.id)
                    sys.exit(1)
                else:
                    import Bio.Data.CodonTable
                    try:
                        translation = str(utils.get_aa_translation(seq_record, cdsfeature))
                    except Bio.Data.CodonTable.TranslationError as e:
                        logging.error('Getting amino acid sequences from %s, CDS %r failed: %s',
                                seq_record.name, cdsfeature.id, e)
                        sys.exit(1)
                    cdsfeature.qualifiers['translation'] = [translation]


def add_seq_record_seq(seq_records):
    for seq_record in seq_records:
        if len(seq_record.seq) == 0:
            seqmax = max([cds.location.start for cds in utils.get_cds_features(seq_record)] + [cds.location.end for cds in utils.get_cds_features(seq_record)])
            seq_record.seq = Seq(seqmax * "n")


def check_duplicate_gene_ids(sequences):
    "Fix duplicate locus tags so that they are different"
    NO_TAG = "no_tag_found"
    high_water_mark = 0
    all_ids = defaultdict(lambda: False)
    for sequence in sequences:
        seq_ids = utils.get_cds_features(sequence)
        for cdsfeature in seq_ids:
            gene_id = utils.get_gene_id(cdsfeature)
            if not all_ids[gene_id]:
                all_ids[gene_id] = True
            else:
                if gene_id == NO_TAG:
                    x = high_water_mark + 1
                else:
                    x = 1
                id_str = "%s_%s" % ( gene_id[:8], x)
                while all_ids[id_str]:
                    x += 1
                    id_str = "%s_%s" % ( gene_id[:8], x)
                logging.debug("generated id %r", id_str)
                cdsfeature.qualifiers['product'] = [id_str]
                cdsfeature.qualifiers['locus_tag'] = [id_str]
                all_ids[id_str] = True
                if gene_id == NO_TAG:
                    high_water_mark = x


def fix_id_lines(options, filename):

    with open(filename, 'r') as fh:
        file_content = fh.read()

    while file_content[:2] == "\n\n":
        file_content = file_content[1:]

    try:
        filetype = seqio._get_seqtype_from_ext(filename)
    except ValueError as e:
        logging.debug("Got ValueError %s trying to guess the file format", e)
        return filename

    needed_fixing = False

    if filetype == "fasta":
        if len([line for line in file_content.split("\n") if (len(line) > 0 and line[0] == ">" and " " in line)]) > 0:
            lines = []
            for line in file_content.split("\n"):
                if (len(line) > 0 and line[0] == ">" and " " in line):
                    lines.append(line.replace(" ", ""))
                else:
                    lines.append(line)
            file_content = "\n".join(lines)
            needed_fixing = True
    elif filetype == "genbank":
        if "LOCUS       " not in file_content.partition("\n")[0]:
            file_content = "LOCUS       A01                    0 bp        DNA              BCT 01-JAN-2000\n" + file_content
            needed_fixing = True
    elif filetype == "embl":
        if "ID   " not in file_content.partition("\n")[0]:
            file_content = "ID   A01; SV 1; linear; unassigned DNA; STD; PRO; 0 BP.\nXX\n" + file_content
            needed_fixing = True

    if needed_fixing:
        relative_name = path.basename(filename)
        name, ext = path.splitext(relative_name)
        new_name = path.join(options.full_outputfolder_path, "{}_fixed{}".format(name, ext))
        filename = new_name
        with open(filename, 'w') as fh:
            fh.write(file_content)

    return filename


def parse_input_sequences(options):
    "Parse the input sequences from given filename"
    filenames = options.sequences
    logging.info('Parsing input sequences %r', filenames)

    sequences = []
    for filename in filenames:

        if not path.exists(filename):
            logging.error('No sequence file found at %r', filename)
            sys.exit(1)

        infile = open(filename, "r")
        file_content = infile.read()
        infile.close()
        if "Resource temporarily unavailable" in file_content[:200] or \
                "<h1>Server Error</h1>" in file_content[:500] or \
                "NCBI - WWW Error" in file_content[:500]:
            logging.error('ERROR: NCBI server temporarily unavailable: downloading %s failed.', os.path.basename(filename))
            sys.exit(1)

        if options.fix_id_line:
            filename = fix_id_lines(options, filename)

        try:
            record_list = list(seqio.parse(filename))
            if len(record_list) == 0:
                logging.error('No sequence in file %r', filename)
            sequences.extend([rec for rec in record_list if len(rec.seq) > options.minlength or \
                ('contig' in rec.annotations or 'wgs_scafld' in rec.annotations or \
                'wgs' in rec.annotations)])
        except (ValueError, AssertionError) as e:
            logging.error('Parsing %r failed: %s', filename, e)
            sys.exit(1)
        except Exception as e:
            logging.error('Parsing %r failed with unhandled exception: %s',
                          filename, e)
            sys.exit(1)
    #Check if seq_records have appropriate content
    i = 0
    while i < len(sequences):
        sequence = sequences[i]
        if not isinstance(sequence.seq, UnknownSeq):
            cleanseq = str(sequence.seq).replace("-", "")
            cleanseq = cleanseq.replace(":", "")
            sequence.seq = Seq(cleanseq)
        #Check if seq_record has either a sequence or has at least 80% of CDS features with 'translation' qualifier
        cdsfeatures = utils.get_cds_features(sequence)
        cdsfeatures_with_translations = [cdsfeature for cdsfeature in cdsfeatures if 'translation' in cdsfeature.qualifiers]
        if len(sequence.seq) == 0 or (
                options.input_type == 'nucl' and \
                len(str(sequence.seq).replace("N","")) == 0 and \
                len(cdsfeatures_with_translations) < 0.8 * len(cdsfeatures)):
            logging.error("Record %s has no sequence, skipping.", sequence.id)
            sequences.pop(i)
            continue

        if options.input_type == 'prot':
            if is_nucl_seq(sequence.seq):
                logging.error("Record %s is a nucleotide record, skipping.", sequence.id)
                sequences.pop(i)
                continue
        elif options.input_type == 'nucl':
            if not isinstance(sequence.seq.alphabet, NucleotideAlphabet) and not is_nucl_seq(sequence.seq):
                logging.error("Record %s is a protein record, skipping.", sequence.id)
                sequences.pop(i)
                continue
            if sequence.seq.alphabet != generic_dna:
                sequence.seq.alphabet = generic_dna

        i += 1

    #If protein input, convert all protein seq_records to one nucleotide seq_record
    if options.input_type == 'prot':
        sequences = generate_nucl_seq_record(sequences)

    #Handle WGS master or supercontig entries
    sequences = utils.process_wgs_master_scaffolds(sequences)

    #Now remove small contigs < minimum length again
    sequences = [rec for rec in sequences if len(rec.seq) > options.minlength]

    # Make sure we don't waste weeks of runtime on huge records, unless requested by the user
    old_len = len(sequences)
    if options.limit > -1:
        sequences = sequences[:options.limit]

    new_len = len(sequences)
    if new_len < old_len:
        options.triggered_limit = True
        logging.warning("Only analysing the first %d records (increase via --limit)" % options.limit)

    #Check if no duplicate locus tags / gene IDs are found
    check_duplicate_gene_ids(sequences)

    #If no CDS entries in records, run gene finding
    options.record_idx = 1

    #Store IDs for all entries
    options.all_record_ids = {seq.id for seq in sequences}

    # For retaining the correct contig numbers, a second counter is required also including removed sequences without genes
    options.orig_record_idx = 1

    # Check GFF suitability
    if options.gff3:
        try:
            gff_parser.check_gff_suitability(options, sequences)
        except ValueError as e:
            sys.exit(1)

    i = 0
    matched_filter = 'limit_to_record' not in options # defaults to True
    while i < len(sequences):
        sequence = sequences[i]
        #Fix sequence name (will be ID) if it contains illegal chars
        illegal_chars  = '''!"#$%&()*+,:; \r\n\t=>?@[]^`'{|}/ '''
        for char in sequence.name:
            if char in illegal_chars:
                sequence.name = sequence.name.replace(char, "_")
        if 'limit_to_record' in options:
            if sequence.id != options.limit_to_record:
                logging.debug("Skipping record %s, doesn't match filter %s",
                              sequence.name, options.limit_to_record)
                options.orig_record_idx += 1
                i += 1
                continue
            else:
                matched_filter = True
        #Iterate through sequence objects
        if len(utils.get_cds_features(sequence)) < 1:
            if options.gff3:
                logging.info("No CDS features found in record %r but GFF3 file provided, running GFF parser.", sequence.id)
                gff_parser.run(sequence, options)
                check_duplicate_gene_ids(sequences)
            else:
                logging.info("No CDS features found in record %r, running gene finding.", sequence.id)
                genefinding.find_genes(sequence, options)
            if len(utils.get_cds_features(sequence)) < 1:
                logging.info("No genes found, skipping record")
                sequences.pop(i)
                options.orig_record_idx += 1
                continue
        #Fix locus tags
        utils.fix_locus_tags(sequence, options)
        options.record_idx += 1
        options.orig_record_idx += 1
        i += 1

    if not matched_filter:
        raise ValueError("No records matched given record filter: %s" % options.limit_to_record)

    #Make sure that all CDS entries in all seq_records have translation tags, otherwise add them
    add_translations(sequences)

    #Make sure that all seq_records have a sequence
    add_seq_record_seq(sequences)

    if len(sequences) > 1:
        options.start = -1
        options.end = -1
        logging.info("Discarding --from and --to options, as multiple entries are used.")

    i = 0
    while i < len(sequences):
        sequence = sequences[i]

        if options.start > 1:
            if options.start > len(sequence):
                logging.error('Specified analysis start point is at %r, which is larger ' \
                              'than record size %r', options.start, len(sequence))
                sys.exit(1)
            sequence = sequence[options.start-1:]
            # new sequence is shorter, so fix the end calculation
            options.end -= options.start
            sequences[i] = sequence

        if options.end > 0:
            if options.end > len(sequence):
                logging.error('Specified analysis end point is at %r, which is larger ' \
                              'than record size %r', options.end, len(sequence))
                sys.exit(1)
            sequence = sequence[:options.end]
            sequences[i] = sequence

        # Some programs write gaps as - not N, but translate() hates that
        if sequence.seq.find('-') > -1:
            sequence.seq = Seq(str(sequence.seq).replace('-', 'N'),
                               alphabet=sequence.seq.alphabet)

        # Some programs like to write gaps as X, translate() hates that
        if sequence.seq.find('X') > -1:
            sequence.seq = Seq(str(sequence.seq).replace('X', 'N'),
                               alphabet=sequence.seq.alphabet)
        if sequence.seq.find('x') > -1:
            sequence.seq = Seq(str(sequence.seq).replace('x', 'N'),
                               alphabet=sequence.seq.alphabet)

        i += 1

    #Fix sequence record IDs to be unique
    ids_used = []
    for sequence in sequences:
        seq_id = sequence.id
        if seq_id not in ids_used:
            ids_used.append(seq_id)
        else:
            x = 0
            #Make sure the length of the ID does not exceed 16
            if len(seq_id) <= 12:
                while "%s_%i" % (seq_id, x) in ids_used:
                    x += 1
                sequence.id = "%s_%i" % (seq_id, x)
            else:
                while "%s_%i" % (seq_id[:-4], x) in ids_used:
                    x += 1
                sequence.id = "%s_%i" % (seq_id[:-4], x)
            ids_used.append(sequence.id)
            options.all_record_ids.add(sequence.id) #Update all_record_ids with new record
    return sequences


def check_prereqs(plugins, options):
    failure_messages = []
    failure_messages.extend(generic_check_prereqs(options))
    for plugin in plugins:
        if 'check_prereqs' in dir(plugin):
            failure_messages.extend(plugin.check_prereqs())

    for msg in failure_messages:
        logging.error(msg)

    return len(failure_messages)


def detect_signature_genes(seq_record, clustertypes, options):
    "Detect different secondary metabolite clusters based on HMM signatures"
    logging.info('Looking for secondary metabolite cluster signatures')
    hmm_detection.detect_signature_genes(seq_record, clustertypes, options)


def detect_geneclusters(seq_record, options):
    if options.input_type == 'nucl':
        utils.log_status("Detecting secondary metabolite signature genes " \
                         "for contig #%d" % options.record_idx)
    else:
        utils.log_status("Detecting secondary metabolite signature genes " \
                         "for supplied amino acid sequences")
    detect_signature_genes(seq_record, options.enabled_cluster_types, options)
    if options.inclusive:
        utils.log_status("Detecting secondary metabolite clusters using "\
                         "inclusive ClusterFinder algorithm for contig #%d" % options.record_idx)
        fullhmmer.run(seq_record, options)
        clusterfinder.run_cluster_finder(seq_record, options)
        options.full_hmmer = False

    if options.cassis:
        utils.log_status("Detecting secondary metabolite cluster borders "
                         "using CASSIS algorithm for contig #{}".format(
            options.record_idx
        ))
        cassis.detect(seq_record, options)


def cluster_specific_analysis(plugins, seq_record, options):
    "Run specific analysis steps for every detected gene cluster"
    logging.info('Running cluster-specific analyses')

    for plugin in plugins:
        if 'specific_analysis' in dir(plugin):
            logging.debug('Running analyses specific to %s clusters',
                          plugin.short_description)
            plugin.specific_analysis(seq_record, options)
        else:
            logging.debug('No specific analyses implemented for %s clusters',
                          plugin.short_description)


def unspecific_analysis(seq_record, options):
    "Run analyses independent of specific clusters"
    logging.info('Running analyses independent of specific cluster types')

    if options.full_hmmer:
        utils.log_status("Running full-genome PFAM analysis for contig #%d" % \
                         options.record_idx)
        fullhmmer.run(seq_record, options)

    # TODO: offer fullblast here


if __name__ == "__main__":
    main()
