#  _________________________________________________________________________
#
#  Pyomo: Python Optimization Modeling Objects
#  Copyright (c) 2014 Sandia Corporation.
#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
#  the U.S. Government retains certain rights in this software.
#  This software is distributed under the BSD License.
#  _________________________________________________________________________

import sys
import logging
import os
import re
import time
import math

import pyutilib.services
from pyutilib.misc import Bunch, Options

import pyomo.util.plugin
from pyomo.opt.base import *
from pyomo.opt.base.solvers import _extract_version
from pyomo.opt.results import *
from pyomo.opt.solver import *
from pyomo.core.base import (SymbolMap,
                             ComponentMap,
                             NumericLabeler,
                             TextLabeler,
                             value)
from pyomo.repn import generate_canonical_repn
from pyomo.solvers import wrappers

from six import itervalues, iterkeys, iteritems, advance_iterator
from six.moves import xrange

logger = logging.getLogger('pyomo.solvers')

try:
    unicode
except:
    basestring = str

_cplex_version = None
try:
    import cplex
    from cplex.exceptions import CplexError, CplexSolverError
    # create a version tuple of length 4
    _cplex_version = tuple(int(i) for i in cplex.Cplex().get_version().split('.'))
    while(len(_cplex_version) < 4):
        _cplex_version += (0,)
    _cplex_version = _cplex_version[:4]
    cplex_import_available=True
except ImportError:
    cplex_import_available=False

class CplexSolverWrapper(wrappers.MIPSolverWrapper):

    def __init__(self, solver):
        self.cplex = solver

    def add(self, constraint):
        """TODO"""
        pass


class ModelSOS(object):
    def __init__(self):
        self.sosType = {}
        self.sosName = {}
        self.varnames = {}
        self.varids = {}
        self.weights = {}
        self.block_cntr = 0

    def count_constraint(self,symbol_map,labeler,variable_label_map,soscondata):

        sos_items = list(soscondata.get_items())
        level = soscondata.level

        if len(sos_items) == 0:
            return

        self.block_cntr += 1
        varnames = self.varnames[self.block_cntr] = []
        varids = self.varids[self.block_cntr] = []
        weights = self.weights[self.block_cntr] = []
        if level == 1:
            self.sosType[self.block_cntr] = cplex.Cplex.SOS.type.SOS1
        elif level == 2:
            self.sosType[self.block_cntr] = cplex.Cplex.SOS.type.SOS2
        else:
            raise ValueError("Unsupported SOSConstraint level %s" % level)

        self.sosName[self.block_cntr] = symbol_map.getSymbol(soscondata,labeler)

        for vardata, weight in sos_items:
            if vardata.fixed:
                raise RuntimeError("SOSConstraint '%s' includes a fixed variable '%s'. "
                                   "This is currently not supported. Deactivate this constraint "
                                   "in order to proceed" % (soscondata.cname(True), vardata.cname(True)))
            varids.append(id(vardata))
            varnames.append(variable_label_map.getSymbol(vardata))
            weights.append(weight)


class CPLEXPersistent(PersistentSolver):
    """The CPLEX LP/MIP solver
    """

    pyomo.util.plugin.alias('_cplex_persistent',  doc='Persistent Python interface to the CPLEX LP/MIP solver')

    # TBD: Eventually, the CPLEX direct plugin should be merged in with the CPLEX persistent solver plugin -
    #      the capabilities are 99% similar, and should be able to co-exist following introduction of a flag.

    def __init__(self, **kwds):
        #
        # Call base class constructor
        #
        kwds['type'] = 'cplexpersistent'
        OptSolver.__init__(self, **kwds)

        # this interface doesn't use files, but we can create a log
        # file is requested
        self._keepfiles = False
        # do we warmstart
        self._warm_start_solve = False
        # io_options
        self._symbolic_solver_labels = False
        self._output_fixed_variable_bounds = False

        # The working problem instance, via CPLEX python constructs.
        self._active_cplex_instance = None

        # Repeatedly extracting the set of variable names (which are
        # needed to construct a results solution object) via the CPLEX
        # Python API is expensive - so we don't do it. Populated
        # following initial instance compilation, it needs to be
        # updated when new variables are added to the model.
        self._cplex_variable_names = []

        # for purposes of re-compiling variables, it is useful to
        # construct a map between the var_data labels (generated by
        # the labeler object) and the corresponding numeric identifier
        # in CPLEX - name-based indexing is expensive, taking time for
        # map lookups.
        self._cplex_variable_ids = {}

        # Various attributes of the active cplex instance, cached for efficiency purposes.
        self._has_quadratic_constraints = False
        self._has_quadratic_objective = False

        # Note: Undefined capabilities default to 'None' due to the Options class implementation.
        self._capabilities = Options()
        self._capabilities.linear = True
        self._capabilities.quadratic_objective = True
        self._capabilities.quadratic_constraint = True
        self._capabilities.integer = True
        self._capabilities.sos1 = True
        self._capabilities.sos2 = True

        # flag allowing for the use, during solves, of user-defined callbacks.
        self._allow_callbacks = True

        # the CPLEX python API doesn't provide a mechanism to track
        # user/system/wall clock time, so it's up to us. stored as an
        # attribute of the plugin to facilitate persistance across
        # various portions of the method invocations.
        self._solve_user_time = None

        # collection of id(_VarData).
        self._referenced_variable_ids = set()

        # TBD - this will have to expand significantly.

    def available(self, exception_flag=True):
        """ True if the solver is available """

        if exception_flag is False:
            return cplex_import_available
        else:
            if cplex_import_available is False:
                raise ApplicationError("No CPLEX <-> Python bindings available "
                                       "- persistent CPLEX solver functionality "
                                       "is not available")
            else:
                return True

    def _get_version(self):
        if _cplex_version is None:
            return _extract_version('')
        return _cplex_version

    #
    # TBD
    #
    def _evaluate_bound(self, exp):

        from pyomo.core.base import expr

        if exp.is_fixed():
            return exp()
        else:
            raise ValueError("ERROR: non-fixed bound: " + str(exp))

    #
    # CPLEX requires objective expressions to be specified via something other than a sparse pair!
    # NOTE: The returned offset is guaranteed to be a float.
    #
    def _encode_constraint_body_linear(self, expression, labeler, as_pairs=False):

        variables = [] # string names of variables
        coefficients = [] # variable coefficients

        pairs = []

        hash_to_variable_map = expression[-1]
        self_variable_label_map = self._variable_label_map

        for var_hash, var_coefficient in iteritems(expression[1]):

            vardata = hash_to_variable_map[var_hash]
            self._referenced_variable_ids.add(id(vardata))
            variable_name = self_variable_label_map.getSymbol(vardata)

            if as_pairs is True:
                pairs.append((variable_name, var_coefficient))
            else:
                variables.append(variable_name)
                coefficients.append(var_coefficient)

        offset=0.0
        if 0 in expression:
            offset = expression[0][None]

        if as_pairs is True:
            return pairs, offset
        else:
            expr = cplex.SparsePair(ind=variables, val=coefficients)
            return expr, offset

    #
    # CPLEX requires objective expressions to be specified via something other than a sparse pair!
    # NOTE: The returned offset is guaranteed to be a float.
    # NOTE: This function is a variant of the above, specialized for LinearCanonicalRepn objects.
    #
    def _encode_constraint_body_linear_specialized(self, linear_repn, labeler, as_pairs=False):

        variables = [] # string names of variables
        coefficients = [] # variable coefficients

        pairs = []

        self_variable_label_map = self._variable_label_map
        self_cplex_variable_ids = self._cplex_variable_ids

        for i in xrange(0,len(linear_repn.linear)):

            var_coefficient = linear_repn.linear[i]
            var_value = linear_repn.variables[i]
            self._referenced_variable_ids.add(id(var_value))
            variable_name = self_variable_label_map.getSymbol(var_value)
            variable_id = self_cplex_variable_ids[variable_name]

            if as_pairs is True:
                pairs.append((variable_name, var_coefficient))
            else:
                variables.append(variable_id)
                coefficients.append(var_coefficient)

        offset=0.0
        if linear_repn.constant != None:
            offset = linear_repn.constant

        if as_pairs is True:
            return pairs, offset
        else:
            expr = cplex.SparsePair(ind=variables, val=coefficients)
            return expr, offset

    #
    #Handle quadratic constraints and objectives
    #
    def _encode_constraint_body_quadratic(self, expression, labeler, as_triples=False, is_obj=1.0):

        variables1 = [] # string names of variables
        variables2 = [] # string names of variables
        coefficients = [] # variable coefficients

        triples = []

        hash_to_variable_map = expression[-1]

        self_variable_label_map = self._variable_label_map
        for vrs, coeff in iteritems(expression[2]):

            variable_hash_iter = iterkeys(vrs)
            vardata = hash_to_variable_map[advance_iterator(variable_hash_iter)]
            self._referenced_variable_ids.add(id(vardata))
            var1 = self_variable_label_map.getSymbol(vardata)
            if len(vrs)==2:
                vardata = hash_to_variable_map[advance_iterator(variable_hash_iter)]
                self._referenced_variable_ids.add(id(vardata))
                var2 = self_variable_label_map.getSymbol(vardata)
            else:
                var2 = var1

            if as_triples is True:
                triples.append((var1, var2, is_obj*coeff))
            else:
                variables1.append(var1)
                variables2.append(var2)
                coefficients.append(coeff)

        if as_triples is True:
            return triples
        else:
            expr = cplex.SparseTriple(ind1=variables1,ind2=variables2,val=coefficients)
            return expr

    #
    # updates all variable bounds in the compiled model - handles
    # fixed variables and related issues.  re-does everything from
    # scratch by default, ignoring whatever was specified
    # previously. if the value associated with the keyword
    # vars_to_update is a non-empty list (assumed to be variable name
    # / index pairs), then only the bounds for those variables are
    # updated.  this function assumes that the variables themselves
    # already exist in the compiled model.
    #
    def compile_variable_bounds(self, pyomo_instance, vars_to_update):

        from pyomo.core.base import Var

        if self._active_cplex_instance is None:
            raise RuntimeError("***The CPLEXPersistent solver plugin "
                               "cannot compile variable bounds - no "
                               "instance is presently compiled")

        # the bound update entries should be name-value pairs
        new_lower_bounds = []
        new_upper_bounds = []

        # operates through side effects on the above lists!
        def update_bounds_lists(var_name):

            var_lb = None
            var_ub = None

            if var_data.fixed and self._output_fixed_variable_bounds:
                var_lb = var_ub = var_data.value
            elif var_data.fixed:
                # if we've been directed to not deal with fixed
                # variables, then skip - they should have been
                # compiled out of any description of the constraints
                return
            else:
                if var_data.lb is None:
                    var_lb = -cplex.infinity
                else:
                    var_lb = value(var_data.lb)

                if var_data.ub is None:
                    var_ub = cplex.infinity
                else:
                    var_ub= value(var_data.ub)

            var_cplex_id = self._cplex_variable_ids[var_name]

            new_lower_bounds.append((var_cplex_id, var_lb))
            new_upper_bounds.append((var_cplex_id, var_ub))

        if len(vars_to_update) == 0:
            for var_data in pyomo_instance.component_data_objects(Var, active=True):
                var_name = self._symbol_map.getSymbol(var_data, self._labeler)
                update_bounds_lists(var_name)
        else:
            for var_name, var_index in vars_to_update:
                var = pyomo_instance.find_component(var_name)
                # TBD - do some error checking!
                var_data = var[var_index]
                var_name = self._symbol_map.getSymbol(var_data, self._labeler)
                update_bounds_lists(var_name)

        self._active_cplex_instance.variables.set_lower_bounds(new_lower_bounds)
        self._active_cplex_instance.variables.set_upper_bounds(new_upper_bounds)

    #
    # method to compile objective of the input pyomo instance.
    # TBD:
    #   it may be smarter just to track the associated pyomo instance,
    #   and re-compile it automatically from a cached local attribute.
    #   this would ensure consistency, among other things!
    #
    def compile_objective(self, pyomo_instance):

        from pyomo.core.base import Objective
        from pyomo.repn import canonical_is_constant, LinearCanonicalRepn

        if self._active_cplex_instance is None:
            raise RuntimeError("***The CPLEXPersistent solver plugin "
                               "cannot compile objective - no "
                               "instance is presently compiled")

        cplex_instance = self._active_cplex_instance

        cntr = 0
        for block in pyomo_instance.block_data_objects(active=True):
            gen_obj_canonical_repn = \
                getattr(block, "_gen_obj_canonical_repn", True)
            # Get/Create the ComponentMap for the repn
            if not hasattr(block,'_canonical_repn'):
                block._canonical_repn = ComponentMap()
            block_canonical_repn = block._canonical_repn

            for obj_data in block.component_data_objects(Objective, active=True, descend_into=False):

                cntr += 1
                if cntr > 1:
                    raise ValueError(
                        "Multiple active objectives found on Pyomo instance '%s'. "
                        "Solver '%s' will only handle a single active objective" \
                        % (pyomo_instance.cname(True), self.type))

                if obj_data.is_minimizing():
                    cplex_instance.objective.set_sense(
                        cplex_instance.objective.sense.minimize)
                else:
                    cplex_instance.objective.set_sense(
                        cplex_instance.objective.sense.maximize)

                cplex_instance.objective.set_name(
                    self._symbol_map.getSymbol(obj_data,
                                               self._labeler))

                if gen_obj_canonical_repn:
                    obj_repn = generate_canonical_repn(obj_data.expr)
                    block_canonical_repn[obj_data] = obj_repn
                else:
                    obj_repn = block_canonical_repn[obj_data]

                if (isinstance(obj_repn, LinearCanonicalRepn) and \
                    (obj_repn.linear == None)) or \
                    canonical_is_constant(obj_repn):
                    print("Warning: Constant objective detected, replacing "
                          "with a placeholder to prevent solver failure.")

                    objective_expression = [("ONE_VAR_CONSTANT",offset)]
                    cplex_instance.objective.set_linear(objective_expression)

                else:

                    if isinstance(obj_repn, LinearCanonicalRepn):
                        objective_expression, offset = \
                            self._encode_constraint_body_linear_specialized(obj_repn,
                                                                            self._labeler,
                                                                            as_pairs=True)
                        if offset != 0.0:
                            objective_expression.append(("ONE_VAR_CONSTANT",offset))
                        cplex_instance.objective.set_linear(objective_expression)

                    else:
                        #Linear terms
                        if 1 in obj_repn:
                            objective_expression, offset = \
                                self._encode_constraint_body_linear(obj_repn,
                                                                    self._labeler,
                                                                    as_pairs=True)
                            if offset != 0.0:
                                objective_expression.append(("ONE_VAR_CONSTANT",offset))
                            cplex_instance.objective.set_linear(objective_expression)

                        #Quadratic terms
                        if 2 in obj_repn:
                            self._has_quadratic_objective = True
                            objective_expression = \
                                self._encode_constraint_body_quadratic(obj_repn,
                                                                       self._labeler,
                                                                       as_triples=True,
                                                                       is_obj=2.0)
                            cplex_instance.objective.set_quadratic_coefficients(objective_expression)

    #
    # method to populate the CPLEX problem instance (interface) from
    # the supplied Pyomo problem instance.
    #
    def compile_instance(self,
                         pyomo_instance,
                         symbolic_solver_labels=False,
                         output_fixed_variable_bounds=False):

        from pyomo.core.base import Var, Constraint, SOSConstraint
        from pyomo.repn import canonical_is_constant, LinearCanonicalRepn

        self._symbolic_solver_labels = symbolic_solver_labels
        self._output_fixed_variable_bounds = output_fixed_variable_bounds

        self._has_quadratic_constraints = False
        self._has_quadratic_objective = False
        used_sos_constraints = False

        self._active_cplex_instance = cplex.Cplex()

        if self._symbolic_solver_labels:
            labeler = self._labeler = TextLabeler()
        else:
            labeler = self._labeler = NumericLabeler('x')

        self._symbol_map = SymbolMap()
        pyomo_instance.solutions.add_symbol_map(self._symbol_map)
        self._smap_id = id(self._symbol_map)
        # we use this when iterating over the constraints because it will have a much smaller hash
        # table, we also use this for the warm start code after it is cleaned to only contain
        # variables referenced in the constraints
        self._variable_label_map = SymbolMap()

        # cplex wants the caller to set the problem type, which is (for current
        # purposes) strictly based on variable type counts.
        num_binary_variables = 0
        num_integer_variables = 0
        num_continuous_variables = 0

        # transfer the variables from pyomo to cplex.
        var_names = []
        var_lbs = []
        var_ubs = []
        var_types = []

        self._referenced_variable_ids.clear()

        # maps pyomo var data labels to the corresponding CPLEX variable id.
        self._cplex_variable_ids.clear()

        # cached in the loop below - used to update the symbol map
        # immediately following loop termination.
        var_label_pairs = []

        for var_data in pyomo_instance.component_data_objects(Var, active=True):
            if var_data.fixed and not self._output_fixed_variable_bounds:
                # if a variable is fixed, and we're preprocessing
                # fixed variables (as in not outputting them), there
                # is no need to add them to the compiled model.
                continue

            var_name = self._symbol_map.getSymbol(var_data, labeler)
            var_names.append(var_name)
            var_label_pairs.append((var_data, var_name))

            self._cplex_variable_ids[var_name] = len(self._cplex_variable_ids)

            if var_data.lb is None:
                var_lbs.append(-cplex.infinity)
            else:
                var_lbs.append(value(var_data.lb))
            if var_data.ub is None:
                var_ubs.append(cplex.infinity)
            else:
                var_ubs.append(value(var_data.ub))

            if var_data.is_integer():
                var_types.append(self._active_cplex_instance.variables.type.integer)
                num_integer_variables += 1
            elif var_data.is_binary():
                var_types.append(self._active_cplex_instance.variables.type.binary)
                num_binary_variables += 1
            elif var_data.is_continuous():
                var_types.append(self._active_cplex_instance.variables.type.continuous)
                num_continuous_variables += 1
            else:
                raise TypeError("Invalid domain type for variable with name '%s'. "
                                "Variable is not continuous, integer, or binary.")

        self._active_cplex_instance.variables.add(names=var_names, lb=var_lbs, ub=var_ubs, types=var_types)
        self._active_cplex_instance.variables.add(lb=[1],ub=[1],names=["ONE_VAR_CONSTANT"])

        self._variable_label_map.addSymbols(var_label_pairs)
        self._cplex_variable_names = self._active_cplex_instance.variables.get_names()

        # transfer the constraints.
        expressions = []
        senses = []
        rhss = []
        range_values = []
        names = []

        qexpressions = []
        qlinears = []
        qsenses = []
        qrhss = []
        qnames = []

        for block in pyomo_instance.block_data_objects(active=True):

            gen_con_canonical_repn = \
                getattr(block, "_gen_con_canonical_repn", True)
            # Get/Create the ComponentMap for the repn
            if not hasattr(block,'_canonical_repn'):
                block._canonical_repn = ComponentMap()
            block_canonical_repn = block._canonical_repn

            for con in block.component_data_objects(Constraint,
                                                    active=True,
                                                    descend_into=False):

                if gen_con_canonical_repn:
                    con_repn = generate_canonical_repn(con.body)
                    block_canonical_repn[con] = con_repn
                else:
                    con_repn = block_canonical_repn[con]

                # There are conditions, e.g., when fixing variables, under which
                # a constraint block might be empty.  Ignore these, for both
                # practical reasons and the fact that the CPLEX LP format
                # requires a variable in the constraint body.  It is also
                # possible that the body of the constraint consists of only a
                # constant, in which case the "variable" of
                if isinstance(con_repn, LinearCanonicalRepn):
                    if con_repn.linear == None:
                       continue
                else:
                   if canonical_is_constant(con_repn):
                       continue

                name=self._symbol_map.getSymbol(con,labeler)
                expr=None
                qexpr=None

                #Linear constraints
                quadratic = False
                if isinstance(con_repn, LinearCanonicalRepn):
                    expr, offset = self._encode_constraint_body_linear_specialized(con_repn, labeler)
                elif 2 in con_repn:
                    quadratic=True
                elif 1 in con_repn:
                    expr, offset = self._encode_constraint_body_linear(con_repn, labeler)

                #Quadratic constraints
                if quadratic is True:
                    if expr is None:
                        expr = cplex.SparsePair(ind=[0],val=[0.0])
                    self._has_quadratic_constraints = True

                    qexpr = self._encode_constraint_body_quadratic(con_repn,labeler)
                    qnames.append(name)

                    if con._equality:
                        # equality constraint.
                        qsenses.append('E')
                        bound_expr = con.lower
                        bound = self._evaluate_bound(bound_expr)
                        qrhss.append(bound)

                    elif con.lower is not None:
                        assert con.upper is not None
                        qsenses.append('G')
                        bound_expr = con.lower
                        bound = self._evaluate_bound(bound_expr)
                        qrhss.append(bound)

                    else:
                        qsenses.append('L')
                        bound_expr = con.upper
                        bound = self._evaluate_bound(bound_expr)
                        qrhss.append(bound)

                    qlinears.append(expr)
                    qexpressions.append(qexpr)

                else:
                    names.append(name)
                    expressions.append(expr)

                    if con._equality:
                        # equality constraint.
                        senses.append('E')
                        bound_expr = con.lower
                        bound = self._evaluate_bound(bound_expr) - offset
                        rhss.append(bound)
                        range_values.append(0.0)

                    elif (con.lower is not None) and (con.upper is not None):
                        # ranged constraint.
                        senses.append('R')
                        lower_bound_expr = con.lower # TBD - watch the offset - why not subtract?
                        lower_bound = self._evaluate_bound(lower_bound_expr)
                        upper_bound_expr = con.upper # TBD - watch the offset - why not subtract?
                        upper_bound = self._evaluate_bound(upper_bound_expr)
                        rhss.append(lower_bound)
                        range_values.append(upper_bound-lower_bound)

                    elif con.lower is not None:
                        senses.append('G')
                        bound_expr = con.lower
                        bound = self._evaluate_bound(bound_expr) - offset
                        rhss.append(bound)
                        range_values.append(0.0)

                    else:
                        senses.append('L')
                        bound_expr = con.upper
                        bound = self._evaluate_bound(bound_expr) - offset
                        rhss.append(bound)
                        range_values.append(0.0)

        # SOS constraints - largely taken from cpxlp.py so updates there,
        # should be applied here
        # TODO: Allow users to specify the variables coefficients for custom
        # branching/set orders - refer to cpxlp.py
        sosn = self._capabilities.sosn
        sos1 = self._capabilities.sos1
        sos2 = self._capabilities.sos2
        modelSOS = ModelSOS()
        for soscondata in pyomo_instance.component_data_objects(SOSConstraint, active=True):
            level = soscondata.level
            if (level == 1 and not sos1) or (level == 2 and not sos2) or (level > 2 and not sosn):
                raise Exception("Solver does not support SOS level %s constraints" % (level,))
            modelSOS.count_constraint(self._symbol_map,
                                      labeler,
                                      self._variable_label_map,
                                      soscondata)

        if modelSOS.sosType:
            for key in modelSOS.sosType:
                self._active_cplex_instance.SOS.add(type = modelSOS.sosType[key], \
                                       name = modelSOS.sosName[key], \
                                       SOS = [modelSOS.varnames[key], modelSOS.weights[key]] )
                self._referenced_variable_ids.update(modelSOS.varids[key])
            used_sos_constraints = True

        self._active_cplex_instance.linear_constraints.add(
            lin_expr=expressions,
            senses=senses,
            rhs=rhss,
            range_values=range_values,
            names=names)

        for index in xrange(len(qexpressions)):
            self._active_cplex_instance.quadratic_constraints.add(
                lin_expr=qlinears[index],
                quad_expr=qexpressions[index],
                sense=qsenses[index],
                rhs=qrhss[index],
                name=qnames[index])

        # transfer the objective.
        self.compile_objective(pyomo_instance)

        # set the problem type based on the variable counts.
        if (self._has_quadratic_objective is True) or (self._has_quadratic_constraints is True):
            if (num_integer_variables > 0) or (num_binary_variables > 0) or (used_sos_constraints):
                if self._has_quadratic_constraints is True:
                    self._active_cplex_instance.set_problem_type(self._active_cplex_instance.problem_type.MIQCP)
                else:
                    self._active_cplex_instance.set_problem_type(self._active_cplex_instance.problem_type.MIQP)
            else:
                if self._has_quadratic_constraints is True:
                    self._active_cplex_instance.set_problem_type(self._active_cplex_instance.problem_type.QCP)
                else:
                    self._active_cplex_instance.set_problem_type(self._active_cplex_instance.problem_type.QP)
        elif (num_integer_variables > 0) or (num_binary_variables > 0) or (used_sos_constraints):
            self._active_cplex_instance.set_problem_type(self._active_cplex_instance.problem_type.MILP)
        else:
            self._active_cplex_instance.set_problem_type(self._active_cplex_instance.problem_type.LP)

    #
    # simple method to query whether a Pyomo instance has already been
    # compiled.
    #
    def instance_compiled(self):

        return self._active_cplex_instance != None

    #
    # warm-starting is built-in - whatever values are present in the
    # populated CPLEX model will be used by default.
    #
    def warm_start_capable(self):

        return True

    #
    # propagate variable values from the Pyomo _VarData objects to the
    # corresponding CPLEX variable entries.
    #
    def _warm_start(self, instance):

        if self._active_cplex_instance is None:
            raise RuntimeError("***The CPLEXPersistent solver plugin "
                               "cannot warm start - no instance is "
                               "presently compiled")

        # clear any existing warm starts.
        self._active_cplex_instance.MIP_starts.delete()

        # the iteration order is identical to that used in generating
        # the cplex instance, so all should be well.
        variable_ids = []
        variable_values = []

        # IMPT: the var_data returned is a weak ref!
        for label, var_data in iteritems(self._variable_label_map.bySymbol):
            cplex_id = self._cplex_variable_ids[label]
            if var_data().fixed and not self._output_fixed_variable_bounds:
                continue
            elif var_data().value is not None:
                variable_ids.append(cplex_id)
                variable_values.append(var_data().value)

        if len(variable_ids):
            self._active_cplex_instance.MIP_starts.add(
                [variable_ids, variable_values],
                self._active_cplex_instance.MIP_starts.effort_level.auto)

    # over-ride presolve to extract the warm-start keyword, if specified.
    def _presolve(self, *args, **kwds):

        if self._active_cplex_instance is None:
            raise RuntimeError("***The CPLEXPersistent solver plugin"
                               " cannot presolve - no instance is "
                               "presently compiled")

        # create a context in the temporary file manager for
        # this plugin - is "pop"ed in the _postsolve method.
        pyutilib.services.TempfileManager.push()

        self._warm_start_solve = kwds.pop('warmstart', False)
        self._keepfiles = kwds.pop('keepfiles', False)
        # extract io_options here as well, since there is
        # way to tell what kwds were consumed inside
        # OptSolver._presolve. It will be up to that method
        # to decide if remaining kwds are error worthy

        # These specific options need to be handled in the
        # compile_instance method
        symbolic_solver_labels = \
            kwds.pop('symbolic_solver_labels', False)
        assert self._symbolic_solver_labels == \
            symbolic_solver_labels
        output_fixed_variable_bounds = \
            kwds.pop('output_fixed_variable_bounds', False)
        assert self._output_fixed_variable_bounds == \
            output_fixed_variable_bounds
        # TODO: A bad name for it here, but possibly still
        #       useful (perhaps generalize the name)
        #self._file_determinism = \
        #    kwds.pop('file_determinism', 1)

        # this implies we have a custom solution "parser",
        # preventing the OptSolver _presolve method from
        # creating one
        self._results_format = ResultsFormat.soln
        # use the base class _presolve to consume the
        # important keywords
        OptSolver._presolve(self, *args, **kwds)

        if self._log_file is None:
            self._log_file = pyutilib.services.TempfileManager.\
                             create_tempfile(suffix = '.cplex.log')

        # Possible TODOs
        if self._timelimit is not None:
            logger.warn("The 'timelimit' keyword will be ignored "
                        "for solver="+self.type)
        if self._soln_file is not None:
            logger.warn("The 'soln_file' keyword will be ignored "
                        "for solver="+self.type)

        self.available()

        # like other solver plugins, persistent solver plugins can
        # take an instance as an input argument. the only context in
        # which this instance is used, however, is for warm-starting.
        if len(args) > 2:
            raise ValueError("The CPLEXPersistent plugin method "
                             "'_presolve' can be supplied at most "
                             "one problem instance - %s were "
                             "supplied" % len(args))

        # TBD - not sure about this stuff
        # Clean up the symbol map to only contain variables referenced in the constraints
        # **NOTE**: The warmstart method (if called below), relies on a "clean" symbol map
        vars_to_delete = set(self._variable_label_map.byObject.keys())-self._referenced_variable_ids
        sm_byObject = self._symbol_map.byObject
        sm_bySymbol = self._symbol_map.bySymbol
        assert(len(self._symbol_map.aliases) == 0)
        var_sm_byObject = self._variable_label_map.byObject
        var_sm_bySymbol = self._variable_label_map.bySymbol
        for varid in vars_to_delete:
            symbol = var_sm_byObject[varid]
            del sm_byObject[varid]
            del sm_bySymbol[symbol]
            del var_sm_byObject[varid]
            del var_sm_bySymbol[symbol]

        if 'write' in self.options:
            fname = self.options.write
            self._active_cplex_instance.write(fname)

        # Handle other keywords

        # if the first argument is a string (representing a filename),
        # then we don't have an instance => the solver is being applied
        # to a file.

        # FIXME: This appears to be a bogus test: we raise an exception
        # above if len(args) != 1 or type(args[0]) != Model
        if (len(args) > 0) and not isinstance(args[0], basestring):

            # write the warm-start file - currently only supports MIPs.
            # we only know how to deal with a single problem instance.
            if self._warm_start_solve:

                if len(args) != 1:
                    msg = "CPLEX _presolve method can only handle a single " \
                          "problem instance - %s were supplied"
                    raise ValueError(msg % len(args))

                cplex_instance = self._active_cplex_instance
                if cplex_instance.get_problem_type() in (cplex_instance.problem_type.MILP,
                                                         cplex_instance.problem_type.MIQP,
                                                         cplex_instance.problem_type.MIQCP):
                    start_time = time.time()
                    self._warm_start(args[0])
                    end_time = time.time()
                    if self._report_timing is True:
                        print("Warm start write time=%.2f seconds" % (end_time-start_time))

            # Re-add the symbol map id if it was cleared
            # after a previous solution load
            if id(self._symbol_map) not in args[0].solutions.symbol_map:
                args[0].solutions.add_symbol_map(self._symbol_map)
                self._smap_id = id(self._symbol_map)

    #
    # invoke the solver on the currently compiled instance!!!
    #
    def _apply_solver(self):

        if self._active_cplex_instance is None:
            raise RuntimeError("***The CPLEXPersistent solver plugin cannot "
                               "apply solver - no instance is presently compiled")

        # set up all user-specified parameters.
        if (self.options.mipgap is not None) and (self.options.mipgap > 0.0):
            self._active_cplex_instance.parameters.mip.tolerances.mipgap.set(self.options.mipgap)

        for key in self.options:
            if key == 'relax_integrality' or key == 'mipgap' or key == 'write':
                continue
            else:
                opt_cmd = self._active_cplex_instance.parameters
                key_pieces = key.split('_')
                for key_piece in key_pieces:
                    opt_cmd = getattr(opt_cmd,key_piece)
                opt_cmd.set(self.options[key])

        if 'relax_integrality' in self.options:
            self._active_cplex_instance.set_problem_type(
                self._active_cplex_instance.problem_type.LP)

        if self._tee:
            def _process_stream(arg):
                sys.stdout.write(arg)
                return arg
            self._active_cplex_instance.set_results_stream(
                self._log_file,
                _process_stream)
        else:
            self._active_cplex_instance.set_results_stream(
                self._log_file)

        if self._keepfiles:
            print("Solver log file: "+self._log_file)

        #
        # Kick off the solve.
        #

        # NOTE:
        # CPLEX maintains the pool of feasible solutions from the
        # prior solve as the set of mip starts for the next solve.
        # and evaluating multiple mip starts (and there can be many)
        # is expensive. so if the warm_start method is not invoked,
        # there will potentially be a lot of time wasted.

        # apparently some versions of the CPLEX Python bindings do not
        # have the get_time - so check before accessing.
        if hasattr(self._active_cplex_instance, "get_time"):
            solve_start_time = self._active_cplex_instance.get_time()
            self._active_cplex_instance.solve()
            self._solve_user_time = self._active_cplex_instance.get_time() - solve_start_time
        else:
            self._active_cplex_instance.solve()
            self._solve_user_time = None

        # FIXME: can we get a return code indicating if CPLEX had a
        # significant failure?
        return Bunch(rc=None, log=None)

    def _postsolve(self):

        if self._active_cplex_instance is None:
            raise RuntimeError("***The CPLEXPersistent solver plugin "
                               "cannot postsolve - no instance is "
                               "presently compiled")

        # the only suffixes that we extract from CPLEX are
        # constraint duals, constraint slacks, and variable
        # reduced-costs. scan through the solver suffix list
        # and throw an exception if the user has specified
        # any others.
        extract_duals = False
        extract_slacks = False
        extract_reduced_costs = False
        for suffix in self._suffixes:
            flag=False
            if re.match(suffix,"dual"):
                extract_duals = True
                flag=True
            if re.match(suffix,"slack"):
                extract_slacks = True
                flag=True
            if re.match(suffix,"rc"):
                extract_reduced_costs = True
                flag=True
            if not flag:
                raise RuntimeError("***The CPLEXPersistent solver plugin "
                                   "cannot extract solution suffix="+suffix)

        instance = self._active_cplex_instance

        if instance.get_problem_type() in [instance.problem_type.MILP,
                                           instance.problem_type.MIQP,
                                           instance.problem_type.MIQCP]:
            extract_reduced_costs = False
            extract_duals = False

        results = SolverResults()
        results.problem.name = instance.get_problem_name()
        results.problem.lower_bound = None
        results.problem.upper_bound = None
        results.problem.number_of_variables = instance.variables.get_num()
        results.problem.number_of_constraints = instance.linear_constraints.get_num() \
                                                + instance.quadratic_constraints.get_num() \
                                                + instance.indicator_constraints.get_num() \
                                                + instance.SOS.get_num()
        results.problem.number_of_nonzeros = None
        results.problem.number_of_binary_variables = instance.variables.get_num_binary()
        results.problem.number_of_integer_variables = instance.variables.get_num_integer()
        results.problem.number_of_continuous_variables = instance.variables.get_num() \
                                                            - instance.variables.get_num_binary() \
                                                            - instance.variables.get_num_integer() \
                                                            - instance.variables.get_num_semiinteger()
                                                            #TODO: Does this double-count semi-integers?
                                                            #Should we also remove semi-continuous?
        results.problem.number_of_objectives = 1

        results.solver.name = "CPLEX "+instance.get_version()
#        results.solver.status = None
        results.solver.return_code = None
        results.solver.message = None
        results.solver.user_time = self._solve_user_time
        results.solver.system_time = None
        results.solver.wallclock_time = None
        results.solver.termination_message = None

        soln = Solution()
        soln_variable = soln.variable
        soln_constraint = soln.constraint

        soln.gap = None # until proven otherwise

        #Get solution status -- for now, if CPLEX returns anything we don't recognize, mark as an error
        soln_status = instance.solution.get_status()
        if soln_status in [1, 101, 102]:
            results.solver.termination_condition = TerminationCondition.optimal
            soln.status = SolutionStatus.optimal
        elif soln_status in [2, 4, 118, 119]:
            # Note: soln_status of 4 means infeasible or unbounded
            #       and 119 means MIP infeasible or unbounded
            results.solver.termination_condition = TerminationCondition.unbounded
            soln.status = SolutionStatus.unbounded
        elif soln_status in [3, 103]:
            results.solver.termination_condition = TerminationCondition.infeasible
            soln.status = SolutionStatus.infeasible
        else:
            soln.status = SolutionStatus.error

        # TBD - need to expand the follow significantly, based on problem type and other factors.

        # the definition of relative gap in the case of CPLEX MIP is |best - bestinteger| / ((1e-10)+|bestinteger|).
        # for some reason, the CPLEX Python interface doesn't appear to support extraction of the absolute
        # gap, so we have to compute it.
        m = instance.solution.quality_metric
        if instance.get_problem_type() in [instance.problem_type.MILP,
                                           instance.problem_type.MIQP,
                                           instance.problem_type.MIQCP]:
            try:
                relative_gap = instance.solution.MIP.get_mip_relative_gap()
                best_integer = instance.solution.MIP.get_best_objective()
                diff = relative_gap * (1.0e-10 + math.fabs(best_integer))
                soln.gap = diff
            except CplexSolverError:
                # something went wrong during the solve and no solution
                # exists
                pass

        # Only try to get objective and variable values if a solution exists
        soln_type = instance.solution.get_solution_type()
        if soln_type > 0:

            soln.objective[instance.objective.get_name()] = {"Value": instance.solution.get_objective_value()}

            num_variables = instance.variables.get_num()
            variable_names = self._cplex_variable_names
            variable_values = instance.solution.get_values()
            for i in xrange(num_variables):
                variable_name = variable_names[i]
                soln_variable[variable_name] = {"Value" : variable_values[i]}

            if extract_reduced_costs:
                # get variable reduced costs
                rc_values = instance.solution.get_reduced_costs()
                for i in xrange(num_variables):
                    soln_variable[variable_names[i]]["Rc"] = rc_values[i]

            if extract_slacks or extract_duals:

                num_linear_constraints = instance.linear_constraints.get_num()
                num_quadratic_constraints = instance.quadratic_constraints.get_num()

                constraint_names = instance.linear_constraints.get_names()
                q_constraint_names = instance.quadratic_constraints.get_names()

                for i in xrange(num_linear_constraints):
                    soln_constraint[constraint_names[i]] = {}

            if extract_duals:
                # get duals (linear constraints only)
                dual_values = instance.solution.get_dual_values()
                for i in xrange(num_linear_constraints):
                    soln_constraint[constraint_names[i]]["Dual"] = dual_values[i]

                # CPLEX PYTHON API DOES NOT SUPPORT QUADRATIC DUAL COLLECTION

            if extract_slacks:
                # get linear slacks
                slack_values = instance.solution.get_linear_slacks()
                for i in xrange(num_linear_constraints):
                    # if both U and L exist (i.e., a range constraint) then
                    # R_ = U-L
                    R_ = instance.linear_constraints.get_range_values(i)
                    if R_ == 0.0:
                        soln_constraint[constraint_names[i]]["Slack"] = slack_values[i]
                    else:
                        # This is a range constraint for which cplex always returns the
                        # value of f(x)-L. In the spirit of conforming with the other writer,
                        # I will return the max (in absolute value) of L-f(x) and U-f(x)
                        Ls_ = slack_values[i]
                        Us_ = R_ - slack_values[i]
                        if Us_ > Ls_:
                            soln_constraint[constraint_names[i]]["Slack"] = Us_
                        else:
                            soln_constraint[constraint_names[i]]["Slack"] = -Ls_

                # get quadratic slacks
                slack_values = instance.solution.get_quadratic_slacks()
                for i in xrange(num_quadratic_constraints):
                    # if both U and L exist (i.e., a range constraint) then
                    # R_ = U-L
                    soln_constraint[q_constraint_names[i]] = {"Slack" : slack_values[i]}

            byObject = self._symbol_map.byObject
            referenced_varnames = set(byObject[varid] for varid in self._referenced_variable_ids)
            names_to_delete = set(soln_variable.keys())-referenced_varnames
            for varname in names_to_delete:
                del soln_variable[varname]

            results.solution.insert(soln)

        self.results = results

        # don't know if any of this is necessary!

        # take care of the annoying (and empty) CPLEX temporary files in
        # the current directory.  this approach doesn't seem overly
        # efficient, but python os module functions don't accept regular
        # expression directly.
        filename_list = os.listdir(".")
        clone_re = re.compile('clone\d+\.log')
        for filename in filename_list:
            # CPLEX temporary files come in two flavors - cplex.log and
            # clone*.log.  the latter is the case for multi-processor
            # environments.
            #
            # IMPT: trap the possible exception raised by the file not existing.
            #       this can occur in pyro environments where > 1 workers are
            #       running CPLEX, and were started from the same directory.
            #       these logs don't matter anyway (we redirect everything),
            #       and are largely an annoyance.
            try:
                if filename == 'cplex.log':
                    os.remove(filename)
                elif clone_re.match(filename):
                    os.remove(filename)
            except OSError:
                pass

        # finally, clean any temporary files registered with the temp file
        # manager, created populated *directly* by this plugin.
        pyutilib.services.TempfileManager.pop(remove=not self._keepfiles)

        # let the base class deal with returning results.
        return OptSolver._postsolve(self)

    def _initialize_callbacks(self, model):
        #
        # Called from OptSolver
        #
        cplex_callback = {
            "node-callback":        cplex.callbacks.NodeCallback,
            "solve-callback":       cplex.callbacks.SolveCallback,
            "branch-callback":      cplex.callbacks.BranchCallback,
            "heuristic-callback":   cplex.callbacks.HeuristicCallback,
            "incumbent-callback":   cplex.callbacks.IncumbentCallback,
            "cut-callback":         cplex.callbacks.UserCutCallback,
            "lazycut-callback":     cplex.callbacks.LazyConstraintCallback,
            "crossover-callback":   cplex.callbacks.CrossoverCallback,
            "barrier-callback":     cplex.callbacks.BarrierCallback,
            "simplex-callback":     cplex.callbacks.SimplexCallback,
            "presolve-callback":    cplex.callbacks.PresolveCallback,
            "tuning-callback":      cplex.callbacks.TuningCallback
            }
        #
        for name in self._callback:
            try:
                cb_class = cplex_callback[name]
            except KeyError:
                raise ValueError("Unknown callback name: %s" % name)
            #
            def call_fn(self, *args, **kwds):
                try:
                    self.solver = CplexSolverWrapper(self)
                    self._callback[self.name](self.solver, model)
                except Exception(e):
                    # Should we raise this exception?
                    print("ERROR: "+str(e))
            CallbackClass = type('CallbackClass_'+name.replace('-','_'), (cb_class,object), {"_callback":self._callback, "name":name, "__call__":call_fn})
            self._active_cplex_instance.register_callback(CallbackClass)


if cplex_import_available is False:
    SolverFactory().deactivate('_cplex_persistent')
    SolverFactory().deactivate('_mock_cplexpersistent')
