# -*- coding: utf-8 -*-
# Copyright 2018 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Shared utility structures and methods that require importing boto.

This module also imports httplib2 (as it is Boto's http transport and closely
tied to some of Boto's core functionality) and oauth2client.
"""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals

import os
import pkgutil
import tempfile
import textwrap

import six
import boto
from boto import config
import boto.auth
from boto.exception import NoAuthHandlerFound
from boto.gs.connection import GSConnection
from boto.provider import Provider
from boto.pyami.config import BotoConfigLocations

import gslib
from gslib.exception import CommandException
from gslib.utils import system_util
from gslib.utils.constants import DEFAULT_GCS_JSON_API_VERSION
from gslib.utils.constants import DEFAULT_GSUTIL_STATE_DIR
from gslib.utils.constants import SSL_TIMEOUT_SEC
from gslib.utils.constants import UTF8
from gslib.utils.unit_util import HumanReadableToBytes
from gslib.utils.unit_util import ONE_MIB

import httplib2
from oauth2client.client import HAS_CRYPTO

if six.PY3:
  long = int

# Globals in this module are set according to values in the boto config.
BOTO_IS_SECURE = config.get('Boto', 'is_secure', True)
CERTIFICATE_VALIDATION_ENABLED = config.get('Boto',
                                            'https_validate_certificates', True)

configured_certs_file = None  # Single certs file for use across all processes.
temp_certs_file = None  # Temporary certs file for cleanup upon exit.


def ConfigureCertsFile():
  """Configures and returns the CA Certificates file.

  If one is already configured, use it. Otherwise, use the cert roots
  distributed with gsutil.

  Returns:
    string filename of the certs file to use.
  """
  certs_file = boto.config.get('Boto', 'ca_certificates_file', None)
  # The 'system' keyword indicates to use the system installed certs. Some
  # Linux distributions patch the stack such that the Python SSL
  # infrastructure picks up the system installed certs by default, thus no
  #  action necessary on our part
  if certs_file == 'system':
    return None
  if not certs_file:
    global configured_certs_file, temp_certs_file
    if not configured_certs_file:
      configured_certs_file = os.path.abspath(
          os.path.join(gslib.GSLIB_DIR, 'data', 'cacerts.txt'))
      if not os.path.exists(configured_certs_file):
        # If the file is not present on disk, this means the gslib module
        # doesn't actually exist on disk anywhere. This can happen if it's
        # being imported from a zip file. Unfortunately, we have to copy the
        # certs file to a local temp file on disk because the underlying SSL
        # socket requires it to be a filesystem path.
        certs_data = pkgutil.get_data('gslib', 'data/cacerts.txt')
        if not certs_data:
          raise CommandException('Certificates file not found. Please '
                                 'reinstall gsutil from scratch')
        certs_data = six.ensure_str(certs_data)
        fd, fname = tempfile.mkstemp(suffix='.txt', prefix='gsutil-cacerts')
        f = os.fdopen(fd, 'w')
        f.write(certs_data)
        f.close()
        temp_certs_file = fname
        configured_certs_file = temp_certs_file
    certs_file = configured_certs_file
  return certs_file


def ConfigureNoOpAuthIfNeeded():
  """Sets up no-op auth handler if no boto credentials are configured."""
  if not HasConfiguredCredentials():
    if (config.has_option('Credentials', 'gs_service_client_id') and
        not HAS_CRYPTO):
      if system_util.InvokedViaCloudSdk():
        raise CommandException('\n'.join(
            textwrap.wrap(
                'Your gsutil is configured with an OAuth2 service account, but '
                'you do not have PyOpenSSL or PyCrypto 2.6 or later installed. '
                'Service account authentication requires one of these libraries; '
                'please reactivate your service account via the gcloud auth '
                'command and ensure any gcloud packages necessary for '
                'service accounts are present.')))
      else:
        raise CommandException('\n'.join(
            textwrap.wrap(
                'Your gsutil is configured with an OAuth2 service account, but '
                'you do not have PyOpenSSL or PyCrypto 2.6 or later installed. '
                'Service account authentication requires one of these libraries; '
                'please install either of them to proceed, or configure a '
                'different type of credentials with "gsutil config".')))
    else:
      # With no boto config file the user can still access publicly readable
      # buckets and objects.
      from gslib import no_op_auth_plugin  # pylint: disable=unused-variable


def GetCertsFile():
  return configured_certs_file


def GetCleanupFiles():
  """Returns a list of temp files to delete (if possible) when program exits."""
  return [temp_certs_file] if temp_certs_file else []


def GetConfigFilePaths():
  """Returns a list of the path(s) to the boto config file(s) to be loaded."""
  potential_config_paths = BotoConfigLocations

  # When Boto's pyami.config.Config class is initialized, it attempts to read
  # this file, transform it into valid Boto config syntax, and load credentials
  # from it, but it does not add this file to BotoConfigLocations.
  if 'AWS_CREDENTIAL_FILE' in os.environ:
    potential_config_paths.append(os.environ['AWS_CREDENTIAL_FILE'])
  # Provider-specific config files, like the one below, are checked for and
  # loaded when Boto's Provider class is initialized. These aren't listed in
  # BotoConfigLocations, but can still affect what credentials are loaded, so
  # we display them in our config list to make auth'n debugging easier.
  aws_cred_file = os.path.join(os.path.expanduser('~'), '.aws', 'credentials')
  if os.path.isfile(aws_cred_file):
    potential_config_paths.append(aws_cred_file)

  # Only return credential files which we have permission to read (and thus can
  # actually load credentials from).
  readable_config_paths = []
  for path in potential_config_paths:
    try:
      with open(path, 'r'):
        readable_config_paths.append(path)
    except IOError:
      pass
  return readable_config_paths


def GetFriendlyConfigFilePaths():
  """Like GetConfigFilePaths but returns a not-found message if paths empty."""
  readable_config_paths = GetConfigFilePaths()
  if len(readable_config_paths) == 0:
    readable_config_paths.append('No config found')
  return readable_config_paths


def GetGsutilStateDir():
  """Returns the location of the directory for gsutil state files.

  Certain operations, such as cross-process credential sharing and
  resumable transfer tracking, need a known location for state files which
  are created by gsutil as-needed.

  This location should only be used for storing data that is required to be in
  a static location.

  Returns:
    Path to directory for gsutil static state files.
  """
  config_file_dir = config.get('GSUtil', 'state_dir', DEFAULT_GSUTIL_STATE_DIR)
  system_util.CreateDirIfNeeded(config_file_dir)
  return config_file_dir


def GetCredentialStoreFilename():
  # As of gsutil 4.29, this changed from 'credstore' to 'credstore2' because
  # of a change to the underlying credential storage format.
  return os.path.join(GetGsutilStateDir(), 'credstore2')


def GetGceCredentialCacheFilename():
  return os.path.join(GetGsutilStateDir(), 'gcecredcache')


def GetGcsJsonApiVersion():
  return config.get('GSUtil', 'json_api_version', DEFAULT_GCS_JSON_API_VERSION)


# Resumable downloads and uploads make one HTTP call per chunk (and must be
# in multiples of 256KiB). Overridable for testing.
def GetJsonResumableChunkSize():
  chunk_size = config.getint('GSUtil', 'json_resumable_chunk_size',
                             long(1024 * 1024 * 100))
  if chunk_size == 0:
    chunk_size = long(1024 * 256)
  elif chunk_size % long(1024 * 256) != 0:
    chunk_size += (long(1024 * 256) - (chunk_size % (long(1024 * 256))))
  return chunk_size


def GetLastCheckedForGsutilUpdateTimestampFile():
  return os.path.join(GetGsutilStateDir(), '.last_software_update_check')


def GetMaxConcurrentCompressedUploads():
  """Gets the max concurrent transport compressed uploads allowed in parallel.

  Returns:
    The max number of concurrent transport compressed uploads allowed in
    parallel without exceeding the max_upload_compression_buffer_size.
  """
  upload_chunk_size = GetJsonResumableChunkSize()
  # From apitools compression.py.
  compression_chunk_size = 16 * ONE_MIB
  total_upload_size = (upload_chunk_size + compression_chunk_size + 17 + 5 *
                       (((compression_chunk_size - 1) / 16383) + 1))
  max_concurrent_uploads = (GetMaxUploadCompressionBufferSize() /
                            total_upload_size)
  if max_concurrent_uploads <= 0:
    max_concurrent_uploads = 1
  return max_concurrent_uploads


def GetMaxRetryDelay():
  return config.getint('Boto', 'max_retry_delay', 32)


def GetMaxUploadCompressionBufferSize():
  """Get the max amount of memory compressed transport uploads may buffer."""
  return HumanReadableToBytes(
      config.get('GSUtil', 'max_upload_compression_buffer_size', '2GiB'))


def GetNewHttp(http_class=httplib2.Http, **kwargs):
  """Creates and returns a new httplib2.Http instance.

  Args:
    http_class: Optional custom Http class to use.
    **kwargs: Arguments to pass to http_class constructor.

  Returns:
    An initialized httplib2.Http instance.
  """

  ##Get Proxy configuration from boto file, defaults are None, 0 and False
  proxy_host = config.get('Boto', 'proxy', None)  #needed for if clause below
  boto_proxy_config = {
      'proxy_host':
          proxy_host,
      'proxy_type':
          config.get('Boto', 'proxy_type', 'http'),
      'proxy_port':
          config.getint('Boto', 'proxy_port'),
      'proxy_user':
          config.get('Boto', 'proxy_user', None),
      'proxy_pass':
          config.get('Boto', 'proxy_pass', None),
      'proxy_rdns':
          config.get('Boto', 'proxy_rdns', True if proxy_host else None)
  }

  #Use SetProxyInfo to convert boto config to httplib2.proxyinfo object
  proxy_info = SetProxyInfo(boto_proxy_config)

  # Some installers don't package a certs file with httplib2, so use the
  # one included with gsutil.
  kwargs['ca_certs'] = GetCertsFile()
  # Use a non-infinite SSL timeout to avoid hangs during network flakiness.
  kwargs['timeout'] = SSL_TIMEOUT_SEC
  http = http_class(proxy_info=proxy_info, **kwargs)
  http.disable_ssl_certificate_validation = (not config.getbool(
      'Boto', 'https_validate_certificates'))
  return http


# Retry for 10 minutes with exponential backoff, which corresponds to
# the maximum Downtime Period specified in the GCS SLA
# (https://cloud.google.com/storage/sla)
def GetNumRetries():
  return config.getint('Boto', 'num_retries', 23)


def GetTabCompletionLogFilename():
  return os.path.join(GetGsutilStateDir(), 'tab-completion-logs')


def GetTabCompletionCacheFilename():
  tab_completion_dir = os.path.join(GetGsutilStateDir(), 'tab-completion')
  # Limit read permissions on the directory to owner for privacy.
  system_util.CreateDirIfNeeded(tab_completion_dir, mode=0o700)
  return os.path.join(tab_completion_dir, 'cache')


def HasConfiguredCredentials():
  """Determines if boto credential/config file exists."""
  has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and
                    config.has_option('Credentials', 'gs_secret_access_key'))
  has_amzn_creds = (config.has_option('Credentials', 'aws_access_key_id') and
                    config.has_option('Credentials', 'aws_secret_access_key'))
  has_oauth_creds = (config.has_option('Credentials',
                                       'gs_oauth2_refresh_token'))
  has_service_account_creds = (
      HAS_CRYPTO and
      config.has_option('Credentials', 'gs_service_client_id') and
      config.has_option('Credentials', 'gs_service_key_file'))

  if (has_goog_creds or has_amzn_creds or has_oauth_creds or
      has_service_account_creds):
    return True

  valid_auth_handler = None
  try:
    valid_auth_handler = boto.auth.get_auth_handler(GSConnection.DefaultHost,
                                                    config,
                                                    Provider('google'),
                                                    requested_capability=['s3'])
    # Exclude the no-op auth handler as indicating credentials are configured.
    # Note we can't use isinstance() here because the no-op module may not be
    # imported so we can't get a reference to the class type.
    if 'NoOpAuth' == getattr(
        getattr(valid_auth_handler, '__class__', None),
        '__name__',
        None):  # yapf: disable
      valid_auth_handler = None
  except NoAuthHandlerFound:
    pass

  return valid_auth_handler


def JsonResumableChunkSizeDefined():
  chunk_size_defined = config.get('GSUtil', 'json_resumable_chunk_size', None)
  return chunk_size_defined is not None


def MonkeyPatchBoto():
  """Apply gsutil-specific patches to Boto.

  Here be dragons. Sorry.

  Note that this method should not be used as a replacement for contributing
  fixes to the upstream Boto library. However, the Boto library has historically
  not been consistent about release cadence, so upstream fixes may not be
  available immediately in a version which we can pin to. Also, some fixes may
  only be applicable to gsutil. In such cases, patches should be applied to the
  Boto library here (and removed if/when they are included in the upstream
  repository and included in an official new release that we pull in). This
  method should be invoked after all other Boto-related initialization has been
  completed.
  """
  # We have to do all sorts of gross things here (dynamic imports, invalid names
  # to resolve symbols in copy/pasted methods, invalid spacing from copy/pasted
  # methods, etc.), so we just disable pylint warnings for this whole method.
  # yapf: disable
  # pylint: disable=all

  # This should have already been imported if this method was called in the
  # correct place, but we need to perform the import within this method for
  # this module to recognize it.  We don't import this at module level because
  # its import timing is important and is managed elsewhere.
  import gcs_oauth2_boto_plugin

  # TODO(https://github.com/boto/boto/issues/3831): Remove this if the GitHub
  # issue is ever addressed.
  orig_get_plugin_method = boto.plugin.get_plugin

  def _PatchedGetPluginMethod(cls, requested_capability=None):
    handler_subclasses = orig_get_plugin_method(
        cls, requested_capability=requested_capability)
    # In Boto's logic, higher precedence handlers should be closer to the end
    # of the list.  We always want to prefer OAuth2 credentials over HMAC
    # credentials if both are present, so we shuffle OAuth2 handlers to the end
    # of the handler list.
    xml_oauth2_handlers = (
        gcs_oauth2_boto_plugin.oauth2_plugin.OAuth2ServiceAccountAuth,
        gcs_oauth2_boto_plugin.oauth2_plugin.OAuth2Auth)
    new_result = (
        # We need to sort each of these to avoid inconsistent handler selection
        # when multiple credential types are configured in Python 3.5. See
        # https://issuetracker.google.com/issues/135709541 for specific logs.
        sorted(
            [r for r in handler_subclasses if r not in xml_oauth2_handlers],
            # Types aren't sortable, so we use their names:
            key=(lambda handler_t: handler_t.__name__),
        ) +  # Now append XML handlers to the end (highest precedence)
        sorted(
            [r for r in handler_subclasses if r in xml_oauth2_handlers],
            key=(lambda handler_t: handler_t.__name__),
        ))
    return new_result

  boto.plugin.get_plugin = _PatchedGetPluginMethod

  #########################################################################

  # TODO(boto>2.49.0): Remove this.
  # Fixes SSL issue where SNI was not being used for OpenSSL 1.1.1+
  # https://github.com/boto/boto/pull/3843/files

  # Import modules and resolve symbols used by our patch method.
  import socket
  import ssl
  InvalidCertificateException = (
      boto.https_connection.InvalidCertificateException)
  ValidateCertificateHostname = (
      boto.https_connection.ValidateCertificateHostname)

  def _PatchedConnectMethod(self):
    # The lines below were copied directly from the Boto file, so we don't lint
    # or otherwise alter them.
    if hasattr(self, "timeout"):
      sock = socket.create_connection((self.host, self.port), self.timeout)
    else:
      sock = socket.create_connection((self.host, self.port))
    msg = "wrapping ssl socket; "
    if self.ca_certs:
      msg += "CA certificate file=%s" % self.ca_certs
    else:
      msg += "using system provided SSL certs"
    boto.log.debug(msg)
    if hasattr(ssl, 'SSLContext') and getattr(ssl, 'HAS_SNI', False):
      # Use SSLContext so we can specify server_hostname for SNI
      # (Required for connections to storage.googleapis.com)
      context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
      context.verify_mode = ssl.CERT_REQUIRED
      if self.ca_certs:
        context.load_verify_locations(self.ca_certs)
      if self.cert_file:
        context.load_cert_chain(self.cert_file, self.key_file)
      self.sock = context.wrap_socket(sock, server_hostname=self.host)
      # Add attributes only set in SSLSocket constructor without context:
      self.sock.keyfile = self.key_file
      self.sock.certfile = self.cert_file
      self.sock.cert_reqs = context.verify_mode
      self.sock.ssl_version = ssl.PROTOCOL_SSLv23
      self.sock.ca_certs = self.ca_certs
      self.sock.ciphers = None
    else:
      self.sock = ssl.wrap_socket(sock,
                                  keyfile=self.key_file,
                                  certfile=self.cert_file,
                                  cert_reqs=ssl.CERT_REQUIRED,
                                  ca_certs=self.ca_certs)
    cert = self.sock.getpeercert()
    hostname = self.host.split(':', 0)[0]
    if not ValidateCertificateHostname(cert, hostname):
      raise InvalidCertificateException(
          hostname, cert, 'remote hostname "%s" does not match '
          'certificate' % hostname)
    # End `_PatchedConnectMethod` declaration.

  boto.https_connection.CertValidatingHTTPSConnection.connect = (
      _PatchedConnectMethod)


def ProxyInfoFromEnvironmentVar(proxy_env_var):
  """Reads proxy info from the environment and converts to httplib2.ProxyInfo.

  Args:
    proxy_env_var: Environment variable string to read, such as http_proxy or
       https_proxy.

  Returns:
    httplib2.ProxyInfo constructed from the environment string.
  """
  proxy_url = os.environ.get(proxy_env_var)
  if not proxy_url or not proxy_env_var.lower().startswith('http'):
    return httplib2.ProxyInfo(httplib2.socks.PROXY_TYPE_HTTP, None, 0)
  proxy_protocol = proxy_env_var.lower().split('_')[0]
  if not proxy_url.lower().startswith('http'):
    # proxy_info_from_url requires a protocol, which is always http or https.
    proxy_url = proxy_protocol + '://' + proxy_url
  return httplib2.proxy_info_from_url(proxy_url, method=proxy_protocol)


def ResumableThreshold():
  return config.getint('GSUtil', 'resumable_threshold', 8 * ONE_MIB)


def SetProxyInfo(boto_proxy_config):
  """Sets proxy info from boto and environment and converts to httplib2.ProxyInfo.

  Args:
    dict: Values read from the .boto file

  Returns:
    httplib2.ProxyInfo constructed from boto or environment variable string.
  """
  #Defining proxy_type based on httplib2 library, accounting for None entry too.
  proxy_type_spec = {'socks4': 1, 'socks5': 2, 'http': 3, 'https': 3}

  #proxy_type defaults to 'http (3)' for backwards compatibility
  proxy_type = proxy_type_spec.get(
      boto_proxy_config.get('proxy_type').lower(), proxy_type_spec['http'])
  proxy_host = boto_proxy_config.get('proxy_host')
  proxy_port = boto_proxy_config.get('proxy_port')
  proxy_user = boto_proxy_config.get('proxy_user')
  proxy_pass = boto_proxy_config.get('proxy_pass')
  proxy_rdns = bool(boto_proxy_config.get('proxy_rdns'))

  #For proxy_info below, proxy_rdns fails for socks4 and socks5 so restricting use
  #to http only
  proxy_info = httplib2.ProxyInfo(proxy_host=proxy_host,
                                  proxy_type=proxy_type,
                                  proxy_port=proxy_port,
                                  proxy_user=proxy_user,
                                  proxy_pass=proxy_pass,
                                  proxy_rdns=proxy_rdns)

  #Added to force socks proxies not to use rdns
  if not (proxy_info.proxy_type == proxy_type_spec['http']):
    proxy_info.proxy_rdns = False

  if not (proxy_info.proxy_host and proxy_info.proxy_port):
    # Fall back to using the environment variable. Use only http proxies.
    for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']:
      if proxy_env_var in os.environ and os.environ[proxy_env_var]:
        proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var)
        # Assume proxy_rnds is True if a proxy environment variable exists.
        if boto_proxy_config.get('proxy_rdns') == None:
          proxy_info.proxy_rdns = True
        break

  return proxy_info


def UsingCrcmodExtension():
  boto_opt = boto.config.get('GSUtil', 'test_assume_fast_crcmod', None)
  if boto_opt is not None:
    return boto_opt
  # Python 3 makes this attribute tough to access due to the way the top-level
  # crcmod package imports and (identically) names its crcmod module. The only
  # way to get it is "from crcmod.crcmod import _usingExtension". This is the
  # alternative form of that statement, but doesn't pollute this module's
  # namespace with a "_usingExtension" attribute. This also works in both Python
  # 2.7 and 3.5+.
  nested_crcmod = __import__(
      'crcmod.crcmod',
      globals(),
      locals(),
      ['_usingExtension'],
      0,
  )
  return getattr(nested_crcmod, '_usingExtension', False)


# TODO(boto-2.49.0): Remove when we pull in the next version of Boto.
def _PatchedShouldRetryMethod(self, response, chunked_transfer=False):
  """Replaces boto.s3.key's should_retry() to handle KMS-encrypted objects."""
  # We copy/pasted this from boto and slightly modified it; keep the old
  # formatting style.
  # yapf: disable
  provider = self.bucket.connection.provider

  if not chunked_transfer:
    if response.status in [500, 503]:
      # 500 & 503 can be plain retries.
      return True

    if response.getheader('location'):
      # If there's a redirect, plain retry.
      return True

  if 200 <= response.status <= 299:
    self.etag = response.getheader('etag')
    md5 = self.md5
    if isinstance(md5, bytes):
      md5 = md5.decode(UTF8)

    # If you use customer-provided encryption keys, the ETag value that
    # Amazon S3 returns in the response will not be the MD5 of the
    # object.
    amz_server_side_encryption_customer_algorithm = response.getheader(
        'x-amz-server-side-encryption-customer-algorithm', None)
    # The same is applicable for KMS-encrypted objects in gs buckets.
    goog_customer_managed_encryption = response.getheader(
        'x-goog-encryption-kms-key-name', None)
    if (amz_server_side_encryption_customer_algorithm is None and
            goog_customer_managed_encryption is None):
      if self.etag != '"%s"' % md5:
        raise provider.storage_data_error(
            'ETag from S3 did not match computed MD5. '
            '%s vs. %s' % (self.etag, self.md5))

    return True

  if response.status == 400:
    # The 400 must be trapped so the retry handler can check to
    # see if it was a timeout.
    # If ``RequestTimeout`` is present, we'll retry. Otherwise, bomb
    # out.
    body = response.read()
    err = provider.storage_response_error(
        response.status,
        response.reason,
        body
    )

    if err.error_code in ['RequestTimeout']:
      raise boto.exception.PleaseRetryException(
          "Saw %s, retrying" % err.error_code,
          response=response
      )

  return False
