internal/lib/base_cloud_client.py

#!/usr/bin/env python
#
# Copyright 2016 - The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Base Cloud API Client.

BasicCloudApiCliend does basic setup for a cloud API.
"""
import httplib
import logging
import socket
import ssl

# pylint: disable=import-error
from apiclient import errors as gerrors
from apiclient.discovery import build
import apiclient.http
import httplib2
from oauth2client import client

from acloud import errors
from acloud.internal.lib import utils

logger = logging.getLogger(__name__)


class BaseCloudApiClient(object):
    """A class that does basic setup for a cloud API."""

    # To be overriden by subclasses.
    API_NAME = ""
    API_VERSION = "v1"
    SCOPE = ""

    # Defaults for retry.
    RETRY_COUNT = 5
    RETRY_BACKOFF_FACTOR = 1.5
    RETRY_SLEEP_MULTIPLIER = 2
    RETRY_HTTP_CODES = [
        # 403 is to retry the "Rate Limit Exceeded" error.
        # We could retry on a finer-grained error message later if necessary.
        403,
        500,  # Internal Server Error
        502,  # Bad Gateway
        503,  # Service Unavailable
    ]
    RETRIABLE_ERRORS = (httplib.HTTPException, httplib2.HttpLib2Error,
                        socket.error, ssl.SSLError)
    RETRIABLE_AUTH_ERRORS = (client.AccessTokenRefreshError, )

    def __init__(self, oauth2_credentials):
        """Initialize.

        Args:
            oauth2_credentials: An oauth2client.OAuth2Credentials instance.
        """
        self._service = self.InitResourceHandle(oauth2_credentials)

    @classmethod
    def InitResourceHandle(cls, oauth2_credentials):
        """Authenticate and initialize a Resource object.

        Authenticate http and create a Resource object with methods
        for interacting with the service.

        Args:
            oauth2_credentials: An oauth2client.OAuth2Credentials instance.

        Returns:
            An apiclient.discovery.Resource object
        """
        http_auth = oauth2_credentials.authorize(httplib2.Http())
        return utils.RetryExceptionType(
            exception_types=cls.RETRIABLE_AUTH_ERRORS,
            max_retries=cls.RETRY_COUNT,
            functor=build,
            sleep_multiplier=cls.RETRY_SLEEP_MULTIPLIER,
            retry_backoff_factor=cls.RETRY_BACKOFF_FACTOR,
            serviceName=cls.API_NAME,
            version=cls.API_VERSION,
            # This is workaround for a known issue of some veriosn
            # of api client.
            # https://github.com/google/google-api-python-client/issues/435
            cache_discovery=False,
            http=http_auth)

    @staticmethod
    def _ShouldRetry(exception, retry_http_codes,
                     other_retriable_errors):
        """Check if exception is retriable.

        Args:
            exception: An instance of Exception.
            retry_http_codes: a list of integers, retriable HTTP codes of
                              HttpError
            other_retriable_errors: a tuple of error types to retry other than
                                    HttpError.

        Returns:
            Boolean, True if retriable, False otherwise.
        """
        if isinstance(exception, other_retriable_errors):
            return True

        if isinstance(exception, errors.HttpError):
            if exception.code in retry_http_codes:
                return True
            else:
                logger.debug("_ShouldRetry: Exception code %s not in %s: %s",
                             exception.code, retry_http_codes, str(exception))

        logger.debug("_ShouldRetry: Exception %s is not one of %s: %s",
                     type(exception),
                     list(other_retriable_errors) + [errors.HttpError],
                     str(exception))
        return False

    @staticmethod
    def _TranslateError(exception):
        """Translate the exception to a desired type.

        Args:
            exception: An instance of Exception.

        Returns:
            gerrors.HttpError will be translated to errors.HttpError.
            If the error code is errors.HTTP_NOT_FOUND_CODE, it will
            be translated to errors.ResourceNotFoundError.
            Unrecognized error type will not be translated and will
            be returned as is.
        """
        if isinstance(exception, gerrors.HttpError):
            exception = errors.HttpError.CreateFromHttpError(exception)
            if exception.code == errors.HTTP_NOT_FOUND_CODE:
                exception = errors.ResourceNotFoundError(
                    exception.code, str(exception))
        return exception

    def ExecuteOnce(self, api):
        """Execute an api and parse the errors.

        Args:
            api: An apiclient.http.HttpRequest, representing the api to execute.

        Returns:
            Execution result of the api.

        Raises:
            errors.ResourceNotFoundError: For 404 error.
            errors.HttpError: For other types of http error.
        """
        try:
            return api.execute()
        except gerrors.HttpError as e:
            raise self._TranslateError(e)

    def Execute(self,
                api,
                retry_http_codes=None,
                max_retry=None,
                sleep=None,
                backoff_factor=None,
                other_retriable_errors=None):
        """Execute an api with retry.

        Call ExecuteOnce and retry on http error with given codes.

        Args:
            api: An apiclient.http.HttpRequest, representing the api to execute:
            retry_http_codes: A list of http codes to retry.
            max_retry: See utils.Retry.
            sleep: See utils.Retry.
            backoff_factor: See utils.Retry.
            other_retriable_errors: A tuple of error types that should be retried
                                    other than errors.HttpError.

        Returns:
          Execution result of the api.

        Raises:
          See ExecuteOnce.
        """
        retry_http_codes = (self.RETRY_HTTP_CODES
                            if retry_http_codes is None else retry_http_codes)
        max_retry = (self.RETRY_COUNT if max_retry is None else max_retry)
        sleep = (self.RETRY_SLEEP_MULTIPLIER if sleep is None else sleep)
        backoff_factor = (self.RETRY_BACKOFF_FACTOR
                          if backoff_factor is None else backoff_factor)
        other_retriable_errors = (self.RETRIABLE_ERRORS
                                  if other_retriable_errors is None else
                                  other_retriable_errors)

        def _Handler(exc):
            """Check if |exc| is a retriable exception.

            Args:
                exc: An exception.

            Returns:
                True if exc is an errors.HttpError and code exists in |retry_http_codes|
                False otherwise.
            """
            if self._ShouldRetry(exc, retry_http_codes,
                                 other_retriable_errors):
                logger.debug("Will retry error: %s", str(exc))
                return True
            return False

        return utils.Retry(
            _Handler,
            max_retries=max_retry,
            functor=self.ExecuteOnce,
            sleep_multiplier=sleep,
            retry_backoff_factor=backoff_factor,
            api=api)

    def BatchExecuteOnce(self, requests):
        """Execute requests in a batch.

        Args:
            requests: A dictionary where key is request id and value
                      is an http request.

        Returns:
            results, a dictionary in the following format
            {request_id: (response, exception)}
            request_ids are those from requests; response
            is the http response for the request or None on error;
            exception is an instance of DriverError or None if no error.
        """
        results = {}

        def _CallBack(request_id, response, exception):
            results[request_id] = (response, self._TranslateError(exception))

        batch = apiclient.http.BatchHttpRequest()
        for request_id, request in requests.iteritems():
            batch.add(
                request=request, callback=_CallBack, request_id=request_id)
        batch.execute()
        return results

    def BatchExecute(self,
                     requests,
                     retry_http_codes=None,
                     max_retry=None,
                     sleep=None,
                     backoff_factor=None,
                     other_retriable_errors=None):
        """Batch execute multiple requests with retry.

        Call BatchExecuteOnce and retry on http error with given codes.

        Args:
            requests: A dictionary where key is request id picked by caller,
                      and value is a apiclient.http.HttpRequest.
            retry_http_codes: A list of http codes to retry.
            max_retry: See utils.Retry.
            sleep: See utils.Retry.
            backoff_factor: See utils.Retry.
            other_retriable_errors: A tuple of error types that should be retried
                                    other than errors.HttpError.

        Returns:
            results, a dictionary in the following format
            {request_id: (response, exception)}
            request_ids are those from requests; response
            is the http response for the request or None on error;
            exception is an instance of DriverError or None if no error.
        """
        executor = utils.BatchHttpRequestExecutor(
            self.BatchExecuteOnce,
            requests=requests,
            retry_http_codes=retry_http_codes or self.RETRY_HTTP_CODES,
            max_retry=max_retry or self.RETRY_COUNT,
            sleep=sleep or self.RETRY_SLEEP_MULTIPLIER,
            backoff_factor=backoff_factor or self.RETRY_BACKOFF_FACTOR,
            other_retriable_errors=other_retriable_errors
            or self.RETRIABLE_ERRORS)
        executor.Execute()
        return executor.GetResults()

    def ListWithMultiPages(self, api_resource, *args, **kwargs):
        """Call an api that list a type of resource.

        Multiple google services support listing a type of
        resource (e.g list gce instances, list storage objects).
        The querying pattern is similar --
        Step 1: execute the api and get a response object like,
        {
            "items": [..list of resource..],
            # The continuation token that can be used
            # to get the next page.
            "nextPageToken": "A String",
        }
        Step 2: execute the api again with the nextPageToken to
        retrieve more pages and get a response object.

        Step 3: Repeat Step 2 until no more page.

        This method encapsulates the generic logic of
        calling such listing api.

        Args:
            api_resource: An apiclient.discovery.Resource object
                used to create an http request for the listing api.
            *args: Arguments used to create the http request.
            **kwargs: Keyword based arguments to create the http
                      request.

        Returns:
            A list of items.
        """
        items = []
        next_page_token = None
        while True:
            api = api_resource(pageToken=next_page_token, *args, **kwargs)
            response = self.Execute(api)
            items.extend(response.get("items", []))
            next_page_token = response.get("nextPageToken")
            if not next_page_token:
                break
        return items

    @property
    def service(self):
        """Return self._service as a property."""
        return self._service