internal/lib/base_cloud_client.py

#!/usr/bin/env python
#
# Copyright 2016 - The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Base Cloud API Client.

BasicCloudApiCliend does basic setup for a cloud API.
"""
import httplib
import logging
import os
import socket
import ssl

import google3

from apiclient import errors as gerrors
from apiclient.discovery import build
import apiclient.http
import httplib2
from oauth2client import client

from acloud.internal.lib import utils
from acloud.public import errors

logger = logging.getLogger(__name__)


class BaseCloudApiClient(object):
    """A class that does basic setup for a cloud API."""

    # To be overriden by subclasses.
    API_NAME = ""
    API_VERSION = "v1"
    SCOPE = ""

    # Defaults for retry.
    RETRY_COUNT = 5
    RETRY_BACKOFF_FACTOR = 1.5
    RETRY_SLEEP_MULTIPLIER = 2
    RETRY_HTTP_CODES = [
        # 403 is to retry the "Rate Limit Exceeded" error.
        # We could retry on a finer-grained error message later if necessary.
        403,
        500,  # Internal Server Error
        502,  # Bad Gateway
        503,  # Service Unavailable
    ]
    RETRIABLE_ERRORS = (httplib.HTTPException, httplib2.HttpLib2Error,
                        socket.error, ssl.SSLError)
    RETRIABLE_AUTH_ERRORS = (client.AccessTokenRefreshError, )

    def __init__(self, oauth2_credentials):
        """Initialize.

        Args:
            oauth2_credentials: An oauth2client.OAuth2Credentials instance.
        """
        self._service = self.InitResourceHandle(oauth2_credentials)

    @classmethod
    def InitResourceHandle(cls, oauth2_credentials):
        """Authenticate and initialize a Resource object.

        Authenticate http and create a Resource object with methods
        for interacting with the service.

        Args:
            oauth2_credentials: An oauth2client.OAuth2Credentials instance.

        Returns:
            An apiclient.discovery.Resource object
        """
        http_auth = oauth2_credentials.authorize(httplib2.Http())
        return utils.RetryException(exc_retry=cls.RETRIABLE_AUTH_ERRORS,
                                    max_retry=cls.RETRY_COUNT,
                                    functor=build,
                                    sleep=cls.RETRY_SLEEP_MULTIPLIER,
                                    backoff_factor=cls.RETRY_BACKOFF_FACTOR,
                                    serviceName=cls.API_NAME,
                                    version=cls.API_VERSION,
                                    http=http_auth)

    def _ShouldRetry(self, exception, retry_http_codes,
                     other_retriable_errors):
        """Check if exception is retriable.

        Args:
            exception: An instance of Exception.
            retry_http_codes: a list of integers, retriable HTTP codes of
                              HttpError
            other_retriable_errors: a tuple of error types to retry other than
                                    HttpError.

        Returns:
            Boolean, True if retriable, False otherwise.
        """
        if isinstance(exception, other_retriable_errors):
            return True

        if isinstance(exception, errors.HttpError):
            if exception.code in retry_http_codes:
                return True
            else:
                logger.debug("_ShouldRetry: Exception code %s not in %s: %s",
                             exception.code, retry_http_codes, str(exception))

        logger.debug(
            "_ShouldRetry: Exception %s is not one of %s: %s", type(exception),
            list(other_retriable_errors) + [errors.HttpError], str(exception))
        return False

    def _TranslateError(self, exception):
        """Translate the exception to a desired type.

        Args:
            exception: An instance of Exception.

        Returns:
            gerrors.HttpError will be translated to errors.HttpError.
            If the error code is errors.HTTP_NOT_FOUND_CODE, it will
            be translated to errors.ResourceNotFoundError.
            Unrecognized error type will not be translated and will
            be returned as is.
        """
        if isinstance(exception, gerrors.HttpError):
            exception = errors.HttpError.CreateFromHttpError(exception)
            if exception.code == errors.HTTP_NOT_FOUND_CODE:
                exception = errors.ResourceNotFoundError(exception.code,
                                                         str(exception))
        return exception

    def ExecuteOnce(self, api):
        """Execute an api and parse the errors.

        Args:
            api: An apiclient.http.HttpRequest, representing the api to execute.

        Returns:
            Execution result of the api.

        Raises:
            errors.ResourceNotFoundError: For 404 error.
            errors.HttpError: For other types of http error.
        """
        try:
            return api.execute()
        except gerrors.HttpError as e:
            raise self._TranslateError(e)

    def Execute(self,
                api,
                retry_http_codes=None,
                max_retry=None,
                sleep=None,
                backoff_factor=None,
                other_retriable_errors=None):
        """Execute an api with retry.

        Call ExecuteOnce and retry on http error with given codes.

        Args:
            api: An apiclient.http.HttpRequest, representing the api to execute:
            retry_http_codes: A list of http codes to retry.
            max_retry: See utils.GenericRetry.
            sleep: See utils.GenericRetry.
            backoff_factor: See utils.GenericRetry.
            other_retriable_errors: A tuple of error types that should be retried
                                    other than errors.HttpError.

        Returns:
          Execution result of the api.

        Raises:
          See ExecuteOnce.
        """
        retry_http_codes = (self.RETRY_HTTP_CODES if retry_http_codes is None
                            else retry_http_codes)
        max_retry = (self.RETRY_COUNT if max_retry is None else max_retry)
        sleep = (self.RETRY_SLEEP_MULTIPLIER if sleep is None else sleep)
        backoff_factor = (self.RETRY_BACKOFF_FACTOR if backoff_factor is None
                          else backoff_factor)
        other_retriable_errors = (self.RETRIABLE_ERRORS
                                  if other_retriable_errors is None else
                                  other_retriable_errors)

        def _Handler(exc):
            """Check if |exc| is a retriable exception.

            Args:
                exc: An exception.

            Returns:
                True if exc is an errors.HttpError and code exists in |retry_http_codes|
                False otherwise.
            """
            if self._ShouldRetry(exc, retry_http_codes,
                                 other_retriable_errors):
                logger.debug("Will retry error: %s", str(exc))
                return True
            return False

        return utils.GenericRetry(_Handler,
                                  max_retry=max_retry,
                                  functor=self.ExecuteOnce,
                                  sleep=sleep,
                                  backoff_factor=backoff_factor,
                                  api=api)

    def BatchExecuteOnce(self, requests):
        """Execute requests in a batch.

        Args:
            requests: A dictionary where key is request id and value
                      is an http request.

        Returns:
            results, a dictionary in the following format
            {request_id: (response, exception)}
            request_ids are those from requests; response
            is the http response for the request or None on error;
            exception is an instance of DriverError or None if no error.
        """
        results = {}

        def _CallBack(request_id, response, exception):
            results[request_id] = (response, self._TranslateError(exception))

        batch = apiclient.http.BatchHttpRequest()
        for request_id, request in requests.iteritems():
            batch.add(request=request,
                      callback=_CallBack,
                      request_id=request_id)
        batch.execute()
        return results

    def BatchExecute(self,
                     requests,
                     retry_http_codes=None,
                     max_retry=None,
                     sleep=None,
                     backoff_factor=None,
                     other_retriable_errors=None):
        """Batch execute multiple requests with retry.

        Call BatchExecuteOnce and retry on http error with given codes.

        Args:
            requests: A dictionary where key is request id picked by caller,
                      and value is a apiclient.http.HttpRequest.
            retry_http_codes: A list of http codes to retry.
            max_retry: See utils.GenericRetry.
            sleep: See utils.GenericRetry.
            backoff_factor: See utils.GenericRetry.
            other_retriable_errors: A tuple of error types that should be retried
                                    other than errors.HttpError.

        Returns:
            results, a dictionary in the following format
            {request_id: (response, exception)}
            request_ids are those from requests; response
            is the http response for the request or None on error;
            exception is an instance of DriverError or None if no error.
        """
        executor = utils.BatchHttpRequestExecutor(
            self.BatchExecuteOnce,
            requests=requests,
            retry_http_codes=retry_http_codes or self.RETRY_HTTP_CODES,
            max_retry=max_retry or self.RETRY_COUNT,
            sleep=sleep or self.RETRY_SLEEP_MULTIPLIER,
            backoff_factor=backoff_factor or self.RETRY_BACKOFF_FACTOR,
            other_retriable_errors=other_retriable_errors or
            self.RETRIABLE_ERRORS)
        executor.Execute()
        return executor.GetResults()

    def ListWithMultiPages(self, api_resource, *args, **kwargs):
        """Call an api that list a type of resource.

        Multiple google services support listing a type of
        resource (e.g list gce instances, list storage objects).
        The querying pattern is similar --
        Step 1: execute the api and get a response object like,
        {
            "items": [..list of resource..],
            # The continuation token that can be used
            # to get the next page.
            "nextPageToken": "A String",
        }
        Step 2: execute the api again with the nextPageToken to
        retrieve more pages and get a response object.

        Step 3: Repeat Step 2 until no more page.

        This method encapsulates the generic logic of
        calling such listing api.

        Args:
            api_resource: An apiclient.discovery.Resource object
                used to create an http request for the listing api.
            *args: Arguments used to create the http request.
            **kwargs: Keyword based arguments to create the http
                      request.

        Returns:
            A list of items.
        """
        items = []
        next_page_token = None
        while True:
            api = api_resource(pageToken=next_page_token, *args, **kwargs)
            response = self.Execute(api)
            items.extend(response.get("items", []))
            next_page_token = response.get("nextPageToken")
            if not next_page_token:
                break
        return items

    @property
    def service(self):
        """Return self._service as a property."""
        return self._service