internal/lib/base_cloud_client.py - platform/tools/acloud - Gitiles

 #!/usr/bin/env python
 #
 # Copyright 2016 - The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.

 """Base Cloud API Client.

 BasicCloudApiCliend does basic setup for a cloud API.
 """
 import httplib
 import logging
 import os
 import socket
 import ssl

 from apiclient import errors as gerrors
 from apiclient.discovery import build
 import apiclient.http
 import httplib2
 from oauth2client import client

 from acloud.internal.lib import utils
 from acloud.public import errors

 logger = logging.getLogger(__name__)


 class BaseCloudApiClient(object):
     """A class that does basic setup for a cloud API."""

     # To be overriden by subclasses.
     API_NAME = ""
     API_VERSION = "v1"
     SCOPE = ""

     # Defaults for retry.
     RETRY_COUNT = 5
     RETRY_BACKOFF_FACTOR = 1.5
     RETRY_SLEEP_MULTIPLIER = 2
     RETRY_HTTP_CODES = [
         # 403 is to retry the "Rate Limit Exceeded" error.
         # We could retry on a finer-grained error message later if necessary.
         403,
         500,  # Internal Server Error
         502,  # Bad Gateway
         503,  # Service Unavailable
     ]
     RETRIABLE_ERRORS = (httplib.HTTPException, httplib2.HttpLib2Error,
                         socket.error, ssl.SSLError)
     RETRIABLE_AUTH_ERRORS = (client.AccessTokenRefreshError, )

     def __init__(self, oauth2_credentials):
         """Initialize.

         Args:
             oauth2_credentials: An oauth2client.OAuth2Credentials instance.
         """
         self._service = self.InitResourceHandle(oauth2_credentials)

     @classmethod
     def InitResourceHandle(cls, oauth2_credentials):
         """Authenticate and initialize a Resource object.

         Authenticate http and create a Resource object with methods
         for interacting with the service.

         Args:
             oauth2_credentials: An oauth2client.OAuth2Credentials instance.

         Returns:
             An apiclient.discovery.Resource object
         """
         http_auth = oauth2_credentials.authorize(httplib2.Http())
         return utils.RetryExceptionType(
                 exception_types=cls.RETRIABLE_AUTH_ERRORS,
                 max_retries=cls.RETRY_COUNT,
                 functor=build,
                 sleep_multiplier=cls.RETRY_SLEEP_MULTIPLIER,
                 retry_backoff_factor=cls.RETRY_BACKOFF_FACTOR,
                 serviceName=cls.API_NAME,
                 version=cls.API_VERSION,
                 http=http_auth)

     def _ShouldRetry(self, exception, retry_http_codes,
                      other_retriable_errors):
         """Check if exception is retriable.

         Args:
             exception: An instance of Exception.
             retry_http_codes: a list of integers, retriable HTTP codes of
                               HttpError
             other_retriable_errors: a tuple of error types to retry other than
                                     HttpError.

         Returns:
             Boolean, True if retriable, False otherwise.
         """
         if isinstance(exception, other_retriable_errors):
             return True

         if isinstance(exception, errors.HttpError):
             if exception.code in retry_http_codes:
                 return True
             else:
                 logger.debug("_ShouldRetry: Exception code %s not in %s: %s",
                              exception.code, retry_http_codes, str(exception))

         logger.debug(
             "_ShouldRetry: Exception %s is not one of %s: %s", type(exception),
             list(other_retriable_errors) + [errors.HttpError], str(exception))
         return False

     def _TranslateError(self, exception):
         """Translate the exception to a desired type.

         Args:
             exception: An instance of Exception.

         Returns:
             gerrors.HttpError will be translated to errors.HttpError.
             If the error code is errors.HTTP_NOT_FOUND_CODE, it will
             be translated to errors.ResourceNotFoundError.
             Unrecognized error type will not be translated and will
             be returned as is.
         """
         if isinstance(exception, gerrors.HttpError):
             exception = errors.HttpError.CreateFromHttpError(exception)
             if exception.code == errors.HTTP_NOT_FOUND_CODE:
                 exception = errors.ResourceNotFoundError(exception.code,
                                                          str(exception))
         return exception

     def ExecuteOnce(self, api):
         """Execute an api and parse the errors.

         Args:
             api: An apiclient.http.HttpRequest, representing the api to execute.

         Returns:
             Execution result of the api.

         Raises:
             errors.ResourceNotFoundError: For 404 error.
             errors.HttpError: For other types of http error.
         """
         try:
             return api.execute()
         except gerrors.HttpError as e:
             raise self._TranslateError(e)

     def Execute(self,
                 api,
                 retry_http_codes=None,
                 max_retry=None,
                 sleep=None,
                 backoff_factor=None,
                 other_retriable_errors=None):
         """Execute an api with retry.

         Call ExecuteOnce and retry on http error with given codes.

         Args:
             api: An apiclient.http.HttpRequest, representing the api to execute:
             retry_http_codes: A list of http codes to retry.
             max_retry: See utils.Retry.
             sleep: See utils.Retry.
             backoff_factor: See utils.Retry.
             other_retriable_errors: A tuple of error types that should be retried
                                     other than errors.HttpError.

         Returns:
           Execution result of the api.

         Raises:
           See ExecuteOnce.
         """
         retry_http_codes = (self.RETRY_HTTP_CODES if retry_http_codes is None
                             else retry_http_codes)
         max_retry = (self.RETRY_COUNT if max_retry is None else max_retry)
         sleep = (self.RETRY_SLEEP_MULTIPLIER if sleep is None else sleep)
         backoff_factor = (self.RETRY_BACKOFF_FACTOR if backoff_factor is None
                           else backoff_factor)
         other_retriable_errors = (self.RETRIABLE_ERRORS
                                   if other_retriable_errors is None else
                                   other_retriable_errors)

         def _Handler(exc):
             """Check if |exc| is a retriable exception.

             Args:
                 exc: An exception.

             Returns:
                 True if exc is an errors.HttpError and code exists in |retry_http_codes|
                 False otherwise.
             """
             if self._ShouldRetry(exc, retry_http_codes,
                                  other_retriable_errors):
                 logger.debug("Will retry error: %s", str(exc))
                 return True
             return False

         return utils.Retry(
              _Handler, max_retries=max_retry, functor=self.ExecuteOnce,
              sleep_multiplier=sleep, retry_backoff_factor=backoff_factor,
              api=api)

     def BatchExecuteOnce(self, requests):
         """Execute requests in a batch.

         Args:
             requests: A dictionary where key is request id and value
                       is an http request.

         Returns:
             results, a dictionary in the following format
             {request_id: (response, exception)}
             request_ids are those from requests; response
             is the http response for the request or None on error;
             exception is an instance of DriverError or None if no error.
         """
         results = {}

         def _CallBack(request_id, response, exception):
             results[request_id] = (response, self._TranslateError(exception))

         batch = apiclient.http.BatchHttpRequest()
         for request_id, request in requests.iteritems():
             batch.add(request=request,
                       callback=_CallBack,
                       request_id=request_id)
         batch.execute()
         return results

     def BatchExecute(self,
                      requests,
                      retry_http_codes=None,
                      max_retry=None,
                      sleep=None,
                      backoff_factor=None,
                      other_retriable_errors=None):
         """Batch execute multiple requests with retry.

         Call BatchExecuteOnce and retry on http error with given codes.

         Args:
             requests: A dictionary where key is request id picked by caller,
                       and value is a apiclient.http.HttpRequest.
             retry_http_codes: A list of http codes to retry.
             max_retry: See utils.Retry.
             sleep: See utils.Retry.
             backoff_factor: See utils.Retry.
             other_retriable_errors: A tuple of error types that should be retried
                                     other than errors.HttpError.

         Returns:
             results, a dictionary in the following format
             {request_id: (response, exception)}
             request_ids are those from requests; response
             is the http response for the request or None on error;
             exception is an instance of DriverError or None if no error.
         """
         executor = utils.BatchHttpRequestExecutor(
             self.BatchExecuteOnce,
             requests=requests,
             retry_http_codes=retry_http_codes or self.RETRY_HTTP_CODES,
             max_retry=max_retry or self.RETRY_COUNT,
             sleep=sleep or self.RETRY_SLEEP_MULTIPLIER,
             backoff_factor=backoff_factor or self.RETRY_BACKOFF_FACTOR,
             other_retriable_errors=other_retriable_errors or
             self.RETRIABLE_ERRORS)
         executor.Execute()
         return executor.GetResults()

     def ListWithMultiPages(self, api_resource, *args, **kwargs):
         """Call an api that list a type of resource.

         Multiple google services support listing a type of
         resource (e.g list gce instances, list storage objects).
         The querying pattern is similar --
         Step 1: execute the api and get a response object like,
         {
             "items": [..list of resource..],
             # The continuation token that can be used
             # to get the next page.
             "nextPageToken": "A String",
         }
         Step 2: execute the api again with the nextPageToken to
         retrieve more pages and get a response object.

         Step 3: Repeat Step 2 until no more page.

         This method encapsulates the generic logic of
         calling such listing api.

         Args:
             api_resource: An apiclient.discovery.Resource object
                 used to create an http request for the listing api.
             *args: Arguments used to create the http request.
             **kwargs: Keyword based arguments to create the http
                       request.

         Returns:
             A list of items.
         """
         items = []
         next_page_token = None
         while True:
             api = api_resource(pageToken=next_page_token, *args, **kwargs)
             response = self.Execute(api)
             items.extend(response.get("items", []))
             next_page_token = response.get("nextPageToken")
             if not next_page_token:
                 break
         return items

     @property
     def service(self):
         """Return self._service as a property."""
         return self._service
	#!/usr/bin/env python
	#
	# Copyright 2016 - The Android Open Source Project
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	"""Base Cloud API Client.

	BasicCloudApiCliend does basic setup for a cloud API.
	"""
	import httplib
	import logging
	import os
	import socket
	import ssl

	from apiclient import errors as gerrors
	from apiclient.discovery import build
	import apiclient.http
	import httplib2
	from oauth2client import client

	from acloud.internal.lib import utils
	from acloud.public import errors

	logger = logging.getLogger(__name__)


	class BaseCloudApiClient(object):
	"""A class that does basic setup for a cloud API."""

	# To be overriden by subclasses.
	API_NAME = ""
	API_VERSION = "v1"
	SCOPE = ""

	# Defaults for retry.
	RETRY_COUNT = 5
	RETRY_BACKOFF_FACTOR = 1.5
	RETRY_SLEEP_MULTIPLIER = 2
	RETRY_HTTP_CODES = [
	# 403 is to retry the "Rate Limit Exceeded" error.
	# We could retry on a finer-grained error message later if necessary.
	403,
	500, # Internal Server Error
	502, # Bad Gateway
	503, # Service Unavailable
	]
	RETRIABLE_ERRORS = (httplib.HTTPException, httplib2.HttpLib2Error,
	socket.error, ssl.SSLError)
	RETRIABLE_AUTH_ERRORS = (client.AccessTokenRefreshError, )

	def __init__(self, oauth2_credentials):
	"""Initialize.

	Args:
	oauth2_credentials: An oauth2client.OAuth2Credentials instance.
	"""
	self._service = self.InitResourceHandle(oauth2_credentials)

	@classmethod
	def InitResourceHandle(cls, oauth2_credentials):
	"""Authenticate and initialize a Resource object.

	Authenticate http and create a Resource object with methods
	for interacting with the service.

	Args:
	oauth2_credentials: An oauth2client.OAuth2Credentials instance.

	Returns:
	An apiclient.discovery.Resource object
	"""
	http_auth = oauth2_credentials.authorize(httplib2.Http())
	return utils.RetryExceptionType(
	exception_types=cls.RETRIABLE_AUTH_ERRORS,
	max_retries=cls.RETRY_COUNT,
	functor=build,
	sleep_multiplier=cls.RETRY_SLEEP_MULTIPLIER,
	retry_backoff_factor=cls.RETRY_BACKOFF_FACTOR,
	serviceName=cls.API_NAME,
	version=cls.API_VERSION,
	http=http_auth)

	def _ShouldRetry(self, exception, retry_http_codes,
	other_retriable_errors):
	"""Check if exception is retriable.

	Args:
	exception: An instance of Exception.
	retry_http_codes: a list of integers, retriable HTTP codes of
	HttpError
	other_retriable_errors: a tuple of error types to retry other than
	HttpError.

	Returns:
	Boolean, True if retriable, False otherwise.
	"""
	if isinstance(exception, other_retriable_errors):
	return True

	if isinstance(exception, errors.HttpError):
	if exception.code in retry_http_codes:
	return True
	else:
	logger.debug("_ShouldRetry: Exception code %s not in %s: %s",
	exception.code, retry_http_codes, str(exception))

	logger.debug(
	"_ShouldRetry: Exception %s is not one of %s: %s", type(exception),
	list(other_retriable_errors) + [errors.HttpError], str(exception))
	return False

	def _TranslateError(self, exception):
	"""Translate the exception to a desired type.

	Args:
	exception: An instance of Exception.

	Returns:
	gerrors.HttpError will be translated to errors.HttpError.
	If the error code is errors.HTTP_NOT_FOUND_CODE, it will
	be translated to errors.ResourceNotFoundError.
	Unrecognized error type will not be translated and will
	be returned as is.
	"""
	if isinstance(exception, gerrors.HttpError):
	exception = errors.HttpError.CreateFromHttpError(exception)
	if exception.code == errors.HTTP_NOT_FOUND_CODE:
	exception = errors.ResourceNotFoundError(exception.code,
	str(exception))
	return exception

	def ExecuteOnce(self, api):
	"""Execute an api and parse the errors.

	Args:
	api: An apiclient.http.HttpRequest, representing the api to execute.

	Returns:
	Execution result of the api.

	Raises:
	errors.ResourceNotFoundError: For 404 error.
	errors.HttpError: For other types of http error.
	"""
	try:
	return api.execute()
	except gerrors.HttpError as e:
	raise self._TranslateError(e)

	def Execute(self,
	api,
	retry_http_codes=None,
	max_retry=None,
	sleep=None,
	backoff_factor=None,
	other_retriable_errors=None):
	"""Execute an api with retry.

	Call ExecuteOnce and retry on http error with given codes.

	Args:
	api: An apiclient.http.HttpRequest, representing the api to execute:
	retry_http_codes: A list of http codes to retry.
	max_retry: See utils.Retry.
	sleep: See utils.Retry.
	backoff_factor: See utils.Retry.
	other_retriable_errors: A tuple of error types that should be retried
	other than errors.HttpError.

	Returns:
	Execution result of the api.

	Raises:
	See ExecuteOnce.
	"""
	retry_http_codes = (self.RETRY_HTTP_CODES if retry_http_codes is None
	else retry_http_codes)
	max_retry = (self.RETRY_COUNT if max_retry is None else max_retry)
	sleep = (self.RETRY_SLEEP_MULTIPLIER if sleep is None else sleep)
	backoff_factor = (self.RETRY_BACKOFF_FACTOR if backoff_factor is None
	else backoff_factor)
	other_retriable_errors = (self.RETRIABLE_ERRORS
	if other_retriable_errors is None else
	other_retriable_errors)

	def _Handler(exc):
	"""Check if \|exc\| is a retriable exception.

	Args:
	exc: An exception.

	Returns:
	True if exc is an errors.HttpError and code exists in \|retry_http_codes\|
	False otherwise.
	"""
	if self._ShouldRetry(exc, retry_http_codes,
	other_retriable_errors):
	logger.debug("Will retry error: %s", str(exc))
	return True
	return False

	return utils.Retry(
	_Handler, max_retries=max_retry, functor=self.ExecuteOnce,
	sleep_multiplier=sleep, retry_backoff_factor=backoff_factor,
	api=api)

	def BatchExecuteOnce(self, requests):
	"""Execute requests in a batch.

	Args:
	requests: A dictionary where key is request id and value
	is an http request.

	Returns:
	results, a dictionary in the following format
	{request_id: (response, exception)}
	request_ids are those from requests; response
	is the http response for the request or None on error;
	exception is an instance of DriverError or None if no error.
	"""
	results = {}

	def _CallBack(request_id, response, exception):
	results[request_id] = (response, self._TranslateError(exception))

	batch = apiclient.http.BatchHttpRequest()
	for request_id, request in requests.iteritems():
	batch.add(request=request,
	callback=_CallBack,
	request_id=request_id)
	batch.execute()
	return results

	def BatchExecute(self,
	requests,
	retry_http_codes=None,
	max_retry=None,
	sleep=None,
	backoff_factor=None,
	other_retriable_errors=None):
	"""Batch execute multiple requests with retry.

	Call BatchExecuteOnce and retry on http error with given codes.

	Args:
	requests: A dictionary where key is request id picked by caller,
	and value is a apiclient.http.HttpRequest.
	retry_http_codes: A list of http codes to retry.
	max_retry: See utils.Retry.
	sleep: See utils.Retry.
	backoff_factor: See utils.Retry.
	other_retriable_errors: A tuple of error types that should be retried
	other than errors.HttpError.

	Returns:
	results, a dictionary in the following format
	{request_id: (response, exception)}
	request_ids are those from requests; response
	is the http response for the request or None on error;
	exception is an instance of DriverError or None if no error.
	"""
	executor = utils.BatchHttpRequestExecutor(
	self.BatchExecuteOnce,
	requests=requests,
	retry_http_codes=retry_http_codes or self.RETRY_HTTP_CODES,
	max_retry=max_retry or self.RETRY_COUNT,
	sleep=sleep or self.RETRY_SLEEP_MULTIPLIER,
	backoff_factor=backoff_factor or self.RETRY_BACKOFF_FACTOR,
	other_retriable_errors=other_retriable_errors or
	self.RETRIABLE_ERRORS)
	executor.Execute()
	return executor.GetResults()

	def ListWithMultiPages(self, api_resource, args, *kwargs):
	"""Call an api that list a type of resource.

	Multiple google services support listing a type of
	resource (e.g list gce instances, list storage objects).
	The querying pattern is similar --
	Step 1: execute the api and get a response object like,
	{
	"items": [..list of resource..],
	# The continuation token that can be used
	# to get the next page.
	"nextPageToken": "A String",
	}
	Step 2: execute the api again with the nextPageToken to
	retrieve more pages and get a response object.

	Step 3: Repeat Step 2 until no more page.

	This method encapsulates the generic logic of
	calling such listing api.

	Args:
	api_resource: An apiclient.discovery.Resource object
	used to create an http request for the listing api.
	*args: Arguments used to create the http request.
	**kwargs: Keyword based arguments to create the http
	request.

	Returns:
	A list of items.
	"""
	items = []
	next_page_token = None
	while True:
	api = api_resource(pageToken=next_page_token, args, *kwargs)
	response = self.Execute(api)
	items.extend(response.get("items", []))
	next_page_token = response.get("nextPageToken")
	if not next_page_token:
	break
	return items

	@property
	def service(self):
	"""Return self._service as a property."""
	return self._service