initial commit of acloud for open sourcing
The Cloud Android Driver Binaries (namely, acloud) in this project
provide the standard APIs to access and control Cloud Android devices
(i.e., Android Virtual Devices on Google Compute Engine) instantiated
by using the Android source code (e.g., device/google/gce* projects).
No code change required in the initial commit which is to track
all the changes submitted after the initial commit.
Unit tests are not part of this initial commit and thus will be
submitted as the second commit due to their current dependencies
Test: no build rule defined for python yet
Change-Id: Ib6aaadf33fa110f4532ba2d5b7be91e8ddc632a9
diff --git a/internal/lib/base_cloud_client.py b/internal/lib/base_cloud_client.py
new file mode 100755
index 0000000..5566793
--- /dev/null
+++ b/internal/lib/base_cloud_client.py
@@ -0,0 +1,334 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 - The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Base Cloud API Client.
+
+BasicCloudApiCliend does basic setup for a cloud API.
+"""
+import httplib
+import logging
+import os
+import socket
+import ssl
+
+import google3
+
+from apiclient import errors as gerrors
+from apiclient.discovery import build
+import apiclient.http
+import httplib2
+from oauth2client import client
+
+from acloud.internal.lib import utils
+from acloud.public import errors
+
+logger = logging.getLogger(__name__)
+
+
+class BaseCloudApiClient(object):
+ """A class that does basic setup for a cloud API."""
+
+ # To be overriden by subclasses.
+ API_NAME = ""
+ API_VERSION = "v1"
+ SCOPE = ""
+
+ # Defaults for retry.
+ RETRY_COUNT = 5
+ RETRY_BACKOFF_FACTOR = 1.5
+ RETRY_SLEEP_MULTIPLIER = 2
+ RETRY_HTTP_CODES = [
+ # 403 is to retry the "Rate Limit Exceeded" error.
+ # We could retry on a finer-grained error message later if necessary.
+ 403,
+ 500, # Internal Server Error
+ 502, # Bad Gateway
+ 503, # Service Unavailable
+ ]
+ RETRIABLE_ERRORS = (httplib.HTTPException, httplib2.HttpLib2Error,
+ socket.error, ssl.SSLError)
+ RETRIABLE_AUTH_ERRORS = (client.AccessTokenRefreshError, )
+
+ def __init__(self, oauth2_credentials):
+ """Initialize.
+
+ Args:
+ oauth2_credentials: An oauth2client.OAuth2Credentials instance.
+ """
+ self._service = self.InitResourceHandle(oauth2_credentials)
+
+ @classmethod
+ def InitResourceHandle(cls, oauth2_credentials):
+ """Authenticate and initialize a Resource object.
+
+ Authenticate http and create a Resource object with methods
+ for interacting with the service.
+
+ Args:
+ oauth2_credentials: An oauth2client.OAuth2Credentials instance.
+
+ Returns:
+ An apiclient.discovery.Resource object
+ """
+ http_auth = oauth2_credentials.authorize(httplib2.Http())
+ return utils.RetryException(exc_retry=cls.RETRIABLE_AUTH_ERRORS,
+ max_retry=cls.RETRY_COUNT,
+ functor=build,
+ sleep=cls.RETRY_SLEEP_MULTIPLIER,
+ backoff_factor=cls.RETRY_BACKOFF_FACTOR,
+ serviceName=cls.API_NAME,
+ version=cls.API_VERSION,
+ http=http_auth)
+
+ def _ShouldRetry(self, exception, retry_http_codes,
+ other_retriable_errors):
+ """Check if exception is retriable.
+
+ Args:
+ exception: An instance of Exception.
+ retry_http_codes: a list of integers, retriable HTTP codes of
+ HttpError
+ other_retriable_errors: a tuple of error types to retry other than
+ HttpError.
+
+ Returns:
+ Boolean, True if retriable, False otherwise.
+ """
+ if isinstance(exception, other_retriable_errors):
+ return True
+
+ if isinstance(exception, errors.HttpError):
+ if exception.code in retry_http_codes:
+ return True
+ else:
+ logger.debug("_ShouldRetry: Exception code %s not in %s: %s",
+ exception.code, retry_http_codes, str(exception))
+
+ logger.debug(
+ "_ShouldRetry: Exception %s is not one of %s: %s", type(exception),
+ list(other_retriable_errors) + [errors.HttpError], str(exception))
+ return False
+
+ def _TranslateError(self, exception):
+ """Translate the exception to a desired type.
+
+ Args:
+ exception: An instance of Exception.
+
+ Returns:
+ gerrors.HttpError will be translated to errors.HttpError.
+ If the error code is errors.HTTP_NOT_FOUND_CODE, it will
+ be translated to errors.ResourceNotFoundError.
+ Unrecognized error type will not be translated and will
+ be returned as is.
+ """
+ if isinstance(exception, gerrors.HttpError):
+ exception = errors.HttpError.CreateFromHttpError(exception)
+ if exception.code == errors.HTTP_NOT_FOUND_CODE:
+ exception = errors.ResourceNotFoundError(exception.code,
+ str(exception))
+ return exception
+
+ def ExecuteOnce(self, api):
+ """Execute an api and parse the errors.
+
+ Args:
+ api: An apiclient.http.HttpRequest, representing the api to execute.
+
+ Returns:
+ Execution result of the api.
+
+ Raises:
+ errors.ResourceNotFoundError: For 404 error.
+ errors.HttpError: For other types of http error.
+ """
+ try:
+ return api.execute()
+ except gerrors.HttpError as e:
+ raise self._TranslateError(e)
+
+ def Execute(self,
+ api,
+ retry_http_codes=None,
+ max_retry=None,
+ sleep=None,
+ backoff_factor=None,
+ other_retriable_errors=None):
+ """Execute an api with retry.
+
+ Call ExecuteOnce and retry on http error with given codes.
+
+ Args:
+ api: An apiclient.http.HttpRequest, representing the api to execute:
+ retry_http_codes: A list of http codes to retry.
+ max_retry: See utils.GenericRetry.
+ sleep: See utils.GenericRetry.
+ backoff_factor: See utils.GenericRetry.
+ other_retriable_errors: A tuple of error types that should be retried
+ other than errors.HttpError.
+
+ Returns:
+ Execution result of the api.
+
+ Raises:
+ See ExecuteOnce.
+ """
+ retry_http_codes = (self.RETRY_HTTP_CODES if retry_http_codes is None
+ else retry_http_codes)
+ max_retry = (self.RETRY_COUNT if max_retry is None else max_retry)
+ sleep = (self.RETRY_SLEEP_MULTIPLIER if sleep is None else sleep)
+ backoff_factor = (self.RETRY_BACKOFF_FACTOR if backoff_factor is None
+ else backoff_factor)
+ other_retriable_errors = (self.RETRIABLE_ERRORS
+ if other_retriable_errors is None else
+ other_retriable_errors)
+
+ def _Handler(exc):
+ """Check if |exc| is a retriable exception.
+
+ Args:
+ exc: An exception.
+
+ Returns:
+ True if exc is an errors.HttpError and code exists in |retry_http_codes|
+ False otherwise.
+ """
+ if self._ShouldRetry(exc, retry_http_codes,
+ other_retriable_errors):
+ logger.debug("Will retry error: %s", str(exc))
+ return True
+ return False
+
+ return utils.GenericRetry(_Handler,
+ max_retry=max_retry,
+ functor=self.ExecuteOnce,
+ sleep=sleep,
+ backoff_factor=backoff_factor,
+ api=api)
+
+ def BatchExecuteOnce(self, requests):
+ """Execute requests in a batch.
+
+ Args:
+ requests: A dictionary where key is request id and value
+ is an http request.
+
+ Returns:
+ results, a dictionary in the following format
+ {request_id: (response, exception)}
+ request_ids are those from requests; response
+ is the http response for the request or None on error;
+ exception is an instance of DriverError or None if no error.
+ """
+ results = {}
+
+ def _CallBack(request_id, response, exception):
+ results[request_id] = (response, self._TranslateError(exception))
+
+ batch = apiclient.http.BatchHttpRequest()
+ for request_id, request in requests.iteritems():
+ batch.add(request=request,
+ callback=_CallBack,
+ request_id=request_id)
+ batch.execute()
+ return results
+
+ def BatchExecute(self,
+ requests,
+ retry_http_codes=None,
+ max_retry=None,
+ sleep=None,
+ backoff_factor=None,
+ other_retriable_errors=None):
+ """Batch execute multiple requests with retry.
+
+ Call BatchExecuteOnce and retry on http error with given codes.
+
+ Args:
+ requests: A dictionary where key is request id picked by caller,
+ and value is a apiclient.http.HttpRequest.
+ retry_http_codes: A list of http codes to retry.
+ max_retry: See utils.GenericRetry.
+ sleep: See utils.GenericRetry.
+ backoff_factor: See utils.GenericRetry.
+ other_retriable_errors: A tuple of error types that should be retried
+ other than errors.HttpError.
+
+ Returns:
+ results, a dictionary in the following format
+ {request_id: (response, exception)}
+ request_ids are those from requests; response
+ is the http response for the request or None on error;
+ exception is an instance of DriverError or None if no error.
+ """
+ executor = utils.BatchHttpRequestExecutor(
+ self.BatchExecuteOnce,
+ requests=requests,
+ retry_http_codes=retry_http_codes or self.RETRY_HTTP_CODES,
+ max_retry=max_retry or self.RETRY_COUNT,
+ sleep=sleep or self.RETRY_SLEEP_MULTIPLIER,
+ backoff_factor=backoff_factor or self.RETRY_BACKOFF_FACTOR,
+ other_retriable_errors=other_retriable_errors or
+ self.RETRIABLE_ERRORS)
+ executor.Execute()
+ return executor.GetResults()
+
+ def ListWithMultiPages(self, api_resource, *args, **kwargs):
+ """Call an api that list a type of resource.
+
+ Multiple google services support listing a type of
+ resource (e.g list gce instances, list storage objects).
+ The querying pattern is similar --
+ Step 1: execute the api and get a response object like,
+ {
+ "items": [..list of resource..],
+ # The continuation token that can be used
+ # to get the next page.
+ "nextPageToken": "A String",
+ }
+ Step 2: execute the api again with the nextPageToken to
+ retrieve more pages and get a response object.
+
+ Step 3: Repeat Step 2 until no more page.
+
+ This method encapsulates the generic logic of
+ calling such listing api.
+
+ Args:
+ api_resource: An apiclient.discovery.Resource object
+ used to create an http request for the listing api.
+ *args: Arguments used to create the http request.
+ **kwargs: Keyword based arguments to create the http
+ request.
+
+ Returns:
+ A list of items.
+ """
+ items = []
+ next_page_token = None
+ while True:
+ api = api_resource(pageToken=next_page_token, *args, **kwargs)
+ response = self.Execute(api)
+ items.extend(response.get("items", []))
+ next_page_token = response.get("nextPageToken")
+ if not next_page_token:
+ break
+ return items
+
+ @property
+ def service(self):
+ """Return self._service as a property."""
+ return self._service