Add api_core package (#4210)
* Add api_core package
* Address review comments
diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 0000000..d097511
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,13 @@
+[run]
+branch = True
+
+[report]
+fail_under = 100
+show_missing = True
+exclude_lines =
+ # Re-enable the standard pragma
+ pragma: NO COVER
+ # Ignore debug-only repr
+ def __repr__
+ # Ignore abstract methods
+ raise NotImplementedError
diff --git a/.flake8 b/.flake8
new file mode 100644
index 0000000..3db9b73
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,10 @@
+[flake8]
+import-order-style=google
+# Note: this forces all google imports to be in the third group. See
+# https://github.com/PyCQA/flake8-import-order/issues/111
+application-import-names=google
+exclude =
+ __pycache__,
+ .git,
+ *.pyc,
+ conf.py
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..d645695
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..1fbc0d0
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,3 @@
+include README.rst LICENSE
+recursive-include tests *
+global-exclude *.pyc __pycache__
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000..afb4774
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,14 @@
+Core Library for Google Client Libraries
+========================================
+
+|pypi| |versions|
+
+This library is not meant to stand-alone. Instead it defines
+common helpers used by all Google API clients. For more information, see the
+`documentation`_.
+
+.. |pypi| image:: https://img.shields.io/pypi/v/google-cloud-core.svg
+ :target: https://pypi.org/project/google-cloud-core/
+.. |versions| image:: https://img.shields.io/pypi/pyversions/google-cloud-core.svg
+ :target: https://pypi.org/project/google-cloud-core/
+.. _documentation: https://googlecloudplatform.github.io/google-cloud-python/latest/core/
diff --git a/google/__init__.py b/google/__init__.py
new file mode 100644
index 0000000..a35569c
--- /dev/null
+++ b/google/__init__.py
@@ -0,0 +1,22 @@
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Google namespace package."""
+
+try:
+ import pkg_resources
+ pkg_resources.declare_namespace(__name__)
+except ImportError:
+ import pkgutil
+ __path__ = pkgutil.extend_path(__path__, __name__)
diff --git a/google/api_core/__init__.py b/google/api_core/__init__.py
new file mode 100644
index 0000000..9c1ab88
--- /dev/null
+++ b/google/api_core/__init__.py
@@ -0,0 +1,23 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Google API Core.
+
+This package contains common code and utilties used by Google client libraries.
+"""
+
+from pkg_resources import get_distribution
+
+
+__version__ = get_distribution('google-api-core').version
diff --git a/google/api_core/datetime_helpers.py b/google/api_core/datetime_helpers.py
new file mode 100644
index 0000000..cfc817b
--- /dev/null
+++ b/google/api_core/datetime_helpers.py
@@ -0,0 +1,22 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for :mod:`datetime`."""
+
+import datetime
+
+
+def utcnow():
+ """A :meth:`datetime.datetime.utcnow()` alias to allow mocking in tests."""
+ return datetime.datetime.utcnow()
diff --git a/google/api_core/exceptions.py b/google/api_core/exceptions.py
new file mode 100644
index 0000000..eb1d548
--- /dev/null
+++ b/google/api_core/exceptions.py
@@ -0,0 +1,443 @@
+# Copyright 2014 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Exceptions raised by Google API core & clients.
+
+This module provides base classes for all errors raised by libraries based
+on :mod:`google.api_core`, including both HTTP and gRPC clients.
+"""
+
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import six
+from six.moves import http_client
+
+try:
+ import grpc
+except ImportError: # pragma: NO COVER
+ grpc = None
+
+# Lookup tables for mapping exceptions from HTTP and gRPC transports.
+# Populated by _APICallErrorMeta
+_HTTP_CODE_TO_EXCEPTION = {}
+_GRPC_CODE_TO_EXCEPTION = {}
+
+
+class GoogleAPIError(Exception):
+ """Base class for all exceptions raised by Google API Clients."""
+ pass
+
+
+@six.python_2_unicode_compatible
+class RetryError(GoogleAPIError):
+ """Raised when a function has exhausted all of its available retries.
+
+ Args:
+ message (str): The exception message.
+ cause (Exception): The last exception raised when retring the
+ function.
+ """
+ def __init__(self, message, cause):
+ super(RetryError, self).__init__(message)
+ self.message = message
+ self._cause = cause
+
+ @property
+ def cause(self):
+ """The last exception raised when retrying the function."""
+ return self._cause
+
+ def __str__(self):
+ return '{}, last exception: {}'.format(self.message, self.cause)
+
+
+class _GoogleAPICallErrorMeta(type):
+ """Metaclass for registering GoogleAPICallError subclasses."""
+ def __new__(mcs, name, bases, class_dict):
+ cls = type.__new__(mcs, name, bases, class_dict)
+ if cls.code is not None:
+ _HTTP_CODE_TO_EXCEPTION.setdefault(cls.code, cls)
+ if cls.grpc_status_code is not None:
+ _GRPC_CODE_TO_EXCEPTION.setdefault(cls.grpc_status_code, cls)
+ return cls
+
+
+@six.python_2_unicode_compatible
+@six.add_metaclass(_GoogleAPICallErrorMeta)
+class GoogleAPICallError(GoogleAPIError):
+ """Base class for exceptions raised by calling API methods.
+
+ Args:
+ message (str): The exception message.
+ errors (Sequence[Any]): An optional list of error details.
+ response (Union[requests.Request, grpc.Call]): The response or
+ gRPC call metadata.
+ """
+
+ code = None
+ """Optional[int]: The HTTP status code associated with this error.
+
+ This may be ``None`` if the exception does not have a direct mapping
+ to an HTTP error.
+
+ See http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
+ """
+
+ grpc_status_code = None
+ """Optional[grpc.StatusCode]: The gRPC status code associated with this
+ error.
+
+ This may be ``None`` if the exception does not match up to a gRPC error.
+ """
+
+ def __init__(self, message, errors=(), response=None):
+ super(GoogleAPICallError, self).__init__(message)
+ self.message = message
+ """str: The exception message."""
+ self._errors = errors
+ self._response = response
+
+ def __str__(self):
+ return '{} {}'.format(self.code, self.message)
+
+ @property
+ def errors(self):
+ """Detailed error information.
+
+ Returns:
+ Sequence[Any]: A list of additional error details.
+ """
+ return list(self._errors)
+
+ @property
+ def response(self):
+ """Optional[Union[requests.Request, grpc.Call]]: The response or
+ gRPC call metadata."""
+ return self._response
+
+
+class Redirection(GoogleAPICallError):
+ """Base class for for all redirection (HTTP 3xx) responses."""
+
+
+class MovedPermanently(Redirection):
+ """Exception mapping a ``301 Moved Permanently`` response."""
+ code = http_client.MOVED_PERMANENTLY
+
+
+class NotModified(Redirection):
+ """Exception mapping a ``304 Not Modified`` response."""
+ code = http_client.NOT_MODIFIED
+
+
+class TemporaryRedirect(Redirection):
+ """Exception mapping a ``307 Temporary Redirect`` response."""
+ code = http_client.TEMPORARY_REDIRECT
+
+
+class ResumeIncomplete(Redirection):
+ """Exception mapping a ``308 Resume Incomplete`` response.
+
+ .. note:: :attr:`http_client.PERMANENT_REDIRECT` is ``308``, but Google
+ APIs differ in their use of this status code.
+ """
+ code = 308
+
+
+class ClientError(GoogleAPICallError):
+ """Base class for all client error (HTTP 4xx) responses."""
+
+
+class BadRequest(ClientError):
+ """Exception mapping a ``400 Bad Request`` response."""
+ code = http_client.BAD_REQUEST
+
+
+class InvalidArgument(BadRequest):
+ """Exception mapping a :attr:`grpc.StatusCode.INVALID_ARGUMENT` error."""
+ grpc_status_code = (
+ grpc.StatusCode.INVALID_ARGUMENT if grpc is not None else None)
+
+
+class FailedPrecondition(BadRequest):
+ """Exception mapping a :attr:`grpc.StatusCode.FAILED_PRECONDITION`
+ error."""
+ grpc_status_code = (
+ grpc.StatusCode.FAILED_PRECONDITION if grpc is not None else None)
+
+
+class OutOfRange(BadRequest):
+ """Exception mapping a :attr:`grpc.StatusCode.OUT_OF_RANGE` error."""
+ grpc_status_code = (
+ grpc.StatusCode.OUT_OF_RANGE if grpc is not None else None)
+
+
+class Unauthorized(ClientError):
+ """Exception mapping a ``401 Unauthorized`` response."""
+ code = http_client.UNAUTHORIZED
+
+
+class Unauthenticated(Unauthorized):
+ """Exception mapping a :attr:`grpc.StatusCode.UNAUTHENTICATED` error."""
+ grpc_status_code = (
+ grpc.StatusCode.UNAUTHENTICATED if grpc is not None else None)
+
+
+class Forbidden(ClientError):
+ """Exception mapping a ``403 Forbidden`` response."""
+ code = http_client.FORBIDDEN
+
+
+class PermissionDenied(Forbidden):
+ """Exception mapping a :attr:`grpc.StatusCode.PERMISSION_DENIED` error."""
+ grpc_status_code = (
+ grpc.StatusCode.PERMISSION_DENIED if grpc is not None else None)
+
+
+class NotFound(ClientError):
+ """Exception mapping a ``404 Not Found`` response or a
+ :attr:`grpc.StatusCode.NOT_FOUND` error."""
+ code = http_client.NOT_FOUND
+ grpc_status_code = (
+ grpc.StatusCode.NOT_FOUND if grpc is not None else None)
+
+
+class MethodNotAllowed(ClientError):
+ """Exception mapping a ``405 Method Not Allowed`` response."""
+ code = http_client.METHOD_NOT_ALLOWED
+
+
+class Conflict(ClientError):
+ """Exception mapping a ``409 Conflict`` response."""
+ code = http_client.CONFLICT
+
+
+class AlreadyExists(Conflict):
+ """Exception mapping a :attr:`grpc.StatusCode.ALREADY_EXISTS` error."""
+ grpc_status_code = (
+ grpc.StatusCode.ALREADY_EXISTS if grpc is not None else None)
+
+
+class Aborted(Conflict):
+ """Exception mapping a :attr:`grpc.StatusCode.ABORTED` error."""
+ grpc_status_code = (
+ grpc.StatusCode.ABORTED if grpc is not None else None)
+
+
+class LengthRequired(ClientError):
+ """Exception mapping a ``411 Length Required`` response."""
+ code = http_client.LENGTH_REQUIRED
+
+
+class PreconditionFailed(ClientError):
+ """Exception mapping a ``412 Precondition Failed`` response."""
+ code = http_client.PRECONDITION_FAILED
+
+
+class RequestRangeNotSatisfiable(ClientError):
+ """Exception mapping a ``416 Request Range Not Satisfiable`` response."""
+ code = http_client.REQUESTED_RANGE_NOT_SATISFIABLE
+
+
+class TooManyRequests(ClientError):
+ """Exception mapping a ``429 Too Many Requests`` response."""
+ # http_client does not define a constant for this in Python 2.
+ code = 429
+
+
+class ResourceExhausted(TooManyRequests):
+ """Exception mapping a :attr:`grpc.StatusCode.RESOURCE_EXHAUSTED` error."""
+ grpc_status_code = (
+ grpc.StatusCode.RESOURCE_EXHAUSTED if grpc is not None else None)
+
+
+class Cancelled(ClientError):
+ """Exception mapping a :attr:`grpc.StatusCode.CANCELLED` error."""
+ # This maps to HTTP status code 499. See
+ # https://github.com/googleapis/googleapis/blob/master/google/rpc\
+ # /code.proto
+ code = 499
+ grpc_status_code = grpc.StatusCode.CANCELLED if grpc is not None else None
+
+
+class ServerError(GoogleAPICallError):
+ """Base for 5xx responses."""
+
+
+class InternalServerError(ServerError):
+ """Exception mapping a ``500 Internal Server Error`` response. or a
+ :attr:`grpc.StatusCode.INTERNAL` error."""
+ code = http_client.INTERNAL_SERVER_ERROR
+ grpc_status_code = grpc.StatusCode.INTERNAL if grpc is not None else None
+
+
+class Unknown(ServerError):
+ """Exception mapping a :attr:`grpc.StatusCode.UNKNOWN` error."""
+ grpc_status_code = grpc.StatusCode.UNKNOWN if grpc is not None else None
+
+
+class DataLoss(ServerError):
+ """Exception mapping a :attr:`grpc.StatusCode.DATA_LOSS` error."""
+ grpc_status_code = grpc.StatusCode.DATA_LOSS if grpc is not None else None
+
+
+class MethodNotImplemented(ServerError):
+ """Exception mapping a ``501 Not Implemented`` response or a
+ :attr:`grpc.StatusCode.UNIMPLEMENTED` error."""
+ code = http_client.NOT_IMPLEMENTED
+ grpc_status_code = (
+ grpc.StatusCode.UNIMPLEMENTED if grpc is not None else None)
+
+
+class BadGateway(ServerError):
+ """Exception mapping a ``502 Bad Gateway`` response."""
+ code = http_client.BAD_GATEWAY
+
+
+class ServiceUnavailable(ServerError):
+ """Exception mapping a ``503 Service Unavailable`` response or a
+ :attr:`grpc.StatusCode.UNAVAILABLE` error."""
+ code = http_client.SERVICE_UNAVAILABLE
+ grpc_status_code = (
+ grpc.StatusCode.UNAVAILABLE if grpc is not None else None)
+
+
+class GatewayTimeout(ServerError):
+ """Exception mapping a ``504 Gateway Timeout`` response."""
+ code = http_client.GATEWAY_TIMEOUT
+
+
+class DeadlineExceeded(GatewayTimeout):
+ """Exception mapping a :attr:`grpc.StatusCode.DEADLINE_EXCEEDED` error."""
+ grpc_status_code = (
+ grpc.StatusCode.DEADLINE_EXCEEDED if grpc is not None else None)
+
+
+def exception_class_for_http_status(status_code):
+ """Return the exception class for a specific HTTP status code.
+
+ Args:
+ status_code (int): The HTTP status code.
+
+ Returns:
+ :func:`type`: the appropriate subclass of :class:`GoogleAPICallError`.
+ """
+ return _HTTP_CODE_TO_EXCEPTION.get(status_code, GoogleAPICallError)
+
+
+def from_http_status(status_code, message, **kwargs):
+ """Create a :class:`GoogleAPICallError` from an HTTP status code.
+
+ Args:
+ status_code (int): The HTTP status code.
+ message (str): The exception message.
+ kwargs: Additional arguments passed to the :class:`GoogleAPICallError`
+ constructor.
+
+ Returns:
+ GoogleAPICallError: An instance of the appropriate subclass of
+ :class:`GoogleAPICallError`.
+ """
+ error_class = exception_class_for_http_status(status_code)
+ error = error_class(message, **kwargs)
+
+ if error.code is None:
+ error.code = status_code
+
+ return error
+
+
+def from_http_response(response):
+ """Create a :class:`GoogleAPICallError` from a :class:`requests.Response`.
+
+ Args:
+ response (requests.Response): The HTTP response.
+
+ Returns:
+ GoogleAPICallError: An instance of the appropriate subclass of
+ :class:`GoogleAPICallError`, with the message and errors populated
+ from the response.
+ """
+ try:
+ payload = response.json()
+ except ValueError:
+ payload = {'error': {'message': response.text or 'unknown error'}}
+
+ error_message = payload.get('error', {}).get('message', 'unknown error')
+ errors = payload.get('error', {}).get('errors', ())
+
+ message = '{method} {url}: {error}'.format(
+ method=response.request.method,
+ url=response.request.url,
+ error=error_message)
+
+ exception = from_http_status(
+ response.status_code, message, errors=errors, response=response)
+ return exception
+
+
+def exception_class_for_grpc_status(status_code):
+ """Return the exception class for a specific :class:`grpc.StatusCode`.
+
+ Args:
+ status_code (grpc.StatusCode): The gRPC status code.
+
+ Returns:
+ :func:`type`: the appropriate subclass of :class:`GoogleAPICallError`.
+ """
+ return _GRPC_CODE_TO_EXCEPTION.get(status_code, GoogleAPICallError)
+
+
+def from_grpc_status(status_code, message, **kwargs):
+ """Create a :class:`GoogleAPICallError` from a :class:`grpc.StatusCode`.
+
+ Args:
+ status_code (grpc.StatusCode): The gRPC status code.
+ message (str): The exception message.
+ kwargs: Additional arguments passed to the :class:`GoogleAPICallError`
+ constructor.
+
+ Returns:
+ GoogleAPICallError: An instance of the appropriate subclass of
+ :class:`GoogleAPICallError`.
+ """
+ error_class = exception_class_for_grpc_status(status_code)
+ error = error_class(message, **kwargs)
+
+ if error.grpc_status_code is None:
+ error.grpc_status_code = status_code
+
+ return error
+
+
+def from_grpc_error(rpc_exc):
+ """Create a :class:`GoogleAPICallError` from a :class:`grpc.RpcError`.
+
+ Args:
+ rpc_exc (grpc.RpcError): The gRPC error.
+
+ Returns:
+ GoogleAPICallError: An instance of the appropriate subclass of
+ :class:`GoogleAPICallError`.
+ """
+ if isinstance(rpc_exc, grpc.Call):
+ return from_grpc_status(
+ rpc_exc.code(),
+ rpc_exc.details(),
+ errors=(rpc_exc,),
+ response=rpc_exc)
+ else:
+ return GoogleAPICallError(
+ str(rpc_exc), errors=(rpc_exc,), response=rpc_exc)
diff --git a/google/api_core/future/__init__.py b/google/api_core/future/__init__.py
new file mode 100644
index 0000000..82ab739
--- /dev/null
+++ b/google/api_core/future/__init__.py
@@ -0,0 +1,21 @@
+# Copyright 2017, Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Futures for dealing with asynchronous operations."""
+
+from google.api_core.future.base import Future
+
+__all__ = [
+ 'Future',
+]
diff --git a/google/api_core/future/_helpers.py b/google/api_core/future/_helpers.py
new file mode 100644
index 0000000..933d0b8
--- /dev/null
+++ b/google/api_core/future/_helpers.py
@@ -0,0 +1,39 @@
+# Copyright 2017, Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Private helpers for futures."""
+
+import logging
+import threading
+
+
+_LOGGER = logging.getLogger(__name__)
+
+
+def start_daemon_thread(*args, **kwargs):
+ """Starts a thread and marks it as a daemon thread."""
+ thread = threading.Thread(*args, **kwargs)
+ thread.daemon = True
+ thread.start()
+ return thread
+
+
+def safe_invoke_callback(callback, *args, **kwargs):
+ """Invoke a callback, swallowing and logging any exceptions."""
+ # pylint: disable=bare-except
+ # We intentionally want to swallow all exceptions.
+ try:
+ return callback(*args, **kwargs)
+ except:
+ _LOGGER.exception('Error while executing Future callback.')
diff --git a/google/api_core/future/base.py b/google/api_core/future/base.py
new file mode 100644
index 0000000..2439136
--- /dev/null
+++ b/google/api_core/future/base.py
@@ -0,0 +1,67 @@
+# Copyright 2017, Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Abstract and helper bases for Future implementations."""
+
+import abc
+
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Future(object):
+ # pylint: disable=missing-docstring
+ # We inherit the interfaces here from concurrent.futures.
+
+ """Future interface.
+
+ This interface is based on :class:`concurrent.futures.Future`.
+ """
+
+ @abc.abstractmethod
+ def cancel(self):
+ raise NotImplementedError()
+
+ @abc.abstractmethod
+ def cancelled(self):
+ raise NotImplementedError()
+
+ @abc.abstractmethod
+ def running(self):
+ raise NotImplementedError()
+
+ @abc.abstractmethod
+ def done(self):
+ raise NotImplementedError()
+
+ @abc.abstractmethod
+ def result(self, timeout=None):
+ raise NotImplementedError()
+
+ @abc.abstractmethod
+ def exception(self, timeout=None):
+ raise NotImplementedError()
+
+ @abc.abstractmethod
+ def add_done_callback(self, fn):
+ # pylint: disable=invalid-name
+ raise NotImplementedError()
+
+ @abc.abstractmethod
+ def set_result(self, result):
+ raise NotImplementedError()
+
+ @abc.abstractmethod
+ def set_exception(self, exception):
+ raise NotImplementedError()
diff --git a/google/api_core/future/polling.py b/google/api_core/future/polling.py
new file mode 100644
index 0000000..eea9624
--- /dev/null
+++ b/google/api_core/future/polling.py
@@ -0,0 +1,165 @@
+# Copyright 2017, Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Abstract and helper bases for Future implementations."""
+
+import abc
+import concurrent.futures
+
+from google.api_core import exceptions
+from google.api_core import retry
+from google.api_core.future import _helpers
+from google.api_core.future import base
+
+
+class _OperationNotComplete(Exception):
+ """Private exception used for polling via retry."""
+ pass
+
+
+class PollingFuture(base.Future):
+ """A Future that needs to poll some service to check its status.
+
+ The :meth:`done` method should be implemented by subclasses. The polling
+ behavior will repeatedly call ``done`` until it returns True.
+
+ .. note: Privacy here is intended to prevent the final class from
+ overexposing, not to prevent subclasses from accessing methods.
+ """
+ def __init__(self):
+ super(PollingFuture, self).__init__()
+ self._result = None
+ self._exception = None
+ self._result_set = False
+ """bool: Set to True when the result has been set via set_result or
+ set_exception."""
+ self._polling_thread = None
+ self._done_callbacks = []
+
+ @abc.abstractmethod
+ def done(self):
+ """Checks to see if the operation is complete.
+
+ Returns:
+ bool: True if the operation is complete, False otherwise.
+ """
+ # pylint: disable=redundant-returns-doc, missing-raises-doc
+ raise NotImplementedError()
+
+ def _done_or_raise(self):
+ """Check if the future is done and raise if it's not."""
+ if not self.done():
+ raise _OperationNotComplete()
+
+ def running(self):
+ """True if the operation is currently running."""
+ return not self.done()
+
+ def _blocking_poll(self, timeout=None):
+ """Poll and wait for the Future to be resolved.
+
+ Args:
+ timeout (int):
+ How long (in seconds) to wait for the operation to complete.
+ If None, wait indefinitely.
+ """
+ if self._result_set:
+ return
+
+ retry_ = retry.Retry(
+ predicate=retry.if_exception_type(_OperationNotComplete),
+ deadline=timeout)
+
+ try:
+ retry_(self._done_or_raise)()
+ except exceptions.RetryError:
+ raise concurrent.futures.TimeoutError(
+ 'Operation did not complete within the designated '
+ 'timeout.')
+
+ def result(self, timeout=None):
+ """Get the result of the operation, blocking if necessary.
+
+ Args:
+ timeout (int):
+ How long (in seconds) to wait for the operation to complete.
+ If None, wait indefinitely.
+
+ Returns:
+ google.protobuf.Message: The Operation's result.
+
+ Raises:
+ google.gax.GaxError: If the operation errors or if the timeout is
+ reached before the operation completes.
+ """
+ self._blocking_poll(timeout=timeout)
+
+ if self._exception is not None:
+ # pylint: disable=raising-bad-type
+ # Pylint doesn't recognize that this is valid in this case.
+ raise self._exception
+
+ return self._result
+
+ def exception(self, timeout=None):
+ """Get the exception from the operation, blocking if necessary.
+
+ Args:
+ timeout (int): How long to wait for the operation to complete.
+ If None, wait indefinitely.
+
+ Returns:
+ Optional[google.gax.GaxError]: The operation's error.
+ """
+ self._blocking_poll()
+ return self._exception
+
+ def add_done_callback(self, fn):
+ """Add a callback to be executed when the operation is complete.
+
+ If the operation is not already complete, this will start a helper
+ thread to poll for the status of the operation in the background.
+
+ Args:
+ fn (Callable[Future]): The callback to execute when the operation
+ is complete.
+ """
+ if self._result_set:
+ _helpers.safe_invoke_callback(fn, self)
+ return
+
+ self._done_callbacks.append(fn)
+
+ if self._polling_thread is None:
+ # The polling thread will exit on its own as soon as the operation
+ # is done.
+ self._polling_thread = _helpers.start_daemon_thread(
+ target=self._blocking_poll)
+
+ def _invoke_callbacks(self, *args, **kwargs):
+ """Invoke all done callbacks."""
+ for callback in self._done_callbacks:
+ _helpers.safe_invoke_callback(callback, *args, **kwargs)
+
+ def set_result(self, result):
+ """Set the Future's result."""
+ self._result = result
+ self._result_set = True
+ self._invoke_callbacks(self)
+
+ def set_exception(self, exception):
+ """Set the Future's exception."""
+ self._exception = exception
+ self._result_set = True
+ self._invoke_callbacks(self)
diff --git a/google/api_core/gapic_v1/__init__.py b/google/api_core/gapic_v1/__init__.py
new file mode 100644
index 0000000..7a588cf
--- /dev/null
+++ b/google/api_core/gapic_v1/__init__.py
@@ -0,0 +1,21 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from google.api_core.gapic_v1 import config
+from google.api_core.gapic_v1 import method
+
+__all__ = [
+ 'config',
+ 'method',
+]
diff --git a/google/api_core/gapic_v1/config.py b/google/api_core/gapic_v1/config.py
new file mode 100644
index 0000000..376e4f9
--- /dev/null
+++ b/google/api_core/gapic_v1/config.py
@@ -0,0 +1,169 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for loading gapic configuration data.
+
+The Google API generator creates supplementary configuration for each RPC
+method to tell the client library how to deal with retries and timeouts.
+"""
+
+import collections
+
+import grpc
+import six
+
+from google.api_core import exceptions
+from google.api_core import retry
+from google.api_core import timeout
+
+
+_MILLIS_PER_SECOND = 1000.0
+
+
+def _exception_class_for_grpc_status_name(name):
+ """Returns the Google API exception class for a gRPC error code name.
+
+ Args:
+ name (str): The name of the gRPC status code, for example,
+ ``UNAVAILABLE``.
+
+ Returns:
+ :func:`type`: The appropriate subclass of
+ :class:`google.api_core.exceptions.GoogleAPICallError`.
+ """
+ return exceptions.exception_class_for_grpc_status(
+ getattr(grpc.StatusCode, name))
+
+
+def _retry_from_retry_config(retry_params, retry_codes):
+ """Creates a Retry object given a gapic retry configuration.
+
+ Args:
+ retry_params (dict): The retry parameter values, for example::
+
+ {
+ "initial_retry_delay_millis": 1000,
+ "retry_delay_multiplier": 2.5,
+ "max_retry_delay_millis": 120000,
+ "initial_rpc_timeout_millis": 120000,
+ "rpc_timeout_multiplier": 1.0,
+ "max_rpc_timeout_millis": 120000,
+ "total_timeout_millis": 600000
+ }
+
+ retry_codes (sequence[str]): The list of retryable gRPC error code
+ names.
+
+ Returns:
+ google.api_core.retry.Retry: The default retry object for the method.
+ """
+ exception_classes = [
+ _exception_class_for_grpc_status_name(code) for code in retry_codes]
+ return retry.Retry(
+ retry.if_exception_type(*exception_classes),
+ initial=(
+ retry_params['initial_retry_delay_millis'] / _MILLIS_PER_SECOND),
+ maximum=(
+ retry_params['max_retry_delay_millis'] / _MILLIS_PER_SECOND),
+ multiplier=retry_params['retry_delay_multiplier'],
+ deadline=retry_params['total_timeout_millis'] / _MILLIS_PER_SECOND)
+
+
+def _timeout_from_retry_config(retry_params):
+ """Creates a ExponentialTimeout object given a gapic retry configuration.
+
+ Args:
+ retry_params (dict): The retry parameter values, for example::
+
+ {
+ "initial_retry_delay_millis": 1000,
+ "retry_delay_multiplier": 2.5,
+ "max_retry_delay_millis": 120000,
+ "initial_rpc_timeout_millis": 120000,
+ "rpc_timeout_multiplier": 1.0,
+ "max_rpc_timeout_millis": 120000,
+ "total_timeout_millis": 600000
+ }
+
+ Returns:
+ google.api_core.retry.ExponentialTimeout: The default time object for
+ the method.
+ """
+ return timeout.ExponentialTimeout(
+ initial=(
+ retry_params['initial_rpc_timeout_millis'] / _MILLIS_PER_SECOND),
+ maximum=(
+ retry_params['max_rpc_timeout_millis'] / _MILLIS_PER_SECOND),
+ multiplier=retry_params['rpc_timeout_multiplier'],
+ deadline=(
+ retry_params['total_timeout_millis'] / _MILLIS_PER_SECOND))
+
+
+MethodConfig = collections.namedtuple('MethodConfig', ['retry', 'timeout'])
+
+
+def parse_method_configs(interface_config):
+ """Creates default retry and timeout objects for each method in a gapic
+ interface config.
+
+ Args:
+ interface_config (Mapping): The interface config section of the full
+ gapic library config. For example, If the full configuration has
+ an interface named ``google.example.v1.ExampleService`` you would
+ pass in just that interface's configuration, for example
+ ``gapic_config['interfaces']['google.example.v1.ExampleService']``.
+
+ Returns:
+ Mapping[str, MethodConfig]: A mapping of RPC method names to their
+ configuration.
+ """
+ # Grab all the retry codes
+ retry_codes_map = {
+ name: retry_codes
+ for name, retry_codes
+ in six.iteritems(interface_config.get('retry_codes', {}))
+ }
+
+ # Grab all of the retry params
+ retry_params_map = {
+ name: retry_params
+ for name, retry_params
+ in six.iteritems(interface_config.get('retry_params', {}))
+ }
+
+ # Iterate through all the API methods and create a flat MethodConfig
+ # instance for each one.
+ method_configs = {}
+
+ for method_name, method_params in six.iteritems(
+ interface_config.get('methods', {})):
+ retry_params_name = method_params.get('retry_params_name')
+
+ if retry_params_name is not None:
+ retry_params = retry_params_map[retry_params_name]
+ retry_ = _retry_from_retry_config(
+ retry_params,
+ retry_codes_map[method_params['retry_codes_name']])
+ timeout_ = _timeout_from_retry_config(retry_params)
+
+ # No retry config, so this is a non-retryable method.
+ else:
+ retry_ = None
+ timeout_ = timeout.ConstantTimeout(
+ method_params['timeout_millis'] / _MILLIS_PER_SECOND)
+
+ method_configs[method_name] = MethodConfig(
+ retry=retry_, timeout=timeout_)
+
+ return method_configs
diff --git a/google/api_core/gapic_v1/method.py b/google/api_core/gapic_v1/method.py
new file mode 100644
index 0000000..88e5a57
--- /dev/null
+++ b/google/api_core/gapic_v1/method.py
@@ -0,0 +1,292 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for wrapping low-level gRPC methods with common functionality.
+
+This is used by gapic clients to provide common error mapping, retry, timeout,
+pagination, and long-running operations to gRPC methods.
+"""
+
+import functools
+import platform
+
+import pkg_resources
+import six
+
+from google.api_core import general_helpers
+from google.api_core import grpc_helpers
+from google.api_core import page_iterator
+from google.api_core import timeout
+
+_PY_VERSION = platform.python_version()
+_GRPC_VERSION = pkg_resources.get_distribution('grpcio').version
+_API_CORE_VERSION = pkg_resources.get_distribution('google-api-core').version
+METRICS_METADATA_KEY = 'x-goog-api-client'
+USE_DEFAULT_METADATA = object()
+DEFAULT = object()
+"""Sentinel value indicating that a retry or timeout argument was unspecified,
+so the default should be used."""
+
+
+def _is_not_none_or_false(value):
+ return value is not None and value is not False
+
+
+def _apply_decorators(func, decorators):
+ """Apply a list of decorators to a given function.
+
+ ``decorators`` may contain items that are ``None`` or ``False`` which will
+ be ignored.
+ """
+ decorators = filter(_is_not_none_or_false, reversed(decorators))
+
+ for decorator in decorators:
+ func = decorator(func)
+
+ return func
+
+
+def _prepare_metadata(metadata):
+ """Transforms metadata to gRPC format and adds global metrics.
+
+ Args:
+ metadata (Mapping[str, str]): Any current metadata.
+
+ Returns:
+ Sequence[Tuple(str, str)]: The gRPC-friendly metadata keys and values.
+ """
+ client_metadata = 'api-core/{} gl-python/{} grpc/{}'.format(
+ _API_CORE_VERSION, _PY_VERSION, _GRPC_VERSION)
+
+ # Merge this with any existing metric metadata.
+ if METRICS_METADATA_KEY in metadata:
+ client_metadata = '{} {}'.format(
+ client_metadata, metadata[METRICS_METADATA_KEY])
+
+ metadata[METRICS_METADATA_KEY] = client_metadata
+
+ return list(metadata.items())
+
+
+def _determine_timeout(default_timeout, specified_timeout, retry):
+ """Determines how timeout should be applied to a wrapped method.
+
+ Args:
+ default_timeout (Optional[Timeout]): The default timeout specified
+ at method creation time.
+ specified_timeout (Optional[Timeout]): The timeout specified at
+ invocation time. If :attr:`DEFAULT`, this will be set to
+ the ``default_timeout``.
+ retry (Optional[Retry]): The retry specified at invocation time.
+
+ Returns:
+ Optional[Timeout]: The timeout to apply to the method or ``None``.
+ """
+ if specified_timeout is DEFAULT:
+ specified_timeout = default_timeout
+
+ if specified_timeout is default_timeout:
+ # If timeout is the default and the default timeout is exponential and
+ # a non-default retry is specified, make sure the timeout's deadline
+ # matches the retry's. This handles the case where the user leaves
+ # the timeout default but specifies a lower deadline via the retry.
+ if (retry and retry is not DEFAULT
+ and isinstance(default_timeout, timeout.ExponentialTimeout)):
+ return default_timeout.with_deadline(retry._deadline)
+ else:
+ return default_timeout
+
+ # If timeout is specified as a number instead of a Timeout instance,
+ # convert it to a ConstantTimeout.
+ if isinstance(specified_timeout, (int, float)):
+ return timeout.ConstantTimeout(specified_timeout)
+ else:
+ return specified_timeout
+
+
+class _GapicCallable(object):
+ """Callable that applies retry, timeout, and metadata logic.
+
+ Args:
+ target (Callable): The low-level RPC method.
+ retry (google.api_core.retry.Retry): The default retry for the
+ callable. If ``None``, this callable will not retry by default
+ timeout (google.api_core.timeout.Timeout): The default timeout
+ for the callable. If ``None``, this callable will not specify
+ a timeout argument to the low-level RPC method by default.
+ metadata (Optional[Sequence[Tuple[str, str]]]): gRPC call metadata
+ that's passed to the low-level RPC method. If ``None``, no metadata
+ will be passed to the low-level RPC method.
+ """
+
+ def __init__(self, target, retry, timeout, metadata):
+ self._target = target
+ self._retry = retry
+ self._timeout = timeout
+ self._metadata = metadata
+
+ def __call__(self, *args, **kwargs):
+ """Invoke the low-level RPC with retry, timeout, and metadata."""
+ # Note: Due to Python 2 lacking keyword-only arguments we use kwargs to
+ # extract the retry and timeout params.
+ timeout_ = _determine_timeout(
+ self._timeout,
+ kwargs.pop('timeout', self._timeout),
+ # Use only the invocation-specified retry only for this, as we only
+ # want to adjust the timeout deadline if the *user* specified
+ # a different retry.
+ kwargs.get('retry', None))
+
+ retry = kwargs.pop('retry', self._retry)
+
+ if retry is DEFAULT:
+ retry = self._retry
+
+ # Apply all applicable decorators.
+ wrapped_func = _apply_decorators(self._target, [retry, timeout_])
+
+ # Set the metadata for the call using the metadata calculated by
+ # _prepare_metadata.
+ if self._metadata is not None:
+ kwargs['metadata'] = self._metadata
+
+ return wrapped_func(*args, **kwargs)
+
+
+def wrap_method(
+ func, default_retry=None, default_timeout=None,
+ metadata=USE_DEFAULT_METADATA):
+ """Wrap an RPC method with common behavior.
+
+ This applies common error wrapping, retry, and timeout behavior a function.
+ The wrapped function will take optional ``retry`` and ``timeout``
+ arguments.
+
+ For example::
+
+ import google.api_core.gapic_v1.method
+ from google.api_core import retry
+ from google.api_core import timeout
+
+ # The original RPC method.
+ def get_topic(name, timeout=None):
+ request = publisher_v2.GetTopicRequest(name=name)
+ return publisher_stub.GetTopic(request, timeout=timeout)
+
+ default_retry = retry.Retry(deadline=60)
+ default_timeout = timeout.Timeout(deadline=60)
+ wrapped_get_topic = google.api_core.gapic_v1.method.wrap_method(
+ get_topic, default_retry)
+
+ # Execute get_topic with default retry and timeout:
+ response = wrapped_get_topic()
+
+ # Execute get_topic without doing any retying but with the default
+ # timeout:
+ response = wrapped_get_topic(retry=None)
+
+ # Execute get_topic but only retry on 5xx errors:
+ my_retry = retry.Retry(retry.if_exception_type(
+ exceptions.InternalServerError))
+ response = wrapped_get_topic(retry=my_retry)
+
+ The way this works is by late-wrapping the given function with the retry
+ and timeout decorators. Essentially, when ``wrapped_get_topic()`` is
+ called:
+
+ * ``get_topic()`` is first wrapped with the ``timeout`` into
+ ``get_topic_with_timeout``.
+ * ``get_topic_with_timeout`` is wrapped with the ``retry`` into
+ ``get_topic_with_timeout_and_retry()``.
+ * The final ``get_topic_with_timeout_and_retry`` is called passing through
+ the ``args`` and ``kwargs``.
+
+ The callstack is therefore::
+
+ method.__call__() ->
+ Retry.__call__() ->
+ Timeout.__call__() ->
+ wrap_errors() ->
+ get_topic()
+
+ Note that if ``timeout`` or ``retry`` is ``None``, then they are not
+ applied to the function. For example,
+ ``wrapped_get_topic(timeout=None, retry=None)`` is more or less
+ equivalent to just calling ``get_topic`` but with error re-mapping.
+
+ Args:
+ func (Callable[Any]): The function to wrap. It should accept an
+ optional ``timeout`` argument. If ``metadata`` is not ``None``, it
+ should accept a ``metadata`` argument.
+ default_retry (Optional[google.api_core.Retry]): The default retry
+ strategy. If ``None``, the method will not retry by default.
+ default_timeout (Optional[google.api_core.Timeout]): The default
+ timeout strategy. Can also be specified as an int or float. If
+ ``None``, the method will not have timeout specified by default.
+ metadata (Optional(Mapping[str, str])): A dict of metadata keys and
+ values. This will be augmented with common ``x-google-api-client``
+ metadata. If ``None``, metadata will not be passed to the function
+ at all, if :attr:`USE_DEFAULT_METADATA` (the default) then only the
+ common metadata will be provided.
+
+ Returns:
+ Callable: A new callable that takes optional ``retry`` and ``timeout``
+ arguments and applies the common error mapping, retry, timeout,
+ and metadata behavior to the low-level RPC method.
+ """
+ func = grpc_helpers.wrap_errors(func)
+
+ if metadata is USE_DEFAULT_METADATA:
+ metadata = {}
+
+ if metadata is not None:
+ metadata = _prepare_metadata(metadata)
+
+ return general_helpers.wraps(func)(
+ _GapicCallable(func, default_retry, default_timeout, metadata))
+
+
+def wrap_with_paging(
+ func, items_field, request_token_field, response_token_field):
+ """Wrap an RPC method to return a page iterator.
+
+ Args:
+ func (Callable): The RPC method. This should already have been
+ wrapped with common functionality using :func:`wrap_method`.
+ request (protobuf.Message): The request message.
+ items_field (str): The field in the response message that has the
+ items for the page.
+ request_token_field (str): The field in the request message used to
+ specify the page token.
+ response_token_field (str): The field in the response message that has
+ the token for the next page.
+
+ Returns:
+ Callable: Returns a callable that when invoked will call the RPC
+ method and return a
+ :class:`google.api_core.page_iterator.Iterator`.
+ """
+ @six.wraps(func)
+ def paged_method(request, **kwargs):
+ """Wrapper that invokes a method and returns a page iterator."""
+ iterator = page_iterator.GRPCIterator(
+ client=None,
+ method=functools.partial(func, **kwargs),
+ request=request,
+ items_field=items_field,
+ request_token_field=request_token_field,
+ response_token_field=response_token_field)
+ return iterator
+
+ return paged_method
diff --git a/google/api_core/general_helpers.py b/google/api_core/general_helpers.py
new file mode 100644
index 0000000..0c8e408
--- /dev/null
+++ b/google/api_core/general_helpers.py
@@ -0,0 +1,32 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for general Python functionality."""
+
+import functools
+
+import six
+
+
+# functools.partial objects lack several attributes present on real function
+# objects. In Python 2 wraps fails on this so use a restricted set instead.
+_PARTIAL_VALID_ASSIGNMENTS = ('__doc__',)
+
+
+def wraps(wrapped):
+ """A functools.wraps helper that handles partial objects on Python 2."""
+ if isinstance(wrapped, functools.partial):
+ return six.wraps(wrapped, assigned=_PARTIAL_VALID_ASSIGNMENTS)
+ else:
+ return six.wraps(wrapped)
diff --git a/google/api_core/grpc_helpers.py b/google/api_core/grpc_helpers.py
new file mode 100644
index 0000000..e4781db
--- /dev/null
+++ b/google/api_core/grpc_helpers.py
@@ -0,0 +1,134 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for :mod:`grpc`."""
+
+import grpc
+import six
+
+from google.api_core import exceptions
+from google.api_core import general_helpers
+import google.auth
+import google.auth.transport.grpc
+import google.auth.transport.requests
+
+
+# The list of gRPC Callable interfaces that return iterators.
+_STREAM_WRAP_CLASSES = (
+ grpc.UnaryStreamMultiCallable,
+ grpc.StreamStreamMultiCallable,
+)
+
+
+def _patch_callable_name(callable_):
+ """Fix-up gRPC callable attributes.
+
+ gRPC callable lack the ``__name__`` attribute which causes
+ :func:`functools.wraps` to error. This adds the attribute if needed.
+ """
+ if not hasattr(callable_, '__name__'):
+ callable_.__name__ = callable_.__class__.__name__
+
+
+def _wrap_unary_errors(callable_):
+ """Map errors for Unary-Unary and Stream-Unary gRPC callables."""
+ _patch_callable_name(callable_)
+
+ @six.wraps(callable_)
+ def error_remapped_callable(*args, **kwargs):
+ try:
+ return callable_(*args, **kwargs)
+ except grpc.RpcError as exc:
+ six.raise_from(exceptions.from_grpc_error(exc), exc)
+
+ return error_remapped_callable
+
+
+def _wrap_stream_errors(callable_):
+ """Wrap errors for Unary-Stream and Stream-Stream gRPC callables.
+
+ The callables that return iterators require a bit more logic to re-map
+ errors when iterating. This wraps both the initial invocation and the
+ iterator of the return value to re-map errors.
+ """
+ _patch_callable_name(callable_)
+
+ @general_helpers.wraps(callable_)
+ def error_remapped_callable(*args, **kwargs):
+ try:
+ result = callable_(*args, **kwargs)
+ # Note: we are patching the private grpc._channel._Rendezvous._next
+ # method as magic methods (__next__ in this case) can not be
+ # patched on a per-instance basis (see
+ # https://docs.python.org/3/reference/datamodel.html
+ # #special-lookup).
+ # In an ideal world, gRPC would return a *specific* interface
+ # from *StreamMultiCallables, but they return a God class that's
+ # a combination of basically every interface in gRPC making it
+ # untenable for us to implement a wrapper object using the same
+ # interface.
+ result._next = _wrap_unary_errors(result._next)
+ return result
+ except grpc.RpcError as exc:
+ six.raise_from(exceptions.from_grpc_error(exc), exc)
+
+ return error_remapped_callable
+
+
+def wrap_errors(callable_):
+ """Wrap a gRPC callable and map :class:`grpc.RpcErrors` to friendly error
+ classes.
+
+ Errors raised by the gRPC callable are mapped to the appropriate
+ :class:`google.api_core.exceptions.GoogleAPICallError` subclasses.
+ The original `grpc.RpcError` (which is usually also a `grpc.Call`) is
+ available from the ``response`` property on the mapped exception. This
+ is useful for extracting metadata from the original error.
+
+ Args:
+ callable_ (Callable): A gRPC callable.
+
+ Returns:
+ Callable: The wrapped gRPC callable.
+ """
+ if isinstance(callable_, _STREAM_WRAP_CLASSES):
+ return _wrap_stream_errors(callable_)
+ else:
+ return _wrap_unary_errors(callable_)
+
+
+def create_channel(target, credentials=None, scopes=None, **kwargs):
+ """Create a secure channel with credentials.
+
+ Args:
+ target (str): The target service address in the format 'hostname:port'.
+ credentials (google.auth.credentials.Credentials): The credentials. If
+ not specified, then this function will attempt to ascertain the
+ credentials from the environment using :func:`google.auth.default`.
+ scopes (Sequence[str]): A optional list of scopes needed for this
+ service. These are only used when credentials are not specified and
+ are passed to :func:`google.auth.default`.
+ kwargs: Additional key-word args passed to
+ :func:`google.auth.transport.grpc.secure_authorized_channel`.
+
+ Returns:
+ grpc.Channel: The created channel.
+ """
+ if credentials is None:
+ credentials, _ = google.auth.default(scopes=scopes)
+
+ request = google.auth.transport.requests.Request()
+
+ return google.auth.transport.grpc.secure_authorized_channel(
+ credentials, request, target, **kwargs)
diff --git a/google/api_core/operation.py b/google/api_core/operation.py
new file mode 100644
index 0000000..2136d95
--- /dev/null
+++ b/google/api_core/operation.py
@@ -0,0 +1,297 @@
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Futures for long-running operations returned from Google Cloud APIs.
+
+These futures can be used to synchronously wait for the result of a
+long-running operation using :meth:`Operation.result`:
+
+
+.. code-block:: python
+
+ operation = my_api_client.long_running_method()
+ result = operation.result()
+
+Or asynchronously using callbacks and :meth:`Operation.add_done_callback`:
+
+.. code-block:: python
+
+ operation = my_api_client.long_running_method()
+
+ def my_callback(future):
+ result = future.result()
+
+ operation.add_done_callback(my_callback)
+
+"""
+
+import functools
+import threading
+
+from google.api_core import exceptions
+from google.api_core import protobuf_helpers
+from google.api_core.future import polling
+from google.longrunning import operations_pb2
+from google.protobuf import json_format
+from google.rpc import code_pb2
+
+
+class Operation(polling.PollingFuture):
+ """A Future for interacting with a Google API Long-Running Operation.
+
+ Args:
+ operation (google.longrunning.operations_pb2.Operation): The
+ initial operation.
+ refresh (Callable[[], Operation]): A callable that returns the
+ latest state of the operation.
+ cancel (Callable[[], None]): A callable that tries to cancel
+ the operation.
+ result_type (func:`type`): The protobuf type for the operation's
+ result.
+ metadata_type (func:`type`): The protobuf type for the operation's
+ metadata.
+ """
+
+ def __init__(
+ self, operation, refresh, cancel,
+ result_type, metadata_type=None):
+ super(Operation, self).__init__()
+ self._operation = operation
+ self._refresh = refresh
+ self._cancel = cancel
+ self._result_type = result_type
+ self._metadata_type = metadata_type
+ self._completion_lock = threading.Lock()
+ # Invoke this in case the operation came back already complete.
+ self._set_result_from_operation()
+
+ @property
+ def operation(self):
+ """google.longrunning.Operation: The current long-running operation."""
+ return self._operation
+
+ @property
+ def metadata(self):
+ """google.protobuf.Message: the current operation metadata."""
+ if not self._operation.HasField('metadata'):
+ return None
+
+ return protobuf_helpers.from_any_pb(
+ self._metadata_type, self._operation.metadata)
+
+ def _set_result_from_operation(self):
+ """Set the result or exception from the operation if it is complete."""
+ # This must be done in a lock to prevent the polling thread
+ # and main thread from both executing the completion logic
+ # at the same time.
+ with self._completion_lock:
+ # If the operation isn't complete or if the result has already been
+ # set, do not call set_result/set_exception again.
+ # Note: self._result_set is set to True in set_result and
+ # set_exception, in case those methods are invoked directly.
+ if not self._operation.done or self._result_set:
+ return
+
+ if self._operation.HasField('response'):
+ response = protobuf_helpers.from_any_pb(
+ self._result_type, self._operation.response)
+ self.set_result(response)
+ elif self._operation.HasField('error'):
+ exception = exceptions.GoogleAPICallError(
+ self._operation.error.message,
+ errors=(self._operation.error),
+ response=self._operation)
+ self.set_exception(exception)
+ else:
+ exception = exceptions.GoogleAPICallError(
+ 'Unexpected state: Long-running operation had neither '
+ 'response nor error set.')
+ self.set_exception(exception)
+
+ def _refresh_and_update(self):
+ """Refresh the operation and update the result if needed."""
+ # If the currently cached operation is done, no need to make another
+ # RPC as it will not change once done.
+ if not self._operation.done:
+ self._operation = self._refresh()
+ self._set_result_from_operation()
+
+ def done(self):
+ """Checks to see if the operation is complete.
+
+ Returns:
+ bool: True if the operation is complete, False otherwise.
+ """
+ self._refresh_and_update()
+ return self._operation.done
+
+ def cancel(self):
+ """Attempt to cancel the operation.
+
+ Returns:
+ bool: True if the cancel RPC was made, False if the operation is
+ already complete.
+ """
+ if self.done():
+ return False
+
+ self._cancel()
+ return True
+
+ def cancelled(self):
+ """True if the operation was cancelled."""
+ self._refresh_and_update()
+ return (self._operation.HasField('error') and
+ self._operation.error.code == code_pb2.CANCELLED)
+
+
+def _refresh_http(api_request, operation_name):
+ """Refresh an operation using a JSON/HTTP client.
+
+ Args:
+ api_request (Callable): A callable used to make an API request. This
+ should generally be
+ :meth:`google.cloud._http.Connection.api_request`.
+ operation_name (str): The name of the operation.
+
+ Returns:
+ google.longrunning.operations_pb2.Operation: The operation.
+ """
+ path = 'operations/{}'.format(operation_name)
+ api_response = api_request(method='GET', path=path)
+ return json_format.ParseDict(
+ api_response, operations_pb2.Operation())
+
+
+def _cancel_http(api_request, operation_name):
+ """Cancel an operation using a JSON/HTTP client.
+
+ Args:
+ api_request (Callable): A callable used to make an API request. This
+ should generally be
+ :meth:`google.cloud._http.Connection.api_request`.
+ operation_name (str): The name of the operation.
+ """
+ path = 'operations/{}:cancel'.format(operation_name)
+ api_request(method='POST', path=path)
+
+
+def from_http_json(operation, api_request, result_type, **kwargs):
+ """Create an operation future using a HTTP/JSON client.
+
+ This interacts with the long-running operations `service`_ (specific
+ to a given API) vis `HTTP/JSON`_.
+
+ .. _HTTP/JSON: https://cloud.google.com/speech/reference/rest/\
+ v1beta1/operations#Operation
+
+ Args:
+ operation (dict): Operation as a dictionary.
+ api_request (Callable): A callable used to make an API request. This
+ should generally be
+ :meth:`google.cloud._http.Connection.api_request`.
+ result_type (:func:`type`): The protobuf result type.
+ kwargs: Keyword args passed into the :class:`Operation` constructor.
+
+ Returns:
+ Operation: The operation future to track the given operation.
+ """
+ operation_proto = json_format.ParseDict(
+ operation, operations_pb2.Operation())
+ refresh = functools.partial(
+ _refresh_http, api_request, operation_proto.name)
+ cancel = functools.partial(
+ _cancel_http, api_request, operation_proto.name)
+ return Operation(operation_proto, refresh, cancel, result_type, **kwargs)
+
+
+def _refresh_grpc(operations_stub, operation_name):
+ """Refresh an operation using a gRPC client.
+
+ Args:
+ operations_stub (google.longrunning.operations_pb2.OperationsStub):
+ The gRPC operations stub.
+ operation_name (str): The name of the operation.
+
+ Returns:
+ google.longrunning.operations_pb2.Operation: The operation.
+ """
+ request_pb = operations_pb2.GetOperationRequest(name=operation_name)
+ return operations_stub.GetOperation(request_pb)
+
+
+def _cancel_grpc(operations_stub, operation_name):
+ """Cancel an operation using a gRPC client.
+
+ Args:
+ operations_stub (google.longrunning.operations_pb2.OperationsStub):
+ The gRPC operations stub.
+ operation_name (str): The name of the operation.
+ """
+ request_pb = operations_pb2.CancelOperationRequest(name=operation_name)
+ operations_stub.CancelOperation(request_pb)
+
+
+def from_grpc(operation, operations_stub, result_type, **kwargs):
+ """Create an operation future using a gRPC client.
+
+ This interacts with the long-running operations `service`_ (specific
+ to a given API) via gRPC.
+
+ .. _service: https://github.com/googleapis/googleapis/blob/\
+ 050400df0fdb16f63b63e9dee53819044bffc857/\
+ google/longrunning/operations.proto#L38
+
+ Args:
+ operation (google.longrunning.operations_pb2.Operation): The operation.
+ operations_stub (google.longrunning.operations_pb2.OperationsStub):
+ The operations stub.
+ result_type (:func:`type`): The protobuf result type.
+ kwargs: Keyword args passed into the :class:`Operation` constructor.
+
+ Returns:
+ Operation: The operation future to track the given operation.
+ """
+ refresh = functools.partial(
+ _refresh_grpc, operations_stub, operation.name)
+ cancel = functools.partial(
+ _cancel_grpc, operations_stub, operation.name)
+ return Operation(operation, refresh, cancel, result_type, **kwargs)
+
+
+def from_gapic(operation, operations_client, result_type, **kwargs):
+ """Create an operation future from a gapic client.
+
+ This interacts with the long-running operations `service`_ (specific
+ to a given API) via a gapic client.
+
+ .. _service: https://github.com/googleapis/googleapis/blob/\
+ 050400df0fdb16f63b63e9dee53819044bffc857/\
+ google/longrunning/operations.proto#L38
+
+ Args:
+ operation (google.longrunning.operations_pb2.Operation): The operation.
+ operations_client (google.api_core.operations_v1.OperationsClient):
+ The operations client.
+ result_type (:func:`type`): The protobuf result type.
+ kwargs: Keyword args passed into the :class:`Operation` constructor.
+
+ Returns:
+ Operation: The operation future to track the given operation.
+ """
+ refresh = functools.partial(
+ operations_client.get_operation, operation.name)
+ cancel = functools.partial(
+ operations_client.cancel_operation, operation.name)
+ return Operation(operation, refresh, cancel, result_type, **kwargs)
diff --git a/google/api_core/operations_v1/__init__.py b/google/api_core/operations_v1/__init__.py
new file mode 100644
index 0000000..e9f88f4
--- /dev/null
+++ b/google/api_core/operations_v1/__init__.py
@@ -0,0 +1,21 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Package for interacting with the google.longrunning.operations meta-API."""
+
+from google.api_core.operations_v1.operations_client import OperationsClient
+
+__all__ = [
+ 'OperationsClient'
+]
diff --git a/google/api_core/operations_v1/operations_client.py b/google/api_core/operations_v1/operations_client.py
new file mode 100644
index 0000000..93a823f
--- /dev/null
+++ b/google/api_core/operations_v1/operations_client.py
@@ -0,0 +1,271 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A client for the google.longrunning.operations meta-API.
+
+This is a client that deals with long-running operations that follow the
+pattern outlined by the `Google API Style Guide`_.
+
+When an API method normally takes long time to complete, it can be designed to
+return ``Operation`` to the client, and the client can use this interface to
+receive the real response asynchronously by polling the operation resource to
+receive the response.
+
+It is not a separate service, but rather an interface implemented by a larger
+service. The protocol-level definition is available at
+`google/longrunning/operations.proto`_. Typically, this will be constructed
+automatically by another client class to deal with operations.
+
+.. _Google API Style Guide:
+ https://cloud.google.com/apis/design/design_pattern
+ s#long_running_operations
+.. _google/longrunning/operations.proto:
+ https://github.com/googleapis/googleapis/blob/master/google/longrunning
+ /operations.proto
+"""
+
+from google.api_core import gapic_v1
+from google.api_core.operations_v1 import operations_client_config
+from google.longrunning import operations_pb2
+
+
+class OperationsClient(object):
+ """Client for interacting with long-running operations within a service.
+
+ Args:
+ channel (grpc.Channel): The gRPC channel associated with the service
+ that implements the ``google.longrunning.operations`` interface.
+ client_config (dict):
+ A dictionary of call options for each method. If not specified
+ the default configuration is used.
+ """
+
+ def __init__(self, channel, client_config=operations_client_config.config):
+ # Create the gRPC client stub.
+ self.operations_stub = operations_pb2.OperationsStub(channel)
+
+ # Create all wrapped methods using the interface configuration.
+ # The interface config contains all of the default settings for retry
+ # and timeout for each RPC method.
+ interfaces = client_config['interfaces']
+ interface_config = interfaces['google.longrunning.Operations']
+ method_configs = gapic_v1.config.parse_method_configs(interface_config)
+
+ self._get_operation = gapic_v1.method.wrap_method(
+ self.operations_stub.GetOperation,
+ default_retry=method_configs['GetOperation'].retry,
+ default_timeout=method_configs['GetOperation'].timeout)
+
+ self._list_operations = gapic_v1.method.wrap_method(
+ self.operations_stub.ListOperations,
+ default_retry=method_configs['ListOperations'].retry,
+ default_timeout=method_configs['ListOperations'].timeout)
+
+ self._list_operations = gapic_v1.method.wrap_with_paging(
+ self._list_operations,
+ 'operations',
+ 'page_token',
+ 'next_page_token')
+
+ self._cancel_operation = gapic_v1.method.wrap_method(
+ self.operations_stub.CancelOperation,
+ default_retry=method_configs['CancelOperation'].retry,
+ default_timeout=method_configs['CancelOperation'].timeout)
+
+ self._delete_operation = gapic_v1.method.wrap_method(
+ self.operations_stub.DeleteOperation,
+ default_retry=method_configs['DeleteOperation'].retry,
+ default_timeout=method_configs['DeleteOperation'].timeout)
+
+ # Service calls
+ def get_operation(
+ self, name,
+ retry=gapic_v1.method.DEFAULT, timeout=gapic_v1.method.DEFAULT):
+ """Gets the latest state of a long-running operation.
+
+ Clients can use this method to poll the operation result at intervals
+ as recommended by the API service.
+
+ Example:
+ >>> from google.api_core import operations_v1
+ >>> api = operations_v1.OperationsClient()
+ >>> name = ''
+ >>> response = api.get_operation(name)
+
+ Args:
+ name (str): The name of the operation resource.
+ retry (google.api_core.retry.Retry): The retry strategy to use
+ when invoking the RPC. If unspecified, the default retry from
+ the client configuration will be used. If ``None``, then this
+ method will not retry the RPC at all.
+ timeout (float): The amount of time in seconds to wait for the RPC
+ to complete. Note that if ``retry`` is used, this timeout
+ applies to each individual attempt and the overall time it
+ takes for this method to complete may be longer. If
+ unspecified, the the default timeout in the client
+ configuration is used. If ``None``, then the RPC method will
+ not time out.
+
+ Returns:
+ google.longrunning.operations_pb2.Operation: The state of the
+ operation.
+
+ Raises:
+ google.api_core.exceptions.GoogleAPICallError: If an error occurred
+ while invoking the RPC, the appropriate ``GoogleAPICallError``
+ subclass will be raised.
+ """
+ request = operations_pb2.GetOperationRequest(name=name)
+ return self._get_operation(request, retry=retry, timeout=timeout)
+
+ def list_operations(
+ self, name, filter_,
+ retry=gapic_v1.method.DEFAULT, timeout=gapic_v1.method.DEFAULT):
+ """
+ Lists operations that match the specified filter in the request.
+
+ Example:
+ >>> from google.api_core import operations_v1
+ >>> api = operations_v1.OperationsClient()
+ >>> name = ''
+ >>>
+ >>> # Iterate over all results
+ >>> for operation in api.list_operations(name):
+ >>> # process operation
+ >>> pass
+ >>>
+ >>> # Or iterate over results one page at a time
+ >>> iter = api.list_operations(name)
+ >>> for page in iter.pages:
+ >>> for operation in page:
+ >>> # process operation
+ >>> pass
+
+ Args:
+ name (str): The name of the operation collection.
+ filter_ (str): The standard list filter.
+ retry (google.api_core.retry.Retry): The retry strategy to use
+ when invoking the RPC. If unspecified, the default retry from
+ the client configuration will be used. If ``None``, then this
+ method will not retry the RPC at all.
+ timeout (float): The amount of time in seconds to wait for the RPC
+ to complete. Note that if ``retry`` is used, this timeout
+ applies to each individual attempt and the overall time it
+ takes for this method to complete may be longer. If
+ unspecified, the the default timeout in the client
+ configuration is used. If ``None``, then the RPC method will
+ not time out.
+
+ Returns:
+ google.api_core.page_iterator.Iterator: An iterator that yields
+ :class:`google.longrunning.operations_pb2.Operation` instances.
+
+ Raises:
+ google.api_core.exceptions.MethodNotImplemented: If the server
+ does not support this method. Services are not required to
+ implement this method.
+ google.api_core.exceptions.GoogleAPICallError: If an error occurred
+ while invoking the RPC, the appropriate ``GoogleAPICallError``
+ subclass will be raised.
+ """
+ # Create the request object.
+ request = operations_pb2.ListOperationsRequest(
+ name=name, filter=filter_)
+ return self._list_operations(request, retry=retry, timeout=timeout)
+
+ def cancel_operation(
+ self, name,
+ retry=gapic_v1.method.DEFAULT, timeout=gapic_v1.method.DEFAULT):
+ """Starts asynchronous cancellation on a long-running operation.
+
+ The server makes a best effort to cancel the operation, but success is
+ not guaranteed. Clients can use :meth:`get_operation` or service-
+ specific methods to check whether the cancellation succeeded or whether
+ the operation completed despite cancellation. On successful
+ cancellation, the operation is not deleted; instead, it becomes an
+ operation with an ``Operation.error`` value with a
+ ``google.rpc.Status.code`` of ``1``, corresponding to
+ ``Code.CANCELLED``.
+
+ Example:
+ >>> from google.api_core import operations_v1
+ >>> api = operations_v1.OperationsClient()
+ >>> name = ''
+ >>> api.cancel_operation(name)
+
+ Args:
+ name (str): The name of the operation resource to be cancelled.
+ retry (google.api_core.retry.Retry): The retry strategy to use
+ when invoking the RPC. If unspecified, the default retry from
+ the client configuration will be used. If ``None``, then this
+ method will not retry the RPC at all.
+ timeout (float): The amount of time in seconds to wait for the RPC
+ to complete. Note that if ``retry`` is used, this timeout
+ applies to each individual attempt and the overall time it
+ takes for this method to complete may be longer. If
+ unspecified, the the default timeout in the client
+ configuration is used. If ``None``, then the RPC method will
+ not time out.
+
+ Raises:
+ google.api_core.exceptions.MethodNotImplemented: If the server
+ does not support this method. Services are not required to
+ implement this method.
+ google.api_core.exceptions.GoogleAPICallError: If an error occurred
+ while invoking the RPC, the appropriate ``GoogleAPICallError``
+ subclass will be raised.
+ """
+ # Create the request object.
+ request = operations_pb2.CancelOperationRequest(name=name)
+ self._cancel_operation(request, retry=retry, timeout=timeout)
+
+ def delete_operation(
+ self, name,
+ retry=gapic_v1.method.DEFAULT, timeout=gapic_v1.method.DEFAULT):
+ """Deletes a long-running operation.
+
+ This method indicates that the client is no longer interested in the
+ operation result. It does not cancel the operation.
+
+ Example:
+ >>> from google.api_core import operations_v1
+ >>> api = operations_v1.OperationsClient()
+ >>> name = ''
+ >>> api.delete_operation(name)
+
+ Args:
+ name (str): The name of the operation resource to be deleted.
+ retry (google.api_core.retry.Retry): The retry strategy to use
+ when invoking the RPC. If unspecified, the default retry from
+ the client configuration will be used. If ``None``, then this
+ method will not retry the RPC at all.
+ timeout (float): The amount of time in seconds to wait for the RPC
+ to complete. Note that if ``retry`` is used, this timeout
+ applies to each individual attempt and the overall time it
+ takes for this method to complete may be longer. If
+ unspecified, the the default timeout in the client
+ configuration is used. If ``None``, then the RPC method will
+ not time out.
+
+ Raises:
+ google.api_core.exceptions.MethodNotImplemented: If the server
+ does not support this method. Services are not required to
+ implement this method.
+ google.api_core.exceptions.GoogleAPICallError: If an error occurred
+ while invoking the RPC, the appropriate ``GoogleAPICallError``
+ subclass will be raised.
+ """
+ # Create the request object.
+ request = operations_pb2.DeleteOperationRequest(name=name)
+ self._delete_operation(request, retry=retry, timeout=timeout)
diff --git a/google/api_core/operations_v1/operations_client_config.py b/google/api_core/operations_v1/operations_client_config.py
new file mode 100644
index 0000000..bd79fd5
--- /dev/null
+++ b/google/api_core/operations_v1/operations_client_config.py
@@ -0,0 +1,62 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""gapic configuration for the googe.longrunning.operations client."""
+
+config = {
+ "interfaces": {
+ "google.longrunning.Operations": {
+ "retry_codes": {
+ "idempotent": [
+ "DEADLINE_EXCEEDED",
+ "UNAVAILABLE"
+ ],
+ "non_idempotent": []
+ },
+ "retry_params": {
+ "default": {
+ "initial_retry_delay_millis": 100,
+ "retry_delay_multiplier": 1.3,
+ "max_retry_delay_millis": 60000,
+ "initial_rpc_timeout_millis": 20000,
+ "rpc_timeout_multiplier": 1.0,
+ "max_rpc_timeout_millis": 600000,
+ "total_timeout_millis": 600000
+ }
+ },
+ "methods": {
+ "GetOperation": {
+ "timeout_millis": 60000,
+ "retry_codes_name": "idempotent",
+ "retry_params_name": "default"
+ },
+ "ListOperations": {
+ "timeout_millis": 60000,
+ "retry_codes_name": "idempotent",
+ "retry_params_name": "default"
+ },
+ "CancelOperation": {
+ "timeout_millis": 60000,
+ "retry_codes_name": "idempotent",
+ "retry_params_name": "default"
+ },
+ "DeleteOperation": {
+ "timeout_millis": 60000,
+ "retry_codes_name": "idempotent",
+ "retry_params_name": "default"
+ }
+ }
+ }
+ }
+}
diff --git a/google/api_core/page_iterator.py b/google/api_core/page_iterator.py
new file mode 100644
index 0000000..c3e5a6d
--- /dev/null
+++ b/google/api_core/page_iterator.py
@@ -0,0 +1,522 @@
+# Copyright 2015 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Iterators for paging through paged API methods.
+
+These iterators simplify the process of paging through API responses
+where the request takes a page token and the response is a list of results with
+a token for the next page. See `list pagination`_ in the Google API Style Guide
+for more details.
+
+.. _list pagination:
+ https://cloud.google.com/apis/design/design_patterns#list_pagination
+
+API clients that have methods that follow the list pagination pattern can
+return an :class:`.Iterator`. You can use this iterator to get **all** of
+the results across all pages::
+
+ >>> results_iterator = client.list_resources()
+ >>> list(results_iterator) # Convert to a list (consumes all values).
+
+Or you can walk your way through items and call off the search early if
+you find what you're looking for (resulting in possibly fewer requests)::
+
+ >>> for resource in results_iterator:
+ ... print(resource.name)
+ ... if not resource.is_valid:
+ ... break
+
+At any point, you may check the number of items consumed by referencing the
+``num_results`` property of the iterator::
+
+ >>> for my_item in results_iterator:
+ ... if results_iterator.num_results >= 10:
+ ... break
+
+When iterating, not every new item will send a request to the server.
+To iterate based on each page of items (where a page corresponds to
+a request)::
+
+ >>> for page in results_iterator.pages:
+ ... print('=' * 20)
+ ... print(' Page number: {:d}'.format(iterator.page_number))
+ ... print(' Items in page: {:d}'.format(page.num_items))
+ ... print(' First item: {!r}'.format(next(page)))
+ ... print('Items remaining: {:d}'.format(page.remaining))
+ ... print('Next page token: {}'.format(iterator.next_page_token))
+ ====================
+ Page number: 1
+ Items in page: 1
+ First item: <MyItemClass at 0x7f1d3cccf690>
+ Items remaining: 0
+ Next page token: eav1OzQB0OM8rLdGXOEsyQWSG
+ ====================
+ Page number: 2
+ Items in page: 19
+ First item: <MyItemClass at 0x7f1d3cccffd0>
+ Items remaining: 18
+ Next page token: None
+
+Then, for each page you can get all the resources on that page by iterating
+through it or using :func:`list`::
+
+ >>> list(page)
+ [
+ <MyItemClass at 0x7fd64a098ad0>,
+ <MyItemClass at 0x7fd64a098ed0>,
+ <MyItemClass at 0x7fd64a098e90>,
+ ]
+"""
+
+import abc
+
+import six
+
+
+class Page(object):
+ """Single page of results in an iterator.
+
+ Args:
+ parent (google.api_core.page_iterator.Iterator): The iterator that owns
+ the current page.
+ items (Sequence[Any]): An iterable (that also defines __len__) of items
+ from a raw API response.
+ item_to_value (Callable[google.api_core.page_iterator.Iterator, Any]):
+ Callable to convert an item from the type in the raw API response
+ into the native object. Will be called with the iterator and a
+ single item.
+ """
+
+ def __init__(self, parent, items, item_to_value):
+ self._parent = parent
+ self._num_items = len(items)
+ self._remaining = self._num_items
+ self._item_iter = iter(items)
+ self._item_to_value = item_to_value
+
+ @property
+ def num_items(self):
+ """int: Total items in the page."""
+ return self._num_items
+
+ @property
+ def remaining(self):
+ """int: Remaining items in the page."""
+ return self._remaining
+
+ def __iter__(self):
+ """The :class:`Page` is an iterator of items."""
+ return self
+
+ def next(self):
+ """Get the next value in the page."""
+ item = six.next(self._item_iter)
+ result = self._item_to_value(self._parent, item)
+ # Since we've successfully got the next value from the
+ # iterator, we update the number of remaining.
+ self._remaining -= 1
+ return result
+
+ # Alias needed for Python 2/3 support.
+ __next__ = next
+
+
+def _item_to_value_identity(iterator, item):
+ """An item to value transformer that returns the item un-changed."""
+ # pylint: disable=unused-argument
+ # We are conforming to the interface defined by Iterator.
+ return item
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Iterator(object):
+ """A generic class for iterating through API list responses.
+
+ Args:
+ client(google.cloud.client.Client): The API client.
+ item_to_value (Callable[google.api_core.page_iterator.Iterator, Any]):
+ Callable to convert an item from the type in the raw API response
+ into the native object. Will be called with the iterator and a
+ single item.
+ page_token (str): A token identifying a page in a result set to start
+ fetching results from.
+ max_results (int): The maximum number of results to fetch.
+ """
+
+ def __init__(self, client, item_to_value=_item_to_value_identity,
+ page_token=None, max_results=None):
+ self._started = False
+ self.client = client
+ self._item_to_value = item_to_value
+ self.max_results = max_results
+ # The attributes below will change over the life of the iterator.
+ self.page_number = 0
+ self.next_page_token = page_token
+ self.num_results = 0
+
+ @property
+ def pages(self):
+ """Iterator of pages in the response.
+
+ returns:
+ types.GeneratorType[google.api_core.page_iterator.Page]: A
+ generator of page instances.
+
+ raises:
+ ValueError: If the iterator has already been started.
+ """
+ if self._started:
+ raise ValueError('Iterator has already started', self)
+ self._started = True
+ return self._page_iter(increment=True)
+
+ def _items_iter(self):
+ """Iterator for each item returned."""
+ for page in self._page_iter(increment=False):
+ for item in page:
+ self.num_results += 1
+ yield item
+
+ def __iter__(self):
+ """Iterator for each item returned.
+
+ Returns:
+ types.GeneratorType[Any]: A generator of items from the API.
+
+ Raises:
+ ValueError: If the iterator has already been started.
+ """
+ if self._started:
+ raise ValueError('Iterator has already started', self)
+ self._started = True
+ return self._items_iter()
+
+ def _page_iter(self, increment):
+ """Generator of pages of API responses.
+
+ Args:
+ increment (bool): Flag indicating if the total number of results
+ should be incremented on each page. This is useful since a page
+ iterator will want to increment by results per page while an
+ items iterator will want to increment per item.
+
+ Yields:
+ Page: each page of items from the API.
+ """
+ page = self._next_page()
+ while page is not None:
+ self.page_number += 1
+ if increment:
+ self.num_results += page.num_items
+ yield page
+ page = self._next_page()
+
+ @abc.abstractmethod
+ def _next_page(self):
+ """Get the next page in the iterator.
+
+ This does nothing and is intended to be over-ridden by subclasses
+ to return the next :class:`Page`.
+
+ Raises:
+ NotImplementedError: Always, this method is abstract.
+ """
+ raise NotImplementedError
+
+
+def _do_nothing_page_start(iterator, page, response):
+ """Helper to provide custom behavior after a :class:`Page` is started.
+
+ This is a do-nothing stand-in as the default value.
+
+ Args:
+ iterator (Iterator): An iterator that holds some request info.
+ page (Page): The page that was just created.
+ response (Any): The API response for a page.
+ """
+ # pylint: disable=unused-argument
+ pass
+
+
+class HTTPIterator(Iterator):
+ """A generic class for iterating through HTTP/JSON API list responses.
+
+ To make an iterator work, you'll need to provide a way to convert a JSON
+ item returned from the API into the object of your choice (via
+ ``item_to_value``). You also may need to specify a custom ``items_key`` so
+ that a given response (containing a page of results) can be parsed into an
+ iterable page of the actual objects you want.
+
+ Args:
+ client (google.cloud.client.Client): The API client.
+ api_request (Callable): The function to use to make API requests.
+ Generally, this will be
+ :meth:`google.cloud._http.JSONConnection.api_request`.
+ path (str): The method path to query for the list of items.
+ item_to_value (Callable[google.api_core.page_iterator.Iterator, Any]):
+ Callable to convert an item from the type in the JSON response into
+ a native object. Will be called with the iterator and a single
+ item.
+ items_key (str): The key in the API response where the list of items
+ can be found.
+ page_token (str): A token identifying a page in a result set to start
+ fetching results from.
+ max_results (int): The maximum number of results to fetch.
+ extra_params (dict): Extra query string parameters for the
+ API call.
+ page_start (Callable[
+ google.api_core.page_iterator.Iterator,
+ google.api_core.page_iterator.Page, dict]): Callable to provide
+ any special behavior after a new page has been created. Assumed
+ signature takes the :class:`.Iterator` that started the page,
+ the :class:`.Page` that was started and the dictionary containing
+ the page response.
+ next_token (str): The name of the field used in the response for page
+ tokens.
+
+ .. autoattribute:: pages
+ """
+
+ _DEFAULT_ITEMS_KEY = 'items'
+ _PAGE_TOKEN = 'pageToken'
+ _MAX_RESULTS = 'maxResults'
+ _NEXT_TOKEN = 'nextPageToken'
+ _RESERVED_PARAMS = frozenset([_PAGE_TOKEN])
+ _HTTP_METHOD = 'GET'
+
+ def __init__(self, client, api_request, path, item_to_value,
+ items_key=_DEFAULT_ITEMS_KEY,
+ page_token=None, max_results=None, extra_params=None,
+ page_start=_do_nothing_page_start, next_token=_NEXT_TOKEN):
+ super(HTTPIterator, self).__init__(
+ client, item_to_value, page_token=page_token,
+ max_results=max_results)
+ self.api_request = api_request
+ self.path = path
+ self._items_key = items_key
+ self.extra_params = extra_params
+ self._page_start = page_start
+ self._next_token = next_token
+ # Verify inputs / provide defaults.
+ if self.extra_params is None:
+ self.extra_params = {}
+ self._verify_params()
+
+ def _verify_params(self):
+ """Verifies the parameters don't use any reserved parameter.
+
+ Raises:
+ ValueError: If a reserved parameter is used.
+ """
+ reserved_in_use = self._RESERVED_PARAMS.intersection(
+ self.extra_params)
+ if reserved_in_use:
+ raise ValueError('Using a reserved parameter',
+ reserved_in_use)
+
+ def _next_page(self):
+ """Get the next page in the iterator.
+
+ Returns:
+ Optional[Page]: The next page in the iterator or :data:`None` if
+ there are no pages left.
+ """
+ if self._has_next_page():
+ response = self._get_next_page_response()
+ items = response.get(self._items_key, ())
+ page = Page(self, items, self._item_to_value)
+ self._page_start(self, page, response)
+ self.next_page_token = response.get(self._next_token)
+ return page
+ else:
+ return None
+
+ def _has_next_page(self):
+ """Determines whether or not there are more pages with results.
+
+ Returns:
+ bool: Whether the iterator has more pages.
+ """
+ if self.page_number == 0:
+ return True
+
+ if self.max_results is not None:
+ if self.num_results >= self.max_results:
+ return False
+
+ return self.next_page_token is not None
+
+ def _get_query_params(self):
+ """Getter for query parameters for the next request.
+
+ Returns:
+ dict: A dictionary of query parameters.
+ """
+ result = {}
+ if self.next_page_token is not None:
+ result[self._PAGE_TOKEN] = self.next_page_token
+ if self.max_results is not None:
+ result[self._MAX_RESULTS] = self.max_results - self.num_results
+ result.update(self.extra_params)
+ return result
+
+ def _get_next_page_response(self):
+ """Requests the next page from the path provided.
+
+ Returns:
+ dict: The parsed JSON response of the next page's contents.
+
+ Raises:
+ ValueError: If the HTTP method is not ``GET`` or ``POST``.
+ """
+ params = self._get_query_params()
+ if self._HTTP_METHOD == 'GET':
+ return self.api_request(
+ method=self._HTTP_METHOD,
+ path=self.path,
+ query_params=params)
+ elif self._HTTP_METHOD == 'POST':
+ return self.api_request(
+ method=self._HTTP_METHOD,
+ path=self.path,
+ data=params)
+ else:
+ raise ValueError('Unexpected HTTP method', self._HTTP_METHOD)
+
+
+class _GAXIterator(Iterator):
+ """A generic class for iterating through Cloud gRPC APIs list responses.
+
+ Any:
+ client (google.cloud.client.Client): The API client.
+ page_iter (google.gax.PageIterator): A GAX page iterator to be wrapped
+ to conform to the :class:`Iterator` interface.
+ item_to_value (Callable[Iterator, Any]): Callable to convert an item
+ from the the protobuf response into a native object. Will
+ be called with the iterator and a single item.
+ max_results (int): The maximum number of results to fetch.
+
+ .. autoattribute:: pages
+ """
+
+ def __init__(self, client, page_iter, item_to_value, max_results=None):
+ super(_GAXIterator, self).__init__(
+ client, item_to_value, page_token=page_iter.page_token,
+ max_results=max_results)
+ self._gax_page_iter = page_iter
+
+ def _next_page(self):
+ """Get the next page in the iterator.
+
+ Wraps the response from the :class:`~google.gax.PageIterator` in a
+ :class:`Page` instance and captures some state at each page.
+
+ Returns:
+ Optional[Page]: The next page in the iterator or :data:`None` if
+ there are no pages left.
+ """
+ try:
+ items = six.next(self._gax_page_iter)
+ page = Page(self, items, self._item_to_value)
+ self.next_page_token = self._gax_page_iter.page_token or None
+ return page
+ except StopIteration:
+ return None
+
+
+class GRPCIterator(Iterator):
+ """A generic class for iterating through gRPC list responses.
+
+ .. note:: The class does not take a ``page_token`` argument because it can
+ just be specified in the ``request``.
+
+ Args:
+ client (google.cloud.client.Client): The API client. This unused by
+ this class, but kept to satisfy the :class:`Iterator` interface.
+ method (Callable[protobuf.Message]): A bound gRPC method that should
+ take a single message for the request.
+ request (protobuf.Message): The request message.
+ items_field (str): The field in the response message that has the
+ items for the page.
+ item_to_value (Callable[GRPCIterator, Any]): Callable to convert an
+ item from the type in the JSON response into a native object. Will
+ be called with the iterator and a single item.
+ request_token_field (str): The field in the request message used to
+ specify the page token.
+ response_token_field (str): The field in the response message that has
+ the token for the next page.
+ max_results (int): The maximum number of results to fetch.
+
+ .. autoattribute:: pages
+ """
+
+ _DEFAULT_REQUEST_TOKEN_FIELD = 'page_token'
+ _DEFAULT_RESPONSE_TOKEN_FIELD = 'next_page_token'
+
+ def __init__(
+ self,
+ client,
+ method,
+ request,
+ items_field,
+ item_to_value=_item_to_value_identity,
+ request_token_field=_DEFAULT_REQUEST_TOKEN_FIELD,
+ response_token_field=_DEFAULT_RESPONSE_TOKEN_FIELD,
+ max_results=None):
+ super(GRPCIterator, self).__init__(
+ client, item_to_value, max_results=max_results)
+ self._method = method
+ self._request = request
+ self._items_field = items_field
+ self._request_token_field = request_token_field
+ self._response_token_field = response_token_field
+
+ def _next_page(self):
+ """Get the next page in the iterator.
+
+ Returns:
+ Page: The next page in the iterator or :data:`None` if
+ there are no pages left.
+ """
+ if not self._has_next_page():
+ return None
+
+ if self.next_page_token is not None:
+ setattr(
+ self._request, self._request_token_field, self.next_page_token)
+
+ response = self._method(self._request)
+
+ self.next_page_token = getattr(response, self._response_token_field)
+ items = getattr(response, self._items_field)
+ page = Page(self, items, self._item_to_value)
+
+ return page
+
+ def _has_next_page(self):
+ """Determines whether or not there are more pages with results.
+
+ Returns:
+ bool: Whether the iterator has more pages.
+ """
+ if self.page_number == 0:
+ return True
+
+ if self.max_results is not None:
+ if self.num_results >= self.max_results:
+ return False
+
+ # Note: intentionally a falsy check instead of a None check. The RPC
+ # can return an empty string indicating no more pages.
+ return True if self.next_page_token else False
diff --git a/google/api_core/path_template.py b/google/api_core/path_template.py
new file mode 100644
index 0000000..e1cfae3
--- /dev/null
+++ b/google/api_core/path_template.py
@@ -0,0 +1,198 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Expand and validate URL path templates.
+
+This module provides the :func:`expand` and :func:`validate` functions for
+interacting with Google-style URL `path templates`_ which are commonly used
+in Google APIs for `resource names`_.
+
+.. _path templates: https://github.com/googleapis/googleapis/blob
+ /57e2d376ac7ef48681554204a3ba78a414f2c533/google/api/http.proto#L212
+.. _resource names: https://cloud.google.com/apis/design/resource_names
+"""
+
+from __future__ import unicode_literals
+
+import functools
+import re
+
+import six
+
+# Regular expression for extracting variable parts from a path template.
+# The variables can be expressed as:
+#
+# - "*": a single-segment positional variable, for example: "books/*"
+# - "**": a multi-segment positional variable, for example: "shelf/**/book/*"
+# - "{name}": a single-segment wildcard named variable, for example
+# "books/{name}"
+# - "{name=*}: same as above.
+# - "{name=**}": a multi-segment wildcard named variable, for example
+# "shelf/{name=**}"
+# - "{name=/path/*/**}": a multi-segment named variable with a sub-template.
+_VARIABLE_RE = re.compile(r"""
+ ( # Capture the entire variable expression
+ (?P<positional>\*\*?) # Match & capture * and ** positional variables.
+ |
+ # Match & capture named variables {name}
+ {
+ (?P<name>[^/]+?)
+ # Optionally match and capture the named variable's template.
+ (?:=(?P<template>.+?))?
+ }
+ )
+ """, re.VERBOSE)
+
+# Segment expressions used for validating paths against a template.
+_SINGLE_SEGMENT_PATTERN = r'([^/]+)'
+_MULTI_SEGMENT_PATTERN = r'(.+)'
+
+
+def _expand_variable_match(positional_vars, named_vars, match):
+ """Expand a matched variable with its value.
+
+ Args:
+ positional_vars (list): A list of positonal variables. This list will
+ be modified.
+ named_vars (dict): A dictionary of named variables.
+ match (re.Match): A regular expression match.
+
+ Returns:
+ str: The expanded variable to replace the match.
+
+ Raises:
+ ValueError: If a positional or named variable is required by the
+ template but not specified or if an unexpected template expression
+ is encountered.
+ """
+ positional = match.group('positional')
+ name = match.group('name')
+ if name is not None:
+ try:
+ return six.text_type(named_vars[name])
+ except KeyError:
+ raise ValueError(
+ 'Named variable \'{}\' not specified and needed by template '
+ '`{}` at position {}'.format(
+ name, match.string, match.start()))
+ elif positional is not None:
+ try:
+ return six.text_type(positional_vars.pop(0))
+ except IndexError:
+ raise ValueError(
+ 'Positional variable not specified and needed by template '
+ '`{}` at position {}'.format(
+ match.string, match.start()))
+ else:
+ raise ValueError(
+ 'Unknown template expression {}'.format(
+ match.group(0)))
+
+
+def expand(tmpl, *args, **kwargs):
+ """Expand a path template with the given variables.
+
+ ..code-block:: python
+
+ >>> expand('users/*/messages/*', 'me', '123')
+ users/me/messages/123
+ >>> expand('/v1/{name=shelves/*/books/*}', name='shelves/1/books/3')
+ /v1/shelves/1/books/3
+
+ Args:
+ tmpl (str): The path template.
+ args: The positional variables for the path.
+ kwargs: The named variables for the path.
+
+ Returns:
+ str: The expanded path
+
+ Raises:
+ ValueError: If a positional or named variable is required by the
+ template but not specified or if an unexpected template expression
+ is encountered.
+ """
+ replacer = functools.partial(_expand_variable_match, list(args), kwargs)
+ return _VARIABLE_RE.sub(replacer, tmpl)
+
+
+def _replace_variable_with_pattern(match):
+ """Replace a variable match with a pattern that can be used to validate it.
+
+ Args:
+ match (re.Match): A regular expression match
+
+ Returns:
+ str: A regular expression pattern that can be used to validate the
+ variable in an expanded path.
+
+ Raises:
+ ValueError: If an unexpected template expression is encountered.
+ """
+ positional = match.group('positional')
+ name = match.group('name')
+ template = match.group('template')
+ if name is not None:
+ if not template:
+ return _SINGLE_SEGMENT_PATTERN.format(name)
+ elif template == '**':
+ return _MULTI_SEGMENT_PATTERN.format(name)
+ else:
+ return _generate_pattern_for_template(template)
+ elif positional == '*':
+ return _SINGLE_SEGMENT_PATTERN
+ elif positional == '**':
+ return _MULTI_SEGMENT_PATTERN
+ else:
+ raise ValueError(
+ 'Unknown template expression {}'.format(
+ match.group(0)))
+
+
+def _generate_pattern_for_template(tmpl):
+ """Generate a pattern that can validate a path template.
+
+ Args:
+ tmpl (str): The path template
+
+ Returns:
+ str: A regular expression pattern that can be used to validate an
+ expanded path template.
+ """
+ return _VARIABLE_RE.sub(_replace_variable_with_pattern, tmpl)
+
+
+def validate(tmpl, path):
+ """Validate a path against the path template.
+
+ .. code-block:: python
+
+ >>> validate('users/*/messages/*', 'users/me/messages/123')
+ True
+ >>> validate('users/*/messages/*', 'users/me/drafts/123')
+ False
+ >>> validate('/v1/{name=shelves/*/books/*}', /v1/shelves/1/books/3)
+ True
+ >>> validate('/v1/{name=shelves/*/books/*}', /v1/shelves/1/tapes/3)
+ False
+
+ Args:
+ tmpl (str): The path template.
+ path (str): The expanded path.
+
+ Returns:
+ bool: True if the path matches.
+ """
+ pattern = _generate_pattern_for_template(tmpl) + '$'
+ return True if re.match(pattern, path) is not None else False
diff --git a/google/api_core/protobuf_helpers.py b/google/api_core/protobuf_helpers.py
new file mode 100644
index 0000000..35eb574
--- /dev/null
+++ b/google/api_core/protobuf_helpers.py
@@ -0,0 +1,38 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for :mod:`protobuf`."""
+
+
+def from_any_pb(pb_type, any_pb):
+ """Converts an ``Any`` protobuf to the specified message type.
+
+ Args:
+ pb_type (type): the type of the message that any_pb stores an instance
+ of.
+ any_pb (google.protobuf.any_pb2.Any): the object to be converted.
+
+ Returns:
+ pb_type: An instance of the pb_type message.
+
+ Raises:
+ TypeError: if the message could not be converted.
+ """
+ msg = pb_type()
+ if not any_pb.Unpack(msg):
+ raise TypeError(
+ 'Could not convert {} to {}'.format(
+ any_pb.__class__.__name__, pb_type.__name__))
+
+ return msg
diff --git a/google/api_core/retry.py b/google/api_core/retry.py
new file mode 100644
index 0000000..ccbb883
--- /dev/null
+++ b/google/api_core/retry.py
@@ -0,0 +1,323 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for retrying functions with exponential back-off.
+
+The :class:`Retry` decorator can be used to retry functions that raise
+exceptions using exponential backoff. Because a exponential sleep algorithm is
+used, the retry is limited by a `deadline`. The deadline is the maxmimum amount
+of time a method can block. This is used instead of total number of retries
+because it is difficult to ascertain the amount of time a function can block
+when using total number of retries and exponential backoff.
+
+By default, this decorator will retry transient
+API errors (see :func:`if_transient_error`). For example:
+
+.. code-block:: python
+
+ @retry.Retry()
+ def call_flaky_rpc():
+ return client.flaky_rpc()
+
+ # Will retry flaky_rpc() if it raises transient API errors.
+ result = call_flaky_rpc()
+
+You can pass a custom predicate to retry on different exceptions, such as
+waiting for an eventually consistent item to be available:
+
+.. code-block:: python
+
+ @retry.Retry(predicate=if_exception_type(exceptions.NotFound))
+ def check_if_exists():
+ return client.does_thing_exist()
+
+ is_available = check_if_exists()
+
+Some client library methods apply retry automatically. These methods can accept
+a ``retry`` parameter that allows you to configure the behavior:
+
+.. code-block:: python
+
+ my_retry = retry.Retry(deadline=60)
+ result = client.some_method(retry=my_retry)
+
+"""
+
+from __future__ import unicode_literals
+
+import datetime
+import functools
+import logging
+import random
+import time
+
+import six
+
+from google.api_core import datetime_helpers
+from google.api_core import exceptions
+from google.api_core import general_helpers
+
+_LOGGER = logging.getLogger(__name__)
+_DEFAULT_INITIAL_DELAY = 1.0 # seconds
+_DEFAULT_MAXIMUM_DELAY = 60.0 # seconds
+_DEFAULT_DELAY_MULTIPLIER = 2.0
+_DEFAULT_DEADLINE = 60.0 * 2.0 # seconds
+
+
+def if_exception_type(*exception_types):
+ """Creates a predicate to check if the exception is of a given type.
+
+ Args:
+ exception_types (Sequence[:func:`type`]): The exception types to check
+ for.
+
+ Returns:
+ Callable[Exception]: A predicate that returns True if the provided
+ exception is of the given type(s).
+ """
+ def if_exception_type_predicate(exception):
+ """Bound predicate for checking an exception type."""
+ return isinstance(exception, exception_types)
+ return if_exception_type_predicate
+
+
+# pylint: disable=invalid-name
+# Pylint sees this as a constant, but it is also an alias that should be
+# considered a function.
+if_transient_error = if_exception_type((
+ exceptions.InternalServerError,
+ exceptions.TooManyRequests))
+"""A predicate that checks if an exception is a transient API error.
+
+The following server errors are considered transient:
+
+- :class:`google.api_core.exceptions.InternalServerError` - HTTP 500, gRPC
+ ``INTERNAL(13)`` and its subclasses.
+- :class:`google.api_core.exceptions.TooManyRequests` - HTTP 429
+- :class:`google.api_core.exceptions.ResourceExhausted` - gRPC
+ ``RESOURCE_EXHAUSTED(8)``
+"""
+# pylint: enable=invalid-name
+
+
+def exponential_sleep_generator(
+ initial, maximum, multiplier=_DEFAULT_DELAY_MULTIPLIER):
+ """Generates sleep intervals based on the exponential back-off algorithm.
+
+ This implements the `Truncated Exponential Back-off`_ algorithm.
+
+ .. _Truncated Exponential Back-off:
+ https://cloud.google.com/storage/docs/exponential-backoff
+
+ Args:
+ initial (float): The minimum about of time to delay. This must
+ be greater than 0.
+ maximum (float): The maximum about of time to delay.
+ multiplier (float): The multiplier applied to the delay.
+
+ Yields:
+ float: successive sleep intervals.
+ """
+ delay = initial
+ while True:
+ # Introduce jitter by yielding a delay that is uniformly distributed
+ # to average out to the delay time.
+ yield min(random.uniform(0.0, delay * 2.0), maximum)
+ delay = delay * multiplier
+
+
+def retry_target(target, predicate, sleep_generator, deadline, on_error=None):
+ """Call a function and retry if it fails.
+
+ This is the lowest-level retry helper. Generally, you'll use the
+ higher-level retry helper :class:`Retry`.
+
+ Args:
+ target(Callable): The function to call and retry. This must be a
+ nullary function - apply arguments with `functools.partial`.
+ predicate (Callable[Exception]): A callable used to determine if an
+ exception raised by the target should be considered retryable.
+ It should return True to retry or False otherwise.
+ sleep_generator (Iterable[float]): An infinite iterator that determines
+ how long to sleep between retries.
+ deadline (float): How long to keep retrying the target.
+ on_error (Callable): A function to call while processing a retryable
+ exception. Any error raised by this function will *not* be
+ caught.
+
+ Returns:
+ Any: the return value of the target function.
+
+ Raises:
+ google.api_core.RetryError: If the deadline is exceeded while retrying.
+ ValueError: If the sleep generator stops yielding values.
+ Exception: If the target raises a method that isn't retryable.
+ """
+ if deadline is not None:
+ deadline_datetime = (
+ datetime_helpers.utcnow() + datetime.timedelta(seconds=deadline))
+ else:
+ deadline_datetime = None
+
+ last_exc = None
+
+ for sleep in sleep_generator:
+ try:
+ return target()
+
+ # pylint: disable=broad-except
+ # This function explicitly must deal with broad exceptions.
+ except Exception as exc:
+ if not predicate(exc):
+ raise
+ last_exc = exc
+ if on_error is not None:
+ on_error(exc)
+
+ now = datetime_helpers.utcnow()
+ if deadline_datetime is not None and deadline_datetime < now:
+ six.raise_from(
+ exceptions.RetryError(
+ 'Deadline of {:.1f}s exceeded while calling {}'.format(
+ deadline, target),
+ last_exc),
+ last_exc)
+
+ _LOGGER.debug('Retrying due to {}, sleeping {:.1f}s ...'.format(
+ last_exc, sleep))
+ time.sleep(sleep)
+
+ raise ValueError('Sleep generator stopped yielding sleep values.')
+
+
+@six.python_2_unicode_compatible
+class Retry(object):
+ """Exponential retry decorator.
+
+ This class is a decorator used to add exponential back-off retry behavior
+ to an RPC call.
+
+ Although the default behavior is to retry transient API errors, a
+ different predicate can be provided to retry other exceptions.
+
+ Args:
+ predicate (Callable[Exception]): A callable that should return ``True``
+ if the given exception is retryable.
+ initial (float): The minimum about of time to delay in seconds. This
+ must be greater than 0.
+ maximum (float): The maximum about of time to delay in seconds.
+ multiplier (float): The multiplier applied to the delay.
+ deadline (float): How long to keep retrying in seconds.
+ """
+ def __init__(
+ self,
+ predicate=if_transient_error,
+ initial=_DEFAULT_INITIAL_DELAY,
+ maximum=_DEFAULT_MAXIMUM_DELAY,
+ multiplier=_DEFAULT_DELAY_MULTIPLIER,
+ deadline=_DEFAULT_DEADLINE):
+ self._predicate = predicate
+ self._initial = initial
+ self._multiplier = multiplier
+ self._maximum = maximum
+ self._deadline = deadline
+
+ def __call__(self, func, on_error=None):
+ """Wrap a callable with retry behavior.
+
+ Args:
+ func (Callable): The callable to add retry behavior to.
+ on_error (Callable): A function to call while processing a
+ retryable exception. Any error raised by this function will
+ *not* be caught.
+
+ Returns:
+ Callable: A callable that will invoke ``func`` with retry
+ behavior.
+ """
+ @general_helpers.wraps(func)
+ def retry_wrapped_func(*args, **kwargs):
+ """A wrapper that calls target function with retry."""
+ target = functools.partial(func, *args, **kwargs)
+ sleep_generator = exponential_sleep_generator(
+ self._initial, self._maximum, multiplier=self._multiplier)
+ return retry_target(
+ target,
+ self._predicate,
+ sleep_generator,
+ self._deadline,
+ on_error=on_error,
+ )
+
+ return retry_wrapped_func
+
+ def with_deadline(self, deadline):
+ """Return a copy of this retry with the given deadline.
+
+ Args:
+ deadline (float): How long to keep retrying.
+
+ Returns:
+ Retry: A new retry instance with the given deadline.
+ """
+ return Retry(
+ predicate=self._predicate,
+ initial=self._initial,
+ maximum=self._maximum,
+ multiplier=self._multiplier,
+ deadline=deadline)
+
+ def with_predicate(self, predicate):
+ """Return a copy of this retry with the given predicate.
+
+ Args:
+ predicate (Callable[Exception]): A callable that should return
+ ``True`` if the given exception is retryable.
+
+ Returns:
+ Retry: A new retry instance with the given predicate.
+ """
+ return Retry(
+ predicate=predicate,
+ initial=self._initial,
+ maximum=self._maximum,
+ multiplier=self._multiplier,
+ deadline=self._deadline)
+
+ def with_delay(
+ self, initial=None, maximum=None, multiplier=None):
+ """Return a copy of this retry with the given delay options.
+
+ Args:
+ initial (float): The minimum about of time to delay. This must
+ be greater than 0.
+ maximum (float): The maximum about of time to delay.
+ multiplier (float): The multiplier applied to the delay.
+
+ Returns:
+ Retry: A new retry instance with the given predicate.
+ """
+ return Retry(
+ predicate=self._predicate,
+ initial=initial if initial is not None else self._initial,
+ maximum=maximum if maximum is not None else self._maximum,
+ multiplier=multiplier if maximum is not None else self._multiplier,
+ deadline=self._deadline)
+
+ def __str__(self):
+ return (
+ '<Retry predicate={}, initial={:.1f}, maximum={:.1f}, '
+ 'multiplier={:.1f}, deadline={:.1f}>'.format(
+ self._predicate, self._initial, self._maximum,
+ self._multiplier, self._deadline))
diff --git a/google/api_core/timeout.py b/google/api_core/timeout.py
new file mode 100644
index 0000000..12fb9fc
--- /dev/null
+++ b/google/api_core/timeout.py
@@ -0,0 +1,215 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Decorators for applying timeout arguments to functions.
+
+These decorators are used to wrap API methods to apply either a constant
+or exponential timeout argument.
+
+For example, imagine an API method that can take a while to return results,
+such as one that might block until a resource is ready:
+
+.. code-block:: python
+
+ def is_thing_ready(timeout=None):
+ response = requests.get('https://example.com/is_thing_ready')
+ response.raise_for_status()
+ return response.json()
+
+This module allows a function like this to be wrapped so that timeouts are
+automatically determined, for example:
+
+.. code-block:: python
+
+ timeout_ = timeout.ExponentialTimeout()
+ is_thing_ready_with_timeout = timeout_(is_thing_ready)
+
+ for n in range(10):
+ try:
+ is_thing_ready_with_timeout({'example': 'data'})
+ except:
+ pass
+
+In this example the first call to ``is_thing_ready`` will have a relatively
+small timeout (like 1 second). If the resource is available and the request
+completes quickly, the loop exits. But, if the resource isn't yet available
+and the request times out, it'll be retried - this time with a larger timeout.
+
+In the broader context these decorators are typically combined with
+:mod:`google.api_core.retry` to implement API methods with a signature that
+matches ``api_method(request, timeout=None, retry=None)``.
+"""
+
+from __future__ import unicode_literals
+
+import datetime
+
+import six
+
+from google.api_core import datetime_helpers
+from google.api_core import general_helpers
+
+_DEFAULT_INITIAL_TIMEOUT = 5.0 # seconds
+_DEFAULT_MAXIMUM_TIMEOUT = 30.0 # seconds
+_DEFAULT_TIMEOUT_MULTIPLIER = 2.0
+# If specified, must be in seconds. If none, deadline is not used in the
+# timeout calculation.
+_DEFAULT_DEADLINE = None
+
+
+@six.python_2_unicode_compatible
+class ConstantTimeout(object):
+ """A decorator that adds a constant timeout argument.
+
+ This is effectively equivalent to
+ ``functools.partial(func, timeout=timeout)``.
+
+ Args:
+ timeout (Optional[float]): the timeout (in seconds) to applied to the
+ wrapped function. If `None`, the target function is expected to
+ never timeout.
+ """
+ def __init__(self, timeout=None):
+ self._timeout = timeout
+
+ def __call__(self, func):
+ """Apply the timeout decorator.
+
+ Args:
+ func (Callable): The function to apply the timeout argument to.
+ This function must accept a timeout keyword argument.
+
+ Returns:
+ Callable: The wrapped function.
+ """
+ @general_helpers.wraps(func)
+ def func_with_timeout(*args, **kwargs):
+ """Wrapped function that adds timeout."""
+ kwargs['timeout'] = self._timeout
+ return func(*args, **kwargs)
+ return func_with_timeout
+
+ def __str__(self):
+ return '<ConstantTimeout timeout={:.1f}>'.format(self._timeout)
+
+
+def _exponential_timeout_generator(initial, maximum, multiplier, deadline):
+ """A generator that yields exponential timeout values.
+
+ Args:
+ initial (float): The initial timeout.
+ maximum (float): The maximum timeout.
+ multiplier (float): The multiplier applied to the timeout.
+ deadline (float): The overall deadline across all invocations.
+
+ Yields:
+ float: A timeout value.
+ """
+ if deadline is not None:
+ deadline_datetime = (
+ datetime_helpers.utcnow() +
+ datetime.timedelta(seconds=deadline))
+ else:
+ deadline_datetime = datetime.datetime.max
+
+ timeout = initial
+ while True:
+ now = datetime_helpers.utcnow()
+ yield min(
+ # The calculated timeout based on invocations.
+ timeout,
+ # The set maximum timeout.
+ maximum,
+ # The remaining time before the deadline is reached.
+ float((deadline_datetime - now).seconds))
+ timeout = timeout * multiplier
+
+
+@six.python_2_unicode_compatible
+class ExponentialTimeout(object):
+ """A decorator that adds an exponentially increasing timeout argument.
+
+ This is useful if a function is called multiple times. Each time the
+ function is called this decorator will calculate a new timeout parameter
+ based on the the number of times the function has been called.
+
+ For example
+
+ .. code-block:: python
+
+ Args:
+ initial (float): The initial timeout to pass.
+ maximum (float): The maximum timeout for any one call.
+ multiplier (float): The multiplier applied to the timeout for each
+ invocation.
+ deadline (Optional[float]): The overall deadline across all
+ invocations. This is used to prevent a very large calculated
+ timeout from pushing the overall execution time over the deadline.
+ This is especially useful in conjuction with
+ :mod:`google.api_core.retry`. If ``None``, the timeouts will not
+ be adjusted to accomodate an overall deadline.
+ """
+ def __init__(
+ self,
+ initial=_DEFAULT_INITIAL_TIMEOUT,
+ maximum=_DEFAULT_MAXIMUM_TIMEOUT,
+ multiplier=_DEFAULT_TIMEOUT_MULTIPLIER,
+ deadline=_DEFAULT_DEADLINE):
+ self._initial = initial
+ self._maximum = maximum
+ self._multiplier = multiplier
+ self._deadline = deadline
+
+ def with_deadline(self, deadline):
+ """Return a copy of this teimout with the given deadline.
+
+ Args:
+ deadline (float): The overall deadline across all invocations.
+
+ Returns:
+ ExponentialTimeout: A new instance with the given deadline.
+ """
+ return ExponentialTimeout(
+ initial=self._initial,
+ maximum=self._maximum,
+ multiplier=self._multiplier,
+ deadline=deadline)
+
+ def __call__(self, func):
+ """Apply the timeout decorator.
+
+ Args:
+ func (Callable): The function to apply the timeout argument to.
+ This function must accept a timeout keyword argument.
+
+ Returns:
+ Callable: The wrapped function.
+ """
+ timeouts = _exponential_timeout_generator(
+ self._initial, self._maximum, self._multiplier, self._deadline)
+
+ @general_helpers.wraps(func)
+ def func_with_timeout(*args, **kwargs):
+ """Wrapped function that adds timeout."""
+ kwargs['timeout'] = next(timeouts)
+ return func(*args, **kwargs)
+
+ return func_with_timeout
+
+ def __str__(self):
+ return (
+ '<ExponentialTimeout initial={:.1f}, maximum={:.1f}, '
+ 'multiplier={:.1f}, deadline={:.1f}>'.format(
+ self._initial, self._maximum, self._multiplier,
+ self._deadline))
diff --git a/nox.py b/nox.py
new file mode 100644
index 0000000..bb10c6c
--- /dev/null
+++ b/nox.py
@@ -0,0 +1,93 @@
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+import os
+
+import nox
+
+
+@nox.session
+@nox.parametrize('python_version', ['2.7', '3.4', '3.5', '3.6'])
+def unit_tests(session, python_version):
+ """Run the unit test suite."""
+
+ # Run unit tests against all supported versions of Python.
+ session.interpreter = 'python{}'.format(python_version)
+
+ # Set the virtualenv dirname.
+ session.virtualenv_dirname = 'unit-' + python_version
+
+ # Install all test dependencies, then install this package in-place.
+ session.install(
+ 'mock',
+ 'pytest',
+ 'pytest-cov',
+ 'grpcio >= 1.0.2',
+ )
+ session.install('-e', '.')
+
+ # Run py.test against the unit tests.
+ session.run(
+ 'py.test',
+ '--quiet',
+ '--cov=google.cloud',
+ '--cov=google.api_core',
+ '--cov=tests.unit',
+ '--cov-append',
+ '--cov-config=.coveragerc',
+ '--cov-report=',
+ '--cov-fail-under=97',
+ os.path.join('tests', 'unit'),
+ *session.posargs
+ )
+
+
+@nox.session
+def lint(session):
+ """Run linters.
+
+ Returns a failure if the linters find linting errors or sufficiently
+ serious code quality issues.
+ """
+ session.interpreter = 'python3.6'
+ session.install('flake8', 'flake8-import-order')
+ session.install('.')
+ session.run('flake8', 'google', 'tests')
+
+
+@nox.session
+def lint_setup_py(session):
+ """Verify that setup.py is valid (including RST check)."""
+ session.interpreter = 'python3.6'
+
+ # Set the virtualenv dirname.
+ session.virtualenv_dirname = 'setup'
+
+ session.install('docutils', 'Pygments')
+ session.run(
+ 'python', 'setup.py', 'check', '--restructuredtext', '--strict')
+
+
+@nox.session
+def cover(session):
+ """Run the final coverage report.
+
+ This outputs the coverage report aggregating coverage from the unit
+ test runs (not system test runs), and then erases coverage data.
+ """
+ session.interpreter = 'python3.6'
+ session.install('coverage', 'pytest-cov')
+ session.run('coverage', 'report', '--show-missing', '--fail-under=100')
+ session.run('coverage', 'erase')
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..2a9acf1
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal = 1
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..fa6cdf7
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,76 @@
+# Copyright 2016 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from setuptools import find_packages
+from setuptools import setup
+
+
+PACKAGE_ROOT = os.path.abspath(os.path.dirname(__file__))
+
+with open(os.path.join(PACKAGE_ROOT, 'README.rst')) as file_obj:
+ README = file_obj.read()
+
+
+SETUP_BASE = {
+ 'author': 'Google Cloud Platform',
+ 'author_email': 'googleapis-publisher@google.com',
+ 'scripts': [],
+ 'url': 'https://github.com/GoogleCloudPlatform/google-cloud-python',
+ 'license': 'Apache 2.0',
+ 'platforms': 'Posix; MacOS X; Windows',
+ 'include_package_data': True,
+ 'zip_safe': False,
+ 'classifiers': [
+ 'Development Status :: 4 - Beta',
+ 'Intended Audience :: Developers',
+ 'License :: OSI Approved :: Apache Software License',
+ 'Operating System :: OS Independent',
+ 'Programming Language :: Python :: 2',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.4',
+ 'Programming Language :: Python :: 3.5',
+ 'Programming Language :: Python :: 3.6',
+ 'Topic :: Internet',
+ ],
+}
+
+
+REQUIREMENTS = [
+ 'googleapis-common-protos >= 1.5.3, < 2.0dev',
+ 'protobuf >= 3.0.0',
+ 'google-auth >= 0.4.0, < 2.0.0dev',
+ 'requests >= 2.18.0, < 3.0.0dev',
+ 'setuptools >= 34.0.0',
+ 'six >= 1.10.0',
+]
+
+EXTRAS_REQUIREMENTS = {
+ ':python_version<"3.2"': ['futures >= 3.0.0'],
+ 'grpc': ['grpcio >= 1.2.0, < 1.6dev'],
+}
+
+setup(
+ name='google-api-core',
+ version='0.1.0',
+ description='Core Google API Client Library',
+ long_description=README,
+ namespace_packages=['google'],
+ packages=find_packages(exclude=('tests*',)),
+ install_requires=REQUIREMENTS,
+ extras_require=EXTRAS_REQUIREMENTS,
+ **SETUP_BASE
+)
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/__init__.py
diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/unit/__init__.py
diff --git a/tests/unit/future/__init__.py b/tests/unit/future/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/unit/future/__init__.py
diff --git a/tests/unit/future/test__helpers.py b/tests/unit/future/test__helpers.py
new file mode 100644
index 0000000..660d23a
--- /dev/null
+++ b/tests/unit/future/test__helpers.py
@@ -0,0 +1,37 @@
+# Copyright 2017, Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+from google.api_core.future import _helpers
+
+
+@mock.patch('threading.Thread', autospec=True)
+def test_start_deamon_thread(unused_thread):
+ deamon_thread = _helpers.start_daemon_thread(target=mock.sentinel.target)
+ assert deamon_thread.daemon is True
+
+
+def test_safe_invoke_callback():
+ callback = mock.Mock(spec=['__call__'], return_value=42)
+ result = _helpers.safe_invoke_callback(callback, 'a', b='c')
+ assert result == 42
+ callback.assert_called_once_with('a', b='c')
+
+
+def test_safe_invoke_callback_exception():
+ callback = mock.Mock(spec=['__call__'], side_effect=ValueError())
+ result = _helpers.safe_invoke_callback(callback, 'a', b='c')
+ assert result is None
+ callback.assert_called_once_with('a', b='c')
diff --git a/tests/unit/future/test_polling.py b/tests/unit/future/test_polling.py
new file mode 100644
index 0000000..7ad9aee
--- /dev/null
+++ b/tests/unit/future/test_polling.py
@@ -0,0 +1,157 @@
+# Copyright 2017, Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import concurrent.futures
+import threading
+import time
+
+import mock
+import pytest
+
+from google.api_core.future import polling
+
+
+class PollingFutureImpl(polling.PollingFuture):
+ def done(self):
+ return False
+
+ def cancel(self):
+ return True
+
+ def cancelled(self):
+ return False
+
+ def running(self):
+ return True
+
+
+def test_polling_future_constructor():
+ future = PollingFutureImpl()
+ assert not future.done()
+ assert not future.cancelled()
+ assert future.running()
+ assert future.cancel()
+
+
+def test_set_result():
+ future = PollingFutureImpl()
+ callback = mock.Mock()
+
+ future.set_result(1)
+
+ assert future.result() == 1
+ future.add_done_callback(callback)
+ callback.assert_called_once_with(future)
+
+
+def test_set_exception():
+ future = PollingFutureImpl()
+ exception = ValueError('meep')
+
+ future.set_exception(exception)
+
+ assert future.exception() == exception
+ with pytest.raises(ValueError):
+ future.result()
+
+ callback = mock.Mock()
+ future.add_done_callback(callback)
+ callback.assert_called_once_with(future)
+
+
+def test_invoke_callback_exception():
+ future = PollingFutureImplWithPoll()
+ future.set_result(42)
+
+ # This should not raise, despite the callback causing an exception.
+ callback = mock.Mock(side_effect=ValueError)
+ future.add_done_callback(callback)
+ callback.assert_called_once_with(future)
+
+
+class PollingFutureImplWithPoll(PollingFutureImpl):
+ def __init__(self):
+ super(PollingFutureImplWithPoll, self).__init__()
+ self.poll_count = 0
+ self.event = threading.Event()
+
+ def done(self):
+ self.poll_count += 1
+ self.event.wait()
+ self.set_result(42)
+ return True
+
+
+def test_result_with_polling():
+ future = PollingFutureImplWithPoll()
+
+ future.event.set()
+ result = future.result()
+
+ assert result == 42
+ assert future.poll_count == 1
+ # Repeated calls should not cause additional polling
+ assert future.result() == result
+ assert future.poll_count == 1
+
+
+class PollingFutureImplTimeout(PollingFutureImplWithPoll):
+ def done(self):
+ time.sleep(1)
+ return False
+
+
+def test_result_timeout():
+ future = PollingFutureImplTimeout()
+ with pytest.raises(concurrent.futures.TimeoutError):
+ future.result(timeout=1)
+
+
+def test_callback_background_thread():
+ future = PollingFutureImplWithPoll()
+ callback = mock.Mock()
+
+ future.add_done_callback(callback)
+
+ assert future._polling_thread is not None
+
+ # Give the thread a second to poll
+ time.sleep(1)
+ assert future.poll_count == 1
+
+ future.event.set()
+ future._polling_thread.join()
+
+ callback.assert_called_once_with(future)
+
+
+def test_double_callback_background_thread():
+ future = PollingFutureImplWithPoll()
+ callback = mock.Mock()
+ callback2 = mock.Mock()
+
+ future.add_done_callback(callback)
+ current_thread = future._polling_thread
+ assert current_thread is not None
+
+ # only one polling thread should be created.
+ future.add_done_callback(callback2)
+ assert future._polling_thread is current_thread
+
+ future.event.set()
+ future._polling_thread.join()
+
+ assert future.poll_count == 1
+ callback.assert_called_once_with(future)
+ callback2.assert_called_once_with(future)
diff --git a/tests/unit/gapic/test_config.py b/tests/unit/gapic/test_config.py
new file mode 100644
index 0000000..75a6e1c
--- /dev/null
+++ b/tests/unit/gapic/test_config.py
@@ -0,0 +1,89 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from google.api_core import exceptions
+from google.api_core.gapic_v1 import config
+
+
+INTERFACE_CONFIG = {
+ 'retry_codes': {
+ 'idempotent': ['DEADLINE_EXCEEDED', 'UNAVAILABLE'],
+ 'other': ['FAILED_PRECONDITION'],
+ 'non_idempotent': []
+ },
+ 'retry_params': {
+ 'default': {
+ 'initial_retry_delay_millis': 1000,
+ 'retry_delay_multiplier': 2.5,
+ 'max_retry_delay_millis': 120000,
+ 'initial_rpc_timeout_millis': 120000,
+ 'rpc_timeout_multiplier': 1.0,
+ 'max_rpc_timeout_millis': 120000,
+ 'total_timeout_millis': 600000
+ },
+ 'other': {
+ 'initial_retry_delay_millis': 1000,
+ 'retry_delay_multiplier': 1,
+ 'max_retry_delay_millis': 1000,
+ 'initial_rpc_timeout_millis': 1000,
+ 'rpc_timeout_multiplier': 1,
+ 'max_rpc_timeout_millis': 1000,
+ 'total_timeout_millis': 1000
+ },
+ },
+ 'methods': {
+ 'AnnotateVideo': {
+ 'timeout_millis': 60000,
+ 'retry_codes_name': 'idempotent',
+ 'retry_params_name': 'default'
+ },
+ 'Other': {
+ 'timeout_millis': 60000,
+ 'retry_codes_name': 'other',
+ 'retry_params_name': 'other'
+ },
+ 'Plain': {
+ 'timeout_millis': 30000
+ }
+ }
+}
+
+
+def test_create_method_configs():
+ method_configs = config.parse_method_configs(INTERFACE_CONFIG)
+
+ retry, timeout = method_configs['AnnotateVideo']
+ assert retry._predicate(exceptions.DeadlineExceeded(None))
+ assert retry._predicate(exceptions.ServiceUnavailable(None))
+ assert retry._initial == 1.0
+ assert retry._multiplier == 2.5
+ assert retry._maximum == 120.0
+ assert retry._deadline == 600.0
+ assert timeout._initial == 120.0
+ assert timeout._multiplier == 1.0
+ assert timeout._maximum == 120.0
+
+ retry, timeout = method_configs['Other']
+ assert retry._predicate(exceptions.FailedPrecondition(None))
+ assert retry._initial == 1.0
+ assert retry._multiplier == 1.0
+ assert retry._maximum == 1.0
+ assert retry._deadline == 1.0
+ assert timeout._initial == 1.0
+ assert timeout._multiplier == 1.0
+ assert timeout._maximum == 1.0
+
+ retry, timeout = method_configs['Plain']
+ assert retry is None
+ assert timeout._timeout == 30.0
diff --git a/tests/unit/gapic/test_method.py b/tests/unit/gapic/test_method.py
new file mode 100644
index 0000000..35ac144
--- /dev/null
+++ b/tests/unit/gapic/test_method.py
@@ -0,0 +1,226 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+
+import mock
+
+from google.api_core import exceptions
+from google.api_core import retry
+from google.api_core import timeout
+import google.api_core.gapic_v1.method
+import google.api_core.page_iterator
+
+
+def _utcnow_monotonic():
+ curr_value = datetime.datetime.min
+ delta = datetime.timedelta(seconds=0.5)
+ while True:
+ yield curr_value
+ curr_value += delta
+
+
+def test_wrap_method_basic():
+ method = mock.Mock(spec=['__call__'], return_value=42)
+
+ wrapped_method = google.api_core.gapic_v1.method.wrap_method(
+ method, metadata=None)
+
+ result = wrapped_method(1, 2, meep='moop')
+
+ assert result == 42
+ method.assert_called_once_with(1, 2, meep='moop')
+
+
+def test_wrap_method_with_default_metadata():
+ method = mock.Mock(spec=['__call__'])
+
+ wrapped_method = google.api_core.gapic_v1.method.wrap_method(method)
+
+ wrapped_method(1, 2, meep='moop')
+
+ method.assert_called_once_with(1, 2, meep='moop', metadata=mock.ANY)
+
+ metadata = method.call_args[1]['metadata']
+ assert len(metadata) == 1
+ assert metadata[0][0] == 'x-goog-api-client'
+ assert 'api-core' in metadata[0][1]
+
+
+def test_wrap_method_with_custom_metadata():
+ method = mock.Mock(spec=['__call__'])
+
+ wrapped_method = google.api_core.gapic_v1.method.wrap_method(
+ method, metadata={'foo': 'bar'})
+
+ wrapped_method(1, 2, meep='moop')
+
+ method.assert_called_once_with(1, 2, meep='moop', metadata=mock.ANY)
+
+ metadata = method.call_args[1]['metadata']
+ assert len(metadata) == 2
+ assert ('foo', 'bar') in metadata
+
+
+def test_wrap_method_with_merged_metadata():
+ method = mock.Mock(spec=['__call__'])
+
+ wrapped_method = google.api_core.gapic_v1.method.wrap_method(
+ method, metadata={'x-goog-api-client': 'foo/1.2.3'})
+
+ wrapped_method(1, 2, meep='moop')
+
+ method.assert_called_once_with(1, 2, meep='moop', metadata=mock.ANY)
+
+ metadata = method.call_args[1]['metadata']
+ assert len(metadata) == 1
+ assert metadata[0][0] == 'x-goog-api-client'
+ assert metadata[0][1].endswith(' foo/1.2.3')
+
+
+@mock.patch('time.sleep')
+def test_wrap_method_with_default_retry_and_timeout(unusued_sleep):
+ method = mock.Mock(
+ spec=['__call__'],
+ side_effect=[exceptions.InternalServerError(None), 42]
+ )
+ default_retry = retry.Retry()
+ default_timeout = timeout.ConstantTimeout(60)
+ wrapped_method = google.api_core.gapic_v1.method.wrap_method(
+ method, default_retry, default_timeout)
+
+ result = wrapped_method()
+
+ assert result == 42
+ assert method.call_count == 2
+ method.assert_called_with(timeout=60, metadata=mock.ANY)
+
+
+@mock.patch('time.sleep')
+def test_wrap_method_with_default_retry_and_timeout_using_sentinel(
+ unusued_sleep):
+ method = mock.Mock(
+ spec=['__call__'],
+ side_effect=[exceptions.InternalServerError(None), 42]
+ )
+ default_retry = retry.Retry()
+ default_timeout = timeout.ConstantTimeout(60)
+ wrapped_method = google.api_core.gapic_v1.method.wrap_method(
+ method, default_retry, default_timeout)
+
+ result = wrapped_method(
+ retry=google.api_core.gapic_v1.method.DEFAULT,
+ timeout=google.api_core.gapic_v1.method.DEFAULT)
+
+ assert result == 42
+ assert method.call_count == 2
+ method.assert_called_with(timeout=60, metadata=mock.ANY)
+
+
+@mock.patch('time.sleep')
+def test_wrap_method_with_overriding_retry_and_timeout(unusued_sleep):
+ method = mock.Mock(
+ spec=['__call__'],
+ side_effect=[exceptions.NotFound(None), 42]
+ )
+ default_retry = retry.Retry()
+ default_timeout = timeout.ConstantTimeout(60)
+ wrapped_method = google.api_core.gapic_v1.method.wrap_method(
+ method, default_retry, default_timeout)
+
+ result = wrapped_method(
+ retry=retry.Retry(retry.if_exception_type(exceptions.NotFound)),
+ timeout=timeout.ConstantTimeout(22))
+
+ assert result == 42
+ assert method.call_count == 2
+ method.assert_called_with(timeout=22, metadata=mock.ANY)
+
+
+@mock.patch('time.sleep')
+@mock.patch(
+ 'google.api_core.datetime_helpers.utcnow',
+ side_effect=_utcnow_monotonic(),
+ autospec=True)
+def test_wrap_method_with_overriding_retry_deadline(utcnow, unused_sleep):
+ method = mock.Mock(
+ spec=['__call__'],
+ side_effect=([exceptions.InternalServerError(None)] * 4) + [42]
+ )
+ default_retry = retry.Retry()
+ default_timeout = timeout.ExponentialTimeout(deadline=60)
+ wrapped_method = google.api_core.gapic_v1.method.wrap_method(
+ method, default_retry, default_timeout)
+
+ # Overriding only the retry's deadline should also override the timeout's
+ # deadline.
+ result = wrapped_method(
+ retry=default_retry.with_deadline(30))
+
+ assert result == 42
+ timeout_args = [call[1]['timeout'] for call in method.call_args_list]
+ assert timeout_args == [5.0, 10.0, 20.0, 26.0, 25.0]
+ assert utcnow.call_count == (
+ 1 + # First to set the deadline.
+ 5 + # One for each min(timeout, maximum, (DEADLINE - NOW).seconds)
+ 5
+ )
+
+
+def test_wrap_method_with_overriding_timeout_as_a_number():
+ method = mock.Mock(spec=['__call__'], return_value=42)
+ default_retry = retry.Retry()
+ default_timeout = timeout.ConstantTimeout(60)
+ wrapped_method = google.api_core.gapic_v1.method.wrap_method(
+ method, default_retry, default_timeout)
+
+ result = wrapped_method(timeout=22)
+
+ assert result == 42
+ method.assert_called_once_with(timeout=22, metadata=mock.ANY)
+
+
+def test_wrap_with_paging():
+ page_one = mock.Mock(
+ spec=['items', 'page_token', 'next_page_token'],
+ items=[1, 2],
+ next_page_token='icanhasnextpls')
+ page_two = mock.Mock(
+ spec=['items', 'page_token', 'next_page_token'],
+ items=[3, 4],
+ next_page_token=None)
+ method = mock.Mock(
+ spec=['__call__', '__name__'], side_effect=(page_one, page_two))
+ method.__name__ = 'mockmethod'
+
+ wrapped_method = google.api_core.gapic_v1.method.wrap_with_paging(
+ method, 'items', 'page_token', 'next_page_token')
+
+ request = mock.Mock(spec=['page_token'], page_token=None)
+ result = wrapped_method(request, extra='param')
+
+ # Should return an iterator and should not have actually called the
+ # method yet.
+ assert isinstance(result, google.api_core.page_iterator.Iterator)
+ method.assert_not_called()
+ assert request.page_token is None
+
+ # Draining the iterator should call the method until no more pages are
+ # returned.
+ results = list(result)
+
+ assert results == [1, 2, 3, 4]
+ assert method.call_count == 2
+ method.assert_called_with(request, extra='param')
+ assert request.page_token == 'icanhasnextpls'
diff --git a/tests/unit/operations_v1/__init__.py b/tests/unit/operations_v1/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/unit/operations_v1/__init__.py
diff --git a/tests/unit/operations_v1/test_operations_client.py b/tests/unit/operations_v1/test_operations_client.py
new file mode 100644
index 0000000..60b11b7
--- /dev/null
+++ b/tests/unit/operations_v1/test_operations_client.py
@@ -0,0 +1,101 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+from google.api_core import operations_v1
+from google.api_core import page_iterator
+from google.longrunning import operations_pb2
+
+
+def make_operations_stub(channel):
+ return mock.Mock(
+ spec=[
+ 'GetOperation', 'DeleteOperation', 'ListOperations',
+ 'CancelOperation'])
+
+
+operations_stub_patch = mock.patch(
+ 'google.longrunning.operations_pb2.OperationsStub',
+ autospec=True,
+ side_effect=make_operations_stub)
+
+
+@operations_stub_patch
+def test_constructor(operations_stub):
+ stub = make_operations_stub(None)
+ operations_stub.side_effect = None
+ operations_stub.return_value = stub
+
+ client = operations_v1.OperationsClient(mock.sentinel.channel)
+
+ assert client.operations_stub == stub
+ operations_stub.assert_called_once_with(mock.sentinel.channel)
+
+
+@operations_stub_patch
+def test_get_operation(operations_stub):
+ client = operations_v1.OperationsClient(mock.sentinel.channel)
+ client.operations_stub.GetOperation.return_value = mock.sentinel.operation
+
+ response = client.get_operation('name')
+
+ request = client.operations_stub.GetOperation.call_args[0][0]
+ assert isinstance(request, operations_pb2.GetOperationRequest)
+ assert request.name == 'name'
+
+ assert response == mock.sentinel.operation
+
+
+@operations_stub_patch
+def test_list_operations(operations_stub):
+ client = operations_v1.OperationsClient(mock.sentinel.channel)
+ operations = [
+ operations_pb2.Operation(name='1'),
+ operations_pb2.Operation(name='2')]
+ list_response = operations_pb2.ListOperationsResponse(
+ operations=operations)
+ client.operations_stub.ListOperations.return_value = list_response
+
+ response = client.list_operations('name', 'filter')
+
+ assert isinstance(response, page_iterator.Iterator)
+ assert list(response) == operations
+
+ request = client.operations_stub.ListOperations.call_args[0][0]
+ assert isinstance(request, operations_pb2.ListOperationsRequest)
+ assert request.name == 'name'
+ assert request.filter == 'filter'
+
+
+@operations_stub_patch
+def test_delete_operation(operations_stub):
+ client = operations_v1.OperationsClient(mock.sentinel.channel)
+
+ client.delete_operation('name')
+
+ request = client.operations_stub.DeleteOperation.call_args[0][0]
+ assert isinstance(request, operations_pb2.DeleteOperationRequest)
+ assert request.name == 'name'
+
+
+@operations_stub_patch
+def test_cancel_operation(operations_stub):
+ client = operations_v1.OperationsClient(mock.sentinel.channel)
+
+ client.cancel_operation('name')
+
+ request = client.operations_stub.CancelOperation.call_args[0][0]
+ assert isinstance(request, operations_pb2.CancelOperationRequest)
+ assert request.name == 'name'
diff --git a/tests/unit/test_datetime_helpers.py b/tests/unit/test_datetime_helpers.py
new file mode 100644
index 0000000..24f8dbd
--- /dev/null
+++ b/tests/unit/test_datetime_helpers.py
@@ -0,0 +1,22 @@
+# Copyright 2017, Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+
+from google.api_core import datetime_helpers
+
+
+def test_utcnow():
+ result = datetime_helpers.utcnow()
+ assert isinstance(result, datetime.datetime)
diff --git a/tests/unit/test_exceptions.py b/tests/unit/test_exceptions.py
new file mode 100644
index 0000000..df159be
--- /dev/null
+++ b/tests/unit/test_exceptions.py
@@ -0,0 +1,201 @@
+# Copyright 2014 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import grpc
+import mock
+import requests
+from six.moves import http_client
+
+from google.api_core import exceptions
+
+
+def test_create_google_cloud_error():
+ exception = exceptions.GoogleAPICallError('Testing')
+ exception.code = 600
+ assert str(exception) == '600 Testing'
+ assert exception.message == 'Testing'
+ assert exception.errors == []
+ assert exception.response is None
+
+
+def test_create_google_cloud_error_with_args():
+ error = {
+ 'domain': 'global',
+ 'location': 'test',
+ 'locationType': 'testing',
+ 'message': 'Testing',
+ 'reason': 'test',
+ }
+ response = mock.sentinel.response
+ exception = exceptions.GoogleAPICallError(
+ 'Testing', [error], response=response)
+ exception.code = 600
+ assert str(exception) == '600 Testing'
+ assert exception.message == 'Testing'
+ assert exception.errors == [error]
+ assert exception.response == response
+
+
+def test_from_http_status():
+ message = 'message'
+ exception = exceptions.from_http_status(http_client.NOT_FOUND, message)
+ assert exception.code == http_client.NOT_FOUND
+ assert exception.message == message
+ assert exception.errors == []
+
+
+def test_from_http_status_with_errors_and_response():
+ message = 'message'
+ errors = ['1', '2']
+ response = mock.sentinel.response
+ exception = exceptions.from_http_status(
+ http_client.NOT_FOUND, message, errors=errors, response=response)
+
+ assert isinstance(exception, exceptions.NotFound)
+ assert exception.code == http_client.NOT_FOUND
+ assert exception.message == message
+ assert exception.errors == errors
+ assert exception.response == response
+
+
+def test_from_http_status_unknown_code():
+ message = 'message'
+ status_code = 156
+ exception = exceptions.from_http_status(status_code, message)
+ assert exception.code == status_code
+ assert exception.message == message
+
+
+def make_response(content):
+ response = requests.Response()
+ response._content = content
+ response.status_code = http_client.NOT_FOUND
+ response.request = requests.Request(
+ method='POST', url='https://example.com').prepare()
+ return response
+
+
+def test_from_http_response_no_content():
+ response = make_response(None)
+
+ exception = exceptions.from_http_response(response)
+
+ assert isinstance(exception, exceptions.NotFound)
+ assert exception.code == http_client.NOT_FOUND
+ assert exception.message == 'POST https://example.com/: unknown error'
+ assert exception.response == response
+
+
+def test_from_http_response_text_content():
+ response = make_response(b'message')
+
+ exception = exceptions.from_http_response(response)
+
+ assert isinstance(exception, exceptions.NotFound)
+ assert exception.code == http_client.NOT_FOUND
+ assert exception.message == 'POST https://example.com/: message'
+
+
+def test_from_http_response_json_content():
+ response = make_response(json.dumps({
+ 'error': {
+ 'message': 'json message',
+ 'errors': ['1', '2']
+ }
+ }).encode('utf-8'))
+
+ exception = exceptions.from_http_response(response)
+
+ assert isinstance(exception, exceptions.NotFound)
+ assert exception.code == http_client.NOT_FOUND
+ assert exception.message == 'POST https://example.com/: json message'
+ assert exception.errors == ['1', '2']
+
+
+def test_from_http_response_bad_json_content():
+ response = make_response(json.dumps({'meep': 'moop'}).encode('utf-8'))
+
+ exception = exceptions.from_http_response(response)
+
+ assert isinstance(exception, exceptions.NotFound)
+ assert exception.code == http_client.NOT_FOUND
+ assert exception.message == 'POST https://example.com/: unknown error'
+
+
+def test_from_grpc_status():
+ message = 'message'
+ exception = exceptions.from_grpc_status(
+ grpc.StatusCode.OUT_OF_RANGE, message)
+ assert isinstance(exception, exceptions.BadRequest)
+ assert isinstance(exception, exceptions.OutOfRange)
+ assert exception.code == http_client.BAD_REQUEST
+ assert exception.grpc_status_code == grpc.StatusCode.OUT_OF_RANGE
+ assert exception.message == message
+ assert exception.errors == []
+
+
+def test_from_grpc_status_with_errors_and_response():
+ message = 'message'
+ response = mock.sentinel.response
+ errors = ['1', '2']
+ exception = exceptions.from_grpc_status(
+ grpc.StatusCode.OUT_OF_RANGE, message,
+ errors=errors, response=response)
+
+ assert isinstance(exception, exceptions.OutOfRange)
+ assert exception.message == message
+ assert exception.errors == errors
+ assert exception.response == response
+
+
+def test_from_grpc_status_unknown_code():
+ message = 'message'
+ exception = exceptions.from_grpc_status(
+ grpc.StatusCode.OK, message)
+ assert exception.grpc_status_code == grpc.StatusCode.OK
+ assert exception.message == message
+
+
+def test_from_grpc_error():
+ message = 'message'
+ error = mock.create_autospec(grpc.Call, instance=True)
+ error.code.return_value = grpc.StatusCode.INVALID_ARGUMENT
+ error.details.return_value = message
+
+ exception = exceptions.from_grpc_error(error)
+
+ assert isinstance(exception, exceptions.BadRequest)
+ assert isinstance(exception, exceptions.InvalidArgument)
+ assert exception.code == http_client.BAD_REQUEST
+ assert exception.grpc_status_code == grpc.StatusCode.INVALID_ARGUMENT
+ assert exception.message == message
+ assert exception.errors == [error]
+ assert exception.response == error
+
+
+def test_from_grpc_error_non_call():
+ message = 'message'
+ error = mock.create_autospec(grpc.RpcError, instance=True)
+ error.__str__.return_value = message
+
+ exception = exceptions.from_grpc_error(error)
+
+ assert isinstance(exception, exceptions.GoogleAPICallError)
+ assert exception.code is None
+ assert exception.grpc_status_code is None
+ assert exception.message == message
+ assert exception.errors == [error]
+ assert exception.response == error
diff --git a/tests/unit/test_general_helpers.py b/tests/unit/test_general_helpers.py
new file mode 100644
index 0000000..b878cc5
--- /dev/null
+++ b/tests/unit/test_general_helpers.py
@@ -0,0 +1,43 @@
+# Copyright 2017, Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+
+from google.api_core import general_helpers
+
+
+def test_wraps_normal_func():
+
+ def func():
+ return 42
+
+ @general_helpers.wraps(func)
+ def replacement():
+ return func()
+
+ assert replacement() == 42
+
+
+def test_wraps_partial():
+
+ def func():
+ return 42
+
+ partial = functools.partial(func)
+
+ @general_helpers.wraps(partial)
+ def replacement():
+ return func()
+
+ assert replacement() == 42
diff --git a/tests/unit/test_grpc_helpers.py b/tests/unit/test_grpc_helpers.py
new file mode 100644
index 0000000..d5e0b3c
--- /dev/null
+++ b/tests/unit/test_grpc_helpers.py
@@ -0,0 +1,171 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import grpc
+import mock
+import pytest
+
+from google.api_core import exceptions
+from google.api_core import grpc_helpers
+
+
+def test__patch_callable_name():
+ callable = mock.Mock(spec=['__class__'])
+ callable.__class__ = mock.Mock(spec=['__name__'])
+ callable.__class__.__name__ = 'TestCallable'
+
+ grpc_helpers._patch_callable_name(callable)
+
+ assert callable.__name__ == 'TestCallable'
+
+
+def test__patch_callable_name_no_op():
+ callable = mock.Mock(spec=['__name__'])
+ callable.__name__ = 'test_callable'
+
+ grpc_helpers._patch_callable_name(callable)
+
+ assert callable.__name__ == 'test_callable'
+
+
+class RpcErrorImpl(grpc.RpcError, grpc.Call):
+ def __init__(self, code):
+ super(RpcErrorImpl, self).__init__()
+ self._code = code
+
+ def code(self):
+ return self._code
+
+ def details(self):
+ return None
+
+
+def test_wrap_unary_errors():
+ grpc_error = RpcErrorImpl(grpc.StatusCode.INVALID_ARGUMENT)
+ callable_ = mock.Mock(spec=['__call__'], side_effect=grpc_error)
+
+ wrapped_callable = grpc_helpers._wrap_unary_errors(callable_)
+
+ with pytest.raises(exceptions.InvalidArgument) as exc_info:
+ wrapped_callable(1, 2, three='four')
+
+ callable_.assert_called_once_with(1, 2, three='four')
+ assert exc_info.value.response == grpc_error
+
+
+def test_wrap_stream_errors_invocation():
+ grpc_error = RpcErrorImpl(grpc.StatusCode.INVALID_ARGUMENT)
+ callable_ = mock.Mock(spec=['__call__'], side_effect=grpc_error)
+
+ wrapped_callable = grpc_helpers._wrap_stream_errors(callable_)
+
+ with pytest.raises(exceptions.InvalidArgument) as exc_info:
+ wrapped_callable(1, 2, three='four')
+
+ callable_.assert_called_once_with(1, 2, three='four')
+ assert exc_info.value.response == grpc_error
+
+
+class RpcResponseIteratorImpl(object):
+ def __init__(self, exception):
+ self._exception = exception
+
+ # Note: This matches grpc._channel._Rendezvous._next which is what is
+ # patched by _wrap_stream_errors.
+ def _next(self):
+ raise self._exception
+
+ def __next__(self): # pragma: NO COVER
+ return self._next()
+
+ def next(self): # pragma: NO COVER
+ return self._next()
+
+
+def test_wrap_stream_errors_iterator():
+ grpc_error = RpcErrorImpl(grpc.StatusCode.UNAVAILABLE)
+ response_iter = RpcResponseIteratorImpl(grpc_error)
+ callable_ = mock.Mock(spec=['__call__'], return_value=response_iter)
+
+ wrapped_callable = grpc_helpers._wrap_stream_errors(callable_)
+
+ got_iterator = wrapped_callable(1, 2, three='four')
+
+ with pytest.raises(exceptions.ServiceUnavailable) as exc_info:
+ next(got_iterator)
+
+ assert got_iterator == response_iter
+ callable_.assert_called_once_with(1, 2, three='four')
+ assert exc_info.value.response == grpc_error
+
+
+@mock.patch('google.api_core.grpc_helpers._wrap_unary_errors')
+def test_wrap_errors_non_streaming(wrap_unary_errors):
+ callable_ = mock.create_autospec(grpc.UnaryUnaryMultiCallable)
+
+ result = grpc_helpers.wrap_errors(callable_)
+
+ assert result == wrap_unary_errors.return_value
+ wrap_unary_errors.assert_called_once_with(callable_)
+
+
+@mock.patch('google.api_core.grpc_helpers._wrap_stream_errors')
+def test_wrap_errors_streaming(wrap_stream_errors):
+ callable_ = mock.create_autospec(grpc.UnaryStreamMultiCallable)
+
+ result = grpc_helpers.wrap_errors(callable_)
+
+ assert result == wrap_stream_errors.return_value
+ wrap_stream_errors.assert_called_once_with(callable_)
+
+
+@mock.patch(
+ 'google.auth.default',
+ return_value=(mock.sentinel.credentials, mock.sentinel.projet))
+@mock.patch('google.auth.transport.grpc.secure_authorized_channel')
+def test_create_channel_implicit(secure_authorized_channel, default):
+ target = 'example.com:443'
+
+ channel = grpc_helpers.create_channel(target)
+
+ assert channel is secure_authorized_channel.return_value
+ default.assert_called_once_with(scopes=None)
+ secure_authorized_channel.assert_called_once_with(
+ mock.sentinel.credentials, mock.ANY, target)
+
+
+@mock.patch(
+ 'google.auth.default',
+ return_value=(mock.sentinel.credentials, mock.sentinel.projet))
+@mock.patch('google.auth.transport.grpc.secure_authorized_channel')
+def test_create_channel_implicit_with_scopes(
+ secure_authorized_channel, default):
+ target = 'example.com:443'
+
+ channel = grpc_helpers.create_channel(target, scopes=['one', 'two'])
+
+ assert channel is secure_authorized_channel.return_value
+ default.assert_called_once_with(scopes=['one', 'two'])
+
+
+@mock.patch('google.auth.transport.grpc.secure_authorized_channel')
+def test_create_channel_explicit(secure_authorized_channel):
+ target = 'example.com:443'
+
+ channel = grpc_helpers.create_channel(
+ target, credentials=mock.sentinel.credentials)
+
+ assert channel is secure_authorized_channel.return_value
+ secure_authorized_channel.assert_called_once_with(
+ mock.sentinel.credentials, mock.ANY, target)
diff --git a/tests/unit/test_operation.py b/tests/unit/test_operation.py
new file mode 100644
index 0000000..1d765cc
--- /dev/null
+++ b/tests/unit/test_operation.py
@@ -0,0 +1,223 @@
+# Copyright 2017, Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import mock
+
+from google.api_core import operation
+from google.api_core import operations_v1
+from google.longrunning import operations_pb2
+from google.protobuf import struct_pb2
+from google.rpc import code_pb2
+from google.rpc import status_pb2
+
+TEST_OPERATION_NAME = 'test/operation'
+
+
+def make_operation_proto(
+ name=TEST_OPERATION_NAME, metadata=None, response=None,
+ error=None, **kwargs):
+ operation_proto = operations_pb2.Operation(
+ name=name, **kwargs)
+
+ if metadata is not None:
+ operation_proto.metadata.Pack(metadata)
+
+ if response is not None:
+ operation_proto.response.Pack(response)
+
+ if error is not None:
+ operation_proto.error.CopyFrom(error)
+
+ return operation_proto
+
+
+def make_operation_future(client_operations_responses=None):
+ if client_operations_responses is None:
+ client_operations_responses = [make_operation_proto()]
+
+ refresh = mock.Mock(
+ spec=['__call__'], side_effect=client_operations_responses)
+ refresh.responses = client_operations_responses
+ cancel = mock.Mock(spec=['__call__'])
+ operation_future = operation.Operation(
+ client_operations_responses[0],
+ refresh,
+ cancel,
+ result_type=struct_pb2.Struct,
+ metadata_type=struct_pb2.Struct)
+
+ return operation_future, refresh, cancel
+
+
+def test_constructor():
+ future, refresh, _ = make_operation_future()
+
+ assert future.operation == refresh.responses[0]
+ assert future.operation.done is False
+ assert future.operation.name == TEST_OPERATION_NAME
+ assert future.metadata is None
+ assert future.running()
+
+
+def test_metadata():
+ expected_metadata = struct_pb2.Struct()
+ future, _, _ = make_operation_future(
+ [make_operation_proto(metadata=expected_metadata)])
+
+ assert future.metadata == expected_metadata
+
+
+def test_cancellation():
+ responses = [
+ make_operation_proto(),
+ # Second response indicates that the operation was cancelled.
+ make_operation_proto(
+ done=True,
+ error=status_pb2.Status(code=code_pb2.CANCELLED))]
+ future, _, cancel = make_operation_future(responses)
+
+ assert future.cancel()
+ assert future.cancelled()
+ cancel.assert_called_once_with()
+
+ # Cancelling twice should have no effect.
+ assert not future.cancel()
+ cancel.assert_called_once_with()
+
+
+def test_result():
+ expected_result = struct_pb2.Struct()
+ responses = [
+ make_operation_proto(),
+ # Second operation response includes the result.
+ make_operation_proto(done=True, response=expected_result)]
+ future, _, _ = make_operation_future(responses)
+
+ result = future.result()
+
+ assert result == expected_result
+ assert future.done()
+
+
+def test_exception():
+ expected_exception = status_pb2.Status(message='meep')
+ responses = [
+ make_operation_proto(),
+ # Second operation response includes the error.
+ make_operation_proto(done=True, error=expected_exception)]
+ future, _, _ = make_operation_future(responses)
+
+ exception = future.exception()
+
+ assert expected_exception.message in '{!r}'.format(exception)
+
+
+def test_unexpected_result():
+ responses = [
+ make_operation_proto(),
+ # Second operation response is done, but has not error or response.
+ make_operation_proto(done=True)]
+ future, _, _ = make_operation_future(responses)
+
+ exception = future.exception()
+
+ assert 'Unexpected state' in '{!r}'.format(exception)
+
+
+def test__refresh_http():
+ api_request = mock.Mock(
+ return_value={'name': TEST_OPERATION_NAME, 'done': True})
+
+ result = operation._refresh_http(api_request, TEST_OPERATION_NAME)
+
+ assert result.name == TEST_OPERATION_NAME
+ assert result.done is True
+ api_request.assert_called_once_with(
+ method='GET', path='operations/{}'.format(TEST_OPERATION_NAME))
+
+
+def test__cancel_http():
+ api_request = mock.Mock()
+
+ operation._cancel_http(api_request, TEST_OPERATION_NAME)
+
+ api_request.assert_called_once_with(
+ method='POST', path='operations/{}:cancel'.format(TEST_OPERATION_NAME))
+
+
+def test_from_http_json():
+ operation_json = {'name': TEST_OPERATION_NAME, 'done': True}
+ api_request = mock.sentinel.api_request
+
+ future = operation.from_http_json(
+ operation_json, api_request, struct_pb2.Struct,
+ metadata_type=struct_pb2.Struct)
+
+ assert future._result_type == struct_pb2.Struct
+ assert future._metadata_type == struct_pb2.Struct
+ assert future.operation.name == TEST_OPERATION_NAME
+ assert future.done
+
+
+def test__refresh_grpc():
+ operations_stub = mock.Mock(spec=['GetOperation'])
+ expected_result = make_operation_proto(done=True)
+ operations_stub.GetOperation.return_value = expected_result
+
+ result = operation._refresh_grpc(operations_stub, TEST_OPERATION_NAME)
+
+ assert result == expected_result
+ expected_request = operations_pb2.GetOperationRequest(
+ name=TEST_OPERATION_NAME)
+ operations_stub.GetOperation.assert_called_once_with(expected_request)
+
+
+def test__cancel_grpc():
+ operations_stub = mock.Mock(spec=['CancelOperation'])
+
+ operation._cancel_grpc(operations_stub, TEST_OPERATION_NAME)
+
+ expected_request = operations_pb2.CancelOperationRequest(
+ name=TEST_OPERATION_NAME)
+ operations_stub.CancelOperation.assert_called_once_with(expected_request)
+
+
+def test_from_grpc():
+ operation_proto = make_operation_proto(done=True)
+ operations_stub = mock.sentinel.operations_stub
+
+ future = operation.from_grpc(
+ operation_proto, operations_stub, struct_pb2.Struct,
+ metadata_type=struct_pb2.Struct)
+
+ assert future._result_type == struct_pb2.Struct
+ assert future._metadata_type == struct_pb2.Struct
+ assert future.operation.name == TEST_OPERATION_NAME
+ assert future.done
+
+
+def test_from_gapic():
+ operation_proto = make_operation_proto(done=True)
+ operations_client = mock.create_autospec(
+ operations_v1.OperationsClient, instance=True)
+
+ future = operation.from_gapic(
+ operation_proto, operations_client, struct_pb2.Struct,
+ metadata_type=struct_pb2.Struct)
+
+ assert future._result_type == struct_pb2.Struct
+ assert future._metadata_type == struct_pb2.Struct
+ assert future.operation.name == TEST_OPERATION_NAME
+ assert future.done
diff --git a/tests/unit/test_page_iterator.py b/tests/unit/test_page_iterator.py
new file mode 100644
index 0000000..5cecac8
--- /dev/null
+++ b/tests/unit/test_page_iterator.py
@@ -0,0 +1,545 @@
+# Copyright 2015 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import types
+
+import mock
+import pytest
+import six
+
+from google.api_core import page_iterator
+
+
+def test__do_nothing_page_start():
+ assert page_iterator._do_nothing_page_start(None, None, None) is None
+
+
+class TestPage(object):
+
+ def test_constructor(self):
+ parent = mock.sentinel.parent
+ item_to_value = mock.sentinel.item_to_value
+
+ page = page_iterator.Page(parent, (1, 2, 3), item_to_value)
+
+ assert page.num_items == 3
+ assert page.remaining == 3
+ assert page._parent is parent
+ assert page._item_to_value is item_to_value
+
+ def test___iter__(self):
+ page = page_iterator.Page(None, (), None)
+ assert iter(page) is page
+
+ def test_iterator_calls_parent_item_to_value(self):
+ parent = mock.sentinel.parent
+
+ item_to_value = mock.Mock(
+ side_effect=lambda iterator, value: value, spec=['__call__'])
+
+ page = page_iterator.Page(parent, (10, 11, 12), item_to_value)
+ page._remaining = 100
+
+ assert item_to_value.call_count == 0
+ assert page.remaining == 100
+
+ assert six.next(page) == 10
+ assert item_to_value.call_count == 1
+ item_to_value.assert_called_with(parent, 10)
+ assert page.remaining == 99
+
+ assert six.next(page) == 11
+ assert item_to_value.call_count == 2
+ item_to_value.assert_called_with(parent, 11)
+ assert page.remaining == 98
+
+ assert six.next(page) == 12
+ assert item_to_value.call_count == 3
+ item_to_value.assert_called_with(parent, 12)
+ assert page.remaining == 97
+
+
+class PageIteratorImpl(page_iterator.Iterator):
+ def _next_page(self):
+ return mock.create_autospec(page_iterator.Page, instance=True)
+
+
+class TestIterator(object):
+
+ def test_constructor(self):
+ client = mock.sentinel.client
+ item_to_value = mock.sentinel.item_to_value
+ token = 'ab13nceor03'
+ max_results = 1337
+
+ iterator = PageIteratorImpl(
+ client, item_to_value, page_token=token, max_results=max_results)
+
+ assert not iterator._started
+ assert iterator.client is client
+ assert iterator._item_to_value == item_to_value
+ assert iterator.max_results == max_results
+ # Changing attributes.
+ assert iterator.page_number == 0
+ assert iterator.next_page_token == token
+ assert iterator.num_results == 0
+
+ def test_pages_property_starts(self):
+ iterator = PageIteratorImpl(None, None)
+
+ assert not iterator._started
+
+ assert isinstance(iterator.pages, types.GeneratorType)
+
+ assert iterator._started
+
+ def test_pages_property_restart(self):
+ iterator = PageIteratorImpl(None, None)
+
+ assert iterator.pages
+
+ # Make sure we cannot restart.
+ with pytest.raises(ValueError):
+ assert iterator.pages
+
+ def test__page_iter_increment(self):
+ iterator = PageIteratorImpl(None, None)
+ page = page_iterator.Page(
+ iterator, ('item',), page_iterator._item_to_value_identity)
+ iterator._next_page = mock.Mock(side_effect=[page, None])
+
+ assert iterator.num_results == 0
+
+ page_iter = iterator._page_iter(increment=True)
+ next(page_iter)
+
+ assert iterator.num_results == 1
+
+ def test__page_iter_no_increment(self):
+ iterator = PageIteratorImpl(None, None)
+
+ assert iterator.num_results == 0
+
+ page_iter = iterator._page_iter(increment=False)
+ next(page_iter)
+
+ # results should still be 0 after fetching a page.
+ assert iterator.num_results == 0
+
+ def test__items_iter(self):
+ # Items to be returned.
+ item1 = 17
+ item2 = 100
+ item3 = 211
+
+ # Make pages from mock responses
+ parent = mock.sentinel.parent
+ page1 = page_iterator.Page(
+ parent, (item1, item2), page_iterator._item_to_value_identity)
+ page2 = page_iterator.Page(
+ parent, (item3,), page_iterator._item_to_value_identity)
+
+ iterator = PageIteratorImpl(None, None)
+ iterator._next_page = mock.Mock(side_effect=[page1, page2, None])
+
+ items_iter = iterator._items_iter()
+
+ assert isinstance(items_iter, types.GeneratorType)
+
+ # Consume items and check the state of the iterator.
+ assert iterator.num_results == 0
+
+ assert six.next(items_iter) == item1
+ assert iterator.num_results == 1
+
+ assert six.next(items_iter) == item2
+ assert iterator.num_results == 2
+
+ assert six.next(items_iter) == item3
+ assert iterator.num_results == 3
+
+ with pytest.raises(StopIteration):
+ six.next(items_iter)
+
+ def test___iter__(self):
+ iterator = PageIteratorImpl(None, None)
+ iterator._next_page = mock.Mock(side_effect=[(1, 2), (3,), None])
+
+ assert not iterator._started
+
+ result = list(iterator)
+
+ assert result == [1, 2, 3]
+ assert iterator._started
+
+ def test___iter__restart(self):
+ iterator = PageIteratorImpl(None, None)
+
+ iter(iterator)
+
+ # Make sure we cannot restart.
+ with pytest.raises(ValueError):
+ iter(iterator)
+
+ def test___iter___restart_after_page(self):
+ iterator = PageIteratorImpl(None, None)
+
+ assert iterator.pages
+
+ # Make sure we cannot restart after starting the page iterator
+ with pytest.raises(ValueError):
+ iter(iterator)
+
+
+class TestHTTPIterator(object):
+
+ def test_constructor(self):
+ client = mock.sentinel.client
+ path = '/foo'
+ iterator = page_iterator.HTTPIterator(
+ client, mock.sentinel.api_request,
+ path, mock.sentinel.item_to_value)
+
+ assert not iterator._started
+ assert iterator.client is client
+ assert iterator.path == path
+ assert iterator._item_to_value is mock.sentinel.item_to_value
+ assert iterator._items_key == 'items'
+ assert iterator.max_results is None
+ assert iterator.extra_params == {}
+ assert iterator._page_start == page_iterator._do_nothing_page_start
+ # Changing attributes.
+ assert iterator.page_number == 0
+ assert iterator.next_page_token is None
+ assert iterator.num_results == 0
+
+ def test_constructor_w_extra_param_collision(self):
+ extra_params = {'pageToken': 'val'}
+
+ with pytest.raises(ValueError):
+ page_iterator.HTTPIterator(
+ mock.sentinel.client,
+ mock.sentinel.api_request,
+ mock.sentinel.path,
+ mock.sentinel.item_to_value,
+ extra_params=extra_params)
+
+ def test_iterate(self):
+ path = '/foo'
+ item1 = {'name': '1'}
+ item2 = {'name': '2'}
+ api_request = mock.Mock(return_value={'items': [item1, item2]})
+ iterator = page_iterator.HTTPIterator(
+ mock.sentinel.client, api_request, path=path,
+ item_to_value=page_iterator._item_to_value_identity)
+
+ assert iterator.num_results == 0
+
+ items_iter = iter(iterator)
+
+ val1 = six.next(items_iter)
+ assert val1 == item1
+ assert iterator.num_results == 1
+
+ val2 = six.next(items_iter)
+ assert val2 == item2
+ assert iterator.num_results == 2
+
+ with pytest.raises(StopIteration):
+ six.next(items_iter)
+
+ api_request.assert_called_once_with(
+ method='GET', path=path, query_params={})
+
+ def test__has_next_page_new(self):
+ iterator = page_iterator.HTTPIterator(
+ mock.sentinel.client,
+ mock.sentinel.api_request,
+ mock.sentinel.path,
+ mock.sentinel.item_to_value)
+
+ # The iterator should *always* indicate that it has a next page
+ # when created so that it can fetch the initial page.
+ assert iterator._has_next_page()
+
+ def test__has_next_page_without_token(self):
+ iterator = page_iterator.HTTPIterator(
+ mock.sentinel.client,
+ mock.sentinel.api_request,
+ mock.sentinel.path,
+ mock.sentinel.item_to_value)
+
+ iterator.page_number = 1
+
+ # The iterator should not indicate that it has a new page if the
+ # initial page has been requested and there's no page token.
+ assert not iterator._has_next_page()
+
+ def test__has_next_page_w_number_w_token(self):
+ iterator = page_iterator.HTTPIterator(
+ mock.sentinel.client,
+ mock.sentinel.api_request,
+ mock.sentinel.path,
+ mock.sentinel.item_to_value)
+
+ iterator.page_number = 1
+ iterator.next_page_token = mock.sentinel.token
+
+ # The iterator should indicate that it has a new page if the
+ # initial page has been requested and there's is a page token.
+ assert iterator._has_next_page()
+
+ def test__has_next_page_w_max_results_not_done(self):
+ iterator = page_iterator.HTTPIterator(
+ mock.sentinel.client,
+ mock.sentinel.api_request,
+ mock.sentinel.path,
+ mock.sentinel.item_to_value,
+ max_results=3,
+ page_token=mock.sentinel.token)
+
+ iterator.page_number = 1
+
+ # The iterator should indicate that it has a new page if there
+ # is a page token and it has not consumed more than max_results.
+ assert iterator.num_results < iterator.max_results
+ assert iterator._has_next_page()
+
+ def test__has_next_page_w_max_results_done(self):
+
+ iterator = page_iterator.HTTPIterator(
+ mock.sentinel.client,
+ mock.sentinel.api_request,
+ mock.sentinel.path,
+ mock.sentinel.item_to_value,
+ max_results=3,
+ page_token=mock.sentinel.token)
+
+ iterator.page_number = 1
+ iterator.num_results = 3
+
+ # The iterator should not indicate that it has a new page if there
+ # if it has consumed more than max_results.
+ assert iterator.num_results == iterator.max_results
+ assert not iterator._has_next_page()
+
+ def test__get_query_params_no_token(self):
+ iterator = page_iterator.HTTPIterator(
+ mock.sentinel.client,
+ mock.sentinel.api_request,
+ mock.sentinel.path,
+ mock.sentinel.item_to_value)
+
+ assert iterator._get_query_params() == {}
+
+ def test__get_query_params_w_token(self):
+ iterator = page_iterator.HTTPIterator(
+ mock.sentinel.client,
+ mock.sentinel.api_request,
+ mock.sentinel.path,
+ mock.sentinel.item_to_value)
+ iterator.next_page_token = 'token'
+
+ assert iterator._get_query_params() == {
+ 'pageToken': iterator.next_page_token}
+
+ def test__get_query_params_w_max_results(self):
+ max_results = 3
+ iterator = page_iterator.HTTPIterator(
+ mock.sentinel.client,
+ mock.sentinel.api_request,
+ mock.sentinel.path,
+ mock.sentinel.item_to_value,
+ max_results=max_results)
+
+ iterator.num_results = 1
+ local_max = max_results - iterator.num_results
+
+ assert iterator._get_query_params() == {
+ 'maxResults': local_max}
+
+ def test__get_query_params_extra_params(self):
+ extra_params = {'key': 'val'}
+ iterator = page_iterator.HTTPIterator(
+ mock.sentinel.client,
+ mock.sentinel.api_request,
+ mock.sentinel.path,
+ mock.sentinel.item_to_value,
+ extra_params=extra_params)
+
+ assert iterator._get_query_params() == extra_params
+
+ def test__get_next_page_response_with_post(self):
+ path = '/foo'
+ page_response = {'items': ['one', 'two']}
+ api_request = mock.Mock(return_value=page_response)
+ iterator = page_iterator.HTTPIterator(
+ mock.sentinel.client, api_request, path=path,
+ item_to_value=page_iterator._item_to_value_identity)
+ iterator._HTTP_METHOD = 'POST'
+
+ response = iterator._get_next_page_response()
+
+ assert response == page_response
+
+ api_request.assert_called_once_with(
+ method='POST', path=path, data={})
+
+ def test__get_next_page_bad_http_method(self):
+ iterator = page_iterator.HTTPIterator(
+ mock.sentinel.client,
+ mock.sentinel.api_request,
+ mock.sentinel.path,
+ mock.sentinel.item_to_value)
+ iterator._HTTP_METHOD = 'NOT-A-VERB'
+
+ with pytest.raises(ValueError):
+ iterator._get_next_page_response()
+
+
+class TestGRPCIterator(object):
+
+ def test_constructor(self):
+ client = mock.sentinel.client
+ items_field = 'items'
+ iterator = page_iterator.GRPCIterator(
+ client, mock.sentinel.method, mock.sentinel.request, items_field)
+
+ assert not iterator._started
+ assert iterator.client is client
+ assert iterator.max_results is None
+ assert iterator._method == mock.sentinel.method
+ assert iterator._request == mock.sentinel.request
+ assert iterator._items_field == items_field
+ assert iterator._item_to_value is page_iterator._item_to_value_identity
+ assert (iterator._request_token_field ==
+ page_iterator.GRPCIterator._DEFAULT_REQUEST_TOKEN_FIELD)
+ assert (iterator._response_token_field ==
+ page_iterator.GRPCIterator._DEFAULT_RESPONSE_TOKEN_FIELD)
+ # Changing attributes.
+ assert iterator.page_number == 0
+ assert iterator.next_page_token is None
+ assert iterator.num_results == 0
+
+ def test_constructor_options(self):
+ client = mock.sentinel.client
+ items_field = 'items'
+ request_field = 'request'
+ response_field = 'response'
+ iterator = page_iterator.GRPCIterator(
+ client, mock.sentinel.method, mock.sentinel.request, items_field,
+ item_to_value=mock.sentinel.item_to_value,
+ request_token_field=request_field,
+ response_token_field=response_field,
+ max_results=42)
+
+ assert iterator.client is client
+ assert iterator.max_results == 42
+ assert iterator._method == mock.sentinel.method
+ assert iterator._request == mock.sentinel.request
+ assert iterator._items_field == items_field
+ assert iterator._item_to_value is mock.sentinel.item_to_value
+ assert iterator._request_token_field == request_field
+ assert iterator._response_token_field == response_field
+
+ def test_iterate(self):
+ request = mock.Mock(spec=['page_token'], page_token=None)
+ response1 = mock.Mock(items=['a', 'b'], next_page_token='1')
+ response2 = mock.Mock(items=['c'], next_page_token='2')
+ response3 = mock.Mock(items=['d'], next_page_token='')
+ method = mock.Mock(side_effect=[response1, response2, response3])
+ iterator = page_iterator.GRPCIterator(
+ mock.sentinel.client, method, request, 'items')
+
+ assert iterator.num_results == 0
+
+ items = list(iterator)
+ assert items == ['a', 'b', 'c', 'd']
+
+ method.assert_called_with(request)
+ assert method.call_count == 3
+ assert request.page_token == '2'
+
+ def test_iterate_with_max_results(self):
+ request = mock.Mock(spec=['page_token'], page_token=None)
+ response1 = mock.Mock(items=['a', 'b'], next_page_token='1')
+ response2 = mock.Mock(items=['c'], next_page_token='2')
+ response3 = mock.Mock(items=['d'], next_page_token='')
+ method = mock.Mock(side_effect=[response1, response2, response3])
+ iterator = page_iterator.GRPCIterator(
+ mock.sentinel.client, method, request, 'items', max_results=3)
+
+ assert iterator.num_results == 0
+
+ items = list(iterator)
+
+ assert items == ['a', 'b', 'c']
+ assert iterator.num_results == 3
+
+ method.assert_called_with(request)
+ assert method.call_count == 2
+ assert request.page_token is '1'
+
+
+class GAXPageIterator(object):
+ """Fake object that matches gax.PageIterator"""
+ def __init__(self, pages, page_token=None):
+ self._pages = iter(pages)
+ self.page_token = page_token
+
+ def next(self):
+ return six.next(self._pages)
+
+ __next__ = next
+
+
+class TestGAXIterator(object):
+
+ def test_constructor(self):
+ client = mock.sentinel.client
+ token = 'zzzyy78kl'
+ page_iter = GAXPageIterator((), page_token=token)
+ item_to_value = page_iterator._item_to_value_identity
+ max_results = 1337
+ iterator = page_iterator._GAXIterator(
+ client, page_iter, item_to_value, max_results=max_results)
+
+ assert not iterator._started
+ assert iterator.client is client
+ assert iterator._item_to_value is item_to_value
+ assert iterator.max_results == max_results
+ assert iterator._gax_page_iter is page_iter
+ # Changing attributes.
+ assert iterator.page_number == 0
+ assert iterator.next_page_token == token
+ assert iterator.num_results == 0
+
+ def test__next_page(self):
+ page_items = (29, 31)
+ page_token = '2sde98ds2s0hh'
+ page_iter = GAXPageIterator([page_items], page_token=page_token)
+ iterator = page_iterator._GAXIterator(
+ mock.sentinel.client,
+ page_iter,
+ page_iterator._item_to_value_identity)
+
+ page = iterator._next_page()
+
+ assert iterator.next_page_token == page_token
+ assert isinstance(page, page_iterator.Page)
+ assert list(page) == list(page_items)
+
+ next_page = iterator._next_page()
+
+ assert next_page is None
diff --git a/tests/unit/test_path_template.py b/tests/unit/test_path_template.py
new file mode 100644
index 0000000..daeeeec
--- /dev/null
+++ b/tests/unit/test_path_template.py
@@ -0,0 +1,90 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import unicode_literals
+
+import mock
+import pytest
+
+from google.api_core import path_template
+
+
+@pytest.mark.parametrize('tmpl, args, kwargs, expected_result', [
+ # Basic positional params
+ ['/v1/*', ['a'], {}, '/v1/a'],
+ ['/v1/**', ['a/b'], {}, '/v1/a/b'],
+ ['/v1/*/*', ['a', 'b'], {}, '/v1/a/b'],
+ ['/v1/*/*/**', ['a', 'b', 'c/d'], {}, '/v1/a/b/c/d'],
+ # Basic named params
+ ['/v1/{name}', [], {'name': 'parent'}, '/v1/parent'],
+ ['/v1/{name=**}', [], {'name': 'parent/child'}, '/v1/parent/child'],
+ # Named params with a sub-template
+ ['/v1/{name=parent/*}', [], {'name': 'parent/child'}, '/v1/parent/child'],
+ ['/v1/{name=parent/**}', [], {'name': 'parent/child/object'},
+ '/v1/parent/child/object'],
+ # Combining positional and named params
+ ['/v1/*/{name}', ['a'], {'name': 'parent'}, '/v1/a/parent'],
+ ['/v1/{name}/*', ['a'], {'name': 'parent'}, '/v1/parent/a'],
+ ['/v1/{parent}/*/{child}/*', ['a', 'b'],
+ {'parent': 'thor', 'child': 'thorson'}, '/v1/thor/a/thorson/b'],
+ ['/v1/{name}/**', ['a/b'], {'name': 'parent'}, '/v1/parent/a/b'],
+ # Combining positional and named params with sub-templates.
+ ['/v1/{name=parent/*}/*', ['a'], {'name': 'parent/child'},
+ '/v1/parent/child/a'],
+ ['/v1/*/{name=parent/**}', ['a'], {'name': 'parent/child/object'},
+ '/v1/a/parent/child/object'],
+])
+def test_expand_success(tmpl, args, kwargs, expected_result):
+ result = path_template.expand(tmpl, *args, **kwargs)
+ assert result == expected_result
+ assert path_template.validate(tmpl, result)
+
+
+@pytest.mark.parametrize('tmpl, args, kwargs, exc_match', [
+ # Missing positional arg.
+ ['v1/*', [], {}, 'Positional'],
+ # Missing named arg.
+ ['v1/{name}', [], {}, 'Named'],
+])
+def test_expanded_failure(tmpl, args, kwargs, exc_match):
+ with pytest.raises(ValueError, match=exc_match):
+ path_template.expand(tmpl, *args, **kwargs)
+
+
+@pytest.mark.parametrize('tmpl, path', [
+ # Single segment template, but multi segment value
+ ['v1/*', 'v1/a/b'],
+ ['v1/*/*', 'v1/a/b/c'],
+ # Single segement named template, but multi segment value
+ ['v1/{name}', 'v1/a/b'],
+ ['v1/{name}/{value}', 'v1/a/b/c'],
+ # Named value with a sub-template but invalid value
+ ['v1/{name=parent/*}', 'v1/grandparent/child'],
+])
+def test_validate_failure(tmpl, path):
+ assert not path_template.validate(tmpl, path)
+
+
+def test__expand_variable_match_unexpected():
+ match = mock.Mock(spec=['group'])
+ match.group.return_value = None
+ with pytest.raises(ValueError, match='Unknown'):
+ path_template._expand_variable_match([], {}, match)
+
+
+def test__replace_variable_with_pattern():
+ match = mock.Mock(spec=['group'])
+ match.group.return_value = None
+ with pytest.raises(ValueError, match='Unknown'):
+ path_template._replace_variable_with_pattern(match)
diff --git a/tests/unit/test_protobuf_helpers.py b/tests/unit/test_protobuf_helpers.py
new file mode 100644
index 0000000..6233536
--- /dev/null
+++ b/tests/unit/test_protobuf_helpers.py
@@ -0,0 +1,37 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from google.api_core import protobuf_helpers
+from google.protobuf import any_pb2
+from google.type import date_pb2
+from google.type import timeofday_pb2
+
+
+def test_from_any_pb_success():
+ in_message = date_pb2.Date(year=1990)
+ in_message_any = any_pb2.Any()
+ in_message_any.Pack(in_message)
+ out_message = protobuf_helpers.from_any_pb(date_pb2.Date, in_message_any)
+
+ assert in_message == out_message
+
+
+def test_from_any_pb_failure():
+ in_message = any_pb2.Any()
+ in_message.Pack(date_pb2.Date(year=1990))
+
+ with pytest.raises(TypeError):
+ protobuf_helpers.from_any_pb(timeofday_pb2.TimeOfDay, in_message)
diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py
new file mode 100644
index 0000000..a671ad3
--- /dev/null
+++ b/tests/unit/test_retry.py
@@ -0,0 +1,255 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import itertools
+import re
+
+import mock
+import pytest
+
+from google.api_core import exceptions
+from google.api_core import retry
+
+
+def test_if_exception_type():
+ predicate = retry.if_exception_type(ValueError)
+
+ assert predicate(ValueError())
+ assert not predicate(TypeError())
+
+
+def test_if_exception_type_multiple():
+ predicate = retry.if_exception_type(ValueError, TypeError)
+
+ assert predicate(ValueError())
+ assert predicate(TypeError())
+ assert not predicate(RuntimeError())
+
+
+def test_if_transient_error():
+ assert retry.if_transient_error(exceptions.InternalServerError(''))
+ assert retry.if_transient_error(exceptions.TooManyRequests(''))
+ assert not retry.if_transient_error(exceptions.InvalidArgument(''))
+
+
+# Make uniform return half of its maximum, which will be the calculated
+# sleep time.
+@mock.patch('random.uniform', autospec=True, side_effect=lambda m, n: n/2.0)
+def test_exponential_sleep_generator_base_2(uniform):
+ gen = retry.exponential_sleep_generator(
+ 1, 60, multiplier=2)
+
+ result = list(itertools.islice(gen, 8))
+ assert result == [1, 2, 4, 8, 16, 32, 60, 60]
+
+
+@mock.patch('time.sleep', autospec=True)
+@mock.patch(
+ 'google.api_core.datetime_helpers.utcnow',
+ return_value=datetime.datetime.min,
+ autospec=True)
+def test_retry_target_success(utcnow, sleep):
+ predicate = retry.if_exception_type(ValueError)
+ call_count = [0]
+
+ def target():
+ call_count[0] += 1
+ if call_count[0] < 3:
+ raise ValueError()
+ return 42
+
+ result = retry.retry_target(target, predicate, range(10), None)
+
+ assert result == 42
+ assert call_count[0] == 3
+ sleep.assert_has_calls([mock.call(0), mock.call(1)])
+
+
+@mock.patch('time.sleep', autospec=True)
+@mock.patch(
+ 'google.api_core.datetime_helpers.utcnow',
+ return_value=datetime.datetime.min,
+ autospec=True)
+def test_retry_target_w_on_error(utcnow, sleep):
+ predicate = retry.if_exception_type(ValueError)
+ call_count = {'target': 0}
+ to_raise = ValueError()
+
+ def target():
+ call_count['target'] += 1
+ if call_count['target'] < 3:
+ raise to_raise
+ return 42
+
+ on_error = mock.Mock()
+
+ result = retry.retry_target(
+ target, predicate, range(10), None, on_error=on_error)
+
+ assert result == 42
+ assert call_count['target'] == 3
+
+ on_error.assert_has_calls([mock.call(to_raise), mock.call(to_raise)])
+ sleep.assert_has_calls([mock.call(0), mock.call(1)])
+
+
+@mock.patch('time.sleep', autospec=True)
+@mock.patch(
+ 'google.api_core.datetime_helpers.utcnow',
+ return_value=datetime.datetime.min,
+ autospec=True)
+def test_retry_target_non_retryable_error(utcnow, sleep):
+ predicate = retry.if_exception_type(ValueError)
+ exception = TypeError()
+ target = mock.Mock(side_effect=exception)
+
+ with pytest.raises(TypeError) as exc_info:
+ retry.retry_target(target, predicate, range(10), None)
+
+ assert exc_info.value == exception
+ sleep.assert_not_called()
+
+
+@mock.patch('time.sleep', autospec=True)
+@mock.patch(
+ 'google.api_core.datetime_helpers.utcnow', autospec=True)
+def test_retry_target_deadline_exceeded(utcnow, sleep):
+ predicate = retry.if_exception_type(ValueError)
+ exception = ValueError('meep')
+ target = mock.Mock(side_effect=exception)
+ # Setup the timeline so that the first call takes 5 seconds but the second
+ # call takes 6, which puts the retry over the deadline.
+ utcnow.side_effect = [
+ # The first call to utcnow establishes the start of the timeline.
+ datetime.datetime.min,
+ datetime.datetime.min + datetime.timedelta(seconds=5),
+ datetime.datetime.min + datetime.timedelta(seconds=11)]
+
+ with pytest.raises(exceptions.RetryError) as exc_info:
+ retry.retry_target(target, predicate, range(10), deadline=10)
+
+ assert exc_info.value.cause == exception
+ assert exc_info.match('Deadline of 10.0s exceeded')
+ assert exc_info.match('last exception: meep')
+ assert target.call_count == 2
+
+
+def test_retry_target_bad_sleep_generator():
+ with pytest.raises(ValueError, match='Sleep generator'):
+ retry.retry_target(
+ mock.sentinel.target, mock.sentinel.predicate, [], None)
+
+
+class TestRetry(object):
+ def test_constructor_defaults(self):
+ retry_ = retry.Retry()
+ assert retry_._predicate == retry.if_transient_error
+ assert retry_._initial == 1
+ assert retry_._maximum == 60
+ assert retry_._multiplier == 2
+ assert retry_._deadline == 120
+
+ def test_constructor_options(self):
+ retry_ = retry.Retry(
+ predicate=mock.sentinel.predicate,
+ initial=1,
+ maximum=2,
+ multiplier=3,
+ deadline=4,
+ )
+ assert retry_._predicate == mock.sentinel.predicate
+ assert retry_._initial == 1
+ assert retry_._maximum == 2
+ assert retry_._multiplier == 3
+ assert retry_._deadline == 4
+
+ def test_with_deadline(self):
+ retry_ = retry.Retry()
+ new_retry = retry_.with_deadline(42)
+ assert retry_ is not new_retry
+ assert new_retry._deadline == 42
+
+ def test_with_predicate(self):
+ retry_ = retry.Retry()
+ new_retry = retry_.with_predicate(mock.sentinel.predicate)
+ assert retry_ is not new_retry
+ assert new_retry._predicate == mock.sentinel.predicate
+
+ def test_with_delay_noop(self):
+ retry_ = retry.Retry()
+ new_retry = retry_.with_delay()
+ assert retry_ is not new_retry
+ assert new_retry._initial == retry_._initial
+ assert new_retry._maximum == retry_._maximum
+ assert new_retry._multiplier == retry_._multiplier
+
+ def test_with_delay(self):
+ retry_ = retry.Retry()
+ new_retry = retry_.with_delay(
+ initial=1, maximum=2, multiplier=3)
+ assert retry_ is not new_retry
+ assert new_retry._initial == 1
+ assert new_retry._maximum == 2
+ assert new_retry._multiplier == 3
+
+ def test___str__(self):
+ retry_ = retry.Retry()
+ assert re.match((
+ r'<Retry predicate=<function.*?if_exception_type.*?>, '
+ r'initial=1.0, maximum=60.0, multiplier=2.0, deadline=120.0>'),
+ str(retry_))
+
+ @mock.patch('time.sleep', autospec=True)
+ def test___call___and_execute_success(self, sleep):
+ retry_ = retry.Retry()
+ target = mock.Mock(spec=['__call__'], return_value=42)
+ # __name__ is needed by functools.partial.
+ target.__name__ = 'target'
+
+ decorated = retry_(target)
+ target.assert_not_called()
+
+ result = decorated('meep')
+
+ assert result == 42
+ target.assert_called_once_with('meep')
+ sleep.assert_not_called()
+
+ # Make uniform return half of its maximum, which will be the calculated
+ # sleep time.
+ @mock.patch(
+ 'random.uniform', autospec=True, side_effect=lambda m, n: n/2.0)
+ @mock.patch('time.sleep', autospec=True)
+ def test___call___and_execute_retry(self, sleep, uniform):
+
+ on_error = mock.Mock(spec=['__call__'], side_effect=[None])
+ retry_ = retry.Retry(
+ predicate=retry.if_exception_type(ValueError),
+ )
+
+ target = mock.Mock(spec=['__call__'], side_effect=[ValueError(), 42])
+ # __name__ is needed by functools.partial.
+ target.__name__ = 'target'
+
+ decorated = retry_(target, on_error=on_error)
+ target.assert_not_called()
+
+ result = decorated('meep')
+
+ assert result == 42
+ assert target.call_count == 2
+ target.assert_has_calls([mock.call('meep'), mock.call('meep')])
+ sleep.assert_called_once_with(retry_._initial)
+ assert on_error.call_count == 1
diff --git a/tests/unit/test_timeout.py b/tests/unit/test_timeout.py
new file mode 100644
index 0000000..40caef4
--- /dev/null
+++ b/tests/unit/test_timeout.py
@@ -0,0 +1,132 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import itertools
+
+import mock
+
+from google.api_core import timeout
+
+
+def test__exponential_timeout_generator_base_2():
+ gen = timeout._exponential_timeout_generator(
+ 1.0, 60.0, 2.0, deadline=None)
+
+ result = list(itertools.islice(gen, 8))
+ assert result == [1, 2, 4, 8, 16, 32, 60, 60]
+
+
+@mock.patch('google.api_core.datetime_helpers.utcnow', autospec=True)
+def test__exponential_timeout_generator_base_deadline(utcnow):
+ # Make each successive call to utcnow() advance one second.
+ utcnow.side_effect = [
+ datetime.datetime.min + datetime.timedelta(seconds=n)
+ for n in range(15)]
+
+ gen = timeout._exponential_timeout_generator(
+ 1.0, 60.0, 2.0, deadline=30.0)
+
+ result = list(itertools.islice(gen, 14))
+ # Should grow until the cumulative time is > 30s, then start decreasing as
+ # the cumulative time approaches 60s.
+ assert result == [1, 2, 4, 8, 16, 24, 23, 22, 21, 20, 19, 18, 17, 16]
+
+
+class TestConstantTimeout(object):
+
+ def test_constructor(self):
+ timeout_ = timeout.ConstantTimeout()
+ assert timeout_._timeout is None
+
+ def test_constructor_args(self):
+ timeout_ = timeout.ConstantTimeout(42.0)
+ assert timeout_._timeout == 42.0
+
+ def test___str__(self):
+ timeout_ = timeout.ConstantTimeout(1)
+ assert str(timeout_) == '<ConstantTimeout timeout=1.0>'
+
+ def test_apply(self):
+ target = mock.Mock(spec=['__call__', '__name__'], __name__='target')
+ timeout_ = timeout.ConstantTimeout(42.0)
+ wrapped = timeout_(target)
+
+ wrapped()
+
+ target.assert_called_once_with(timeout=42.0)
+
+ def test_apply_passthrough(self):
+ target = mock.Mock(spec=['__call__', '__name__'], __name__='target')
+ timeout_ = timeout.ConstantTimeout(42.0)
+ wrapped = timeout_(target)
+
+ wrapped(1, 2, meep='moop')
+
+ target.assert_called_once_with(1, 2, meep='moop', timeout=42.0)
+
+
+class TestExponentialTimeout(object):
+
+ def test_constructor(self):
+ timeout_ = timeout.ExponentialTimeout()
+ assert timeout_._initial == timeout._DEFAULT_INITIAL_TIMEOUT
+ assert timeout_._maximum == timeout._DEFAULT_MAXIMUM_TIMEOUT
+ assert timeout_._multiplier == timeout._DEFAULT_TIMEOUT_MULTIPLIER
+ assert timeout_._deadline == timeout._DEFAULT_DEADLINE
+
+ def test_constructor_args(self):
+ timeout_ = timeout.ExponentialTimeout(1, 2, 3, 4)
+ assert timeout_._initial == 1
+ assert timeout_._maximum == 2
+ assert timeout_._multiplier == 3
+ assert timeout_._deadline == 4
+
+ def test_with_timeout(self):
+ original_timeout = timeout.ExponentialTimeout()
+ timeout_ = original_timeout.with_deadline(42)
+ assert original_timeout is not timeout_
+ assert timeout_._initial == timeout._DEFAULT_INITIAL_TIMEOUT
+ assert timeout_._maximum == timeout._DEFAULT_MAXIMUM_TIMEOUT
+ assert timeout_._multiplier == timeout._DEFAULT_TIMEOUT_MULTIPLIER
+ assert timeout_._deadline == 42
+
+ def test___str__(self):
+ timeout_ = timeout.ExponentialTimeout(1, 2, 3, 4)
+ assert str(timeout_) == (
+ '<ExponentialTimeout initial=1.0, maximum=2.0, multiplier=3.0, '
+ 'deadline=4.0>')
+
+ def test_apply(self):
+ target = mock.Mock(spec=['__call__', '__name__'], __name__='target')
+ timeout_ = timeout.ExponentialTimeout(1, 10, 2)
+ wrapped = timeout_(target)
+
+ wrapped()
+ target.assert_called_with(timeout=1)
+
+ wrapped()
+ target.assert_called_with(timeout=2)
+
+ wrapped()
+ target.assert_called_with(timeout=4)
+
+ def test_apply_passthrough(self):
+ target = mock.Mock(spec=['__call__', '__name__'], __name__='target')
+ timeout_ = timeout.ExponentialTimeout(42.0, 100, 2)
+ wrapped = timeout_(target)
+
+ wrapped(1, 2, meep='moop')
+
+ target.assert_called_once_with(1, 2, meep='moop', timeout=42.0)