Craig Citro | 751b7fb | 2014-09-23 11:20:38 -0700 | [diff] [blame] | 1 | # Copyright 2014 Google Inc. All Rights Reserved. |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
| 14 | |
| 15 | """Classes to encapsulate a single HTTP request. |
| 16 | |
| 17 | The classes implement a command pattern, with every |
| 18 | object supporting an execute() method that does the |
| 19 | actuall HTTP request. |
| 20 | """ |
INADA Naoki | 0bceb33 | 2014-08-20 15:27:52 +0900 | [diff] [blame] | 21 | from __future__ import absolute_import |
INADA Naoki | e4ea1a9 | 2015-03-04 03:45:42 +0900 | [diff] [blame] | 22 | import six |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 23 | from six.moves import http_client |
INADA Naoki | e4ea1a9 | 2015-03-04 03:45:42 +0900 | [diff] [blame] | 24 | from six.moves import range |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 25 | |
| 26 | __author__ = 'jcgregorio@google.com (Joe Gregorio)' |
| 27 | |
Pat Ferate | ed9affd | 2015-03-03 16:03:15 -0800 | [diff] [blame] | 28 | from six import BytesIO, StringIO |
Pat Ferate | d5b61bd | 2015-03-03 16:04:11 -0800 | [diff] [blame] | 29 | from six.moves.urllib.parse import urlparse, urlunparse, quote, unquote |
Pat Ferate | ed9affd | 2015-03-03 16:03:15 -0800 | [diff] [blame] | 30 | |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 31 | import base64 |
| 32 | import copy |
| 33 | import gzip |
| 34 | import httplib2 |
Craig Citro | 6ae34d7 | 2014-08-18 23:10:09 -0700 | [diff] [blame] | 35 | import json |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 36 | import logging |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 37 | import mimetypes |
| 38 | import os |
| 39 | import random |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 40 | import socket |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 41 | import sys |
| 42 | import time |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 43 | import uuid |
| 44 | |
Tay Ray Chuan | 3146c92 | 2016-04-20 16:38:19 +0000 | [diff] [blame] | 45 | # TODO(issue 221): Remove this conditional import jibbajabba. |
| 46 | try: |
| 47 | import ssl |
| 48 | except ImportError: |
| 49 | _ssl_SSLError = object() |
| 50 | else: |
| 51 | _ssl_SSLError = ssl.SSLError |
| 52 | |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 53 | from email.generator import Generator |
| 54 | from email.mime.multipart import MIMEMultipart |
| 55 | from email.mime.nonmultipart import MIMENonMultipart |
| 56 | from email.parser import FeedParser |
Pat Ferate | b240c17 | 2015-03-03 16:23:51 -0800 | [diff] [blame] | 57 | |
Jon Wayne Parrott | 6755f61 | 2016-08-15 10:52:26 -0700 | [diff] [blame] | 58 | # Oauth2client < 3 has the positional helper in 'util', >= 3 has it |
| 59 | # in '_helpers'. |
| 60 | try: |
| 61 | from oauth2client import util |
| 62 | except ImportError: |
| 63 | from oauth2client import _helpers as util |
| 64 | |
Pat Ferate | b240c17 | 2015-03-03 16:23:51 -0800 | [diff] [blame] | 65 | from googleapiclient import mimeparse |
| 66 | from googleapiclient.errors import BatchError |
| 67 | from googleapiclient.errors import HttpError |
| 68 | from googleapiclient.errors import InvalidChunkSizeError |
| 69 | from googleapiclient.errors import ResumableUploadError |
| 70 | from googleapiclient.errors import UnexpectedBodyError |
| 71 | from googleapiclient.errors import UnexpectedMethodError |
| 72 | from googleapiclient.model import JsonModel |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 73 | |
| 74 | |
Emmett Butler | 0969915 | 2016-02-08 14:26:00 -0800 | [diff] [blame] | 75 | LOGGER = logging.getLogger(__name__) |
| 76 | |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 77 | DEFAULT_CHUNK_SIZE = 512*1024 |
| 78 | |
| 79 | MAX_URI_LENGTH = 2048 |
| 80 | |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 81 | _TOO_MANY_REQUESTS = 429 |
| 82 | |
Igor Maravić | 2243529 | 2017-01-19 22:28:22 +0100 | [diff] [blame^] | 83 | DEFAULT_HTTP_TIMEOUT_SEC = 60 |
| 84 | |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 85 | |
| 86 | def _should_retry_response(resp_status, content): |
| 87 | """Determines whether a response should be retried. |
| 88 | |
| 89 | Args: |
| 90 | resp_status: The response status received. |
| 91 | content: The response content body. |
| 92 | |
| 93 | Returns: |
| 94 | True if the response should be retried, otherwise False. |
| 95 | """ |
| 96 | # Retry on 5xx errors. |
| 97 | if resp_status >= 500: |
| 98 | return True |
| 99 | |
| 100 | # Retry on 429 errors. |
| 101 | if resp_status == _TOO_MANY_REQUESTS: |
| 102 | return True |
| 103 | |
| 104 | # For 403 errors, we have to check for the `reason` in the response to |
| 105 | # determine if we should retry. |
| 106 | if resp_status == six.moves.http_client.FORBIDDEN: |
| 107 | # If there's no details about the 403 type, don't retry. |
| 108 | if not content: |
| 109 | return False |
| 110 | |
| 111 | # Content is in JSON format. |
| 112 | try: |
| 113 | data = json.loads(content.decode('utf-8')) |
| 114 | reason = data['error']['errors'][0]['reason'] |
| 115 | except (UnicodeDecodeError, ValueError, KeyError): |
| 116 | LOGGER.warning('Invalid JSON content from response: %s', content) |
| 117 | return False |
| 118 | |
| 119 | LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason) |
| 120 | |
| 121 | # Only retry on rate limit related failures. |
| 122 | if reason in ('userRateLimitExceeded', 'rateLimitExceeded', ): |
| 123 | return True |
| 124 | |
| 125 | # Everything else is a success or non-retriable so break. |
| 126 | return False |
| 127 | |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 128 | |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 129 | def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, |
| 130 | **kwargs): |
| 131 | """Retries an HTTP request multiple times while handling errors. |
| 132 | |
| 133 | If after all retries the request still fails, last error is either returned as |
| 134 | return value (for HTTP 5xx errors) or thrown (for ssl.SSLError). |
| 135 | |
| 136 | Args: |
| 137 | http: Http object to be used to execute request. |
| 138 | num_retries: Maximum number of retries. |
| 139 | req_type: Type of the request (used for logging retries). |
| 140 | sleep, rand: Functions to sleep for random time between retries. |
| 141 | uri: URI to be requested. |
| 142 | method: HTTP method to be used. |
| 143 | args, kwargs: Additional arguments passed to http.request. |
| 144 | |
| 145 | Returns: |
| 146 | resp, content - Response from the http request (may be HTTP 5xx). |
| 147 | """ |
| 148 | resp = None |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 149 | content = None |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 150 | for retry_num in range(num_retries + 1): |
| 151 | if retry_num > 0: |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 152 | # Sleep before retrying. |
| 153 | sleep_time = rand() * 2 ** retry_num |
Emmett Butler | 0969915 | 2016-02-08 14:26:00 -0800 | [diff] [blame] | 154 | LOGGER.warning( |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 155 | 'Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s', |
| 156 | sleep_time, retry_num, num_retries, req_type, method, uri, |
| 157 | resp.status if resp else exception) |
| 158 | sleep(sleep_time) |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 159 | |
| 160 | try: |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 161 | exception = None |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 162 | resp, content = http.request(uri, method, *args, **kwargs) |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 163 | # Retry on SSL errors and socket timeout errors. |
Tay Ray Chuan | 3146c92 | 2016-04-20 16:38:19 +0000 | [diff] [blame] | 164 | except _ssl_SSLError as ssl_error: |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 165 | exception = ssl_error |
| 166 | except socket.error as socket_error: |
| 167 | # errno's contents differ by platform, so we have to match by name. |
| 168 | if socket.errno.errorcode.get(socket_error.errno) not in ( |
Thomas Bonfort | 88ab76b | 2016-04-19 08:48:53 +0200 | [diff] [blame] | 169 | 'WSAETIMEDOUT', 'ETIMEDOUT', 'EPIPE', 'ECONNABORTED', ): |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 170 | raise |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 171 | exception = socket_error |
| 172 | |
| 173 | if exception: |
| 174 | if retry_num == num_retries: |
| 175 | raise exception |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 176 | else: |
| 177 | continue |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 178 | |
| 179 | if not _should_retry_response(resp.status, content): |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 180 | break |
| 181 | |
| 182 | return resp, content |
| 183 | |
| 184 | |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 185 | class MediaUploadProgress(object): |
| 186 | """Status of a resumable upload.""" |
| 187 | |
| 188 | def __init__(self, resumable_progress, total_size): |
| 189 | """Constructor. |
| 190 | |
| 191 | Args: |
| 192 | resumable_progress: int, bytes sent so far. |
| 193 | total_size: int, total bytes in complete upload, or None if the total |
| 194 | upload size isn't known ahead of time. |
| 195 | """ |
| 196 | self.resumable_progress = resumable_progress |
| 197 | self.total_size = total_size |
| 198 | |
| 199 | def progress(self): |
| 200 | """Percent of upload completed, as a float. |
| 201 | |
| 202 | Returns: |
| 203 | the percentage complete as a float, returning 0.0 if the total size of |
| 204 | the upload is unknown. |
| 205 | """ |
| 206 | if self.total_size is not None: |
| 207 | return float(self.resumable_progress) / float(self.total_size) |
| 208 | else: |
| 209 | return 0.0 |
| 210 | |
| 211 | |
| 212 | class MediaDownloadProgress(object): |
| 213 | """Status of a resumable download.""" |
| 214 | |
| 215 | def __init__(self, resumable_progress, total_size): |
| 216 | """Constructor. |
| 217 | |
| 218 | Args: |
| 219 | resumable_progress: int, bytes received so far. |
| 220 | total_size: int, total bytes in complete download. |
| 221 | """ |
| 222 | self.resumable_progress = resumable_progress |
| 223 | self.total_size = total_size |
| 224 | |
| 225 | def progress(self): |
| 226 | """Percent of download completed, as a float. |
| 227 | |
| 228 | Returns: |
| 229 | the percentage complete as a float, returning 0.0 if the total size of |
| 230 | the download is unknown. |
| 231 | """ |
| 232 | if self.total_size is not None: |
| 233 | return float(self.resumable_progress) / float(self.total_size) |
| 234 | else: |
| 235 | return 0.0 |
| 236 | |
| 237 | |
| 238 | class MediaUpload(object): |
| 239 | """Describes a media object to upload. |
| 240 | |
| 241 | Base class that defines the interface of MediaUpload subclasses. |
| 242 | |
| 243 | Note that subclasses of MediaUpload may allow you to control the chunksize |
| 244 | when uploading a media object. It is important to keep the size of the chunk |
| 245 | as large as possible to keep the upload efficient. Other factors may influence |
| 246 | the size of the chunk you use, particularly if you are working in an |
| 247 | environment where individual HTTP requests may have a hardcoded time limit, |
| 248 | such as under certain classes of requests under Google App Engine. |
| 249 | |
| 250 | Streams are io.Base compatible objects that support seek(). Some MediaUpload |
| 251 | subclasses support using streams directly to upload data. Support for |
| 252 | streaming may be indicated by a MediaUpload sub-class and if appropriate for a |
| 253 | platform that stream will be used for uploading the media object. The support |
| 254 | for streaming is indicated by has_stream() returning True. The stream() method |
| 255 | should return an io.Base object that supports seek(). On platforms where the |
| 256 | underlying httplib module supports streaming, for example Python 2.6 and |
| 257 | later, the stream will be passed into the http library which will result in |
| 258 | less memory being used and possibly faster uploads. |
| 259 | |
| 260 | If you need to upload media that can't be uploaded using any of the existing |
| 261 | MediaUpload sub-class then you can sub-class MediaUpload for your particular |
| 262 | needs. |
| 263 | """ |
| 264 | |
| 265 | def chunksize(self): |
| 266 | """Chunk size for resumable uploads. |
| 267 | |
| 268 | Returns: |
| 269 | Chunk size in bytes. |
| 270 | """ |
| 271 | raise NotImplementedError() |
| 272 | |
| 273 | def mimetype(self): |
| 274 | """Mime type of the body. |
| 275 | |
| 276 | Returns: |
| 277 | Mime type. |
| 278 | """ |
| 279 | return 'application/octet-stream' |
| 280 | |
| 281 | def size(self): |
| 282 | """Size of upload. |
| 283 | |
| 284 | Returns: |
| 285 | Size of the body, or None of the size is unknown. |
| 286 | """ |
| 287 | return None |
| 288 | |
| 289 | def resumable(self): |
| 290 | """Whether this upload is resumable. |
| 291 | |
| 292 | Returns: |
| 293 | True if resumable upload or False. |
| 294 | """ |
| 295 | return False |
| 296 | |
| 297 | def getbytes(self, begin, end): |
| 298 | """Get bytes from the media. |
| 299 | |
| 300 | Args: |
| 301 | begin: int, offset from beginning of file. |
| 302 | length: int, number of bytes to read, starting at begin. |
| 303 | |
| 304 | Returns: |
| 305 | A string of bytes read. May be shorter than length if EOF was reached |
| 306 | first. |
| 307 | """ |
| 308 | raise NotImplementedError() |
| 309 | |
| 310 | def has_stream(self): |
| 311 | """Does the underlying upload support a streaming interface. |
| 312 | |
| 313 | Streaming means it is an io.IOBase subclass that supports seek, i.e. |
| 314 | seekable() returns True. |
| 315 | |
| 316 | Returns: |
| 317 | True if the call to stream() will return an instance of a seekable io.Base |
| 318 | subclass. |
| 319 | """ |
| 320 | return False |
| 321 | |
| 322 | def stream(self): |
| 323 | """A stream interface to the data being uploaded. |
| 324 | |
| 325 | Returns: |
| 326 | The returned value is an io.IOBase subclass that supports seek, i.e. |
| 327 | seekable() returns True. |
| 328 | """ |
| 329 | raise NotImplementedError() |
| 330 | |
| 331 | @util.positional(1) |
| 332 | def _to_json(self, strip=None): |
| 333 | """Utility function for creating a JSON representation of a MediaUpload. |
| 334 | |
| 335 | Args: |
| 336 | strip: array, An array of names of members to not include in the JSON. |
| 337 | |
| 338 | Returns: |
| 339 | string, a JSON representation of this instance, suitable to pass to |
| 340 | from_json(). |
| 341 | """ |
| 342 | t = type(self) |
| 343 | d = copy.copy(self.__dict__) |
| 344 | if strip is not None: |
| 345 | for member in strip: |
| 346 | del d[member] |
| 347 | d['_class'] = t.__name__ |
| 348 | d['_module'] = t.__module__ |
Craig Citro | 6ae34d7 | 2014-08-18 23:10:09 -0700 | [diff] [blame] | 349 | return json.dumps(d) |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 350 | |
| 351 | def to_json(self): |
| 352 | """Create a JSON representation of an instance of MediaUpload. |
| 353 | |
| 354 | Returns: |
| 355 | string, a JSON representation of this instance, suitable to pass to |
| 356 | from_json(). |
| 357 | """ |
| 358 | return self._to_json() |
| 359 | |
| 360 | @classmethod |
| 361 | def new_from_json(cls, s): |
| 362 | """Utility class method to instantiate a MediaUpload subclass from a JSON |
| 363 | representation produced by to_json(). |
| 364 | |
| 365 | Args: |
| 366 | s: string, JSON from to_json(). |
| 367 | |
| 368 | Returns: |
| 369 | An instance of the subclass of MediaUpload that was serialized with |
| 370 | to_json(). |
| 371 | """ |
Craig Citro | 6ae34d7 | 2014-08-18 23:10:09 -0700 | [diff] [blame] | 372 | data = json.loads(s) |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 373 | # Find and call the right classmethod from_json() to restore the object. |
| 374 | module = data['_module'] |
| 375 | m = __import__(module, fromlist=module.split('.')[:-1]) |
| 376 | kls = getattr(m, data['_class']) |
| 377 | from_json = getattr(kls, 'from_json') |
| 378 | return from_json(s) |
| 379 | |
| 380 | |
| 381 | class MediaIoBaseUpload(MediaUpload): |
| 382 | """A MediaUpload for a io.Base objects. |
| 383 | |
| 384 | Note that the Python file object is compatible with io.Base and can be used |
| 385 | with this class also. |
| 386 | |
Pat Ferate | ed9affd | 2015-03-03 16:03:15 -0800 | [diff] [blame] | 387 | fh = BytesIO('...Some data to upload...') |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 388 | media = MediaIoBaseUpload(fh, mimetype='image/png', |
| 389 | chunksize=1024*1024, resumable=True) |
| 390 | farm.animals().insert( |
| 391 | id='cow', |
| 392 | name='cow.png', |
| 393 | media_body=media).execute() |
| 394 | |
| 395 | Depending on the platform you are working on, you may pass -1 as the |
| 396 | chunksize, which indicates that the entire file should be uploaded in a single |
| 397 | request. If the underlying platform supports streams, such as Python 2.6 or |
| 398 | later, then this can be very efficient as it avoids multiple connections, and |
| 399 | also avoids loading the entire file into memory before sending it. Note that |
| 400 | Google App Engine has a 5MB limit on request size, so you should never set |
| 401 | your chunksize larger than 5MB, or to -1. |
| 402 | """ |
| 403 | |
| 404 | @util.positional(3) |
| 405 | def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE, |
| 406 | resumable=False): |
| 407 | """Constructor. |
| 408 | |
| 409 | Args: |
| 410 | fd: io.Base or file object, The source of the bytes to upload. MUST be |
| 411 | opened in blocking mode, do not use streams opened in non-blocking mode. |
| 412 | The given stream must be seekable, that is, it must be able to call |
| 413 | seek() on fd. |
| 414 | mimetype: string, Mime-type of the file. |
| 415 | chunksize: int, File will be uploaded in chunks of this many bytes. Only |
| 416 | used if resumable=True. Pass in a value of -1 if the file is to be |
| 417 | uploaded as a single chunk. Note that Google App Engine has a 5MB limit |
| 418 | on request size, so you should never set your chunksize larger than 5MB, |
| 419 | or to -1. |
| 420 | resumable: bool, True if this is a resumable upload. False means upload |
| 421 | in a single request. |
| 422 | """ |
| 423 | super(MediaIoBaseUpload, self).__init__() |
| 424 | self._fd = fd |
| 425 | self._mimetype = mimetype |
| 426 | if not (chunksize == -1 or chunksize > 0): |
| 427 | raise InvalidChunkSizeError() |
| 428 | self._chunksize = chunksize |
| 429 | self._resumable = resumable |
| 430 | |
| 431 | self._fd.seek(0, os.SEEK_END) |
| 432 | self._size = self._fd.tell() |
| 433 | |
| 434 | def chunksize(self): |
| 435 | """Chunk size for resumable uploads. |
| 436 | |
| 437 | Returns: |
| 438 | Chunk size in bytes. |
| 439 | """ |
| 440 | return self._chunksize |
| 441 | |
| 442 | def mimetype(self): |
| 443 | """Mime type of the body. |
| 444 | |
| 445 | Returns: |
| 446 | Mime type. |
| 447 | """ |
| 448 | return self._mimetype |
| 449 | |
| 450 | def size(self): |
| 451 | """Size of upload. |
| 452 | |
| 453 | Returns: |
| 454 | Size of the body, or None of the size is unknown. |
| 455 | """ |
| 456 | return self._size |
| 457 | |
| 458 | def resumable(self): |
| 459 | """Whether this upload is resumable. |
| 460 | |
| 461 | Returns: |
| 462 | True if resumable upload or False. |
| 463 | """ |
| 464 | return self._resumable |
| 465 | |
| 466 | def getbytes(self, begin, length): |
| 467 | """Get bytes from the media. |
| 468 | |
| 469 | Args: |
| 470 | begin: int, offset from beginning of file. |
| 471 | length: int, number of bytes to read, starting at begin. |
| 472 | |
| 473 | Returns: |
| 474 | A string of bytes read. May be shorted than length if EOF was reached |
| 475 | first. |
| 476 | """ |
| 477 | self._fd.seek(begin) |
| 478 | return self._fd.read(length) |
| 479 | |
| 480 | def has_stream(self): |
| 481 | """Does the underlying upload support a streaming interface. |
| 482 | |
| 483 | Streaming means it is an io.IOBase subclass that supports seek, i.e. |
| 484 | seekable() returns True. |
| 485 | |
| 486 | Returns: |
| 487 | True if the call to stream() will return an instance of a seekable io.Base |
| 488 | subclass. |
| 489 | """ |
| 490 | return True |
| 491 | |
| 492 | def stream(self): |
| 493 | """A stream interface to the data being uploaded. |
| 494 | |
| 495 | Returns: |
| 496 | The returned value is an io.IOBase subclass that supports seek, i.e. |
| 497 | seekable() returns True. |
| 498 | """ |
| 499 | return self._fd |
| 500 | |
| 501 | def to_json(self): |
| 502 | """This upload type is not serializable.""" |
| 503 | raise NotImplementedError('MediaIoBaseUpload is not serializable.') |
| 504 | |
| 505 | |
| 506 | class MediaFileUpload(MediaIoBaseUpload): |
| 507 | """A MediaUpload for a file. |
| 508 | |
| 509 | Construct a MediaFileUpload and pass as the media_body parameter of the |
| 510 | method. For example, if we had a service that allowed uploading images: |
| 511 | |
| 512 | |
| 513 | media = MediaFileUpload('cow.png', mimetype='image/png', |
| 514 | chunksize=1024*1024, resumable=True) |
| 515 | farm.animals().insert( |
| 516 | id='cow', |
| 517 | name='cow.png', |
| 518 | media_body=media).execute() |
| 519 | |
| 520 | Depending on the platform you are working on, you may pass -1 as the |
| 521 | chunksize, which indicates that the entire file should be uploaded in a single |
| 522 | request. If the underlying platform supports streams, such as Python 2.6 or |
| 523 | later, then this can be very efficient as it avoids multiple connections, and |
| 524 | also avoids loading the entire file into memory before sending it. Note that |
| 525 | Google App Engine has a 5MB limit on request size, so you should never set |
| 526 | your chunksize larger than 5MB, or to -1. |
| 527 | """ |
| 528 | |
| 529 | @util.positional(2) |
| 530 | def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE, |
| 531 | resumable=False): |
| 532 | """Constructor. |
| 533 | |
| 534 | Args: |
| 535 | filename: string, Name of the file. |
| 536 | mimetype: string, Mime-type of the file. If None then a mime-type will be |
| 537 | guessed from the file extension. |
| 538 | chunksize: int, File will be uploaded in chunks of this many bytes. Only |
| 539 | used if resumable=True. Pass in a value of -1 if the file is to be |
| 540 | uploaded in a single chunk. Note that Google App Engine has a 5MB limit |
| 541 | on request size, so you should never set your chunksize larger than 5MB, |
| 542 | or to -1. |
| 543 | resumable: bool, True if this is a resumable upload. False means upload |
| 544 | in a single request. |
| 545 | """ |
| 546 | self._filename = filename |
| 547 | fd = open(self._filename, 'rb') |
| 548 | if mimetype is None: |
Nam T. Nguyen | dc13631 | 2015-12-01 10:18:56 -0800 | [diff] [blame] | 549 | # No mimetype provided, make a guess. |
| 550 | mimetype, _ = mimetypes.guess_type(filename) |
| 551 | if mimetype is None: |
| 552 | # Guess failed, use octet-stream. |
| 553 | mimetype = 'application/octet-stream' |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 554 | super(MediaFileUpload, self).__init__(fd, mimetype, chunksize=chunksize, |
| 555 | resumable=resumable) |
| 556 | |
| 557 | def to_json(self): |
| 558 | """Creating a JSON representation of an instance of MediaFileUpload. |
| 559 | |
| 560 | Returns: |
| 561 | string, a JSON representation of this instance, suitable to pass to |
| 562 | from_json(). |
| 563 | """ |
| 564 | return self._to_json(strip=['_fd']) |
| 565 | |
| 566 | @staticmethod |
| 567 | def from_json(s): |
Craig Citro | 6ae34d7 | 2014-08-18 23:10:09 -0700 | [diff] [blame] | 568 | d = json.loads(s) |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 569 | return MediaFileUpload(d['_filename'], mimetype=d['_mimetype'], |
| 570 | chunksize=d['_chunksize'], resumable=d['_resumable']) |
| 571 | |
| 572 | |
| 573 | class MediaInMemoryUpload(MediaIoBaseUpload): |
| 574 | """MediaUpload for a chunk of bytes. |
| 575 | |
| 576 | DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for |
| 577 | the stream. |
| 578 | """ |
| 579 | |
| 580 | @util.positional(2) |
| 581 | def __init__(self, body, mimetype='application/octet-stream', |
| 582 | chunksize=DEFAULT_CHUNK_SIZE, resumable=False): |
| 583 | """Create a new MediaInMemoryUpload. |
| 584 | |
| 585 | DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for |
| 586 | the stream. |
| 587 | |
| 588 | Args: |
| 589 | body: string, Bytes of body content. |
| 590 | mimetype: string, Mime-type of the file or default of |
| 591 | 'application/octet-stream'. |
| 592 | chunksize: int, File will be uploaded in chunks of this many bytes. Only |
| 593 | used if resumable=True. |
| 594 | resumable: bool, True if this is a resumable upload. False means upload |
| 595 | in a single request. |
| 596 | """ |
Pat Ferate | ed9affd | 2015-03-03 16:03:15 -0800 | [diff] [blame] | 597 | fd = BytesIO(body) |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 598 | super(MediaInMemoryUpload, self).__init__(fd, mimetype, chunksize=chunksize, |
| 599 | resumable=resumable) |
| 600 | |
| 601 | |
| 602 | class MediaIoBaseDownload(object): |
| 603 | """"Download media resources. |
| 604 | |
| 605 | Note that the Python file object is compatible with io.Base and can be used |
| 606 | with this class also. |
| 607 | |
| 608 | |
| 609 | Example: |
| 610 | request = farms.animals().get_media(id='cow') |
| 611 | fh = io.FileIO('cow.png', mode='wb') |
| 612 | downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024) |
| 613 | |
| 614 | done = False |
| 615 | while done is False: |
| 616 | status, done = downloader.next_chunk() |
| 617 | if status: |
| 618 | print "Download %d%%." % int(status.progress() * 100) |
| 619 | print "Download Complete!" |
| 620 | """ |
| 621 | |
| 622 | @util.positional(3) |
| 623 | def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE): |
| 624 | """Constructor. |
| 625 | |
| 626 | Args: |
| 627 | fd: io.Base or file object, The stream in which to write the downloaded |
| 628 | bytes. |
| 629 | request: googleapiclient.http.HttpRequest, the media request to perform in |
| 630 | chunks. |
| 631 | chunksize: int, File will be downloaded in chunks of this many bytes. |
| 632 | """ |
| 633 | self._fd = fd |
| 634 | self._request = request |
| 635 | self._uri = request.uri |
| 636 | self._chunksize = chunksize |
| 637 | self._progress = 0 |
| 638 | self._total_size = None |
| 639 | self._done = False |
| 640 | |
| 641 | # Stubs for testing. |
| 642 | self._sleep = time.sleep |
| 643 | self._rand = random.random |
| 644 | |
| 645 | @util.positional(1) |
| 646 | def next_chunk(self, num_retries=0): |
| 647 | """Get the next chunk of the download. |
| 648 | |
| 649 | Args: |
Zhihao Yuan | cc6d398 | 2016-07-27 11:40:45 -0500 | [diff] [blame] | 650 | num_retries: Integer, number of times to retry with randomized |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 651 | exponential backoff. If all retries fail, the raised HttpError |
| 652 | represents the last request. If zero (default), we attempt the |
| 653 | request only once. |
| 654 | |
| 655 | Returns: |
| 656 | (status, done): (MediaDownloadStatus, boolean) |
| 657 | The value of 'done' will be True when the media has been fully |
| 658 | downloaded. |
| 659 | |
| 660 | Raises: |
| 661 | googleapiclient.errors.HttpError if the response was not a 2xx. |
| 662 | httplib2.HttpLib2Error if a transport error has occured. |
| 663 | """ |
| 664 | headers = { |
| 665 | 'range': 'bytes=%d-%d' % ( |
| 666 | self._progress, self._progress + self._chunksize) |
| 667 | } |
| 668 | http = self._request.http |
| 669 | |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 670 | resp, content = _retry_request( |
| 671 | http, num_retries, 'media download', self._sleep, self._rand, self._uri, |
| 672 | 'GET', headers=headers) |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 673 | |
| 674 | if resp.status in [200, 206]: |
| 675 | if 'content-location' in resp and resp['content-location'] != self._uri: |
| 676 | self._uri = resp['content-location'] |
| 677 | self._progress += len(content) |
| 678 | self._fd.write(content) |
| 679 | |
| 680 | if 'content-range' in resp: |
| 681 | content_range = resp['content-range'] |
| 682 | length = content_range.rsplit('/', 1)[1] |
| 683 | self._total_size = int(length) |
jack | ac8df21 | 2015-02-17 12:16:19 -0800 | [diff] [blame] | 684 | elif 'content-length' in resp: |
jack | 77c63c9 | 2015-02-10 12:11:00 -0800 | [diff] [blame] | 685 | self._total_size = int(resp['content-length']) |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 686 | |
| 687 | if self._progress == self._total_size: |
| 688 | self._done = True |
| 689 | return MediaDownloadProgress(self._progress, self._total_size), self._done |
| 690 | else: |
| 691 | raise HttpError(resp, content, uri=self._uri) |
| 692 | |
| 693 | |
| 694 | class _StreamSlice(object): |
| 695 | """Truncated stream. |
| 696 | |
| 697 | Takes a stream and presents a stream that is a slice of the original stream. |
| 698 | This is used when uploading media in chunks. In later versions of Python a |
| 699 | stream can be passed to httplib in place of the string of data to send. The |
| 700 | problem is that httplib just blindly reads to the end of the stream. This |
| 701 | wrapper presents a virtual stream that only reads to the end of the chunk. |
| 702 | """ |
| 703 | |
| 704 | def __init__(self, stream, begin, chunksize): |
| 705 | """Constructor. |
| 706 | |
| 707 | Args: |
| 708 | stream: (io.Base, file object), the stream to wrap. |
| 709 | begin: int, the seek position the chunk begins at. |
| 710 | chunksize: int, the size of the chunk. |
| 711 | """ |
| 712 | self._stream = stream |
| 713 | self._begin = begin |
| 714 | self._chunksize = chunksize |
| 715 | self._stream.seek(begin) |
| 716 | |
| 717 | def read(self, n=-1): |
| 718 | """Read n bytes. |
| 719 | |
| 720 | Args: |
| 721 | n, int, the number of bytes to read. |
| 722 | |
| 723 | Returns: |
| 724 | A string of length 'n', or less if EOF is reached. |
| 725 | """ |
| 726 | # The data left available to read sits in [cur, end) |
| 727 | cur = self._stream.tell() |
| 728 | end = self._begin + self._chunksize |
| 729 | if n == -1 or cur + n > end: |
| 730 | n = end - cur |
| 731 | return self._stream.read(n) |
| 732 | |
| 733 | |
| 734 | class HttpRequest(object): |
| 735 | """Encapsulates a single HTTP request.""" |
| 736 | |
| 737 | @util.positional(4) |
| 738 | def __init__(self, http, postproc, uri, |
| 739 | method='GET', |
| 740 | body=None, |
| 741 | headers=None, |
| 742 | methodId=None, |
| 743 | resumable=None): |
| 744 | """Constructor for an HttpRequest. |
| 745 | |
| 746 | Args: |
| 747 | http: httplib2.Http, the transport object to use to make a request |
| 748 | postproc: callable, called on the HTTP response and content to transform |
| 749 | it into a data object before returning, or raising an exception |
| 750 | on an error. |
| 751 | uri: string, the absolute URI to send the request to |
| 752 | method: string, the HTTP method to use |
| 753 | body: string, the request body of the HTTP request, |
| 754 | headers: dict, the HTTP request headers |
| 755 | methodId: string, a unique identifier for the API method being called. |
| 756 | resumable: MediaUpload, None if this is not a resumbale request. |
| 757 | """ |
| 758 | self.uri = uri |
| 759 | self.method = method |
| 760 | self.body = body |
| 761 | self.headers = headers or {} |
| 762 | self.methodId = methodId |
| 763 | self.http = http |
| 764 | self.postproc = postproc |
| 765 | self.resumable = resumable |
| 766 | self.response_callbacks = [] |
| 767 | self._in_error_state = False |
| 768 | |
| 769 | # Pull the multipart boundary out of the content-type header. |
| 770 | major, minor, params = mimeparse.parse_mime_type( |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 771 | self.headers.get('content-type', 'application/json')) |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 772 | |
| 773 | # The size of the non-media part of the request. |
| 774 | self.body_size = len(self.body or '') |
| 775 | |
| 776 | # The resumable URI to send chunks to. |
| 777 | self.resumable_uri = None |
| 778 | |
| 779 | # The bytes that have been uploaded. |
| 780 | self.resumable_progress = 0 |
| 781 | |
| 782 | # Stubs for testing. |
| 783 | self._rand = random.random |
| 784 | self._sleep = time.sleep |
| 785 | |
| 786 | @util.positional(1) |
| 787 | def execute(self, http=None, num_retries=0): |
| 788 | """Execute the request. |
| 789 | |
| 790 | Args: |
| 791 | http: httplib2.Http, an http object to be used in place of the |
| 792 | one the HttpRequest request object was constructed with. |
Zhihao Yuan | cc6d398 | 2016-07-27 11:40:45 -0500 | [diff] [blame] | 793 | num_retries: Integer, number of times to retry with randomized |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 794 | exponential backoff. If all retries fail, the raised HttpError |
| 795 | represents the last request. If zero (default), we attempt the |
| 796 | request only once. |
| 797 | |
| 798 | Returns: |
| 799 | A deserialized object model of the response body as determined |
| 800 | by the postproc. |
| 801 | |
| 802 | Raises: |
| 803 | googleapiclient.errors.HttpError if the response was not a 2xx. |
| 804 | httplib2.HttpLib2Error if a transport error has occured. |
| 805 | """ |
| 806 | if http is None: |
| 807 | http = self.http |
| 808 | |
| 809 | if self.resumable: |
| 810 | body = None |
| 811 | while body is None: |
| 812 | _, body = self.next_chunk(http=http, num_retries=num_retries) |
| 813 | return body |
| 814 | |
| 815 | # Non-resumable case. |
| 816 | |
| 817 | if 'content-length' not in self.headers: |
| 818 | self.headers['content-length'] = str(self.body_size) |
| 819 | # If the request URI is too long then turn it into a POST request. |
| 820 | if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET': |
| 821 | self.method = 'POST' |
| 822 | self.headers['x-http-method-override'] = 'GET' |
| 823 | self.headers['content-type'] = 'application/x-www-form-urlencoded' |
Pat Ferate | d5b61bd | 2015-03-03 16:04:11 -0800 | [diff] [blame] | 824 | parsed = urlparse(self.uri) |
| 825 | self.uri = urlunparse( |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 826 | (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None, |
| 827 | None) |
| 828 | ) |
| 829 | self.body = parsed.query |
| 830 | self.headers['content-length'] = str(len(self.body)) |
| 831 | |
| 832 | # Handle retries for server-side errors. |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 833 | resp, content = _retry_request( |
| 834 | http, num_retries, 'request', self._sleep, self._rand, str(self.uri), |
| 835 | method=str(self.method), body=self.body, headers=self.headers) |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 836 | |
| 837 | for callback in self.response_callbacks: |
| 838 | callback(resp) |
| 839 | if resp.status >= 300: |
| 840 | raise HttpError(resp, content, uri=self.uri) |
| 841 | return self.postproc(resp, content) |
| 842 | |
| 843 | @util.positional(2) |
| 844 | def add_response_callback(self, cb): |
| 845 | """add_response_headers_callback |
| 846 | |
| 847 | Args: |
| 848 | cb: Callback to be called on receiving the response headers, of signature: |
| 849 | |
| 850 | def cb(resp): |
| 851 | # Where resp is an instance of httplib2.Response |
| 852 | """ |
| 853 | self.response_callbacks.append(cb) |
| 854 | |
| 855 | @util.positional(1) |
| 856 | def next_chunk(self, http=None, num_retries=0): |
| 857 | """Execute the next step of a resumable upload. |
| 858 | |
| 859 | Can only be used if the method being executed supports media uploads and |
| 860 | the MediaUpload object passed in was flagged as using resumable upload. |
| 861 | |
| 862 | Example: |
| 863 | |
| 864 | media = MediaFileUpload('cow.png', mimetype='image/png', |
| 865 | chunksize=1000, resumable=True) |
| 866 | request = farm.animals().insert( |
| 867 | id='cow', |
| 868 | name='cow.png', |
| 869 | media_body=media) |
| 870 | |
| 871 | response = None |
| 872 | while response is None: |
| 873 | status, response = request.next_chunk() |
| 874 | if status: |
| 875 | print "Upload %d%% complete." % int(status.progress() * 100) |
| 876 | |
| 877 | |
| 878 | Args: |
| 879 | http: httplib2.Http, an http object to be used in place of the |
| 880 | one the HttpRequest request object was constructed with. |
Zhihao Yuan | cc6d398 | 2016-07-27 11:40:45 -0500 | [diff] [blame] | 881 | num_retries: Integer, number of times to retry with randomized |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 882 | exponential backoff. If all retries fail, the raised HttpError |
| 883 | represents the last request. If zero (default), we attempt the |
| 884 | request only once. |
| 885 | |
| 886 | Returns: |
| 887 | (status, body): (ResumableMediaStatus, object) |
| 888 | The body will be None until the resumable media is fully uploaded. |
| 889 | |
| 890 | Raises: |
| 891 | googleapiclient.errors.HttpError if the response was not a 2xx. |
| 892 | httplib2.HttpLib2Error if a transport error has occured. |
| 893 | """ |
| 894 | if http is None: |
| 895 | http = self.http |
| 896 | |
| 897 | if self.resumable.size() is None: |
| 898 | size = '*' |
| 899 | else: |
| 900 | size = str(self.resumable.size()) |
| 901 | |
| 902 | if self.resumable_uri is None: |
| 903 | start_headers = copy.copy(self.headers) |
| 904 | start_headers['X-Upload-Content-Type'] = self.resumable.mimetype() |
| 905 | if size != '*': |
| 906 | start_headers['X-Upload-Content-Length'] = size |
| 907 | start_headers['content-length'] = str(self.body_size) |
| 908 | |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 909 | resp, content = _retry_request( |
| 910 | http, num_retries, 'resumable URI request', self._sleep, self._rand, |
| 911 | self.uri, method=self.method, body=self.body, headers=start_headers) |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 912 | |
| 913 | if resp.status == 200 and 'location' in resp: |
| 914 | self.resumable_uri = resp['location'] |
| 915 | else: |
| 916 | raise ResumableUploadError(resp, content) |
| 917 | elif self._in_error_state: |
| 918 | # If we are in an error state then query the server for current state of |
| 919 | # the upload by sending an empty PUT and reading the 'range' header in |
| 920 | # the response. |
| 921 | headers = { |
| 922 | 'Content-Range': 'bytes */%s' % size, |
| 923 | 'content-length': '0' |
| 924 | } |
| 925 | resp, content = http.request(self.resumable_uri, 'PUT', |
| 926 | headers=headers) |
| 927 | status, body = self._process_response(resp, content) |
| 928 | if body: |
| 929 | # The upload was complete. |
| 930 | return (status, body) |
| 931 | |
e00E | fafe858 | 2015-10-10 18:19:37 +0200 | [diff] [blame] | 932 | if self.resumable.has_stream(): |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 933 | data = self.resumable.stream() |
| 934 | if self.resumable.chunksize() == -1: |
| 935 | data.seek(self.resumable_progress) |
| 936 | chunk_end = self.resumable.size() - self.resumable_progress - 1 |
| 937 | else: |
| 938 | # Doing chunking with a stream, so wrap a slice of the stream. |
| 939 | data = _StreamSlice(data, self.resumable_progress, |
| 940 | self.resumable.chunksize()) |
| 941 | chunk_end = min( |
| 942 | self.resumable_progress + self.resumable.chunksize() - 1, |
| 943 | self.resumable.size() - 1) |
| 944 | else: |
| 945 | data = self.resumable.getbytes( |
| 946 | self.resumable_progress, self.resumable.chunksize()) |
| 947 | |
| 948 | # A short read implies that we are at EOF, so finish the upload. |
| 949 | if len(data) < self.resumable.chunksize(): |
| 950 | size = str(self.resumable_progress + len(data)) |
| 951 | |
| 952 | chunk_end = self.resumable_progress + len(data) - 1 |
| 953 | |
| 954 | headers = { |
| 955 | 'Content-Range': 'bytes %d-%d/%s' % ( |
| 956 | self.resumable_progress, chunk_end, size), |
| 957 | # Must set the content-length header here because httplib can't |
| 958 | # calculate the size when working with _StreamSlice. |
| 959 | 'Content-Length': str(chunk_end - self.resumable_progress + 1) |
| 960 | } |
| 961 | |
INADA Naoki | e4ea1a9 | 2015-03-04 03:45:42 +0900 | [diff] [blame] | 962 | for retry_num in range(num_retries + 1): |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 963 | if retry_num > 0: |
| 964 | self._sleep(self._rand() * 2**retry_num) |
Emmett Butler | 0969915 | 2016-02-08 14:26:00 -0800 | [diff] [blame] | 965 | LOGGER.warning( |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 966 | 'Retry #%d for media upload: %s %s, following status: %d' |
| 967 | % (retry_num, self.method, self.uri, resp.status)) |
| 968 | |
| 969 | try: |
| 970 | resp, content = http.request(self.resumable_uri, method='PUT', |
| 971 | body=data, |
| 972 | headers=headers) |
| 973 | except: |
| 974 | self._in_error_state = True |
| 975 | raise |
Zhihao Yuan | cc6d398 | 2016-07-27 11:40:45 -0500 | [diff] [blame] | 976 | if not _should_retry_response(resp.status, content): |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 977 | break |
| 978 | |
| 979 | return self._process_response(resp, content) |
| 980 | |
| 981 | def _process_response(self, resp, content): |
| 982 | """Process the response from a single chunk upload. |
| 983 | |
| 984 | Args: |
| 985 | resp: httplib2.Response, the response object. |
| 986 | content: string, the content of the response. |
| 987 | |
| 988 | Returns: |
| 989 | (status, body): (ResumableMediaStatus, object) |
| 990 | The body will be None until the resumable media is fully uploaded. |
| 991 | |
| 992 | Raises: |
| 993 | googleapiclient.errors.HttpError if the response was not a 2xx or a 308. |
| 994 | """ |
| 995 | if resp.status in [200, 201]: |
| 996 | self._in_error_state = False |
| 997 | return None, self.postproc(resp, content) |
| 998 | elif resp.status == 308: |
| 999 | self._in_error_state = False |
| 1000 | # A "308 Resume Incomplete" indicates we are not done. |
Matt Carroll | 94a5394 | 2016-12-20 13:56:43 -0800 | [diff] [blame] | 1001 | try: |
| 1002 | self.resumable_progress = int(resp['range'].split('-')[1]) + 1 |
| 1003 | except KeyError: |
| 1004 | # If resp doesn't contain range header, resumable progress is 0 |
| 1005 | self.resumable_progress = 0 |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1006 | if 'location' in resp: |
| 1007 | self.resumable_uri = resp['location'] |
| 1008 | else: |
| 1009 | self._in_error_state = True |
| 1010 | raise HttpError(resp, content, uri=self.uri) |
| 1011 | |
| 1012 | return (MediaUploadProgress(self.resumable_progress, self.resumable.size()), |
| 1013 | None) |
| 1014 | |
| 1015 | def to_json(self): |
| 1016 | """Returns a JSON representation of the HttpRequest.""" |
| 1017 | d = copy.copy(self.__dict__) |
| 1018 | if d['resumable'] is not None: |
| 1019 | d['resumable'] = self.resumable.to_json() |
| 1020 | del d['http'] |
| 1021 | del d['postproc'] |
| 1022 | del d['_sleep'] |
| 1023 | del d['_rand'] |
| 1024 | |
Craig Citro | 6ae34d7 | 2014-08-18 23:10:09 -0700 | [diff] [blame] | 1025 | return json.dumps(d) |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1026 | |
| 1027 | @staticmethod |
| 1028 | def from_json(s, http, postproc): |
| 1029 | """Returns an HttpRequest populated with info from a JSON object.""" |
Craig Citro | 6ae34d7 | 2014-08-18 23:10:09 -0700 | [diff] [blame] | 1030 | d = json.loads(s) |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1031 | if d['resumable'] is not None: |
| 1032 | d['resumable'] = MediaUpload.new_from_json(d['resumable']) |
| 1033 | return HttpRequest( |
| 1034 | http, |
| 1035 | postproc, |
| 1036 | uri=d['uri'], |
| 1037 | method=d['method'], |
| 1038 | body=d['body'], |
| 1039 | headers=d['headers'], |
| 1040 | methodId=d['methodId'], |
| 1041 | resumable=d['resumable']) |
| 1042 | |
| 1043 | |
| 1044 | class BatchHttpRequest(object): |
| 1045 | """Batches multiple HttpRequest objects into a single HTTP request. |
| 1046 | |
| 1047 | Example: |
| 1048 | from googleapiclient.http import BatchHttpRequest |
| 1049 | |
| 1050 | def list_animals(request_id, response, exception): |
| 1051 | \"\"\"Do something with the animals list response.\"\"\" |
| 1052 | if exception is not None: |
| 1053 | # Do something with the exception. |
| 1054 | pass |
| 1055 | else: |
| 1056 | # Do something with the response. |
| 1057 | pass |
| 1058 | |
| 1059 | def list_farmers(request_id, response, exception): |
| 1060 | \"\"\"Do something with the farmers list response.\"\"\" |
| 1061 | if exception is not None: |
| 1062 | # Do something with the exception. |
| 1063 | pass |
| 1064 | else: |
| 1065 | # Do something with the response. |
| 1066 | pass |
| 1067 | |
| 1068 | service = build('farm', 'v2') |
| 1069 | |
| 1070 | batch = BatchHttpRequest() |
| 1071 | |
| 1072 | batch.add(service.animals().list(), list_animals) |
| 1073 | batch.add(service.farmers().list(), list_farmers) |
| 1074 | batch.execute(http=http) |
| 1075 | """ |
| 1076 | |
| 1077 | @util.positional(1) |
| 1078 | def __init__(self, callback=None, batch_uri=None): |
| 1079 | """Constructor for a BatchHttpRequest. |
| 1080 | |
| 1081 | Args: |
| 1082 | callback: callable, A callback to be called for each response, of the |
| 1083 | form callback(id, response, exception). The first parameter is the |
| 1084 | request id, and the second is the deserialized response object. The |
| 1085 | third is an googleapiclient.errors.HttpError exception object if an HTTP error |
| 1086 | occurred while processing the request, or None if no error occurred. |
| 1087 | batch_uri: string, URI to send batch requests to. |
| 1088 | """ |
| 1089 | if batch_uri is None: |
| 1090 | batch_uri = 'https://www.googleapis.com/batch' |
| 1091 | self._batch_uri = batch_uri |
| 1092 | |
| 1093 | # Global callback to be called for each individual response in the batch. |
| 1094 | self._callback = callback |
| 1095 | |
| 1096 | # A map from id to request. |
| 1097 | self._requests = {} |
| 1098 | |
| 1099 | # A map from id to callback. |
| 1100 | self._callbacks = {} |
| 1101 | |
| 1102 | # List of request ids, in the order in which they were added. |
| 1103 | self._order = [] |
| 1104 | |
| 1105 | # The last auto generated id. |
| 1106 | self._last_auto_id = 0 |
| 1107 | |
| 1108 | # Unique ID on which to base the Content-ID headers. |
| 1109 | self._base_id = None |
| 1110 | |
| 1111 | # A map from request id to (httplib2.Response, content) response pairs |
| 1112 | self._responses = {} |
| 1113 | |
| 1114 | # A map of id(Credentials) that have been refreshed. |
| 1115 | self._refreshed_credentials = {} |
| 1116 | |
| 1117 | def _refresh_and_apply_credentials(self, request, http): |
| 1118 | """Refresh the credentials and apply to the request. |
| 1119 | |
| 1120 | Args: |
| 1121 | request: HttpRequest, the request. |
| 1122 | http: httplib2.Http, the global http object for the batch. |
| 1123 | """ |
| 1124 | # For the credentials to refresh, but only once per refresh_token |
| 1125 | # If there is no http per the request then refresh the http passed in |
| 1126 | # via execute() |
| 1127 | creds = None |
| 1128 | if request.http is not None and hasattr(request.http.request, |
| 1129 | 'credentials'): |
| 1130 | creds = request.http.request.credentials |
| 1131 | elif http is not None and hasattr(http.request, 'credentials'): |
| 1132 | creds = http.request.credentials |
| 1133 | if creds is not None: |
| 1134 | if id(creds) not in self._refreshed_credentials: |
| 1135 | creds.refresh(http) |
| 1136 | self._refreshed_credentials[id(creds)] = 1 |
| 1137 | |
| 1138 | # Only apply the credentials if we are using the http object passed in, |
| 1139 | # otherwise apply() will get called during _serialize_request(). |
| 1140 | if request.http is None or not hasattr(request.http.request, |
| 1141 | 'credentials'): |
| 1142 | creds.apply(request.headers) |
| 1143 | |
| 1144 | def _id_to_header(self, id_): |
| 1145 | """Convert an id to a Content-ID header value. |
| 1146 | |
| 1147 | Args: |
| 1148 | id_: string, identifier of individual request. |
| 1149 | |
| 1150 | Returns: |
| 1151 | A Content-ID header with the id_ encoded into it. A UUID is prepended to |
| 1152 | the value because Content-ID headers are supposed to be universally |
| 1153 | unique. |
| 1154 | """ |
| 1155 | if self._base_id is None: |
| 1156 | self._base_id = uuid.uuid4() |
| 1157 | |
Pat Ferate | d5b61bd | 2015-03-03 16:04:11 -0800 | [diff] [blame] | 1158 | return '<%s+%s>' % (self._base_id, quote(id_)) |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1159 | |
| 1160 | def _header_to_id(self, header): |
| 1161 | """Convert a Content-ID header value to an id. |
| 1162 | |
| 1163 | Presumes the Content-ID header conforms to the format that _id_to_header() |
| 1164 | returns. |
| 1165 | |
| 1166 | Args: |
| 1167 | header: string, Content-ID header value. |
| 1168 | |
| 1169 | Returns: |
| 1170 | The extracted id value. |
| 1171 | |
| 1172 | Raises: |
| 1173 | BatchError if the header is not in the expected format. |
| 1174 | """ |
| 1175 | if header[0] != '<' or header[-1] != '>': |
| 1176 | raise BatchError("Invalid value for Content-ID: %s" % header) |
| 1177 | if '+' not in header: |
| 1178 | raise BatchError("Invalid value for Content-ID: %s" % header) |
| 1179 | base, id_ = header[1:-1].rsplit('+', 1) |
| 1180 | |
Pat Ferate | d5b61bd | 2015-03-03 16:04:11 -0800 | [diff] [blame] | 1181 | return unquote(id_) |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1182 | |
| 1183 | def _serialize_request(self, request): |
| 1184 | """Convert an HttpRequest object into a string. |
| 1185 | |
| 1186 | Args: |
| 1187 | request: HttpRequest, the request to serialize. |
| 1188 | |
| 1189 | Returns: |
| 1190 | The request as a string in application/http format. |
| 1191 | """ |
| 1192 | # Construct status line |
Pat Ferate | d5b61bd | 2015-03-03 16:04:11 -0800 | [diff] [blame] | 1193 | parsed = urlparse(request.uri) |
| 1194 | request_line = urlunparse( |
Pat Ferate | c9abbbd | 2015-03-03 18:00:38 -0800 | [diff] [blame] | 1195 | ('', '', parsed.path, parsed.params, parsed.query, '') |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1196 | ) |
| 1197 | status_line = request.method + ' ' + request_line + ' HTTP/1.1\n' |
| 1198 | major, minor = request.headers.get('content-type', 'application/json').split('/') |
| 1199 | msg = MIMENonMultipart(major, minor) |
| 1200 | headers = request.headers.copy() |
| 1201 | |
| 1202 | if request.http is not None and hasattr(request.http.request, |
| 1203 | 'credentials'): |
| 1204 | request.http.request.credentials.apply(headers) |
| 1205 | |
| 1206 | # MIMENonMultipart adds its own Content-Type header. |
| 1207 | if 'content-type' in headers: |
| 1208 | del headers['content-type'] |
| 1209 | |
INADA Naoki | e4ea1a9 | 2015-03-04 03:45:42 +0900 | [diff] [blame] | 1210 | for key, value in six.iteritems(headers): |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1211 | msg[key] = value |
| 1212 | msg['Host'] = parsed.netloc |
| 1213 | msg.set_unixfrom(None) |
| 1214 | |
| 1215 | if request.body is not None: |
| 1216 | msg.set_payload(request.body) |
| 1217 | msg['content-length'] = str(len(request.body)) |
| 1218 | |
| 1219 | # Serialize the mime message. |
Pat Ferate | ed9affd | 2015-03-03 16:03:15 -0800 | [diff] [blame] | 1220 | fp = StringIO() |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1221 | # maxheaderlen=0 means don't line wrap headers. |
| 1222 | g = Generator(fp, maxheaderlen=0) |
| 1223 | g.flatten(msg, unixfrom=False) |
| 1224 | body = fp.getvalue() |
| 1225 | |
Pat Ferate | c9abbbd | 2015-03-03 18:00:38 -0800 | [diff] [blame] | 1226 | return status_line + body |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1227 | |
| 1228 | def _deserialize_response(self, payload): |
| 1229 | """Convert string into httplib2 response and content. |
| 1230 | |
| 1231 | Args: |
| 1232 | payload: string, headers and body as a string. |
| 1233 | |
| 1234 | Returns: |
| 1235 | A pair (resp, content), such as would be returned from httplib2.request. |
| 1236 | """ |
| 1237 | # Strip off the status line |
| 1238 | status_line, payload = payload.split('\n', 1) |
| 1239 | protocol, status, reason = status_line.split(' ', 2) |
| 1240 | |
| 1241 | # Parse the rest of the response |
| 1242 | parser = FeedParser() |
| 1243 | parser.feed(payload) |
| 1244 | msg = parser.close() |
| 1245 | msg['status'] = status |
| 1246 | |
| 1247 | # Create httplib2.Response from the parsed headers. |
| 1248 | resp = httplib2.Response(msg) |
| 1249 | resp.reason = reason |
| 1250 | resp.version = int(protocol.split('/', 1)[1].replace('.', '')) |
| 1251 | |
| 1252 | content = payload.split('\r\n\r\n', 1)[1] |
| 1253 | |
| 1254 | return resp, content |
| 1255 | |
| 1256 | def _new_id(self): |
| 1257 | """Create a new id. |
| 1258 | |
| 1259 | Auto incrementing number that avoids conflicts with ids already used. |
| 1260 | |
| 1261 | Returns: |
| 1262 | string, a new unique id. |
| 1263 | """ |
| 1264 | self._last_auto_id += 1 |
| 1265 | while str(self._last_auto_id) in self._requests: |
| 1266 | self._last_auto_id += 1 |
| 1267 | return str(self._last_auto_id) |
| 1268 | |
| 1269 | @util.positional(2) |
| 1270 | def add(self, request, callback=None, request_id=None): |
| 1271 | """Add a new request. |
| 1272 | |
| 1273 | Every callback added will be paired with a unique id, the request_id. That |
| 1274 | unique id will be passed back to the callback when the response comes back |
| 1275 | from the server. The default behavior is to have the library generate it's |
| 1276 | own unique id. If the caller passes in a request_id then they must ensure |
| 1277 | uniqueness for each request_id, and if they are not an exception is |
| 1278 | raised. Callers should either supply all request_ids or nevery supply a |
| 1279 | request id, to avoid such an error. |
| 1280 | |
| 1281 | Args: |
| 1282 | request: HttpRequest, Request to add to the batch. |
| 1283 | callback: callable, A callback to be called for this response, of the |
| 1284 | form callback(id, response, exception). The first parameter is the |
| 1285 | request id, and the second is the deserialized response object. The |
| 1286 | third is an googleapiclient.errors.HttpError exception object if an HTTP error |
| 1287 | occurred while processing the request, or None if no errors occurred. |
| 1288 | request_id: string, A unique id for the request. The id will be passed to |
| 1289 | the callback with the response. |
| 1290 | |
| 1291 | Returns: |
| 1292 | None |
| 1293 | |
| 1294 | Raises: |
| 1295 | BatchError if a media request is added to a batch. |
| 1296 | KeyError is the request_id is not unique. |
| 1297 | """ |
| 1298 | if request_id is None: |
| 1299 | request_id = self._new_id() |
| 1300 | if request.resumable is not None: |
| 1301 | raise BatchError("Media requests cannot be used in a batch request.") |
| 1302 | if request_id in self._requests: |
| 1303 | raise KeyError("A request with this ID already exists: %s" % request_id) |
| 1304 | self._requests[request_id] = request |
| 1305 | self._callbacks[request_id] = callback |
| 1306 | self._order.append(request_id) |
| 1307 | |
| 1308 | def _execute(self, http, order, requests): |
| 1309 | """Serialize batch request, send to server, process response. |
| 1310 | |
| 1311 | Args: |
| 1312 | http: httplib2.Http, an http object to be used to make the request with. |
| 1313 | order: list, list of request ids in the order they were added to the |
| 1314 | batch. |
| 1315 | request: list, list of request objects to send. |
| 1316 | |
| 1317 | Raises: |
| 1318 | httplib2.HttpLib2Error if a transport error has occured. |
| 1319 | googleapiclient.errors.BatchError if the response is the wrong format. |
| 1320 | """ |
| 1321 | message = MIMEMultipart('mixed') |
| 1322 | # Message should not write out it's own headers. |
| 1323 | setattr(message, '_write_headers', lambda self: None) |
| 1324 | |
| 1325 | # Add all the individual requests. |
| 1326 | for request_id in order: |
| 1327 | request = requests[request_id] |
| 1328 | |
| 1329 | msg = MIMENonMultipart('application', 'http') |
| 1330 | msg['Content-Transfer-Encoding'] = 'binary' |
| 1331 | msg['Content-ID'] = self._id_to_header(request_id) |
| 1332 | |
| 1333 | body = self._serialize_request(request) |
| 1334 | msg.set_payload(body) |
| 1335 | message.attach(msg) |
| 1336 | |
Craig Citro | 72389b7 | 2014-07-15 17:12:50 -0700 | [diff] [blame] | 1337 | # encode the body: note that we can't use `as_string`, because |
| 1338 | # it plays games with `From ` lines. |
Pat Ferate | ed9affd | 2015-03-03 16:03:15 -0800 | [diff] [blame] | 1339 | fp = StringIO() |
Craig Citro | 72389b7 | 2014-07-15 17:12:50 -0700 | [diff] [blame] | 1340 | g = Generator(fp, mangle_from_=False) |
| 1341 | g.flatten(message, unixfrom=False) |
| 1342 | body = fp.getvalue() |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1343 | |
| 1344 | headers = {} |
| 1345 | headers['content-type'] = ('multipart/mixed; ' |
| 1346 | 'boundary="%s"') % message.get_boundary() |
| 1347 | |
| 1348 | resp, content = http.request(self._batch_uri, method='POST', body=body, |
| 1349 | headers=headers) |
| 1350 | |
| 1351 | if resp.status >= 300: |
| 1352 | raise HttpError(resp, content, uri=self._batch_uri) |
| 1353 | |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1354 | # Prepend with a content-type header so FeedParser can handle it. |
| 1355 | header = 'content-type: %s\r\n\r\n' % resp['content-type'] |
INADA Naoki | 0915761 | 2015-03-25 01:51:03 +0900 | [diff] [blame] | 1356 | # PY3's FeedParser only accepts unicode. So we should decode content |
| 1357 | # here, and encode each payload again. |
| 1358 | if six.PY3: |
| 1359 | content = content.decode('utf-8') |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1360 | for_parser = header + content |
| 1361 | |
| 1362 | parser = FeedParser() |
| 1363 | parser.feed(for_parser) |
| 1364 | mime_response = parser.close() |
| 1365 | |
| 1366 | if not mime_response.is_multipart(): |
| 1367 | raise BatchError("Response not in multipart/mixed format.", resp=resp, |
| 1368 | content=content) |
| 1369 | |
| 1370 | for part in mime_response.get_payload(): |
| 1371 | request_id = self._header_to_id(part['Content-ID']) |
| 1372 | response, content = self._deserialize_response(part.get_payload()) |
INADA Naoki | 0915761 | 2015-03-25 01:51:03 +0900 | [diff] [blame] | 1373 | # We encode content here to emulate normal http response. |
| 1374 | if isinstance(content, six.text_type): |
| 1375 | content = content.encode('utf-8') |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1376 | self._responses[request_id] = (response, content) |
| 1377 | |
| 1378 | @util.positional(1) |
| 1379 | def execute(self, http=None): |
| 1380 | """Execute all the requests as a single batched HTTP request. |
| 1381 | |
| 1382 | Args: |
| 1383 | http: httplib2.Http, an http object to be used in place of the one the |
| 1384 | HttpRequest request object was constructed with. If one isn't supplied |
| 1385 | then use a http object from the requests in this batch. |
| 1386 | |
| 1387 | Returns: |
| 1388 | None |
| 1389 | |
| 1390 | Raises: |
| 1391 | httplib2.HttpLib2Error if a transport error has occured. |
| 1392 | googleapiclient.errors.BatchError if the response is the wrong format. |
| 1393 | """ |
Mohamed Zenadi | 1b5350d | 2015-07-30 11:52:39 +0200 | [diff] [blame] | 1394 | # If we have no requests return |
| 1395 | if len(self._order) == 0: |
| 1396 | return None |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1397 | |
| 1398 | # If http is not supplied use the first valid one given in the requests. |
| 1399 | if http is None: |
| 1400 | for request_id in self._order: |
| 1401 | request = self._requests[request_id] |
| 1402 | if request is not None: |
| 1403 | http = request.http |
| 1404 | break |
| 1405 | |
| 1406 | if http is None: |
| 1407 | raise ValueError("Missing a valid http object.") |
| 1408 | |
Gabriel Garcia | 23174be | 2016-05-25 17:28:07 +0200 | [diff] [blame] | 1409 | # Special case for OAuth2Credentials-style objects which have not yet been |
| 1410 | # refreshed with an initial access_token. |
| 1411 | if getattr(http.request, 'credentials', None) is not None: |
| 1412 | creds = http.request.credentials |
| 1413 | if not getattr(creds, 'access_token', None): |
| 1414 | LOGGER.info('Attempting refresh to obtain initial access_token') |
| 1415 | creds.refresh(http) |
| 1416 | |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1417 | self._execute(http, self._order, self._requests) |
| 1418 | |
| 1419 | # Loop over all the requests and check for 401s. For each 401 request the |
| 1420 | # credentials should be refreshed and then sent again in a separate batch. |
| 1421 | redo_requests = {} |
| 1422 | redo_order = [] |
| 1423 | |
| 1424 | for request_id in self._order: |
| 1425 | resp, content = self._responses[request_id] |
| 1426 | if resp['status'] == '401': |
| 1427 | redo_order.append(request_id) |
| 1428 | request = self._requests[request_id] |
| 1429 | self._refresh_and_apply_credentials(request, http) |
| 1430 | redo_requests[request_id] = request |
| 1431 | |
| 1432 | if redo_requests: |
| 1433 | self._execute(http, redo_order, redo_requests) |
| 1434 | |
| 1435 | # Now process all callbacks that are erroring, and raise an exception for |
| 1436 | # ones that return a non-2xx response? Or add extra parameter to callback |
| 1437 | # that contains an HttpError? |
| 1438 | |
| 1439 | for request_id in self._order: |
| 1440 | resp, content = self._responses[request_id] |
| 1441 | |
| 1442 | request = self._requests[request_id] |
| 1443 | callback = self._callbacks[request_id] |
| 1444 | |
| 1445 | response = None |
| 1446 | exception = None |
| 1447 | try: |
| 1448 | if resp.status >= 300: |
| 1449 | raise HttpError(resp, content, uri=request.uri) |
| 1450 | response = request.postproc(resp, content) |
INADA Naoki | c1505df | 2014-08-20 15:19:53 +0900 | [diff] [blame] | 1451 | except HttpError as e: |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1452 | exception = e |
| 1453 | |
| 1454 | if callback is not None: |
| 1455 | callback(request_id, response, exception) |
| 1456 | if self._callback is not None: |
| 1457 | self._callback(request_id, response, exception) |
| 1458 | |
| 1459 | |
| 1460 | class HttpRequestMock(object): |
| 1461 | """Mock of HttpRequest. |
| 1462 | |
| 1463 | Do not construct directly, instead use RequestMockBuilder. |
| 1464 | """ |
| 1465 | |
| 1466 | def __init__(self, resp, content, postproc): |
| 1467 | """Constructor for HttpRequestMock |
| 1468 | |
| 1469 | Args: |
| 1470 | resp: httplib2.Response, the response to emulate coming from the request |
| 1471 | content: string, the response body |
| 1472 | postproc: callable, the post processing function usually supplied by |
| 1473 | the model class. See model.JsonModel.response() as an example. |
| 1474 | """ |
| 1475 | self.resp = resp |
| 1476 | self.content = content |
| 1477 | self.postproc = postproc |
| 1478 | if resp is None: |
| 1479 | self.resp = httplib2.Response({'status': 200, 'reason': 'OK'}) |
| 1480 | if 'reason' in self.resp: |
| 1481 | self.resp.reason = self.resp['reason'] |
| 1482 | |
| 1483 | def execute(self, http=None): |
| 1484 | """Execute the request. |
| 1485 | |
| 1486 | Same behavior as HttpRequest.execute(), but the response is |
| 1487 | mocked and not really from an HTTP request/response. |
| 1488 | """ |
| 1489 | return self.postproc(self.resp, self.content) |
| 1490 | |
| 1491 | |
| 1492 | class RequestMockBuilder(object): |
| 1493 | """A simple mock of HttpRequest |
| 1494 | |
| 1495 | Pass in a dictionary to the constructor that maps request methodIds to |
| 1496 | tuples of (httplib2.Response, content, opt_expected_body) that should be |
| 1497 | returned when that method is called. None may also be passed in for the |
| 1498 | httplib2.Response, in which case a 200 OK response will be generated. |
| 1499 | If an opt_expected_body (str or dict) is provided, it will be compared to |
| 1500 | the body and UnexpectedBodyError will be raised on inequality. |
| 1501 | |
| 1502 | Example: |
| 1503 | response = '{"data": {"id": "tag:google.c...' |
| 1504 | requestBuilder = RequestMockBuilder( |
| 1505 | { |
| 1506 | 'plus.activities.get': (None, response), |
| 1507 | } |
| 1508 | ) |
| 1509 | googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder) |
| 1510 | |
| 1511 | Methods that you do not supply a response for will return a |
| 1512 | 200 OK with an empty string as the response content or raise an excpetion |
| 1513 | if check_unexpected is set to True. The methodId is taken from the rpcName |
| 1514 | in the discovery document. |
| 1515 | |
| 1516 | For more details see the project wiki. |
| 1517 | """ |
| 1518 | |
| 1519 | def __init__(self, responses, check_unexpected=False): |
| 1520 | """Constructor for RequestMockBuilder |
| 1521 | |
| 1522 | The constructed object should be a callable object |
| 1523 | that can replace the class HttpResponse. |
| 1524 | |
| 1525 | responses - A dictionary that maps methodIds into tuples |
| 1526 | of (httplib2.Response, content). The methodId |
| 1527 | comes from the 'rpcName' field in the discovery |
| 1528 | document. |
| 1529 | check_unexpected - A boolean setting whether or not UnexpectedMethodError |
| 1530 | should be raised on unsupplied method. |
| 1531 | """ |
| 1532 | self.responses = responses |
| 1533 | self.check_unexpected = check_unexpected |
| 1534 | |
| 1535 | def __call__(self, http, postproc, uri, method='GET', body=None, |
| 1536 | headers=None, methodId=None, resumable=None): |
| 1537 | """Implements the callable interface that discovery.build() expects |
| 1538 | of requestBuilder, which is to build an object compatible with |
| 1539 | HttpRequest.execute(). See that method for the description of the |
| 1540 | parameters and the expected response. |
| 1541 | """ |
| 1542 | if methodId in self.responses: |
| 1543 | response = self.responses[methodId] |
| 1544 | resp, content = response[:2] |
| 1545 | if len(response) > 2: |
| 1546 | # Test the body against the supplied expected_body. |
| 1547 | expected_body = response[2] |
| 1548 | if bool(expected_body) != bool(body): |
| 1549 | # Not expecting a body and provided one |
| 1550 | # or expecting a body and not provided one. |
| 1551 | raise UnexpectedBodyError(expected_body, body) |
| 1552 | if isinstance(expected_body, str): |
Craig Citro | 6ae34d7 | 2014-08-18 23:10:09 -0700 | [diff] [blame] | 1553 | expected_body = json.loads(expected_body) |
| 1554 | body = json.loads(body) |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1555 | if body != expected_body: |
| 1556 | raise UnexpectedBodyError(expected_body, body) |
| 1557 | return HttpRequestMock(resp, content, postproc) |
| 1558 | elif self.check_unexpected: |
| 1559 | raise UnexpectedMethodError(methodId=methodId) |
| 1560 | else: |
| 1561 | model = JsonModel(False) |
| 1562 | return HttpRequestMock(None, '{}', model.response) |
| 1563 | |
| 1564 | |
| 1565 | class HttpMock(object): |
| 1566 | """Mock of httplib2.Http""" |
| 1567 | |
| 1568 | def __init__(self, filename=None, headers=None): |
| 1569 | """ |
| 1570 | Args: |
| 1571 | filename: string, absolute filename to read response from |
| 1572 | headers: dict, header to return with response |
| 1573 | """ |
| 1574 | if headers is None: |
Craig Gurnik | 8e55b76 | 2015-01-20 15:00:10 -0500 | [diff] [blame] | 1575 | headers = {'status': '200'} |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1576 | if filename: |
Alan Briolat | 26b0100 | 2015-08-14 00:13:57 +0100 | [diff] [blame] | 1577 | f = open(filename, 'rb') |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1578 | self.data = f.read() |
| 1579 | f.close() |
| 1580 | else: |
| 1581 | self.data = None |
| 1582 | self.response_headers = headers |
| 1583 | self.headers = None |
| 1584 | self.uri = None |
| 1585 | self.method = None |
| 1586 | self.body = None |
| 1587 | self.headers = None |
| 1588 | |
| 1589 | |
| 1590 | def request(self, uri, |
| 1591 | method='GET', |
| 1592 | body=None, |
| 1593 | headers=None, |
| 1594 | redirections=1, |
| 1595 | connection_type=None): |
| 1596 | self.uri = uri |
| 1597 | self.method = method |
| 1598 | self.body = body |
| 1599 | self.headers = headers |
| 1600 | return httplib2.Response(self.response_headers), self.data |
| 1601 | |
| 1602 | |
| 1603 | class HttpMockSequence(object): |
| 1604 | """Mock of httplib2.Http |
| 1605 | |
| 1606 | Mocks a sequence of calls to request returning different responses for each |
| 1607 | call. Create an instance initialized with the desired response headers |
| 1608 | and content and then use as if an httplib2.Http instance. |
| 1609 | |
| 1610 | http = HttpMockSequence([ |
| 1611 | ({'status': '401'}, ''), |
| 1612 | ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'), |
| 1613 | ({'status': '200'}, 'echo_request_headers'), |
| 1614 | ]) |
| 1615 | resp, content = http.request("http://examples.com") |
| 1616 | |
| 1617 | There are special values you can pass in for content to trigger |
| 1618 | behavours that are helpful in testing. |
| 1619 | |
| 1620 | 'echo_request_headers' means return the request headers in the response body |
| 1621 | 'echo_request_headers_as_json' means return the request headers in |
| 1622 | the response body |
| 1623 | 'echo_request_body' means return the request body in the response body |
| 1624 | 'echo_request_uri' means return the request uri in the response body |
| 1625 | """ |
| 1626 | |
| 1627 | def __init__(self, iterable): |
| 1628 | """ |
| 1629 | Args: |
| 1630 | iterable: iterable, a sequence of pairs of (headers, body) |
| 1631 | """ |
| 1632 | self._iterable = iterable |
| 1633 | self.follow_redirects = True |
| 1634 | |
| 1635 | def request(self, uri, |
| 1636 | method='GET', |
| 1637 | body=None, |
| 1638 | headers=None, |
| 1639 | redirections=1, |
| 1640 | connection_type=None): |
| 1641 | resp, content = self._iterable.pop(0) |
| 1642 | if content == 'echo_request_headers': |
| 1643 | content = headers |
| 1644 | elif content == 'echo_request_headers_as_json': |
Craig Citro | 6ae34d7 | 2014-08-18 23:10:09 -0700 | [diff] [blame] | 1645 | content = json.dumps(headers) |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1646 | elif content == 'echo_request_body': |
| 1647 | if hasattr(body, 'read'): |
| 1648 | content = body.read() |
| 1649 | else: |
| 1650 | content = body |
| 1651 | elif content == 'echo_request_uri': |
| 1652 | content = uri |
INADA Naoki | 0915761 | 2015-03-25 01:51:03 +0900 | [diff] [blame] | 1653 | if isinstance(content, six.text_type): |
| 1654 | content = content.encode('utf-8') |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1655 | return httplib2.Response(resp), content |
| 1656 | |
| 1657 | |
| 1658 | def set_user_agent(http, user_agent): |
| 1659 | """Set the user-agent on every request. |
| 1660 | |
| 1661 | Args: |
| 1662 | http - An instance of httplib2.Http |
| 1663 | or something that acts like it. |
| 1664 | user_agent: string, the value for the user-agent header. |
| 1665 | |
| 1666 | Returns: |
| 1667 | A modified instance of http that was passed in. |
| 1668 | |
| 1669 | Example: |
| 1670 | |
| 1671 | h = httplib2.Http() |
| 1672 | h = set_user_agent(h, "my-app-name/6.0") |
| 1673 | |
| 1674 | Most of the time the user-agent will be set doing auth, this is for the rare |
| 1675 | cases where you are accessing an unauthenticated endpoint. |
| 1676 | """ |
| 1677 | request_orig = http.request |
| 1678 | |
| 1679 | # The closure that will replace 'httplib2.Http.request'. |
| 1680 | def new_request(uri, method='GET', body=None, headers=None, |
| 1681 | redirections=httplib2.DEFAULT_MAX_REDIRECTS, |
| 1682 | connection_type=None): |
| 1683 | """Modify the request headers to add the user-agent.""" |
| 1684 | if headers is None: |
| 1685 | headers = {} |
| 1686 | if 'user-agent' in headers: |
| 1687 | headers['user-agent'] = user_agent + ' ' + headers['user-agent'] |
| 1688 | else: |
| 1689 | headers['user-agent'] = user_agent |
| 1690 | resp, content = request_orig(uri, method, body, headers, |
| 1691 | redirections, connection_type) |
| 1692 | return resp, content |
| 1693 | |
| 1694 | http.request = new_request |
| 1695 | return http |
| 1696 | |
| 1697 | |
| 1698 | def tunnel_patch(http): |
| 1699 | """Tunnel PATCH requests over POST. |
| 1700 | Args: |
| 1701 | http - An instance of httplib2.Http |
| 1702 | or something that acts like it. |
| 1703 | |
| 1704 | Returns: |
| 1705 | A modified instance of http that was passed in. |
| 1706 | |
| 1707 | Example: |
| 1708 | |
| 1709 | h = httplib2.Http() |
| 1710 | h = tunnel_patch(h, "my-app-name/6.0") |
| 1711 | |
| 1712 | Useful if you are running on a platform that doesn't support PATCH. |
| 1713 | Apply this last if you are using OAuth 1.0, as changing the method |
| 1714 | will result in a different signature. |
| 1715 | """ |
| 1716 | request_orig = http.request |
| 1717 | |
| 1718 | # The closure that will replace 'httplib2.Http.request'. |
| 1719 | def new_request(uri, method='GET', body=None, headers=None, |
| 1720 | redirections=httplib2.DEFAULT_MAX_REDIRECTS, |
| 1721 | connection_type=None): |
| 1722 | """Modify the request headers to add the user-agent.""" |
| 1723 | if headers is None: |
| 1724 | headers = {} |
| 1725 | if method == 'PATCH': |
| 1726 | if 'oauth_token' in headers.get('authorization', ''): |
Emmett Butler | 0969915 | 2016-02-08 14:26:00 -0800 | [diff] [blame] | 1727 | LOGGER.warning( |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 1728 | 'OAuth 1.0 request made with Credentials after tunnel_patch.') |
| 1729 | headers['x-http-method-override'] = "PATCH" |
| 1730 | method = 'POST' |
| 1731 | resp, content = request_orig(uri, method, body, headers, |
| 1732 | redirections, connection_type) |
| 1733 | return resp, content |
| 1734 | |
| 1735 | http.request = new_request |
| 1736 | return http |
Igor Maravić | 2243529 | 2017-01-19 22:28:22 +0100 | [diff] [blame^] | 1737 | |
| 1738 | |
| 1739 | def build_http(): |
| 1740 | """Builds httplib2.Http object |
| 1741 | |
| 1742 | Returns: |
| 1743 | A httplib2.Http object, which is used to make http requests, and which has timeout set by default. |
| 1744 | To override default timeout call |
| 1745 | |
| 1746 | socket.setdefaulttimeout(timeout_in_sec) |
| 1747 | |
| 1748 | before interacting with this method. |
| 1749 | """ |
| 1750 | if socket.getdefaulttimeout() is not None: |
| 1751 | http_timeout = socket.getdefaulttimeout() |
| 1752 | else: |
| 1753 | http_timeout = DEFAULT_HTTP_TIMEOUT_SEC |
| 1754 | return httplib2.Http(timeout=http_timeout) |