blob: 6a47bf7623e7d4cedeb8a556d74d7e84d7ef55a8 [file] [log] [blame]
Craig Citro751b7fb2014-09-23 11:20:38 -07001# Copyright 2014 Google Inc. All Rights Reserved.
John Asmuth864311d2014-04-24 15:46:08 -04002#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""Classes to encapsulate a single HTTP request.
16
17The classes implement a command pattern, with every
18object supporting an execute() method that does the
cspeidelfbaf9d72018-05-10 12:50:12 -060019actual HTTP request.
John Asmuth864311d2014-04-24 15:46:08 -040020"""
INADA Naoki0bceb332014-08-20 15:27:52 +090021from __future__ import absolute_import
INADA Naokie4ea1a92015-03-04 03:45:42 +090022import six
eesheeshc6425a02016-02-12 15:07:06 +000023from six.moves import http_client
INADA Naokie4ea1a92015-03-04 03:45:42 +090024from six.moves import range
John Asmuth864311d2014-04-24 15:46:08 -040025
26__author__ = 'jcgregorio@google.com (Joe Gregorio)'
27
Pat Ferateed9affd2015-03-03 16:03:15 -080028from six import BytesIO, StringIO
Pat Ferated5b61bd2015-03-03 16:04:11 -080029from six.moves.urllib.parse import urlparse, urlunparse, quote, unquote
Pat Ferateed9affd2015-03-03 16:03:15 -080030
John Asmuth864311d2014-04-24 15:46:08 -040031import base64
32import copy
33import gzip
34import httplib2
Craig Citro6ae34d72014-08-18 23:10:09 -070035import json
John Asmuth864311d2014-04-24 15:46:08 -040036import logging
John Asmuth864311d2014-04-24 15:46:08 -040037import mimetypes
38import os
39import random
eesheeshc6425a02016-02-12 15:07:06 +000040import socket
John Asmuth864311d2014-04-24 15:46:08 -040041import sys
42import time
John Asmuth864311d2014-04-24 15:46:08 -040043import uuid
44
Tay Ray Chuan3146c922016-04-20 16:38:19 +000045# TODO(issue 221): Remove this conditional import jibbajabba.
46try:
47 import ssl
48except ImportError:
49 _ssl_SSLError = object()
50else:
51 _ssl_SSLError = ssl.SSLError
52
John Asmuth864311d2014-04-24 15:46:08 -040053from email.generator import Generator
54from email.mime.multipart import MIMEMultipart
55from email.mime.nonmultipart import MIMENonMultipart
56from email.parser import FeedParser
Pat Ferateb240c172015-03-03 16:23:51 -080057
Helen Koikede13e3b2018-04-26 16:05:16 -030058from googleapiclient import _helpers as util
Jon Wayne Parrott6755f612016-08-15 10:52:26 -070059
Jon Wayne Parrottd3a5cf42017-06-19 17:55:04 -070060from googleapiclient import _auth
Pat Ferateb240c172015-03-03 16:23:51 -080061from googleapiclient.errors import BatchError
62from googleapiclient.errors import HttpError
63from googleapiclient.errors import InvalidChunkSizeError
64from googleapiclient.errors import ResumableUploadError
65from googleapiclient.errors import UnexpectedBodyError
66from googleapiclient.errors import UnexpectedMethodError
67from googleapiclient.model import JsonModel
John Asmuth864311d2014-04-24 15:46:08 -040068
69
Emmett Butler09699152016-02-08 14:26:00 -080070LOGGER = logging.getLogger(__name__)
71
Craig Citrod1aa6912018-02-28 15:31:08 -080072DEFAULT_CHUNK_SIZE = 100*1024*1024
John Asmuth864311d2014-04-24 15:46:08 -040073
74MAX_URI_LENGTH = 2048
75
eesheeshc6425a02016-02-12 15:07:06 +000076_TOO_MANY_REQUESTS = 429
77
Igor Maravić22435292017-01-19 22:28:22 +010078DEFAULT_HTTP_TIMEOUT_SEC = 60
79
Jon Wayne Parrottbae748a2018-03-28 10:21:12 -070080_LEGACY_BATCH_URI = 'https://www.googleapis.com/batch'
81
eesheeshc6425a02016-02-12 15:07:06 +000082
83def _should_retry_response(resp_status, content):
84 """Determines whether a response should be retried.
85
86 Args:
87 resp_status: The response status received.
Nilayan Bhattacharya90ffb852017-12-05 15:30:32 -080088 content: The response content body.
eesheeshc6425a02016-02-12 15:07:06 +000089
90 Returns:
91 True if the response should be retried, otherwise False.
92 """
93 # Retry on 5xx errors.
94 if resp_status >= 500:
95 return True
96
97 # Retry on 429 errors.
98 if resp_status == _TOO_MANY_REQUESTS:
99 return True
100
101 # For 403 errors, we have to check for the `reason` in the response to
102 # determine if we should retry.
103 if resp_status == six.moves.http_client.FORBIDDEN:
104 # If there's no details about the 403 type, don't retry.
105 if not content:
106 return False
107
108 # Content is in JSON format.
109 try:
110 data = json.loads(content.decode('utf-8'))
Nilayan Bhattacharya90ffb852017-12-05 15:30:32 -0800111 if isinstance(data, dict):
112 reason = data['error']['errors'][0]['reason']
113 else:
114 reason = data[0]['error']['errors']['reason']
eesheeshc6425a02016-02-12 15:07:06 +0000115 except (UnicodeDecodeError, ValueError, KeyError):
116 LOGGER.warning('Invalid JSON content from response: %s', content)
117 return False
118
119 LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason)
120
121 # Only retry on rate limit related failures.
122 if reason in ('userRateLimitExceeded', 'rateLimitExceeded', ):
123 return True
124
125 # Everything else is a success or non-retriable so break.
126 return False
127
John Asmuth864311d2014-04-24 15:46:08 -0400128
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100129def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args,
130 **kwargs):
131 """Retries an HTTP request multiple times while handling errors.
132
133 If after all retries the request still fails, last error is either returned as
134 return value (for HTTP 5xx errors) or thrown (for ssl.SSLError).
135
136 Args:
137 http: Http object to be used to execute request.
138 num_retries: Maximum number of retries.
139 req_type: Type of the request (used for logging retries).
140 sleep, rand: Functions to sleep for random time between retries.
141 uri: URI to be requested.
142 method: HTTP method to be used.
143 args, kwargs: Additional arguments passed to http.request.
144
145 Returns:
146 resp, content - Response from the http request (may be HTTP 5xx).
147 """
148 resp = None
eesheeshc6425a02016-02-12 15:07:06 +0000149 content = None
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100150 for retry_num in range(num_retries + 1):
151 if retry_num > 0:
eesheeshc6425a02016-02-12 15:07:06 +0000152 # Sleep before retrying.
153 sleep_time = rand() * 2 ** retry_num
Emmett Butler09699152016-02-08 14:26:00 -0800154 LOGGER.warning(
eesheeshc6425a02016-02-12 15:07:06 +0000155 'Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s',
156 sleep_time, retry_num, num_retries, req_type, method, uri,
157 resp.status if resp else exception)
158 sleep(sleep_time)
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100159
160 try:
eesheeshc6425a02016-02-12 15:07:06 +0000161 exception = None
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100162 resp, content = http.request(uri, method, *args, **kwargs)
eesheeshc6425a02016-02-12 15:07:06 +0000163 # Retry on SSL errors and socket timeout errors.
Tay Ray Chuan3146c922016-04-20 16:38:19 +0000164 except _ssl_SSLError as ssl_error:
eesheeshc6425a02016-02-12 15:07:06 +0000165 exception = ssl_error
Alexander Mohrfff3ae52018-04-27 13:39:53 -0700166 except socket.timeout as socket_timeout:
167 # It's important that this be before socket.error as it's a subclass
168 # socket.timeout has no errorcode
169 exception = socket_timeout
eesheeshc6425a02016-02-12 15:07:06 +0000170 except socket.error as socket_error:
171 # errno's contents differ by platform, so we have to match by name.
Alexander Mohrfff3ae52018-04-27 13:39:53 -0700172 if socket.errno.errorcode.get(socket_error.errno) not in {
173 'WSAETIMEDOUT', 'ETIMEDOUT', 'EPIPE', 'ECONNABORTED'}:
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100174 raise
eesheeshc6425a02016-02-12 15:07:06 +0000175 exception = socket_error
Bashir Sadjadc35150f2018-06-25 11:46:09 -0400176 except httplib2.ServerNotFoundError as server_not_found_error:
177 exception = server_not_found_error
eesheeshc6425a02016-02-12 15:07:06 +0000178
179 if exception:
180 if retry_num == num_retries:
181 raise exception
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100182 else:
183 continue
eesheeshc6425a02016-02-12 15:07:06 +0000184
185 if not _should_retry_response(resp.status, content):
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100186 break
187
188 return resp, content
189
190
John Asmuth864311d2014-04-24 15:46:08 -0400191class MediaUploadProgress(object):
192 """Status of a resumable upload."""
193
194 def __init__(self, resumable_progress, total_size):
195 """Constructor.
196
197 Args:
198 resumable_progress: int, bytes sent so far.
199 total_size: int, total bytes in complete upload, or None if the total
200 upload size isn't known ahead of time.
201 """
202 self.resumable_progress = resumable_progress
203 self.total_size = total_size
204
205 def progress(self):
206 """Percent of upload completed, as a float.
207
208 Returns:
209 the percentage complete as a float, returning 0.0 if the total size of
210 the upload is unknown.
211 """
andrewnestera4a44cf2017-03-31 16:09:31 +0300212 if self.total_size is not None and self.total_size != 0:
John Asmuth864311d2014-04-24 15:46:08 -0400213 return float(self.resumable_progress) / float(self.total_size)
214 else:
215 return 0.0
216
217
218class MediaDownloadProgress(object):
219 """Status of a resumable download."""
220
221 def __init__(self, resumable_progress, total_size):
222 """Constructor.
223
224 Args:
225 resumable_progress: int, bytes received so far.
226 total_size: int, total bytes in complete download.
227 """
228 self.resumable_progress = resumable_progress
229 self.total_size = total_size
230
231 def progress(self):
232 """Percent of download completed, as a float.
233
234 Returns:
235 the percentage complete as a float, returning 0.0 if the total size of
236 the download is unknown.
237 """
andrewnestera4a44cf2017-03-31 16:09:31 +0300238 if self.total_size is not None and self.total_size != 0:
John Asmuth864311d2014-04-24 15:46:08 -0400239 return float(self.resumable_progress) / float(self.total_size)
240 else:
241 return 0.0
242
243
244class MediaUpload(object):
245 """Describes a media object to upload.
246
247 Base class that defines the interface of MediaUpload subclasses.
248
249 Note that subclasses of MediaUpload may allow you to control the chunksize
250 when uploading a media object. It is important to keep the size of the chunk
251 as large as possible to keep the upload efficient. Other factors may influence
252 the size of the chunk you use, particularly if you are working in an
253 environment where individual HTTP requests may have a hardcoded time limit,
254 such as under certain classes of requests under Google App Engine.
255
256 Streams are io.Base compatible objects that support seek(). Some MediaUpload
257 subclasses support using streams directly to upload data. Support for
258 streaming may be indicated by a MediaUpload sub-class and if appropriate for a
259 platform that stream will be used for uploading the media object. The support
260 for streaming is indicated by has_stream() returning True. The stream() method
261 should return an io.Base object that supports seek(). On platforms where the
262 underlying httplib module supports streaming, for example Python 2.6 and
263 later, the stream will be passed into the http library which will result in
264 less memory being used and possibly faster uploads.
265
266 If you need to upload media that can't be uploaded using any of the existing
267 MediaUpload sub-class then you can sub-class MediaUpload for your particular
268 needs.
269 """
270
271 def chunksize(self):
272 """Chunk size for resumable uploads.
273
274 Returns:
275 Chunk size in bytes.
276 """
277 raise NotImplementedError()
278
279 def mimetype(self):
280 """Mime type of the body.
281
282 Returns:
283 Mime type.
284 """
285 return 'application/octet-stream'
286
287 def size(self):
288 """Size of upload.
289
290 Returns:
291 Size of the body, or None of the size is unknown.
292 """
293 return None
294
295 def resumable(self):
296 """Whether this upload is resumable.
297
298 Returns:
299 True if resumable upload or False.
300 """
301 return False
302
303 def getbytes(self, begin, end):
304 """Get bytes from the media.
305
306 Args:
307 begin: int, offset from beginning of file.
308 length: int, number of bytes to read, starting at begin.
309
310 Returns:
311 A string of bytes read. May be shorter than length if EOF was reached
312 first.
313 """
314 raise NotImplementedError()
315
316 def has_stream(self):
317 """Does the underlying upload support a streaming interface.
318
319 Streaming means it is an io.IOBase subclass that supports seek, i.e.
320 seekable() returns True.
321
322 Returns:
323 True if the call to stream() will return an instance of a seekable io.Base
324 subclass.
325 """
326 return False
327
328 def stream(self):
329 """A stream interface to the data being uploaded.
330
331 Returns:
332 The returned value is an io.IOBase subclass that supports seek, i.e.
333 seekable() returns True.
334 """
335 raise NotImplementedError()
336
337 @util.positional(1)
338 def _to_json(self, strip=None):
339 """Utility function for creating a JSON representation of a MediaUpload.
340
341 Args:
342 strip: array, An array of names of members to not include in the JSON.
343
344 Returns:
345 string, a JSON representation of this instance, suitable to pass to
346 from_json().
347 """
348 t = type(self)
349 d = copy.copy(self.__dict__)
350 if strip is not None:
351 for member in strip:
352 del d[member]
353 d['_class'] = t.__name__
354 d['_module'] = t.__module__
Craig Citro6ae34d72014-08-18 23:10:09 -0700355 return json.dumps(d)
John Asmuth864311d2014-04-24 15:46:08 -0400356
357 def to_json(self):
358 """Create a JSON representation of an instance of MediaUpload.
359
360 Returns:
361 string, a JSON representation of this instance, suitable to pass to
362 from_json().
363 """
364 return self._to_json()
365
366 @classmethod
367 def new_from_json(cls, s):
368 """Utility class method to instantiate a MediaUpload subclass from a JSON
369 representation produced by to_json().
370
371 Args:
372 s: string, JSON from to_json().
373
374 Returns:
375 An instance of the subclass of MediaUpload that was serialized with
376 to_json().
377 """
Craig Citro6ae34d72014-08-18 23:10:09 -0700378 data = json.loads(s)
John Asmuth864311d2014-04-24 15:46:08 -0400379 # Find and call the right classmethod from_json() to restore the object.
380 module = data['_module']
381 m = __import__(module, fromlist=module.split('.')[:-1])
382 kls = getattr(m, data['_class'])
383 from_json = getattr(kls, 'from_json')
384 return from_json(s)
385
386
387class MediaIoBaseUpload(MediaUpload):
388 """A MediaUpload for a io.Base objects.
389
390 Note that the Python file object is compatible with io.Base and can be used
391 with this class also.
392
Pat Ferateed9affd2015-03-03 16:03:15 -0800393 fh = BytesIO('...Some data to upload...')
John Asmuth864311d2014-04-24 15:46:08 -0400394 media = MediaIoBaseUpload(fh, mimetype='image/png',
395 chunksize=1024*1024, resumable=True)
396 farm.animals().insert(
397 id='cow',
398 name='cow.png',
399 media_body=media).execute()
400
401 Depending on the platform you are working on, you may pass -1 as the
402 chunksize, which indicates that the entire file should be uploaded in a single
403 request. If the underlying platform supports streams, such as Python 2.6 or
404 later, then this can be very efficient as it avoids multiple connections, and
405 also avoids loading the entire file into memory before sending it. Note that
406 Google App Engine has a 5MB limit on request size, so you should never set
407 your chunksize larger than 5MB, or to -1.
408 """
409
410 @util.positional(3)
411 def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE,
412 resumable=False):
413 """Constructor.
414
415 Args:
416 fd: io.Base or file object, The source of the bytes to upload. MUST be
417 opened in blocking mode, do not use streams opened in non-blocking mode.
418 The given stream must be seekable, that is, it must be able to call
419 seek() on fd.
420 mimetype: string, Mime-type of the file.
421 chunksize: int, File will be uploaded in chunks of this many bytes. Only
422 used if resumable=True. Pass in a value of -1 if the file is to be
423 uploaded as a single chunk. Note that Google App Engine has a 5MB limit
424 on request size, so you should never set your chunksize larger than 5MB,
425 or to -1.
426 resumable: bool, True if this is a resumable upload. False means upload
427 in a single request.
428 """
429 super(MediaIoBaseUpload, self).__init__()
430 self._fd = fd
431 self._mimetype = mimetype
432 if not (chunksize == -1 or chunksize > 0):
433 raise InvalidChunkSizeError()
434 self._chunksize = chunksize
435 self._resumable = resumable
436
437 self._fd.seek(0, os.SEEK_END)
438 self._size = self._fd.tell()
439
440 def chunksize(self):
441 """Chunk size for resumable uploads.
442
443 Returns:
444 Chunk size in bytes.
445 """
446 return self._chunksize
447
448 def mimetype(self):
449 """Mime type of the body.
450
451 Returns:
452 Mime type.
453 """
454 return self._mimetype
455
456 def size(self):
457 """Size of upload.
458
459 Returns:
460 Size of the body, or None of the size is unknown.
461 """
462 return self._size
463
464 def resumable(self):
465 """Whether this upload is resumable.
466
467 Returns:
468 True if resumable upload or False.
469 """
470 return self._resumable
471
472 def getbytes(self, begin, length):
473 """Get bytes from the media.
474
475 Args:
476 begin: int, offset from beginning of file.
477 length: int, number of bytes to read, starting at begin.
478
479 Returns:
480 A string of bytes read. May be shorted than length if EOF was reached
481 first.
482 """
483 self._fd.seek(begin)
484 return self._fd.read(length)
485
486 def has_stream(self):
487 """Does the underlying upload support a streaming interface.
488
489 Streaming means it is an io.IOBase subclass that supports seek, i.e.
490 seekable() returns True.
491
492 Returns:
493 True if the call to stream() will return an instance of a seekable io.Base
494 subclass.
495 """
496 return True
497
498 def stream(self):
499 """A stream interface to the data being uploaded.
500
501 Returns:
502 The returned value is an io.IOBase subclass that supports seek, i.e.
503 seekable() returns True.
504 """
505 return self._fd
506
507 def to_json(self):
508 """This upload type is not serializable."""
509 raise NotImplementedError('MediaIoBaseUpload is not serializable.')
510
511
512class MediaFileUpload(MediaIoBaseUpload):
513 """A MediaUpload for a file.
514
515 Construct a MediaFileUpload and pass as the media_body parameter of the
516 method. For example, if we had a service that allowed uploading images:
517
John Asmuth864311d2014-04-24 15:46:08 -0400518 media = MediaFileUpload('cow.png', mimetype='image/png',
519 chunksize=1024*1024, resumable=True)
520 farm.animals().insert(
521 id='cow',
522 name='cow.png',
523 media_body=media).execute()
524
525 Depending on the platform you are working on, you may pass -1 as the
526 chunksize, which indicates that the entire file should be uploaded in a single
527 request. If the underlying platform supports streams, such as Python 2.6 or
528 later, then this can be very efficient as it avoids multiple connections, and
529 also avoids loading the entire file into memory before sending it. Note that
530 Google App Engine has a 5MB limit on request size, so you should never set
531 your chunksize larger than 5MB, or to -1.
532 """
533
534 @util.positional(2)
535 def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE,
536 resumable=False):
537 """Constructor.
538
539 Args:
540 filename: string, Name of the file.
541 mimetype: string, Mime-type of the file. If None then a mime-type will be
542 guessed from the file extension.
543 chunksize: int, File will be uploaded in chunks of this many bytes. Only
544 used if resumable=True. Pass in a value of -1 if the file is to be
545 uploaded in a single chunk. Note that Google App Engine has a 5MB limit
546 on request size, so you should never set your chunksize larger than 5MB,
547 or to -1.
548 resumable: bool, True if this is a resumable upload. False means upload
549 in a single request.
550 """
551 self._filename = filename
552 fd = open(self._filename, 'rb')
553 if mimetype is None:
Nam T. Nguyendc136312015-12-01 10:18:56 -0800554 # No mimetype provided, make a guess.
555 mimetype, _ = mimetypes.guess_type(filename)
556 if mimetype is None:
557 # Guess failed, use octet-stream.
558 mimetype = 'application/octet-stream'
John Asmuth864311d2014-04-24 15:46:08 -0400559 super(MediaFileUpload, self).__init__(fd, mimetype, chunksize=chunksize,
560 resumable=resumable)
561
562 def to_json(self):
563 """Creating a JSON representation of an instance of MediaFileUpload.
564
565 Returns:
566 string, a JSON representation of this instance, suitable to pass to
567 from_json().
568 """
569 return self._to_json(strip=['_fd'])
570
571 @staticmethod
572 def from_json(s):
Craig Citro6ae34d72014-08-18 23:10:09 -0700573 d = json.loads(s)
John Asmuth864311d2014-04-24 15:46:08 -0400574 return MediaFileUpload(d['_filename'], mimetype=d['_mimetype'],
575 chunksize=d['_chunksize'], resumable=d['_resumable'])
576
577
578class MediaInMemoryUpload(MediaIoBaseUpload):
579 """MediaUpload for a chunk of bytes.
580
581 DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for
582 the stream.
583 """
584
585 @util.positional(2)
586 def __init__(self, body, mimetype='application/octet-stream',
587 chunksize=DEFAULT_CHUNK_SIZE, resumable=False):
588 """Create a new MediaInMemoryUpload.
589
590 DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for
591 the stream.
592
593 Args:
594 body: string, Bytes of body content.
595 mimetype: string, Mime-type of the file or default of
596 'application/octet-stream'.
597 chunksize: int, File will be uploaded in chunks of this many bytes. Only
598 used if resumable=True.
599 resumable: bool, True if this is a resumable upload. False means upload
600 in a single request.
601 """
Pat Ferateed9affd2015-03-03 16:03:15 -0800602 fd = BytesIO(body)
John Asmuth864311d2014-04-24 15:46:08 -0400603 super(MediaInMemoryUpload, self).__init__(fd, mimetype, chunksize=chunksize,
604 resumable=resumable)
605
606
607class MediaIoBaseDownload(object):
608 """"Download media resources.
609
610 Note that the Python file object is compatible with io.Base and can be used
611 with this class also.
612
613
614 Example:
615 request = farms.animals().get_media(id='cow')
616 fh = io.FileIO('cow.png', mode='wb')
617 downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024)
618
619 done = False
620 while done is False:
621 status, done = downloader.next_chunk()
622 if status:
623 print "Download %d%%." % int(status.progress() * 100)
624 print "Download Complete!"
625 """
626
627 @util.positional(3)
628 def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE):
629 """Constructor.
630
631 Args:
632 fd: io.Base or file object, The stream in which to write the downloaded
633 bytes.
634 request: googleapiclient.http.HttpRequest, the media request to perform in
635 chunks.
636 chunksize: int, File will be downloaded in chunks of this many bytes.
637 """
638 self._fd = fd
639 self._request = request
640 self._uri = request.uri
641 self._chunksize = chunksize
642 self._progress = 0
643 self._total_size = None
644 self._done = False
645
646 # Stubs for testing.
647 self._sleep = time.sleep
648 self._rand = random.random
649
650 @util.positional(1)
651 def next_chunk(self, num_retries=0):
652 """Get the next chunk of the download.
653
654 Args:
Zhihao Yuancc6d3982016-07-27 11:40:45 -0500655 num_retries: Integer, number of times to retry with randomized
John Asmuth864311d2014-04-24 15:46:08 -0400656 exponential backoff. If all retries fail, the raised HttpError
657 represents the last request. If zero (default), we attempt the
658 request only once.
659
660 Returns:
Nilayan Bhattacharya89906ac2017-10-27 13:47:23 -0700661 (status, done): (MediaDownloadProgress, boolean)
John Asmuth864311d2014-04-24 15:46:08 -0400662 The value of 'done' will be True when the media has been fully
Daniel44067782018-01-16 23:17:56 +0100663 downloaded or the total size of the media is unknown.
John Asmuth864311d2014-04-24 15:46:08 -0400664
665 Raises:
666 googleapiclient.errors.HttpError if the response was not a 2xx.
667 httplib2.HttpLib2Error if a transport error has occured.
668 """
669 headers = {
670 'range': 'bytes=%d-%d' % (
671 self._progress, self._progress + self._chunksize)
672 }
673 http = self._request.http
674
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100675 resp, content = _retry_request(
676 http, num_retries, 'media download', self._sleep, self._rand, self._uri,
677 'GET', headers=headers)
John Asmuth864311d2014-04-24 15:46:08 -0400678
679 if resp.status in [200, 206]:
680 if 'content-location' in resp and resp['content-location'] != self._uri:
681 self._uri = resp['content-location']
682 self._progress += len(content)
683 self._fd.write(content)
684
685 if 'content-range' in resp:
686 content_range = resp['content-range']
687 length = content_range.rsplit('/', 1)[1]
688 self._total_size = int(length)
jackac8df212015-02-17 12:16:19 -0800689 elif 'content-length' in resp:
jack77c63c92015-02-10 12:11:00 -0800690 self._total_size = int(resp['content-length'])
John Asmuth864311d2014-04-24 15:46:08 -0400691
Daniel44067782018-01-16 23:17:56 +0100692 if self._total_size is None or self._progress == self._total_size:
John Asmuth864311d2014-04-24 15:46:08 -0400693 self._done = True
694 return MediaDownloadProgress(self._progress, self._total_size), self._done
695 else:
696 raise HttpError(resp, content, uri=self._uri)
697
698
699class _StreamSlice(object):
700 """Truncated stream.
701
702 Takes a stream and presents a stream that is a slice of the original stream.
703 This is used when uploading media in chunks. In later versions of Python a
704 stream can be passed to httplib in place of the string of data to send. The
705 problem is that httplib just blindly reads to the end of the stream. This
706 wrapper presents a virtual stream that only reads to the end of the chunk.
707 """
708
709 def __init__(self, stream, begin, chunksize):
710 """Constructor.
711
712 Args:
713 stream: (io.Base, file object), the stream to wrap.
714 begin: int, the seek position the chunk begins at.
715 chunksize: int, the size of the chunk.
716 """
717 self._stream = stream
718 self._begin = begin
719 self._chunksize = chunksize
720 self._stream.seek(begin)
721
722 def read(self, n=-1):
723 """Read n bytes.
724
725 Args:
726 n, int, the number of bytes to read.
727
728 Returns:
729 A string of length 'n', or less if EOF is reached.
730 """
731 # The data left available to read sits in [cur, end)
732 cur = self._stream.tell()
733 end = self._begin + self._chunksize
734 if n == -1 or cur + n > end:
735 n = end - cur
736 return self._stream.read(n)
737
738
739class HttpRequest(object):
740 """Encapsulates a single HTTP request."""
741
742 @util.positional(4)
743 def __init__(self, http, postproc, uri,
744 method='GET',
745 body=None,
746 headers=None,
747 methodId=None,
748 resumable=None):
749 """Constructor for an HttpRequest.
750
751 Args:
752 http: httplib2.Http, the transport object to use to make a request
753 postproc: callable, called on the HTTP response and content to transform
754 it into a data object before returning, or raising an exception
755 on an error.
756 uri: string, the absolute URI to send the request to
757 method: string, the HTTP method to use
758 body: string, the request body of the HTTP request,
759 headers: dict, the HTTP request headers
760 methodId: string, a unique identifier for the API method being called.
761 resumable: MediaUpload, None if this is not a resumbale request.
762 """
763 self.uri = uri
764 self.method = method
765 self.body = body
766 self.headers = headers or {}
767 self.methodId = methodId
768 self.http = http
769 self.postproc = postproc
770 self.resumable = resumable
771 self.response_callbacks = []
772 self._in_error_state = False
773
John Asmuth864311d2014-04-24 15:46:08 -0400774 # The size of the non-media part of the request.
775 self.body_size = len(self.body or '')
776
777 # The resumable URI to send chunks to.
778 self.resumable_uri = None
779
780 # The bytes that have been uploaded.
781 self.resumable_progress = 0
782
783 # Stubs for testing.
784 self._rand = random.random
785 self._sleep = time.sleep
786
787 @util.positional(1)
788 def execute(self, http=None, num_retries=0):
789 """Execute the request.
790
791 Args:
792 http: httplib2.Http, an http object to be used in place of the
793 one the HttpRequest request object was constructed with.
Zhihao Yuancc6d3982016-07-27 11:40:45 -0500794 num_retries: Integer, number of times to retry with randomized
John Asmuth864311d2014-04-24 15:46:08 -0400795 exponential backoff. If all retries fail, the raised HttpError
796 represents the last request. If zero (default), we attempt the
797 request only once.
798
799 Returns:
800 A deserialized object model of the response body as determined
801 by the postproc.
802
803 Raises:
804 googleapiclient.errors.HttpError if the response was not a 2xx.
805 httplib2.HttpLib2Error if a transport error has occured.
806 """
807 if http is None:
808 http = self.http
809
810 if self.resumable:
811 body = None
812 while body is None:
813 _, body = self.next_chunk(http=http, num_retries=num_retries)
814 return body
815
816 # Non-resumable case.
817
818 if 'content-length' not in self.headers:
819 self.headers['content-length'] = str(self.body_size)
820 # If the request URI is too long then turn it into a POST request.
Thomas Coffee20af04d2017-02-10 15:24:44 -0800821 # Assume that a GET request never contains a request body.
John Asmuth864311d2014-04-24 15:46:08 -0400822 if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET':
823 self.method = 'POST'
824 self.headers['x-http-method-override'] = 'GET'
825 self.headers['content-type'] = 'application/x-www-form-urlencoded'
Pat Ferated5b61bd2015-03-03 16:04:11 -0800826 parsed = urlparse(self.uri)
827 self.uri = urlunparse(
John Asmuth864311d2014-04-24 15:46:08 -0400828 (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None,
829 None)
830 )
831 self.body = parsed.query
832 self.headers['content-length'] = str(len(self.body))
833
834 # Handle retries for server-side errors.
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100835 resp, content = _retry_request(
836 http, num_retries, 'request', self._sleep, self._rand, str(self.uri),
837 method=str(self.method), body=self.body, headers=self.headers)
John Asmuth864311d2014-04-24 15:46:08 -0400838
839 for callback in self.response_callbacks:
840 callback(resp)
841 if resp.status >= 300:
842 raise HttpError(resp, content, uri=self.uri)
843 return self.postproc(resp, content)
844
845 @util.positional(2)
846 def add_response_callback(self, cb):
847 """add_response_headers_callback
848
849 Args:
850 cb: Callback to be called on receiving the response headers, of signature:
851
852 def cb(resp):
853 # Where resp is an instance of httplib2.Response
854 """
855 self.response_callbacks.append(cb)
856
857 @util.positional(1)
858 def next_chunk(self, http=None, num_retries=0):
859 """Execute the next step of a resumable upload.
860
861 Can only be used if the method being executed supports media uploads and
862 the MediaUpload object passed in was flagged as using resumable upload.
863
864 Example:
865
866 media = MediaFileUpload('cow.png', mimetype='image/png',
867 chunksize=1000, resumable=True)
868 request = farm.animals().insert(
869 id='cow',
870 name='cow.png',
871 media_body=media)
872
873 response = None
874 while response is None:
875 status, response = request.next_chunk()
876 if status:
877 print "Upload %d%% complete." % int(status.progress() * 100)
878
879
880 Args:
881 http: httplib2.Http, an http object to be used in place of the
882 one the HttpRequest request object was constructed with.
Zhihao Yuancc6d3982016-07-27 11:40:45 -0500883 num_retries: Integer, number of times to retry with randomized
John Asmuth864311d2014-04-24 15:46:08 -0400884 exponential backoff. If all retries fail, the raised HttpError
885 represents the last request. If zero (default), we attempt the
886 request only once.
887
888 Returns:
889 (status, body): (ResumableMediaStatus, object)
890 The body will be None until the resumable media is fully uploaded.
891
892 Raises:
893 googleapiclient.errors.HttpError if the response was not a 2xx.
894 httplib2.HttpLib2Error if a transport error has occured.
895 """
896 if http is None:
897 http = self.http
898
899 if self.resumable.size() is None:
900 size = '*'
901 else:
902 size = str(self.resumable.size())
903
904 if self.resumable_uri is None:
905 start_headers = copy.copy(self.headers)
906 start_headers['X-Upload-Content-Type'] = self.resumable.mimetype()
907 if size != '*':
908 start_headers['X-Upload-Content-Length'] = size
909 start_headers['content-length'] = str(self.body_size)
910
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100911 resp, content = _retry_request(
912 http, num_retries, 'resumable URI request', self._sleep, self._rand,
913 self.uri, method=self.method, body=self.body, headers=start_headers)
John Asmuth864311d2014-04-24 15:46:08 -0400914
915 if resp.status == 200 and 'location' in resp:
916 self.resumable_uri = resp['location']
917 else:
918 raise ResumableUploadError(resp, content)
919 elif self._in_error_state:
920 # If we are in an error state then query the server for current state of
921 # the upload by sending an empty PUT and reading the 'range' header in
922 # the response.
923 headers = {
924 'Content-Range': 'bytes */%s' % size,
925 'content-length': '0'
926 }
927 resp, content = http.request(self.resumable_uri, 'PUT',
928 headers=headers)
929 status, body = self._process_response(resp, content)
930 if body:
931 # The upload was complete.
932 return (status, body)
933
e00Efafe8582015-10-10 18:19:37 +0200934 if self.resumable.has_stream():
John Asmuth864311d2014-04-24 15:46:08 -0400935 data = self.resumable.stream()
936 if self.resumable.chunksize() == -1:
937 data.seek(self.resumable_progress)
938 chunk_end = self.resumable.size() - self.resumable_progress - 1
939 else:
940 # Doing chunking with a stream, so wrap a slice of the stream.
941 data = _StreamSlice(data, self.resumable_progress,
942 self.resumable.chunksize())
943 chunk_end = min(
944 self.resumable_progress + self.resumable.chunksize() - 1,
945 self.resumable.size() - 1)
946 else:
947 data = self.resumable.getbytes(
948 self.resumable_progress, self.resumable.chunksize())
949
950 # A short read implies that we are at EOF, so finish the upload.
951 if len(data) < self.resumable.chunksize():
952 size = str(self.resumable_progress + len(data))
953
954 chunk_end = self.resumable_progress + len(data) - 1
955
956 headers = {
957 'Content-Range': 'bytes %d-%d/%s' % (
958 self.resumable_progress, chunk_end, size),
959 # Must set the content-length header here because httplib can't
960 # calculate the size when working with _StreamSlice.
961 'Content-Length': str(chunk_end - self.resumable_progress + 1)
962 }
963
INADA Naokie4ea1a92015-03-04 03:45:42 +0900964 for retry_num in range(num_retries + 1):
John Asmuth864311d2014-04-24 15:46:08 -0400965 if retry_num > 0:
966 self._sleep(self._rand() * 2**retry_num)
Emmett Butler09699152016-02-08 14:26:00 -0800967 LOGGER.warning(
John Asmuth864311d2014-04-24 15:46:08 -0400968 'Retry #%d for media upload: %s %s, following status: %d'
969 % (retry_num, self.method, self.uri, resp.status))
970
971 try:
972 resp, content = http.request(self.resumable_uri, method='PUT',
973 body=data,
974 headers=headers)
975 except:
976 self._in_error_state = True
977 raise
Zhihao Yuancc6d3982016-07-27 11:40:45 -0500978 if not _should_retry_response(resp.status, content):
John Asmuth864311d2014-04-24 15:46:08 -0400979 break
980
981 return self._process_response(resp, content)
982
983 def _process_response(self, resp, content):
984 """Process the response from a single chunk upload.
985
986 Args:
987 resp: httplib2.Response, the response object.
988 content: string, the content of the response.
989
990 Returns:
991 (status, body): (ResumableMediaStatus, object)
992 The body will be None until the resumable media is fully uploaded.
993
994 Raises:
995 googleapiclient.errors.HttpError if the response was not a 2xx or a 308.
996 """
997 if resp.status in [200, 201]:
998 self._in_error_state = False
999 return None, self.postproc(resp, content)
1000 elif resp.status == 308:
1001 self._in_error_state = False
1002 # A "308 Resume Incomplete" indicates we are not done.
Matt Carroll94a53942016-12-20 13:56:43 -08001003 try:
1004 self.resumable_progress = int(resp['range'].split('-')[1]) + 1
1005 except KeyError:
1006 # If resp doesn't contain range header, resumable progress is 0
1007 self.resumable_progress = 0
John Asmuth864311d2014-04-24 15:46:08 -04001008 if 'location' in resp:
1009 self.resumable_uri = resp['location']
1010 else:
1011 self._in_error_state = True
1012 raise HttpError(resp, content, uri=self.uri)
1013
1014 return (MediaUploadProgress(self.resumable_progress, self.resumable.size()),
1015 None)
1016
1017 def to_json(self):
1018 """Returns a JSON representation of the HttpRequest."""
1019 d = copy.copy(self.__dict__)
1020 if d['resumable'] is not None:
1021 d['resumable'] = self.resumable.to_json()
1022 del d['http']
1023 del d['postproc']
1024 del d['_sleep']
1025 del d['_rand']
1026
Craig Citro6ae34d72014-08-18 23:10:09 -07001027 return json.dumps(d)
John Asmuth864311d2014-04-24 15:46:08 -04001028
1029 @staticmethod
1030 def from_json(s, http, postproc):
1031 """Returns an HttpRequest populated with info from a JSON object."""
Craig Citro6ae34d72014-08-18 23:10:09 -07001032 d = json.loads(s)
John Asmuth864311d2014-04-24 15:46:08 -04001033 if d['resumable'] is not None:
1034 d['resumable'] = MediaUpload.new_from_json(d['resumable'])
1035 return HttpRequest(
1036 http,
1037 postproc,
1038 uri=d['uri'],
1039 method=d['method'],
1040 body=d['body'],
1041 headers=d['headers'],
1042 methodId=d['methodId'],
1043 resumable=d['resumable'])
1044
1045
1046class BatchHttpRequest(object):
1047 """Batches multiple HttpRequest objects into a single HTTP request.
1048
1049 Example:
1050 from googleapiclient.http import BatchHttpRequest
1051
1052 def list_animals(request_id, response, exception):
1053 \"\"\"Do something with the animals list response.\"\"\"
1054 if exception is not None:
1055 # Do something with the exception.
1056 pass
1057 else:
1058 # Do something with the response.
1059 pass
1060
1061 def list_farmers(request_id, response, exception):
1062 \"\"\"Do something with the farmers list response.\"\"\"
1063 if exception is not None:
1064 # Do something with the exception.
1065 pass
1066 else:
1067 # Do something with the response.
1068 pass
1069
1070 service = build('farm', 'v2')
1071
1072 batch = BatchHttpRequest()
1073
1074 batch.add(service.animals().list(), list_animals)
1075 batch.add(service.farmers().list(), list_farmers)
1076 batch.execute(http=http)
1077 """
1078
1079 @util.positional(1)
1080 def __init__(self, callback=None, batch_uri=None):
1081 """Constructor for a BatchHttpRequest.
1082
1083 Args:
1084 callback: callable, A callback to be called for each response, of the
1085 form callback(id, response, exception). The first parameter is the
1086 request id, and the second is the deserialized response object. The
1087 third is an googleapiclient.errors.HttpError exception object if an HTTP error
1088 occurred while processing the request, or None if no error occurred.
1089 batch_uri: string, URI to send batch requests to.
1090 """
1091 if batch_uri is None:
Jon Wayne Parrottbae748a2018-03-28 10:21:12 -07001092 batch_uri = _LEGACY_BATCH_URI
1093
1094 if batch_uri == _LEGACY_BATCH_URI:
1095 LOGGER.warn(
1096 "You have constructed a BatchHttpRequest using the legacy batch "
1097 "endpoint %s. This endpoint will be turned down on March 25, 2019. "
1098 "Please provide the API-specific endpoint or use "
1099 "service.new_batch_http_request(). For more details see "
1100 "https://developers.googleblog.com/2018/03/discontinuing-support-for-json-rpc-and.html"
1101 "and https://developers.google.com/api-client-library/python/guide/batch.",
1102 _LEGACY_BATCH_URI)
John Asmuth864311d2014-04-24 15:46:08 -04001103 self._batch_uri = batch_uri
1104
1105 # Global callback to be called for each individual response in the batch.
1106 self._callback = callback
1107
1108 # A map from id to request.
1109 self._requests = {}
1110
1111 # A map from id to callback.
1112 self._callbacks = {}
1113
1114 # List of request ids, in the order in which they were added.
1115 self._order = []
1116
1117 # The last auto generated id.
1118 self._last_auto_id = 0
1119
1120 # Unique ID on which to base the Content-ID headers.
1121 self._base_id = None
1122
1123 # A map from request id to (httplib2.Response, content) response pairs
1124 self._responses = {}
1125
1126 # A map of id(Credentials) that have been refreshed.
1127 self._refreshed_credentials = {}
1128
1129 def _refresh_and_apply_credentials(self, request, http):
1130 """Refresh the credentials and apply to the request.
1131
1132 Args:
1133 request: HttpRequest, the request.
1134 http: httplib2.Http, the global http object for the batch.
1135 """
1136 # For the credentials to refresh, but only once per refresh_token
1137 # If there is no http per the request then refresh the http passed in
1138 # via execute()
1139 creds = None
Jon Wayne Parrottd3a5cf42017-06-19 17:55:04 -07001140 request_credentials = False
1141
1142 if request.http is not None:
1143 creds = _auth.get_credentials_from_http(request.http)
1144 request_credentials = True
1145
1146 if creds is None and http is not None:
1147 creds = _auth.get_credentials_from_http(http)
1148
John Asmuth864311d2014-04-24 15:46:08 -04001149 if creds is not None:
1150 if id(creds) not in self._refreshed_credentials:
Jon Wayne Parrottd3a5cf42017-06-19 17:55:04 -07001151 _auth.refresh_credentials(creds)
John Asmuth864311d2014-04-24 15:46:08 -04001152 self._refreshed_credentials[id(creds)] = 1
1153
1154 # Only apply the credentials if we are using the http object passed in,
1155 # otherwise apply() will get called during _serialize_request().
Jon Wayne Parrottd3a5cf42017-06-19 17:55:04 -07001156 if request.http is None or not request_credentials:
1157 _auth.apply_credentials(creds, request.headers)
1158
John Asmuth864311d2014-04-24 15:46:08 -04001159
1160 def _id_to_header(self, id_):
1161 """Convert an id to a Content-ID header value.
1162
1163 Args:
1164 id_: string, identifier of individual request.
1165
1166 Returns:
1167 A Content-ID header with the id_ encoded into it. A UUID is prepended to
1168 the value because Content-ID headers are supposed to be universally
1169 unique.
1170 """
1171 if self._base_id is None:
1172 self._base_id = uuid.uuid4()
1173
Chris McDonough3cf5e602018-07-18 16:18:38 -04001174 # NB: we intentionally leave whitespace between base/id and '+', so RFC2822
1175 # line folding works properly on Python 3; see
1176 # https://github.com/google/google-api-python-client/issues/164
1177 return '<%s + %s>' % (self._base_id, quote(id_))
John Asmuth864311d2014-04-24 15:46:08 -04001178
1179 def _header_to_id(self, header):
1180 """Convert a Content-ID header value to an id.
1181
1182 Presumes the Content-ID header conforms to the format that _id_to_header()
1183 returns.
1184
1185 Args:
1186 header: string, Content-ID header value.
1187
1188 Returns:
1189 The extracted id value.
1190
1191 Raises:
1192 BatchError if the header is not in the expected format.
1193 """
1194 if header[0] != '<' or header[-1] != '>':
1195 raise BatchError("Invalid value for Content-ID: %s" % header)
1196 if '+' not in header:
1197 raise BatchError("Invalid value for Content-ID: %s" % header)
Chris McDonough3cf5e602018-07-18 16:18:38 -04001198 base, id_ = header[1:-1].split(' + ', 1)
John Asmuth864311d2014-04-24 15:46:08 -04001199
Pat Ferated5b61bd2015-03-03 16:04:11 -08001200 return unquote(id_)
John Asmuth864311d2014-04-24 15:46:08 -04001201
1202 def _serialize_request(self, request):
1203 """Convert an HttpRequest object into a string.
1204
1205 Args:
1206 request: HttpRequest, the request to serialize.
1207
1208 Returns:
1209 The request as a string in application/http format.
1210 """
1211 # Construct status line
Pat Ferated5b61bd2015-03-03 16:04:11 -08001212 parsed = urlparse(request.uri)
1213 request_line = urlunparse(
Pat Feratec9abbbd2015-03-03 18:00:38 -08001214 ('', '', parsed.path, parsed.params, parsed.query, '')
John Asmuth864311d2014-04-24 15:46:08 -04001215 )
1216 status_line = request.method + ' ' + request_line + ' HTTP/1.1\n'
1217 major, minor = request.headers.get('content-type', 'application/json').split('/')
1218 msg = MIMENonMultipart(major, minor)
1219 headers = request.headers.copy()
1220
Jon Wayne Parrottd3a5cf42017-06-19 17:55:04 -07001221 if request.http is not None:
1222 credentials = _auth.get_credentials_from_http(request.http)
1223 if credentials is not None:
1224 _auth.apply_credentials(credentials, headers)
John Asmuth864311d2014-04-24 15:46:08 -04001225
1226 # MIMENonMultipart adds its own Content-Type header.
1227 if 'content-type' in headers:
1228 del headers['content-type']
1229
INADA Naokie4ea1a92015-03-04 03:45:42 +09001230 for key, value in six.iteritems(headers):
John Asmuth864311d2014-04-24 15:46:08 -04001231 msg[key] = value
1232 msg['Host'] = parsed.netloc
1233 msg.set_unixfrom(None)
1234
1235 if request.body is not None:
1236 msg.set_payload(request.body)
1237 msg['content-length'] = str(len(request.body))
1238
1239 # Serialize the mime message.
Pat Ferateed9affd2015-03-03 16:03:15 -08001240 fp = StringIO()
John Asmuth864311d2014-04-24 15:46:08 -04001241 # maxheaderlen=0 means don't line wrap headers.
1242 g = Generator(fp, maxheaderlen=0)
1243 g.flatten(msg, unixfrom=False)
1244 body = fp.getvalue()
1245
Pat Feratec9abbbd2015-03-03 18:00:38 -08001246 return status_line + body
John Asmuth864311d2014-04-24 15:46:08 -04001247
1248 def _deserialize_response(self, payload):
1249 """Convert string into httplib2 response and content.
1250
1251 Args:
1252 payload: string, headers and body as a string.
1253
1254 Returns:
1255 A pair (resp, content), such as would be returned from httplib2.request.
1256 """
1257 # Strip off the status line
1258 status_line, payload = payload.split('\n', 1)
1259 protocol, status, reason = status_line.split(' ', 2)
1260
1261 # Parse the rest of the response
1262 parser = FeedParser()
1263 parser.feed(payload)
1264 msg = parser.close()
1265 msg['status'] = status
1266
1267 # Create httplib2.Response from the parsed headers.
1268 resp = httplib2.Response(msg)
1269 resp.reason = reason
1270 resp.version = int(protocol.split('/', 1)[1].replace('.', ''))
1271
1272 content = payload.split('\r\n\r\n', 1)[1]
1273
1274 return resp, content
1275
1276 def _new_id(self):
1277 """Create a new id.
1278
1279 Auto incrementing number that avoids conflicts with ids already used.
1280
1281 Returns:
1282 string, a new unique id.
1283 """
1284 self._last_auto_id += 1
1285 while str(self._last_auto_id) in self._requests:
1286 self._last_auto_id += 1
1287 return str(self._last_auto_id)
1288
1289 @util.positional(2)
1290 def add(self, request, callback=None, request_id=None):
1291 """Add a new request.
1292
1293 Every callback added will be paired with a unique id, the request_id. That
1294 unique id will be passed back to the callback when the response comes back
1295 from the server. The default behavior is to have the library generate it's
1296 own unique id. If the caller passes in a request_id then they must ensure
1297 uniqueness for each request_id, and if they are not an exception is
cspeidelfbaf9d72018-05-10 12:50:12 -06001298 raised. Callers should either supply all request_ids or never supply a
John Asmuth864311d2014-04-24 15:46:08 -04001299 request id, to avoid such an error.
1300
1301 Args:
1302 request: HttpRequest, Request to add to the batch.
1303 callback: callable, A callback to be called for this response, of the
1304 form callback(id, response, exception). The first parameter is the
1305 request id, and the second is the deserialized response object. The
1306 third is an googleapiclient.errors.HttpError exception object if an HTTP error
1307 occurred while processing the request, or None if no errors occurred.
Chris McDonough3cf5e602018-07-18 16:18:38 -04001308 request_id: string, A unique id for the request. The id will be passed
1309 to the callback with the response.
John Asmuth864311d2014-04-24 15:46:08 -04001310
1311 Returns:
1312 None
1313
1314 Raises:
1315 BatchError if a media request is added to a batch.
1316 KeyError is the request_id is not unique.
1317 """
1318 if request_id is None:
1319 request_id = self._new_id()
1320 if request.resumable is not None:
1321 raise BatchError("Media requests cannot be used in a batch request.")
1322 if request_id in self._requests:
1323 raise KeyError("A request with this ID already exists: %s" % request_id)
1324 self._requests[request_id] = request
1325 self._callbacks[request_id] = callback
1326 self._order.append(request_id)
1327
1328 def _execute(self, http, order, requests):
1329 """Serialize batch request, send to server, process response.
1330
1331 Args:
1332 http: httplib2.Http, an http object to be used to make the request with.
1333 order: list, list of request ids in the order they were added to the
1334 batch.
1335 request: list, list of request objects to send.
1336
1337 Raises:
1338 httplib2.HttpLib2Error if a transport error has occured.
1339 googleapiclient.errors.BatchError if the response is the wrong format.
1340 """
1341 message = MIMEMultipart('mixed')
1342 # Message should not write out it's own headers.
1343 setattr(message, '_write_headers', lambda self: None)
1344
1345 # Add all the individual requests.
1346 for request_id in order:
1347 request = requests[request_id]
1348
1349 msg = MIMENonMultipart('application', 'http')
1350 msg['Content-Transfer-Encoding'] = 'binary'
1351 msg['Content-ID'] = self._id_to_header(request_id)
1352
1353 body = self._serialize_request(request)
1354 msg.set_payload(body)
1355 message.attach(msg)
1356
Craig Citro72389b72014-07-15 17:12:50 -07001357 # encode the body: note that we can't use `as_string`, because
1358 # it plays games with `From ` lines.
Pat Ferateed9affd2015-03-03 16:03:15 -08001359 fp = StringIO()
Craig Citro72389b72014-07-15 17:12:50 -07001360 g = Generator(fp, mangle_from_=False)
1361 g.flatten(message, unixfrom=False)
1362 body = fp.getvalue()
John Asmuth864311d2014-04-24 15:46:08 -04001363
1364 headers = {}
1365 headers['content-type'] = ('multipart/mixed; '
1366 'boundary="%s"') % message.get_boundary()
1367
1368 resp, content = http.request(self._batch_uri, method='POST', body=body,
1369 headers=headers)
1370
1371 if resp.status >= 300:
1372 raise HttpError(resp, content, uri=self._batch_uri)
1373
John Asmuth864311d2014-04-24 15:46:08 -04001374 # Prepend with a content-type header so FeedParser can handle it.
1375 header = 'content-type: %s\r\n\r\n' % resp['content-type']
INADA Naoki09157612015-03-25 01:51:03 +09001376 # PY3's FeedParser only accepts unicode. So we should decode content
1377 # here, and encode each payload again.
1378 if six.PY3:
1379 content = content.decode('utf-8')
John Asmuth864311d2014-04-24 15:46:08 -04001380 for_parser = header + content
1381
1382 parser = FeedParser()
1383 parser.feed(for_parser)
1384 mime_response = parser.close()
1385
1386 if not mime_response.is_multipart():
1387 raise BatchError("Response not in multipart/mixed format.", resp=resp,
1388 content=content)
1389
1390 for part in mime_response.get_payload():
1391 request_id = self._header_to_id(part['Content-ID'])
1392 response, content = self._deserialize_response(part.get_payload())
INADA Naoki09157612015-03-25 01:51:03 +09001393 # We encode content here to emulate normal http response.
1394 if isinstance(content, six.text_type):
1395 content = content.encode('utf-8')
John Asmuth864311d2014-04-24 15:46:08 -04001396 self._responses[request_id] = (response, content)
1397
1398 @util.positional(1)
1399 def execute(self, http=None):
1400 """Execute all the requests as a single batched HTTP request.
1401
1402 Args:
1403 http: httplib2.Http, an http object to be used in place of the one the
1404 HttpRequest request object was constructed with. If one isn't supplied
1405 then use a http object from the requests in this batch.
1406
1407 Returns:
1408 None
1409
1410 Raises:
1411 httplib2.HttpLib2Error if a transport error has occured.
1412 googleapiclient.errors.BatchError if the response is the wrong format.
1413 """
Mohamed Zenadi1b5350d2015-07-30 11:52:39 +02001414 # If we have no requests return
1415 if len(self._order) == 0:
1416 return None
John Asmuth864311d2014-04-24 15:46:08 -04001417
1418 # If http is not supplied use the first valid one given in the requests.
1419 if http is None:
1420 for request_id in self._order:
1421 request = self._requests[request_id]
1422 if request is not None:
1423 http = request.http
1424 break
1425
1426 if http is None:
1427 raise ValueError("Missing a valid http object.")
1428
Gabriel Garcia23174be2016-05-25 17:28:07 +02001429 # Special case for OAuth2Credentials-style objects which have not yet been
1430 # refreshed with an initial access_token.
Jon Wayne Parrottd3a5cf42017-06-19 17:55:04 -07001431 creds = _auth.get_credentials_from_http(http)
1432 if creds is not None:
1433 if not _auth.is_valid(creds):
Gabriel Garcia23174be2016-05-25 17:28:07 +02001434 LOGGER.info('Attempting refresh to obtain initial access_token')
Jon Wayne Parrottd3a5cf42017-06-19 17:55:04 -07001435 _auth.refresh_credentials(creds)
Gabriel Garcia23174be2016-05-25 17:28:07 +02001436
John Asmuth864311d2014-04-24 15:46:08 -04001437 self._execute(http, self._order, self._requests)
1438
1439 # Loop over all the requests and check for 401s. For each 401 request the
1440 # credentials should be refreshed and then sent again in a separate batch.
1441 redo_requests = {}
1442 redo_order = []
1443
1444 for request_id in self._order:
1445 resp, content = self._responses[request_id]
1446 if resp['status'] == '401':
1447 redo_order.append(request_id)
1448 request = self._requests[request_id]
1449 self._refresh_and_apply_credentials(request, http)
1450 redo_requests[request_id] = request
1451
1452 if redo_requests:
1453 self._execute(http, redo_order, redo_requests)
1454
1455 # Now process all callbacks that are erroring, and raise an exception for
1456 # ones that return a non-2xx response? Or add extra parameter to callback
1457 # that contains an HttpError?
1458
1459 for request_id in self._order:
1460 resp, content = self._responses[request_id]
1461
1462 request = self._requests[request_id]
1463 callback = self._callbacks[request_id]
1464
1465 response = None
1466 exception = None
1467 try:
1468 if resp.status >= 300:
1469 raise HttpError(resp, content, uri=request.uri)
1470 response = request.postproc(resp, content)
INADA Naokic1505df2014-08-20 15:19:53 +09001471 except HttpError as e:
John Asmuth864311d2014-04-24 15:46:08 -04001472 exception = e
1473
1474 if callback is not None:
1475 callback(request_id, response, exception)
1476 if self._callback is not None:
1477 self._callback(request_id, response, exception)
1478
1479
1480class HttpRequestMock(object):
1481 """Mock of HttpRequest.
1482
1483 Do not construct directly, instead use RequestMockBuilder.
1484 """
1485
1486 def __init__(self, resp, content, postproc):
1487 """Constructor for HttpRequestMock
1488
1489 Args:
1490 resp: httplib2.Response, the response to emulate coming from the request
1491 content: string, the response body
1492 postproc: callable, the post processing function usually supplied by
1493 the model class. See model.JsonModel.response() as an example.
1494 """
1495 self.resp = resp
1496 self.content = content
1497 self.postproc = postproc
1498 if resp is None:
1499 self.resp = httplib2.Response({'status': 200, 'reason': 'OK'})
1500 if 'reason' in self.resp:
1501 self.resp.reason = self.resp['reason']
1502
1503 def execute(self, http=None):
1504 """Execute the request.
1505
1506 Same behavior as HttpRequest.execute(), but the response is
1507 mocked and not really from an HTTP request/response.
1508 """
1509 return self.postproc(self.resp, self.content)
1510
1511
1512class RequestMockBuilder(object):
1513 """A simple mock of HttpRequest
1514
1515 Pass in a dictionary to the constructor that maps request methodIds to
1516 tuples of (httplib2.Response, content, opt_expected_body) that should be
1517 returned when that method is called. None may also be passed in for the
1518 httplib2.Response, in which case a 200 OK response will be generated.
1519 If an opt_expected_body (str or dict) is provided, it will be compared to
1520 the body and UnexpectedBodyError will be raised on inequality.
1521
1522 Example:
1523 response = '{"data": {"id": "tag:google.c...'
1524 requestBuilder = RequestMockBuilder(
1525 {
1526 'plus.activities.get': (None, response),
1527 }
1528 )
1529 googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder)
1530
1531 Methods that you do not supply a response for will return a
1532 200 OK with an empty string as the response content or raise an excpetion
1533 if check_unexpected is set to True. The methodId is taken from the rpcName
1534 in the discovery document.
1535
1536 For more details see the project wiki.
1537 """
1538
1539 def __init__(self, responses, check_unexpected=False):
1540 """Constructor for RequestMockBuilder
1541
1542 The constructed object should be a callable object
1543 that can replace the class HttpResponse.
1544
1545 responses - A dictionary that maps methodIds into tuples
1546 of (httplib2.Response, content). The methodId
1547 comes from the 'rpcName' field in the discovery
1548 document.
1549 check_unexpected - A boolean setting whether or not UnexpectedMethodError
1550 should be raised on unsupplied method.
1551 """
1552 self.responses = responses
1553 self.check_unexpected = check_unexpected
1554
1555 def __call__(self, http, postproc, uri, method='GET', body=None,
1556 headers=None, methodId=None, resumable=None):
1557 """Implements the callable interface that discovery.build() expects
1558 of requestBuilder, which is to build an object compatible with
1559 HttpRequest.execute(). See that method for the description of the
1560 parameters and the expected response.
1561 """
1562 if methodId in self.responses:
1563 response = self.responses[methodId]
1564 resp, content = response[:2]
1565 if len(response) > 2:
1566 # Test the body against the supplied expected_body.
1567 expected_body = response[2]
1568 if bool(expected_body) != bool(body):
1569 # Not expecting a body and provided one
1570 # or expecting a body and not provided one.
1571 raise UnexpectedBodyError(expected_body, body)
1572 if isinstance(expected_body, str):
Craig Citro6ae34d72014-08-18 23:10:09 -07001573 expected_body = json.loads(expected_body)
1574 body = json.loads(body)
John Asmuth864311d2014-04-24 15:46:08 -04001575 if body != expected_body:
1576 raise UnexpectedBodyError(expected_body, body)
1577 return HttpRequestMock(resp, content, postproc)
1578 elif self.check_unexpected:
1579 raise UnexpectedMethodError(methodId=methodId)
1580 else:
1581 model = JsonModel(False)
1582 return HttpRequestMock(None, '{}', model.response)
1583
1584
1585class HttpMock(object):
1586 """Mock of httplib2.Http"""
1587
1588 def __init__(self, filename=None, headers=None):
1589 """
1590 Args:
1591 filename: string, absolute filename to read response from
1592 headers: dict, header to return with response
1593 """
1594 if headers is None:
Craig Gurnik8e55b762015-01-20 15:00:10 -05001595 headers = {'status': '200'}
John Asmuth864311d2014-04-24 15:46:08 -04001596 if filename:
Alan Briolat26b01002015-08-14 00:13:57 +01001597 f = open(filename, 'rb')
John Asmuth864311d2014-04-24 15:46:08 -04001598 self.data = f.read()
1599 f.close()
1600 else:
1601 self.data = None
1602 self.response_headers = headers
1603 self.headers = None
1604 self.uri = None
1605 self.method = None
1606 self.body = None
1607 self.headers = None
1608
1609
1610 def request(self, uri,
1611 method='GET',
1612 body=None,
1613 headers=None,
1614 redirections=1,
1615 connection_type=None):
1616 self.uri = uri
1617 self.method = method
1618 self.body = body
1619 self.headers = headers
1620 return httplib2.Response(self.response_headers), self.data
1621
1622
1623class HttpMockSequence(object):
1624 """Mock of httplib2.Http
1625
1626 Mocks a sequence of calls to request returning different responses for each
1627 call. Create an instance initialized with the desired response headers
1628 and content and then use as if an httplib2.Http instance.
1629
1630 http = HttpMockSequence([
1631 ({'status': '401'}, ''),
1632 ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'),
1633 ({'status': '200'}, 'echo_request_headers'),
1634 ])
1635 resp, content = http.request("http://examples.com")
1636
1637 There are special values you can pass in for content to trigger
1638 behavours that are helpful in testing.
1639
1640 'echo_request_headers' means return the request headers in the response body
1641 'echo_request_headers_as_json' means return the request headers in
1642 the response body
1643 'echo_request_body' means return the request body in the response body
1644 'echo_request_uri' means return the request uri in the response body
1645 """
1646
1647 def __init__(self, iterable):
1648 """
1649 Args:
1650 iterable: iterable, a sequence of pairs of (headers, body)
1651 """
1652 self._iterable = iterable
1653 self.follow_redirects = True
1654
1655 def request(self, uri,
1656 method='GET',
1657 body=None,
1658 headers=None,
1659 redirections=1,
1660 connection_type=None):
1661 resp, content = self._iterable.pop(0)
1662 if content == 'echo_request_headers':
1663 content = headers
1664 elif content == 'echo_request_headers_as_json':
Craig Citro6ae34d72014-08-18 23:10:09 -07001665 content = json.dumps(headers)
John Asmuth864311d2014-04-24 15:46:08 -04001666 elif content == 'echo_request_body':
1667 if hasattr(body, 'read'):
1668 content = body.read()
1669 else:
1670 content = body
1671 elif content == 'echo_request_uri':
1672 content = uri
INADA Naoki09157612015-03-25 01:51:03 +09001673 if isinstance(content, six.text_type):
1674 content = content.encode('utf-8')
John Asmuth864311d2014-04-24 15:46:08 -04001675 return httplib2.Response(resp), content
1676
1677
1678def set_user_agent(http, user_agent):
1679 """Set the user-agent on every request.
1680
1681 Args:
1682 http - An instance of httplib2.Http
1683 or something that acts like it.
1684 user_agent: string, the value for the user-agent header.
1685
1686 Returns:
1687 A modified instance of http that was passed in.
1688
1689 Example:
1690
1691 h = httplib2.Http()
1692 h = set_user_agent(h, "my-app-name/6.0")
1693
1694 Most of the time the user-agent will be set doing auth, this is for the rare
1695 cases where you are accessing an unauthenticated endpoint.
1696 """
1697 request_orig = http.request
1698
1699 # The closure that will replace 'httplib2.Http.request'.
1700 def new_request(uri, method='GET', body=None, headers=None,
1701 redirections=httplib2.DEFAULT_MAX_REDIRECTS,
1702 connection_type=None):
1703 """Modify the request headers to add the user-agent."""
1704 if headers is None:
1705 headers = {}
1706 if 'user-agent' in headers:
1707 headers['user-agent'] = user_agent + ' ' + headers['user-agent']
1708 else:
1709 headers['user-agent'] = user_agent
1710 resp, content = request_orig(uri, method, body, headers,
1711 redirections, connection_type)
1712 return resp, content
1713
1714 http.request = new_request
1715 return http
1716
1717
1718def tunnel_patch(http):
1719 """Tunnel PATCH requests over POST.
1720 Args:
1721 http - An instance of httplib2.Http
1722 or something that acts like it.
1723
1724 Returns:
1725 A modified instance of http that was passed in.
1726
1727 Example:
1728
1729 h = httplib2.Http()
1730 h = tunnel_patch(h, "my-app-name/6.0")
1731
1732 Useful if you are running on a platform that doesn't support PATCH.
1733 Apply this last if you are using OAuth 1.0, as changing the method
1734 will result in a different signature.
1735 """
1736 request_orig = http.request
1737
1738 # The closure that will replace 'httplib2.Http.request'.
1739 def new_request(uri, method='GET', body=None, headers=None,
1740 redirections=httplib2.DEFAULT_MAX_REDIRECTS,
1741 connection_type=None):
1742 """Modify the request headers to add the user-agent."""
1743 if headers is None:
1744 headers = {}
1745 if method == 'PATCH':
1746 if 'oauth_token' in headers.get('authorization', ''):
Emmett Butler09699152016-02-08 14:26:00 -08001747 LOGGER.warning(
John Asmuth864311d2014-04-24 15:46:08 -04001748 'OAuth 1.0 request made with Credentials after tunnel_patch.')
1749 headers['x-http-method-override'] = "PATCH"
1750 method = 'POST'
1751 resp, content = request_orig(uri, method, body, headers,
1752 redirections, connection_type)
1753 return resp, content
1754
1755 http.request = new_request
1756 return http
Igor Maravić22435292017-01-19 22:28:22 +01001757
1758
1759def build_http():
1760 """Builds httplib2.Http object
1761
1762 Returns:
1763 A httplib2.Http object, which is used to make http requests, and which has timeout set by default.
1764 To override default timeout call
1765
1766 socket.setdefaulttimeout(timeout_in_sec)
1767
1768 before interacting with this method.
1769 """
1770 if socket.getdefaulttimeout() is not None:
1771 http_timeout = socket.getdefaulttimeout()
1772 else:
1773 http_timeout = DEFAULT_HTTP_TIMEOUT_SEC
1774 return httplib2.Http(timeout=http_timeout)