Package apiclient :: Module http
[hide private]
[frames] | no frames]

Source Code for Module apiclient.http

   1  # Copyright (C) 2012 Google Inc. 
   2  # 
   3  # Licensed under the Apache License, Version 2.0 (the "License"); 
   4  # you may not use this file except in compliance with the License. 
   5  # You may obtain a copy of the License at 
   6  # 
   7  #      http://www.apache.org/licenses/LICENSE-2.0 
   8  # 
   9  # Unless required by applicable law or agreed to in writing, software 
  10  # distributed under the License is distributed on an "AS IS" BASIS, 
  11  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
  12  # See the License for the specific language governing permissions and 
  13  # limitations under the License. 
  14   
  15  """Classes to encapsulate a single HTTP request. 
  16   
  17  The classes implement a command pattern, with every 
  18  object supporting an execute() method that does the 
  19  actuall HTTP request. 
  20  """ 
  21   
  22  __author__ = 'jcgregorio@google.com (Joe Gregorio)' 
  23   
  24  import StringIO 
  25  import base64 
  26  import copy 
  27  import gzip 
  28  import httplib2 
  29  import mimeparse 
  30  import mimetypes 
  31  import os 
  32  import sys 
  33  import urllib 
  34  import urlparse 
  35  import uuid 
  36   
  37  from email.generator import Generator 
  38  from email.mime.multipart import MIMEMultipart 
  39  from email.mime.nonmultipart import MIMENonMultipart 
  40  from email.parser import FeedParser 
  41  from errors import BatchError 
  42  from errors import HttpError 
  43  from errors import InvalidChunkSizeError 
  44  from errors import ResumableUploadError 
  45  from errors import UnexpectedBodyError 
  46  from errors import UnexpectedMethodError 
  47  from model import JsonModel 
  48  from oauth2client import util 
  49  from oauth2client.anyjson import simplejson 
  50   
  51   
  52  DEFAULT_CHUNK_SIZE = 512*1024 
  53   
  54  MAX_URI_LENGTH = 2048 
55 56 57 -class MediaUploadProgress(object):
58 """Status of a resumable upload.""" 59
60 - def __init__(self, resumable_progress, total_size):
61 """Constructor. 62 63 Args: 64 resumable_progress: int, bytes sent so far. 65 total_size: int, total bytes in complete upload, or None if the total 66 upload size isn't known ahead of time. 67 """ 68 self.resumable_progress = resumable_progress 69 self.total_size = total_size
70
71 - def progress(self):
72 """Percent of upload completed, as a float. 73 74 Returns: 75 the percentage complete as a float, returning 0.0 if the total size of 76 the upload is unknown. 77 """ 78 if self.total_size is not None: 79 return float(self.resumable_progress) / float(self.total_size) 80 else: 81 return 0.0
82
83 84 -class MediaDownloadProgress(object):
85 """Status of a resumable download.""" 86
87 - def __init__(self, resumable_progress, total_size):
88 """Constructor. 89 90 Args: 91 resumable_progress: int, bytes received so far. 92 total_size: int, total bytes in complete download. 93 """ 94 self.resumable_progress = resumable_progress 95 self.total_size = total_size
96
97 - def progress(self):
98 """Percent of download completed, as a float. 99 100 Returns: 101 the percentage complete as a float, returning 0.0 if the total size of 102 the download is unknown. 103 """ 104 if self.total_size is not None: 105 return float(self.resumable_progress) / float(self.total_size) 106 else: 107 return 0.0
108
109 110 -class MediaUpload(object):
111 """Describes a media object to upload. 112 113 Base class that defines the interface of MediaUpload subclasses. 114 115 Note that subclasses of MediaUpload may allow you to control the chunksize 116 when uploading a media object. It is important to keep the size of the chunk 117 as large as possible to keep the upload efficient. Other factors may influence 118 the size of the chunk you use, particularly if you are working in an 119 environment where individual HTTP requests may have a hardcoded time limit, 120 such as under certain classes of requests under Google App Engine. 121 122 Streams are io.Base compatible objects that support seek(). Some MediaUpload 123 subclasses support using streams directly to upload data. Support for 124 streaming may be indicated by a MediaUpload sub-class and if appropriate for a 125 platform that stream will be used for uploading the media object. The support 126 for streaming is indicated by has_stream() returning True. The stream() method 127 should return an io.Base object that supports seek(). On platforms where the 128 underlying httplib module supports streaming, for example Python 2.6 and 129 later, the stream will be passed into the http library which will result in 130 less memory being used and possibly faster uploads. 131 132 If you need to upload media that can't be uploaded using any of the existing 133 MediaUpload sub-class then you can sub-class MediaUpload for your particular 134 needs. 135 """ 136
137 - def chunksize(self):
138 """Chunk size for resumable uploads. 139 140 Returns: 141 Chunk size in bytes. 142 """ 143 raise NotImplementedError()
144
145 - def mimetype(self):
146 """Mime type of the body. 147 148 Returns: 149 Mime type. 150 """ 151 return 'application/octet-stream'
152
153 - def size(self):
154 """Size of upload. 155 156 Returns: 157 Size of the body, or None of the size is unknown. 158 """ 159 return None
160
161 - def resumable(self):
162 """Whether this upload is resumable. 163 164 Returns: 165 True if resumable upload or False. 166 """ 167 return False
168
169 - def getbytes(self, begin, end):
170 """Get bytes from the media. 171 172 Args: 173 begin: int, offset from beginning of file. 174 length: int, number of bytes to read, starting at begin. 175 176 Returns: 177 A string of bytes read. May be shorter than length if EOF was reached 178 first. 179 """ 180 raise NotImplementedError()
181
182 - def has_stream(self):
183 """Does the underlying upload support a streaming interface. 184 185 Streaming means it is an io.IOBase subclass that supports seek, i.e. 186 seekable() returns True. 187 188 Returns: 189 True if the call to stream() will return an instance of a seekable io.Base 190 subclass. 191 """ 192 return False
193
194 - def stream(self):
195 """A stream interface to the data being uploaded. 196 197 Returns: 198 The returned value is an io.IOBase subclass that supports seek, i.e. 199 seekable() returns True. 200 """ 201 raise NotImplementedError()
202 203 @util.positional(1)
204 - def _to_json(self, strip=None):
205 """Utility function for creating a JSON representation of a MediaUpload. 206 207 Args: 208 strip: array, An array of names of members to not include in the JSON. 209 210 Returns: 211 string, a JSON representation of this instance, suitable to pass to 212 from_json(). 213 """ 214 t = type(self) 215 d = copy.copy(self.__dict__) 216 if strip is not None: 217 for member in strip: 218 del d[member] 219 d['_class'] = t.__name__ 220 d['_module'] = t.__module__ 221 return simplejson.dumps(d)
222
223 - def to_json(self):
224 """Create a JSON representation of an instance of MediaUpload. 225 226 Returns: 227 string, a JSON representation of this instance, suitable to pass to 228 from_json(). 229 """ 230 return self._to_json()
231 232 @classmethod
233 - def new_from_json(cls, s):
234 """Utility class method to instantiate a MediaUpload subclass from a JSON 235 representation produced by to_json(). 236 237 Args: 238 s: string, JSON from to_json(). 239 240 Returns: 241 An instance of the subclass of MediaUpload that was serialized with 242 to_json(). 243 """ 244 data = simplejson.loads(s) 245 # Find and call the right classmethod from_json() to restore the object. 246 module = data['_module'] 247 m = __import__(module, fromlist=module.split('.')[:-1]) 248 kls = getattr(m, data['_class']) 249 from_json = getattr(kls, 'from_json') 250 return from_json(s)
251
252 253 -class MediaIoBaseUpload(MediaUpload):
254 """A MediaUpload for a io.Base objects. 255 256 Note that the Python file object is compatible with io.Base and can be used 257 with this class also. 258 259 fh = io.BytesIO('...Some data to upload...') 260 media = MediaIoBaseUpload(fh, mimetype='image/png', 261 chunksize=1024*1024, resumable=True) 262 farm.animals().insert( 263 id='cow', 264 name='cow.png', 265 media_body=media).execute() 266 267 Depending on the platform you are working on, you may pass -1 as the 268 chunksize, which indicates that the entire file should be uploaded in a single 269 request. If the underlying platform supports streams, such as Python 2.6 or 270 later, then this can be very efficient as it avoids multiple connections, and 271 also avoids loading the entire file into memory before sending it. Note that 272 Google App Engine has a 5MB limit on request size, so you should never set 273 your chunksize larger than 5MB, or to -1. 274 """ 275 276 @util.positional(3)
277 - def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE, 278 resumable=False):
279 """Constructor. 280 281 Args: 282 fd: io.Base or file object, The source of the bytes to upload. MUST be 283 opened in blocking mode, do not use streams opened in non-blocking mode. 284 The given stream must be seekable, that is, it must be able to call 285 seek() on fd. 286 mimetype: string, Mime-type of the file. 287 chunksize: int, File will be uploaded in chunks of this many bytes. Only 288 used if resumable=True. Pass in a value of -1 if the file is to be 289 uploaded as a single chunk. Note that Google App Engine has a 5MB limit 290 on request size, so you should never set your chunksize larger than 5MB, 291 or to -1. 292 resumable: bool, True if this is a resumable upload. False means upload 293 in a single request. 294 """ 295 super(MediaIoBaseUpload, self).__init__() 296 self._fd = fd 297 self._mimetype = mimetype 298 if not (chunksize == -1 or chunksize > 0): 299 raise InvalidChunkSizeError() 300 self._chunksize = chunksize 301 self._resumable = resumable 302 303 self._fd.seek(0, os.SEEK_END) 304 self._size = self._fd.tell()
305
306 - def chunksize(self):
307 """Chunk size for resumable uploads. 308 309 Returns: 310 Chunk size in bytes. 311 """ 312 return self._chunksize
313
314 - def mimetype(self):
315 """Mime type of the body. 316 317 Returns: 318 Mime type. 319 """ 320 return self._mimetype
321
322 - def size(self):
323 """Size of upload. 324 325 Returns: 326 Size of the body, or None of the size is unknown. 327 """ 328 return self._size
329
330 - def resumable(self):
331 """Whether this upload is resumable. 332 333 Returns: 334 True if resumable upload or False. 335 """ 336 return self._resumable
337
338 - def getbytes(self, begin, length):
339 """Get bytes from the media. 340 341 Args: 342 begin: int, offset from beginning of file. 343 length: int, number of bytes to read, starting at begin. 344 345 Returns: 346 A string of bytes read. May be shorted than length if EOF was reached 347 first. 348 """ 349 self._fd.seek(begin) 350 return self._fd.read(length)
351
352 - def has_stream(self):
353 """Does the underlying upload support a streaming interface. 354 355 Streaming means it is an io.IOBase subclass that supports seek, i.e. 356 seekable() returns True. 357 358 Returns: 359 True if the call to stream() will return an instance of a seekable io.Base 360 subclass. 361 """ 362 return True
363
364 - def stream(self):
365 """A stream interface to the data being uploaded. 366 367 Returns: 368 The returned value is an io.IOBase subclass that supports seek, i.e. 369 seekable() returns True. 370 """ 371 return self._fd
372
373 - def to_json(self):
374 """This upload type is not serializable.""" 375 raise NotImplementedError('MediaIoBaseUpload is not serializable.')
376
377 378 -class MediaFileUpload(MediaIoBaseUpload):
379 """A MediaUpload for a file. 380 381 Construct a MediaFileUpload and pass as the media_body parameter of the 382 method. For example, if we had a service that allowed uploading images: 383 384 385 media = MediaFileUpload('cow.png', mimetype='image/png', 386 chunksize=1024*1024, resumable=True) 387 farm.animals().insert( 388 id='cow', 389 name='cow.png', 390 media_body=media).execute() 391 392 Depending on the platform you are working on, you may pass -1 as the 393 chunksize, which indicates that the entire file should be uploaded in a single 394 request. If the underlying platform supports streams, such as Python 2.6 or 395 later, then this can be very efficient as it avoids multiple connections, and 396 also avoids loading the entire file into memory before sending it. Note that 397 Google App Engine has a 5MB limit on request size, so you should never set 398 your chunksize larger than 5MB, or to -1. 399 """ 400 401 @util.positional(2)
402 - def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE, 403 resumable=False):
404 """Constructor. 405 406 Args: 407 filename: string, Name of the file. 408 mimetype: string, Mime-type of the file. If None then a mime-type will be 409 guessed from the file extension. 410 chunksize: int, File will be uploaded in chunks of this many bytes. Only 411 used if resumable=True. Pass in a value of -1 if the file is to be 412 uploaded in a single chunk. Note that Google App Engine has a 5MB limit 413 on request size, so you should never set your chunksize larger than 5MB, 414 or to -1. 415 resumable: bool, True if this is a resumable upload. False means upload 416 in a single request. 417 """ 418 self._filename = filename 419 fd = open(self._filename, 'rb') 420 if mimetype is None: 421 (mimetype, encoding) = mimetypes.guess_type(filename) 422 super(MediaFileUpload, self).__init__(fd, mimetype, chunksize=chunksize, 423 resumable=resumable)
424
425 - def to_json(self):
426 """Creating a JSON representation of an instance of MediaFileUpload. 427 428 Returns: 429 string, a JSON representation of this instance, suitable to pass to 430 from_json(). 431 """ 432 return self._to_json(strip=['_fd'])
433 434 @staticmethod
435 - def from_json(s):
436 d = simplejson.loads(s) 437 return MediaFileUpload(d['_filename'], mimetype=d['_mimetype'], 438 chunksize=d['_chunksize'], resumable=d['_resumable'])
439
440 441 -class MediaInMemoryUpload(MediaIoBaseUpload):
442 """MediaUpload for a chunk of bytes. 443 444 DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for 445 the stream. 446 """ 447 448 @util.positional(2)
449 - def __init__(self, body, mimetype='application/octet-stream', 450 chunksize=DEFAULT_CHUNK_SIZE, resumable=False):
451 """Create a new MediaInMemoryUpload. 452 453 DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for 454 the stream. 455 456 Args: 457 body: string, Bytes of body content. 458 mimetype: string, Mime-type of the file or default of 459 'application/octet-stream'. 460 chunksize: int, File will be uploaded in chunks of this many bytes. Only 461 used if resumable=True. 462 resumable: bool, True if this is a resumable upload. False means upload 463 in a single request. 464 """ 465 fd = StringIO.StringIO(body) 466 super(MediaInMemoryUpload, self).__init__(fd, mimetype, chunksize=chunksize, 467 resumable=resumable)
468
469 470 -class MediaIoBaseDownload(object):
471 """"Download media resources. 472 473 Note that the Python file object is compatible with io.Base and can be used 474 with this class also. 475 476 477 Example: 478 request = farms.animals().get_media(id='cow') 479 fh = io.FileIO('cow.png', mode='wb') 480 downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024) 481 482 done = False 483 while done is False: 484 status, done = downloader.next_chunk() 485 if status: 486 print "Download %d%%." % int(status.progress() * 100) 487 print "Download Complete!" 488 """ 489 490 @util.positional(3)
491 - def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE):
492 """Constructor. 493 494 Args: 495 fd: io.Base or file object, The stream in which to write the downloaded 496 bytes. 497 request: apiclient.http.HttpRequest, the media request to perform in 498 chunks. 499 chunksize: int, File will be downloaded in chunks of this many bytes. 500 """ 501 self._fd = fd 502 self._request = request 503 self._uri = request.uri 504 self._chunksize = chunksize 505 self._progress = 0 506 self._total_size = None 507 self._done = False
508
509 - def next_chunk(self):
510 """Get the next chunk of the download. 511 512 Returns: 513 (status, done): (MediaDownloadStatus, boolean) 514 The value of 'done' will be True when the media has been fully 515 downloaded. 516 517 Raises: 518 apiclient.errors.HttpError if the response was not a 2xx. 519 httplib2.HttpLib2Error if a transport error has occured. 520 """ 521 headers = { 522 'range': 'bytes=%d-%d' % ( 523 self._progress, self._progress + self._chunksize) 524 } 525 http = self._request.http 526 http.follow_redirects = False 527 528 resp, content = http.request(self._uri, headers=headers) 529 if resp.status in [301, 302, 303, 307, 308] and 'location' in resp: 530 self._uri = resp['location'] 531 resp, content = http.request(self._uri, headers=headers) 532 if resp.status in [200, 206]: 533 self._progress += len(content) 534 self._fd.write(content) 535 536 if 'content-range' in resp: 537 content_range = resp['content-range'] 538 length = content_range.rsplit('/', 1)[1] 539 self._total_size = int(length) 540 541 if self._progress == self._total_size: 542 self._done = True 543 return MediaDownloadProgress(self._progress, self._total_size), self._done 544 else: 545 raise HttpError(resp, content, uri=self._uri)
546
547 548 -class _StreamSlice(object):
549 """Truncated stream. 550 551 Takes a stream and presents a stream that is a slice of the original stream. 552 This is used when uploading media in chunks. In later versions of Python a 553 stream can be passed to httplib in place of the string of data to send. The 554 problem is that httplib just blindly reads to the end of the stream. This 555 wrapper presents a virtual stream that only reads to the end of the chunk. 556 """ 557
558 - def __init__(self, stream, begin, chunksize):
559 """Constructor. 560 561 Args: 562 stream: (io.Base, file object), the stream to wrap. 563 begin: int, the seek position the chunk begins at. 564 chunksize: int, the size of the chunk. 565 """ 566 self._stream = stream 567 self._begin = begin 568 self._chunksize = chunksize 569 self._stream.seek(begin)
570
571 - def read(self, n=-1):
572 """Read n bytes. 573 574 Args: 575 n, int, the number of bytes to read. 576 577 Returns: 578 A string of length 'n', or less if EOF is reached. 579 """ 580 # The data left available to read sits in [cur, end) 581 cur = self._stream.tell() 582 end = self._begin + self._chunksize 583 if n == -1 or cur + n > end: 584 n = end - cur 585 return self._stream.read(n)
586
587 588 -class HttpRequest(object):
589 """Encapsulates a single HTTP request.""" 590 591 @util.positional(4)
592 - def __init__(self, http, postproc, uri, 593 method='GET', 594 body=None, 595 headers=None, 596 methodId=None, 597 resumable=None):
598 """Constructor for an HttpRequest. 599 600 Args: 601 http: httplib2.Http, the transport object to use to make a request 602 postproc: callable, called on the HTTP response and content to transform 603 it into a data object before returning, or raising an exception 604 on an error. 605 uri: string, the absolute URI to send the request to 606 method: string, the HTTP method to use 607 body: string, the request body of the HTTP request, 608 headers: dict, the HTTP request headers 609 methodId: string, a unique identifier for the API method being called. 610 resumable: MediaUpload, None if this is not a resumbale request. 611 """ 612 self.uri = uri 613 self.method = method 614 self.body = body 615 self.headers = headers or {} 616 self.methodId = methodId 617 self.http = http 618 self.postproc = postproc 619 self.resumable = resumable 620 self.response_callbacks = [] 621 self._in_error_state = False 622 623 # Pull the multipart boundary out of the content-type header. 624 major, minor, params = mimeparse.parse_mime_type( 625 headers.get('content-type', 'application/json')) 626 627 # The size of the non-media part of the request. 628 self.body_size = len(self.body or '') 629 630 # The resumable URI to send chunks to. 631 self.resumable_uri = None 632 633 # The bytes that have been uploaded. 634 self.resumable_progress = 0
635 636 @util.positional(1)
637 - def execute(self, http=None):
638 """Execute the request. 639 640 Args: 641 http: httplib2.Http, an http object to be used in place of the 642 one the HttpRequest request object was constructed with. 643 644 Returns: 645 A deserialized object model of the response body as determined 646 by the postproc. 647 648 Raises: 649 apiclient.errors.HttpError if the response was not a 2xx. 650 httplib2.HttpLib2Error if a transport error has occured. 651 """ 652 if http is None: 653 http = self.http 654 if self.resumable: 655 body = None 656 while body is None: 657 _, body = self.next_chunk(http=http) 658 return body 659 else: 660 if 'content-length' not in self.headers: 661 self.headers['content-length'] = str(self.body_size) 662 # If the request URI is too long then turn it into a POST request. 663 if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET': 664 self.method = 'POST' 665 self.headers['x-http-method-override'] = 'GET' 666 self.headers['content-type'] = 'application/x-www-form-urlencoded' 667 parsed = urlparse.urlparse(self.uri) 668 self.uri = urlparse.urlunparse( 669 (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None, 670 None) 671 ) 672 self.body = parsed.query 673 self.headers['content-length'] = str(len(self.body)) 674 675 resp, content = http.request(str(self.uri), method=str(self.method), 676 body=self.body, headers=self.headers) 677 for callback in self.response_callbacks: 678 callback(resp) 679 if resp.status >= 300: 680 raise HttpError(resp, content, uri=self.uri) 681 return self.postproc(resp, content)
682 683 @util.positional(2)
684 - def add_response_callback(self, cb):
685 """add_response_headers_callback 686 687 Args: 688 cb: Callback to be called on receiving the response headers, of signature: 689 690 def cb(resp): 691 # Where resp is an instance of httplib2.Response 692 """ 693 self.response_callbacks.append(cb)
694 695 @util.positional(1)
696 - def next_chunk(self, http=None):
697 """Execute the next step of a resumable upload. 698 699 Can only be used if the method being executed supports media uploads and 700 the MediaUpload object passed in was flagged as using resumable upload. 701 702 Example: 703 704 media = MediaFileUpload('cow.png', mimetype='image/png', 705 chunksize=1000, resumable=True) 706 request = farm.animals().insert( 707 id='cow', 708 name='cow.png', 709 media_body=media) 710 711 response = None 712 while response is None: 713 status, response = request.next_chunk() 714 if status: 715 print "Upload %d%% complete." % int(status.progress() * 100) 716 717 718 Returns: 719 (status, body): (ResumableMediaStatus, object) 720 The body will be None until the resumable media is fully uploaded. 721 722 Raises: 723 apiclient.errors.HttpError if the response was not a 2xx. 724 httplib2.HttpLib2Error if a transport error has occured. 725 """ 726 if http is None: 727 http = self.http 728 729 if self.resumable.size() is None: 730 size = '*' 731 else: 732 size = str(self.resumable.size()) 733 734 if self.resumable_uri is None: 735 start_headers = copy.copy(self.headers) 736 start_headers['X-Upload-Content-Type'] = self.resumable.mimetype() 737 if size != '*': 738 start_headers['X-Upload-Content-Length'] = size 739 start_headers['content-length'] = str(self.body_size) 740 741 resp, content = http.request(self.uri, self.method, 742 body=self.body, 743 headers=start_headers) 744 if resp.status == 200 and 'location' in resp: 745 self.resumable_uri = resp['location'] 746 else: 747 raise ResumableUploadError(resp, content) 748 elif self._in_error_state: 749 # If we are in an error state then query the server for current state of 750 # the upload by sending an empty PUT and reading the 'range' header in 751 # the response. 752 headers = { 753 'Content-Range': 'bytes */%s' % size, 754 'content-length': '0' 755 } 756 resp, content = http.request(self.resumable_uri, 'PUT', 757 headers=headers) 758 status, body = self._process_response(resp, content) 759 if body: 760 # The upload was complete. 761 return (status, body) 762 763 # The httplib.request method can take streams for the body parameter, but 764 # only in Python 2.6 or later. If a stream is available under those 765 # conditions then use it as the body argument. 766 if self.resumable.has_stream() and sys.version_info[1] >= 6: 767 data = self.resumable.stream() 768 if self.resumable.chunksize() == -1: 769 data.seek(self.resumable_progress) 770 chunk_end = self.resumable.size() - self.resumable_progress - 1 771 else: 772 # Doing chunking with a stream, so wrap a slice of the stream. 773 data = _StreamSlice(data, self.resumable_progress, 774 self.resumable.chunksize()) 775 chunk_end = min( 776 self.resumable_progress + self.resumable.chunksize() - 1, 777 self.resumable.size() - 1) 778 else: 779 data = self.resumable.getbytes( 780 self.resumable_progress, self.resumable.chunksize()) 781 782 # A short read implies that we are at EOF, so finish the upload. 783 if len(data) < self.resumable.chunksize(): 784 size = str(self.resumable_progress + len(data)) 785 786 chunk_end = self.resumable_progress + len(data) - 1 787 788 headers = { 789 'Content-Range': 'bytes %d-%d/%s' % ( 790 self.resumable_progress, chunk_end, size), 791 # Must set the content-length header here because httplib can't 792 # calculate the size when working with _StreamSlice. 793 'Content-Length': str(chunk_end - self.resumable_progress + 1) 794 } 795 try: 796 resp, content = http.request(self.resumable_uri, 'PUT', 797 body=data, 798 headers=headers) 799 except: 800 self._in_error_state = True 801 raise 802 803 return self._process_response(resp, content)
804
805 - def _process_response(self, resp, content):
806 """Process the response from a single chunk upload. 807 808 Args: 809 resp: httplib2.Response, the response object. 810 content: string, the content of the response. 811 812 Returns: 813 (status, body): (ResumableMediaStatus, object) 814 The body will be None until the resumable media is fully uploaded. 815 816 Raises: 817 apiclient.errors.HttpError if the response was not a 2xx or a 308. 818 """ 819 if resp.status in [200, 201]: 820 self._in_error_state = False 821 return None, self.postproc(resp, content) 822 elif resp.status == 308: 823 self._in_error_state = False 824 # A "308 Resume Incomplete" indicates we are not done. 825 self.resumable_progress = int(resp['range'].split('-')[1]) + 1 826 if 'location' in resp: 827 self.resumable_uri = resp['location'] 828 else: 829 self._in_error_state = True 830 raise HttpError(resp, content, uri=self.uri) 831 832 return (MediaUploadProgress(self.resumable_progress, self.resumable.size()), 833 None)
834
835 - def to_json(self):
836 """Returns a JSON representation of the HttpRequest.""" 837 d = copy.copy(self.__dict__) 838 if d['resumable'] is not None: 839 d['resumable'] = self.resumable.to_json() 840 del d['http'] 841 del d['postproc'] 842 843 return simplejson.dumps(d)
844 845 @staticmethod
846 - def from_json(s, http, postproc):
847 """Returns an HttpRequest populated with info from a JSON object.""" 848 d = simplejson.loads(s) 849 if d['resumable'] is not None: 850 d['resumable'] = MediaUpload.new_from_json(d['resumable']) 851 return HttpRequest( 852 http, 853 postproc, 854 uri=d['uri'], 855 method=d['method'], 856 body=d['body'], 857 headers=d['headers'], 858 methodId=d['methodId'], 859 resumable=d['resumable'])
860
861 862 -class BatchHttpRequest(object):
863 """Batches multiple HttpRequest objects into a single HTTP request. 864 865 Example: 866 from apiclient.http import BatchHttpRequest 867 868 def list_animals(request_id, response, exception): 869 \"\"\"Do something with the animals list response.\"\"\" 870 if exception is not None: 871 # Do something with the exception. 872 pass 873 else: 874 # Do something with the response. 875 pass 876 877 def list_farmers(request_id, response, exception): 878 \"\"\"Do something with the farmers list response.\"\"\" 879 if exception is not None: 880 # Do something with the exception. 881 pass 882 else: 883 # Do something with the response. 884 pass 885 886 service = build('farm', 'v2') 887 888 batch = BatchHttpRequest() 889 890 batch.add(service.animals().list(), list_animals) 891 batch.add(service.farmers().list(), list_farmers) 892 batch.execute(http=http) 893 """ 894 895 @util.positional(1)
896 - def __init__(self, callback=None, batch_uri=None):
897 """Constructor for a BatchHttpRequest. 898 899 Args: 900 callback: callable, A callback to be called for each response, of the 901 form callback(id, response, exception). The first parameter is the 902 request id, and the second is the deserialized response object. The 903 third is an apiclient.errors.HttpError exception object if an HTTP error 904 occurred while processing the request, or None if no error occurred. 905 batch_uri: string, URI to send batch requests to. 906 """ 907 if batch_uri is None: 908 batch_uri = 'https://www.googleapis.com/batch' 909 self._batch_uri = batch_uri 910 911 # Global callback to be called for each individual response in the batch. 912 self._callback = callback 913 914 # A map from id to request. 915 self._requests = {} 916 917 # A map from id to callback. 918 self._callbacks = {} 919 920 # List of request ids, in the order in which they were added. 921 self._order = [] 922 923 # The last auto generated id. 924 self._last_auto_id = 0 925 926 # Unique ID on which to base the Content-ID headers. 927 self._base_id = None 928 929 # A map from request id to (httplib2.Response, content) response pairs 930 self._responses = {} 931 932 # A map of id(Credentials) that have been refreshed. 933 self._refreshed_credentials = {}
934
935 - def _refresh_and_apply_credentials(self, request, http):
936 """Refresh the credentials and apply to the request. 937 938 Args: 939 request: HttpRequest, the request. 940 http: httplib2.Http, the global http object for the batch. 941 """ 942 # For the credentials to refresh, but only once per refresh_token 943 # If there is no http per the request then refresh the http passed in 944 # via execute() 945 creds = None 946 if request.http is not None and hasattr(request.http.request, 947 'credentials'): 948 creds = request.http.request.credentials 949 elif http is not None and hasattr(http.request, 'credentials'): 950 creds = http.request.credentials 951 if creds is not None: 952 if id(creds) not in self._refreshed_credentials: 953 creds.refresh(http) 954 self._refreshed_credentials[id(creds)] = 1 955 956 # Only apply the credentials if we are using the http object passed in, 957 # otherwise apply() will get called during _serialize_request(). 958 if request.http is None or not hasattr(request.http.request, 959 'credentials'): 960 creds.apply(request.headers)
961
962 - def _id_to_header(self, id_):
963 """Convert an id to a Content-ID header value. 964 965 Args: 966 id_: string, identifier of individual request. 967 968 Returns: 969 A Content-ID header with the id_ encoded into it. A UUID is prepended to 970 the value because Content-ID headers are supposed to be universally 971 unique. 972 """ 973 if self._base_id is None: 974 self._base_id = uuid.uuid4() 975 976 return '<%s+%s>' % (self._base_id, urllib.quote(id_))
977
978 - def _header_to_id(self, header):
979 """Convert a Content-ID header value to an id. 980 981 Presumes the Content-ID header conforms to the format that _id_to_header() 982 returns. 983 984 Args: 985 header: string, Content-ID header value. 986 987 Returns: 988 The extracted id value. 989 990 Raises: 991 BatchError if the header is not in the expected format. 992 """ 993 if header[0] != '<' or header[-1] != '>': 994 raise BatchError("Invalid value for Content-ID: %s" % header) 995 if '+' not in header: 996 raise BatchError("Invalid value for Content-ID: %s" % header) 997 base, id_ = header[1:-1].rsplit('+', 1) 998 999 return urllib.unquote(id_)
1000
1001 - def _serialize_request(self, request):
1002 """Convert an HttpRequest object into a string. 1003 1004 Args: 1005 request: HttpRequest, the request to serialize. 1006 1007 Returns: 1008 The request as a string in application/http format. 1009 """ 1010 # Construct status line 1011 parsed = urlparse.urlparse(request.uri) 1012 request_line = urlparse.urlunparse( 1013 (None, None, parsed.path, parsed.params, parsed.query, None) 1014 ) 1015 status_line = request.method + ' ' + request_line + ' HTTP/1.1\n' 1016 major, minor = request.headers.get('content-type', 'application/json').split('/') 1017 msg = MIMENonMultipart(major, minor) 1018 headers = request.headers.copy() 1019 1020 if request.http is not None and hasattr(request.http.request, 1021 'credentials'): 1022 request.http.request.credentials.apply(headers) 1023 1024 # MIMENonMultipart adds its own Content-Type header. 1025 if 'content-type' in headers: 1026 del headers['content-type'] 1027 1028 for key, value in headers.iteritems(): 1029 msg[key] = value 1030 msg['Host'] = parsed.netloc 1031 msg.set_unixfrom(None) 1032 1033 if request.body is not None: 1034 msg.set_payload(request.body) 1035 msg['content-length'] = str(len(request.body)) 1036 1037 # Serialize the mime message. 1038 fp = StringIO.StringIO() 1039 # maxheaderlen=0 means don't line wrap headers. 1040 g = Generator(fp, maxheaderlen=0) 1041 g.flatten(msg, unixfrom=False) 1042 body = fp.getvalue() 1043 1044 # Strip off the \n\n that the MIME lib tacks onto the end of the payload. 1045 if request.body is None: 1046 body = body[:-2] 1047 1048 return status_line.encode('utf-8') + body
1049
1050 - def _deserialize_response(self, payload):
1051 """Convert string into httplib2 response and content. 1052 1053 Args: 1054 payload: string, headers and body as a string. 1055 1056 Returns: 1057 A pair (resp, content), such as would be returned from httplib2.request. 1058 """ 1059 # Strip off the status line 1060 status_line, payload = payload.split('\n', 1) 1061 protocol, status, reason = status_line.split(' ', 2) 1062 1063 # Parse the rest of the response 1064 parser = FeedParser() 1065 parser.feed(payload) 1066 msg = parser.close() 1067 msg['status'] = status 1068 1069 # Create httplib2.Response from the parsed headers. 1070 resp = httplib2.Response(msg) 1071 resp.reason = reason 1072 resp.version = int(protocol.split('/', 1)[1].replace('.', '')) 1073 1074 content = payload.split('\r\n\r\n', 1)[1] 1075 1076 return resp, content
1077
1078 - def _new_id(self):
1079 """Create a new id. 1080 1081 Auto incrementing number that avoids conflicts with ids already used. 1082 1083 Returns: 1084 string, a new unique id. 1085 """ 1086 self._last_auto_id += 1 1087 while str(self._last_auto_id) in self._requests: 1088 self._last_auto_id += 1 1089 return str(self._last_auto_id)
1090 1091 @util.positional(2)
1092 - def add(self, request, callback=None, request_id=None):
1093 """Add a new request. 1094 1095 Every callback added will be paired with a unique id, the request_id. That 1096 unique id will be passed back to the callback when the response comes back 1097 from the server. The default behavior is to have the library generate it's 1098 own unique id. If the caller passes in a request_id then they must ensure 1099 uniqueness for each request_id, and if they are not an exception is 1100 raised. Callers should either supply all request_ids or nevery supply a 1101 request id, to avoid such an error. 1102 1103 Args: 1104 request: HttpRequest, Request to add to the batch. 1105 callback: callable, A callback to be called for this response, of the 1106 form callback(id, response, exception). The first parameter is the 1107 request id, and the second is the deserialized response object. The 1108 third is an apiclient.errors.HttpError exception object if an HTTP error 1109 occurred while processing the request, or None if no errors occurred. 1110 request_id: string, A unique id for the request. The id will be passed to 1111 the callback with the response. 1112 1113 Returns: 1114 None 1115 1116 Raises: 1117 BatchError if a media request is added to a batch. 1118 KeyError is the request_id is not unique. 1119 """ 1120 if request_id is None: 1121 request_id = self._new_id() 1122 if request.resumable is not None: 1123 raise BatchError("Media requests cannot be used in a batch request.") 1124 if request_id in self._requests: 1125 raise KeyError("A request with this ID already exists: %s" % request_id) 1126 self._requests[request_id] = request 1127 self._callbacks[request_id] = callback 1128 self._order.append(request_id)
1129
1130 - def _execute(self, http, order, requests):
1131 """Serialize batch request, send to server, process response. 1132 1133 Args: 1134 http: httplib2.Http, an http object to be used to make the request with. 1135 order: list, list of request ids in the order they were added to the 1136 batch. 1137 request: list, list of request objects to send. 1138 1139 Raises: 1140 httplib2.HttpLib2Error if a transport error has occured. 1141 apiclient.errors.BatchError if the response is the wrong format. 1142 """ 1143 message = MIMEMultipart('mixed') 1144 # Message should not write out it's own headers. 1145 setattr(message, '_write_headers', lambda self: None) 1146 1147 # Add all the individual requests. 1148 for request_id in order: 1149 request = requests[request_id] 1150 1151 msg = MIMENonMultipart('application', 'http') 1152 msg['Content-Transfer-Encoding'] = 'binary' 1153 msg['Content-ID'] = self._id_to_header(request_id) 1154 1155 body = self._serialize_request(request) 1156 msg.set_payload(body) 1157 message.attach(msg) 1158 1159 body = message.as_string() 1160 1161 headers = {} 1162 headers['content-type'] = ('multipart/mixed; ' 1163 'boundary="%s"') % message.get_boundary() 1164 1165 resp, content = http.request(self._batch_uri, 'POST', body=body, 1166 headers=headers) 1167 1168 if resp.status >= 300: 1169 raise HttpError(resp, content, uri=self._batch_uri) 1170 1171 # Now break out the individual responses and store each one. 1172 boundary, _ = content.split(None, 1) 1173 1174 # Prepend with a content-type header so FeedParser can handle it. 1175 header = 'content-type: %s\r\n\r\n' % resp['content-type'] 1176 for_parser = header + content 1177 1178 parser = FeedParser() 1179 parser.feed(for_parser) 1180 mime_response = parser.close() 1181 1182 if not mime_response.is_multipart(): 1183 raise BatchError("Response not in multipart/mixed format.", resp=resp, 1184 content=content) 1185 1186 for part in mime_response.get_payload(): 1187 request_id = self._header_to_id(part['Content-ID']) 1188 response, content = self._deserialize_response(part.get_payload()) 1189 self._responses[request_id] = (response, content)
1190 1191 @util.positional(1)
1192 - def execute(self, http=None):
1193 """Execute all the requests as a single batched HTTP request. 1194 1195 Args: 1196 http: httplib2.Http, an http object to be used in place of the one the 1197 HttpRequest request object was constructed with. If one isn't supplied 1198 then use a http object from the requests in this batch. 1199 1200 Returns: 1201 None 1202 1203 Raises: 1204 httplib2.HttpLib2Error if a transport error has occured. 1205 apiclient.errors.BatchError if the response is the wrong format. 1206 """ 1207 1208 # If http is not supplied use the first valid one given in the requests. 1209 if http is None: 1210 for request_id in self._order: 1211 request = self._requests[request_id] 1212 if request is not None: 1213 http = request.http 1214 break 1215 1216 if http is None: 1217 raise ValueError("Missing a valid http object.") 1218 1219 self._execute(http, self._order, self._requests) 1220 1221 # Loop over all the requests and check for 401s. For each 401 request the 1222 # credentials should be refreshed and then sent again in a separate batch. 1223 redo_requests = {} 1224 redo_order = [] 1225 1226 for request_id in self._order: 1227 resp, content = self._responses[request_id] 1228 if resp['status'] == '401': 1229 redo_order.append(request_id) 1230 request = self._requests[request_id] 1231 self._refresh_and_apply_credentials(request, http) 1232 redo_requests[request_id] = request 1233 1234 if redo_requests: 1235 self._execute(http, redo_order, redo_requests) 1236 1237 # Now process all callbacks that are erroring, and raise an exception for 1238 # ones that return a non-2xx response? Or add extra parameter to callback 1239 # that contains an HttpError? 1240 1241 for request_id in self._order: 1242 resp, content = self._responses[request_id] 1243 1244 request = self._requests[request_id] 1245 callback = self._callbacks[request_id] 1246 1247 response = None 1248 exception = None 1249 try: 1250 if resp.status >= 300: 1251 raise HttpError(resp, content, uri=request.uri) 1252 response = request.postproc(resp, content) 1253 except HttpError, e: 1254 exception = e 1255 1256 if callback is not None: 1257 callback(request_id, response, exception) 1258 if self._callback is not None: 1259 self._callback(request_id, response, exception)
1260
1261 1262 -class HttpRequestMock(object):
1263 """Mock of HttpRequest. 1264 1265 Do not construct directly, instead use RequestMockBuilder. 1266 """ 1267
1268 - def __init__(self, resp, content, postproc):
1269 """Constructor for HttpRequestMock 1270 1271 Args: 1272 resp: httplib2.Response, the response to emulate coming from the request 1273 content: string, the response body 1274 postproc: callable, the post processing function usually supplied by 1275 the model class. See model.JsonModel.response() as an example. 1276 """ 1277 self.resp = resp 1278 self.content = content 1279 self.postproc = postproc 1280 if resp is None: 1281 self.resp = httplib2.Response({'status': 200, 'reason': 'OK'}) 1282 if 'reason' in self.resp: 1283 self.resp.reason = self.resp['reason']
1284
1285 - def execute(self, http=None):
1286 """Execute the request. 1287 1288 Same behavior as HttpRequest.execute(), but the response is 1289 mocked and not really from an HTTP request/response. 1290 """ 1291 return self.postproc(self.resp, self.content)
1292
1293 1294 -class RequestMockBuilder(object):
1295 """A simple mock of HttpRequest 1296 1297 Pass in a dictionary to the constructor that maps request methodIds to 1298 tuples of (httplib2.Response, content, opt_expected_body) that should be 1299 returned when that method is called. None may also be passed in for the 1300 httplib2.Response, in which case a 200 OK response will be generated. 1301 If an opt_expected_body (str or dict) is provided, it will be compared to 1302 the body and UnexpectedBodyError will be raised on inequality. 1303 1304 Example: 1305 response = '{"data": {"id": "tag:google.c...' 1306 requestBuilder = RequestMockBuilder( 1307 { 1308 'plus.activities.get': (None, response), 1309 } 1310 ) 1311 apiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder) 1312 1313 Methods that you do not supply a response for will return a 1314 200 OK with an empty string as the response content or raise an excpetion 1315 if check_unexpected is set to True. The methodId is taken from the rpcName 1316 in the discovery document. 1317 1318 For more details see the project wiki. 1319 """ 1320
1321 - def __init__(self, responses, check_unexpected=False):
1322 """Constructor for RequestMockBuilder 1323 1324 The constructed object should be a callable object 1325 that can replace the class HttpResponse. 1326 1327 responses - A dictionary that maps methodIds into tuples 1328 of (httplib2.Response, content). The methodId 1329 comes from the 'rpcName' field in the discovery 1330 document. 1331 check_unexpected - A boolean setting whether or not UnexpectedMethodError 1332 should be raised on unsupplied method. 1333 """ 1334 self.responses = responses 1335 self.check_unexpected = check_unexpected
1336
1337 - def __call__(self, http, postproc, uri, method='GET', body=None, 1338 headers=None, methodId=None, resumable=None):
1339 """Implements the callable interface that discovery.build() expects 1340 of requestBuilder, which is to build an object compatible with 1341 HttpRequest.execute(). See that method for the description of the 1342 parameters and the expected response. 1343 """ 1344 if methodId in self.responses: 1345 response = self.responses[methodId] 1346 resp, content = response[:2] 1347 if len(response) > 2: 1348 # Test the body against the supplied expected_body. 1349 expected_body = response[2] 1350 if bool(expected_body) != bool(body): 1351 # Not expecting a body and provided one 1352 # or expecting a body and not provided one. 1353 raise UnexpectedBodyError(expected_body, body) 1354 if isinstance(expected_body, str): 1355 expected_body = simplejson.loads(expected_body) 1356 body = simplejson.loads(body) 1357 if body != expected_body: 1358 raise UnexpectedBodyError(expected_body, body) 1359 return HttpRequestMock(resp, content, postproc) 1360 elif self.check_unexpected: 1361 raise UnexpectedMethodError(methodId=methodId) 1362 else: 1363 model = JsonModel(False) 1364 return HttpRequestMock(None, '{}', model.response)
1365
1366 1367 -class HttpMock(object):
1368 """Mock of httplib2.Http""" 1369
1370 - def __init__(self, filename=None, headers=None):
1371 """ 1372 Args: 1373 filename: string, absolute filename to read response from 1374 headers: dict, header to return with response 1375 """ 1376 if headers is None: 1377 headers = {'status': '200 OK'} 1378 if filename: 1379 f = file(filename, 'r') 1380 self.data = f.read() 1381 f.close() 1382 else: 1383 self.data = None 1384 self.response_headers = headers 1385 self.headers = None 1386 self.uri = None 1387 self.method = None 1388 self.body = None 1389 self.headers = None
1390 1391
1392 - def request(self, uri, 1393 method='GET', 1394 body=None, 1395 headers=None, 1396 redirections=1, 1397 connection_type=None):
1398 self.uri = uri 1399 self.method = method 1400 self.body = body 1401 self.headers = headers 1402 return httplib2.Response(self.response_headers), self.data
1403
1404 1405 -class HttpMockSequence(object):
1406 """Mock of httplib2.Http 1407 1408 Mocks a sequence of calls to request returning different responses for each 1409 call. Create an instance initialized with the desired response headers 1410 and content and then use as if an httplib2.Http instance. 1411 1412 http = HttpMockSequence([ 1413 ({'status': '401'}, ''), 1414 ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'), 1415 ({'status': '200'}, 'echo_request_headers'), 1416 ]) 1417 resp, content = http.request("http://examples.com") 1418 1419 There are special values you can pass in for content to trigger 1420 behavours that are helpful in testing. 1421 1422 'echo_request_headers' means return the request headers in the response body 1423 'echo_request_headers_as_json' means return the request headers in 1424 the response body 1425 'echo_request_body' means return the request body in the response body 1426 'echo_request_uri' means return the request uri in the response body 1427 """ 1428
1429 - def __init__(self, iterable):
1430 """ 1431 Args: 1432 iterable: iterable, a sequence of pairs of (headers, body) 1433 """ 1434 self._iterable = iterable 1435 self.follow_redirects = True
1436
1437 - def request(self, uri, 1438 method='GET', 1439 body=None, 1440 headers=None, 1441 redirections=1, 1442 connection_type=None):
1443 resp, content = self._iterable.pop(0) 1444 if content == 'echo_request_headers': 1445 content = headers 1446 elif content == 'echo_request_headers_as_json': 1447 content = simplejson.dumps(headers) 1448 elif content == 'echo_request_body': 1449 if hasattr(body, 'read'): 1450 content = body.read() 1451 else: 1452 content = body 1453 elif content == 'echo_request_uri': 1454 content = uri 1455 return httplib2.Response(resp), content
1456
1457 1458 -def set_user_agent(http, user_agent):
1459 """Set the user-agent on every request. 1460 1461 Args: 1462 http - An instance of httplib2.Http 1463 or something that acts like it. 1464 user_agent: string, the value for the user-agent header. 1465 1466 Returns: 1467 A modified instance of http that was passed in. 1468 1469 Example: 1470 1471 h = httplib2.Http() 1472 h = set_user_agent(h, "my-app-name/6.0") 1473 1474 Most of the time the user-agent will be set doing auth, this is for the rare 1475 cases where you are accessing an unauthenticated endpoint. 1476 """ 1477 request_orig = http.request 1478 1479 # The closure that will replace 'httplib2.Http.request'. 1480 def new_request(uri, method='GET', body=None, headers=None, 1481 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1482 connection_type=None): 1483 """Modify the request headers to add the user-agent.""" 1484 if headers is None: 1485 headers = {} 1486 if 'user-agent' in headers: 1487 headers['user-agent'] = user_agent + ' ' + headers['user-agent'] 1488 else: 1489 headers['user-agent'] = user_agent 1490 resp, content = request_orig(uri, method, body, headers, 1491 redirections, connection_type) 1492 return resp, content
1493 1494 http.request = new_request 1495 return http 1496
1497 1498 -def tunnel_patch(http):
1499 """Tunnel PATCH requests over POST. 1500 Args: 1501 http - An instance of httplib2.Http 1502 or something that acts like it. 1503 1504 Returns: 1505 A modified instance of http that was passed in. 1506 1507 Example: 1508 1509 h = httplib2.Http() 1510 h = tunnel_patch(h, "my-app-name/6.0") 1511 1512 Useful if you are running on a platform that doesn't support PATCH. 1513 Apply this last if you are using OAuth 1.0, as changing the method 1514 will result in a different signature. 1515 """ 1516 request_orig = http.request 1517 1518 # The closure that will replace 'httplib2.Http.request'. 1519 def new_request(uri, method='GET', body=None, headers=None, 1520 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1521 connection_type=None): 1522 """Modify the request headers to add the user-agent.""" 1523 if headers is None: 1524 headers = {} 1525 if method == 'PATCH': 1526 if 'oauth_token' in headers.get('authorization', ''): 1527 logging.warning( 1528 'OAuth 1.0 request made with Credentials after tunnel_patch.') 1529 headers['x-http-method-override'] = "PATCH" 1530 method = 'POST' 1531 resp, content = request_orig(uri, method, body, headers, 1532 redirections, connection_type) 1533 return resp, content
1534 1535 http.request = new_request 1536 return http 1537