Package apiclient :: Module http
[hide private]
[frames] | no frames]

Source Code for Module apiclient.http

   1  # Copyright (C) 2012 Google Inc. 
   2  # 
   3  # Licensed under the Apache License, Version 2.0 (the "License"); 
   4  # you may not use this file except in compliance with the License. 
   5  # You may obtain a copy of the License at 
   6  # 
   7  #      http://www.apache.org/licenses/LICENSE-2.0 
   8  # 
   9  # Unless required by applicable law or agreed to in writing, software 
  10  # distributed under the License is distributed on an "AS IS" BASIS, 
  11  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
  12  # See the License for the specific language governing permissions and 
  13  # limitations under the License. 
  14   
  15  """Classes to encapsulate a single HTTP request. 
  16   
  17  The classes implement a command pattern, with every 
  18  object supporting an execute() method that does the 
  19  actuall HTTP request. 
  20  """ 
  21   
  22  __author__ = 'jcgregorio@google.com (Joe Gregorio)' 
  23   
  24  import StringIO 
  25  import base64 
  26  import copy 
  27  import gzip 
  28  import httplib2 
  29  import mimeparse 
  30  import mimetypes 
  31  import os 
  32  import urllib 
  33  import urlparse 
  34  import uuid 
  35   
  36  from email.generator import Generator 
  37  from email.mime.multipart import MIMEMultipart 
  38  from email.mime.nonmultipart import MIMENonMultipart 
  39  from email.parser import FeedParser 
  40  from errors import BatchError 
  41  from errors import HttpError 
  42  from errors import ResumableUploadError 
  43  from errors import UnexpectedBodyError 
  44  from errors import UnexpectedMethodError 
  45  from model import JsonModel 
  46  from oauth2client.anyjson import simplejson 
  47   
  48   
  49  DEFAULT_CHUNK_SIZE = 512*1024 
50 51 52 -class MediaUploadProgress(object):
53 """Status of a resumable upload.""" 54
55 - def __init__(self, resumable_progress, total_size):
56 """Constructor. 57 58 Args: 59 resumable_progress: int, bytes sent so far. 60 total_size: int, total bytes in complete upload, or None if the total 61 upload size isn't known ahead of time. 62 """ 63 self.resumable_progress = resumable_progress 64 self.total_size = total_size
65
66 - def progress(self):
67 """Percent of upload completed, as a float. 68 69 Returns: 70 the percentage complete as a float, returning 0.0 if the total size of 71 the upload is unknown. 72 """ 73 if self.total_size is not None: 74 return float(self.resumable_progress) / float(self.total_size) 75 else: 76 return 0.0
77
78 79 -class MediaDownloadProgress(object):
80 """Status of a resumable download.""" 81
82 - def __init__(self, resumable_progress, total_size):
83 """Constructor. 84 85 Args: 86 resumable_progress: int, bytes received so far. 87 total_size: int, total bytes in complete download. 88 """ 89 self.resumable_progress = resumable_progress 90 self.total_size = total_size
91
92 - def progress(self):
93 """Percent of download completed, as a float. 94 95 Returns: 96 the percentage complete as a float, returning 0.0 if the total size of 97 the download is unknown. 98 """ 99 if self.total_size is not None: 100 return float(self.resumable_progress) / float(self.total_size) 101 else: 102 return 0.0
103
104 105 -class MediaUpload(object):
106 """Describes a media object to upload. 107 108 Base class that defines the interface of MediaUpload subclasses. 109 110 Note that subclasses of MediaUpload may allow you to control the chunksize 111 when upload a media object. It is important to keep the size of the chunk as 112 large as possible to keep the upload efficient. Other factors may influence 113 the size of the chunk you use, particularly if you are working in an 114 environment where individual HTTP requests may have a hardcoded time limit, 115 such as under certain classes of requests under Google App Engine. 116 """ 117
118 - def chunksize(self):
119 """Chunk size for resumable uploads. 120 121 Returns: 122 Chunk size in bytes. 123 """ 124 raise NotImplementedError()
125
126 - def mimetype(self):
127 """Mime type of the body. 128 129 Returns: 130 Mime type. 131 """ 132 return 'application/octet-stream'
133
134 - def size(self):
135 """Size of upload. 136 137 Returns: 138 Size of the body, or None of the size is unknown. 139 """ 140 return None
141
142 - def resumable(self):
143 """Whether this upload is resumable. 144 145 Returns: 146 True if resumable upload or False. 147 """ 148 return False
149
150 - def getbytes(self, begin, end):
151 """Get bytes from the media. 152 153 Args: 154 begin: int, offset from beginning of file. 155 length: int, number of bytes to read, starting at begin. 156 157 Returns: 158 A string of bytes read. May be shorter than length if EOF was reached 159 first. 160 """ 161 raise NotImplementedError()
162
163 - def _to_json(self, strip=None):
164 """Utility function for creating a JSON representation of a MediaUpload. 165 166 Args: 167 strip: array, An array of names of members to not include in the JSON. 168 169 Returns: 170 string, a JSON representation of this instance, suitable to pass to 171 from_json(). 172 """ 173 t = type(self) 174 d = copy.copy(self.__dict__) 175 if strip is not None: 176 for member in strip: 177 del d[member] 178 d['_class'] = t.__name__ 179 d['_module'] = t.__module__ 180 return simplejson.dumps(d)
181
182 - def to_json(self):
183 """Create a JSON representation of an instance of MediaUpload. 184 185 Returns: 186 string, a JSON representation of this instance, suitable to pass to 187 from_json(). 188 """ 189 return self._to_json()
190 191 @classmethod
192 - def new_from_json(cls, s):
193 """Utility class method to instantiate a MediaUpload subclass from a JSON 194 representation produced by to_json(). 195 196 Args: 197 s: string, JSON from to_json(). 198 199 Returns: 200 An instance of the subclass of MediaUpload that was serialized with 201 to_json(). 202 """ 203 data = simplejson.loads(s) 204 # Find and call the right classmethod from_json() to restore the object. 205 module = data['_module'] 206 m = __import__(module, fromlist=module.split('.')[:-1]) 207 kls = getattr(m, data['_class']) 208 from_json = getattr(kls, 'from_json') 209 return from_json(s)
210
211 212 -class MediaFileUpload(MediaUpload):
213 """A MediaUpload for a file. 214 215 Construct a MediaFileUpload and pass as the media_body parameter of the 216 method. For example, if we had a service that allowed uploading images: 217 218 219 media = MediaFileUpload('cow.png', mimetype='image/png', 220 chunksize=1024*1024, resumable=True) 221 farm.animals()..insert( 222 id='cow', 223 name='cow.png', 224 media_body=media).execute() 225 """ 226
227 - def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE, resumable=False):
228 """Constructor. 229 230 Args: 231 filename: string, Name of the file. 232 mimetype: string, Mime-type of the file. If None then a mime-type will be 233 guessed from the file extension. 234 chunksize: int, File will be uploaded in chunks of this many bytes. Only 235 used if resumable=True. 236 resumable: bool, True if this is a resumable upload. False means upload 237 in a single request. 238 """ 239 self._filename = filename 240 self._size = os.path.getsize(filename) 241 self._fd = None 242 if mimetype is None: 243 (mimetype, encoding) = mimetypes.guess_type(filename) 244 self._mimetype = mimetype 245 self._chunksize = chunksize 246 self._resumable = resumable
247
248 - def chunksize(self):
249 """Chunk size for resumable uploads. 250 251 Returns: 252 Chunk size in bytes. 253 """ 254 return self._chunksize
255
256 - def mimetype(self):
257 """Mime type of the body. 258 259 Returns: 260 Mime type. 261 """ 262 return self._mimetype
263
264 - def size(self):
265 """Size of upload. 266 267 Returns: 268 Size of the body, or None of the size is unknown. 269 """ 270 return self._size
271
272 - def resumable(self):
273 """Whether this upload is resumable. 274 275 Returns: 276 True if resumable upload or False. 277 """ 278 return self._resumable
279
280 - def getbytes(self, begin, length):
281 """Get bytes from the media. 282 283 Args: 284 begin: int, offset from beginning of file. 285 length: int, number of bytes to read, starting at begin. 286 287 Returns: 288 A string of bytes read. May be shorted than length if EOF was reached 289 first. 290 """ 291 if self._fd is None: 292 self._fd = open(self._filename, 'rb') 293 self._fd.seek(begin) 294 return self._fd.read(length)
295
296 - def to_json(self):
297 """Creating a JSON representation of an instance of MediaFileUpload. 298 299 Returns: 300 string, a JSON representation of this instance, suitable to pass to 301 from_json(). 302 """ 303 return self._to_json(['_fd'])
304 305 @staticmethod
306 - def from_json(s):
307 d = simplejson.loads(s) 308 return MediaFileUpload( 309 d['_filename'], d['_mimetype'], d['_chunksize'], d['_resumable'])
310
311 312 -class MediaIoBaseUpload(MediaUpload):
313 """A MediaUpload for a io.Base objects. 314 315 Note that the Python file object is compatible with io.Base and can be used 316 with this class also. 317 318 fh = io.BytesIO('...Some data to upload...') 319 media = MediaIoBaseUpload(fh, mimetype='image/png', 320 chunksize=1024*1024, resumable=True) 321 farm.animals().insert( 322 id='cow', 323 name='cow.png', 324 media_body=media).execute() 325 """ 326
327 - def __init__(self, fh, mimetype, chunksize=DEFAULT_CHUNK_SIZE, 328 resumable=False):
329 """Constructor. 330 331 Args: 332 fh: io.Base or file object, The source of the bytes to upload. MUST be 333 opened in blocking mode, do not use streams opened in non-blocking mode. 334 mimetype: string, Mime-type of the file. If None then a mime-type will be 335 guessed from the file extension. 336 chunksize: int, File will be uploaded in chunks of this many bytes. Only 337 used if resumable=True. 338 resumable: bool, True if this is a resumable upload. False means upload 339 in a single request. 340 """ 341 self._fh = fh 342 self._mimetype = mimetype 343 self._chunksize = chunksize 344 self._resumable = resumable 345 self._size = None 346 try: 347 if hasattr(self._fh, 'fileno'): 348 fileno = self._fh.fileno() 349 350 # Pipes and such show up as 0 length files. 351 size = os.fstat(fileno).st_size 352 if size: 353 self._size = os.fstat(fileno).st_size 354 except IOError: 355 pass
356
357 - def chunksize(self):
358 """Chunk size for resumable uploads. 359 360 Returns: 361 Chunk size in bytes. 362 """ 363 return self._chunksize
364
365 - def mimetype(self):
366 """Mime type of the body. 367 368 Returns: 369 Mime type. 370 """ 371 return self._mimetype
372
373 - def size(self):
374 """Size of upload. 375 376 Returns: 377 Size of the body, or None of the size is unknown. 378 """ 379 return self._size
380
381 - def resumable(self):
382 """Whether this upload is resumable. 383 384 Returns: 385 True if resumable upload or False. 386 """ 387 return self._resumable
388
389 - def getbytes(self, begin, length):
390 """Get bytes from the media. 391 392 Args: 393 begin: int, offset from beginning of file. 394 length: int, number of bytes to read, starting at begin. 395 396 Returns: 397 A string of bytes read. May be shorted than length if EOF was reached 398 first. 399 """ 400 self._fh.seek(begin) 401 return self._fh.read(length)
402
403 - def to_json(self):
404 """This upload type is not serializable.""" 405 raise NotImplementedError('MediaIoBaseUpload is not serializable.')
406
407 408 -class MediaInMemoryUpload(MediaUpload):
409 """MediaUpload for a chunk of bytes. 410 411 Construct a MediaFileUpload and pass as the media_body parameter of the 412 method. 413 """ 414
415 - def __init__(self, body, mimetype='application/octet-stream', 416 chunksize=DEFAULT_CHUNK_SIZE, resumable=False):
417 """Create a new MediaBytesUpload. 418 419 Args: 420 body: string, Bytes of body content. 421 mimetype: string, Mime-type of the file or default of 422 'application/octet-stream'. 423 chunksize: int, File will be uploaded in chunks of this many bytes. Only 424 used if resumable=True. 425 resumable: bool, True if this is a resumable upload. False means upload 426 in a single request. 427 """ 428 self._body = body 429 self._mimetype = mimetype 430 self._resumable = resumable 431 self._chunksize = chunksize
432
433 - def chunksize(self):
434 """Chunk size for resumable uploads. 435 436 Returns: 437 Chunk size in bytes. 438 """ 439 return self._chunksize
440
441 - def mimetype(self):
442 """Mime type of the body. 443 444 Returns: 445 Mime type. 446 """ 447 return self._mimetype
448
449 - def size(self):
450 """Size of upload. 451 452 Returns: 453 Size of the body, or None of the size is unknown. 454 """ 455 return len(self._body)
456
457 - def resumable(self):
458 """Whether this upload is resumable. 459 460 Returns: 461 True if resumable upload or False. 462 """ 463 return self._resumable
464
465 - def getbytes(self, begin, length):
466 """Get bytes from the media. 467 468 Args: 469 begin: int, offset from beginning of file. 470 length: int, number of bytes to read, starting at begin. 471 472 Returns: 473 A string of bytes read. May be shorter than length if EOF was reached 474 first. 475 """ 476 return self._body[begin:begin + length]
477
478 - def to_json(self):
479 """Create a JSON representation of a MediaInMemoryUpload. 480 481 Returns: 482 string, a JSON representation of this instance, suitable to pass to 483 from_json(). 484 """ 485 t = type(self) 486 d = copy.copy(self.__dict__) 487 del d['_body'] 488 d['_class'] = t.__name__ 489 d['_module'] = t.__module__ 490 d['_b64body'] = base64.b64encode(self._body) 491 return simplejson.dumps(d)
492 493 @staticmethod
494 - def from_json(s):
495 d = simplejson.loads(s) 496 return MediaInMemoryUpload(base64.b64decode(d['_b64body']), 497 d['_mimetype'], d['_chunksize'], 498 d['_resumable'])
499
500 501 -class MediaIoBaseDownload(object):
502 """"Download media resources. 503 504 Note that the Python file object is compatible with io.Base and can be used 505 with this class also. 506 507 508 Example: 509 request = farms.animals().get_media(id='cow') 510 fh = io.FileIO('cow.png', mode='wb') 511 downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024) 512 513 done = False 514 while done is False: 515 status, done = downloader.next_chunk() 516 if status: 517 print "Download %d%%." % int(status.progress() * 100) 518 print "Download Complete!" 519 """ 520
521 - def __init__(self, fh, request, chunksize=DEFAULT_CHUNK_SIZE):
522 """Constructor. 523 524 Args: 525 fh: io.Base or file object, The stream in which to write the downloaded 526 bytes. 527 request: apiclient.http.HttpRequest, the media request to perform in 528 chunks. 529 chunksize: int, File will be downloaded in chunks of this many bytes. 530 """ 531 self.fh_ = fh 532 self.request_ = request 533 self.uri_ = request.uri 534 self.chunksize_ = chunksize 535 self.progress_ = 0 536 self.total_size_ = None 537 self.done_ = False
538
539 - def next_chunk(self):
540 """Get the next chunk of the download. 541 542 Returns: 543 (status, done): (MediaDownloadStatus, boolean) 544 The value of 'done' will be True when the media has been fully 545 downloaded. 546 547 Raises: 548 apiclient.errors.HttpError if the response was not a 2xx. 549 httplib2.Error if a transport error has occured. 550 """ 551 headers = { 552 'range': 'bytes=%d-%d' % ( 553 self.progress_, self.progress_ + self.chunksize_) 554 } 555 http = self.request_.http 556 http.follow_redirects = False 557 558 resp, content = http.request(self.uri_, headers=headers) 559 if resp.status in [301, 302, 303, 307, 308] and 'location' in resp: 560 self.uri_ = resp['location'] 561 resp, content = http.request(self.uri_, headers=headers) 562 if resp.status in [200, 206]: 563 self.progress_ += len(content) 564 self.fh_.write(content) 565 566 if 'content-range' in resp: 567 content_range = resp['content-range'] 568 length = content_range.rsplit('/', 1)[1] 569 self.total_size_ = int(length) 570 571 if self.progress_ == self.total_size_: 572 self.done_ = True 573 return MediaDownloadProgress(self.progress_, self.total_size_), self.done_ 574 else: 575 raise HttpError(resp, content, self.uri_)
576
577 578 -class HttpRequest(object):
579 """Encapsulates a single HTTP request.""" 580
581 - def __init__(self, http, postproc, uri, 582 method='GET', 583 body=None, 584 headers=None, 585 methodId=None, 586 resumable=None):
587 """Constructor for an HttpRequest. 588 589 Args: 590 http: httplib2.Http, the transport object to use to make a request 591 postproc: callable, called on the HTTP response and content to transform 592 it into a data object before returning, or raising an exception 593 on an error. 594 uri: string, the absolute URI to send the request to 595 method: string, the HTTP method to use 596 body: string, the request body of the HTTP request, 597 headers: dict, the HTTP request headers 598 methodId: string, a unique identifier for the API method being called. 599 resumable: MediaUpload, None if this is not a resumbale request. 600 """ 601 self.uri = uri 602 self.method = method 603 self.body = body 604 self.headers = headers or {} 605 self.methodId = methodId 606 self.http = http 607 self.postproc = postproc 608 self.resumable = resumable 609 self._in_error_state = False 610 611 # Pull the multipart boundary out of the content-type header. 612 major, minor, params = mimeparse.parse_mime_type( 613 headers.get('content-type', 'application/json')) 614 615 # The size of the non-media part of the request. 616 self.body_size = len(self.body or '') 617 618 # The resumable URI to send chunks to. 619 self.resumable_uri = None 620 621 # The bytes that have been uploaded. 622 self.resumable_progress = 0
623
624 - def execute(self, http=None):
625 """Execute the request. 626 627 Args: 628 http: httplib2.Http, an http object to be used in place of the 629 one the HttpRequest request object was constructed with. 630 631 Returns: 632 A deserialized object model of the response body as determined 633 by the postproc. 634 635 Raises: 636 apiclient.errors.HttpError if the response was not a 2xx. 637 httplib2.Error if a transport error has occured. 638 """ 639 if http is None: 640 http = self.http 641 if self.resumable: 642 body = None 643 while body is None: 644 _, body = self.next_chunk(http) 645 return body 646 else: 647 if 'content-length' not in self.headers: 648 self.headers['content-length'] = str(self.body_size) 649 resp, content = http.request(self.uri, self.method, 650 body=self.body, 651 headers=self.headers) 652 653 if resp.status >= 300: 654 raise HttpError(resp, content, self.uri) 655 return self.postproc(resp, content)
656
657 - def next_chunk(self, http=None):
658 """Execute the next step of a resumable upload. 659 660 Can only be used if the method being executed supports media uploads and 661 the MediaUpload object passed in was flagged as using resumable upload. 662 663 Example: 664 665 media = MediaFileUpload('cow.png', mimetype='image/png', 666 chunksize=1000, resumable=True) 667 request = farm.animals().insert( 668 id='cow', 669 name='cow.png', 670 media_body=media) 671 672 response = None 673 while response is None: 674 status, response = request.next_chunk() 675 if status: 676 print "Upload %d%% complete." % int(status.progress() * 100) 677 678 679 Returns: 680 (status, body): (ResumableMediaStatus, object) 681 The body will be None until the resumable media is fully uploaded. 682 683 Raises: 684 apiclient.errors.HttpError if the response was not a 2xx. 685 httplib2.Error if a transport error has occured. 686 """ 687 if http is None: 688 http = self.http 689 690 if self.resumable.size() is None: 691 size = '*' 692 else: 693 size = str(self.resumable.size()) 694 695 if self.resumable_uri is None: 696 start_headers = copy.copy(self.headers) 697 start_headers['X-Upload-Content-Type'] = self.resumable.mimetype() 698 if size != '*': 699 start_headers['X-Upload-Content-Length'] = size 700 start_headers['content-length'] = str(self.body_size) 701 702 resp, content = http.request(self.uri, self.method, 703 body=self.body, 704 headers=start_headers) 705 if resp.status == 200 and 'location' in resp: 706 self.resumable_uri = resp['location'] 707 else: 708 raise ResumableUploadError("Failed to retrieve starting URI.") 709 elif self._in_error_state: 710 # If we are in an error state then query the server for current state of 711 # the upload by sending an empty PUT and reading the 'range' header in 712 # the response. 713 headers = { 714 'Content-Range': 'bytes */%s' % size, 715 'content-length': '0' 716 } 717 resp, content = http.request(self.resumable_uri, 'PUT', 718 headers=headers) 719 status, body = self._process_response(resp, content) 720 if body: 721 # The upload was complete. 722 return (status, body) 723 724 data = self.resumable.getbytes( 725 self.resumable_progress, self.resumable.chunksize()) 726 727 # A short read implies that we are at EOF, so finish the upload. 728 if len(data) < self.resumable.chunksize(): 729 size = str(self.resumable_progress + len(data)) 730 731 headers = { 732 'Content-Range': 'bytes %d-%d/%s' % ( 733 self.resumable_progress, self.resumable_progress + len(data) - 1, 734 size) 735 } 736 try: 737 resp, content = http.request(self.resumable_uri, 'PUT', 738 body=data, 739 headers=headers) 740 except: 741 self._in_error_state = True 742 raise 743 744 return self._process_response(resp, content)
745
746 - def _process_response(self, resp, content):
747 """Process the response from a single chunk upload. 748 749 Args: 750 resp: httplib2.Response, the response object. 751 content: string, the content of the response. 752 753 Returns: 754 (status, body): (ResumableMediaStatus, object) 755 The body will be None until the resumable media is fully uploaded. 756 757 Raises: 758 apiclient.errors.HttpError if the response was not a 2xx or a 308. 759 """ 760 if resp.status in [200, 201]: 761 self._in_error_state = False 762 return None, self.postproc(resp, content) 763 elif resp.status == 308: 764 self._in_error_state = False 765 # A "308 Resume Incomplete" indicates we are not done. 766 self.resumable_progress = int(resp['range'].split('-')[1]) + 1 767 if 'location' in resp: 768 self.resumable_uri = resp['location'] 769 else: 770 self._in_error_state = True 771 raise HttpError(resp, content, self.uri) 772 773 return (MediaUploadProgress(self.resumable_progress, self.resumable.size()), 774 None)
775
776 - def to_json(self):
777 """Returns a JSON representation of the HttpRequest.""" 778 d = copy.copy(self.__dict__) 779 if d['resumable'] is not None: 780 d['resumable'] = self.resumable.to_json() 781 del d['http'] 782 del d['postproc'] 783 784 return simplejson.dumps(d)
785 786 @staticmethod
787 - def from_json(s, http, postproc):
788 """Returns an HttpRequest populated with info from a JSON object.""" 789 d = simplejson.loads(s) 790 if d['resumable'] is not None: 791 d['resumable'] = MediaUpload.new_from_json(d['resumable']) 792 return HttpRequest( 793 http, 794 postproc, 795 uri=d['uri'], 796 method=d['method'], 797 body=d['body'], 798 headers=d['headers'], 799 methodId=d['methodId'], 800 resumable=d['resumable'])
801
802 803 -class BatchHttpRequest(object):
804 """Batches multiple HttpRequest objects into a single HTTP request. 805 806 Example: 807 from apiclient.http import BatchHttpRequest 808 809 def list_animals(request_id, response): 810 \"\"\"Do something with the animals list response.\"\"\" 811 pass 812 813 def list_farmers(request_id, response): 814 \"\"\"Do something with the farmers list response.\"\"\" 815 pass 816 817 service = build('farm', 'v2') 818 819 batch = BatchHttpRequest() 820 821 batch.add(service.animals().list(), list_animals) 822 batch.add(service.farmers().list(), list_farmers) 823 batch.execute(http) 824 """ 825
826 - def __init__(self, callback=None, batch_uri=None):
827 """Constructor for a BatchHttpRequest. 828 829 Args: 830 callback: callable, A callback to be called for each response, of the 831 form callback(id, response). The first parameter is the request id, and 832 the second is the deserialized response object. 833 batch_uri: string, URI to send batch requests to. 834 """ 835 if batch_uri is None: 836 batch_uri = 'https://www.googleapis.com/batch' 837 self._batch_uri = batch_uri 838 839 # Global callback to be called for each individual response in the batch. 840 self._callback = callback 841 842 # A map from id to request. 843 self._requests = {} 844 845 # A map from id to callback. 846 self._callbacks = {} 847 848 # List of request ids, in the order in which they were added. 849 self._order = [] 850 851 # The last auto generated id. 852 self._last_auto_id = 0 853 854 # Unique ID on which to base the Content-ID headers. 855 self._base_id = None 856 857 # A map from request id to (headers, content) response pairs 858 self._responses = {} 859 860 # A map of id(Credentials) that have been refreshed. 861 self._refreshed_credentials = {}
862
863 - def _refresh_and_apply_credentials(self, request, http):
864 """Refresh the credentials and apply to the request. 865 866 Args: 867 request: HttpRequest, the request. 868 http: httplib2.Http, the global http object for the batch. 869 """ 870 # For the credentials to refresh, but only once per refresh_token 871 # If there is no http per the request then refresh the http passed in 872 # via execute() 873 creds = None 874 if request.http is not None and hasattr(request.http.request, 875 'credentials'): 876 creds = request.http.request.credentials 877 elif http is not None and hasattr(http.request, 'credentials'): 878 creds = http.request.credentials 879 if creds is not None: 880 if id(creds) not in self._refreshed_credentials: 881 creds.refresh(http) 882 self._refreshed_credentials[id(creds)] = 1 883 884 # Only apply the credentials if we are using the http object passed in, 885 # otherwise apply() will get called during _serialize_request(). 886 if request.http is None or not hasattr(request.http.request, 887 'credentials'): 888 creds.apply(request.headers)
889
890 - def _id_to_header(self, id_):
891 """Convert an id to a Content-ID header value. 892 893 Args: 894 id_: string, identifier of individual request. 895 896 Returns: 897 A Content-ID header with the id_ encoded into it. A UUID is prepended to 898 the value because Content-ID headers are supposed to be universally 899 unique. 900 """ 901 if self._base_id is None: 902 self._base_id = uuid.uuid4() 903 904 return '<%s+%s>' % (self._base_id, urllib.quote(id_))
905
906 - def _header_to_id(self, header):
907 """Convert a Content-ID header value to an id. 908 909 Presumes the Content-ID header conforms to the format that _id_to_header() 910 returns. 911 912 Args: 913 header: string, Content-ID header value. 914 915 Returns: 916 The extracted id value. 917 918 Raises: 919 BatchError if the header is not in the expected format. 920 """ 921 if header[0] != '<' or header[-1] != '>': 922 raise BatchError("Invalid value for Content-ID: %s" % header) 923 if '+' not in header: 924 raise BatchError("Invalid value for Content-ID: %s" % header) 925 base, id_ = header[1:-1].rsplit('+', 1) 926 927 return urllib.unquote(id_)
928
929 - def _serialize_request(self, request):
930 """Convert an HttpRequest object into a string. 931 932 Args: 933 request: HttpRequest, the request to serialize. 934 935 Returns: 936 The request as a string in application/http format. 937 """ 938 # Construct status line 939 parsed = urlparse.urlparse(request.uri) 940 request_line = urlparse.urlunparse( 941 (None, None, parsed.path, parsed.params, parsed.query, None) 942 ) 943 status_line = request.method + ' ' + request_line + ' HTTP/1.1\n' 944 major, minor = request.headers.get('content-type', 'application/json').split('/') 945 msg = MIMENonMultipart(major, minor) 946 headers = request.headers.copy() 947 948 if request.http is not None and hasattr(request.http.request, 949 'credentials'): 950 request.http.request.credentials.apply(headers) 951 952 # MIMENonMultipart adds its own Content-Type header. 953 if 'content-type' in headers: 954 del headers['content-type'] 955 956 for key, value in headers.iteritems(): 957 msg[key] = value 958 msg['Host'] = parsed.netloc 959 msg.set_unixfrom(None) 960 961 if request.body is not None: 962 msg.set_payload(request.body) 963 msg['content-length'] = str(len(request.body)) 964 965 # Serialize the mime message. 966 fp = StringIO.StringIO() 967 # maxheaderlen=0 means don't line wrap headers. 968 g = Generator(fp, maxheaderlen=0) 969 g.flatten(msg, unixfrom=False) 970 body = fp.getvalue() 971 972 # Strip off the \n\n that the MIME lib tacks onto the end of the payload. 973 if request.body is None: 974 body = body[:-2] 975 976 return status_line.encode('utf-8') + body
977
978 - def _deserialize_response(self, payload):
979 """Convert string into httplib2 response and content. 980 981 Args: 982 payload: string, headers and body as a string. 983 984 Returns: 985 A pair (resp, content) like would be returned from httplib2.request. 986 """ 987 # Strip off the status line 988 status_line, payload = payload.split('\n', 1) 989 protocol, status, reason = status_line.split(' ', 2) 990 991 # Parse the rest of the response 992 parser = FeedParser() 993 parser.feed(payload) 994 msg = parser.close() 995 msg['status'] = status 996 997 # Create httplib2.Response from the parsed headers. 998 resp = httplib2.Response(msg) 999 resp.reason = reason 1000 resp.version = int(protocol.split('/', 1)[1].replace('.', '')) 1001 1002 content = payload.split('\r\n\r\n', 1)[1] 1003 1004 return resp, content
1005
1006 - def _new_id(self):
1007 """Create a new id. 1008 1009 Auto incrementing number that avoids conflicts with ids already used. 1010 1011 Returns: 1012 string, a new unique id. 1013 """ 1014 self._last_auto_id += 1 1015 while str(self._last_auto_id) in self._requests: 1016 self._last_auto_id += 1 1017 return str(self._last_auto_id)
1018
1019 - def add(self, request, callback=None, request_id=None):
1020 """Add a new request. 1021 1022 Every callback added will be paired with a unique id, the request_id. That 1023 unique id will be passed back to the callback when the response comes back 1024 from the server. The default behavior is to have the library generate it's 1025 own unique id. If the caller passes in a request_id then they must ensure 1026 uniqueness for each request_id, and if they are not an exception is 1027 raised. Callers should either supply all request_ids or nevery supply a 1028 request id, to avoid such an error. 1029 1030 Args: 1031 request: HttpRequest, Request to add to the batch. 1032 callback: callable, A callback to be called for this response, of the 1033 form callback(id, response). The first parameter is the request id, and 1034 the second is the deserialized response object. 1035 request_id: string, A unique id for the request. The id will be passed to 1036 the callback with the response. 1037 1038 Returns: 1039 None 1040 1041 Raises: 1042 BatchError if a media request is added to a batch. 1043 KeyError is the request_id is not unique. 1044 """ 1045 if request_id is None: 1046 request_id = self._new_id() 1047 if request.resumable is not None: 1048 raise BatchError("Media requests cannot be used in a batch request.") 1049 if request_id in self._requests: 1050 raise KeyError("A request with this ID already exists: %s" % request_id) 1051 self._requests[request_id] = request 1052 self._callbacks[request_id] = callback 1053 self._order.append(request_id)
1054
1055 - def _execute(self, http, order, requests):
1056 """Serialize batch request, send to server, process response. 1057 1058 Args: 1059 http: httplib2.Http, an http object to be used to make the request with. 1060 order: list, list of request ids in the order they were added to the 1061 batch. 1062 request: list, list of request objects to send. 1063 1064 Raises: 1065 httplib2.Error if a transport error has occured. 1066 apiclient.errors.BatchError if the response is the wrong format. 1067 """ 1068 message = MIMEMultipart('mixed') 1069 # Message should not write out it's own headers. 1070 setattr(message, '_write_headers', lambda self: None) 1071 1072 # Add all the individual requests. 1073 for request_id in order: 1074 request = requests[request_id] 1075 1076 msg = MIMENonMultipart('application', 'http') 1077 msg['Content-Transfer-Encoding'] = 'binary' 1078 msg['Content-ID'] = self._id_to_header(request_id) 1079 1080 body = self._serialize_request(request) 1081 msg.set_payload(body) 1082 message.attach(msg) 1083 1084 body = message.as_string() 1085 1086 headers = {} 1087 headers['content-type'] = ('multipart/mixed; ' 1088 'boundary="%s"') % message.get_boundary() 1089 1090 resp, content = http.request(self._batch_uri, 'POST', body=body, 1091 headers=headers) 1092 1093 if resp.status >= 300: 1094 raise HttpError(resp, content, self._batch_uri) 1095 1096 # Now break out the individual responses and store each one. 1097 boundary, _ = content.split(None, 1) 1098 1099 # Prepend with a content-type header so FeedParser can handle it. 1100 header = 'content-type: %s\r\n\r\n' % resp['content-type'] 1101 for_parser = header + content 1102 1103 parser = FeedParser() 1104 parser.feed(for_parser) 1105 mime_response = parser.close() 1106 1107 if not mime_response.is_multipart(): 1108 raise BatchError("Response not in multipart/mixed format.", resp, 1109 content) 1110 1111 for part in mime_response.get_payload(): 1112 request_id = self._header_to_id(part['Content-ID']) 1113 headers, content = self._deserialize_response(part.get_payload()) 1114 self._responses[request_id] = (headers, content)
1115
1116 - def execute(self, http=None):
1117 """Execute all the requests as a single batched HTTP request. 1118 1119 Args: 1120 http: httplib2.Http, an http object to be used in place of the one the 1121 HttpRequest request object was constructed with. If one isn't supplied 1122 then use a http object from the requests in this batch. 1123 1124 Returns: 1125 None 1126 1127 Raises: 1128 httplib2.Error if a transport error has occured. 1129 apiclient.errors.BatchError if the response is the wrong format. 1130 """ 1131 1132 # If http is not supplied use the first valid one given in the requests. 1133 if http is None: 1134 for request_id in self._order: 1135 request = self._requests[request_id] 1136 if request is not None: 1137 http = request.http 1138 break 1139 1140 if http is None: 1141 raise ValueError("Missing a valid http object.") 1142 1143 self._execute(http, self._order, self._requests) 1144 1145 # Loop over all the requests and check for 401s. For each 401 request the 1146 # credentials should be refreshed and then sent again in a separate batch. 1147 redo_requests = {} 1148 redo_order = [] 1149 1150 for request_id in self._order: 1151 headers, content = self._responses[request_id] 1152 if headers['status'] == '401': 1153 redo_order.append(request_id) 1154 request = self._requests[request_id] 1155 self._refresh_and_apply_credentials(request, http) 1156 redo_requests[request_id] = request 1157 1158 if redo_requests: 1159 self._execute(http, redo_order, redo_requests) 1160 1161 # Now process all callbacks that are erroring, and raise an exception for 1162 # ones that return a non-2xx response? Or add extra parameter to callback 1163 # that contains an HttpError? 1164 1165 for request_id in self._order: 1166 headers, content = self._responses[request_id] 1167 1168 request = self._requests[request_id] 1169 callback = self._callbacks[request_id] 1170 1171 response = None 1172 exception = None 1173 try: 1174 r = httplib2.Response(headers) 1175 response = request.postproc(r, content) 1176 except HttpError, e: 1177 exception = e 1178 1179 if callback is not None: 1180 callback(request_id, response, exception) 1181 if self._callback is not None: 1182 self._callback(request_id, response, exception)
1183
1184 1185 -class HttpRequestMock(object):
1186 """Mock of HttpRequest. 1187 1188 Do not construct directly, instead use RequestMockBuilder. 1189 """ 1190
1191 - def __init__(self, resp, content, postproc):
1192 """Constructor for HttpRequestMock 1193 1194 Args: 1195 resp: httplib2.Response, the response to emulate coming from the request 1196 content: string, the response body 1197 postproc: callable, the post processing function usually supplied by 1198 the model class. See model.JsonModel.response() as an example. 1199 """ 1200 self.resp = resp 1201 self.content = content 1202 self.postproc = postproc 1203 if resp is None: 1204 self.resp = httplib2.Response({'status': 200, 'reason': 'OK'}) 1205 if 'reason' in self.resp: 1206 self.resp.reason = self.resp['reason']
1207
1208 - def execute(self, http=None):
1209 """Execute the request. 1210 1211 Same behavior as HttpRequest.execute(), but the response is 1212 mocked and not really from an HTTP request/response. 1213 """ 1214 return self.postproc(self.resp, self.content)
1215
1216 1217 -class RequestMockBuilder(object):
1218 """A simple mock of HttpRequest 1219 1220 Pass in a dictionary to the constructor that maps request methodIds to 1221 tuples of (httplib2.Response, content, opt_expected_body) that should be 1222 returned when that method is called. None may also be passed in for the 1223 httplib2.Response, in which case a 200 OK response will be generated. 1224 If an opt_expected_body (str or dict) is provided, it will be compared to 1225 the body and UnexpectedBodyError will be raised on inequality. 1226 1227 Example: 1228 response = '{"data": {"id": "tag:google.c...' 1229 requestBuilder = RequestMockBuilder( 1230 { 1231 'plus.activities.get': (None, response), 1232 } 1233 ) 1234 apiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder) 1235 1236 Methods that you do not supply a response for will return a 1237 200 OK with an empty string as the response content or raise an excpetion 1238 if check_unexpected is set to True. The methodId is taken from the rpcName 1239 in the discovery document. 1240 1241 For more details see the project wiki. 1242 """ 1243
1244 - def __init__(self, responses, check_unexpected=False):
1245 """Constructor for RequestMockBuilder 1246 1247 The constructed object should be a callable object 1248 that can replace the class HttpResponse. 1249 1250 responses - A dictionary that maps methodIds into tuples 1251 of (httplib2.Response, content). The methodId 1252 comes from the 'rpcName' field in the discovery 1253 document. 1254 check_unexpected - A boolean setting whether or not UnexpectedMethodError 1255 should be raised on unsupplied method. 1256 """ 1257 self.responses = responses 1258 self.check_unexpected = check_unexpected
1259
1260 - def __call__(self, http, postproc, uri, method='GET', body=None, 1261 headers=None, methodId=None, resumable=None):
1262 """Implements the callable interface that discovery.build() expects 1263 of requestBuilder, which is to build an object compatible with 1264 HttpRequest.execute(). See that method for the description of the 1265 parameters and the expected response. 1266 """ 1267 if methodId in self.responses: 1268 response = self.responses[methodId] 1269 resp, content = response[:2] 1270 if len(response) > 2: 1271 # Test the body against the supplied expected_body. 1272 expected_body = response[2] 1273 if bool(expected_body) != bool(body): 1274 # Not expecting a body and provided one 1275 # or expecting a body and not provided one. 1276 raise UnexpectedBodyError(expected_body, body) 1277 if isinstance(expected_body, str): 1278 expected_body = simplejson.loads(expected_body) 1279 body = simplejson.loads(body) 1280 if body != expected_body: 1281 raise UnexpectedBodyError(expected_body, body) 1282 return HttpRequestMock(resp, content, postproc) 1283 elif self.check_unexpected: 1284 raise UnexpectedMethodError(methodId) 1285 else: 1286 model = JsonModel(False) 1287 return HttpRequestMock(None, '{}', model.response)
1288
1289 1290 -class HttpMock(object):
1291 """Mock of httplib2.Http""" 1292
1293 - def __init__(self, filename, headers=None):
1294 """ 1295 Args: 1296 filename: string, absolute filename to read response from 1297 headers: dict, header to return with response 1298 """ 1299 if headers is None: 1300 headers = {'status': '200 OK'} 1301 f = file(filename, 'r') 1302 self.data = f.read() 1303 f.close() 1304 self.headers = headers
1305
1306 - def request(self, uri, 1307 method='GET', 1308 body=None, 1309 headers=None, 1310 redirections=1, 1311 connection_type=None):
1312 return httplib2.Response(self.headers), self.data
1313
1314 1315 -class HttpMockSequence(object):
1316 """Mock of httplib2.Http 1317 1318 Mocks a sequence of calls to request returning different responses for each 1319 call. Create an instance initialized with the desired response headers 1320 and content and then use as if an httplib2.Http instance. 1321 1322 http = HttpMockSequence([ 1323 ({'status': '401'}, ''), 1324 ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'), 1325 ({'status': '200'}, 'echo_request_headers'), 1326 ]) 1327 resp, content = http.request("http://examples.com") 1328 1329 There are special values you can pass in for content to trigger 1330 behavours that are helpful in testing. 1331 1332 'echo_request_headers' means return the request headers in the response body 1333 'echo_request_headers_as_json' means return the request headers in 1334 the response body 1335 'echo_request_body' means return the request body in the response body 1336 'echo_request_uri' means return the request uri in the response body 1337 """ 1338
1339 - def __init__(self, iterable):
1340 """ 1341 Args: 1342 iterable: iterable, a sequence of pairs of (headers, body) 1343 """ 1344 self._iterable = iterable 1345 self.follow_redirects = True
1346
1347 - def request(self, uri, 1348 method='GET', 1349 body=None, 1350 headers=None, 1351 redirections=1, 1352 connection_type=None):
1353 resp, content = self._iterable.pop(0) 1354 if content == 'echo_request_headers': 1355 content = headers 1356 elif content == 'echo_request_headers_as_json': 1357 content = simplejson.dumps(headers) 1358 elif content == 'echo_request_body': 1359 content = body 1360 elif content == 'echo_request_uri': 1361 content = uri 1362 return httplib2.Response(resp), content
1363
1364 1365 -def set_user_agent(http, user_agent):
1366 """Set the user-agent on every request. 1367 1368 Args: 1369 http - An instance of httplib2.Http 1370 or something that acts like it. 1371 user_agent: string, the value for the user-agent header. 1372 1373 Returns: 1374 A modified instance of http that was passed in. 1375 1376 Example: 1377 1378 h = httplib2.Http() 1379 h = set_user_agent(h, "my-app-name/6.0") 1380 1381 Most of the time the user-agent will be set doing auth, this is for the rare 1382 cases where you are accessing an unauthenticated endpoint. 1383 """ 1384 request_orig = http.request 1385 1386 # The closure that will replace 'httplib2.Http.request'. 1387 def new_request(uri, method='GET', body=None, headers=None, 1388 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1389 connection_type=None): 1390 """Modify the request headers to add the user-agent.""" 1391 if headers is None: 1392 headers = {} 1393 if 'user-agent' in headers: 1394 headers['user-agent'] = user_agent + ' ' + headers['user-agent'] 1395 else: 1396 headers['user-agent'] = user_agent 1397 resp, content = request_orig(uri, method, body, headers, 1398 redirections, connection_type) 1399 return resp, content
1400 1401 http.request = new_request 1402 return http 1403
1404 1405 -def tunnel_patch(http):
1406 """Tunnel PATCH requests over POST. 1407 Args: 1408 http - An instance of httplib2.Http 1409 or something that acts like it. 1410 1411 Returns: 1412 A modified instance of http that was passed in. 1413 1414 Example: 1415 1416 h = httplib2.Http() 1417 h = tunnel_patch(h, "my-app-name/6.0") 1418 1419 Useful if you are running on a platform that doesn't support PATCH. 1420 Apply this last if you are using OAuth 1.0, as changing the method 1421 will result in a different signature. 1422 """ 1423 request_orig = http.request 1424 1425 # The closure that will replace 'httplib2.Http.request'. 1426 def new_request(uri, method='GET', body=None, headers=None, 1427 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1428 connection_type=None): 1429 """Modify the request headers to add the user-agent.""" 1430 if headers is None: 1431 headers = {} 1432 if method == 'PATCH': 1433 if 'oauth_token' in headers.get('authorization', ''): 1434 logging.warning( 1435 'OAuth 1.0 request made with Credentials after tunnel_patch.') 1436 headers['x-http-method-override'] = "PATCH" 1437 method = 'POST' 1438 resp, content = request_orig(uri, method, body, headers, 1439 redirections, connection_type) 1440 return resp, content
1441 1442 http.request = new_request 1443 return http 1444