Package apiclient :: Module http
[hide private]
[frames] | no frames]

Source Code for Module apiclient.http

   1  # Copyright (C) 2012 Google Inc. 
   2  # 
   3  # Licensed under the Apache License, Version 2.0 (the "License"); 
   4  # you may not use this file except in compliance with the License. 
   5  # You may obtain a copy of the License at 
   6  # 
   7  #      http://www.apache.org/licenses/LICENSE-2.0 
   8  # 
   9  # Unless required by applicable law or agreed to in writing, software 
  10  # distributed under the License is distributed on an "AS IS" BASIS, 
  11  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
  12  # See the License for the specific language governing permissions and 
  13  # limitations under the License. 
  14   
  15  """Classes to encapsulate a single HTTP request. 
  16   
  17  The classes implement a command pattern, with every 
  18  object supporting an execute() method that does the 
  19  actuall HTTP request. 
  20  """ 
  21   
  22  __author__ = 'jcgregorio@google.com (Joe Gregorio)' 
  23   
  24  import StringIO 
  25  import base64 
  26  import copy 
  27  import gzip 
  28  import httplib2 
  29  import mimeparse 
  30  import mimetypes 
  31  import os 
  32  import urllib 
  33  import urlparse 
  34  import uuid 
  35   
  36  from email.generator import Generator 
  37  from email.mime.multipart import MIMEMultipart 
  38  from email.mime.nonmultipart import MIMENonMultipart 
  39  from email.parser import FeedParser 
  40  from errors import BatchError 
  41  from errors import HttpError 
  42  from errors import ResumableUploadError 
  43  from errors import UnexpectedBodyError 
  44  from errors import UnexpectedMethodError 
  45  from model import JsonModel 
  46  from oauth2client.anyjson import simplejson 
  47   
  48   
  49  DEFAULT_CHUNK_SIZE = 512*1024 
50 51 52 -class MediaUploadProgress(object):
53 """Status of a resumable upload.""" 54
55 - def __init__(self, resumable_progress, total_size):
56 """Constructor. 57 58 Args: 59 resumable_progress: int, bytes sent so far. 60 total_size: int, total bytes in complete upload, or None if the total 61 upload size isn't known ahead of time. 62 """ 63 self.resumable_progress = resumable_progress 64 self.total_size = total_size
65
66 - def progress(self):
67 """Percent of upload completed, as a float. 68 69 Returns: 70 the percentage complete as a float, returning 0.0 if the total size of 71 the upload is unknown. 72 """ 73 if self.total_size is not None: 74 return float(self.resumable_progress) / float(self.total_size) 75 else: 76 return 0.0
77
78 79 -class MediaDownloadProgress(object):
80 """Status of a resumable download.""" 81
82 - def __init__(self, resumable_progress, total_size):
83 """Constructor. 84 85 Args: 86 resumable_progress: int, bytes received so far. 87 total_size: int, total bytes in complete download. 88 """ 89 self.resumable_progress = resumable_progress 90 self.total_size = total_size
91
92 - def progress(self):
93 """Percent of download completed, as a float. 94 95 Returns: 96 the percentage complete as a float, returning 0.0 if the total size of 97 the download is unknown. 98 """ 99 if self.total_size is not None: 100 return float(self.resumable_progress) / float(self.total_size) 101 else: 102 return 0.0
103
104 105 -class MediaUpload(object):
106 """Describes a media object to upload. 107 108 Base class that defines the interface of MediaUpload subclasses. 109 110 Note that subclasses of MediaUpload may allow you to control the chunksize 111 when upload a media object. It is important to keep the size of the chunk as 112 large as possible to keep the upload efficient. Other factors may influence 113 the size of the chunk you use, particularly if you are working in an 114 environment where individual HTTP requests may have a hardcoded time limit, 115 such as under certain classes of requests under Google App Engine. 116 """ 117
118 - def chunksize(self):
119 """Chunk size for resumable uploads. 120 121 Returns: 122 Chunk size in bytes. 123 """ 124 raise NotImplementedError()
125
126 - def mimetype(self):
127 """Mime type of the body. 128 129 Returns: 130 Mime type. 131 """ 132 return 'application/octet-stream'
133
134 - def size(self):
135 """Size of upload. 136 137 Returns: 138 Size of the body, or None of the size is unknown. 139 """ 140 return None
141
142 - def resumable(self):
143 """Whether this upload is resumable. 144 145 Returns: 146 True if resumable upload or False. 147 """ 148 return False
149
150 - def getbytes(self, begin, end):
151 """Get bytes from the media. 152 153 Args: 154 begin: int, offset from beginning of file. 155 length: int, number of bytes to read, starting at begin. 156 157 Returns: 158 A string of bytes read. May be shorter than length if EOF was reached 159 first. 160 """ 161 raise NotImplementedError()
162
163 - def _to_json(self, strip=None):
164 """Utility function for creating a JSON representation of a MediaUpload. 165 166 Args: 167 strip: array, An array of names of members to not include in the JSON. 168 169 Returns: 170 string, a JSON representation of this instance, suitable to pass to 171 from_json(). 172 """ 173 t = type(self) 174 d = copy.copy(self.__dict__) 175 if strip is not None: 176 for member in strip: 177 del d[member] 178 d['_class'] = t.__name__ 179 d['_module'] = t.__module__ 180 return simplejson.dumps(d)
181
182 - def to_json(self):
183 """Create a JSON representation of an instance of MediaUpload. 184 185 Returns: 186 string, a JSON representation of this instance, suitable to pass to 187 from_json(). 188 """ 189 return self._to_json()
190 191 @classmethod
192 - def new_from_json(cls, s):
193 """Utility class method to instantiate a MediaUpload subclass from a JSON 194 representation produced by to_json(). 195 196 Args: 197 s: string, JSON from to_json(). 198 199 Returns: 200 An instance of the subclass of MediaUpload that was serialized with 201 to_json(). 202 """ 203 data = simplejson.loads(s) 204 # Find and call the right classmethod from_json() to restore the object. 205 module = data['_module'] 206 m = __import__(module, fromlist=module.split('.')[:-1]) 207 kls = getattr(m, data['_class']) 208 from_json = getattr(kls, 'from_json') 209 return from_json(s)
210
211 212 -class MediaFileUpload(MediaUpload):
213 """A MediaUpload for a file. 214 215 Construct a MediaFileUpload and pass as the media_body parameter of the 216 method. For example, if we had a service that allowed uploading images: 217 218 219 media = MediaFileUpload('cow.png', mimetype='image/png', 220 chunksize=1024*1024, resumable=True) 221 farm.animals()..insert( 222 id='cow', 223 name='cow.png', 224 media_body=media).execute() 225 """ 226
227 - def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE, resumable=False):
228 """Constructor. 229 230 Args: 231 filename: string, Name of the file. 232 mimetype: string, Mime-type of the file. If None then a mime-type will be 233 guessed from the file extension. 234 chunksize: int, File will be uploaded in chunks of this many bytes. Only 235 used if resumable=True. 236 resumable: bool, True if this is a resumable upload. False means upload 237 in a single request. 238 """ 239 self._filename = filename 240 self._size = os.path.getsize(filename) 241 self._fd = None 242 if mimetype is None: 243 (mimetype, encoding) = mimetypes.guess_type(filename) 244 self._mimetype = mimetype 245 self._chunksize = chunksize 246 self._resumable = resumable
247
248 - def chunksize(self):
249 """Chunk size for resumable uploads. 250 251 Returns: 252 Chunk size in bytes. 253 """ 254 return self._chunksize
255
256 - def mimetype(self):
257 """Mime type of the body. 258 259 Returns: 260 Mime type. 261 """ 262 return self._mimetype
263
264 - def size(self):
265 """Size of upload. 266 267 Returns: 268 Size of the body, or None of the size is unknown. 269 """ 270 return self._size
271
272 - def resumable(self):
273 """Whether this upload is resumable. 274 275 Returns: 276 True if resumable upload or False. 277 """ 278 return self._resumable
279
280 - def getbytes(self, begin, length):
281 """Get bytes from the media. 282 283 Args: 284 begin: int, offset from beginning of file. 285 length: int, number of bytes to read, starting at begin. 286 287 Returns: 288 A string of bytes read. May be shorted than length if EOF was reached 289 first. 290 """ 291 if self._fd is None: 292 self._fd = open(self._filename, 'rb') 293 self._fd.seek(begin) 294 return self._fd.read(length)
295
296 - def to_json(self):
297 """Creating a JSON representation of an instance of MediaFileUpload. 298 299 Returns: 300 string, a JSON representation of this instance, suitable to pass to 301 from_json(). 302 """ 303 return self._to_json(['_fd'])
304 305 @staticmethod
306 - def from_json(s):
307 d = simplejson.loads(s) 308 return MediaFileUpload( 309 d['_filename'], d['_mimetype'], d['_chunksize'], d['_resumable'])
310
311 312 -class MediaIoBaseUpload(MediaUpload):
313 """A MediaUpload for a io.Base objects. 314 315 Note that the Python file object is compatible with io.Base and can be used 316 with this class also. 317 318 fh = io.BytesIO('...Some data to upload...') 319 media = MediaIoBaseUpload(fh, mimetype='image/png', 320 chunksize=1024*1024, resumable=True) 321 farm.animals().insert( 322 id='cow', 323 name='cow.png', 324 media_body=media).execute() 325 """ 326
327 - def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE, 328 resumable=False):
329 """Constructor. 330 331 Args: 332 fd: io.Base or file object, The source of the bytes to upload. MUST be 333 opened in blocking mode, do not use streams opened in non-blocking mode. 334 mimetype: string, Mime-type of the file. If None then a mime-type will be 335 guessed from the file extension. 336 chunksize: int, File will be uploaded in chunks of this many bytes. Only 337 used if resumable=True. 338 resumable: bool, True if this is a resumable upload. False means upload 339 in a single request. 340 """ 341 self._fd = fd 342 self._mimetype = mimetype 343 self._chunksize = chunksize 344 self._resumable = resumable 345 self._size = None 346 try: 347 if hasattr(self._fd, 'fileno'): 348 fileno = self._fd.fileno() 349 350 # Pipes and such show up as 0 length files. 351 size = os.fstat(fileno).st_size 352 if size: 353 self._size = os.fstat(fileno).st_size 354 except IOError: 355 pass
356
357 - def chunksize(self):
358 """Chunk size for resumable uploads. 359 360 Returns: 361 Chunk size in bytes. 362 """ 363 return self._chunksize
364
365 - def mimetype(self):
366 """Mime type of the body. 367 368 Returns: 369 Mime type. 370 """ 371 return self._mimetype
372
373 - def size(self):
374 """Size of upload. 375 376 Returns: 377 Size of the body, or None of the size is unknown. 378 """ 379 return self._size
380
381 - def resumable(self):
382 """Whether this upload is resumable. 383 384 Returns: 385 True if resumable upload or False. 386 """ 387 return self._resumable
388
389 - def getbytes(self, begin, length):
390 """Get bytes from the media. 391 392 Args: 393 begin: int, offset from beginning of file. 394 length: int, number of bytes to read, starting at begin. 395 396 Returns: 397 A string of bytes read. May be shorted than length if EOF was reached 398 first. 399 """ 400 self._fd.seek(begin) 401 return self._fd.read(length)
402
403 - def to_json(self):
404 """This upload type is not serializable.""" 405 raise NotImplementedError('MediaIoBaseUpload is not serializable.')
406
407 408 -class MediaInMemoryUpload(MediaUpload):
409 """MediaUpload for a chunk of bytes. 410 411 Construct a MediaFileUpload and pass as the media_body parameter of the 412 method. 413 """ 414
415 - def __init__(self, body, mimetype='application/octet-stream', 416 chunksize=DEFAULT_CHUNK_SIZE, resumable=False):
417 """Create a new MediaBytesUpload. 418 419 Args: 420 body: string, Bytes of body content. 421 mimetype: string, Mime-type of the file or default of 422 'application/octet-stream'. 423 chunksize: int, File will be uploaded in chunks of this many bytes. Only 424 used if resumable=True. 425 resumable: bool, True if this is a resumable upload. False means upload 426 in a single request. 427 """ 428 self._body = body 429 self._mimetype = mimetype 430 self._resumable = resumable 431 self._chunksize = chunksize
432
433 - def chunksize(self):
434 """Chunk size for resumable uploads. 435 436 Returns: 437 Chunk size in bytes. 438 """ 439 return self._chunksize
440
441 - def mimetype(self):
442 """Mime type of the body. 443 444 Returns: 445 Mime type. 446 """ 447 return self._mimetype
448
449 - def size(self):
450 """Size of upload. 451 452 Returns: 453 Size of the body, or None of the size is unknown. 454 """ 455 return len(self._body)
456
457 - def resumable(self):
458 """Whether this upload is resumable. 459 460 Returns: 461 True if resumable upload or False. 462 """ 463 return self._resumable
464
465 - def getbytes(self, begin, length):
466 """Get bytes from the media. 467 468 Args: 469 begin: int, offset from beginning of file. 470 length: int, number of bytes to read, starting at begin. 471 472 Returns: 473 A string of bytes read. May be shorter than length if EOF was reached 474 first. 475 """ 476 return self._body[begin:begin + length]
477
478 - def to_json(self):
479 """Create a JSON representation of a MediaInMemoryUpload. 480 481 Returns: 482 string, a JSON representation of this instance, suitable to pass to 483 from_json(). 484 """ 485 t = type(self) 486 d = copy.copy(self.__dict__) 487 del d['_body'] 488 d['_class'] = t.__name__ 489 d['_module'] = t.__module__ 490 d['_b64body'] = base64.b64encode(self._body) 491 return simplejson.dumps(d)
492 493 @staticmethod
494 - def from_json(s):
495 d = simplejson.loads(s) 496 return MediaInMemoryUpload(base64.b64decode(d['_b64body']), 497 d['_mimetype'], d['_chunksize'], 498 d['_resumable'])
499
500 501 -class MediaIoBaseDownload(object):
502 """"Download media resources. 503 504 Note that the Python file object is compatible with io.Base and can be used 505 with this class also. 506 507 508 Example: 509 request = farms.animals().get_media(id='cow') 510 fh = io.FileIO('cow.png', mode='wb') 511 downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024) 512 513 done = False 514 while done is False: 515 status, done = downloader.next_chunk() 516 if status: 517 print "Download %d%%." % int(status.progress() * 100) 518 print "Download Complete!" 519 """ 520
521 - def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE):
522 """Constructor. 523 524 Args: 525 fd: io.Base or file object, The stream in which to write the downloaded 526 bytes. 527 request: apiclient.http.HttpRequest, the media request to perform in 528 chunks. 529 chunksize: int, File will be downloaded in chunks of this many bytes. 530 """ 531 self._fd = fd 532 self._request = request 533 self._uri = request.uri 534 self._chunksize = chunksize 535 self._progress = 0 536 self._total_size = None 537 self._done = False
538
539 - def next_chunk(self):
540 """Get the next chunk of the download. 541 542 Returns: 543 (status, done): (MediaDownloadStatus, boolean) 544 The value of 'done' will be True when the media has been fully 545 downloaded. 546 547 Raises: 548 apiclient.errors.HttpError if the response was not a 2xx. 549 httplib2.HttpLib2Error if a transport error has occured. 550 """ 551 headers = { 552 'range': 'bytes=%d-%d' % ( 553 self._progress, self._progress + self._chunksize) 554 } 555 http = self._request.http 556 http.follow_redirects = False 557 558 resp, content = http.request(self._uri, headers=headers) 559 if resp.status in [301, 302, 303, 307, 308] and 'location' in resp: 560 self._uri = resp['location'] 561 resp, content = http.request(self._uri, headers=headers) 562 if resp.status in [200, 206]: 563 self._progress += len(content) 564 self._fd.write(content) 565 566 if 'content-range' in resp: 567 content_range = resp['content-range'] 568 length = content_range.rsplit('/', 1)[1] 569 self._total_size = int(length) 570 571 if self._progress == self._total_size: 572 self._done = True 573 return MediaDownloadProgress(self._progress, self._total_size), self._done 574 else: 575 raise HttpError(resp, content, self._uri)
576
577 578 -class HttpRequest(object):
579 """Encapsulates a single HTTP request.""" 580
581 - def __init__(self, http, postproc, uri, 582 method='GET', 583 body=None, 584 headers=None, 585 methodId=None, 586 resumable=None):
587 """Constructor for an HttpRequest. 588 589 Args: 590 http: httplib2.Http, the transport object to use to make a request 591 postproc: callable, called on the HTTP response and content to transform 592 it into a data object before returning, or raising an exception 593 on an error. 594 uri: string, the absolute URI to send the request to 595 method: string, the HTTP method to use 596 body: string, the request body of the HTTP request, 597 headers: dict, the HTTP request headers 598 methodId: string, a unique identifier for the API method being called. 599 resumable: MediaUpload, None if this is not a resumbale request. 600 """ 601 self.uri = uri 602 self.method = method 603 self.body = body 604 self.headers = headers or {} 605 self.methodId = methodId 606 self.http = http 607 self.postproc = postproc 608 self.resumable = resumable 609 self._in_error_state = False 610 611 # Pull the multipart boundary out of the content-type header. 612 major, minor, params = mimeparse.parse_mime_type( 613 headers.get('content-type', 'application/json')) 614 615 # The size of the non-media part of the request. 616 self.body_size = len(self.body or '') 617 618 # The resumable URI to send chunks to. 619 self.resumable_uri = None 620 621 # The bytes that have been uploaded. 622 self.resumable_progress = 0
623
624 - def execute(self, http=None):
625 """Execute the request. 626 627 Args: 628 http: httplib2.Http, an http object to be used in place of the 629 one the HttpRequest request object was constructed with. 630 631 Returns: 632 A deserialized object model of the response body as determined 633 by the postproc. 634 635 Raises: 636 apiclient.errors.HttpError if the response was not a 2xx. 637 httplib2.HttpLib2Error if a transport error has occured. 638 """ 639 if http is None: 640 http = self.http 641 if self.resumable: 642 body = None 643 while body is None: 644 _, body = self.next_chunk(http) 645 return body 646 else: 647 if 'content-length' not in self.headers: 648 self.headers['content-length'] = str(self.body_size) 649 resp, content = http.request(self.uri, self.method, 650 body=self.body, 651 headers=self.headers) 652 if resp.status >= 300: 653 raise HttpError(resp, content, self.uri) 654 return self.postproc(resp, content)
655
656 - def next_chunk(self, http=None):
657 """Execute the next step of a resumable upload. 658 659 Can only be used if the method being executed supports media uploads and 660 the MediaUpload object passed in was flagged as using resumable upload. 661 662 Example: 663 664 media = MediaFileUpload('cow.png', mimetype='image/png', 665 chunksize=1000, resumable=True) 666 request = farm.animals().insert( 667 id='cow', 668 name='cow.png', 669 media_body=media) 670 671 response = None 672 while response is None: 673 status, response = request.next_chunk() 674 if status: 675 print "Upload %d%% complete." % int(status.progress() * 100) 676 677 678 Returns: 679 (status, body): (ResumableMediaStatus, object) 680 The body will be None until the resumable media is fully uploaded. 681 682 Raises: 683 apiclient.errors.HttpError if the response was not a 2xx. 684 httplib2.HttpLib2Error if a transport error has occured. 685 """ 686 if http is None: 687 http = self.http 688 689 if self.resumable.size() is None: 690 size = '*' 691 else: 692 size = str(self.resumable.size()) 693 694 if self.resumable_uri is None: 695 start_headers = copy.copy(self.headers) 696 start_headers['X-Upload-Content-Type'] = self.resumable.mimetype() 697 if size != '*': 698 start_headers['X-Upload-Content-Length'] = size 699 start_headers['content-length'] = str(self.body_size) 700 701 resp, content = http.request(self.uri, self.method, 702 body=self.body, 703 headers=start_headers) 704 if resp.status == 200 and 'location' in resp: 705 self.resumable_uri = resp['location'] 706 else: 707 raise ResumableUploadError("Failed to retrieve starting URI.") 708 elif self._in_error_state: 709 # If we are in an error state then query the server for current state of 710 # the upload by sending an empty PUT and reading the 'range' header in 711 # the response. 712 headers = { 713 'Content-Range': 'bytes */%s' % size, 714 'content-length': '0' 715 } 716 resp, content = http.request(self.resumable_uri, 'PUT', 717 headers=headers) 718 status, body = self._process_response(resp, content) 719 if body: 720 # The upload was complete. 721 return (status, body) 722 723 data = self.resumable.getbytes( 724 self.resumable_progress, self.resumable.chunksize()) 725 726 # A short read implies that we are at EOF, so finish the upload. 727 if len(data) < self.resumable.chunksize(): 728 size = str(self.resumable_progress + len(data)) 729 730 headers = { 731 'Content-Range': 'bytes %d-%d/%s' % ( 732 self.resumable_progress, self.resumable_progress + len(data) - 1, 733 size) 734 } 735 try: 736 resp, content = http.request(self.resumable_uri, 'PUT', 737 body=data, 738 headers=headers) 739 except: 740 self._in_error_state = True 741 raise 742 743 return self._process_response(resp, content)
744
745 - def _process_response(self, resp, content):
746 """Process the response from a single chunk upload. 747 748 Args: 749 resp: httplib2.Response, the response object. 750 content: string, the content of the response. 751 752 Returns: 753 (status, body): (ResumableMediaStatus, object) 754 The body will be None until the resumable media is fully uploaded. 755 756 Raises: 757 apiclient.errors.HttpError if the response was not a 2xx or a 308. 758 """ 759 if resp.status in [200, 201]: 760 self._in_error_state = False 761 return None, self.postproc(resp, content) 762 elif resp.status == 308: 763 self._in_error_state = False 764 # A "308 Resume Incomplete" indicates we are not done. 765 self.resumable_progress = int(resp['range'].split('-')[1]) + 1 766 if 'location' in resp: 767 self.resumable_uri = resp['location'] 768 else: 769 self._in_error_state = True 770 raise HttpError(resp, content, self.uri) 771 772 return (MediaUploadProgress(self.resumable_progress, self.resumable.size()), 773 None)
774
775 - def to_json(self):
776 """Returns a JSON representation of the HttpRequest.""" 777 d = copy.copy(self.__dict__) 778 if d['resumable'] is not None: 779 d['resumable'] = self.resumable.to_json() 780 del d['http'] 781 del d['postproc'] 782 783 return simplejson.dumps(d)
784 785 @staticmethod
786 - def from_json(s, http, postproc):
787 """Returns an HttpRequest populated with info from a JSON object.""" 788 d = simplejson.loads(s) 789 if d['resumable'] is not None: 790 d['resumable'] = MediaUpload.new_from_json(d['resumable']) 791 return HttpRequest( 792 http, 793 postproc, 794 uri=d['uri'], 795 method=d['method'], 796 body=d['body'], 797 headers=d['headers'], 798 methodId=d['methodId'], 799 resumable=d['resumable'])
800
801 802 -class BatchHttpRequest(object):
803 """Batches multiple HttpRequest objects into a single HTTP request. 804 805 Example: 806 from apiclient.http import BatchHttpRequest 807 808 def list_animals(request_id, response, exception): 809 \"\"\"Do something with the animals list response.\"\"\" 810 if exception is not None: 811 # Do something with the exception. 812 pass 813 else: 814 # Do something with the response. 815 pass 816 817 def list_farmers(request_id, response, exception): 818 \"\"\"Do something with the farmers list response.\"\"\" 819 if exception is not None: 820 # Do something with the exception. 821 pass 822 else: 823 # Do something with the response. 824 pass 825 826 service = build('farm', 'v2') 827 828 batch = BatchHttpRequest() 829 830 batch.add(service.animals().list(), list_animals) 831 batch.add(service.farmers().list(), list_farmers) 832 batch.execute(http) 833 """ 834
835 - def __init__(self, callback=None, batch_uri=None):
836 """Constructor for a BatchHttpRequest. 837 838 Args: 839 callback: callable, A callback to be called for each response, of the 840 form callback(id, response, exception). The first parameter is the 841 request id, and the second is the deserialized response object. The 842 third is an apiclient.errors.HttpError exception object if an HTTP error 843 occurred while processing the request, or None if no error occurred. 844 batch_uri: string, URI to send batch requests to. 845 """ 846 if batch_uri is None: 847 batch_uri = 'https://www.googleapis.com/batch' 848 self._batch_uri = batch_uri 849 850 # Global callback to be called for each individual response in the batch. 851 self._callback = callback 852 853 # A map from id to request. 854 self._requests = {} 855 856 # A map from id to callback. 857 self._callbacks = {} 858 859 # List of request ids, in the order in which they were added. 860 self._order = [] 861 862 # The last auto generated id. 863 self._last_auto_id = 0 864 865 # Unique ID on which to base the Content-ID headers. 866 self._base_id = None 867 868 # A map from request id to (httplib2.Response, content) response pairs 869 self._responses = {} 870 871 # A map of id(Credentials) that have been refreshed. 872 self._refreshed_credentials = {}
873
874 - def _refresh_and_apply_credentials(self, request, http):
875 """Refresh the credentials and apply to the request. 876 877 Args: 878 request: HttpRequest, the request. 879 http: httplib2.Http, the global http object for the batch. 880 """ 881 # For the credentials to refresh, but only once per refresh_token 882 # If there is no http per the request then refresh the http passed in 883 # via execute() 884 creds = None 885 if request.http is not None and hasattr(request.http.request, 886 'credentials'): 887 creds = request.http.request.credentials 888 elif http is not None and hasattr(http.request, 'credentials'): 889 creds = http.request.credentials 890 if creds is not None: 891 if id(creds) not in self._refreshed_credentials: 892 creds.refresh(http) 893 self._refreshed_credentials[id(creds)] = 1 894 895 # Only apply the credentials if we are using the http object passed in, 896 # otherwise apply() will get called during _serialize_request(). 897 if request.http is None or not hasattr(request.http.request, 898 'credentials'): 899 creds.apply(request.headers)
900
901 - def _id_to_header(self, id_):
902 """Convert an id to a Content-ID header value. 903 904 Args: 905 id_: string, identifier of individual request. 906 907 Returns: 908 A Content-ID header with the id_ encoded into it. A UUID is prepended to 909 the value because Content-ID headers are supposed to be universally 910 unique. 911 """ 912 if self._base_id is None: 913 self._base_id = uuid.uuid4() 914 915 return '<%s+%s>' % (self._base_id, urllib.quote(id_))
916
917 - def _header_to_id(self, header):
918 """Convert a Content-ID header value to an id. 919 920 Presumes the Content-ID header conforms to the format that _id_to_header() 921 returns. 922 923 Args: 924 header: string, Content-ID header value. 925 926 Returns: 927 The extracted id value. 928 929 Raises: 930 BatchError if the header is not in the expected format. 931 """ 932 if header[0] != '<' or header[-1] != '>': 933 raise BatchError("Invalid value for Content-ID: %s" % header) 934 if '+' not in header: 935 raise BatchError("Invalid value for Content-ID: %s" % header) 936 base, id_ = header[1:-1].rsplit('+', 1) 937 938 return urllib.unquote(id_)
939
940 - def _serialize_request(self, request):
941 """Convert an HttpRequest object into a string. 942 943 Args: 944 request: HttpRequest, the request to serialize. 945 946 Returns: 947 The request as a string in application/http format. 948 """ 949 # Construct status line 950 parsed = urlparse.urlparse(request.uri) 951 request_line = urlparse.urlunparse( 952 (None, None, parsed.path, parsed.params, parsed.query, None) 953 ) 954 status_line = request.method + ' ' + request_line + ' HTTP/1.1\n' 955 major, minor = request.headers.get('content-type', 'application/json').split('/') 956 msg = MIMENonMultipart(major, minor) 957 headers = request.headers.copy() 958 959 if request.http is not None and hasattr(request.http.request, 960 'credentials'): 961 request.http.request.credentials.apply(headers) 962 963 # MIMENonMultipart adds its own Content-Type header. 964 if 'content-type' in headers: 965 del headers['content-type'] 966 967 for key, value in headers.iteritems(): 968 msg[key] = value 969 msg['Host'] = parsed.netloc 970 msg.set_unixfrom(None) 971 972 if request.body is not None: 973 msg.set_payload(request.body) 974 msg['content-length'] = str(len(request.body)) 975 976 # Serialize the mime message. 977 fp = StringIO.StringIO() 978 # maxheaderlen=0 means don't line wrap headers. 979 g = Generator(fp, maxheaderlen=0) 980 g.flatten(msg, unixfrom=False) 981 body = fp.getvalue() 982 983 # Strip off the \n\n that the MIME lib tacks onto the end of the payload. 984 if request.body is None: 985 body = body[:-2] 986 987 return status_line.encode('utf-8') + body
988
989 - def _deserialize_response(self, payload):
990 """Convert string into httplib2 response and content. 991 992 Args: 993 payload: string, headers and body as a string. 994 995 Returns: 996 A pair (resp, content), such as would be returned from httplib2.request. 997 """ 998 # Strip off the status line 999 status_line, payload = payload.split('\n', 1) 1000 protocol, status, reason = status_line.split(' ', 2) 1001 1002 # Parse the rest of the response 1003 parser = FeedParser() 1004 parser.feed(payload) 1005 msg = parser.close() 1006 msg['status'] = status 1007 1008 # Create httplib2.Response from the parsed headers. 1009 resp = httplib2.Response(msg) 1010 resp.reason = reason 1011 resp.version = int(protocol.split('/', 1)[1].replace('.', '')) 1012 1013 content = payload.split('\r\n\r\n', 1)[1] 1014 1015 return resp, content
1016
1017 - def _new_id(self):
1018 """Create a new id. 1019 1020 Auto incrementing number that avoids conflicts with ids already used. 1021 1022 Returns: 1023 string, a new unique id. 1024 """ 1025 self._last_auto_id += 1 1026 while str(self._last_auto_id) in self._requests: 1027 self._last_auto_id += 1 1028 return str(self._last_auto_id)
1029
1030 - def add(self, request, callback=None, request_id=None):
1031 """Add a new request. 1032 1033 Every callback added will be paired with a unique id, the request_id. That 1034 unique id will be passed back to the callback when the response comes back 1035 from the server. The default behavior is to have the library generate it's 1036 own unique id. If the caller passes in a request_id then they must ensure 1037 uniqueness for each request_id, and if they are not an exception is 1038 raised. Callers should either supply all request_ids or nevery supply a 1039 request id, to avoid such an error. 1040 1041 Args: 1042 request: HttpRequest, Request to add to the batch. 1043 callback: callable, A callback to be called for this response, of the 1044 form callback(id, response, exception). The first parameter is the 1045 request id, and the second is the deserialized response object. The 1046 third is an apiclient.errors.HttpError exception object if an HTTP error 1047 occurred while processing the request, or None if no errors occurred. 1048 request_id: string, A unique id for the request. The id will be passed to 1049 the callback with the response. 1050 1051 Returns: 1052 None 1053 1054 Raises: 1055 BatchError if a media request is added to a batch. 1056 KeyError is the request_id is not unique. 1057 """ 1058 if request_id is None: 1059 request_id = self._new_id() 1060 if request.resumable is not None: 1061 raise BatchError("Media requests cannot be used in a batch request.") 1062 if request_id in self._requests: 1063 raise KeyError("A request with this ID already exists: %s" % request_id) 1064 self._requests[request_id] = request 1065 self._callbacks[request_id] = callback 1066 self._order.append(request_id)
1067
1068 - def _execute(self, http, order, requests):
1069 """Serialize batch request, send to server, process response. 1070 1071 Args: 1072 http: httplib2.Http, an http object to be used to make the request with. 1073 order: list, list of request ids in the order they were added to the 1074 batch. 1075 request: list, list of request objects to send. 1076 1077 Raises: 1078 httplib2.HttpLib2Error if a transport error has occured. 1079 apiclient.errors.BatchError if the response is the wrong format. 1080 """ 1081 message = MIMEMultipart('mixed') 1082 # Message should not write out it's own headers. 1083 setattr(message, '_write_headers', lambda self: None) 1084 1085 # Add all the individual requests. 1086 for request_id in order: 1087 request = requests[request_id] 1088 1089 msg = MIMENonMultipart('application', 'http') 1090 msg['Content-Transfer-Encoding'] = 'binary' 1091 msg['Content-ID'] = self._id_to_header(request_id) 1092 1093 body = self._serialize_request(request) 1094 msg.set_payload(body) 1095 message.attach(msg) 1096 1097 body = message.as_string() 1098 1099 headers = {} 1100 headers['content-type'] = ('multipart/mixed; ' 1101 'boundary="%s"') % message.get_boundary() 1102 1103 resp, content = http.request(self._batch_uri, 'POST', body=body, 1104 headers=headers) 1105 1106 if resp.status >= 300: 1107 raise HttpError(resp, content, self._batch_uri) 1108 1109 # Now break out the individual responses and store each one. 1110 boundary, _ = content.split(None, 1) 1111 1112 # Prepend with a content-type header so FeedParser can handle it. 1113 header = 'content-type: %s\r\n\r\n' % resp['content-type'] 1114 for_parser = header + content 1115 1116 parser = FeedParser() 1117 parser.feed(for_parser) 1118 mime_response = parser.close() 1119 1120 if not mime_response.is_multipart(): 1121 raise BatchError("Response not in multipart/mixed format.", resp, 1122 content) 1123 1124 for part in mime_response.get_payload(): 1125 request_id = self._header_to_id(part['Content-ID']) 1126 response, content = self._deserialize_response(part.get_payload()) 1127 self._responses[request_id] = (response, content)
1128
1129 - def execute(self, http=None):
1130 """Execute all the requests as a single batched HTTP request. 1131 1132 Args: 1133 http: httplib2.Http, an http object to be used in place of the one the 1134 HttpRequest request object was constructed with. If one isn't supplied 1135 then use a http object from the requests in this batch. 1136 1137 Returns: 1138 None 1139 1140 Raises: 1141 httplib2.HttpLib2Error if a transport error has occured. 1142 apiclient.errors.BatchError if the response is the wrong format. 1143 """ 1144 1145 # If http is not supplied use the first valid one given in the requests. 1146 if http is None: 1147 for request_id in self._order: 1148 request = self._requests[request_id] 1149 if request is not None: 1150 http = request.http 1151 break 1152 1153 if http is None: 1154 raise ValueError("Missing a valid http object.") 1155 1156 self._execute(http, self._order, self._requests) 1157 1158 # Loop over all the requests and check for 401s. For each 401 request the 1159 # credentials should be refreshed and then sent again in a separate batch. 1160 redo_requests = {} 1161 redo_order = [] 1162 1163 for request_id in self._order: 1164 resp, content = self._responses[request_id] 1165 if resp['status'] == '401': 1166 redo_order.append(request_id) 1167 request = self._requests[request_id] 1168 self._refresh_and_apply_credentials(request, http) 1169 redo_requests[request_id] = request 1170 1171 if redo_requests: 1172 self._execute(http, redo_order, redo_requests) 1173 1174 # Now process all callbacks that are erroring, and raise an exception for 1175 # ones that return a non-2xx response? Or add extra parameter to callback 1176 # that contains an HttpError? 1177 1178 for request_id in self._order: 1179 resp, content = self._responses[request_id] 1180 1181 request = self._requests[request_id] 1182 callback = self._callbacks[request_id] 1183 1184 response = None 1185 exception = None 1186 try: 1187 if resp.status >= 300: 1188 raise HttpError(resp, content, request.uri) 1189 response = request.postproc(resp, content) 1190 except HttpError, e: 1191 exception = e 1192 1193 if callback is not None: 1194 callback(request_id, response, exception) 1195 if self._callback is not None: 1196 self._callback(request_id, response, exception)
1197
1198 1199 -class HttpRequestMock(object):
1200 """Mock of HttpRequest. 1201 1202 Do not construct directly, instead use RequestMockBuilder. 1203 """ 1204
1205 - def __init__(self, resp, content, postproc):
1206 """Constructor for HttpRequestMock 1207 1208 Args: 1209 resp: httplib2.Response, the response to emulate coming from the request 1210 content: string, the response body 1211 postproc: callable, the post processing function usually supplied by 1212 the model class. See model.JsonModel.response() as an example. 1213 """ 1214 self.resp = resp 1215 self.content = content 1216 self.postproc = postproc 1217 if resp is None: 1218 self.resp = httplib2.Response({'status': 200, 'reason': 'OK'}) 1219 if 'reason' in self.resp: 1220 self.resp.reason = self.resp['reason']
1221
1222 - def execute(self, http=None):
1223 """Execute the request. 1224 1225 Same behavior as HttpRequest.execute(), but the response is 1226 mocked and not really from an HTTP request/response. 1227 """ 1228 return self.postproc(self.resp, self.content)
1229
1230 1231 -class RequestMockBuilder(object):
1232 """A simple mock of HttpRequest 1233 1234 Pass in a dictionary to the constructor that maps request methodIds to 1235 tuples of (httplib2.Response, content, opt_expected_body) that should be 1236 returned when that method is called. None may also be passed in for the 1237 httplib2.Response, in which case a 200 OK response will be generated. 1238 If an opt_expected_body (str or dict) is provided, it will be compared to 1239 the body and UnexpectedBodyError will be raised on inequality. 1240 1241 Example: 1242 response = '{"data": {"id": "tag:google.c...' 1243 requestBuilder = RequestMockBuilder( 1244 { 1245 'plus.activities.get': (None, response), 1246 } 1247 ) 1248 apiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder) 1249 1250 Methods that you do not supply a response for will return a 1251 200 OK with an empty string as the response content or raise an excpetion 1252 if check_unexpected is set to True. The methodId is taken from the rpcName 1253 in the discovery document. 1254 1255 For more details see the project wiki. 1256 """ 1257
1258 - def __init__(self, responses, check_unexpected=False):
1259 """Constructor for RequestMockBuilder 1260 1261 The constructed object should be a callable object 1262 that can replace the class HttpResponse. 1263 1264 responses - A dictionary that maps methodIds into tuples 1265 of (httplib2.Response, content). The methodId 1266 comes from the 'rpcName' field in the discovery 1267 document. 1268 check_unexpected - A boolean setting whether or not UnexpectedMethodError 1269 should be raised on unsupplied method. 1270 """ 1271 self.responses = responses 1272 self.check_unexpected = check_unexpected
1273
1274 - def __call__(self, http, postproc, uri, method='GET', body=None, 1275 headers=None, methodId=None, resumable=None):
1276 """Implements the callable interface that discovery.build() expects 1277 of requestBuilder, which is to build an object compatible with 1278 HttpRequest.execute(). See that method for the description of the 1279 parameters and the expected response. 1280 """ 1281 if methodId in self.responses: 1282 response = self.responses[methodId] 1283 resp, content = response[:2] 1284 if len(response) > 2: 1285 # Test the body against the supplied expected_body. 1286 expected_body = response[2] 1287 if bool(expected_body) != bool(body): 1288 # Not expecting a body and provided one 1289 # or expecting a body and not provided one. 1290 raise UnexpectedBodyError(expected_body, body) 1291 if isinstance(expected_body, str): 1292 expected_body = simplejson.loads(expected_body) 1293 body = simplejson.loads(body) 1294 if body != expected_body: 1295 raise UnexpectedBodyError(expected_body, body) 1296 return HttpRequestMock(resp, content, postproc) 1297 elif self.check_unexpected: 1298 raise UnexpectedMethodError(methodId) 1299 else: 1300 model = JsonModel(False) 1301 return HttpRequestMock(None, '{}', model.response)
1302
1303 1304 -class HttpMock(object):
1305 """Mock of httplib2.Http""" 1306
1307 - def __init__(self, filename, headers=None):
1308 """ 1309 Args: 1310 filename: string, absolute filename to read response from 1311 headers: dict, header to return with response 1312 """ 1313 if headers is None: 1314 headers = {'status': '200 OK'} 1315 f = file(filename, 'r') 1316 self.data = f.read() 1317 f.close() 1318 self.headers = headers
1319
1320 - def request(self, uri, 1321 method='GET', 1322 body=None, 1323 headers=None, 1324 redirections=1, 1325 connection_type=None):
1326 return httplib2.Response(self.headers), self.data
1327
1328 1329 -class HttpMockSequence(object):
1330 """Mock of httplib2.Http 1331 1332 Mocks a sequence of calls to request returning different responses for each 1333 call. Create an instance initialized with the desired response headers 1334 and content and then use as if an httplib2.Http instance. 1335 1336 http = HttpMockSequence([ 1337 ({'status': '401'}, ''), 1338 ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'), 1339 ({'status': '200'}, 'echo_request_headers'), 1340 ]) 1341 resp, content = http.request("http://examples.com") 1342 1343 There are special values you can pass in for content to trigger 1344 behavours that are helpful in testing. 1345 1346 'echo_request_headers' means return the request headers in the response body 1347 'echo_request_headers_as_json' means return the request headers in 1348 the response body 1349 'echo_request_body' means return the request body in the response body 1350 'echo_request_uri' means return the request uri in the response body 1351 """ 1352
1353 - def __init__(self, iterable):
1354 """ 1355 Args: 1356 iterable: iterable, a sequence of pairs of (headers, body) 1357 """ 1358 self._iterable = iterable 1359 self.follow_redirects = True
1360
1361 - def request(self, uri, 1362 method='GET', 1363 body=None, 1364 headers=None, 1365 redirections=1, 1366 connection_type=None):
1367 resp, content = self._iterable.pop(0) 1368 if content == 'echo_request_headers': 1369 content = headers 1370 elif content == 'echo_request_headers_as_json': 1371 content = simplejson.dumps(headers) 1372 elif content == 'echo_request_body': 1373 content = body 1374 elif content == 'echo_request_uri': 1375 content = uri 1376 return httplib2.Response(resp), content
1377
1378 1379 -def set_user_agent(http, user_agent):
1380 """Set the user-agent on every request. 1381 1382 Args: 1383 http - An instance of httplib2.Http 1384 or something that acts like it. 1385 user_agent: string, the value for the user-agent header. 1386 1387 Returns: 1388 A modified instance of http that was passed in. 1389 1390 Example: 1391 1392 h = httplib2.Http() 1393 h = set_user_agent(h, "my-app-name/6.0") 1394 1395 Most of the time the user-agent will be set doing auth, this is for the rare 1396 cases where you are accessing an unauthenticated endpoint. 1397 """ 1398 request_orig = http.request 1399 1400 # The closure that will replace 'httplib2.Http.request'. 1401 def new_request(uri, method='GET', body=None, headers=None, 1402 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1403 connection_type=None): 1404 """Modify the request headers to add the user-agent.""" 1405 if headers is None: 1406 headers = {} 1407 if 'user-agent' in headers: 1408 headers['user-agent'] = user_agent + ' ' + headers['user-agent'] 1409 else: 1410 headers['user-agent'] = user_agent 1411 resp, content = request_orig(uri, method, body, headers, 1412 redirections, connection_type) 1413 return resp, content
1414 1415 http.request = new_request 1416 return http 1417
1418 1419 -def tunnel_patch(http):
1420 """Tunnel PATCH requests over POST. 1421 Args: 1422 http - An instance of httplib2.Http 1423 or something that acts like it. 1424 1425 Returns: 1426 A modified instance of http that was passed in. 1427 1428 Example: 1429 1430 h = httplib2.Http() 1431 h = tunnel_patch(h, "my-app-name/6.0") 1432 1433 Useful if you are running on a platform that doesn't support PATCH. 1434 Apply this last if you are using OAuth 1.0, as changing the method 1435 will result in a different signature. 1436 """ 1437 request_orig = http.request 1438 1439 # The closure that will replace 'httplib2.Http.request'. 1440 def new_request(uri, method='GET', body=None, headers=None, 1441 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1442 connection_type=None): 1443 """Modify the request headers to add the user-agent.""" 1444 if headers is None: 1445 headers = {} 1446 if method == 'PATCH': 1447 if 'oauth_token' in headers.get('authorization', ''): 1448 logging.warning( 1449 'OAuth 1.0 request made with Credentials after tunnel_patch.') 1450 headers['x-http-method-override'] = "PATCH" 1451 method = 'POST' 1452 resp, content = request_orig(uri, method, body, headers, 1453 redirections, connection_type) 1454 return resp, content
1455 1456 http.request = new_request 1457 return http 1458