Package apiclient :: Module http
[hide private]
[frames] | no frames]

Source Code for Module apiclient.http

   1  # Copyright (C) 2012 Google Inc. 
   2  # 
   3  # Licensed under the Apache License, Version 2.0 (the "License"); 
   4  # you may not use this file except in compliance with the License. 
   5  # You may obtain a copy of the License at 
   6  # 
   7  #      http://www.apache.org/licenses/LICENSE-2.0 
   8  # 
   9  # Unless required by applicable law or agreed to in writing, software 
  10  # distributed under the License is distributed on an "AS IS" BASIS, 
  11  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
  12  # See the License for the specific language governing permissions and 
  13  # limitations under the License. 
  14   
  15  """Classes to encapsulate a single HTTP request. 
  16   
  17  The classes implement a command pattern, with every 
  18  object supporting an execute() method that does the 
  19  actuall HTTP request. 
  20  """ 
  21   
  22  __author__ = 'jcgregorio@google.com (Joe Gregorio)' 
  23   
  24  import StringIO 
  25  import base64 
  26  import copy 
  27  import gzip 
  28  import httplib2 
  29  import mimeparse 
  30  import mimetypes 
  31  import os 
  32  import urllib 
  33  import urlparse 
  34  import uuid 
  35   
  36  from email.generator import Generator 
  37  from email.mime.multipart import MIMEMultipart 
  38  from email.mime.nonmultipart import MIMENonMultipart 
  39  from email.parser import FeedParser 
  40  from errors import BatchError 
  41  from errors import HttpError 
  42  from errors import ResumableUploadError 
  43  from errors import UnexpectedBodyError 
  44  from errors import UnexpectedMethodError 
  45  from model import JsonModel 
  46  from oauth2client.anyjson import simplejson 
  47   
  48   
  49  DEFAULT_CHUNK_SIZE = 512*1024 
50 51 52 -class MediaUploadProgress(object):
53 """Status of a resumable upload.""" 54
55 - def __init__(self, resumable_progress, total_size):
56 """Constructor. 57 58 Args: 59 resumable_progress: int, bytes sent so far. 60 total_size: int, total bytes in complete upload, or None if the total 61 upload size isn't known ahead of time. 62 """ 63 self.resumable_progress = resumable_progress 64 self.total_size = total_size
65
66 - def progress(self):
67 """Percent of upload completed, as a float. 68 69 Returns: 70 the percentage complete as a float, returning 0.0 if the total size of 71 the upload is unknown. 72 """ 73 if self.total_size is not None: 74 return float(self.resumable_progress) / float(self.total_size) 75 else: 76 return 0.0
77
78 79 -class MediaDownloadProgress(object):
80 """Status of a resumable download.""" 81
82 - def __init__(self, resumable_progress, total_size):
83 """Constructor. 84 85 Args: 86 resumable_progress: int, bytes received so far. 87 total_size: int, total bytes in complete download. 88 """ 89 self.resumable_progress = resumable_progress 90 self.total_size = total_size
91
92 - def progress(self):
93 """Percent of download completed, as a float. 94 95 Returns: 96 the percentage complete as a float, returning 0.0 if the total size of 97 the download is unknown. 98 """ 99 if self.total_size is not None: 100 return float(self.resumable_progress) / float(self.total_size) 101 else: 102 return 0.0
103
104 105 -class MediaUpload(object):
106 """Describes a media object to upload. 107 108 Base class that defines the interface of MediaUpload subclasses. 109 110 Note that subclasses of MediaUpload may allow you to control the chunksize 111 when upload a media object. It is important to keep the size of the chunk as 112 large as possible to keep the upload efficient. Other factors may influence 113 the size of the chunk you use, particularly if you are working in an 114 environment where individual HTTP requests may have a hardcoded time limit, 115 such as under certain classes of requests under Google App Engine. 116 """ 117
118 - def chunksize(self):
119 """Chunk size for resumable uploads. 120 121 Returns: 122 Chunk size in bytes. 123 """ 124 raise NotImplementedError()
125
126 - def mimetype(self):
127 """Mime type of the body. 128 129 Returns: 130 Mime type. 131 """ 132 return 'application/octet-stream'
133
134 - def size(self):
135 """Size of upload. 136 137 Returns: 138 Size of the body, or None of the size is unknown. 139 """ 140 return None
141
142 - def resumable(self):
143 """Whether this upload is resumable. 144 145 Returns: 146 True if resumable upload or False. 147 """ 148 return False
149
150 - def getbytes(self, begin, end):
151 """Get bytes from the media. 152 153 Args: 154 begin: int, offset from beginning of file. 155 length: int, number of bytes to read, starting at begin. 156 157 Returns: 158 A string of bytes read. May be shorter than length if EOF was reached 159 first. 160 """ 161 raise NotImplementedError()
162
163 - def _to_json(self, strip=None):
164 """Utility function for creating a JSON representation of a MediaUpload. 165 166 Args: 167 strip: array, An array of names of members to not include in the JSON. 168 169 Returns: 170 string, a JSON representation of this instance, suitable to pass to 171 from_json(). 172 """ 173 t = type(self) 174 d = copy.copy(self.__dict__) 175 if strip is not None: 176 for member in strip: 177 del d[member] 178 d['_class'] = t.__name__ 179 d['_module'] = t.__module__ 180 return simplejson.dumps(d)
181
182 - def to_json(self):
183 """Create a JSON representation of an instance of MediaUpload. 184 185 Returns: 186 string, a JSON representation of this instance, suitable to pass to 187 from_json(). 188 """ 189 return self._to_json()
190 191 @classmethod
192 - def new_from_json(cls, s):
193 """Utility class method to instantiate a MediaUpload subclass from a JSON 194 representation produced by to_json(). 195 196 Args: 197 s: string, JSON from to_json(). 198 199 Returns: 200 An instance of the subclass of MediaUpload that was serialized with 201 to_json(). 202 """ 203 data = simplejson.loads(s) 204 # Find and call the right classmethod from_json() to restore the object. 205 module = data['_module'] 206 m = __import__(module, fromlist=module.split('.')[:-1]) 207 kls = getattr(m, data['_class']) 208 from_json = getattr(kls, 'from_json') 209 return from_json(s)
210
211 212 -class MediaFileUpload(MediaUpload):
213 """A MediaUpload for a file. 214 215 Construct a MediaFileUpload and pass as the media_body parameter of the 216 method. For example, if we had a service that allowed uploading images: 217 218 219 media = MediaFileUpload('cow.png', mimetype='image/png', 220 chunksize=1024*1024, resumable=True) 221 farm.animals()..insert( 222 id='cow', 223 name='cow.png', 224 media_body=media).execute() 225 """ 226
227 - def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE, resumable=False):
228 """Constructor. 229 230 Args: 231 filename: string, Name of the file. 232 mimetype: string, Mime-type of the file. If None then a mime-type will be 233 guessed from the file extension. 234 chunksize: int, File will be uploaded in chunks of this many bytes. Only 235 used if resumable=True. 236 resumable: bool, True if this is a resumable upload. False means upload 237 in a single request. 238 """ 239 self._filename = filename 240 self._size = os.path.getsize(filename) 241 self._fd = None 242 if mimetype is None: 243 (mimetype, encoding) = mimetypes.guess_type(filename) 244 self._mimetype = mimetype 245 self._chunksize = chunksize 246 self._resumable = resumable
247
248 - def chunksize(self):
249 """Chunk size for resumable uploads. 250 251 Returns: 252 Chunk size in bytes. 253 """ 254 return self._chunksize
255
256 - def mimetype(self):
257 """Mime type of the body. 258 259 Returns: 260 Mime type. 261 """ 262 return self._mimetype
263
264 - def size(self):
265 """Size of upload. 266 267 Returns: 268 Size of the body, or None of the size is unknown. 269 """ 270 return self._size
271
272 - def resumable(self):
273 """Whether this upload is resumable. 274 275 Returns: 276 True if resumable upload or False. 277 """ 278 return self._resumable
279
280 - def getbytes(self, begin, length):
281 """Get bytes from the media. 282 283 Args: 284 begin: int, offset from beginning of file. 285 length: int, number of bytes to read, starting at begin. 286 287 Returns: 288 A string of bytes read. May be shorted than length if EOF was reached 289 first. 290 """ 291 if self._fd is None: 292 self._fd = open(self._filename, 'rb') 293 self._fd.seek(begin) 294 return self._fd.read(length)
295
296 - def to_json(self):
297 """Creating a JSON representation of an instance of MediaFileUpload. 298 299 Returns: 300 string, a JSON representation of this instance, suitable to pass to 301 from_json(). 302 """ 303 return self._to_json(['_fd'])
304 305 @staticmethod
306 - def from_json(s):
307 d = simplejson.loads(s) 308 return MediaFileUpload( 309 d['_filename'], d['_mimetype'], d['_chunksize'], d['_resumable'])
310
311 312 -class MediaIoBaseUpload(MediaUpload):
313 """A MediaUpload for a io.Base objects. 314 315 Note that the Python file object is compatible with io.Base and can be used 316 with this class also. 317 318 fh = io.BytesIO('...Some data to upload...') 319 media = MediaIoBaseUpload(fh, mimetype='image/png', 320 chunksize=1024*1024, resumable=True) 321 farm.animals().insert( 322 id='cow', 323 name='cow.png', 324 media_body=media).execute() 325 """ 326
327 - def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE, 328 resumable=False):
329 """Constructor. 330 331 Args: 332 fd: io.Base or file object, The source of the bytes to upload. MUST be 333 opened in blocking mode, do not use streams opened in non-blocking mode. 334 mimetype: string, Mime-type of the file. If None then a mime-type will be 335 guessed from the file extension. 336 chunksize: int, File will be uploaded in chunks of this many bytes. Only 337 used if resumable=True. 338 resumable: bool, True if this is a resumable upload. False means upload 339 in a single request. 340 """ 341 self._fd = fd 342 self._mimetype = mimetype 343 self._chunksize = chunksize 344 self._resumable = resumable 345 self._size = None 346 try: 347 if hasattr(self._fd, 'fileno'): 348 fileno = self._fd.fileno() 349 350 # Pipes and such show up as 0 length files. 351 size = os.fstat(fileno).st_size 352 if size: 353 self._size = os.fstat(fileno).st_size 354 except IOError: 355 pass
356
357 - def chunksize(self):
358 """Chunk size for resumable uploads. 359 360 Returns: 361 Chunk size in bytes. 362 """ 363 return self._chunksize
364
365 - def mimetype(self):
366 """Mime type of the body. 367 368 Returns: 369 Mime type. 370 """ 371 return self._mimetype
372
373 - def size(self):
374 """Size of upload. 375 376 Returns: 377 Size of the body, or None of the size is unknown. 378 """ 379 return self._size
380
381 - def resumable(self):
382 """Whether this upload is resumable. 383 384 Returns: 385 True if resumable upload or False. 386 """ 387 return self._resumable
388
389 - def getbytes(self, begin, length):
390 """Get bytes from the media. 391 392 Args: 393 begin: int, offset from beginning of file. 394 length: int, number of bytes to read, starting at begin. 395 396 Returns: 397 A string of bytes read. May be shorted than length if EOF was reached 398 first. 399 """ 400 self._fd.seek(begin) 401 return self._fd.read(length)
402
403 - def to_json(self):
404 """This upload type is not serializable.""" 405 raise NotImplementedError('MediaIoBaseUpload is not serializable.')
406
407 408 -class MediaInMemoryUpload(MediaUpload):
409 """MediaUpload for a chunk of bytes. 410 411 Construct a MediaFileUpload and pass as the media_body parameter of the 412 method. 413 """ 414
415 - def __init__(self, body, mimetype='application/octet-stream', 416 chunksize=DEFAULT_CHUNK_SIZE, resumable=False):
417 """Create a new MediaBytesUpload. 418 419 Args: 420 body: string, Bytes of body content. 421 mimetype: string, Mime-type of the file or default of 422 'application/octet-stream'. 423 chunksize: int, File will be uploaded in chunks of this many bytes. Only 424 used if resumable=True. 425 resumable: bool, True if this is a resumable upload. False means upload 426 in a single request. 427 """ 428 self._body = body 429 self._mimetype = mimetype 430 self._resumable = resumable 431 self._chunksize = chunksize
432
433 - def chunksize(self):
434 """Chunk size for resumable uploads. 435 436 Returns: 437 Chunk size in bytes. 438 """ 439 return self._chunksize
440
441 - def mimetype(self):
442 """Mime type of the body. 443 444 Returns: 445 Mime type. 446 """ 447 return self._mimetype
448
449 - def size(self):
450 """Size of upload. 451 452 Returns: 453 Size of the body, or None of the size is unknown. 454 """ 455 return len(self._body)
456
457 - def resumable(self):
458 """Whether this upload is resumable. 459 460 Returns: 461 True if resumable upload or False. 462 """ 463 return self._resumable
464
465 - def getbytes(self, begin, length):
466 """Get bytes from the media. 467 468 Args: 469 begin: int, offset from beginning of file. 470 length: int, number of bytes to read, starting at begin. 471 472 Returns: 473 A string of bytes read. May be shorter than length if EOF was reached 474 first. 475 """ 476 return self._body[begin:begin + length]
477
478 - def to_json(self):
479 """Create a JSON representation of a MediaInMemoryUpload. 480 481 Returns: 482 string, a JSON representation of this instance, suitable to pass to 483 from_json(). 484 """ 485 t = type(self) 486 d = copy.copy(self.__dict__) 487 del d['_body'] 488 d['_class'] = t.__name__ 489 d['_module'] = t.__module__ 490 d['_b64body'] = base64.b64encode(self._body) 491 return simplejson.dumps(d)
492 493 @staticmethod
494 - def from_json(s):
495 d = simplejson.loads(s) 496 return MediaInMemoryUpload(base64.b64decode(d['_b64body']), 497 d['_mimetype'], d['_chunksize'], 498 d['_resumable'])
499
500 501 -class MediaIoBaseDownload(object):
502 """"Download media resources. 503 504 Note that the Python file object is compatible with io.Base and can be used 505 with this class also. 506 507 508 Example: 509 request = farms.animals().get_media(id='cow') 510 fh = io.FileIO('cow.png', mode='wb') 511 downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024) 512 513 done = False 514 while done is False: 515 status, done = downloader.next_chunk() 516 if status: 517 print "Download %d%%." % int(status.progress() * 100) 518 print "Download Complete!" 519 """ 520
521 - def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE):
522 """Constructor. 523 524 Args: 525 fd: io.Base or file object, The stream in which to write the downloaded 526 bytes. 527 request: apiclient.http.HttpRequest, the media request to perform in 528 chunks. 529 chunksize: int, File will be downloaded in chunks of this many bytes. 530 """ 531 self._fd = fd 532 self._request = request 533 self._uri = request.uri 534 self._chunksize = chunksize 535 self._progress = 0 536 self._total_size = None 537 self._done = False
538
539 - def next_chunk(self):
540 """Get the next chunk of the download. 541 542 Returns: 543 (status, done): (MediaDownloadStatus, boolean) 544 The value of 'done' will be True when the media has been fully 545 downloaded. 546 547 Raises: 548 apiclient.errors.HttpError if the response was not a 2xx. 549 httplib2.Error if a transport error has occured. 550 """ 551 headers = { 552 'range': 'bytes=%d-%d' % ( 553 self._progress, self._progress + self._chunksize) 554 } 555 http = self._request.http 556 http.follow_redirects = False 557 558 resp, content = http.request(self._uri, headers=headers) 559 if resp.status in [301, 302, 303, 307, 308] and 'location' in resp: 560 self._uri = resp['location'] 561 resp, content = http.request(self._uri, headers=headers) 562 if resp.status in [200, 206]: 563 self._progress += len(content) 564 self._fd.write(content) 565 566 if 'content-range' in resp: 567 content_range = resp['content-range'] 568 length = content_range.rsplit('/', 1)[1] 569 self._total_size = int(length) 570 571 if self._progress == self._total_size: 572 self._done = True 573 return MediaDownloadProgress(self._progress, self._total_size), self._done 574 else: 575 raise HttpError(resp, content, self._uri)
576
577 578 -class HttpRequest(object):
579 """Encapsulates a single HTTP request.""" 580
581 - def __init__(self, http, postproc, uri, 582 method='GET', 583 body=None, 584 headers=None, 585 methodId=None, 586 resumable=None):
587 """Constructor for an HttpRequest. 588 589 Args: 590 http: httplib2.Http, the transport object to use to make a request 591 postproc: callable, called on the HTTP response and content to transform 592 it into a data object before returning, or raising an exception 593 on an error. 594 uri: string, the absolute URI to send the request to 595 method: string, the HTTP method to use 596 body: string, the request body of the HTTP request, 597 headers: dict, the HTTP request headers 598 methodId: string, a unique identifier for the API method being called. 599 resumable: MediaUpload, None if this is not a resumbale request. 600 """ 601 self.uri = uri 602 self.method = method 603 self.body = body 604 self.headers = headers or {} 605 self.methodId = methodId 606 self.http = http 607 self.postproc = postproc 608 self.resumable = resumable 609 self._in_error_state = False 610 611 # Pull the multipart boundary out of the content-type header. 612 major, minor, params = mimeparse.parse_mime_type( 613 headers.get('content-type', 'application/json')) 614 615 # The size of the non-media part of the request. 616 self.body_size = len(self.body or '') 617 618 # The resumable URI to send chunks to. 619 self.resumable_uri = None 620 621 # The bytes that have been uploaded. 622 self.resumable_progress = 0
623
624 - def execute(self, http=None):
625 """Execute the request. 626 627 Args: 628 http: httplib2.Http, an http object to be used in place of the 629 one the HttpRequest request object was constructed with. 630 631 Returns: 632 A deserialized object model of the response body as determined 633 by the postproc. 634 635 Raises: 636 apiclient.errors.HttpError if the response was not a 2xx. 637 httplib2.Error if a transport error has occured. 638 """ 639 if http is None: 640 http = self.http 641 if self.resumable: 642 body = None 643 while body is None: 644 _, body = self.next_chunk(http) 645 return body 646 else: 647 if 'content-length' not in self.headers: 648 self.headers['content-length'] = str(self.body_size) 649 resp, content = http.request(self.uri, self.method, 650 body=self.body, 651 headers=self.headers) 652 653 if resp.status >= 300: 654 raise HttpError(resp, content, self.uri) 655 return self.postproc(resp, content)
656
657 - def next_chunk(self, http=None):
658 """Execute the next step of a resumable upload. 659 660 Can only be used if the method being executed supports media uploads and 661 the MediaUpload object passed in was flagged as using resumable upload. 662 663 Example: 664 665 media = MediaFileUpload('cow.png', mimetype='image/png', 666 chunksize=1000, resumable=True) 667 request = farm.animals().insert( 668 id='cow', 669 name='cow.png', 670 media_body=media) 671 672 response = None 673 while response is None: 674 status, response = request.next_chunk() 675 if status: 676 print "Upload %d%% complete." % int(status.progress() * 100) 677 678 679 Returns: 680 (status, body): (ResumableMediaStatus, object) 681 The body will be None until the resumable media is fully uploaded. 682 683 Raises: 684 apiclient.errors.HttpError if the response was not a 2xx. 685 httplib2.Error if a transport error has occured. 686 """ 687 if http is None: 688 http = self.http 689 690 if self.resumable.size() is None: 691 size = '*' 692 else: 693 size = str(self.resumable.size()) 694 695 if self.resumable_uri is None: 696 start_headers = copy.copy(self.headers) 697 start_headers['X-Upload-Content-Type'] = self.resumable.mimetype() 698 if size != '*': 699 start_headers['X-Upload-Content-Length'] = size 700 start_headers['content-length'] = str(self.body_size) 701 702 resp, content = http.request(self.uri, self.method, 703 body=self.body, 704 headers=start_headers) 705 if resp.status == 200 and 'location' in resp: 706 self.resumable_uri = resp['location'] 707 else: 708 raise ResumableUploadError("Failed to retrieve starting URI.") 709 elif self._in_error_state: 710 # If we are in an error state then query the server for current state of 711 # the upload by sending an empty PUT and reading the 'range' header in 712 # the response. 713 headers = { 714 'Content-Range': 'bytes */%s' % size, 715 'content-length': '0' 716 } 717 resp, content = http.request(self.resumable_uri, 'PUT', 718 headers=headers) 719 status, body = self._process_response(resp, content) 720 if body: 721 # The upload was complete. 722 return (status, body) 723 724 data = self.resumable.getbytes( 725 self.resumable_progress, self.resumable.chunksize()) 726 727 # A short read implies that we are at EOF, so finish the upload. 728 if len(data) < self.resumable.chunksize(): 729 size = str(self.resumable_progress + len(data)) 730 731 headers = { 732 'Content-Range': 'bytes %d-%d/%s' % ( 733 self.resumable_progress, self.resumable_progress + len(data) - 1, 734 size) 735 } 736 try: 737 resp, content = http.request(self.resumable_uri, 'PUT', 738 body=data, 739 headers=headers) 740 except: 741 self._in_error_state = True 742 raise 743 744 return self._process_response(resp, content)
745
746 - def _process_response(self, resp, content):
747 """Process the response from a single chunk upload. 748 749 Args: 750 resp: httplib2.Response, the response object. 751 content: string, the content of the response. 752 753 Returns: 754 (status, body): (ResumableMediaStatus, object) 755 The body will be None until the resumable media is fully uploaded. 756 757 Raises: 758 apiclient.errors.HttpError if the response was not a 2xx or a 308. 759 """ 760 if resp.status in [200, 201]: 761 self._in_error_state = False 762 return None, self.postproc(resp, content) 763 elif resp.status == 308: 764 self._in_error_state = False 765 # A "308 Resume Incomplete" indicates we are not done. 766 self.resumable_progress = int(resp['range'].split('-')[1]) + 1 767 if 'location' in resp: 768 self.resumable_uri = resp['location'] 769 else: 770 self._in_error_state = True 771 raise HttpError(resp, content, self.uri) 772 773 return (MediaUploadProgress(self.resumable_progress, self.resumable.size()), 774 None)
775
776 - def to_json(self):
777 """Returns a JSON representation of the HttpRequest.""" 778 d = copy.copy(self.__dict__) 779 if d['resumable'] is not None: 780 d['resumable'] = self.resumable.to_json() 781 del d['http'] 782 del d['postproc'] 783 784 return simplejson.dumps(d)
785 786 @staticmethod
787 - def from_json(s, http, postproc):
788 """Returns an HttpRequest populated with info from a JSON object.""" 789 d = simplejson.loads(s) 790 if d['resumable'] is not None: 791 d['resumable'] = MediaUpload.new_from_json(d['resumable']) 792 return HttpRequest( 793 http, 794 postproc, 795 uri=d['uri'], 796 method=d['method'], 797 body=d['body'], 798 headers=d['headers'], 799 methodId=d['methodId'], 800 resumable=d['resumable'])
801
802 803 -class BatchHttpRequest(object):
804 """Batches multiple HttpRequest objects into a single HTTP request. 805 806 Example: 807 from apiclient.http import BatchHttpRequest 808 809 def list_animals(request_id, response, exception): 810 \"\"\"Do something with the animals list response.\"\"\" 811 if exception is not None: 812 # Do something with the exception. 813 pass 814 else: 815 # Do something with the response. 816 pass 817 818 def list_farmers(request_id, response, exception): 819 \"\"\"Do something with the farmers list response.\"\"\" 820 if exception is not None: 821 # Do something with the exception. 822 pass 823 else: 824 # Do something with the response. 825 pass 826 827 service = build('farm', 'v2') 828 829 batch = BatchHttpRequest() 830 831 batch.add(service.animals().list(), list_animals) 832 batch.add(service.farmers().list(), list_farmers) 833 batch.execute(http) 834 """ 835
836 - def __init__(self, callback=None, batch_uri=None):
837 """Constructor for a BatchHttpRequest. 838 839 Args: 840 callback: callable, A callback to be called for each response, of the 841 form callback(id, response, exception). The first parameter is the 842 request id, and the second is the deserialized response object. The 843 third is an apiclient.errors.HttpError exception object if an HTTP error 844 occurred while processing the request, or None if no error occurred. 845 batch_uri: string, URI to send batch requests to. 846 """ 847 if batch_uri is None: 848 batch_uri = 'https://www.googleapis.com/batch' 849 self._batch_uri = batch_uri 850 851 # Global callback to be called for each individual response in the batch. 852 self._callback = callback 853 854 # A map from id to request. 855 self._requests = {} 856 857 # A map from id to callback. 858 self._callbacks = {} 859 860 # List of request ids, in the order in which they were added. 861 self._order = [] 862 863 # The last auto generated id. 864 self._last_auto_id = 0 865 866 # Unique ID on which to base the Content-ID headers. 867 self._base_id = None 868 869 # A map from request id to (httplib2.Response, content) response pairs 870 self._responses = {} 871 872 # A map of id(Credentials) that have been refreshed. 873 self._refreshed_credentials = {}
874
875 - def _refresh_and_apply_credentials(self, request, http):
876 """Refresh the credentials and apply to the request. 877 878 Args: 879 request: HttpRequest, the request. 880 http: httplib2.Http, the global http object for the batch. 881 """ 882 # For the credentials to refresh, but only once per refresh_token 883 # If there is no http per the request then refresh the http passed in 884 # via execute() 885 creds = None 886 if request.http is not None and hasattr(request.http.request, 887 'credentials'): 888 creds = request.http.request.credentials 889 elif http is not None and hasattr(http.request, 'credentials'): 890 creds = http.request.credentials 891 if creds is not None: 892 if id(creds) not in self._refreshed_credentials: 893 creds.refresh(http) 894 self._refreshed_credentials[id(creds)] = 1 895 896 # Only apply the credentials if we are using the http object passed in, 897 # otherwise apply() will get called during _serialize_request(). 898 if request.http is None or not hasattr(request.http.request, 899 'credentials'): 900 creds.apply(request.headers)
901
902 - def _id_to_header(self, id_):
903 """Convert an id to a Content-ID header value. 904 905 Args: 906 id_: string, identifier of individual request. 907 908 Returns: 909 A Content-ID header with the id_ encoded into it. A UUID is prepended to 910 the value because Content-ID headers are supposed to be universally 911 unique. 912 """ 913 if self._base_id is None: 914 self._base_id = uuid.uuid4() 915 916 return '<%s+%s>' % (self._base_id, urllib.quote(id_))
917
918 - def _header_to_id(self, header):
919 """Convert a Content-ID header value to an id. 920 921 Presumes the Content-ID header conforms to the format that _id_to_header() 922 returns. 923 924 Args: 925 header: string, Content-ID header value. 926 927 Returns: 928 The extracted id value. 929 930 Raises: 931 BatchError if the header is not in the expected format. 932 """ 933 if header[0] != '<' or header[-1] != '>': 934 raise BatchError("Invalid value for Content-ID: %s" % header) 935 if '+' not in header: 936 raise BatchError("Invalid value for Content-ID: %s" % header) 937 base, id_ = header[1:-1].rsplit('+', 1) 938 939 return urllib.unquote(id_)
940
941 - def _serialize_request(self, request):
942 """Convert an HttpRequest object into a string. 943 944 Args: 945 request: HttpRequest, the request to serialize. 946 947 Returns: 948 The request as a string in application/http format. 949 """ 950 # Construct status line 951 parsed = urlparse.urlparse(request.uri) 952 request_line = urlparse.urlunparse( 953 (None, None, parsed.path, parsed.params, parsed.query, None) 954 ) 955 status_line = request.method + ' ' + request_line + ' HTTP/1.1\n' 956 major, minor = request.headers.get('content-type', 'application/json').split('/') 957 msg = MIMENonMultipart(major, minor) 958 headers = request.headers.copy() 959 960 if request.http is not None and hasattr(request.http.request, 961 'credentials'): 962 request.http.request.credentials.apply(headers) 963 964 # MIMENonMultipart adds its own Content-Type header. 965 if 'content-type' in headers: 966 del headers['content-type'] 967 968 for key, value in headers.iteritems(): 969 msg[key] = value 970 msg['Host'] = parsed.netloc 971 msg.set_unixfrom(None) 972 973 if request.body is not None: 974 msg.set_payload(request.body) 975 msg['content-length'] = str(len(request.body)) 976 977 # Serialize the mime message. 978 fp = StringIO.StringIO() 979 # maxheaderlen=0 means don't line wrap headers. 980 g = Generator(fp, maxheaderlen=0) 981 g.flatten(msg, unixfrom=False) 982 body = fp.getvalue() 983 984 # Strip off the \n\n that the MIME lib tacks onto the end of the payload. 985 if request.body is None: 986 body = body[:-2] 987 988 return status_line.encode('utf-8') + body
989
990 - def _deserialize_response(self, payload):
991 """Convert string into httplib2 response and content. 992 993 Args: 994 payload: string, headers and body as a string. 995 996 Returns: 997 A pair (resp, content), such as would be returned from httplib2.request. 998 """ 999 # Strip off the status line 1000 status_line, payload = payload.split('\n', 1) 1001 protocol, status, reason = status_line.split(' ', 2) 1002 1003 # Parse the rest of the response 1004 parser = FeedParser() 1005 parser.feed(payload) 1006 msg = parser.close() 1007 msg['status'] = status 1008 1009 # Create httplib2.Response from the parsed headers. 1010 resp = httplib2.Response(msg) 1011 resp.reason = reason 1012 resp.version = int(protocol.split('/', 1)[1].replace('.', '')) 1013 1014 content = payload.split('\r\n\r\n', 1)[1] 1015 1016 return resp, content
1017
1018 - def _new_id(self):
1019 """Create a new id. 1020 1021 Auto incrementing number that avoids conflicts with ids already used. 1022 1023 Returns: 1024 string, a new unique id. 1025 """ 1026 self._last_auto_id += 1 1027 while str(self._last_auto_id) in self._requests: 1028 self._last_auto_id += 1 1029 return str(self._last_auto_id)
1030
1031 - def add(self, request, callback=None, request_id=None):
1032 """Add a new request. 1033 1034 Every callback added will be paired with a unique id, the request_id. That 1035 unique id will be passed back to the callback when the response comes back 1036 from the server. The default behavior is to have the library generate it's 1037 own unique id. If the caller passes in a request_id then they must ensure 1038 uniqueness for each request_id, and if they are not an exception is 1039 raised. Callers should either supply all request_ids or nevery supply a 1040 request id, to avoid such an error. 1041 1042 Args: 1043 request: HttpRequest, Request to add to the batch. 1044 callback: callable, A callback to be called for this response, of the 1045 form callback(id, response, exception). The first parameter is the 1046 request id, and the second is the deserialized response object. The 1047 third is an apiclient.errors.HttpError exception object if an HTTP error 1048 occurred while processing the request, or None if no errors occurred. 1049 request_id: string, A unique id for the request. The id will be passed to 1050 the callback with the response. 1051 1052 Returns: 1053 None 1054 1055 Raises: 1056 BatchError if a media request is added to a batch. 1057 KeyError is the request_id is not unique. 1058 """ 1059 if request_id is None: 1060 request_id = self._new_id() 1061 if request.resumable is not None: 1062 raise BatchError("Media requests cannot be used in a batch request.") 1063 if request_id in self._requests: 1064 raise KeyError("A request with this ID already exists: %s" % request_id) 1065 self._requests[request_id] = request 1066 self._callbacks[request_id] = callback 1067 self._order.append(request_id)
1068
1069 - def _execute(self, http, order, requests):
1070 """Serialize batch request, send to server, process response. 1071 1072 Args: 1073 http: httplib2.Http, an http object to be used to make the request with. 1074 order: list, list of request ids in the order they were added to the 1075 batch. 1076 request: list, list of request objects to send. 1077 1078 Raises: 1079 httplib2.Error if a transport error has occured. 1080 apiclient.errors.BatchError if the response is the wrong format. 1081 """ 1082 message = MIMEMultipart('mixed') 1083 # Message should not write out it's own headers. 1084 setattr(message, '_write_headers', lambda self: None) 1085 1086 # Add all the individual requests. 1087 for request_id in order: 1088 request = requests[request_id] 1089 1090 msg = MIMENonMultipart('application', 'http') 1091 msg['Content-Transfer-Encoding'] = 'binary' 1092 msg['Content-ID'] = self._id_to_header(request_id) 1093 1094 body = self._serialize_request(request) 1095 msg.set_payload(body) 1096 message.attach(msg) 1097 1098 body = message.as_string() 1099 1100 headers = {} 1101 headers['content-type'] = ('multipart/mixed; ' 1102 'boundary="%s"') % message.get_boundary() 1103 1104 resp, content = http.request(self._batch_uri, 'POST', body=body, 1105 headers=headers) 1106 1107 if resp.status >= 300: 1108 raise HttpError(resp, content, self._batch_uri) 1109 1110 # Now break out the individual responses and store each one. 1111 boundary, _ = content.split(None, 1) 1112 1113 # Prepend with a content-type header so FeedParser can handle it. 1114 header = 'content-type: %s\r\n\r\n' % resp['content-type'] 1115 for_parser = header + content 1116 1117 parser = FeedParser() 1118 parser.feed(for_parser) 1119 mime_response = parser.close() 1120 1121 if not mime_response.is_multipart(): 1122 raise BatchError("Response not in multipart/mixed format.", resp, 1123 content) 1124 1125 for part in mime_response.get_payload(): 1126 request_id = self._header_to_id(part['Content-ID']) 1127 response, content = self._deserialize_response(part.get_payload()) 1128 self._responses[request_id] = (response, content)
1129
1130 - def execute(self, http=None):
1131 """Execute all the requests as a single batched HTTP request. 1132 1133 Args: 1134 http: httplib2.Http, an http object to be used in place of the one the 1135 HttpRequest request object was constructed with. If one isn't supplied 1136 then use a http object from the requests in this batch. 1137 1138 Returns: 1139 None 1140 1141 Raises: 1142 httplib2.Error if a transport error has occured. 1143 apiclient.errors.BatchError if the response is the wrong format. 1144 """ 1145 1146 # If http is not supplied use the first valid one given in the requests. 1147 if http is None: 1148 for request_id in self._order: 1149 request = self._requests[request_id] 1150 if request is not None: 1151 http = request.http 1152 break 1153 1154 if http is None: 1155 raise ValueError("Missing a valid http object.") 1156 1157 self._execute(http, self._order, self._requests) 1158 1159 # Loop over all the requests and check for 401s. For each 401 request the 1160 # credentials should be refreshed and then sent again in a separate batch. 1161 redo_requests = {} 1162 redo_order = [] 1163 1164 for request_id in self._order: 1165 resp, content = self._responses[request_id] 1166 if resp['status'] == '401': 1167 redo_order.append(request_id) 1168 request = self._requests[request_id] 1169 self._refresh_and_apply_credentials(request, http) 1170 redo_requests[request_id] = request 1171 1172 if redo_requests: 1173 self._execute(http, redo_order, redo_requests) 1174 1175 # Now process all callbacks that are erroring, and raise an exception for 1176 # ones that return a non-2xx response? Or add extra parameter to callback 1177 # that contains an HttpError? 1178 1179 for request_id in self._order: 1180 resp, content = self._responses[request_id] 1181 1182 request = self._requests[request_id] 1183 callback = self._callbacks[request_id] 1184 1185 response = None 1186 exception = None 1187 try: 1188 response = request.postproc(resp, content) 1189 except HttpError, e: 1190 exception = e 1191 1192 if callback is not None: 1193 callback(request_id, response, exception) 1194 if self._callback is not None: 1195 self._callback(request_id, response, exception)
1196
1197 1198 -class HttpRequestMock(object):
1199 """Mock of HttpRequest. 1200 1201 Do not construct directly, instead use RequestMockBuilder. 1202 """ 1203
1204 - def __init__(self, resp, content, postproc):
1205 """Constructor for HttpRequestMock 1206 1207 Args: 1208 resp: httplib2.Response, the response to emulate coming from the request 1209 content: string, the response body 1210 postproc: callable, the post processing function usually supplied by 1211 the model class. See model.JsonModel.response() as an example. 1212 """ 1213 self.resp = resp 1214 self.content = content 1215 self.postproc = postproc 1216 if resp is None: 1217 self.resp = httplib2.Response({'status': 200, 'reason': 'OK'}) 1218 if 'reason' in self.resp: 1219 self.resp.reason = self.resp['reason']
1220
1221 - def execute(self, http=None):
1222 """Execute the request. 1223 1224 Same behavior as HttpRequest.execute(), but the response is 1225 mocked and not really from an HTTP request/response. 1226 """ 1227 return self.postproc(self.resp, self.content)
1228
1229 1230 -class RequestMockBuilder(object):
1231 """A simple mock of HttpRequest 1232 1233 Pass in a dictionary to the constructor that maps request methodIds to 1234 tuples of (httplib2.Response, content, opt_expected_body) that should be 1235 returned when that method is called. None may also be passed in for the 1236 httplib2.Response, in which case a 200 OK response will be generated. 1237 If an opt_expected_body (str or dict) is provided, it will be compared to 1238 the body and UnexpectedBodyError will be raised on inequality. 1239 1240 Example: 1241 response = '{"data": {"id": "tag:google.c...' 1242 requestBuilder = RequestMockBuilder( 1243 { 1244 'plus.activities.get': (None, response), 1245 } 1246 ) 1247 apiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder) 1248 1249 Methods that you do not supply a response for will return a 1250 200 OK with an empty string as the response content or raise an excpetion 1251 if check_unexpected is set to True. The methodId is taken from the rpcName 1252 in the discovery document. 1253 1254 For more details see the project wiki. 1255 """ 1256
1257 - def __init__(self, responses, check_unexpected=False):
1258 """Constructor for RequestMockBuilder 1259 1260 The constructed object should be a callable object 1261 that can replace the class HttpResponse. 1262 1263 responses - A dictionary that maps methodIds into tuples 1264 of (httplib2.Response, content). The methodId 1265 comes from the 'rpcName' field in the discovery 1266 document. 1267 check_unexpected - A boolean setting whether or not UnexpectedMethodError 1268 should be raised on unsupplied method. 1269 """ 1270 self.responses = responses 1271 self.check_unexpected = check_unexpected
1272
1273 - def __call__(self, http, postproc, uri, method='GET', body=None, 1274 headers=None, methodId=None, resumable=None):
1275 """Implements the callable interface that discovery.build() expects 1276 of requestBuilder, which is to build an object compatible with 1277 HttpRequest.execute(). See that method for the description of the 1278 parameters and the expected response. 1279 """ 1280 if methodId in self.responses: 1281 response = self.responses[methodId] 1282 resp, content = response[:2] 1283 if len(response) > 2: 1284 # Test the body against the supplied expected_body. 1285 expected_body = response[2] 1286 if bool(expected_body) != bool(body): 1287 # Not expecting a body and provided one 1288 # or expecting a body and not provided one. 1289 raise UnexpectedBodyError(expected_body, body) 1290 if isinstance(expected_body, str): 1291 expected_body = simplejson.loads(expected_body) 1292 body = simplejson.loads(body) 1293 if body != expected_body: 1294 raise UnexpectedBodyError(expected_body, body) 1295 return HttpRequestMock(resp, content, postproc) 1296 elif self.check_unexpected: 1297 raise UnexpectedMethodError(methodId) 1298 else: 1299 model = JsonModel(False) 1300 return HttpRequestMock(None, '{}', model.response)
1301
1302 1303 -class HttpMock(object):
1304 """Mock of httplib2.Http""" 1305
1306 - def __init__(self, filename, headers=None):
1307 """ 1308 Args: 1309 filename: string, absolute filename to read response from 1310 headers: dict, header to return with response 1311 """ 1312 if headers is None: 1313 headers = {'status': '200 OK'} 1314 f = file(filename, 'r') 1315 self.data = f.read() 1316 f.close() 1317 self.headers = headers
1318
1319 - def request(self, uri, 1320 method='GET', 1321 body=None, 1322 headers=None, 1323 redirections=1, 1324 connection_type=None):
1325 return httplib2.Response(self.headers), self.data
1326
1327 1328 -class HttpMockSequence(object):
1329 """Mock of httplib2.Http 1330 1331 Mocks a sequence of calls to request returning different responses for each 1332 call. Create an instance initialized with the desired response headers 1333 and content and then use as if an httplib2.Http instance. 1334 1335 http = HttpMockSequence([ 1336 ({'status': '401'}, ''), 1337 ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'), 1338 ({'status': '200'}, 'echo_request_headers'), 1339 ]) 1340 resp, content = http.request("http://examples.com") 1341 1342 There are special values you can pass in for content to trigger 1343 behavours that are helpful in testing. 1344 1345 'echo_request_headers' means return the request headers in the response body 1346 'echo_request_headers_as_json' means return the request headers in 1347 the response body 1348 'echo_request_body' means return the request body in the response body 1349 'echo_request_uri' means return the request uri in the response body 1350 """ 1351
1352 - def __init__(self, iterable):
1353 """ 1354 Args: 1355 iterable: iterable, a sequence of pairs of (headers, body) 1356 """ 1357 self._iterable = iterable 1358 self.follow_redirects = True
1359
1360 - def request(self, uri, 1361 method='GET', 1362 body=None, 1363 headers=None, 1364 redirections=1, 1365 connection_type=None):
1366 resp, content = self._iterable.pop(0) 1367 if content == 'echo_request_headers': 1368 content = headers 1369 elif content == 'echo_request_headers_as_json': 1370 content = simplejson.dumps(headers) 1371 elif content == 'echo_request_body': 1372 content = body 1373 elif content == 'echo_request_uri': 1374 content = uri 1375 return httplib2.Response(resp), content
1376
1377 1378 -def set_user_agent(http, user_agent):
1379 """Set the user-agent on every request. 1380 1381 Args: 1382 http - An instance of httplib2.Http 1383 or something that acts like it. 1384 user_agent: string, the value for the user-agent header. 1385 1386 Returns: 1387 A modified instance of http that was passed in. 1388 1389 Example: 1390 1391 h = httplib2.Http() 1392 h = set_user_agent(h, "my-app-name/6.0") 1393 1394 Most of the time the user-agent will be set doing auth, this is for the rare 1395 cases where you are accessing an unauthenticated endpoint. 1396 """ 1397 request_orig = http.request 1398 1399 # The closure that will replace 'httplib2.Http.request'. 1400 def new_request(uri, method='GET', body=None, headers=None, 1401 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1402 connection_type=None): 1403 """Modify the request headers to add the user-agent.""" 1404 if headers is None: 1405 headers = {} 1406 if 'user-agent' in headers: 1407 headers['user-agent'] = user_agent + ' ' + headers['user-agent'] 1408 else: 1409 headers['user-agent'] = user_agent 1410 resp, content = request_orig(uri, method, body, headers, 1411 redirections, connection_type) 1412 return resp, content
1413 1414 http.request = new_request 1415 return http 1416
1417 1418 -def tunnel_patch(http):
1419 """Tunnel PATCH requests over POST. 1420 Args: 1421 http - An instance of httplib2.Http 1422 or something that acts like it. 1423 1424 Returns: 1425 A modified instance of http that was passed in. 1426 1427 Example: 1428 1429 h = httplib2.Http() 1430 h = tunnel_patch(h, "my-app-name/6.0") 1431 1432 Useful if you are running on a platform that doesn't support PATCH. 1433 Apply this last if you are using OAuth 1.0, as changing the method 1434 will result in a different signature. 1435 """ 1436 request_orig = http.request 1437 1438 # The closure that will replace 'httplib2.Http.request'. 1439 def new_request(uri, method='GET', body=None, headers=None, 1440 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1441 connection_type=None): 1442 """Modify the request headers to add the user-agent.""" 1443 if headers is None: 1444 headers = {} 1445 if method == 'PATCH': 1446 if 'oauth_token' in headers.get('authorization', ''): 1447 logging.warning( 1448 'OAuth 1.0 request made with Credentials after tunnel_patch.') 1449 headers['x-http-method-override'] = "PATCH" 1450 method = 'POST' 1451 resp, content = request_orig(uri, method, body, headers, 1452 redirections, connection_type) 1453 return resp, content
1454 1455 http.request = new_request 1456 return http 1457