Package apiclient :: Module http
[hide private]
[frames] | no frames]

Source Code for Module apiclient.http

   1  # Copyright (C) 2012 Google Inc. 
   2  # 
   3  # Licensed under the Apache License, Version 2.0 (the "License"); 
   4  # you may not use this file except in compliance with the License. 
   5  # You may obtain a copy of the License at 
   6  # 
   7  #      http://www.apache.org/licenses/LICENSE-2.0 
   8  # 
   9  # Unless required by applicable law or agreed to in writing, software 
  10  # distributed under the License is distributed on an "AS IS" BASIS, 
  11  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
  12  # See the License for the specific language governing permissions and 
  13  # limitations under the License. 
  14   
  15  """Classes to encapsulate a single HTTP request. 
  16   
  17  The classes implement a command pattern, with every 
  18  object supporting an execute() method that does the 
  19  actuall HTTP request. 
  20  """ 
  21   
  22  __author__ = 'jcgregorio@google.com (Joe Gregorio)' 
  23   
  24  import StringIO 
  25  import base64 
  26  import copy 
  27  import gzip 
  28  import httplib2 
  29  import mimeparse 
  30  import mimetypes 
  31  import os 
  32  import urllib 
  33  import urlparse 
  34  import uuid 
  35   
  36  from email.generator import Generator 
  37  from email.mime.multipart import MIMEMultipart 
  38  from email.mime.nonmultipart import MIMENonMultipart 
  39  from email.parser import FeedParser 
  40  from errors import BatchError 
  41  from errors import HttpError 
  42  from errors import ResumableUploadError 
  43  from errors import UnexpectedBodyError 
  44  from errors import UnexpectedMethodError 
  45  from model import JsonModel 
  46  from oauth2client.anyjson import simplejson 
  47   
  48   
  49  DEFAULT_CHUNK_SIZE = 512*1024 
50 51 52 -class MediaUploadProgress(object):
53 """Status of a resumable upload.""" 54
55 - def __init__(self, resumable_progress, total_size):
56 """Constructor. 57 58 Args: 59 resumable_progress: int, bytes sent so far. 60 total_size: int, total bytes in complete upload, or None if the total 61 upload size isn't known ahead of time. 62 """ 63 self.resumable_progress = resumable_progress 64 self.total_size = total_size
65
66 - def progress(self):
67 """Percent of upload completed, as a float. 68 69 Returns: 70 the percentage complete as a float, returning 0.0 if the total size of 71 the upload is unknown. 72 """ 73 if self.total_size is not None: 74 return float(self.resumable_progress) / float(self.total_size) 75 else: 76 return 0.0
77
78 79 -class MediaUpload(object):
80 """Describes a media object to upload. 81 82 Base class that defines the interface of MediaUpload subclasses. 83 84 Note that subclasses of MediaUpload may allow you to control the chunksize 85 when upload a media object. It is important to keep the size of the chunk as 86 large as possible to keep the upload efficient. Other factors may influence 87 the size of the chunk you use, particularly if you are working in an 88 environment where individual HTTP requests may have a hardcoded time limit, 89 such as under certain classes of requests under Google App Engine. 90 """ 91
92 - def chunksize(self):
93 """Chunk size for resumable uploads. 94 95 Returns: 96 Chunk size in bytes. 97 """ 98 raise NotImplementedError()
99
100 - def mimetype(self):
101 """Mime type of the body. 102 103 Returns: 104 Mime type. 105 """ 106 return 'application/octet-stream'
107
108 - def size(self):
109 """Size of upload. 110 111 Returns: 112 Size of the body, or None of the size is unknown. 113 """ 114 return None
115
116 - def resumable(self):
117 """Whether this upload is resumable. 118 119 Returns: 120 True if resumable upload or False. 121 """ 122 return False
123
124 - def getbytes(self, begin, end):
125 """Get bytes from the media. 126 127 Args: 128 begin: int, offset from beginning of file. 129 length: int, number of bytes to read, starting at begin. 130 131 Returns: 132 A string of bytes read. May be shorter than length if EOF was reached 133 first. 134 """ 135 raise NotImplementedError()
136
137 - def _to_json(self, strip=None):
138 """Utility function for creating a JSON representation of a MediaUpload. 139 140 Args: 141 strip: array, An array of names of members to not include in the JSON. 142 143 Returns: 144 string, a JSON representation of this instance, suitable to pass to 145 from_json(). 146 """ 147 t = type(self) 148 d = copy.copy(self.__dict__) 149 if strip is not None: 150 for member in strip: 151 del d[member] 152 d['_class'] = t.__name__ 153 d['_module'] = t.__module__ 154 return simplejson.dumps(d)
155
156 - def to_json(self):
157 """Create a JSON representation of an instance of MediaUpload. 158 159 Returns: 160 string, a JSON representation of this instance, suitable to pass to 161 from_json(). 162 """ 163 return self._to_json()
164 165 @classmethod
166 - def new_from_json(cls, s):
167 """Utility class method to instantiate a MediaUpload subclass from a JSON 168 representation produced by to_json(). 169 170 Args: 171 s: string, JSON from to_json(). 172 173 Returns: 174 An instance of the subclass of MediaUpload that was serialized with 175 to_json(). 176 """ 177 data = simplejson.loads(s) 178 # Find and call the right classmethod from_json() to restore the object. 179 module = data['_module'] 180 m = __import__(module, fromlist=module.split('.')[:-1]) 181 kls = getattr(m, data['_class']) 182 from_json = getattr(kls, 'from_json') 183 return from_json(s)
184
185 186 -class MediaFileUpload(MediaUpload):
187 """A MediaUpload for a file. 188 189 Construct a MediaFileUpload and pass as the media_body parameter of the 190 method. For example, if we had a service that allowed uploading images: 191 192 193 media = MediaFileUpload('smiley.png', mimetype='image/png', 194 chunksize=1024*1024, resumable=True) 195 service.objects().insert( 196 bucket=buckets['items'][0]['id'], 197 name='smiley.png', 198 media_body=media).execute() 199 """ 200
201 - def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE, resumable=False):
202 """Constructor. 203 204 Args: 205 filename: string, Name of the file. 206 mimetype: string, Mime-type of the file. If None then a mime-type will be 207 guessed from the file extension. 208 chunksize: int, File will be uploaded in chunks of this many bytes. Only 209 used if resumable=True. 210 resumable: bool, True if this is a resumable upload. False means upload 211 in a single request. 212 """ 213 self._filename = filename 214 self._size = os.path.getsize(filename) 215 self._fd = None 216 if mimetype is None: 217 (mimetype, encoding) = mimetypes.guess_type(filename) 218 self._mimetype = mimetype 219 self._chunksize = chunksize 220 self._resumable = resumable
221
222 - def chunksize(self):
223 """Chunk size for resumable uploads. 224 225 Returns: 226 Chunk size in bytes. 227 """ 228 return self._chunksize
229
230 - def mimetype(self):
231 """Mime type of the body. 232 233 Returns: 234 Mime type. 235 """ 236 return self._mimetype
237
238 - def size(self):
239 """Size of upload. 240 241 Returns: 242 Size of the body, or None of the size is unknown. 243 """ 244 return self._size
245
246 - def resumable(self):
247 """Whether this upload is resumable. 248 249 Returns: 250 True if resumable upload or False. 251 """ 252 return self._resumable
253
254 - def getbytes(self, begin, length):
255 """Get bytes from the media. 256 257 Args: 258 begin: int, offset from beginning of file. 259 length: int, number of bytes to read, starting at begin. 260 261 Returns: 262 A string of bytes read. May be shorted than length if EOF was reached 263 first. 264 """ 265 if self._fd is None: 266 self._fd = open(self._filename, 'rb') 267 self._fd.seek(begin) 268 return self._fd.read(length)
269
270 - def to_json(self):
271 """Creating a JSON representation of an instance of Credentials. 272 273 Returns: 274 string, a JSON representation of this instance, suitable to pass to 275 from_json(). 276 """ 277 return self._to_json(['_fd'])
278 279 @staticmethod
280 - def from_json(s):
281 d = simplejson.loads(s) 282 return MediaFileUpload( 283 d['_filename'], d['_mimetype'], d['_chunksize'], d['_resumable'])
284
285 286 -class MediaIoBaseUpload(MediaUpload):
287 """A MediaUpload for a io.Base objects. 288 289 Note that the Python file object is compatible with io.Base and can be used 290 with this class also. 291 292 293 fh = io.BytesIO('...Some data to upload...') 294 media = MediaIoBaseUpload(fh, mimetype='image/png', 295 chunksize=1024*1024, resumable=True) 296 service.objects().insert( 297 bucket='a_bucket_id', 298 name='smiley.png', 299 media_body=media).execute() 300 """ 301
302 - def __init__(self, fh, mimetype, chunksize=DEFAULT_CHUNK_SIZE, 303 resumable=False):
304 """Constructor. 305 306 Args: 307 fh: io.Base or file object, The source of the bytes to upload. 308 mimetype: string, Mime-type of the file. If None then a mime-type will be 309 guessed from the file extension. 310 chunksize: int, File will be uploaded in chunks of this many bytes. Only 311 used if resumable=True. 312 resumable: bool, True if this is a resumable upload. False means upload 313 in a single request. 314 """ 315 self._fh = fh 316 self._mimetype = mimetype 317 self._chunksize = chunksize 318 self._resumable = resumable 319 self._size = None 320 try: 321 if hasattr(self._fh, 'fileno'): 322 fileno = self._fh.fileno() 323 self._size = os.fstat(fileno).st_size 324 except IOError: 325 pass
326
327 - def chunksize(self):
328 """Chunk size for resumable uploads. 329 330 Returns: 331 Chunk size in bytes. 332 """ 333 return self._chunksize
334
335 - def mimetype(self):
336 """Mime type of the body. 337 338 Returns: 339 Mime type. 340 """ 341 return self._mimetype
342
343 - def size(self):
344 """Size of upload. 345 346 Returns: 347 Size of the body, or None of the size is unknown. 348 """ 349 return self._size
350
351 - def resumable(self):
352 """Whether this upload is resumable. 353 354 Returns: 355 True if resumable upload or False. 356 """ 357 return self._resumable
358
359 - def getbytes(self, begin, length):
360 """Get bytes from the media. 361 362 Args: 363 begin: int, offset from beginning of file. 364 length: int, number of bytes to read, starting at begin. 365 366 Returns: 367 A string of bytes read. May be shorted than length if EOF was reached 368 first. 369 """ 370 self._fh.seek(begin) 371 return self._fh.read(length)
372
373 - def to_json(self):
374 """This upload type is not serializable.""" 375 raise NotImplementedError('MediaIoBaseUpload is not serializable.')
376
377 378 -class MediaInMemoryUpload(MediaUpload):
379 """MediaUpload for a chunk of bytes. 380 381 Construct a MediaFileUpload and pass as the media_body parameter of the 382 method. For example, if we had a service that allowed plain text: 383 """ 384
385 - def __init__(self, body, mimetype='application/octet-stream', 386 chunksize=DEFAULT_CHUNK_SIZE, resumable=False):
387 """Create a new MediaBytesUpload. 388 389 Args: 390 body: string, Bytes of body content. 391 mimetype: string, Mime-type of the file or default of 392 'application/octet-stream'. 393 chunksize: int, File will be uploaded in chunks of this many bytes. Only 394 used if resumable=True. 395 resumable: bool, True if this is a resumable upload. False means upload 396 in a single request. 397 """ 398 self._body = body 399 self._mimetype = mimetype 400 self._resumable = resumable 401 self._chunksize = chunksize
402
403 - def chunksize(self):
404 """Chunk size for resumable uploads. 405 406 Returns: 407 Chunk size in bytes. 408 """ 409 return self._chunksize
410
411 - def mimetype(self):
412 """Mime type of the body. 413 414 Returns: 415 Mime type. 416 """ 417 return self._mimetype
418
419 - def size(self):
420 """Size of upload. 421 422 Returns: 423 Size of the body, or None of the size is unknown. 424 """ 425 return len(self._body)
426
427 - def resumable(self):
428 """Whether this upload is resumable. 429 430 Returns: 431 True if resumable upload or False. 432 """ 433 return self._resumable
434
435 - def getbytes(self, begin, length):
436 """Get bytes from the media. 437 438 Args: 439 begin: int, offset from beginning of file. 440 length: int, number of bytes to read, starting at begin. 441 442 Returns: 443 A string of bytes read. May be shorter than length if EOF was reached 444 first. 445 """ 446 return self._body[begin:begin + length]
447
448 - def to_json(self):
449 """Create a JSON representation of a MediaInMemoryUpload. 450 451 Returns: 452 string, a JSON representation of this instance, suitable to pass to 453 from_json(). 454 """ 455 t = type(self) 456 d = copy.copy(self.__dict__) 457 del d['_body'] 458 d['_class'] = t.__name__ 459 d['_module'] = t.__module__ 460 d['_b64body'] = base64.b64encode(self._body) 461 return simplejson.dumps(d)
462 463 @staticmethod
464 - def from_json(s):
465 d = simplejson.loads(s) 466 return MediaInMemoryUpload(base64.b64decode(d['_b64body']), 467 d['_mimetype'], d['_chunksize'], 468 d['_resumable'])
469
470 471 -class HttpRequest(object):
472 """Encapsulates a single HTTP request.""" 473
474 - def __init__(self, http, postproc, uri, 475 method='GET', 476 body=None, 477 headers=None, 478 methodId=None, 479 resumable=None):
480 """Constructor for an HttpRequest. 481 482 Args: 483 http: httplib2.Http, the transport object to use to make a request 484 postproc: callable, called on the HTTP response and content to transform 485 it into a data object before returning, or raising an exception 486 on an error. 487 uri: string, the absolute URI to send the request to 488 method: string, the HTTP method to use 489 body: string, the request body of the HTTP request, 490 headers: dict, the HTTP request headers 491 methodId: string, a unique identifier for the API method being called. 492 resumable: MediaUpload, None if this is not a resumbale request. 493 """ 494 self.uri = uri 495 self.method = method 496 self.body = body 497 self.headers = headers or {} 498 self.methodId = methodId 499 self.http = http 500 self.postproc = postproc 501 self.resumable = resumable 502 self._in_error_state = False 503 504 # Pull the multipart boundary out of the content-type header. 505 major, minor, params = mimeparse.parse_mime_type( 506 headers.get('content-type', 'application/json')) 507 508 # The size of the non-media part of the request. 509 self.body_size = len(self.body or '') 510 511 # The resumable URI to send chunks to. 512 self.resumable_uri = None 513 514 # The bytes that have been uploaded. 515 self.resumable_progress = 0
516
517 - def execute(self, http=None):
518 """Execute the request. 519 520 Args: 521 http: httplib2.Http, an http object to be used in place of the 522 one the HttpRequest request object was constructed with. 523 524 Returns: 525 A deserialized object model of the response body as determined 526 by the postproc. 527 528 Raises: 529 apiclient.errors.HttpError if the response was not a 2xx. 530 httplib2.Error if a transport error has occured. 531 """ 532 if http is None: 533 http = self.http 534 if self.resumable: 535 body = None 536 while body is None: 537 _, body = self.next_chunk(http) 538 return body 539 else: 540 if 'content-length' not in self.headers: 541 self.headers['content-length'] = str(self.body_size) 542 resp, content = http.request(self.uri, self.method, 543 body=self.body, 544 headers=self.headers) 545 546 if resp.status >= 300: 547 raise HttpError(resp, content, self.uri) 548 return self.postproc(resp, content)
549
550 - def next_chunk(self, http=None):
551 """Execute the next step of a resumable upload. 552 553 Can only be used if the method being executed supports media uploads and 554 the MediaUpload object passed in was flagged as using resumable upload. 555 556 Example: 557 558 media = MediaFileUpload('smiley.png', mimetype='image/png', 559 chunksize=1000, resumable=True) 560 request = service.objects().insert( 561 bucket=buckets['items'][0]['id'], 562 name='smiley.png', 563 media_body=media) 564 565 response = None 566 while response is None: 567 status, response = request.next_chunk() 568 if status: 569 print "Upload %d%% complete." % int(status.progress() * 100) 570 571 572 Returns: 573 (status, body): (ResumableMediaStatus, object) 574 The body will be None until the resumable media is fully uploaded. 575 576 Raises: 577 apiclient.errors.HttpError if the response was not a 2xx. 578 httplib2.Error if a transport error has occured. 579 """ 580 if http is None: 581 http = self.http 582 583 if self.resumable.size() is None: 584 size = '*' 585 else: 586 size = str(self.resumable.size()) 587 588 if self.resumable_uri is None: 589 start_headers = copy.copy(self.headers) 590 start_headers['X-Upload-Content-Type'] = self.resumable.mimetype() 591 if size != '*': 592 start_headers['X-Upload-Content-Length'] = size 593 start_headers['content-length'] = str(self.body_size) 594 595 resp, content = http.request(self.uri, self.method, 596 body=self.body, 597 headers=start_headers) 598 if resp.status == 200 and 'location' in resp: 599 self.resumable_uri = resp['location'] 600 else: 601 raise ResumableUploadError("Failed to retrieve starting URI.") 602 elif self._in_error_state: 603 # If we are in an error state then query the server for current state of 604 # the upload by sending an empty PUT and reading the 'range' header in 605 # the response. 606 headers = { 607 'Content-Range': 'bytes */%s' % size, 608 'content-length': '0' 609 } 610 resp, content = http.request(self.resumable_uri, 'PUT', 611 headers=headers) 612 status, body = self._process_response(resp, content) 613 if body: 614 # The upload was complete. 615 return (status, body) 616 617 data = self.resumable.getbytes( 618 self.resumable_progress, self.resumable.chunksize()) 619 headers = { 620 'Content-Range': 'bytes %d-%d/%s' % ( 621 self.resumable_progress, self.resumable_progress + len(data) - 1, 622 size) 623 } 624 try: 625 resp, content = http.request(self.resumable_uri, 'PUT', 626 body=data, 627 headers=headers) 628 except: 629 self._in_error_state = True 630 raise 631 632 return self._process_response(resp, content)
633
634 - def _process_response(self, resp, content):
635 """Process the response from a single chunk upload. 636 637 Args: 638 resp: httplib2.Response, the response object. 639 content: string, the content of the response. 640 641 Returns: 642 (status, body): (ResumableMediaStatus, object) 643 The body will be None until the resumable media is fully uploaded. 644 645 Raises: 646 apiclient.errors.HttpError if the response was not a 2xx or a 308. 647 """ 648 if resp.status in [200, 201]: 649 self._in_error_state = False 650 return None, self.postproc(resp, content) 651 elif resp.status == 308: 652 self._in_error_state = False 653 # A "308 Resume Incomplete" indicates we are not done. 654 self.resumable_progress = int(resp['range'].split('-')[1]) + 1 655 if 'location' in resp: 656 self.resumable_uri = resp['location'] 657 else: 658 self._in_error_state = True 659 raise HttpError(resp, content, self.uri) 660 661 return (MediaUploadProgress(self.resumable_progress, self.resumable.size()), 662 None)
663
664 - def to_json(self):
665 """Returns a JSON representation of the HttpRequest.""" 666 d = copy.copy(self.__dict__) 667 if d['resumable'] is not None: 668 d['resumable'] = self.resumable.to_json() 669 del d['http'] 670 del d['postproc'] 671 672 return simplejson.dumps(d)
673 674 @staticmethod
675 - def from_json(s, http, postproc):
676 """Returns an HttpRequest populated with info from a JSON object.""" 677 d = simplejson.loads(s) 678 if d['resumable'] is not None: 679 d['resumable'] = MediaUpload.new_from_json(d['resumable']) 680 return HttpRequest( 681 http, 682 postproc, 683 uri=d['uri'], 684 method=d['method'], 685 body=d['body'], 686 headers=d['headers'], 687 methodId=d['methodId'], 688 resumable=d['resumable'])
689
690 691 -class BatchHttpRequest(object):
692 """Batches multiple HttpRequest objects into a single HTTP request.""" 693
694 - def __init__(self, callback=None, batch_uri=None):
695 """Constructor for a BatchHttpRequest. 696 697 Args: 698 callback: callable, A callback to be called for each response, of the 699 form callback(id, response). The first parameter is the request id, and 700 the second is the deserialized response object. 701 batch_uri: string, URI to send batch requests to. 702 """ 703 if batch_uri is None: 704 batch_uri = 'https://www.googleapis.com/batch' 705 self._batch_uri = batch_uri 706 707 # Global callback to be called for each individual response in the batch. 708 self._callback = callback 709 710 # A map from id to request. 711 self._requests = {} 712 713 # A map from id to callback. 714 self._callbacks = {} 715 716 # List of request ids, in the order in which they were added. 717 self._order = [] 718 719 # The last auto generated id. 720 self._last_auto_id = 0 721 722 # Unique ID on which to base the Content-ID headers. 723 self._base_id = None 724 725 # A map from request id to (headers, content) response pairs 726 self._responses = {} 727 728 # A map of id(Credentials) that have been refreshed. 729 self._refreshed_credentials = {}
730
731 - def _refresh_and_apply_credentials(self, request, http):
732 """Refresh the credentials and apply to the request. 733 734 Args: 735 request: HttpRequest, the request. 736 http: httplib2.Http, the global http object for the batch. 737 """ 738 # For the credentials to refresh, but only once per refresh_token 739 # If there is no http per the request then refresh the http passed in 740 # via execute() 741 creds = None 742 if request.http is not None and hasattr(request.http.request, 743 'credentials'): 744 creds = request.http.request.credentials 745 elif http is not None and hasattr(http.request, 'credentials'): 746 creds = http.request.credentials 747 if creds is not None: 748 if id(creds) not in self._refreshed_credentials: 749 creds.refresh(http) 750 self._refreshed_credentials[id(creds)] = 1 751 752 # Only apply the credentials if we are using the http object passed in, 753 # otherwise apply() will get called during _serialize_request(). 754 if request.http is None or not hasattr(request.http.request, 755 'credentials'): 756 creds.apply(request.headers)
757
758 - def _id_to_header(self, id_):
759 """Convert an id to a Content-ID header value. 760 761 Args: 762 id_: string, identifier of individual request. 763 764 Returns: 765 A Content-ID header with the id_ encoded into it. A UUID is prepended to 766 the value because Content-ID headers are supposed to be universally 767 unique. 768 """ 769 if self._base_id is None: 770 self._base_id = uuid.uuid4() 771 772 return '<%s+%s>' % (self._base_id, urllib.quote(id_))
773
774 - def _header_to_id(self, header):
775 """Convert a Content-ID header value to an id. 776 777 Presumes the Content-ID header conforms to the format that _id_to_header() 778 returns. 779 780 Args: 781 header: string, Content-ID header value. 782 783 Returns: 784 The extracted id value. 785 786 Raises: 787 BatchError if the header is not in the expected format. 788 """ 789 if header[0] != '<' or header[-1] != '>': 790 raise BatchError("Invalid value for Content-ID: %s" % header) 791 if '+' not in header: 792 raise BatchError("Invalid value for Content-ID: %s" % header) 793 base, id_ = header[1:-1].rsplit('+', 1) 794 795 return urllib.unquote(id_)
796
797 - def _serialize_request(self, request):
798 """Convert an HttpRequest object into a string. 799 800 Args: 801 request: HttpRequest, the request to serialize. 802 803 Returns: 804 The request as a string in application/http format. 805 """ 806 # Construct status line 807 parsed = urlparse.urlparse(request.uri) 808 request_line = urlparse.urlunparse( 809 (None, None, parsed.path, parsed.params, parsed.query, None) 810 ) 811 status_line = request.method + ' ' + request_line + ' HTTP/1.1\n' 812 major, minor = request.headers.get('content-type', 'application/json').split('/') 813 msg = MIMENonMultipart(major, minor) 814 headers = request.headers.copy() 815 816 if request.http is not None and hasattr(request.http.request, 817 'credentials'): 818 request.http.request.credentials.apply(headers) 819 820 # MIMENonMultipart adds its own Content-Type header. 821 if 'content-type' in headers: 822 del headers['content-type'] 823 824 for key, value in headers.iteritems(): 825 msg[key] = value 826 msg['Host'] = parsed.netloc 827 msg.set_unixfrom(None) 828 829 if request.body is not None: 830 msg.set_payload(request.body) 831 msg['content-length'] = str(len(request.body)) 832 833 # Serialize the mime message. 834 fp = StringIO.StringIO() 835 # maxheaderlen=0 means don't line wrap headers. 836 g = Generator(fp, maxheaderlen=0) 837 g.flatten(msg, unixfrom=False) 838 body = fp.getvalue() 839 840 # Strip off the \n\n that the MIME lib tacks onto the end of the payload. 841 if request.body is None: 842 body = body[:-2] 843 844 return status_line.encode('utf-8') + body
845
846 - def _deserialize_response(self, payload):
847 """Convert string into httplib2 response and content. 848 849 Args: 850 payload: string, headers and body as a string. 851 852 Returns: 853 A pair (resp, content) like would be returned from httplib2.request. 854 """ 855 # Strip off the status line 856 status_line, payload = payload.split('\n', 1) 857 protocol, status, reason = status_line.split(' ', 2) 858 859 # Parse the rest of the response 860 parser = FeedParser() 861 parser.feed(payload) 862 msg = parser.close() 863 msg['status'] = status 864 865 # Create httplib2.Response from the parsed headers. 866 resp = httplib2.Response(msg) 867 resp.reason = reason 868 resp.version = int(protocol.split('/', 1)[1].replace('.', '')) 869 870 content = payload.split('\r\n\r\n', 1)[1] 871 872 return resp, content
873
874 - def _new_id(self):
875 """Create a new id. 876 877 Auto incrementing number that avoids conflicts with ids already used. 878 879 Returns: 880 string, a new unique id. 881 """ 882 self._last_auto_id += 1 883 while str(self._last_auto_id) in self._requests: 884 self._last_auto_id += 1 885 return str(self._last_auto_id)
886
887 - def add(self, request, callback=None, request_id=None):
888 """Add a new request. 889 890 Every callback added will be paired with a unique id, the request_id. That 891 unique id will be passed back to the callback when the response comes back 892 from the server. The default behavior is to have the library generate it's 893 own unique id. If the caller passes in a request_id then they must ensure 894 uniqueness for each request_id, and if they are not an exception is 895 raised. Callers should either supply all request_ids or nevery supply a 896 request id, to avoid such an error. 897 898 Args: 899 request: HttpRequest, Request to add to the batch. 900 callback: callable, A callback to be called for this response, of the 901 form callback(id, response). The first parameter is the request id, and 902 the second is the deserialized response object. 903 request_id: string, A unique id for the request. The id will be passed to 904 the callback with the response. 905 906 Returns: 907 None 908 909 Raises: 910 BatchError if a resumable request is added to a batch. 911 KeyError is the request_id is not unique. 912 """ 913 if request_id is None: 914 request_id = self._new_id() 915 if request.resumable is not None: 916 raise BatchError("Resumable requests cannot be used in a batch request.") 917 if request_id in self._requests: 918 raise KeyError("A request with this ID already exists: %s" % request_id) 919 self._requests[request_id] = request 920 self._callbacks[request_id] = callback 921 self._order.append(request_id)
922
923 - def _execute(self, http, order, requests):
924 """Serialize batch request, send to server, process response. 925 926 Args: 927 http: httplib2.Http, an http object to be used to make the request with. 928 order: list, list of request ids in the order they were added to the 929 batch. 930 request: list, list of request objects to send. 931 932 Raises: 933 httplib2.Error if a transport error has occured. 934 apiclient.errors.BatchError if the response is the wrong format. 935 """ 936 message = MIMEMultipart('mixed') 937 # Message should not write out it's own headers. 938 setattr(message, '_write_headers', lambda self: None) 939 940 # Add all the individual requests. 941 for request_id in order: 942 request = requests[request_id] 943 944 msg = MIMENonMultipart('application', 'http') 945 msg['Content-Transfer-Encoding'] = 'binary' 946 msg['Content-ID'] = self._id_to_header(request_id) 947 948 body = self._serialize_request(request) 949 msg.set_payload(body) 950 message.attach(msg) 951 952 body = message.as_string() 953 954 headers = {} 955 headers['content-type'] = ('multipart/mixed; ' 956 'boundary="%s"') % message.get_boundary() 957 958 resp, content = http.request(self._batch_uri, 'POST', body=body, 959 headers=headers) 960 961 if resp.status >= 300: 962 raise HttpError(resp, content, self._batch_uri) 963 964 # Now break out the individual responses and store each one. 965 boundary, _ = content.split(None, 1) 966 967 # Prepend with a content-type header so FeedParser can handle it. 968 header = 'content-type: %s\r\n\r\n' % resp['content-type'] 969 for_parser = header + content 970 971 parser = FeedParser() 972 parser.feed(for_parser) 973 mime_response = parser.close() 974 975 if not mime_response.is_multipart(): 976 raise BatchError("Response not in multipart/mixed format.", resp, 977 content) 978 979 for part in mime_response.get_payload(): 980 request_id = self._header_to_id(part['Content-ID']) 981 headers, content = self._deserialize_response(part.get_payload()) 982 self._responses[request_id] = (headers, content)
983
984 - def execute(self, http=None):
985 """Execute all the requests as a single batched HTTP request. 986 987 Args: 988 http: httplib2.Http, an http object to be used in place of the one the 989 HttpRequest request object was constructed with. If one isn't supplied 990 then use a http object from the requests in this batch. 991 992 Returns: 993 None 994 995 Raises: 996 httplib2.Error if a transport error has occured. 997 apiclient.errors.BatchError if the response is the wrong format. 998 """ 999 1000 # If http is not supplied use the first valid one given in the requests. 1001 if http is None: 1002 for request_id in self._order: 1003 request = self._requests[request_id] 1004 if request is not None: 1005 http = request.http 1006 break 1007 1008 if http is None: 1009 raise ValueError("Missing a valid http object.") 1010 1011 self._execute(http, self._order, self._requests) 1012 1013 # Loop over all the requests and check for 401s. For each 401 request the 1014 # credentials should be refreshed and then sent again in a separate batch. 1015 redo_requests = {} 1016 redo_order = [] 1017 1018 for request_id in self._order: 1019 headers, content = self._responses[request_id] 1020 if headers['status'] == '401': 1021 redo_order.append(request_id) 1022 request = self._requests[request_id] 1023 self._refresh_and_apply_credentials(request, http) 1024 redo_requests[request_id] = request 1025 1026 if redo_requests: 1027 self._execute(http, redo_order, redo_requests) 1028 1029 # Now process all callbacks that are erroring, and raise an exception for 1030 # ones that return a non-2xx response? Or add extra parameter to callback 1031 # that contains an HttpError? 1032 1033 for request_id in self._order: 1034 headers, content = self._responses[request_id] 1035 1036 request = self._requests[request_id] 1037 callback = self._callbacks[request_id] 1038 1039 response = None 1040 exception = None 1041 try: 1042 r = httplib2.Response(headers) 1043 response = request.postproc(r, content) 1044 except HttpError, e: 1045 exception = e 1046 1047 if callback is not None: 1048 callback(request_id, response, exception) 1049 if self._callback is not None: 1050 self._callback(request_id, response, exception)
1051
1052 1053 -class HttpRequestMock(object):
1054 """Mock of HttpRequest. 1055 1056 Do not construct directly, instead use RequestMockBuilder. 1057 """ 1058
1059 - def __init__(self, resp, content, postproc):
1060 """Constructor for HttpRequestMock 1061 1062 Args: 1063 resp: httplib2.Response, the response to emulate coming from the request 1064 content: string, the response body 1065 postproc: callable, the post processing function usually supplied by 1066 the model class. See model.JsonModel.response() as an example. 1067 """ 1068 self.resp = resp 1069 self.content = content 1070 self.postproc = postproc 1071 if resp is None: 1072 self.resp = httplib2.Response({'status': 200, 'reason': 'OK'}) 1073 if 'reason' in self.resp: 1074 self.resp.reason = self.resp['reason']
1075
1076 - def execute(self, http=None):
1077 """Execute the request. 1078 1079 Same behavior as HttpRequest.execute(), but the response is 1080 mocked and not really from an HTTP request/response. 1081 """ 1082 return self.postproc(self.resp, self.content)
1083
1084 1085 -class RequestMockBuilder(object):
1086 """A simple mock of HttpRequest 1087 1088 Pass in a dictionary to the constructor that maps request methodIds to 1089 tuples of (httplib2.Response, content, opt_expected_body) that should be 1090 returned when that method is called. None may also be passed in for the 1091 httplib2.Response, in which case a 200 OK response will be generated. 1092 If an opt_expected_body (str or dict) is provided, it will be compared to 1093 the body and UnexpectedBodyError will be raised on inequality. 1094 1095 Example: 1096 response = '{"data": {"id": "tag:google.c...' 1097 requestBuilder = RequestMockBuilder( 1098 { 1099 'plus.activities.get': (None, response), 1100 } 1101 ) 1102 apiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder) 1103 1104 Methods that you do not supply a response for will return a 1105 200 OK with an empty string as the response content or raise an excpetion 1106 if check_unexpected is set to True. The methodId is taken from the rpcName 1107 in the discovery document. 1108 1109 For more details see the project wiki. 1110 """ 1111
1112 - def __init__(self, responses, check_unexpected=False):
1113 """Constructor for RequestMockBuilder 1114 1115 The constructed object should be a callable object 1116 that can replace the class HttpResponse. 1117 1118 responses - A dictionary that maps methodIds into tuples 1119 of (httplib2.Response, content). The methodId 1120 comes from the 'rpcName' field in the discovery 1121 document. 1122 check_unexpected - A boolean setting whether or not UnexpectedMethodError 1123 should be raised on unsupplied method. 1124 """ 1125 self.responses = responses 1126 self.check_unexpected = check_unexpected
1127
1128 - def __call__(self, http, postproc, uri, method='GET', body=None, 1129 headers=None, methodId=None, resumable=None):
1130 """Implements the callable interface that discovery.build() expects 1131 of requestBuilder, which is to build an object compatible with 1132 HttpRequest.execute(). See that method for the description of the 1133 parameters and the expected response. 1134 """ 1135 if methodId in self.responses: 1136 response = self.responses[methodId] 1137 resp, content = response[:2] 1138 if len(response) > 2: 1139 # Test the body against the supplied expected_body. 1140 expected_body = response[2] 1141 if bool(expected_body) != bool(body): 1142 # Not expecting a body and provided one 1143 # or expecting a body and not provided one. 1144 raise UnexpectedBodyError(expected_body, body) 1145 if isinstance(expected_body, str): 1146 expected_body = simplejson.loads(expected_body) 1147 body = simplejson.loads(body) 1148 if body != expected_body: 1149 raise UnexpectedBodyError(expected_body, body) 1150 return HttpRequestMock(resp, content, postproc) 1151 elif self.check_unexpected: 1152 raise UnexpectedMethodError(methodId) 1153 else: 1154 model = JsonModel(False) 1155 return HttpRequestMock(None, '{}', model.response)
1156
1157 1158 -class HttpMock(object):
1159 """Mock of httplib2.Http""" 1160
1161 - def __init__(self, filename, headers=None):
1162 """ 1163 Args: 1164 filename: string, absolute filename to read response from 1165 headers: dict, header to return with response 1166 """ 1167 if headers is None: 1168 headers = {'status': '200 OK'} 1169 f = file(filename, 'r') 1170 self.data = f.read() 1171 f.close() 1172 self.headers = headers
1173
1174 - def request(self, uri, 1175 method='GET', 1176 body=None, 1177 headers=None, 1178 redirections=1, 1179 connection_type=None):
1180 return httplib2.Response(self.headers), self.data
1181
1182 1183 -class HttpMockSequence(object):
1184 """Mock of httplib2.Http 1185 1186 Mocks a sequence of calls to request returning different responses for each 1187 call. Create an instance initialized with the desired response headers 1188 and content and then use as if an httplib2.Http instance. 1189 1190 http = HttpMockSequence([ 1191 ({'status': '401'}, ''), 1192 ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'), 1193 ({'status': '200'}, 'echo_request_headers'), 1194 ]) 1195 resp, content = http.request("http://examples.com") 1196 1197 There are special values you can pass in for content to trigger 1198 behavours that are helpful in testing. 1199 1200 'echo_request_headers' means return the request headers in the response body 1201 'echo_request_headers_as_json' means return the request headers in 1202 the response body 1203 'echo_request_body' means return the request body in the response body 1204 'echo_request_uri' means return the request uri in the response body 1205 """ 1206
1207 - def __init__(self, iterable):
1208 """ 1209 Args: 1210 iterable: iterable, a sequence of pairs of (headers, body) 1211 """ 1212 self._iterable = iterable
1213
1214 - def request(self, uri, 1215 method='GET', 1216 body=None, 1217 headers=None, 1218 redirections=1, 1219 connection_type=None):
1220 resp, content = self._iterable.pop(0) 1221 if content == 'echo_request_headers': 1222 content = headers 1223 elif content == 'echo_request_headers_as_json': 1224 content = simplejson.dumps(headers) 1225 elif content == 'echo_request_body': 1226 content = body 1227 elif content == 'echo_request_uri': 1228 content = uri 1229 return httplib2.Response(resp), content
1230
1231 1232 -def set_user_agent(http, user_agent):
1233 """Set the user-agent on every request. 1234 1235 Args: 1236 http - An instance of httplib2.Http 1237 or something that acts like it. 1238 user_agent: string, the value for the user-agent header. 1239 1240 Returns: 1241 A modified instance of http that was passed in. 1242 1243 Example: 1244 1245 h = httplib2.Http() 1246 h = set_user_agent(h, "my-app-name/6.0") 1247 1248 Most of the time the user-agent will be set doing auth, this is for the rare 1249 cases where you are accessing an unauthenticated endpoint. 1250 """ 1251 request_orig = http.request 1252 1253 # The closure that will replace 'httplib2.Http.request'. 1254 def new_request(uri, method='GET', body=None, headers=None, 1255 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1256 connection_type=None): 1257 """Modify the request headers to add the user-agent.""" 1258 if headers is None: 1259 headers = {} 1260 if 'user-agent' in headers: 1261 headers['user-agent'] = user_agent + ' ' + headers['user-agent'] 1262 else: 1263 headers['user-agent'] = user_agent 1264 resp, content = request_orig(uri, method, body, headers, 1265 redirections, connection_type) 1266 return resp, content
1267 1268 http.request = new_request 1269 return http 1270
1271 1272 -def tunnel_patch(http):
1273 """Tunnel PATCH requests over POST. 1274 Args: 1275 http - An instance of httplib2.Http 1276 or something that acts like it. 1277 1278 Returns: 1279 A modified instance of http that was passed in. 1280 1281 Example: 1282 1283 h = httplib2.Http() 1284 h = tunnel_patch(h, "my-app-name/6.0") 1285 1286 Useful if you are running on a platform that doesn't support PATCH. 1287 Apply this last if you are using OAuth 1.0, as changing the method 1288 will result in a different signature. 1289 """ 1290 request_orig = http.request 1291 1292 # The closure that will replace 'httplib2.Http.request'. 1293 def new_request(uri, method='GET', body=None, headers=None, 1294 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1295 connection_type=None): 1296 """Modify the request headers to add the user-agent.""" 1297 if headers is None: 1298 headers = {} 1299 if method == 'PATCH': 1300 if 'oauth_token' in headers.get('authorization', ''): 1301 logging.warning( 1302 'OAuth 1.0 request made with Credentials after tunnel_patch.') 1303 headers['x-http-method-override'] = "PATCH" 1304 method = 'POST' 1305 resp, content = request_orig(uri, method, body, headers, 1306 redirections, connection_type) 1307 return resp, content
1308 1309 http.request = new_request 1310 return http 1311