apiclient.http

1 # Copyright (C) 2012 Google Inc. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 15 """Classes to encapsulate a single HTTP request. 16 17 The classes implement a command pattern, with every 18 object supporting an execute() method that does the 19 actuall HTTP request. 20 """ 21 22 __author__ = 'jcgregorio@google.com (Joe Gregorio)' 23 24 import StringIO 25 import base64 26 import copy 27 import gzip 28 import httplib2 29 import mimeparse 30 import mimetypes 31 import os 32 import urllib 33 import urlparse 34 import uuid 35 36 from email.generator import Generator 37 from email.mime.multipart import MIMEMultipart 38 from email.mime.nonmultipart import MIMENonMultipart 39 from email.parser import FeedParser 40 from errors import BatchError 41 from errors import HttpError 42 from errors import ResumableUploadError 43 from errors import UnexpectedBodyError 44 from errors import UnexpectedMethodError 45 from model import JsonModel 46 from oauth2client.anyjson import simplejson 47 48 49 DEFAULT_CHUNK_SIZE = 512*1024

50 51 52 -class MediaUploadProgress(object):

53 """Status of a resumable upload.""" 54

55 - def __init__(self, resumable_progress, total_size):

56 """Constructor. 57 58 Args: 59 resumable_progress: int, bytes sent so far. 60 total_size: int, total bytes in complete upload, or None if the total 61 upload size isn't known ahead of time. 62 """ 63 self.resumable_progress = resumable_progress 64 self.total_size = total_size

65

66 - def progress(self):

67 """Percent of upload completed, as a float. 68 69 Returns: 70 the percentage complete as a float, returning 0.0 if the total size of 71 the upload is unknown. 72 """ 73 if self.total_size is not None: 74 return float(self.resumable_progress) / float(self.total_size) 75 else: 76 return 0.0

77

78 79 -class MediaUpload(object):

80 """Describes a media object to upload. 81 82 Base class that defines the interface of MediaUpload subclasses. 83 84 Note that subclasses of MediaUpload may allow you to control the chunksize 85 when upload a media object. It is important to keep the size of the chunk as 86 large as possible to keep the upload efficient. Other factors may influence 87 the size of the chunk you use, particularly if you are working in an 88 environment where individual HTTP requests may have a hardcoded time limit, 89 such as under certain classes of requests under Google App Engine. 90 """ 91

92 - def chunksize(self):

93 """Chunk size for resumable uploads. 94 95 Returns: 96 Chunk size in bytes. 97 """ 98 raise NotImplementedError()

99

100 - def mimetype(self):

101 """Mime type of the body. 102 103 Returns: 104 Mime type. 105 """ 106 return 'application/octet-stream'

107

108 - def size(self):

109 """Size of upload. 110 111 Returns: 112 Size of the body, or None of the size is unknown. 113 """ 114 return None

115

116 - def resumable(self):

117 """Whether this upload is resumable. 118 119 Returns: 120 True if resumable upload or False. 121 """ 122 return False

123

124 - def getbytes(self, begin, end):

125 """Get bytes from the media. 126 127 Args: 128 begin: int, offset from beginning of file. 129 length: int, number of bytes to read, starting at begin. 130 131 Returns: 132 A string of bytes read. May be shorter than length if EOF was reached 133 first. 134 """ 135 raise NotImplementedError()

136

137 - def _to_json(self, strip=None):

138 """Utility function for creating a JSON representation of a MediaUpload. 139 140 Args: 141 strip: array, An array of names of members to not include in the JSON. 142 143 Returns: 144 string, a JSON representation of this instance, suitable to pass to 145 from_json(). 146 """ 147 t = type(self) 148 d = copy.copy(self.__dict__) 149 if strip is not None: 150 for member in strip: 151 del d[member] 152 d['_class'] = t.__name__ 153 d['_module'] = t.__module__ 154 return simplejson.dumps(d)

155

156 - def to_json(self):

157 """Create a JSON representation of an instance of MediaUpload. 158 159 Returns: 160 string, a JSON representation of this instance, suitable to pass to 161 from_json(). 162 """ 163 return self._to_json()

164 165 @classmethod

166 - def new_from_json(cls, s):

167 """Utility class method to instantiate a MediaUpload subclass from a JSON 168 representation produced by to_json(). 169 170 Args: 171 s: string, JSON from to_json(). 172 173 Returns: 174 An instance of the subclass of MediaUpload that was serialized with 175 to_json(). 176 """ 177 data = simplejson.loads(s) 178 # Find and call the right classmethod from_json() to restore the object. 179 module = data['_module'] 180 m = __import__(module, fromlist=module.split('.')[:-1]) 181 kls = getattr(m, data['_class']) 182 from_json = getattr(kls, 'from_json') 183 return from_json(s)

184

185 186 -class MediaFileUpload(MediaUpload):

187 """A MediaUpload for a file. 188 189 Construct a MediaFileUpload and pass as the media_body parameter of the 190 method. For example, if we had a service that allowed uploading images: 191 192 193 media = MediaFileUpload('smiley.png', mimetype='image/png', 194 chunksize=1024*1024, resumable=True) 195 service.objects().insert( 196 bucket=buckets['items'][0]['id'], 197 name='smiley.png', 198 media_body=media).execute() 199 """ 200

201 - def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE, resumable=False):

202 """Constructor. 203 204 Args: 205 filename: string, Name of the file. 206 mimetype: string, Mime-type of the file. If None then a mime-type will be 207 guessed from the file extension. 208 chunksize: int, File will be uploaded in chunks of this many bytes. Only 209 used if resumable=True. 210 resumable: bool, True if this is a resumable upload. False means upload 211 in a single request. 212 """ 213 self._filename = filename 214 self._size = os.path.getsize(filename) 215 self._fd = None 216 if mimetype is None: 217 (mimetype, encoding) = mimetypes.guess_type(filename) 218 self._mimetype = mimetype 219 self._chunksize = chunksize 220 self._resumable = resumable

221

222 - def chunksize(self):

223 """Chunk size for resumable uploads. 224 225 Returns: 226 Chunk size in bytes. 227 """ 228 return self._chunksize

229

230 - def mimetype(self):

231 """Mime type of the body. 232 233 Returns: 234 Mime type. 235 """ 236 return self._mimetype

237

238 - def size(self):

239 """Size of upload. 240 241 Returns: 242 Size of the body, or None of the size is unknown. 243 """ 244 return self._size

245

246 - def resumable(self):

247 """Whether this upload is resumable. 248 249 Returns: 250 True if resumable upload or False. 251 """ 252 return self._resumable

253

254 - def getbytes(self, begin, length):

255 """Get bytes from the media. 256 257 Args: 258 begin: int, offset from beginning of file. 259 length: int, number of bytes to read, starting at begin. 260 261 Returns: 262 A string of bytes read. May be shorted than length if EOF was reached 263 first. 264 """ 265 if self._fd is None: 266 self._fd = open(self._filename, 'rb') 267 self._fd.seek(begin) 268 return self._fd.read(length)

269

270 - def to_json(self):

271 """Creating a JSON representation of an instance of Credentials. 272 273 Returns: 274 string, a JSON representation of this instance, suitable to pass to 275 from_json(). 276 """ 277 return self._to_json(['_fd'])

278 279 @staticmethod

280 - def from_json(s):

281 d = simplejson.loads(s) 282 return MediaFileUpload( 283 d['_filename'], d['_mimetype'], d['_chunksize'], d['_resumable'])

284

285 286 -class MediaIoBaseUpload(MediaUpload):

287 """A MediaUpload for a io.Base objects. 288 289 Note that the Python file object is compatible with io.Base and can be used 290 with this class also. 291 292 fh = io.BytesIO('...Some data to upload...') 293 media = MediaIoBaseUpload(fh, mimetype='image/png', 294 chunksize=1024*1024, resumable=True) 295 service.objects().insert( 296 bucket='a_bucket_id', 297 name='smiley.png', 298 media_body=media).execute() 299 """ 300

301 - def __init__(self, fh, mimetype, chunksize=DEFAULT_CHUNK_SIZE, 302 resumable=False):

303 """Constructor. 304 305 Args: 306 fh: io.Base or file object, The source of the bytes to upload. MUST be 307 opened in blocking mode, do not use streams opened in non-blocking mode. 308 mimetype: string, Mime-type of the file. If None then a mime-type will be 309 guessed from the file extension. 310 chunksize: int, File will be uploaded in chunks of this many bytes. Only 311 used if resumable=True. 312 resumable: bool, True if this is a resumable upload. False means upload 313 in a single request. 314 """ 315 self._fh = fh 316 self._mimetype = mimetype 317 self._chunksize = chunksize 318 self._resumable = resumable 319 self._size = None 320 try: 321 if hasattr(self._fh, 'fileno'): 322 fileno = self._fh.fileno() 323 324 # Pipes and such show up as 0 length files. 325 size = os.fstat(fileno).st_size 326 if size: 327 self._size = os.fstat(fileno).st_size 328 except IOError: 329 pass

330

331 - def chunksize(self):

332 """Chunk size for resumable uploads. 333 334 Returns: 335 Chunk size in bytes. 336 """ 337 return self._chunksize

338

339 - def mimetype(self):

340 """Mime type of the body. 341 342 Returns: 343 Mime type. 344 """ 345 return self._mimetype

346

347 - def size(self):

348 """Size of upload. 349 350 Returns: 351 Size of the body, or None of the size is unknown. 352 """ 353 return self._size

354

355 - def resumable(self):

356 """Whether this upload is resumable. 357 358 Returns: 359 True if resumable upload or False. 360 """ 361 return self._resumable

362

363 - def getbytes(self, begin, length):

364 """Get bytes from the media. 365 366 Args: 367 begin: int, offset from beginning of file. 368 length: int, number of bytes to read, starting at begin. 369 370 Returns: 371 A string of bytes read. May be shorted than length if EOF was reached 372 first. 373 """ 374 self._fh.seek(begin) 375 return self._fh.read(length)

376

377 - def to_json(self):

378 """This upload type is not serializable.""" 379 raise NotImplementedError('MediaIoBaseUpload is not serializable.')

380

381 382 -class MediaInMemoryUpload(MediaUpload):

383 """MediaUpload for a chunk of bytes. 384 385 Construct a MediaFileUpload and pass as the media_body parameter of the 386 method. For example, if we had a service that allowed plain text: 387 """ 388

389 - def __init__(self, body, mimetype='application/octet-stream', 390 chunksize=DEFAULT_CHUNK_SIZE, resumable=False):

391 """Create a new MediaBytesUpload. 392 393 Args: 394 body: string, Bytes of body content. 395 mimetype: string, Mime-type of the file or default of 396 'application/octet-stream'. 397 chunksize: int, File will be uploaded in chunks of this many bytes. Only 398 used if resumable=True. 399 resumable: bool, True if this is a resumable upload. False means upload 400 in a single request. 401 """ 402 self._body = body 403 self._mimetype = mimetype 404 self._resumable = resumable 405 self._chunksize = chunksize

406

407 - def chunksize(self):

408 """Chunk size for resumable uploads. 409 410 Returns: 411 Chunk size in bytes. 412 """ 413 return self._chunksize

414

415 - def mimetype(self):

416 """Mime type of the body. 417 418 Returns: 419 Mime type. 420 """ 421 return self._mimetype

422

423 - def size(self):

424 """Size of upload. 425 426 Returns: 427 Size of the body, or None of the size is unknown. 428 """ 429 return len(self._body)

430

431 - def resumable(self):

432 """Whether this upload is resumable. 433 434 Returns: 435 True if resumable upload or False. 436 """ 437 return self._resumable

438

439 - def getbytes(self, begin, length):

440 """Get bytes from the media. 441 442 Args: 443 begin: int, offset from beginning of file. 444 length: int, number of bytes to read, starting at begin. 445 446 Returns: 447 A string of bytes read. May be shorter than length if EOF was reached 448 first. 449 """ 450 return self._body[begin:begin + length]

451

452 - def to_json(self):

453 """Create a JSON representation of a MediaInMemoryUpload. 454 455 Returns: 456 string, a JSON representation of this instance, suitable to pass to 457 from_json(). 458 """ 459 t = type(self) 460 d = copy.copy(self.__dict__) 461 del d['_body'] 462 d['_class'] = t.__name__ 463 d['_module'] = t.__module__ 464 d['_b64body'] = base64.b64encode(self._body) 465 return simplejson.dumps(d)

466 467 @staticmethod

468 - def from_json(s):

469 d = simplejson.loads(s) 470 return MediaInMemoryUpload(base64.b64decode(d['_b64body']), 471 d['_mimetype'], d['_chunksize'], 472 d['_resumable'])

473

474 475 -class HttpRequest(object):

476 """Encapsulates a single HTTP request.""" 477

478 - def __init__(self, http, postproc, uri, 479 method='GET', 480 body=None, 481 headers=None, 482 methodId=None, 483 resumable=None):

484 """Constructor for an HttpRequest. 485 486 Args: 487 http: httplib2.Http, the transport object to use to make a request 488 postproc: callable, called on the HTTP response and content to transform 489 it into a data object before returning, or raising an exception 490 on an error. 491 uri: string, the absolute URI to send the request to 492 method: string, the HTTP method to use 493 body: string, the request body of the HTTP request, 494 headers: dict, the HTTP request headers 495 methodId: string, a unique identifier for the API method being called. 496 resumable: MediaUpload, None if this is not a resumbale request. 497 """ 498 self.uri = uri 499 self.method = method 500 self.body = body 501 self.headers = headers or {} 502 self.methodId = methodId 503 self.http = http 504 self.postproc = postproc 505 self.resumable = resumable 506 self._in_error_state = False 507 508 # Pull the multipart boundary out of the content-type header. 509 major, minor, params = mimeparse.parse_mime_type( 510 headers.get('content-type', 'application/json')) 511 512 # The size of the non-media part of the request. 513 self.body_size = len(self.body or '') 514 515 # The resumable URI to send chunks to. 516 self.resumable_uri = None 517 518 # The bytes that have been uploaded. 519 self.resumable_progress = 0

520

521 - def execute(self, http=None):

522 """Execute the request. 523 524 Args: 525 http: httplib2.Http, an http object to be used in place of the 526 one the HttpRequest request object was constructed with. 527 528 Returns: 529 A deserialized object model of the response body as determined 530 by the postproc. 531 532 Raises: 533 apiclient.errors.HttpError if the response was not a 2xx. 534 httplib2.Error if a transport error has occured. 535 """ 536 if http is None: 537 http = self.http 538 if self.resumable: 539 body = None 540 while body is None: 541 _, body = self.next_chunk(http) 542 return body 543 else: 544 if 'content-length' not in self.headers: 545 self.headers['content-length'] = str(self.body_size) 546 resp, content = http.request(self.uri, self.method, 547 body=self.body, 548 headers=self.headers) 549 550 if resp.status >= 300: 551 raise HttpError(resp, content, self.uri) 552 return self.postproc(resp, content)

553

554 - def next_chunk(self, http=None):

555 """Execute the next step of a resumable upload. 556 557 Can only be used if the method being executed supports media uploads and 558 the MediaUpload object passed in was flagged as using resumable upload. 559 560 Example: 561 562 media = MediaFileUpload('smiley.png', mimetype='image/png', 563 chunksize=1000, resumable=True) 564 request = service.objects().insert( 565 bucket=buckets['items'][0]['id'], 566 name='smiley.png', 567 media_body=media) 568 569 response = None 570 while response is None: 571 status, response = request.next_chunk() 572 if status: 573 print "Upload %d%% complete." % int(status.progress() * 100) 574 575 576 Returns: 577 (status, body): (ResumableMediaStatus, object) 578 The body will be None until the resumable media is fully uploaded. 579 580 Raises: 581 apiclient.errors.HttpError if the response was not a 2xx. 582 httplib2.Error if a transport error has occured. 583 """ 584 if http is None: 585 http = self.http 586 587 if self.resumable.size() is None: 588 size = '*' 589 else: 590 size = str(self.resumable.size()) 591 592 if self.resumable_uri is None: 593 start_headers = copy.copy(self.headers) 594 start_headers['X-Upload-Content-Type'] = self.resumable.mimetype() 595 if size != '*': 596 start_headers['X-Upload-Content-Length'] = size 597 start_headers['content-length'] = str(self.body_size) 598 599 resp, content = http.request(self.uri, self.method, 600 body=self.body, 601 headers=start_headers) 602 if resp.status == 200 and 'location' in resp: 603 self.resumable_uri = resp['location'] 604 else: 605 raise ResumableUploadError("Failed to retrieve starting URI.") 606 elif self._in_error_state: 607 # If we are in an error state then query the server for current state of 608 # the upload by sending an empty PUT and reading the 'range' header in 609 # the response. 610 headers = { 611 'Content-Range': 'bytes */%s' % size, 612 'content-length': '0' 613 } 614 resp, content = http.request(self.resumable_uri, 'PUT', 615 headers=headers) 616 status, body = self._process_response(resp, content) 617 if body: 618 # The upload was complete. 619 return (status, body) 620 621 data = self.resumable.getbytes( 622 self.resumable_progress, self.resumable.chunksize()) 623 624 # A short read implies that we are at EOF, so finish the upload. 625 if len(data) < self.resumable.chunksize(): 626 size = str(self.resumable_progress + len(data)) 627 628 headers = { 629 'Content-Range': 'bytes %d-%d/%s' % ( 630 self.resumable_progress, self.resumable_progress + len(data) - 1, 631 size) 632 } 633 try: 634 resp, content = http.request(self.resumable_uri, 'PUT', 635 body=data, 636 headers=headers) 637 except: 638 self._in_error_state = True 639 raise 640 641 return self._process_response(resp, content)

642

643 - def _process_response(self, resp, content):

644 """Process the response from a single chunk upload. 645 646 Args: 647 resp: httplib2.Response, the response object. 648 content: string, the content of the response. 649 650 Returns: 651 (status, body): (ResumableMediaStatus, object) 652 The body will be None until the resumable media is fully uploaded. 653 654 Raises: 655 apiclient.errors.HttpError if the response was not a 2xx or a 308. 656 """ 657 if resp.status in [200, 201]: 658 self._in_error_state = False 659 return None, self.postproc(resp, content) 660 elif resp.status == 308: 661 self._in_error_state = False 662 # A "308 Resume Incomplete" indicates we are not done. 663 self.resumable_progress = int(resp['range'].split('-')[1]) + 1 664 if 'location' in resp: 665 self.resumable_uri = resp['location'] 666 else: 667 self._in_error_state = True 668 raise HttpError(resp, content, self.uri) 669 670 return (MediaUploadProgress(self.resumable_progress, self.resumable.size()), 671 None)

672

673 - def to_json(self):

674 """Returns a JSON representation of the HttpRequest.""" 675 d = copy.copy(self.__dict__) 676 if d['resumable'] is not None: 677 d['resumable'] = self.resumable.to_json() 678 del d['http'] 679 del d['postproc'] 680 681 return simplejson.dumps(d)

682 683 @staticmethod

684 - def from_json(s, http, postproc):

685 """Returns an HttpRequest populated with info from a JSON object.""" 686 d = simplejson.loads(s) 687 if d['resumable'] is not None: 688 d['resumable'] = MediaUpload.new_from_json(d['resumable']) 689 return HttpRequest( 690 http, 691 postproc, 692 uri=d['uri'], 693 method=d['method'], 694 body=d['body'], 695 headers=d['headers'], 696 methodId=d['methodId'], 697 resumable=d['resumable'])

698

699 700 -class BatchHttpRequest(object):

701 """Batches multiple HttpRequest objects into a single HTTP request.""" 702

703 - def __init__(self, callback=None, batch_uri=None):

704 """Constructor for a BatchHttpRequest. 705 706 Args: 707 callback: callable, A callback to be called for each response, of the 708 form callback(id, response). The first parameter is the request id, and 709 the second is the deserialized response object. 710 batch_uri: string, URI to send batch requests to. 711 """ 712 if batch_uri is None: 713 batch_uri = 'https://www.googleapis.com/batch' 714 self._batch_uri = batch_uri 715 716 # Global callback to be called for each individual response in the batch. 717 self._callback = callback 718 719 # A map from id to request. 720 self._requests = {} 721 722 # A map from id to callback. 723 self._callbacks = {} 724 725 # List of request ids, in the order in which they were added. 726 self._order = [] 727 728 # The last auto generated id. 729 self._last_auto_id = 0 730 731 # Unique ID on which to base the Content-ID headers. 732 self._base_id = None 733 734 # A map from request id to (headers, content) response pairs 735 self._responses = {} 736 737 # A map of id(Credentials) that have been refreshed. 738 self._refreshed_credentials = {}

739

740 - def _refresh_and_apply_credentials(self, request, http):

741 """Refresh the credentials and apply to the request. 742 743 Args: 744 request: HttpRequest, the request. 745 http: httplib2.Http, the global http object for the batch. 746 """ 747 # For the credentials to refresh, but only once per refresh_token 748 # If there is no http per the request then refresh the http passed in 749 # via execute() 750 creds = None 751 if request.http is not None and hasattr(request.http.request, 752 'credentials'): 753 creds = request.http.request.credentials 754 elif http is not None and hasattr(http.request, 'credentials'): 755 creds = http.request.credentials 756 if creds is not None: 757 if id(creds) not in self._refreshed_credentials: 758 creds.refresh(http) 759 self._refreshed_credentials[id(creds)] = 1 760 761 # Only apply the credentials if we are using the http object passed in, 762 # otherwise apply() will get called during _serialize_request(). 763 if request.http is None or not hasattr(request.http.request, 764 'credentials'): 765 creds.apply(request.headers)

766

767 - def _id_to_header(self, id_):

768 """Convert an id to a Content-ID header value. 769 770 Args: 771 id_: string, identifier of individual request. 772 773 Returns: 774 A Content-ID header with the id_ encoded into it. A UUID is prepended to 775 the value because Content-ID headers are supposed to be universally 776 unique. 777 """ 778 if self._base_id is None: 779 self._base_id = uuid.uuid4() 780 781 return '<%s+%s>' % (self._base_id, urllib.quote(id_))

782

783 - def _header_to_id(self, header):

784 """Convert a Content-ID header value to an id. 785 786 Presumes the Content-ID header conforms to the format that _id_to_header() 787 returns. 788 789 Args: 790 header: string, Content-ID header value. 791 792 Returns: 793 The extracted id value. 794 795 Raises: 796 BatchError if the header is not in the expected format. 797 """ 798 if header[0] != '<' or header[-1] != '>': 799 raise BatchError("Invalid value for Content-ID: %s" % header) 800 if '+' not in header: 801 raise BatchError("Invalid value for Content-ID: %s" % header) 802 base, id_ = header[1:-1].rsplit('+', 1) 803 804 return urllib.unquote(id_)

805

806 - def _serialize_request(self, request):

807 """Convert an HttpRequest object into a string. 808 809 Args: 810 request: HttpRequest, the request to serialize. 811 812 Returns: 813 The request as a string in application/http format. 814 """ 815 # Construct status line 816 parsed = urlparse.urlparse(request.uri) 817 request_line = urlparse.urlunparse( 818 (None, None, parsed.path, parsed.params, parsed.query, None) 819 ) 820 status_line = request.method + ' ' + request_line + ' HTTP/1.1\n' 821 major, minor = request.headers.get('content-type', 'application/json').split('/') 822 msg = MIMENonMultipart(major, minor) 823 headers = request.headers.copy() 824 825 if request.http is not None and hasattr(request.http.request, 826 'credentials'): 827 request.http.request.credentials.apply(headers) 828 829 # MIMENonMultipart adds its own Content-Type header. 830 if 'content-type' in headers: 831 del headers['content-type'] 832 833 for key, value in headers.iteritems(): 834 msg[key] = value 835 msg['Host'] = parsed.netloc 836 msg.set_unixfrom(None) 837 838 if request.body is not None: 839 msg.set_payload(request.body) 840 msg['content-length'] = str(len(request.body)) 841 842 # Serialize the mime message. 843 fp = StringIO.StringIO() 844 # maxheaderlen=0 means don't line wrap headers. 845 g = Generator(fp, maxheaderlen=0) 846 g.flatten(msg, unixfrom=False) 847 body = fp.getvalue() 848 849 # Strip off the \n\n that the MIME lib tacks onto the end of the payload. 850 if request.body is None: 851 body = body[:-2] 852 853 return status_line.encode('utf-8') + body

854

855 - def _deserialize_response(self, payload):

856 """Convert string into httplib2 response and content. 857 858 Args: 859 payload: string, headers and body as a string. 860 861 Returns: 862 A pair (resp, content) like would be returned from httplib2.request. 863 """ 864 # Strip off the status line 865 status_line, payload = payload.split('\n', 1) 866 protocol, status, reason = status_line.split(' ', 2) 867 868 # Parse the rest of the response 869 parser = FeedParser() 870 parser.feed(payload) 871 msg = parser.close() 872 msg['status'] = status 873 874 # Create httplib2.Response from the parsed headers. 875 resp = httplib2.Response(msg) 876 resp.reason = reason 877 resp.version = int(protocol.split('/', 1)[1].replace('.', '')) 878 879 content = payload.split('\r\n\r\n', 1)[1] 880 881 return resp, content

882

883 - def _new_id(self):

884 """Create a new id. 885 886 Auto incrementing number that avoids conflicts with ids already used. 887 888 Returns: 889 string, a new unique id. 890 """ 891 self._last_auto_id += 1 892 while str(self._last_auto_id) in self._requests: 893 self._last_auto_id += 1 894 return str(self._last_auto_id)

895

896 - def add(self, request, callback=None, request_id=None):

897 """Add a new request. 898 899 Every callback added will be paired with a unique id, the request_id. That 900 unique id will be passed back to the callback when the response comes back 901 from the server. The default behavior is to have the library generate it's 902 own unique id. If the caller passes in a request_id then they must ensure 903 uniqueness for each request_id, and if they are not an exception is 904 raised. Callers should either supply all request_ids or nevery supply a 905 request id, to avoid such an error. 906 907 Args: 908 request: HttpRequest, Request to add to the batch. 909 callback: callable, A callback to be called for this response, of the 910 form callback(id, response). The first parameter is the request id, and 911 the second is the deserialized response object. 912 request_id: string, A unique id for the request. The id will be passed to 913 the callback with the response. 914 915 Returns: 916 None 917 918 Raises: 919 BatchError if a resumable request is added to a batch. 920 KeyError is the request_id is not unique. 921 """ 922 if request_id is None: 923 request_id = self._new_id() 924 if request.resumable is not None: 925 raise BatchError("Resumable requests cannot be used in a batch request.") 926 if request_id in self._requests: 927 raise KeyError("A request with this ID already exists: %s" % request_id) 928 self._requests[request_id] = request 929 self._callbacks[request_id] = callback 930 self._order.append(request_id)

931

932 - def _execute(self, http, order, requests):

933 """Serialize batch request, send to server, process response. 934 935 Args: 936 http: httplib2.Http, an http object to be used to make the request with. 937 order: list, list of request ids in the order they were added to the 938 batch. 939 request: list, list of request objects to send. 940 941 Raises: 942 httplib2.Error if a transport error has occured. 943 apiclient.errors.BatchError if the response is the wrong format. 944 """ 945 message = MIMEMultipart('mixed') 946 # Message should not write out it's own headers. 947 setattr(message, '_write_headers', lambda self: None) 948 949 # Add all the individual requests. 950 for request_id in order: 951 request = requests[request_id] 952 953 msg = MIMENonMultipart('application', 'http') 954 msg['Content-Transfer-Encoding'] = 'binary' 955 msg['Content-ID'] = self._id_to_header(request_id) 956 957 body = self._serialize_request(request) 958 msg.set_payload(body) 959 message.attach(msg) 960 961 body = message.as_string() 962 963 headers = {} 964 headers['content-type'] = ('multipart/mixed; ' 965 'boundary="%s"') % message.get_boundary() 966 967 resp, content = http.request(self._batch_uri, 'POST', body=body, 968 headers=headers) 969 970 if resp.status >= 300: 971 raise HttpError(resp, content, self._batch_uri) 972 973 # Now break out the individual responses and store each one. 974 boundary, _ = content.split(None, 1) 975 976 # Prepend with a content-type header so FeedParser can handle it. 977 header = 'content-type: %s\r\n\r\n' % resp['content-type'] 978 for_parser = header + content 979 980 parser = FeedParser() 981 parser.feed(for_parser) 982 mime_response = parser.close() 983 984 if not mime_response.is_multipart(): 985 raise BatchError("Response not in multipart/mixed format.", resp, 986 content) 987 988 for part in mime_response.get_payload(): 989 request_id = self._header_to_id(part['Content-ID']) 990 headers, content = self._deserialize_response(part.get_payload()) 991 self._responses[request_id] = (headers, content)

992

993 - def execute(self, http=None):

994 """Execute all the requests as a single batched HTTP request. 995 996 Args: 997 http: httplib2.Http, an http object to be used in place of the one the 998 HttpRequest request object was constructed with. If one isn't supplied 999 then use a http object from the requests in this batch. 1000 1001 Returns: 1002 None 1003 1004 Raises: 1005 httplib2.Error if a transport error has occured. 1006 apiclient.errors.BatchError if the response is the wrong format. 1007 """ 1008 1009 # If http is not supplied use the first valid one given in the requests. 1010 if http is None: 1011 for request_id in self._order: 1012 request = self._requests[request_id] 1013 if request is not None: 1014 http = request.http 1015 break 1016 1017 if http is None: 1018 raise ValueError("Missing a valid http object.") 1019 1020 self._execute(http, self._order, self._requests) 1021 1022 # Loop over all the requests and check for 401s. For each 401 request the 1023 # credentials should be refreshed and then sent again in a separate batch. 1024 redo_requests = {} 1025 redo_order = [] 1026 1027 for request_id in self._order: 1028 headers, content = self._responses[request_id] 1029 if headers['status'] == '401': 1030 redo_order.append(request_id) 1031 request = self._requests[request_id] 1032 self._refresh_and_apply_credentials(request, http) 1033 redo_requests[request_id] = request 1034 1035 if redo_requests: 1036 self._execute(http, redo_order, redo_requests) 1037 1038 # Now process all callbacks that are erroring, and raise an exception for 1039 # ones that return a non-2xx response? Or add extra parameter to callback 1040 # that contains an HttpError? 1041 1042 for request_id in self._order: 1043 headers, content = self._responses[request_id] 1044 1045 request = self._requests[request_id] 1046 callback = self._callbacks[request_id] 1047 1048 response = None 1049 exception = None 1050 try: 1051 r = httplib2.Response(headers) 1052 response = request.postproc(r, content) 1053 except HttpError, e: 1054 exception = e 1055 1056 if callback is not None: 1057 callback(request_id, response, exception) 1058 if self._callback is not None: 1059 self._callback(request_id, response, exception)

1060

1061 1062 -class HttpRequestMock(object):

1063 """Mock of HttpRequest. 1064 1065 Do not construct directly, instead use RequestMockBuilder. 1066 """ 1067

1068 - def __init__(self, resp, content, postproc):

1069 """Constructor for HttpRequestMock 1070 1071 Args: 1072 resp: httplib2.Response, the response to emulate coming from the request 1073 content: string, the response body 1074 postproc: callable, the post processing function usually supplied by 1075 the model class. See model.JsonModel.response() as an example. 1076 """ 1077 self.resp = resp 1078 self.content = content 1079 self.postproc = postproc 1080 if resp is None: 1081 self.resp = httplib2.Response({'status': 200, 'reason': 'OK'}) 1082 if 'reason' in self.resp: 1083 self.resp.reason = self.resp['reason']

1084

1085 - def execute(self, http=None):

1086 """Execute the request. 1087 1088 Same behavior as HttpRequest.execute(), but the response is 1089 mocked and not really from an HTTP request/response. 1090 """ 1091 return self.postproc(self.resp, self.content)

1092

1093 1094 -class RequestMockBuilder(object):

1095 """A simple mock of HttpRequest 1096 1097 Pass in a dictionary to the constructor that maps request methodIds to 1098 tuples of (httplib2.Response, content, opt_expected_body) that should be 1099 returned when that method is called. None may also be passed in for the 1100 httplib2.Response, in which case a 200 OK response will be generated. 1101 If an opt_expected_body (str or dict) is provided, it will be compared to 1102 the body and UnexpectedBodyError will be raised on inequality. 1103 1104 Example: 1105 response = '{"data": {"id": "tag:google.c...' 1106 requestBuilder = RequestMockBuilder( 1107 { 1108 'plus.activities.get': (None, response), 1109 } 1110 ) 1111 apiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder) 1112 1113 Methods that you do not supply a response for will return a 1114 200 OK with an empty string as the response content or raise an excpetion 1115 if check_unexpected is set to True. The methodId is taken from the rpcName 1116 in the discovery document. 1117 1118 For more details see the project wiki. 1119 """ 1120

1121 - def __init__(self, responses, check_unexpected=False):

1122 """Constructor for RequestMockBuilder 1123 1124 The constructed object should be a callable object 1125 that can replace the class HttpResponse. 1126 1127 responses - A dictionary that maps methodIds into tuples 1128 of (httplib2.Response, content). The methodId 1129 comes from the 'rpcName' field in the discovery 1130 document. 1131 check_unexpected - A boolean setting whether or not UnexpectedMethodError 1132 should be raised on unsupplied method. 1133 """ 1134 self.responses = responses 1135 self.check_unexpected = check_unexpected

1136

1137 - def __call__(self, http, postproc, uri, method='GET', body=None, 1138 headers=None, methodId=None, resumable=None):

1139 """Implements the callable interface that discovery.build() expects 1140 of requestBuilder, which is to build an object compatible with 1141 HttpRequest.execute(). See that method for the description of the 1142 parameters and the expected response. 1143 """ 1144 if methodId in self.responses: 1145 response = self.responses[methodId] 1146 resp, content = response[:2] 1147 if len(response) > 2: 1148 # Test the body against the supplied expected_body. 1149 expected_body = response[2] 1150 if bool(expected_body) != bool(body): 1151 # Not expecting a body and provided one 1152 # or expecting a body and not provided one. 1153 raise UnexpectedBodyError(expected_body, body) 1154 if isinstance(expected_body, str): 1155 expected_body = simplejson.loads(expected_body) 1156 body = simplejson.loads(body) 1157 if body != expected_body: 1158 raise UnexpectedBodyError(expected_body, body) 1159 return HttpRequestMock(resp, content, postproc) 1160 elif self.check_unexpected: 1161 raise UnexpectedMethodError(methodId) 1162 else: 1163 model = JsonModel(False) 1164 return HttpRequestMock(None, '{}', model.response)

1165

1166 1167 -class HttpMock(object):

1168 """Mock of httplib2.Http""" 1169

1170 - def __init__(self, filename, headers=None):

1171 """ 1172 Args: 1173 filename: string, absolute filename to read response from 1174 headers: dict, header to return with response 1175 """ 1176 if headers is None: 1177 headers = {'status': '200 OK'} 1178 f = file(filename, 'r') 1179 self.data = f.read() 1180 f.close() 1181 self.headers = headers

1182

1183 - def request(self, uri, 1184 method='GET', 1185 body=None, 1186 headers=None, 1187 redirections=1, 1188 connection_type=None):

1189 return httplib2.Response(self.headers), self.data

1190

1191 1192 -class HttpMockSequence(object):

1193 """Mock of httplib2.Http 1194 1195 Mocks a sequence of calls to request returning different responses for each 1196 call. Create an instance initialized with the desired response headers 1197 and content and then use as if an httplib2.Http instance. 1198 1199 http = HttpMockSequence([ 1200 ({'status': '401'}, ''), 1201 ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'), 1202 ({'status': '200'}, 'echo_request_headers'), 1203 ]) 1204 resp, content = http.request("http://examples.com") 1205 1206 There are special values you can pass in for content to trigger 1207 behavours that are helpful in testing. 1208 1209 'echo_request_headers' means return the request headers in the response body 1210 'echo_request_headers_as_json' means return the request headers in 1211 the response body 1212 'echo_request_body' means return the request body in the response body 1213 'echo_request_uri' means return the request uri in the response body 1214 """ 1215

1216 - def __init__(self, iterable):

1217 """ 1218 Args: 1219 iterable: iterable, a sequence of pairs of (headers, body) 1220 """ 1221 self._iterable = iterable

1222

1223 - def request(self, uri, 1224 method='GET', 1225 body=None, 1226 headers=None, 1227 redirections=1, 1228 connection_type=None):

1229 resp, content = self._iterable.pop(0) 1230 if content == 'echo_request_headers': 1231 content = headers 1232 elif content == 'echo_request_headers_as_json': 1233 content = simplejson.dumps(headers) 1234 elif content == 'echo_request_body': 1235 content = body 1236 elif content == 'echo_request_uri': 1237 content = uri 1238 return httplib2.Response(resp), content

1239

1240 1241 -def set_user_agent(http, user_agent):

1242 """Set the user-agent on every request. 1243 1244 Args: 1245 http - An instance of httplib2.Http 1246 or something that acts like it. 1247 user_agent: string, the value for the user-agent header. 1248 1249 Returns: 1250 A modified instance of http that was passed in. 1251 1252 Example: 1253 1254 h = httplib2.Http() 1255 h = set_user_agent(h, "my-app-name/6.0") 1256 1257 Most of the time the user-agent will be set doing auth, this is for the rare 1258 cases where you are accessing an unauthenticated endpoint. 1259 """ 1260 request_orig = http.request 1261 1262 # The closure that will replace 'httplib2.Http.request'. 1263 def new_request(uri, method='GET', body=None, headers=None, 1264 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1265 connection_type=None): 1266 """Modify the request headers to add the user-agent.""" 1267 if headers is None: 1268 headers = {} 1269 if 'user-agent' in headers: 1270 headers['user-agent'] = user_agent + ' ' + headers['user-agent'] 1271 else: 1272 headers['user-agent'] = user_agent 1273 resp, content = request_orig(uri, method, body, headers, 1274 redirections, connection_type) 1275 return resp, content

1276 1277 http.request = new_request 1278 return http 1279

1280 1281 -def tunnel_patch(http):

1282 """Tunnel PATCH requests over POST. 1283 Args: 1284 http - An instance of httplib2.Http 1285 or something that acts like it. 1286 1287 Returns: 1288 A modified instance of http that was passed in. 1289 1290 Example: 1291 1292 h = httplib2.Http() 1293 h = tunnel_patch(h, "my-app-name/6.0") 1294 1295 Useful if you are running on a platform that doesn't support PATCH. 1296 Apply this last if you are using OAuth 1.0, as changing the method 1297 will result in a different signature. 1298 """ 1299 request_orig = http.request 1300 1301 # The closure that will replace 'httplib2.Http.request'. 1302 def new_request(uri, method='GET', body=None, headers=None, 1303 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1304 connection_type=None): 1305 """Modify the request headers to add the user-agent.""" 1306 if headers is None: 1307 headers = {} 1308 if method == 'PATCH': 1309 if 'oauth_token' in headers.get('authorization', ''): 1310 logging.warning( 1311 'OAuth 1.0 request made with Credentials after tunnel_patch.') 1312 headers['x-http-method-override'] = "PATCH" 1313 method = 'POST' 1314 resp, content = request_orig(uri, method, body, headers, 1315 redirections, connection_type) 1316 return resp, content

1317 1318 http.request = new_request 1319 return http 1320

Source Code for Module apiclient.http