blob: 8ea5c73bbcbb0b04aeb860aa8944eb25e9ce9b28 [file] [log] [blame]
Joe Gregorio20a5aa92011-04-01 17:44:25 -04001# Copyright (C) 2010 Google Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
Joe Gregorioc5c5a372010-09-22 11:42:32 -040014
Joe Gregorioaf276d22010-12-09 14:26:58 -050015"""Classes to encapsulate a single HTTP request.
Joe Gregorioc5c5a372010-09-22 11:42:32 -040016
Joe Gregorioaf276d22010-12-09 14:26:58 -050017The classes implement a command pattern, with every
18object supporting an execute() method that does the
19actuall HTTP request.
Joe Gregorioc5c5a372010-09-22 11:42:32 -040020"""
21
22__author__ = 'jcgregorio@google.com (Joe Gregorio)'
Joe Gregorioaf276d22010-12-09 14:26:58 -050023__all__ = [
Joe Gregoriocb8103d2011-02-11 23:20:52 -050024 'HttpRequest', 'RequestMockBuilder', 'HttpMock'
Joe Gregoriof4153422011-03-18 22:45:18 -040025 'set_user_agent', 'tunnel_patch'
Joe Gregorioaf276d22010-12-09 14:26:58 -050026 ]
27
Joe Gregorio66f57522011-11-30 11:00:00 -050028import StringIO
Joe Gregoriod0bd3882011-11-22 09:49:47 -050029import copy
Joe Gregorio66f57522011-11-30 11:00:00 -050030import gzip
Joe Gregorioc6722462010-12-20 14:29:28 -050031import httplib2
Joe Gregoriod0bd3882011-11-22 09:49:47 -050032import mimeparse
33import mimetypes
Joe Gregorio66f57522011-11-30 11:00:00 -050034import os
35import urllib
36import urlparse
37import uuid
Joe Gregoriocb8103d2011-02-11 23:20:52 -050038
Joe Gregorio66f57522011-11-30 11:00:00 -050039from anyjson import simplejson
40from email.mime.multipart import MIMEMultipart
41from email.mime.nonmultipart import MIMENonMultipart
42from email.parser import FeedParser
43from errors import BatchError
Joe Gregorio49396552011-03-08 10:39:00 -050044from errors import HttpError
Joe Gregoriod0bd3882011-11-22 09:49:47 -050045from errors import ResumableUploadError
Joe Gregorioa388ce32011-09-09 17:19:13 -040046from errors import UnexpectedBodyError
47from errors import UnexpectedMethodError
Joe Gregorio66f57522011-11-30 11:00:00 -050048from model import JsonModel
Joe Gregorioc5c5a372010-09-22 11:42:32 -040049
50
Joe Gregoriod0bd3882011-11-22 09:49:47 -050051class MediaUploadProgress(object):
52 """Status of a resumable upload."""
53
54 def __init__(self, resumable_progress, total_size):
55 """Constructor.
56
57 Args:
58 resumable_progress: int, bytes sent so far.
59 total_size: int, total bytes in complete upload.
60 """
61 self.resumable_progress = resumable_progress
62 self.total_size = total_size
63
64 def progress(self):
65 """Percent of upload completed, as a float."""
Joe Gregorio66f57522011-11-30 11:00:00 -050066 return float(self.resumable_progress) / float(self.total_size)
Joe Gregoriod0bd3882011-11-22 09:49:47 -050067
68
69class MediaUpload(object):
70 """Describes a media object to upload.
71
72 Base class that defines the interface of MediaUpload subclasses.
73 """
74
75 def getbytes(self, begin, end):
76 raise NotImplementedError()
77
78 def size(self):
79 raise NotImplementedError()
80
81 def chunksize(self):
82 raise NotImplementedError()
83
84 def mimetype(self):
85 return 'application/octet-stream'
86
87 def resumable(self):
88 return False
89
90 def _to_json(self, strip=None):
91 """Utility function for creating a JSON representation of a MediaUpload.
92
93 Args:
94 strip: array, An array of names of members to not include in the JSON.
95
96 Returns:
97 string, a JSON representation of this instance, suitable to pass to
98 from_json().
99 """
100 t = type(self)
101 d = copy.copy(self.__dict__)
102 if strip is not None:
103 for member in strip:
104 del d[member]
105 d['_class'] = t.__name__
106 d['_module'] = t.__module__
107 return simplejson.dumps(d)
108
109 def to_json(self):
110 """Create a JSON representation of an instance of MediaUpload.
111
112 Returns:
113 string, a JSON representation of this instance, suitable to pass to
114 from_json().
115 """
116 return self._to_json()
117
118 @classmethod
119 def new_from_json(cls, s):
120 """Utility class method to instantiate a MediaUpload subclass from a JSON
121 representation produced by to_json().
122
123 Args:
124 s: string, JSON from to_json().
125
126 Returns:
127 An instance of the subclass of MediaUpload that was serialized with
128 to_json().
129 """
130 data = simplejson.loads(s)
131 # Find and call the right classmethod from_json() to restore the object.
132 module = data['_module']
133 m = __import__(module, fromlist=module.split('.')[:-1])
134 kls = getattr(m, data['_class'])
135 from_json = getattr(kls, 'from_json')
136 return from_json(s)
137
Joe Gregorio66f57522011-11-30 11:00:00 -0500138
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500139class MediaFileUpload(MediaUpload):
140 """A MediaUpload for a file.
141
142 Construct a MediaFileUpload and pass as the media_body parameter of the
143 method. For example, if we had a service that allowed uploading images:
144
145
146 media = MediaFileUpload('smiley.png', mimetype='image/png', chunksize=1000,
147 resumable=True)
148 service.objects().insert(
149 bucket=buckets['items'][0]['id'],
150 name='smiley.png',
151 media_body=media).execute()
152 """
153
154 def __init__(self, filename, mimetype=None, chunksize=10000, resumable=False):
155 """Constructor.
156
157 Args:
158 filename: string, Name of the file.
159 mimetype: string, Mime-type of the file. If None then a mime-type will be
160 guessed from the file extension.
161 chunksize: int, File will be uploaded in chunks of this many bytes. Only
162 used if resumable=True.
Joe Gregorio66f57522011-11-30 11:00:00 -0500163 resumable: bool, True if this is a resumable upload. False means upload
164 in a single request.
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500165 """
166 self._filename = filename
167 self._size = os.path.getsize(filename)
168 self._fd = None
169 if mimetype is None:
170 (mimetype, encoding) = mimetypes.guess_type(filename)
171 self._mimetype = mimetype
172 self._chunksize = chunksize
173 self._resumable = resumable
174
175 def mimetype(self):
176 return self._mimetype
177
178 def size(self):
179 return self._size
180
181 def chunksize(self):
182 return self._chunksize
183
184 def resumable(self):
185 return self._resumable
186
187 def getbytes(self, begin, length):
188 """Get bytes from the media.
189
190 Args:
191 begin: int, offset from beginning of file.
192 length: int, number of bytes to read, starting at begin.
193
194 Returns:
195 A string of bytes read. May be shorted than length if EOF was reached
196 first.
197 """
198 if self._fd is None:
199 self._fd = open(self._filename, 'rb')
200 self._fd.seek(begin)
201 return self._fd.read(length)
202
203 def to_json(self):
204 """Creating a JSON representation of an instance of Credentials.
205
206 Returns:
207 string, a JSON representation of this instance, suitable to pass to
208 from_json().
209 """
210 return self._to_json(['_fd'])
211
212 @staticmethod
213 def from_json(s):
214 d = simplejson.loads(s)
215 return MediaFileUpload(
216 d['_filename'], d['_mimetype'], d['_chunksize'], d['_resumable'])
217
218
Joe Gregorioc5c5a372010-09-22 11:42:32 -0400219class HttpRequest(object):
Joe Gregorio66f57522011-11-30 11:00:00 -0500220 """Encapsulates a single HTTP request."""
Joe Gregorioc5c5a372010-09-22 11:42:32 -0400221
Joe Gregoriodeeb0202011-02-15 14:49:57 -0500222 def __init__(self, http, postproc, uri,
Joe Gregorio7c22ab22011-02-16 15:32:39 -0500223 method='GET',
Joe Gregoriodeeb0202011-02-15 14:49:57 -0500224 body=None,
225 headers=None,
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500226 methodId=None,
227 resumable=None):
Joe Gregorioaf276d22010-12-09 14:26:58 -0500228 """Constructor for an HttpRequest.
229
Joe Gregorioaf276d22010-12-09 14:26:58 -0500230 Args:
231 http: httplib2.Http, the transport object to use to make a request
Joe Gregorioabda96f2011-02-11 20:19:33 -0500232 postproc: callable, called on the HTTP response and content to transform
233 it into a data object before returning, or raising an exception
234 on an error.
Joe Gregorioaf276d22010-12-09 14:26:58 -0500235 uri: string, the absolute URI to send the request to
236 method: string, the HTTP method to use
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500237 body: string, the request body of the HTTP request,
Joe Gregorioaf276d22010-12-09 14:26:58 -0500238 headers: dict, the HTTP request headers
Joe Gregorioaf276d22010-12-09 14:26:58 -0500239 methodId: string, a unique identifier for the API method being called.
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500240 resumable: MediaUpload, None if this is not a resumbale request.
Joe Gregorioaf276d22010-12-09 14:26:58 -0500241 """
Joe Gregorioc5c5a372010-09-22 11:42:32 -0400242 self.uri = uri
243 self.method = method
244 self.body = body
245 self.headers = headers or {}
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500246 self.methodId = methodId
Joe Gregorioc5c5a372010-09-22 11:42:32 -0400247 self.http = http
248 self.postproc = postproc
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500249 self.resumable = resumable
250
Joe Gregorio66f57522011-11-30 11:00:00 -0500251 # Pull the multipart boundary out of the content-type header.
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500252 major, minor, params = mimeparse.parse_mime_type(
253 headers.get('content-type', 'application/json'))
Joe Gregoriobd512b52011-12-06 15:39:26 -0500254
255 # Terminating multipart boundary get a trailing '--' appended.
256 self.multipart_boundary = params.get('boundary', '').strip('"') + '--'
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500257
258 # If this was a multipart resumable, the size of the non-media part.
259 self.multipart_size = 0
260
261 # The resumable URI to send chunks to.
262 self.resumable_uri = None
263
264 # The bytes that have been uploaded.
265 self.resumable_progress = 0
266
Joe Gregorio66f57522011-11-30 11:00:00 -0500267 self.total_size = 0
268
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500269 if resumable is not None:
270 if self.body is not None:
271 self.multipart_size = len(self.body)
272 else:
273 self.multipart_size = 0
Joe Gregorio66f57522011-11-30 11:00:00 -0500274 self.total_size = (
275 self.resumable.size() +
276 self.multipart_size +
277 len(self.multipart_boundary))
Joe Gregorioc5c5a372010-09-22 11:42:32 -0400278
279 def execute(self, http=None):
280 """Execute the request.
281
Joe Gregorioaf276d22010-12-09 14:26:58 -0500282 Args:
283 http: httplib2.Http, an http object to be used in place of the
284 one the HttpRequest request object was constructed with.
285
286 Returns:
287 A deserialized object model of the response body as determined
288 by the postproc.
289
290 Raises:
291 apiclient.errors.HttpError if the response was not a 2xx.
292 httplib2.Error if a transport error has occured.
Joe Gregorioc5c5a372010-09-22 11:42:32 -0400293 """
294 if http is None:
295 http = self.http
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500296 if self.resumable:
297 body = None
298 while body is None:
299 _, body = self.next_chunk(http)
300 return body
301 else:
302 resp, content = http.request(self.uri, self.method,
303 body=self.body,
304 headers=self.headers)
Joe Gregorio49396552011-03-08 10:39:00 -0500305
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500306 if resp.status >= 300:
307 raise HttpError(resp, content, self.uri)
Joe Gregorioc5c5a372010-09-22 11:42:32 -0400308 return self.postproc(resp, content)
Joe Gregorioaf276d22010-12-09 14:26:58 -0500309
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500310 def next_chunk(self, http=None):
311 """Execute the next step of a resumable upload.
312
Joe Gregorio66f57522011-11-30 11:00:00 -0500313 Can only be used if the method being executed supports media uploads and
314 the MediaUpload object passed in was flagged as using resumable upload.
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500315
316 Example:
317
Joe Gregorio66f57522011-11-30 11:00:00 -0500318 media = MediaFileUpload('smiley.png', mimetype='image/png',
319 chunksize=1000, resumable=True)
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500320 request = service.objects().insert(
321 bucket=buckets['items'][0]['id'],
322 name='smiley.png',
323 media_body=media)
324
325 response = None
326 while response is None:
327 status, response = request.next_chunk()
328 if status:
329 print "Upload %d%% complete." % int(status.progress() * 100)
330
331
332 Returns:
333 (status, body): (ResumableMediaStatus, object)
334 The body will be None until the resumable media is fully uploaded.
335 """
336 if http is None:
337 http = self.http
338
339 if self.resumable_uri is None:
340 start_headers = copy.copy(self.headers)
341 start_headers['X-Upload-Content-Type'] = self.resumable.mimetype()
342 start_headers['X-Upload-Content-Length'] = str(self.resumable.size())
343 start_headers['Content-Length'] = '0'
344 resp, content = http.request(self.uri, self.method,
345 body="",
346 headers=start_headers)
347 if resp.status == 200 and 'location' in resp:
348 self.resumable_uri = resp['location']
349 else:
350 raise ResumableUploadError("Failed to retrieve starting URI.")
351 if self.body:
352 begin = 0
353 data = self.body
354 else:
355 begin = self.resumable_progress - self.multipart_size
356 data = self.resumable.getbytes(begin, self.resumable.chunksize())
357
358 # Tack on the multipart/related boundary if we are at the end of the file.
359 if begin + self.resumable.chunksize() >= self.resumable.size():
360 data += self.multipart_boundary
361 headers = {
362 'Content-Range': 'bytes %d-%d/%d' % (
363 self.resumable_progress, self.resumable_progress + len(data) - 1,
364 self.total_size),
365 }
366 resp, content = http.request(self.resumable_uri, 'PUT',
367 body=data,
368 headers=headers)
369 if resp.status in [200, 201]:
370 return None, self.postproc(resp, content)
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500371 elif resp.status == 308:
Joe Gregorio66f57522011-11-30 11:00:00 -0500372 # A "308 Resume Incomplete" indicates we are not done.
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500373 self.resumable_progress = int(resp['range'].split('-')[1]) + 1
374 if self.resumable_progress >= self.multipart_size:
375 self.body = None
376 if 'location' in resp:
377 self.resumable_uri = resp['location']
378 else:
379 raise HttpError(resp, content, self.uri)
380
381 return MediaUploadProgress(self.resumable_progress, self.total_size), None
382
383 def to_json(self):
384 """Returns a JSON representation of the HttpRequest."""
385 d = copy.copy(self.__dict__)
386 if d['resumable'] is not None:
387 d['resumable'] = self.resumable.to_json()
388 del d['http']
389 del d['postproc']
390 return simplejson.dumps(d)
391
392 @staticmethod
393 def from_json(s, http, postproc):
394 """Returns an HttpRequest populated with info from a JSON object."""
395 d = simplejson.loads(s)
396 if d['resumable'] is not None:
397 d['resumable'] = MediaUpload.new_from_json(d['resumable'])
398 return HttpRequest(
399 http,
400 postproc,
Joe Gregorio66f57522011-11-30 11:00:00 -0500401 uri=d['uri'],
402 method=d['method'],
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500403 body=d['body'],
404 headers=d['headers'],
405 methodId=d['methodId'],
406 resumable=d['resumable'])
407
Joe Gregorioaf276d22010-12-09 14:26:58 -0500408
Joe Gregorio66f57522011-11-30 11:00:00 -0500409class BatchHttpRequest(object):
410 """Batches multiple HttpRequest objects into a single HTTP request."""
411
412 def __init__(self, callback=None, batch_uri=None):
413 """Constructor for a BatchHttpRequest.
414
415 Args:
416 callback: callable, A callback to be called for each response, of the
417 form callback(id, response). The first parameter is the request id, and
418 the second is the deserialized response object.
419 batch_uri: string, URI to send batch requests to.
420 """
421 if batch_uri is None:
422 batch_uri = 'https://www.googleapis.com/batch'
423 self._batch_uri = batch_uri
424
425 # Global callback to be called for each individual response in the batch.
426 self._callback = callback
427
428 # A map from id to (request, callback) pairs.
429 self._requests = {}
430
431 # List of request ids, in the order in which they were added.
432 self._order = []
433
434 # The last auto generated id.
435 self._last_auto_id = 0
436
437 # Unique ID on which to base the Content-ID headers.
438 self._base_id = None
439
440 def _id_to_header(self, id_):
441 """Convert an id to a Content-ID header value.
442
443 Args:
444 id_: string, identifier of individual request.
445
446 Returns:
447 A Content-ID header with the id_ encoded into it. A UUID is prepended to
448 the value because Content-ID headers are supposed to be universally
449 unique.
450 """
451 if self._base_id is None:
452 self._base_id = uuid.uuid4()
453
454 return '<%s+%s>' % (self._base_id, urllib.quote(id_))
455
456 def _header_to_id(self, header):
457 """Convert a Content-ID header value to an id.
458
459 Presumes the Content-ID header conforms to the format that _id_to_header()
460 returns.
461
462 Args:
463 header: string, Content-ID header value.
464
465 Returns:
466 The extracted id value.
467
468 Raises:
469 BatchError if the header is not in the expected format.
470 """
471 if header[0] != '<' or header[-1] != '>':
472 raise BatchError("Invalid value for Content-ID: %s" % header)
473 if '+' not in header:
474 raise BatchError("Invalid value for Content-ID: %s" % header)
475 base, id_ = header[1:-1].rsplit('+', 1)
476
477 return urllib.unquote(id_)
478
479 def _serialize_request(self, request):
480 """Convert an HttpRequest object into a string.
481
482 Args:
483 request: HttpRequest, the request to serialize.
484
485 Returns:
486 The request as a string in application/http format.
487 """
488 # Construct status line
489 parsed = urlparse.urlparse(request.uri)
490 request_line = urlparse.urlunparse(
491 (None, None, parsed.path, parsed.params, parsed.query, None)
492 )
493 status_line = request.method + ' ' + request_line + ' HTTP/1.1\n'
494 major, minor = request.headers.get('content-type', 'text/plain').split('/')
495 msg = MIMENonMultipart(major, minor)
496 headers = request.headers.copy()
497
498 # MIMENonMultipart adds its own Content-Type header.
499 if 'content-type' in headers:
500 del headers['content-type']
501
502 for key, value in headers.iteritems():
503 msg[key] = value
504 msg['Host'] = parsed.netloc
505 msg.set_unixfrom(None)
506
507 if request.body is not None:
508 msg.set_payload(request.body)
509
510 body = msg.as_string(False)
511 # Strip off the \n\n that the MIME lib tacks onto the end of the payload.
512 if request.body is None:
513 body = body[:-2]
514
515 return status_line + body
516
517 def _deserialize_response(self, payload):
518 """Convert string into httplib2 response and content.
519
520 Args:
521 payload: string, headers and body as a string.
522
523 Returns:
524 A pair (resp, content) like would be returned from httplib2.request.
525 """
526 # Strip off the status line
527 status_line, payload = payload.split('\n', 1)
528 protocol, status, reason = status_line.split(' ')
529
530 # Parse the rest of the response
531 parser = FeedParser()
532 parser.feed(payload)
533 msg = parser.close()
534 msg['status'] = status
535
536 # Create httplib2.Response from the parsed headers.
537 resp = httplib2.Response(msg)
538 resp.reason = reason
539 resp.version = int(protocol.split('/', 1)[1].replace('.', ''))
540
541 content = payload.split('\r\n\r\n', 1)[1]
542
543 return resp, content
544
545 def _new_id(self):
546 """Create a new id.
547
548 Auto incrementing number that avoids conflicts with ids already used.
549
550 Returns:
551 string, a new unique id.
552 """
553 self._last_auto_id += 1
554 while str(self._last_auto_id) in self._requests:
555 self._last_auto_id += 1
556 return str(self._last_auto_id)
557
558 def add(self, request, callback=None, request_id=None):
559 """Add a new request.
560
561 Every callback added will be paired with a unique id, the request_id. That
562 unique id will be passed back to the callback when the response comes back
563 from the server. The default behavior is to have the library generate it's
564 own unique id. If the caller passes in a request_id then they must ensure
565 uniqueness for each request_id, and if they are not an exception is
566 raised. Callers should either supply all request_ids or nevery supply a
567 request id, to avoid such an error.
568
569 Args:
570 request: HttpRequest, Request to add to the batch.
571 callback: callable, A callback to be called for this response, of the
572 form callback(id, response). The first parameter is the request id, and
573 the second is the deserialized response object.
574 request_id: string, A unique id for the request. The id will be passed to
575 the callback with the response.
576
577 Returns:
578 None
579
580 Raises:
581 BatchError if a resumable request is added to a batch.
582 KeyError is the request_id is not unique.
583 """
584 if request_id is None:
585 request_id = self._new_id()
586 if request.resumable is not None:
587 raise BatchError("Resumable requests cannot be used in a batch request.")
588 if request_id in self._requests:
589 raise KeyError("A request with this ID already exists: %s" % request_id)
590 self._requests[request_id] = (request, callback)
591 self._order.append(request_id)
592
593 def execute(self, http=None):
594 """Execute all the requests as a single batched HTTP request.
595
596 Args:
597 http: httplib2.Http, an http object to be used in place of the one the
598 HttpRequest request object was constructed with. If one isn't supplied
599 then use a http object from the requests in this batch.
600
601 Returns:
602 None
603
604 Raises:
605 apiclient.errors.HttpError if the response was not a 2xx.
606 httplib2.Error if a transport error has occured.
607 """
608 if http is None:
609 for request_id in self._order:
610 request, callback = self._requests[request_id]
611 if request is not None:
612 http = request.http
613 break
614 if http is None:
615 raise ValueError("Missing a valid http object.")
616
617
618 msgRoot = MIMEMultipart('mixed')
619 # msgRoot should not write out it's own headers
620 setattr(msgRoot, '_write_headers', lambda self: None)
621
622 # Add all the individual requests.
623 for request_id in self._order:
624 request, callback = self._requests[request_id]
625
626 msg = MIMENonMultipart('application', 'http')
627 msg['Content-Transfer-Encoding'] = 'binary'
628 msg['Content-ID'] = self._id_to_header(request_id)
629
630 body = self._serialize_request(request)
631 msg.set_payload(body)
632 msgRoot.attach(msg)
633
634 body = msgRoot.as_string()
635
636 headers = {}
637 headers['content-type'] = ('multipart/mixed; '
638 'boundary="%s"') % msgRoot.get_boundary()
639
640 resp, content = http.request(self._batch_uri, 'POST', body=body,
641 headers=headers)
642
643 if resp.status >= 300:
644 raise HttpError(resp, content, self._batch_uri)
645
646 # Now break up the response and process each one with the correct postproc
647 # and trigger the right callbacks.
648 boundary, _ = content.split(None, 1)
649
650 # Prepend with a content-type header so FeedParser can handle it.
651 header = 'Content-Type: %s\r\n\r\n' % resp['content-type']
652 content = header + content
653
654 parser = FeedParser()
655 parser.feed(content)
656 respRoot = parser.close()
657
658 if not respRoot.is_multipart():
659 raise BatchError("Response not in multipart/mixed format.")
660
661 parts = respRoot.get_payload()
662 for part in parts:
663 request_id = self._header_to_id(part['Content-ID'])
664
665 headers, content = self._deserialize_response(part.get_payload())
666
667 # TODO(jcgregorio) Remove this temporary hack once the server stops
668 # gzipping individual response bodies.
669 if content[0] != '{':
670 gzipped_content = content
671 content = gzip.GzipFile(
672 fileobj=StringIO.StringIO(gzipped_content)).read()
673
674 request, cb = self._requests[request_id]
675 postproc = request.postproc
676 response = postproc(resp, content)
677 if cb is not None:
678 cb(request_id, response)
679 if self._callback is not None:
680 self._callback(request_id, response)
681
682
Joe Gregorioaf276d22010-12-09 14:26:58 -0500683class HttpRequestMock(object):
684 """Mock of HttpRequest.
685
686 Do not construct directly, instead use RequestMockBuilder.
687 """
688
689 def __init__(self, resp, content, postproc):
690 """Constructor for HttpRequestMock
691
692 Args:
693 resp: httplib2.Response, the response to emulate coming from the request
694 content: string, the response body
695 postproc: callable, the post processing function usually supplied by
696 the model class. See model.JsonModel.response() as an example.
697 """
698 self.resp = resp
699 self.content = content
700 self.postproc = postproc
701 if resp is None:
Joe Gregorioc6722462010-12-20 14:29:28 -0500702 self.resp = httplib2.Response({'status': 200, 'reason': 'OK'})
Joe Gregorioaf276d22010-12-09 14:26:58 -0500703 if 'reason' in self.resp:
704 self.resp.reason = self.resp['reason']
705
706 def execute(self, http=None):
707 """Execute the request.
708
709 Same behavior as HttpRequest.execute(), but the response is
710 mocked and not really from an HTTP request/response.
711 """
712 return self.postproc(self.resp, self.content)
713
714
715class RequestMockBuilder(object):
716 """A simple mock of HttpRequest
717
718 Pass in a dictionary to the constructor that maps request methodIds to
Joe Gregorioa388ce32011-09-09 17:19:13 -0400719 tuples of (httplib2.Response, content, opt_expected_body) that should be
720 returned when that method is called. None may also be passed in for the
721 httplib2.Response, in which case a 200 OK response will be generated.
722 If an opt_expected_body (str or dict) is provided, it will be compared to
723 the body and UnexpectedBodyError will be raised on inequality.
Joe Gregorioaf276d22010-12-09 14:26:58 -0500724
725 Example:
726 response = '{"data": {"id": "tag:google.c...'
727 requestBuilder = RequestMockBuilder(
728 {
Joe Gregorioc4fc0952011-11-09 12:21:11 -0500729 'plus.activities.get': (None, response),
Joe Gregorioaf276d22010-12-09 14:26:58 -0500730 }
731 )
Joe Gregorioc4fc0952011-11-09 12:21:11 -0500732 apiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder)
Joe Gregorioaf276d22010-12-09 14:26:58 -0500733
734 Methods that you do not supply a response for will return a
Joe Gregorio66f57522011-11-30 11:00:00 -0500735 200 OK with an empty string as the response content or raise an excpetion
736 if check_unexpected is set to True. The methodId is taken from the rpcName
Joe Gregorioa388ce32011-09-09 17:19:13 -0400737 in the discovery document.
Joe Gregorioaf276d22010-12-09 14:26:58 -0500738
739 For more details see the project wiki.
740 """
741
Joe Gregorioa388ce32011-09-09 17:19:13 -0400742 def __init__(self, responses, check_unexpected=False):
Joe Gregorioaf276d22010-12-09 14:26:58 -0500743 """Constructor for RequestMockBuilder
744
745 The constructed object should be a callable object
746 that can replace the class HttpResponse.
747
748 responses - A dictionary that maps methodIds into tuples
749 of (httplib2.Response, content). The methodId
750 comes from the 'rpcName' field in the discovery
751 document.
Joe Gregorioa388ce32011-09-09 17:19:13 -0400752 check_unexpected - A boolean setting whether or not UnexpectedMethodError
753 should be raised on unsupplied method.
Joe Gregorioaf276d22010-12-09 14:26:58 -0500754 """
755 self.responses = responses
Joe Gregorioa388ce32011-09-09 17:19:13 -0400756 self.check_unexpected = check_unexpected
Joe Gregorioaf276d22010-12-09 14:26:58 -0500757
Joe Gregorio7c22ab22011-02-16 15:32:39 -0500758 def __call__(self, http, postproc, uri, method='GET', body=None,
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500759 headers=None, methodId=None, resumable=None):
Joe Gregorioaf276d22010-12-09 14:26:58 -0500760 """Implements the callable interface that discovery.build() expects
761 of requestBuilder, which is to build an object compatible with
762 HttpRequest.execute(). See that method for the description of the
763 parameters and the expected response.
764 """
765 if methodId in self.responses:
Joe Gregorioa388ce32011-09-09 17:19:13 -0400766 response = self.responses[methodId]
767 resp, content = response[:2]
768 if len(response) > 2:
769 # Test the body against the supplied expected_body.
770 expected_body = response[2]
771 if bool(expected_body) != bool(body):
772 # Not expecting a body and provided one
773 # or expecting a body and not provided one.
774 raise UnexpectedBodyError(expected_body, body)
775 if isinstance(expected_body, str):
776 expected_body = simplejson.loads(expected_body)
777 body = simplejson.loads(body)
778 if body != expected_body:
779 raise UnexpectedBodyError(expected_body, body)
Joe Gregorioaf276d22010-12-09 14:26:58 -0500780 return HttpRequestMock(resp, content, postproc)
Joe Gregorioa388ce32011-09-09 17:19:13 -0400781 elif self.check_unexpected:
782 raise UnexpectedMethodError(methodId)
Joe Gregorioaf276d22010-12-09 14:26:58 -0500783 else:
Joe Gregoriod433b2a2011-02-22 10:51:51 -0500784 model = JsonModel(False)
Joe Gregorioaf276d22010-12-09 14:26:58 -0500785 return HttpRequestMock(None, '{}', model.response)
Joe Gregoriocb8103d2011-02-11 23:20:52 -0500786
Joe Gregoriodeeb0202011-02-15 14:49:57 -0500787
Joe Gregoriocb8103d2011-02-11 23:20:52 -0500788class HttpMock(object):
789 """Mock of httplib2.Http"""
790
Joe Gregorioec343652011-02-16 16:52:51 -0500791 def __init__(self, filename, headers=None):
Joe Gregoriocb8103d2011-02-11 23:20:52 -0500792 """
793 Args:
794 filename: string, absolute filename to read response from
795 headers: dict, header to return with response
796 """
Joe Gregorioec343652011-02-16 16:52:51 -0500797 if headers is None:
798 headers = {'status': '200 OK'}
Joe Gregoriocb8103d2011-02-11 23:20:52 -0500799 f = file(filename, 'r')
800 self.data = f.read()
801 f.close()
802 self.headers = headers
803
Joe Gregoriodeeb0202011-02-15 14:49:57 -0500804 def request(self, uri,
Joe Gregorio7c22ab22011-02-16 15:32:39 -0500805 method='GET',
Joe Gregoriodeeb0202011-02-15 14:49:57 -0500806 body=None,
807 headers=None,
808 redirections=1,
809 connection_type=None):
Joe Gregoriocb8103d2011-02-11 23:20:52 -0500810 return httplib2.Response(self.headers), self.data
Joe Gregorioccc79542011-02-19 00:05:26 -0500811
812
813class HttpMockSequence(object):
814 """Mock of httplib2.Http
815
816 Mocks a sequence of calls to request returning different responses for each
817 call. Create an instance initialized with the desired response headers
818 and content and then use as if an httplib2.Http instance.
819
820 http = HttpMockSequence([
821 ({'status': '401'}, ''),
822 ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'),
823 ({'status': '200'}, 'echo_request_headers'),
824 ])
825 resp, content = http.request("http://examples.com")
826
827 There are special values you can pass in for content to trigger
828 behavours that are helpful in testing.
829
830 'echo_request_headers' means return the request headers in the response body
Joe Gregorioe9e236f2011-03-21 22:23:14 -0400831 'echo_request_headers_as_json' means return the request headers in
832 the response body
Joe Gregorioccc79542011-02-19 00:05:26 -0500833 'echo_request_body' means return the request body in the response body
Joe Gregorio0bc70912011-05-24 15:30:49 -0400834 'echo_request_uri' means return the request uri in the response body
Joe Gregorioccc79542011-02-19 00:05:26 -0500835 """
836
837 def __init__(self, iterable):
838 """
839 Args:
840 iterable: iterable, a sequence of pairs of (headers, body)
841 """
842 self._iterable = iterable
843
844 def request(self, uri,
845 method='GET',
846 body=None,
847 headers=None,
848 redirections=1,
849 connection_type=None):
850 resp, content = self._iterable.pop(0)
851 if content == 'echo_request_headers':
852 content = headers
Joe Gregoriof4153422011-03-18 22:45:18 -0400853 elif content == 'echo_request_headers_as_json':
854 content = simplejson.dumps(headers)
Joe Gregorioccc79542011-02-19 00:05:26 -0500855 elif content == 'echo_request_body':
856 content = body
Joe Gregorio0bc70912011-05-24 15:30:49 -0400857 elif content == 'echo_request_uri':
858 content = uri
Joe Gregorioccc79542011-02-19 00:05:26 -0500859 return httplib2.Response(resp), content
Joe Gregorio6bcbcea2011-03-10 15:26:05 -0500860
861
862def set_user_agent(http, user_agent):
Joe Gregoriof4153422011-03-18 22:45:18 -0400863 """Set the user-agent on every request.
864
Joe Gregorio6bcbcea2011-03-10 15:26:05 -0500865 Args:
866 http - An instance of httplib2.Http
867 or something that acts like it.
868 user_agent: string, the value for the user-agent header.
869
870 Returns:
871 A modified instance of http that was passed in.
872
873 Example:
874
875 h = httplib2.Http()
876 h = set_user_agent(h, "my-app-name/6.0")
877
878 Most of the time the user-agent will be set doing auth, this is for the rare
879 cases where you are accessing an unauthenticated endpoint.
880 """
881 request_orig = http.request
882
883 # The closure that will replace 'httplib2.Http.request'.
884 def new_request(uri, method='GET', body=None, headers=None,
885 redirections=httplib2.DEFAULT_MAX_REDIRECTS,
886 connection_type=None):
887 """Modify the request headers to add the user-agent."""
888 if headers is None:
889 headers = {}
890 if 'user-agent' in headers:
891 headers['user-agent'] = user_agent + ' ' + headers['user-agent']
892 else:
893 headers['user-agent'] = user_agent
894 resp, content = request_orig(uri, method, body, headers,
895 redirections, connection_type)
896 return resp, content
897
898 http.request = new_request
899 return http
Joe Gregoriof4153422011-03-18 22:45:18 -0400900
901
902def tunnel_patch(http):
903 """Tunnel PATCH requests over POST.
904 Args:
905 http - An instance of httplib2.Http
906 or something that acts like it.
907
908 Returns:
909 A modified instance of http that was passed in.
910
911 Example:
912
913 h = httplib2.Http()
914 h = tunnel_patch(h, "my-app-name/6.0")
915
916 Useful if you are running on a platform that doesn't support PATCH.
917 Apply this last if you are using OAuth 1.0, as changing the method
918 will result in a different signature.
919 """
920 request_orig = http.request
921
922 # The closure that will replace 'httplib2.Http.request'.
923 def new_request(uri, method='GET', body=None, headers=None,
924 redirections=httplib2.DEFAULT_MAX_REDIRECTS,
925 connection_type=None):
926 """Modify the request headers to add the user-agent."""
927 if headers is None:
928 headers = {}
929 if method == 'PATCH':
Joe Gregorio06d852b2011-03-25 15:03:10 -0400930 if 'oauth_token' in headers.get('authorization', ''):
Joe Gregorioe9e236f2011-03-21 22:23:14 -0400931 logging.warning(
Joe Gregorio06d852b2011-03-25 15:03:10 -0400932 'OAuth 1.0 request made with Credentials after tunnel_patch.')
Joe Gregoriof4153422011-03-18 22:45:18 -0400933 headers['x-http-method-override'] = "PATCH"
934 method = 'POST'
935 resp, content = request_orig(uri, method, body, headers,
936 redirections, connection_type)
937 return resp, content
938
939 http.request = new_request
940 return http