blob: 333461e3745e7c7f914ffc688dff520a1cda0c2f [file] [log] [blame]
Joe Gregorio20a5aa92011-04-01 17:44:25 -04001# Copyright (C) 2010 Google Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
Joe Gregorioc5c5a372010-09-22 11:42:32 -040014
Joe Gregorioaf276d22010-12-09 14:26:58 -050015"""Classes to encapsulate a single HTTP request.
Joe Gregorioc5c5a372010-09-22 11:42:32 -040016
Joe Gregorioaf276d22010-12-09 14:26:58 -050017The classes implement a command pattern, with every
18object supporting an execute() method that does the
19actuall HTTP request.
Joe Gregorioc5c5a372010-09-22 11:42:32 -040020"""
21
22__author__ = 'jcgregorio@google.com (Joe Gregorio)'
Joe Gregorioaf276d22010-12-09 14:26:58 -050023__all__ = [
Joe Gregoriocb8103d2011-02-11 23:20:52 -050024 'HttpRequest', 'RequestMockBuilder', 'HttpMock'
Joe Gregoriof4153422011-03-18 22:45:18 -040025 'set_user_agent', 'tunnel_patch'
Joe Gregorioaf276d22010-12-09 14:26:58 -050026 ]
27
Joe Gregorio66f57522011-11-30 11:00:00 -050028import StringIO
Joe Gregoriod0bd3882011-11-22 09:49:47 -050029import copy
Joe Gregorio66f57522011-11-30 11:00:00 -050030import gzip
Joe Gregorioc6722462010-12-20 14:29:28 -050031import httplib2
Joe Gregoriod0bd3882011-11-22 09:49:47 -050032import mimeparse
33import mimetypes
Joe Gregorio66f57522011-11-30 11:00:00 -050034import os
35import urllib
36import urlparse
37import uuid
Joe Gregoriocb8103d2011-02-11 23:20:52 -050038
Joe Gregorio66f57522011-11-30 11:00:00 -050039from anyjson import simplejson
40from email.mime.multipart import MIMEMultipart
41from email.mime.nonmultipart import MIMENonMultipart
42from email.parser import FeedParser
43from errors import BatchError
Joe Gregorio49396552011-03-08 10:39:00 -050044from errors import HttpError
Joe Gregoriod0bd3882011-11-22 09:49:47 -050045from errors import ResumableUploadError
Joe Gregorioa388ce32011-09-09 17:19:13 -040046from errors import UnexpectedBodyError
47from errors import UnexpectedMethodError
Joe Gregorio66f57522011-11-30 11:00:00 -050048from model import JsonModel
Joe Gregorioc5c5a372010-09-22 11:42:32 -040049
50
Joe Gregoriod0bd3882011-11-22 09:49:47 -050051class MediaUploadProgress(object):
52 """Status of a resumable upload."""
53
54 def __init__(self, resumable_progress, total_size):
55 """Constructor.
56
57 Args:
58 resumable_progress: int, bytes sent so far.
59 total_size: int, total bytes in complete upload.
60 """
61 self.resumable_progress = resumable_progress
62 self.total_size = total_size
63
64 def progress(self):
65 """Percent of upload completed, as a float."""
Joe Gregorio66f57522011-11-30 11:00:00 -050066 return float(self.resumable_progress) / float(self.total_size)
Joe Gregoriod0bd3882011-11-22 09:49:47 -050067
68
69class MediaUpload(object):
70 """Describes a media object to upload.
71
72 Base class that defines the interface of MediaUpload subclasses.
73 """
74
75 def getbytes(self, begin, end):
76 raise NotImplementedError()
77
78 def size(self):
79 raise NotImplementedError()
80
81 def chunksize(self):
82 raise NotImplementedError()
83
84 def mimetype(self):
85 return 'application/octet-stream'
86
87 def resumable(self):
88 return False
89
90 def _to_json(self, strip=None):
91 """Utility function for creating a JSON representation of a MediaUpload.
92
93 Args:
94 strip: array, An array of names of members to not include in the JSON.
95
96 Returns:
97 string, a JSON representation of this instance, suitable to pass to
98 from_json().
99 """
100 t = type(self)
101 d = copy.copy(self.__dict__)
102 if strip is not None:
103 for member in strip:
104 del d[member]
105 d['_class'] = t.__name__
106 d['_module'] = t.__module__
107 return simplejson.dumps(d)
108
109 def to_json(self):
110 """Create a JSON representation of an instance of MediaUpload.
111
112 Returns:
113 string, a JSON representation of this instance, suitable to pass to
114 from_json().
115 """
116 return self._to_json()
117
118 @classmethod
119 def new_from_json(cls, s):
120 """Utility class method to instantiate a MediaUpload subclass from a JSON
121 representation produced by to_json().
122
123 Args:
124 s: string, JSON from to_json().
125
126 Returns:
127 An instance of the subclass of MediaUpload that was serialized with
128 to_json().
129 """
130 data = simplejson.loads(s)
131 # Find and call the right classmethod from_json() to restore the object.
132 module = data['_module']
133 m = __import__(module, fromlist=module.split('.')[:-1])
134 kls = getattr(m, data['_class'])
135 from_json = getattr(kls, 'from_json')
136 return from_json(s)
137
Joe Gregorio66f57522011-11-30 11:00:00 -0500138
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500139class MediaFileUpload(MediaUpload):
140 """A MediaUpload for a file.
141
142 Construct a MediaFileUpload and pass as the media_body parameter of the
143 method. For example, if we had a service that allowed uploading images:
144
145
146 media = MediaFileUpload('smiley.png', mimetype='image/png', chunksize=1000,
147 resumable=True)
148 service.objects().insert(
149 bucket=buckets['items'][0]['id'],
150 name='smiley.png',
151 media_body=media).execute()
152 """
153
154 def __init__(self, filename, mimetype=None, chunksize=10000, resumable=False):
155 """Constructor.
156
157 Args:
158 filename: string, Name of the file.
159 mimetype: string, Mime-type of the file. If None then a mime-type will be
160 guessed from the file extension.
161 chunksize: int, File will be uploaded in chunks of this many bytes. Only
162 used if resumable=True.
Joe Gregorio66f57522011-11-30 11:00:00 -0500163 resumable: bool, True if this is a resumable upload. False means upload
164 in a single request.
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500165 """
166 self._filename = filename
167 self._size = os.path.getsize(filename)
168 self._fd = None
169 if mimetype is None:
170 (mimetype, encoding) = mimetypes.guess_type(filename)
171 self._mimetype = mimetype
172 self._chunksize = chunksize
173 self._resumable = resumable
174
175 def mimetype(self):
176 return self._mimetype
177
178 def size(self):
179 return self._size
180
181 def chunksize(self):
182 return self._chunksize
183
184 def resumable(self):
185 return self._resumable
186
187 def getbytes(self, begin, length):
188 """Get bytes from the media.
189
190 Args:
191 begin: int, offset from beginning of file.
192 length: int, number of bytes to read, starting at begin.
193
194 Returns:
195 A string of bytes read. May be shorted than length if EOF was reached
196 first.
197 """
198 if self._fd is None:
199 self._fd = open(self._filename, 'rb')
200 self._fd.seek(begin)
201 return self._fd.read(length)
202
203 def to_json(self):
204 """Creating a JSON representation of an instance of Credentials.
205
206 Returns:
207 string, a JSON representation of this instance, suitable to pass to
208 from_json().
209 """
210 return self._to_json(['_fd'])
211
212 @staticmethod
213 def from_json(s):
214 d = simplejson.loads(s)
215 return MediaFileUpload(
216 d['_filename'], d['_mimetype'], d['_chunksize'], d['_resumable'])
217
218
Joe Gregorioc5c5a372010-09-22 11:42:32 -0400219class HttpRequest(object):
Joe Gregorio66f57522011-11-30 11:00:00 -0500220 """Encapsulates a single HTTP request."""
Joe Gregorioc5c5a372010-09-22 11:42:32 -0400221
Joe Gregoriodeeb0202011-02-15 14:49:57 -0500222 def __init__(self, http, postproc, uri,
Joe Gregorio7c22ab22011-02-16 15:32:39 -0500223 method='GET',
Joe Gregoriodeeb0202011-02-15 14:49:57 -0500224 body=None,
225 headers=None,
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500226 methodId=None,
227 resumable=None):
Joe Gregorioaf276d22010-12-09 14:26:58 -0500228 """Constructor for an HttpRequest.
229
Joe Gregorioaf276d22010-12-09 14:26:58 -0500230 Args:
231 http: httplib2.Http, the transport object to use to make a request
Joe Gregorioabda96f2011-02-11 20:19:33 -0500232 postproc: callable, called on the HTTP response and content to transform
233 it into a data object before returning, or raising an exception
234 on an error.
Joe Gregorioaf276d22010-12-09 14:26:58 -0500235 uri: string, the absolute URI to send the request to
236 method: string, the HTTP method to use
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500237 body: string, the request body of the HTTP request,
Joe Gregorioaf276d22010-12-09 14:26:58 -0500238 headers: dict, the HTTP request headers
Joe Gregorioaf276d22010-12-09 14:26:58 -0500239 methodId: string, a unique identifier for the API method being called.
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500240 resumable: MediaUpload, None if this is not a resumbale request.
Joe Gregorioaf276d22010-12-09 14:26:58 -0500241 """
Joe Gregorioc5c5a372010-09-22 11:42:32 -0400242 self.uri = uri
243 self.method = method
244 self.body = body
245 self.headers = headers or {}
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500246 self.methodId = methodId
Joe Gregorioc5c5a372010-09-22 11:42:32 -0400247 self.http = http
248 self.postproc = postproc
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500249 self.resumable = resumable
250
Joe Gregorio66f57522011-11-30 11:00:00 -0500251 # Pull the multipart boundary out of the content-type header.
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500252 major, minor, params = mimeparse.parse_mime_type(
253 headers.get('content-type', 'application/json'))
254 self.multipart_boundary = params.get('boundary', '').strip('"')
255
256 # If this was a multipart resumable, the size of the non-media part.
257 self.multipart_size = 0
258
259 # The resumable URI to send chunks to.
260 self.resumable_uri = None
261
262 # The bytes that have been uploaded.
263 self.resumable_progress = 0
264
Joe Gregorio66f57522011-11-30 11:00:00 -0500265 self.total_size = 0
266
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500267 if resumable is not None:
268 if self.body is not None:
269 self.multipart_size = len(self.body)
270 else:
271 self.multipart_size = 0
Joe Gregorio66f57522011-11-30 11:00:00 -0500272 self.total_size = (
273 self.resumable.size() +
274 self.multipart_size +
275 len(self.multipart_boundary))
Joe Gregorioc5c5a372010-09-22 11:42:32 -0400276
277 def execute(self, http=None):
278 """Execute the request.
279
Joe Gregorioaf276d22010-12-09 14:26:58 -0500280 Args:
281 http: httplib2.Http, an http object to be used in place of the
282 one the HttpRequest request object was constructed with.
283
284 Returns:
285 A deserialized object model of the response body as determined
286 by the postproc.
287
288 Raises:
289 apiclient.errors.HttpError if the response was not a 2xx.
290 httplib2.Error if a transport error has occured.
Joe Gregorioc5c5a372010-09-22 11:42:32 -0400291 """
292 if http is None:
293 http = self.http
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500294 if self.resumable:
295 body = None
296 while body is None:
297 _, body = self.next_chunk(http)
298 return body
299 else:
300 resp, content = http.request(self.uri, self.method,
301 body=self.body,
302 headers=self.headers)
Joe Gregorio49396552011-03-08 10:39:00 -0500303
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500304 if resp.status >= 300:
305 raise HttpError(resp, content, self.uri)
Joe Gregorioc5c5a372010-09-22 11:42:32 -0400306 return self.postproc(resp, content)
Joe Gregorioaf276d22010-12-09 14:26:58 -0500307
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500308 def next_chunk(self, http=None):
309 """Execute the next step of a resumable upload.
310
Joe Gregorio66f57522011-11-30 11:00:00 -0500311 Can only be used if the method being executed supports media uploads and
312 the MediaUpload object passed in was flagged as using resumable upload.
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500313
314 Example:
315
Joe Gregorio66f57522011-11-30 11:00:00 -0500316 media = MediaFileUpload('smiley.png', mimetype='image/png',
317 chunksize=1000, resumable=True)
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500318 request = service.objects().insert(
319 bucket=buckets['items'][0]['id'],
320 name='smiley.png',
321 media_body=media)
322
323 response = None
324 while response is None:
325 status, response = request.next_chunk()
326 if status:
327 print "Upload %d%% complete." % int(status.progress() * 100)
328
329
330 Returns:
331 (status, body): (ResumableMediaStatus, object)
332 The body will be None until the resumable media is fully uploaded.
333 """
334 if http is None:
335 http = self.http
336
337 if self.resumable_uri is None:
338 start_headers = copy.copy(self.headers)
339 start_headers['X-Upload-Content-Type'] = self.resumable.mimetype()
340 start_headers['X-Upload-Content-Length'] = str(self.resumable.size())
341 start_headers['Content-Length'] = '0'
342 resp, content = http.request(self.uri, self.method,
343 body="",
344 headers=start_headers)
345 if resp.status == 200 and 'location' in resp:
346 self.resumable_uri = resp['location']
347 else:
348 raise ResumableUploadError("Failed to retrieve starting URI.")
349 if self.body:
350 begin = 0
351 data = self.body
352 else:
353 begin = self.resumable_progress - self.multipart_size
354 data = self.resumable.getbytes(begin, self.resumable.chunksize())
355
356 # Tack on the multipart/related boundary if we are at the end of the file.
357 if begin + self.resumable.chunksize() >= self.resumable.size():
358 data += self.multipart_boundary
359 headers = {
360 'Content-Range': 'bytes %d-%d/%d' % (
361 self.resumable_progress, self.resumable_progress + len(data) - 1,
362 self.total_size),
363 }
364 resp, content = http.request(self.resumable_uri, 'PUT',
365 body=data,
366 headers=headers)
367 if resp.status in [200, 201]:
368 return None, self.postproc(resp, content)
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500369 elif resp.status == 308:
Joe Gregorio66f57522011-11-30 11:00:00 -0500370 # A "308 Resume Incomplete" indicates we are not done.
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500371 self.resumable_progress = int(resp['range'].split('-')[1]) + 1
372 if self.resumable_progress >= self.multipart_size:
373 self.body = None
374 if 'location' in resp:
375 self.resumable_uri = resp['location']
376 else:
377 raise HttpError(resp, content, self.uri)
378
379 return MediaUploadProgress(self.resumable_progress, self.total_size), None
380
381 def to_json(self):
382 """Returns a JSON representation of the HttpRequest."""
383 d = copy.copy(self.__dict__)
384 if d['resumable'] is not None:
385 d['resumable'] = self.resumable.to_json()
386 del d['http']
387 del d['postproc']
388 return simplejson.dumps(d)
389
390 @staticmethod
391 def from_json(s, http, postproc):
392 """Returns an HttpRequest populated with info from a JSON object."""
393 d = simplejson.loads(s)
394 if d['resumable'] is not None:
395 d['resumable'] = MediaUpload.new_from_json(d['resumable'])
396 return HttpRequest(
397 http,
398 postproc,
Joe Gregorio66f57522011-11-30 11:00:00 -0500399 uri=d['uri'],
400 method=d['method'],
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500401 body=d['body'],
402 headers=d['headers'],
403 methodId=d['methodId'],
404 resumable=d['resumable'])
405
Joe Gregorioaf276d22010-12-09 14:26:58 -0500406
Joe Gregorio66f57522011-11-30 11:00:00 -0500407class BatchHttpRequest(object):
408 """Batches multiple HttpRequest objects into a single HTTP request."""
409
410 def __init__(self, callback=None, batch_uri=None):
411 """Constructor for a BatchHttpRequest.
412
413 Args:
414 callback: callable, A callback to be called for each response, of the
415 form callback(id, response). The first parameter is the request id, and
416 the second is the deserialized response object.
417 batch_uri: string, URI to send batch requests to.
418 """
419 if batch_uri is None:
420 batch_uri = 'https://www.googleapis.com/batch'
421 self._batch_uri = batch_uri
422
423 # Global callback to be called for each individual response in the batch.
424 self._callback = callback
425
426 # A map from id to (request, callback) pairs.
427 self._requests = {}
428
429 # List of request ids, in the order in which they were added.
430 self._order = []
431
432 # The last auto generated id.
433 self._last_auto_id = 0
434
435 # Unique ID on which to base the Content-ID headers.
436 self._base_id = None
437
438 def _id_to_header(self, id_):
439 """Convert an id to a Content-ID header value.
440
441 Args:
442 id_: string, identifier of individual request.
443
444 Returns:
445 A Content-ID header with the id_ encoded into it. A UUID is prepended to
446 the value because Content-ID headers are supposed to be universally
447 unique.
448 """
449 if self._base_id is None:
450 self._base_id = uuid.uuid4()
451
452 return '<%s+%s>' % (self._base_id, urllib.quote(id_))
453
454 def _header_to_id(self, header):
455 """Convert a Content-ID header value to an id.
456
457 Presumes the Content-ID header conforms to the format that _id_to_header()
458 returns.
459
460 Args:
461 header: string, Content-ID header value.
462
463 Returns:
464 The extracted id value.
465
466 Raises:
467 BatchError if the header is not in the expected format.
468 """
469 if header[0] != '<' or header[-1] != '>':
470 raise BatchError("Invalid value for Content-ID: %s" % header)
471 if '+' not in header:
472 raise BatchError("Invalid value for Content-ID: %s" % header)
473 base, id_ = header[1:-1].rsplit('+', 1)
474
475 return urllib.unquote(id_)
476
477 def _serialize_request(self, request):
478 """Convert an HttpRequest object into a string.
479
480 Args:
481 request: HttpRequest, the request to serialize.
482
483 Returns:
484 The request as a string in application/http format.
485 """
486 # Construct status line
487 parsed = urlparse.urlparse(request.uri)
488 request_line = urlparse.urlunparse(
489 (None, None, parsed.path, parsed.params, parsed.query, None)
490 )
491 status_line = request.method + ' ' + request_line + ' HTTP/1.1\n'
492 major, minor = request.headers.get('content-type', 'text/plain').split('/')
493 msg = MIMENonMultipart(major, minor)
494 headers = request.headers.copy()
495
496 # MIMENonMultipart adds its own Content-Type header.
497 if 'content-type' in headers:
498 del headers['content-type']
499
500 for key, value in headers.iteritems():
501 msg[key] = value
502 msg['Host'] = parsed.netloc
503 msg.set_unixfrom(None)
504
505 if request.body is not None:
506 msg.set_payload(request.body)
507
508 body = msg.as_string(False)
509 # Strip off the \n\n that the MIME lib tacks onto the end of the payload.
510 if request.body is None:
511 body = body[:-2]
512
513 return status_line + body
514
515 def _deserialize_response(self, payload):
516 """Convert string into httplib2 response and content.
517
518 Args:
519 payload: string, headers and body as a string.
520
521 Returns:
522 A pair (resp, content) like would be returned from httplib2.request.
523 """
524 # Strip off the status line
525 status_line, payload = payload.split('\n', 1)
526 protocol, status, reason = status_line.split(' ')
527
528 # Parse the rest of the response
529 parser = FeedParser()
530 parser.feed(payload)
531 msg = parser.close()
532 msg['status'] = status
533
534 # Create httplib2.Response from the parsed headers.
535 resp = httplib2.Response(msg)
536 resp.reason = reason
537 resp.version = int(protocol.split('/', 1)[1].replace('.', ''))
538
539 content = payload.split('\r\n\r\n', 1)[1]
540
541 return resp, content
542
543 def _new_id(self):
544 """Create a new id.
545
546 Auto incrementing number that avoids conflicts with ids already used.
547
548 Returns:
549 string, a new unique id.
550 """
551 self._last_auto_id += 1
552 while str(self._last_auto_id) in self._requests:
553 self._last_auto_id += 1
554 return str(self._last_auto_id)
555
556 def add(self, request, callback=None, request_id=None):
557 """Add a new request.
558
559 Every callback added will be paired with a unique id, the request_id. That
560 unique id will be passed back to the callback when the response comes back
561 from the server. The default behavior is to have the library generate it's
562 own unique id. If the caller passes in a request_id then they must ensure
563 uniqueness for each request_id, and if they are not an exception is
564 raised. Callers should either supply all request_ids or nevery supply a
565 request id, to avoid such an error.
566
567 Args:
568 request: HttpRequest, Request to add to the batch.
569 callback: callable, A callback to be called for this response, of the
570 form callback(id, response). The first parameter is the request id, and
571 the second is the deserialized response object.
572 request_id: string, A unique id for the request. The id will be passed to
573 the callback with the response.
574
575 Returns:
576 None
577
578 Raises:
579 BatchError if a resumable request is added to a batch.
580 KeyError is the request_id is not unique.
581 """
582 if request_id is None:
583 request_id = self._new_id()
584 if request.resumable is not None:
585 raise BatchError("Resumable requests cannot be used in a batch request.")
586 if request_id in self._requests:
587 raise KeyError("A request with this ID already exists: %s" % request_id)
588 self._requests[request_id] = (request, callback)
589 self._order.append(request_id)
590
591 def execute(self, http=None):
592 """Execute all the requests as a single batched HTTP request.
593
594 Args:
595 http: httplib2.Http, an http object to be used in place of the one the
596 HttpRequest request object was constructed with. If one isn't supplied
597 then use a http object from the requests in this batch.
598
599 Returns:
600 None
601
602 Raises:
603 apiclient.errors.HttpError if the response was not a 2xx.
604 httplib2.Error if a transport error has occured.
605 """
606 if http is None:
607 for request_id in self._order:
608 request, callback = self._requests[request_id]
609 if request is not None:
610 http = request.http
611 break
612 if http is None:
613 raise ValueError("Missing a valid http object.")
614
615
616 msgRoot = MIMEMultipart('mixed')
617 # msgRoot should not write out it's own headers
618 setattr(msgRoot, '_write_headers', lambda self: None)
619
620 # Add all the individual requests.
621 for request_id in self._order:
622 request, callback = self._requests[request_id]
623
624 msg = MIMENonMultipart('application', 'http')
625 msg['Content-Transfer-Encoding'] = 'binary'
626 msg['Content-ID'] = self._id_to_header(request_id)
627
628 body = self._serialize_request(request)
629 msg.set_payload(body)
630 msgRoot.attach(msg)
631
632 body = msgRoot.as_string()
633
634 headers = {}
635 headers['content-type'] = ('multipart/mixed; '
636 'boundary="%s"') % msgRoot.get_boundary()
637
638 resp, content = http.request(self._batch_uri, 'POST', body=body,
639 headers=headers)
640
641 if resp.status >= 300:
642 raise HttpError(resp, content, self._batch_uri)
643
644 # Now break up the response and process each one with the correct postproc
645 # and trigger the right callbacks.
646 boundary, _ = content.split(None, 1)
647
648 # Prepend with a content-type header so FeedParser can handle it.
649 header = 'Content-Type: %s\r\n\r\n' % resp['content-type']
650 content = header + content
651
652 parser = FeedParser()
653 parser.feed(content)
654 respRoot = parser.close()
655
656 if not respRoot.is_multipart():
657 raise BatchError("Response not in multipart/mixed format.")
658
659 parts = respRoot.get_payload()
660 for part in parts:
661 request_id = self._header_to_id(part['Content-ID'])
662
663 headers, content = self._deserialize_response(part.get_payload())
664
665 # TODO(jcgregorio) Remove this temporary hack once the server stops
666 # gzipping individual response bodies.
667 if content[0] != '{':
668 gzipped_content = content
669 content = gzip.GzipFile(
670 fileobj=StringIO.StringIO(gzipped_content)).read()
671
672 request, cb = self._requests[request_id]
673 postproc = request.postproc
674 response = postproc(resp, content)
675 if cb is not None:
676 cb(request_id, response)
677 if self._callback is not None:
678 self._callback(request_id, response)
679
680
Joe Gregorioaf276d22010-12-09 14:26:58 -0500681class HttpRequestMock(object):
682 """Mock of HttpRequest.
683
684 Do not construct directly, instead use RequestMockBuilder.
685 """
686
687 def __init__(self, resp, content, postproc):
688 """Constructor for HttpRequestMock
689
690 Args:
691 resp: httplib2.Response, the response to emulate coming from the request
692 content: string, the response body
693 postproc: callable, the post processing function usually supplied by
694 the model class. See model.JsonModel.response() as an example.
695 """
696 self.resp = resp
697 self.content = content
698 self.postproc = postproc
699 if resp is None:
Joe Gregorioc6722462010-12-20 14:29:28 -0500700 self.resp = httplib2.Response({'status': 200, 'reason': 'OK'})
Joe Gregorioaf276d22010-12-09 14:26:58 -0500701 if 'reason' in self.resp:
702 self.resp.reason = self.resp['reason']
703
704 def execute(self, http=None):
705 """Execute the request.
706
707 Same behavior as HttpRequest.execute(), but the response is
708 mocked and not really from an HTTP request/response.
709 """
710 return self.postproc(self.resp, self.content)
711
712
713class RequestMockBuilder(object):
714 """A simple mock of HttpRequest
715
716 Pass in a dictionary to the constructor that maps request methodIds to
Joe Gregorioa388ce32011-09-09 17:19:13 -0400717 tuples of (httplib2.Response, content, opt_expected_body) that should be
718 returned when that method is called. None may also be passed in for the
719 httplib2.Response, in which case a 200 OK response will be generated.
720 If an opt_expected_body (str or dict) is provided, it will be compared to
721 the body and UnexpectedBodyError will be raised on inequality.
Joe Gregorioaf276d22010-12-09 14:26:58 -0500722
723 Example:
724 response = '{"data": {"id": "tag:google.c...'
725 requestBuilder = RequestMockBuilder(
726 {
Joe Gregorioc4fc0952011-11-09 12:21:11 -0500727 'plus.activities.get': (None, response),
Joe Gregorioaf276d22010-12-09 14:26:58 -0500728 }
729 )
Joe Gregorioc4fc0952011-11-09 12:21:11 -0500730 apiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder)
Joe Gregorioaf276d22010-12-09 14:26:58 -0500731
732 Methods that you do not supply a response for will return a
Joe Gregorio66f57522011-11-30 11:00:00 -0500733 200 OK with an empty string as the response content or raise an excpetion
734 if check_unexpected is set to True. The methodId is taken from the rpcName
Joe Gregorioa388ce32011-09-09 17:19:13 -0400735 in the discovery document.
Joe Gregorioaf276d22010-12-09 14:26:58 -0500736
737 For more details see the project wiki.
738 """
739
Joe Gregorioa388ce32011-09-09 17:19:13 -0400740 def __init__(self, responses, check_unexpected=False):
Joe Gregorioaf276d22010-12-09 14:26:58 -0500741 """Constructor for RequestMockBuilder
742
743 The constructed object should be a callable object
744 that can replace the class HttpResponse.
745
746 responses - A dictionary that maps methodIds into tuples
747 of (httplib2.Response, content). The methodId
748 comes from the 'rpcName' field in the discovery
749 document.
Joe Gregorioa388ce32011-09-09 17:19:13 -0400750 check_unexpected - A boolean setting whether or not UnexpectedMethodError
751 should be raised on unsupplied method.
Joe Gregorioaf276d22010-12-09 14:26:58 -0500752 """
753 self.responses = responses
Joe Gregorioa388ce32011-09-09 17:19:13 -0400754 self.check_unexpected = check_unexpected
Joe Gregorioaf276d22010-12-09 14:26:58 -0500755
Joe Gregorio7c22ab22011-02-16 15:32:39 -0500756 def __call__(self, http, postproc, uri, method='GET', body=None,
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500757 headers=None, methodId=None, resumable=None):
Joe Gregorioaf276d22010-12-09 14:26:58 -0500758 """Implements the callable interface that discovery.build() expects
759 of requestBuilder, which is to build an object compatible with
760 HttpRequest.execute(). See that method for the description of the
761 parameters and the expected response.
762 """
763 if methodId in self.responses:
Joe Gregorioa388ce32011-09-09 17:19:13 -0400764 response = self.responses[methodId]
765 resp, content = response[:2]
766 if len(response) > 2:
767 # Test the body against the supplied expected_body.
768 expected_body = response[2]
769 if bool(expected_body) != bool(body):
770 # Not expecting a body and provided one
771 # or expecting a body and not provided one.
772 raise UnexpectedBodyError(expected_body, body)
773 if isinstance(expected_body, str):
774 expected_body = simplejson.loads(expected_body)
775 body = simplejson.loads(body)
776 if body != expected_body:
777 raise UnexpectedBodyError(expected_body, body)
Joe Gregorioaf276d22010-12-09 14:26:58 -0500778 return HttpRequestMock(resp, content, postproc)
Joe Gregorioa388ce32011-09-09 17:19:13 -0400779 elif self.check_unexpected:
780 raise UnexpectedMethodError(methodId)
Joe Gregorioaf276d22010-12-09 14:26:58 -0500781 else:
Joe Gregoriod433b2a2011-02-22 10:51:51 -0500782 model = JsonModel(False)
Joe Gregorioaf276d22010-12-09 14:26:58 -0500783 return HttpRequestMock(None, '{}', model.response)
Joe Gregoriocb8103d2011-02-11 23:20:52 -0500784
Joe Gregoriodeeb0202011-02-15 14:49:57 -0500785
Joe Gregoriocb8103d2011-02-11 23:20:52 -0500786class HttpMock(object):
787 """Mock of httplib2.Http"""
788
Joe Gregorioec343652011-02-16 16:52:51 -0500789 def __init__(self, filename, headers=None):
Joe Gregoriocb8103d2011-02-11 23:20:52 -0500790 """
791 Args:
792 filename: string, absolute filename to read response from
793 headers: dict, header to return with response
794 """
Joe Gregorioec343652011-02-16 16:52:51 -0500795 if headers is None:
796 headers = {'status': '200 OK'}
Joe Gregoriocb8103d2011-02-11 23:20:52 -0500797 f = file(filename, 'r')
798 self.data = f.read()
799 f.close()
800 self.headers = headers
801
Joe Gregoriodeeb0202011-02-15 14:49:57 -0500802 def request(self, uri,
Joe Gregorio7c22ab22011-02-16 15:32:39 -0500803 method='GET',
Joe Gregoriodeeb0202011-02-15 14:49:57 -0500804 body=None,
805 headers=None,
806 redirections=1,
807 connection_type=None):
Joe Gregoriocb8103d2011-02-11 23:20:52 -0500808 return httplib2.Response(self.headers), self.data
Joe Gregorioccc79542011-02-19 00:05:26 -0500809
810
811class HttpMockSequence(object):
812 """Mock of httplib2.Http
813
814 Mocks a sequence of calls to request returning different responses for each
815 call. Create an instance initialized with the desired response headers
816 and content and then use as if an httplib2.Http instance.
817
818 http = HttpMockSequence([
819 ({'status': '401'}, ''),
820 ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'),
821 ({'status': '200'}, 'echo_request_headers'),
822 ])
823 resp, content = http.request("http://examples.com")
824
825 There are special values you can pass in for content to trigger
826 behavours that are helpful in testing.
827
828 'echo_request_headers' means return the request headers in the response body
Joe Gregorioe9e236f2011-03-21 22:23:14 -0400829 'echo_request_headers_as_json' means return the request headers in
830 the response body
Joe Gregorioccc79542011-02-19 00:05:26 -0500831 'echo_request_body' means return the request body in the response body
Joe Gregorio0bc70912011-05-24 15:30:49 -0400832 'echo_request_uri' means return the request uri in the response body
Joe Gregorioccc79542011-02-19 00:05:26 -0500833 """
834
835 def __init__(self, iterable):
836 """
837 Args:
838 iterable: iterable, a sequence of pairs of (headers, body)
839 """
840 self._iterable = iterable
841
842 def request(self, uri,
843 method='GET',
844 body=None,
845 headers=None,
846 redirections=1,
847 connection_type=None):
848 resp, content = self._iterable.pop(0)
849 if content == 'echo_request_headers':
850 content = headers
Joe Gregoriof4153422011-03-18 22:45:18 -0400851 elif content == 'echo_request_headers_as_json':
852 content = simplejson.dumps(headers)
Joe Gregorioccc79542011-02-19 00:05:26 -0500853 elif content == 'echo_request_body':
854 content = body
Joe Gregorio0bc70912011-05-24 15:30:49 -0400855 elif content == 'echo_request_uri':
856 content = uri
Joe Gregorioccc79542011-02-19 00:05:26 -0500857 return httplib2.Response(resp), content
Joe Gregorio6bcbcea2011-03-10 15:26:05 -0500858
859
860def set_user_agent(http, user_agent):
Joe Gregoriof4153422011-03-18 22:45:18 -0400861 """Set the user-agent on every request.
862
Joe Gregorio6bcbcea2011-03-10 15:26:05 -0500863 Args:
864 http - An instance of httplib2.Http
865 or something that acts like it.
866 user_agent: string, the value for the user-agent header.
867
868 Returns:
869 A modified instance of http that was passed in.
870
871 Example:
872
873 h = httplib2.Http()
874 h = set_user_agent(h, "my-app-name/6.0")
875
876 Most of the time the user-agent will be set doing auth, this is for the rare
877 cases where you are accessing an unauthenticated endpoint.
878 """
879 request_orig = http.request
880
881 # The closure that will replace 'httplib2.Http.request'.
882 def new_request(uri, method='GET', body=None, headers=None,
883 redirections=httplib2.DEFAULT_MAX_REDIRECTS,
884 connection_type=None):
885 """Modify the request headers to add the user-agent."""
886 if headers is None:
887 headers = {}
888 if 'user-agent' in headers:
889 headers['user-agent'] = user_agent + ' ' + headers['user-agent']
890 else:
891 headers['user-agent'] = user_agent
892 resp, content = request_orig(uri, method, body, headers,
893 redirections, connection_type)
894 return resp, content
895
896 http.request = new_request
897 return http
Joe Gregoriof4153422011-03-18 22:45:18 -0400898
899
900def tunnel_patch(http):
901 """Tunnel PATCH requests over POST.
902 Args:
903 http - An instance of httplib2.Http
904 or something that acts like it.
905
906 Returns:
907 A modified instance of http that was passed in.
908
909 Example:
910
911 h = httplib2.Http()
912 h = tunnel_patch(h, "my-app-name/6.0")
913
914 Useful if you are running on a platform that doesn't support PATCH.
915 Apply this last if you are using OAuth 1.0, as changing the method
916 will result in a different signature.
917 """
918 request_orig = http.request
919
920 # The closure that will replace 'httplib2.Http.request'.
921 def new_request(uri, method='GET', body=None, headers=None,
922 redirections=httplib2.DEFAULT_MAX_REDIRECTS,
923 connection_type=None):
924 """Modify the request headers to add the user-agent."""
925 if headers is None:
926 headers = {}
927 if method == 'PATCH':
Joe Gregorio06d852b2011-03-25 15:03:10 -0400928 if 'oauth_token' in headers.get('authorization', ''):
Joe Gregorioe9e236f2011-03-21 22:23:14 -0400929 logging.warning(
Joe Gregorio06d852b2011-03-25 15:03:10 -0400930 'OAuth 1.0 request made with Credentials after tunnel_patch.')
Joe Gregoriof4153422011-03-18 22:45:18 -0400931 headers['x-http-method-override'] = "PATCH"
932 method = 'POST'
933 resp, content = request_orig(uri, method, body, headers,
934 redirections, connection_type)
935 return resp, content
936
937 http.request = new_request
938 return http