1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """Classes to encapsulate a single HTTP request.
16
17 The classes implement a command pattern, with every
18 object supporting an execute() method that does the
19 actuall HTTP request.
20 """
21
22 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
23
24 import StringIO
25 import base64
26 import copy
27 import gzip
28 import httplib2
29 import mimeparse
30 import mimetypes
31 import os
32 import urllib
33 import urlparse
34 import uuid
35
36 from email.generator import Generator
37 from email.mime.multipart import MIMEMultipart
38 from email.mime.nonmultipart import MIMENonMultipart
39 from email.parser import FeedParser
40 from errors import BatchError
41 from errors import HttpError
42 from errors import ResumableUploadError
43 from errors import UnexpectedBodyError
44 from errors import UnexpectedMethodError
45 from model import JsonModel
46 from oauth2client.anyjson import simplejson
65
135
215
308
311 """Encapsulates a single HTTP request."""
312
313 - def __init__(self, http, postproc, uri,
314 method='GET',
315 body=None,
316 headers=None,
317 methodId=None,
318 resumable=None):
319 """Constructor for an HttpRequest.
320
321 Args:
322 http: httplib2.Http, the transport object to use to make a request
323 postproc: callable, called on the HTTP response and content to transform
324 it into a data object before returning, or raising an exception
325 on an error.
326 uri: string, the absolute URI to send the request to
327 method: string, the HTTP method to use
328 body: string, the request body of the HTTP request,
329 headers: dict, the HTTP request headers
330 methodId: string, a unique identifier for the API method being called.
331 resumable: MediaUpload, None if this is not a resumbale request.
332 """
333 self.uri = uri
334 self.method = method
335 self.body = body
336 self.headers = headers or {}
337 self.methodId = methodId
338 self.http = http
339 self.postproc = postproc
340 self.resumable = resumable
341
342
343 major, minor, params = mimeparse.parse_mime_type(
344 headers.get('content-type', 'application/json'))
345
346
347 self.body_size = len(self.body or '')
348
349
350 self.resumable_uri = None
351
352
353 self.resumable_progress = 0
354
356 """Execute the request.
357
358 Args:
359 http: httplib2.Http, an http object to be used in place of the
360 one the HttpRequest request object was constructed with.
361
362 Returns:
363 A deserialized object model of the response body as determined
364 by the postproc.
365
366 Raises:
367 apiclient.errors.HttpError if the response was not a 2xx.
368 httplib2.Error if a transport error has occured.
369 """
370 if http is None:
371 http = self.http
372 if self.resumable:
373 body = None
374 while body is None:
375 _, body = self.next_chunk(http)
376 return body
377 else:
378 if 'content-length' not in self.headers:
379 self.headers['content-length'] = str(self.body_size)
380 resp, content = http.request(self.uri, self.method,
381 body=self.body,
382 headers=self.headers)
383
384 if resp.status >= 300:
385 raise HttpError(resp, content, self.uri)
386 return self.postproc(resp, content)
387
389 """Execute the next step of a resumable upload.
390
391 Can only be used if the method being executed supports media uploads and
392 the MediaUpload object passed in was flagged as using resumable upload.
393
394 Example:
395
396 media = MediaFileUpload('smiley.png', mimetype='image/png',
397 chunksize=1000, resumable=True)
398 request = service.objects().insert(
399 bucket=buckets['items'][0]['id'],
400 name='smiley.png',
401 media_body=media)
402
403 response = None
404 while response is None:
405 status, response = request.next_chunk()
406 if status:
407 print "Upload %d%% complete." % int(status.progress() * 100)
408
409
410 Returns:
411 (status, body): (ResumableMediaStatus, object)
412 The body will be None until the resumable media is fully uploaded.
413 """
414 if http is None:
415 http = self.http
416
417 if self.resumable_uri is None:
418 start_headers = copy.copy(self.headers)
419 start_headers['X-Upload-Content-Type'] = self.resumable.mimetype()
420 start_headers['X-Upload-Content-Length'] = str(self.resumable.size())
421 start_headers['content-length'] = str(self.body_size)
422
423 resp, content = http.request(self.uri, self.method,
424 body=self.body,
425 headers=start_headers)
426 if resp.status == 200 and 'location' in resp:
427 self.resumable_uri = resp['location']
428 else:
429 raise ResumableUploadError("Failed to retrieve starting URI.")
430
431 data = self.resumable.getbytes(self.resumable_progress,
432 self.resumable.chunksize())
433
434 headers = {
435 'Content-Range': 'bytes %d-%d/%d' % (
436 self.resumable_progress, self.resumable_progress + len(data) - 1,
437 self.resumable.size()),
438 }
439 resp, content = http.request(self.resumable_uri, 'PUT',
440 body=data,
441 headers=headers)
442 if resp.status in [200, 201]:
443 return None, self.postproc(resp, content)
444 elif resp.status == 308:
445
446 self.resumable_progress = int(resp['range'].split('-')[1]) + 1
447 if 'location' in resp:
448 self.resumable_uri = resp['location']
449 else:
450 raise HttpError(resp, content, self.uri)
451
452 return (MediaUploadProgress(self.resumable_progress, self.resumable.size()),
453 None)
454
456 """Returns a JSON representation of the HttpRequest."""
457 d = copy.copy(self.__dict__)
458 if d['resumable'] is not None:
459 d['resumable'] = self.resumable.to_json()
460 del d['http']
461 del d['postproc']
462 return simplejson.dumps(d)
463
464 @staticmethod
466 """Returns an HttpRequest populated with info from a JSON object."""
467 d = simplejson.loads(s)
468 if d['resumable'] is not None:
469 d['resumable'] = MediaUpload.new_from_json(d['resumable'])
470 return HttpRequest(
471 http,
472 postproc,
473 uri=d['uri'],
474 method=d['method'],
475 body=d['body'],
476 headers=d['headers'],
477 methodId=d['methodId'],
478 resumable=d['resumable'])
479
482 """Batches multiple HttpRequest objects into a single HTTP request."""
483
484 - def __init__(self, callback=None, batch_uri=None):
485 """Constructor for a BatchHttpRequest.
486
487 Args:
488 callback: callable, A callback to be called for each response, of the
489 form callback(id, response). The first parameter is the request id, and
490 the second is the deserialized response object.
491 batch_uri: string, URI to send batch requests to.
492 """
493 if batch_uri is None:
494 batch_uri = 'https://www.googleapis.com/batch'
495 self._batch_uri = batch_uri
496
497
498 self._callback = callback
499
500
501 self._requests = {}
502
503
504 self._callbacks = {}
505
506
507 self._order = []
508
509
510 self._last_auto_id = 0
511
512
513 self._base_id = None
514
515
516 self._responses = {}
517
518
519 self._refreshed_credentials = {}
520
547
549 """Convert an id to a Content-ID header value.
550
551 Args:
552 id_: string, identifier of individual request.
553
554 Returns:
555 A Content-ID header with the id_ encoded into it. A UUID is prepended to
556 the value because Content-ID headers are supposed to be universally
557 unique.
558 """
559 if self._base_id is None:
560 self._base_id = uuid.uuid4()
561
562 return '<%s+%s>' % (self._base_id, urllib.quote(id_))
563
565 """Convert a Content-ID header value to an id.
566
567 Presumes the Content-ID header conforms to the format that _id_to_header()
568 returns.
569
570 Args:
571 header: string, Content-ID header value.
572
573 Returns:
574 The extracted id value.
575
576 Raises:
577 BatchError if the header is not in the expected format.
578 """
579 if header[0] != '<' or header[-1] != '>':
580 raise BatchError("Invalid value for Content-ID: %s" % header)
581 if '+' not in header:
582 raise BatchError("Invalid value for Content-ID: %s" % header)
583 base, id_ = header[1:-1].rsplit('+', 1)
584
585 return urllib.unquote(id_)
586
588 """Convert an HttpRequest object into a string.
589
590 Args:
591 request: HttpRequest, the request to serialize.
592
593 Returns:
594 The request as a string in application/http format.
595 """
596
597 parsed = urlparse.urlparse(request.uri)
598 request_line = urlparse.urlunparse(
599 (None, None, parsed.path, parsed.params, parsed.query, None)
600 )
601 status_line = request.method + ' ' + request_line + ' HTTP/1.1\n'
602 major, minor = request.headers.get('content-type', 'application/json').split('/')
603 msg = MIMENonMultipart(major, minor)
604 headers = request.headers.copy()
605
606 if request.http is not None and hasattr(request.http.request,
607 'credentials'):
608 request.http.request.credentials.apply(headers)
609
610
611 if 'content-type' in headers:
612 del headers['content-type']
613
614 for key, value in headers.iteritems():
615 msg[key] = value
616 msg['Host'] = parsed.netloc
617 msg.set_unixfrom(None)
618
619 if request.body is not None:
620 msg.set_payload(request.body)
621 msg['content-length'] = str(len(request.body))
622
623
624 fp = StringIO.StringIO()
625
626 g = Generator(fp, maxheaderlen=0)
627 g.flatten(msg, unixfrom=False)
628 body = fp.getvalue()
629
630
631 if request.body is None:
632 body = body[:-2]
633
634 return status_line.encode('utf-8') + body
635
637 """Convert string into httplib2 response and content.
638
639 Args:
640 payload: string, headers and body as a string.
641
642 Returns:
643 A pair (resp, content) like would be returned from httplib2.request.
644 """
645
646 status_line, payload = payload.split('\n', 1)
647 protocol, status, reason = status_line.split(' ', 2)
648
649
650 parser = FeedParser()
651 parser.feed(payload)
652 msg = parser.close()
653 msg['status'] = status
654
655
656 resp = httplib2.Response(msg)
657 resp.reason = reason
658 resp.version = int(protocol.split('/', 1)[1].replace('.', ''))
659
660 content = payload.split('\r\n\r\n', 1)[1]
661
662 return resp, content
663
665 """Create a new id.
666
667 Auto incrementing number that avoids conflicts with ids already used.
668
669 Returns:
670 string, a new unique id.
671 """
672 self._last_auto_id += 1
673 while str(self._last_auto_id) in self._requests:
674 self._last_auto_id += 1
675 return str(self._last_auto_id)
676
677 - def add(self, request, callback=None, request_id=None):
678 """Add a new request.
679
680 Every callback added will be paired with a unique id, the request_id. That
681 unique id will be passed back to the callback when the response comes back
682 from the server. The default behavior is to have the library generate it's
683 own unique id. If the caller passes in a request_id then they must ensure
684 uniqueness for each request_id, and if they are not an exception is
685 raised. Callers should either supply all request_ids or nevery supply a
686 request id, to avoid such an error.
687
688 Args:
689 request: HttpRequest, Request to add to the batch.
690 callback: callable, A callback to be called for this response, of the
691 form callback(id, response). The first parameter is the request id, and
692 the second is the deserialized response object.
693 request_id: string, A unique id for the request. The id will be passed to
694 the callback with the response.
695
696 Returns:
697 None
698
699 Raises:
700 BatchError if a resumable request is added to a batch.
701 KeyError is the request_id is not unique.
702 """
703 if request_id is None:
704 request_id = self._new_id()
705 if request.resumable is not None:
706 raise BatchError("Resumable requests cannot be used in a batch request.")
707 if request_id in self._requests:
708 raise KeyError("A request with this ID already exists: %s" % request_id)
709 self._requests[request_id] = request
710 self._callbacks[request_id] = callback
711 self._order.append(request_id)
712
713 - def _execute(self, http, order, requests):
714 """Serialize batch request, send to server, process response.
715
716 Args:
717 http: httplib2.Http, an http object to be used to make the request with.
718 order: list, list of request ids in the order they were added to the
719 batch.
720 request: list, list of request objects to send.
721
722 Raises:
723 httplib2.Error if a transport error has occured.
724 apiclient.errors.BatchError if the response is the wrong format.
725 """
726 message = MIMEMultipart('mixed')
727
728 setattr(message, '_write_headers', lambda self: None)
729
730
731 for request_id in order:
732 request = requests[request_id]
733
734 msg = MIMENonMultipart('application', 'http')
735 msg['Content-Transfer-Encoding'] = 'binary'
736 msg['Content-ID'] = self._id_to_header(request_id)
737
738 body = self._serialize_request(request)
739 msg.set_payload(body)
740 message.attach(msg)
741
742 body = message.as_string()
743
744 headers = {}
745 headers['content-type'] = ('multipart/mixed; '
746 'boundary="%s"') % message.get_boundary()
747
748 resp, content = http.request(self._batch_uri, 'POST', body=body,
749 headers=headers)
750
751 if resp.status >= 300:
752 raise HttpError(resp, content, self._batch_uri)
753
754
755 boundary, _ = content.split(None, 1)
756
757
758 header = 'content-type: %s\r\n\r\n' % resp['content-type']
759 for_parser = header + content
760
761 parser = FeedParser()
762 parser.feed(for_parser)
763 mime_response = parser.close()
764
765 if not mime_response.is_multipart():
766 raise BatchError("Response not in multipart/mixed format.", resp,
767 content)
768
769 for part in mime_response.get_payload():
770 request_id = self._header_to_id(part['Content-ID'])
771 headers, content = self._deserialize_response(part.get_payload())
772 self._responses[request_id] = (headers, content)
773
775 """Execute all the requests as a single batched HTTP request.
776
777 Args:
778 http: httplib2.Http, an http object to be used in place of the one the
779 HttpRequest request object was constructed with. If one isn't supplied
780 then use a http object from the requests in this batch.
781
782 Returns:
783 None
784
785 Raises:
786 httplib2.Error if a transport error has occured.
787 apiclient.errors.BatchError if the response is the wrong format.
788 """
789
790
791 if http is None:
792 for request_id in self._order:
793 request = self._requests[request_id]
794 if request is not None:
795 http = request.http
796 break
797
798 if http is None:
799 raise ValueError("Missing a valid http object.")
800
801 self._execute(http, self._order, self._requests)
802
803
804
805 redo_requests = {}
806 redo_order = []
807
808 for request_id in self._order:
809 headers, content = self._responses[request_id]
810 if headers['status'] == '401':
811 redo_order.append(request_id)
812 request = self._requests[request_id]
813 self._refresh_and_apply_credentials(request, http)
814 redo_requests[request_id] = request
815
816 if redo_requests:
817 self._execute(http, redo_order, redo_requests)
818
819
820
821
822
823 for request_id in self._order:
824 headers, content = self._responses[request_id]
825
826 request = self._requests[request_id]
827 callback = self._callbacks[request_id]
828
829 response = None
830 exception = None
831 try:
832 r = httplib2.Response(headers)
833 response = request.postproc(r, content)
834 except HttpError, e:
835 exception = e
836
837 if callback is not None:
838 callback(request_id, response, exception)
839 if self._callback is not None:
840 self._callback(request_id, response, exception)
841
844 """Mock of HttpRequest.
845
846 Do not construct directly, instead use RequestMockBuilder.
847 """
848
849 - def __init__(self, resp, content, postproc):
850 """Constructor for HttpRequestMock
851
852 Args:
853 resp: httplib2.Response, the response to emulate coming from the request
854 content: string, the response body
855 postproc: callable, the post processing function usually supplied by
856 the model class. See model.JsonModel.response() as an example.
857 """
858 self.resp = resp
859 self.content = content
860 self.postproc = postproc
861 if resp is None:
862 self.resp = httplib2.Response({'status': 200, 'reason': 'OK'})
863 if 'reason' in self.resp:
864 self.resp.reason = self.resp['reason']
865
867 """Execute the request.
868
869 Same behavior as HttpRequest.execute(), but the response is
870 mocked and not really from an HTTP request/response.
871 """
872 return self.postproc(self.resp, self.content)
873
876 """A simple mock of HttpRequest
877
878 Pass in a dictionary to the constructor that maps request methodIds to
879 tuples of (httplib2.Response, content, opt_expected_body) that should be
880 returned when that method is called. None may also be passed in for the
881 httplib2.Response, in which case a 200 OK response will be generated.
882 If an opt_expected_body (str or dict) is provided, it will be compared to
883 the body and UnexpectedBodyError will be raised on inequality.
884
885 Example:
886 response = '{"data": {"id": "tag:google.c...'
887 requestBuilder = RequestMockBuilder(
888 {
889 'plus.activities.get': (None, response),
890 }
891 )
892 apiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder)
893
894 Methods that you do not supply a response for will return a
895 200 OK with an empty string as the response content or raise an excpetion
896 if check_unexpected is set to True. The methodId is taken from the rpcName
897 in the discovery document.
898
899 For more details see the project wiki.
900 """
901
902 - def __init__(self, responses, check_unexpected=False):
903 """Constructor for RequestMockBuilder
904
905 The constructed object should be a callable object
906 that can replace the class HttpResponse.
907
908 responses - A dictionary that maps methodIds into tuples
909 of (httplib2.Response, content). The methodId
910 comes from the 'rpcName' field in the discovery
911 document.
912 check_unexpected - A boolean setting whether or not UnexpectedMethodError
913 should be raised on unsupplied method.
914 """
915 self.responses = responses
916 self.check_unexpected = check_unexpected
917
918 - def __call__(self, http, postproc, uri, method='GET', body=None,
919 headers=None, methodId=None, resumable=None):
920 """Implements the callable interface that discovery.build() expects
921 of requestBuilder, which is to build an object compatible with
922 HttpRequest.execute(). See that method for the description of the
923 parameters and the expected response.
924 """
925 if methodId in self.responses:
926 response = self.responses[methodId]
927 resp, content = response[:2]
928 if len(response) > 2:
929
930 expected_body = response[2]
931 if bool(expected_body) != bool(body):
932
933
934 raise UnexpectedBodyError(expected_body, body)
935 if isinstance(expected_body, str):
936 expected_body = simplejson.loads(expected_body)
937 body = simplejson.loads(body)
938 if body != expected_body:
939 raise UnexpectedBodyError(expected_body, body)
940 return HttpRequestMock(resp, content, postproc)
941 elif self.check_unexpected:
942 raise UnexpectedMethodError(methodId)
943 else:
944 model = JsonModel(False)
945 return HttpRequestMock(None, '{}', model.response)
946
949 """Mock of httplib2.Http"""
950
951 - def __init__(self, filename, headers=None):
952 """
953 Args:
954 filename: string, absolute filename to read response from
955 headers: dict, header to return with response
956 """
957 if headers is None:
958 headers = {'status': '200 OK'}
959 f = file(filename, 'r')
960 self.data = f.read()
961 f.close()
962 self.headers = headers
963
964 - def request(self, uri,
965 method='GET',
966 body=None,
967 headers=None,
968 redirections=1,
969 connection_type=None):
970 return httplib2.Response(self.headers), self.data
971
974 """Mock of httplib2.Http
975
976 Mocks a sequence of calls to request returning different responses for each
977 call. Create an instance initialized with the desired response headers
978 and content and then use as if an httplib2.Http instance.
979
980 http = HttpMockSequence([
981 ({'status': '401'}, ''),
982 ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'),
983 ({'status': '200'}, 'echo_request_headers'),
984 ])
985 resp, content = http.request("http://examples.com")
986
987 There are special values you can pass in for content to trigger
988 behavours that are helpful in testing.
989
990 'echo_request_headers' means return the request headers in the response body
991 'echo_request_headers_as_json' means return the request headers in
992 the response body
993 'echo_request_body' means return the request body in the response body
994 'echo_request_uri' means return the request uri in the response body
995 """
996
998 """
999 Args:
1000 iterable: iterable, a sequence of pairs of (headers, body)
1001 """
1002 self._iterable = iterable
1003
1004 - def request(self, uri,
1005 method='GET',
1006 body=None,
1007 headers=None,
1008 redirections=1,
1009 connection_type=None):
1010 resp, content = self._iterable.pop(0)
1011 if content == 'echo_request_headers':
1012 content = headers
1013 elif content == 'echo_request_headers_as_json':
1014 content = simplejson.dumps(headers)
1015 elif content == 'echo_request_body':
1016 content = body
1017 elif content == 'echo_request_uri':
1018 content = uri
1019 return httplib2.Response(resp), content
1020
1023 """Set the user-agent on every request.
1024
1025 Args:
1026 http - An instance of httplib2.Http
1027 or something that acts like it.
1028 user_agent: string, the value for the user-agent header.
1029
1030 Returns:
1031 A modified instance of http that was passed in.
1032
1033 Example:
1034
1035 h = httplib2.Http()
1036 h = set_user_agent(h, "my-app-name/6.0")
1037
1038 Most of the time the user-agent will be set doing auth, this is for the rare
1039 cases where you are accessing an unauthenticated endpoint.
1040 """
1041 request_orig = http.request
1042
1043
1044 def new_request(uri, method='GET', body=None, headers=None,
1045 redirections=httplib2.DEFAULT_MAX_REDIRECTS,
1046 connection_type=None):
1047 """Modify the request headers to add the user-agent."""
1048 if headers is None:
1049 headers = {}
1050 if 'user-agent' in headers:
1051 headers['user-agent'] = user_agent + ' ' + headers['user-agent']
1052 else:
1053 headers['user-agent'] = user_agent
1054 resp, content = request_orig(uri, method, body, headers,
1055 redirections, connection_type)
1056 return resp, content
1057
1058 http.request = new_request
1059 return http
1060
1063 """Tunnel PATCH requests over POST.
1064 Args:
1065 http - An instance of httplib2.Http
1066 or something that acts like it.
1067
1068 Returns:
1069 A modified instance of http that was passed in.
1070
1071 Example:
1072
1073 h = httplib2.Http()
1074 h = tunnel_patch(h, "my-app-name/6.0")
1075
1076 Useful if you are running on a platform that doesn't support PATCH.
1077 Apply this last if you are using OAuth 1.0, as changing the method
1078 will result in a different signature.
1079 """
1080 request_orig = http.request
1081
1082
1083 def new_request(uri, method='GET', body=None, headers=None,
1084 redirections=httplib2.DEFAULT_MAX_REDIRECTS,
1085 connection_type=None):
1086 """Modify the request headers to add the user-agent."""
1087 if headers is None:
1088 headers = {}
1089 if method == 'PATCH':
1090 if 'oauth_token' in headers.get('authorization', ''):
1091 logging.warning(
1092 'OAuth 1.0 request made with Credentials after tunnel_patch.')
1093 headers['x-http-method-override'] = "PATCH"
1094 method = 'POST'
1095 resp, content = request_orig(uri, method, body, headers,
1096 redirections, connection_type)
1097 return resp, content
1098
1099 http.request = new_request
1100 return http
1101