1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """Classes to encapsulate a single HTTP request.
16
17 The classes implement a command pattern, with every
18 object supporting an execute() method that does the
19 actuall HTTP request.
20 """
21
22 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
23
24 import StringIO
25 import base64
26 import copy
27 import gzip
28 import httplib2
29 import mimeparse
30 import mimetypes
31 import os
32 import urllib
33 import urlparse
34 import uuid
35
36 from email.generator import Generator
37 from email.mime.multipart import MIMEMultipart
38 from email.mime.nonmultipart import MIMENonMultipart
39 from email.parser import FeedParser
40 from errors import BatchError
41 from errors import HttpError
42 from errors import ResumableUploadError
43 from errors import UnexpectedBodyError
44 from errors import UnexpectedMethodError
45 from model import JsonModel
46 from oauth2client.anyjson import simplejson
47
48
49 DEFAULT_CHUNK_SIZE = 512*1024
77
184
284
376
469
472 """Encapsulates a single HTTP request."""
473
474 - def __init__(self, http, postproc, uri,
475 method='GET',
476 body=None,
477 headers=None,
478 methodId=None,
479 resumable=None):
480 """Constructor for an HttpRequest.
481
482 Args:
483 http: httplib2.Http, the transport object to use to make a request
484 postproc: callable, called on the HTTP response and content to transform
485 it into a data object before returning, or raising an exception
486 on an error.
487 uri: string, the absolute URI to send the request to
488 method: string, the HTTP method to use
489 body: string, the request body of the HTTP request,
490 headers: dict, the HTTP request headers
491 methodId: string, a unique identifier for the API method being called.
492 resumable: MediaUpload, None if this is not a resumbale request.
493 """
494 self.uri = uri
495 self.method = method
496 self.body = body
497 self.headers = headers or {}
498 self.methodId = methodId
499 self.http = http
500 self.postproc = postproc
501 self.resumable = resumable
502 self._in_error_state = False
503
504
505 major, minor, params = mimeparse.parse_mime_type(
506 headers.get('content-type', 'application/json'))
507
508
509 self.body_size = len(self.body or '')
510
511
512 self.resumable_uri = None
513
514
515 self.resumable_progress = 0
516
518 """Execute the request.
519
520 Args:
521 http: httplib2.Http, an http object to be used in place of the
522 one the HttpRequest request object was constructed with.
523
524 Returns:
525 A deserialized object model of the response body as determined
526 by the postproc.
527
528 Raises:
529 apiclient.errors.HttpError if the response was not a 2xx.
530 httplib2.Error if a transport error has occured.
531 """
532 if http is None:
533 http = self.http
534 if self.resumable:
535 body = None
536 while body is None:
537 _, body = self.next_chunk(http)
538 return body
539 else:
540 if 'content-length' not in self.headers:
541 self.headers['content-length'] = str(self.body_size)
542 resp, content = http.request(self.uri, self.method,
543 body=self.body,
544 headers=self.headers)
545
546 if resp.status >= 300:
547 raise HttpError(resp, content, self.uri)
548 return self.postproc(resp, content)
549
551 """Execute the next step of a resumable upload.
552
553 Can only be used if the method being executed supports media uploads and
554 the MediaUpload object passed in was flagged as using resumable upload.
555
556 Example:
557
558 media = MediaFileUpload('smiley.png', mimetype='image/png',
559 chunksize=1000, resumable=True)
560 request = service.objects().insert(
561 bucket=buckets['items'][0]['id'],
562 name='smiley.png',
563 media_body=media)
564
565 response = None
566 while response is None:
567 status, response = request.next_chunk()
568 if status:
569 print "Upload %d%% complete." % int(status.progress() * 100)
570
571
572 Returns:
573 (status, body): (ResumableMediaStatus, object)
574 The body will be None until the resumable media is fully uploaded.
575
576 Raises:
577 apiclient.errors.HttpError if the response was not a 2xx.
578 httplib2.Error if a transport error has occured.
579 """
580 if http is None:
581 http = self.http
582
583 if self.resumable.size() is None:
584 size = '*'
585 else:
586 size = str(self.resumable.size())
587
588 if self.resumable_uri is None:
589 start_headers = copy.copy(self.headers)
590 start_headers['X-Upload-Content-Type'] = self.resumable.mimetype()
591 if size != '*':
592 start_headers['X-Upload-Content-Length'] = size
593 start_headers['content-length'] = str(self.body_size)
594
595 resp, content = http.request(self.uri, self.method,
596 body=self.body,
597 headers=start_headers)
598 if resp.status == 200 and 'location' in resp:
599 self.resumable_uri = resp['location']
600 else:
601 raise ResumableUploadError("Failed to retrieve starting URI.")
602 elif self._in_error_state:
603
604
605
606 headers = {
607 'Content-Range': 'bytes */%s' % size,
608 'content-length': '0'
609 }
610 resp, content = http.request(self.resumable_uri, 'PUT',
611 headers=headers)
612 status, body = self._process_response(resp, content)
613 if body:
614
615 return (status, body)
616
617 data = self.resumable.getbytes(
618 self.resumable_progress, self.resumable.chunksize())
619 headers = {
620 'Content-Range': 'bytes %d-%d/%s' % (
621 self.resumable_progress, self.resumable_progress + len(data) - 1,
622 size)
623 }
624 try:
625 resp, content = http.request(self.resumable_uri, 'PUT',
626 body=data,
627 headers=headers)
628 except:
629 self._in_error_state = True
630 raise
631
632 return self._process_response(resp, content)
633
635 """Process the response from a single chunk upload.
636
637 Args:
638 resp: httplib2.Response, the response object.
639 content: string, the content of the response.
640
641 Returns:
642 (status, body): (ResumableMediaStatus, object)
643 The body will be None until the resumable media is fully uploaded.
644
645 Raises:
646 apiclient.errors.HttpError if the response was not a 2xx or a 308.
647 """
648 if resp.status in [200, 201]:
649 self._in_error_state = False
650 return None, self.postproc(resp, content)
651 elif resp.status == 308:
652 self._in_error_state = False
653
654 self.resumable_progress = int(resp['range'].split('-')[1]) + 1
655 if 'location' in resp:
656 self.resumable_uri = resp['location']
657 else:
658 self._in_error_state = True
659 raise HttpError(resp, content, self.uri)
660
661 return (MediaUploadProgress(self.resumable_progress, self.resumable.size()),
662 None)
663
665 """Returns a JSON representation of the HttpRequest."""
666 d = copy.copy(self.__dict__)
667 if d['resumable'] is not None:
668 d['resumable'] = self.resumable.to_json()
669 del d['http']
670 del d['postproc']
671
672 return simplejson.dumps(d)
673
674 @staticmethod
676 """Returns an HttpRequest populated with info from a JSON object."""
677 d = simplejson.loads(s)
678 if d['resumable'] is not None:
679 d['resumable'] = MediaUpload.new_from_json(d['resumable'])
680 return HttpRequest(
681 http,
682 postproc,
683 uri=d['uri'],
684 method=d['method'],
685 body=d['body'],
686 headers=d['headers'],
687 methodId=d['methodId'],
688 resumable=d['resumable'])
689
692 """Batches multiple HttpRequest objects into a single HTTP request."""
693
694 - def __init__(self, callback=None, batch_uri=None):
695 """Constructor for a BatchHttpRequest.
696
697 Args:
698 callback: callable, A callback to be called for each response, of the
699 form callback(id, response). The first parameter is the request id, and
700 the second is the deserialized response object.
701 batch_uri: string, URI to send batch requests to.
702 """
703 if batch_uri is None:
704 batch_uri = 'https://www.googleapis.com/batch'
705 self._batch_uri = batch_uri
706
707
708 self._callback = callback
709
710
711 self._requests = {}
712
713
714 self._callbacks = {}
715
716
717 self._order = []
718
719
720 self._last_auto_id = 0
721
722
723 self._base_id = None
724
725
726 self._responses = {}
727
728
729 self._refreshed_credentials = {}
730
757
759 """Convert an id to a Content-ID header value.
760
761 Args:
762 id_: string, identifier of individual request.
763
764 Returns:
765 A Content-ID header with the id_ encoded into it. A UUID is prepended to
766 the value because Content-ID headers are supposed to be universally
767 unique.
768 """
769 if self._base_id is None:
770 self._base_id = uuid.uuid4()
771
772 return '<%s+%s>' % (self._base_id, urllib.quote(id_))
773
775 """Convert a Content-ID header value to an id.
776
777 Presumes the Content-ID header conforms to the format that _id_to_header()
778 returns.
779
780 Args:
781 header: string, Content-ID header value.
782
783 Returns:
784 The extracted id value.
785
786 Raises:
787 BatchError if the header is not in the expected format.
788 """
789 if header[0] != '<' or header[-1] != '>':
790 raise BatchError("Invalid value for Content-ID: %s" % header)
791 if '+' not in header:
792 raise BatchError("Invalid value for Content-ID: %s" % header)
793 base, id_ = header[1:-1].rsplit('+', 1)
794
795 return urllib.unquote(id_)
796
798 """Convert an HttpRequest object into a string.
799
800 Args:
801 request: HttpRequest, the request to serialize.
802
803 Returns:
804 The request as a string in application/http format.
805 """
806
807 parsed = urlparse.urlparse(request.uri)
808 request_line = urlparse.urlunparse(
809 (None, None, parsed.path, parsed.params, parsed.query, None)
810 )
811 status_line = request.method + ' ' + request_line + ' HTTP/1.1\n'
812 major, minor = request.headers.get('content-type', 'application/json').split('/')
813 msg = MIMENonMultipart(major, minor)
814 headers = request.headers.copy()
815
816 if request.http is not None and hasattr(request.http.request,
817 'credentials'):
818 request.http.request.credentials.apply(headers)
819
820
821 if 'content-type' in headers:
822 del headers['content-type']
823
824 for key, value in headers.iteritems():
825 msg[key] = value
826 msg['Host'] = parsed.netloc
827 msg.set_unixfrom(None)
828
829 if request.body is not None:
830 msg.set_payload(request.body)
831 msg['content-length'] = str(len(request.body))
832
833
834 fp = StringIO.StringIO()
835
836 g = Generator(fp, maxheaderlen=0)
837 g.flatten(msg, unixfrom=False)
838 body = fp.getvalue()
839
840
841 if request.body is None:
842 body = body[:-2]
843
844 return status_line.encode('utf-8') + body
845
847 """Convert string into httplib2 response and content.
848
849 Args:
850 payload: string, headers and body as a string.
851
852 Returns:
853 A pair (resp, content) like would be returned from httplib2.request.
854 """
855
856 status_line, payload = payload.split('\n', 1)
857 protocol, status, reason = status_line.split(' ', 2)
858
859
860 parser = FeedParser()
861 parser.feed(payload)
862 msg = parser.close()
863 msg['status'] = status
864
865
866 resp = httplib2.Response(msg)
867 resp.reason = reason
868 resp.version = int(protocol.split('/', 1)[1].replace('.', ''))
869
870 content = payload.split('\r\n\r\n', 1)[1]
871
872 return resp, content
873
875 """Create a new id.
876
877 Auto incrementing number that avoids conflicts with ids already used.
878
879 Returns:
880 string, a new unique id.
881 """
882 self._last_auto_id += 1
883 while str(self._last_auto_id) in self._requests:
884 self._last_auto_id += 1
885 return str(self._last_auto_id)
886
887 - def add(self, request, callback=None, request_id=None):
888 """Add a new request.
889
890 Every callback added will be paired with a unique id, the request_id. That
891 unique id will be passed back to the callback when the response comes back
892 from the server. The default behavior is to have the library generate it's
893 own unique id. If the caller passes in a request_id then they must ensure
894 uniqueness for each request_id, and if they are not an exception is
895 raised. Callers should either supply all request_ids or nevery supply a
896 request id, to avoid such an error.
897
898 Args:
899 request: HttpRequest, Request to add to the batch.
900 callback: callable, A callback to be called for this response, of the
901 form callback(id, response). The first parameter is the request id, and
902 the second is the deserialized response object.
903 request_id: string, A unique id for the request. The id will be passed to
904 the callback with the response.
905
906 Returns:
907 None
908
909 Raises:
910 BatchError if a resumable request is added to a batch.
911 KeyError is the request_id is not unique.
912 """
913 if request_id is None:
914 request_id = self._new_id()
915 if request.resumable is not None:
916 raise BatchError("Resumable requests cannot be used in a batch request.")
917 if request_id in self._requests:
918 raise KeyError("A request with this ID already exists: %s" % request_id)
919 self._requests[request_id] = request
920 self._callbacks[request_id] = callback
921 self._order.append(request_id)
922
923 - def _execute(self, http, order, requests):
924 """Serialize batch request, send to server, process response.
925
926 Args:
927 http: httplib2.Http, an http object to be used to make the request with.
928 order: list, list of request ids in the order they were added to the
929 batch.
930 request: list, list of request objects to send.
931
932 Raises:
933 httplib2.Error if a transport error has occured.
934 apiclient.errors.BatchError if the response is the wrong format.
935 """
936 message = MIMEMultipart('mixed')
937
938 setattr(message, '_write_headers', lambda self: None)
939
940
941 for request_id in order:
942 request = requests[request_id]
943
944 msg = MIMENonMultipart('application', 'http')
945 msg['Content-Transfer-Encoding'] = 'binary'
946 msg['Content-ID'] = self._id_to_header(request_id)
947
948 body = self._serialize_request(request)
949 msg.set_payload(body)
950 message.attach(msg)
951
952 body = message.as_string()
953
954 headers = {}
955 headers['content-type'] = ('multipart/mixed; '
956 'boundary="%s"') % message.get_boundary()
957
958 resp, content = http.request(self._batch_uri, 'POST', body=body,
959 headers=headers)
960
961 if resp.status >= 300:
962 raise HttpError(resp, content, self._batch_uri)
963
964
965 boundary, _ = content.split(None, 1)
966
967
968 header = 'content-type: %s\r\n\r\n' % resp['content-type']
969 for_parser = header + content
970
971 parser = FeedParser()
972 parser.feed(for_parser)
973 mime_response = parser.close()
974
975 if not mime_response.is_multipart():
976 raise BatchError("Response not in multipart/mixed format.", resp,
977 content)
978
979 for part in mime_response.get_payload():
980 request_id = self._header_to_id(part['Content-ID'])
981 headers, content = self._deserialize_response(part.get_payload())
982 self._responses[request_id] = (headers, content)
983
985 """Execute all the requests as a single batched HTTP request.
986
987 Args:
988 http: httplib2.Http, an http object to be used in place of the one the
989 HttpRequest request object was constructed with. If one isn't supplied
990 then use a http object from the requests in this batch.
991
992 Returns:
993 None
994
995 Raises:
996 httplib2.Error if a transport error has occured.
997 apiclient.errors.BatchError if the response is the wrong format.
998 """
999
1000
1001 if http is None:
1002 for request_id in self._order:
1003 request = self._requests[request_id]
1004 if request is not None:
1005 http = request.http
1006 break
1007
1008 if http is None:
1009 raise ValueError("Missing a valid http object.")
1010
1011 self._execute(http, self._order, self._requests)
1012
1013
1014
1015 redo_requests = {}
1016 redo_order = []
1017
1018 for request_id in self._order:
1019 headers, content = self._responses[request_id]
1020 if headers['status'] == '401':
1021 redo_order.append(request_id)
1022 request = self._requests[request_id]
1023 self._refresh_and_apply_credentials(request, http)
1024 redo_requests[request_id] = request
1025
1026 if redo_requests:
1027 self._execute(http, redo_order, redo_requests)
1028
1029
1030
1031
1032
1033 for request_id in self._order:
1034 headers, content = self._responses[request_id]
1035
1036 request = self._requests[request_id]
1037 callback = self._callbacks[request_id]
1038
1039 response = None
1040 exception = None
1041 try:
1042 r = httplib2.Response(headers)
1043 response = request.postproc(r, content)
1044 except HttpError, e:
1045 exception = e
1046
1047 if callback is not None:
1048 callback(request_id, response, exception)
1049 if self._callback is not None:
1050 self._callback(request_id, response, exception)
1051
1054 """Mock of HttpRequest.
1055
1056 Do not construct directly, instead use RequestMockBuilder.
1057 """
1058
1059 - def __init__(self, resp, content, postproc):
1060 """Constructor for HttpRequestMock
1061
1062 Args:
1063 resp: httplib2.Response, the response to emulate coming from the request
1064 content: string, the response body
1065 postproc: callable, the post processing function usually supplied by
1066 the model class. See model.JsonModel.response() as an example.
1067 """
1068 self.resp = resp
1069 self.content = content
1070 self.postproc = postproc
1071 if resp is None:
1072 self.resp = httplib2.Response({'status': 200, 'reason': 'OK'})
1073 if 'reason' in self.resp:
1074 self.resp.reason = self.resp['reason']
1075
1077 """Execute the request.
1078
1079 Same behavior as HttpRequest.execute(), but the response is
1080 mocked and not really from an HTTP request/response.
1081 """
1082 return self.postproc(self.resp, self.content)
1083
1086 """A simple mock of HttpRequest
1087
1088 Pass in a dictionary to the constructor that maps request methodIds to
1089 tuples of (httplib2.Response, content, opt_expected_body) that should be
1090 returned when that method is called. None may also be passed in for the
1091 httplib2.Response, in which case a 200 OK response will be generated.
1092 If an opt_expected_body (str or dict) is provided, it will be compared to
1093 the body and UnexpectedBodyError will be raised on inequality.
1094
1095 Example:
1096 response = '{"data": {"id": "tag:google.c...'
1097 requestBuilder = RequestMockBuilder(
1098 {
1099 'plus.activities.get': (None, response),
1100 }
1101 )
1102 apiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder)
1103
1104 Methods that you do not supply a response for will return a
1105 200 OK with an empty string as the response content or raise an excpetion
1106 if check_unexpected is set to True. The methodId is taken from the rpcName
1107 in the discovery document.
1108
1109 For more details see the project wiki.
1110 """
1111
1112 - def __init__(self, responses, check_unexpected=False):
1113 """Constructor for RequestMockBuilder
1114
1115 The constructed object should be a callable object
1116 that can replace the class HttpResponse.
1117
1118 responses - A dictionary that maps methodIds into tuples
1119 of (httplib2.Response, content). The methodId
1120 comes from the 'rpcName' field in the discovery
1121 document.
1122 check_unexpected - A boolean setting whether or not UnexpectedMethodError
1123 should be raised on unsupplied method.
1124 """
1125 self.responses = responses
1126 self.check_unexpected = check_unexpected
1127
1128 - def __call__(self, http, postproc, uri, method='GET', body=None,
1129 headers=None, methodId=None, resumable=None):
1130 """Implements the callable interface that discovery.build() expects
1131 of requestBuilder, which is to build an object compatible with
1132 HttpRequest.execute(). See that method for the description of the
1133 parameters and the expected response.
1134 """
1135 if methodId in self.responses:
1136 response = self.responses[methodId]
1137 resp, content = response[:2]
1138 if len(response) > 2:
1139
1140 expected_body = response[2]
1141 if bool(expected_body) != bool(body):
1142
1143
1144 raise UnexpectedBodyError(expected_body, body)
1145 if isinstance(expected_body, str):
1146 expected_body = simplejson.loads(expected_body)
1147 body = simplejson.loads(body)
1148 if body != expected_body:
1149 raise UnexpectedBodyError(expected_body, body)
1150 return HttpRequestMock(resp, content, postproc)
1151 elif self.check_unexpected:
1152 raise UnexpectedMethodError(methodId)
1153 else:
1154 model = JsonModel(False)
1155 return HttpRequestMock(None, '{}', model.response)
1156
1159 """Mock of httplib2.Http"""
1160
1161 - def __init__(self, filename, headers=None):
1162 """
1163 Args:
1164 filename: string, absolute filename to read response from
1165 headers: dict, header to return with response
1166 """
1167 if headers is None:
1168 headers = {'status': '200 OK'}
1169 f = file(filename, 'r')
1170 self.data = f.read()
1171 f.close()
1172 self.headers = headers
1173
1174 - def request(self, uri,
1175 method='GET',
1176 body=None,
1177 headers=None,
1178 redirections=1,
1179 connection_type=None):
1180 return httplib2.Response(self.headers), self.data
1181
1184 """Mock of httplib2.Http
1185
1186 Mocks a sequence of calls to request returning different responses for each
1187 call. Create an instance initialized with the desired response headers
1188 and content and then use as if an httplib2.Http instance.
1189
1190 http = HttpMockSequence([
1191 ({'status': '401'}, ''),
1192 ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'),
1193 ({'status': '200'}, 'echo_request_headers'),
1194 ])
1195 resp, content = http.request("http://examples.com")
1196
1197 There are special values you can pass in for content to trigger
1198 behavours that are helpful in testing.
1199
1200 'echo_request_headers' means return the request headers in the response body
1201 'echo_request_headers_as_json' means return the request headers in
1202 the response body
1203 'echo_request_body' means return the request body in the response body
1204 'echo_request_uri' means return the request uri in the response body
1205 """
1206
1208 """
1209 Args:
1210 iterable: iterable, a sequence of pairs of (headers, body)
1211 """
1212 self._iterable = iterable
1213
1214 - def request(self, uri,
1215 method='GET',
1216 body=None,
1217 headers=None,
1218 redirections=1,
1219 connection_type=None):
1220 resp, content = self._iterable.pop(0)
1221 if content == 'echo_request_headers':
1222 content = headers
1223 elif content == 'echo_request_headers_as_json':
1224 content = simplejson.dumps(headers)
1225 elif content == 'echo_request_body':
1226 content = body
1227 elif content == 'echo_request_uri':
1228 content = uri
1229 return httplib2.Response(resp), content
1230
1233 """Set the user-agent on every request.
1234
1235 Args:
1236 http - An instance of httplib2.Http
1237 or something that acts like it.
1238 user_agent: string, the value for the user-agent header.
1239
1240 Returns:
1241 A modified instance of http that was passed in.
1242
1243 Example:
1244
1245 h = httplib2.Http()
1246 h = set_user_agent(h, "my-app-name/6.0")
1247
1248 Most of the time the user-agent will be set doing auth, this is for the rare
1249 cases where you are accessing an unauthenticated endpoint.
1250 """
1251 request_orig = http.request
1252
1253
1254 def new_request(uri, method='GET', body=None, headers=None,
1255 redirections=httplib2.DEFAULT_MAX_REDIRECTS,
1256 connection_type=None):
1257 """Modify the request headers to add the user-agent."""
1258 if headers is None:
1259 headers = {}
1260 if 'user-agent' in headers:
1261 headers['user-agent'] = user_agent + ' ' + headers['user-agent']
1262 else:
1263 headers['user-agent'] = user_agent
1264 resp, content = request_orig(uri, method, body, headers,
1265 redirections, connection_type)
1266 return resp, content
1267
1268 http.request = new_request
1269 return http
1270
1273 """Tunnel PATCH requests over POST.
1274 Args:
1275 http - An instance of httplib2.Http
1276 or something that acts like it.
1277
1278 Returns:
1279 A modified instance of http that was passed in.
1280
1281 Example:
1282
1283 h = httplib2.Http()
1284 h = tunnel_patch(h, "my-app-name/6.0")
1285
1286 Useful if you are running on a platform that doesn't support PATCH.
1287 Apply this last if you are using OAuth 1.0, as changing the method
1288 will result in a different signature.
1289 """
1290 request_orig = http.request
1291
1292
1293 def new_request(uri, method='GET', body=None, headers=None,
1294 redirections=httplib2.DEFAULT_MAX_REDIRECTS,
1295 connection_type=None):
1296 """Modify the request headers to add the user-agent."""
1297 if headers is None:
1298 headers = {}
1299 if method == 'PATCH':
1300 if 'oauth_token' in headers.get('authorization', ''):
1301 logging.warning(
1302 'OAuth 1.0 request made with Credentials after tunnel_patch.')
1303 headers['x-http-method-override'] = "PATCH"
1304 method = 'POST'
1305 resp, content = request_orig(uri, method, body, headers,
1306 redirections, connection_type)
1307 return resp, content
1308
1309 http.request = new_request
1310 return http
1311