Craig Citro | 15744b1 | 2015-03-02 13:34:32 -0800 | [diff] [blame] | 1 | #!/usr/bin/env python |
Joe Gregorio | 6bcbcea | 2011-03-10 15:26:05 -0500 | [diff] [blame] | 2 | # |
Craig Citro | 751b7fb | 2014-09-23 11:20:38 -0700 | [diff] [blame] | 3 | # Copyright 2014 Google Inc. All Rights Reserved. |
Joe Gregorio | 6bcbcea | 2011-03-10 15:26:05 -0500 | [diff] [blame] | 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | |
| 17 | """Http tests |
| 18 | |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 19 | Unit tests for the googleapiclient.http. |
Joe Gregorio | 6bcbcea | 2011-03-10 15:26:05 -0500 | [diff] [blame] | 20 | """ |
INADA Naoki | d898a37 | 2015-03-04 03:52:46 +0900 | [diff] [blame] | 21 | from __future__ import absolute_import |
| 22 | from six.moves import range |
Joe Gregorio | 6bcbcea | 2011-03-10 15:26:05 -0500 | [diff] [blame] | 23 | |
| 24 | __author__ = 'jcgregorio@google.com (Joe Gregorio)' |
| 25 | |
Pat Ferate | c605087 | 2015-03-03 18:24:59 -0800 | [diff] [blame] | 26 | from six import PY3 |
Pat Ferate | ed9affd | 2015-03-03 16:03:15 -0800 | [diff] [blame] | 27 | from six import BytesIO, StringIO |
| 28 | from io import FileIO |
Pat Ferate | d5b61bd | 2015-03-03 16:04:11 -0800 | [diff] [blame] | 29 | from six.moves.urllib.parse import urlencode |
Pat Ferate | ed9affd | 2015-03-03 16:03:15 -0800 | [diff] [blame] | 30 | |
Joe Gregorio | 7cbceab | 2011-06-27 10:46:54 -0400 | [diff] [blame] | 31 | # Do not remove the httplib2 import |
| 32 | import httplib2 |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 33 | import logging |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 34 | import mock |
Joe Gregorio | d0bd388 | 2011-11-22 09:49:47 -0500 | [diff] [blame] | 35 | import os |
Pat Ferate | 497a90f | 2015-03-09 09:52:54 -0700 | [diff] [blame] | 36 | import unittest2 as unittest |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 37 | import random |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 38 | import socket |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 39 | import ssl |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 40 | import time |
Joe Gregorio | 6bcbcea | 2011-03-10 15:26:05 -0500 | [diff] [blame] | 41 | |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 42 | from googleapiclient.discovery import build |
| 43 | from googleapiclient.errors import BatchError |
| 44 | from googleapiclient.errors import HttpError |
| 45 | from googleapiclient.errors import InvalidChunkSizeError |
Igor Maravić | 2243529 | 2017-01-19 22:28:22 +0100 | [diff] [blame] | 46 | from googleapiclient.http import build_http |
John Asmuth | 864311d | 2014-04-24 15:46:08 -0400 | [diff] [blame] | 47 | from googleapiclient.http import BatchHttpRequest |
| 48 | from googleapiclient.http import HttpMock |
| 49 | from googleapiclient.http import HttpMockSequence |
| 50 | from googleapiclient.http import HttpRequest |
| 51 | from googleapiclient.http import MAX_URI_LENGTH |
| 52 | from googleapiclient.http import MediaFileUpload |
| 53 | from googleapiclient.http import MediaInMemoryUpload |
| 54 | from googleapiclient.http import MediaIoBaseDownload |
| 55 | from googleapiclient.http import MediaIoBaseUpload |
| 56 | from googleapiclient.http import MediaUpload |
| 57 | from googleapiclient.http import _StreamSlice |
| 58 | from googleapiclient.http import set_user_agent |
| 59 | from googleapiclient.model import JsonModel |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 60 | from oauth2client.client import Credentials |
| 61 | |
| 62 | |
| 63 | class MockCredentials(Credentials): |
| 64 | """Mock class for all Credentials objects.""" |
| 65 | def __init__(self, bearer_token): |
| 66 | super(MockCredentials, self).__init__() |
| 67 | self._authorized = 0 |
| 68 | self._refreshed = 0 |
| 69 | self._applied = 0 |
| 70 | self._bearer_token = bearer_token |
| 71 | |
| 72 | def authorize(self, http): |
| 73 | self._authorized += 1 |
| 74 | |
| 75 | request_orig = http.request |
| 76 | |
| 77 | # The closure that will replace 'httplib2.Http.request'. |
| 78 | def new_request(uri, method='GET', body=None, headers=None, |
| 79 | redirections=httplib2.DEFAULT_MAX_REDIRECTS, |
| 80 | connection_type=None): |
| 81 | # Modify the request headers to add the appropriate |
| 82 | # Authorization header. |
| 83 | if headers is None: |
| 84 | headers = {} |
| 85 | self.apply(headers) |
| 86 | |
| 87 | resp, content = request_orig(uri, method, body, headers, |
| 88 | redirections, connection_type) |
| 89 | |
| 90 | return resp, content |
| 91 | |
| 92 | # Replace the request method with our own closure. |
| 93 | http.request = new_request |
| 94 | |
| 95 | # Set credentials as a property of the request method. |
| 96 | setattr(http.request, 'credentials', self) |
| 97 | |
| 98 | return http |
| 99 | |
| 100 | def refresh(self, http): |
| 101 | self._refreshed += 1 |
| 102 | |
| 103 | def apply(self, headers): |
| 104 | self._applied += 1 |
| 105 | headers['authorization'] = self._bearer_token + ' ' + str(self._refreshed) |
Joe Gregorio | 6bcbcea | 2011-03-10 15:26:05 -0500 | [diff] [blame] | 106 | |
| 107 | |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 108 | class HttpMockWithErrors(object): |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 109 | def __init__(self, num_errors, success_json, success_data): |
| 110 | self.num_errors = num_errors |
| 111 | self.success_json = success_json |
| 112 | self.success_data = success_data |
| 113 | |
| 114 | def request(self, *args, **kwargs): |
| 115 | if not self.num_errors: |
| 116 | return httplib2.Response(self.success_json), self.success_data |
| 117 | else: |
| 118 | self.num_errors -= 1 |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 119 | if self.num_errors == 1: |
| 120 | raise ssl.SSLError() |
| 121 | else: |
| 122 | if PY3: |
| 123 | ex = TimeoutError() |
| 124 | else: |
| 125 | ex = socket.error() |
Thomas Bonfort | 88ab76b | 2016-04-19 08:48:53 +0200 | [diff] [blame] | 126 | |
| 127 | if self.num_errors == 2: |
| 128 | #first try a broken pipe error (#218) |
| 129 | ex.errno = socket.errno.EPIPE |
| 130 | else: |
| 131 | # Initialize the timeout error code to the platform's error code. |
| 132 | try: |
| 133 | # For Windows: |
| 134 | ex.errno = socket.errno.WSAETIMEDOUT |
| 135 | except AttributeError: |
| 136 | # For Linux/Mac: |
| 137 | ex.errno = socket.errno.ETIMEDOUT |
| 138 | # Now raise the correct error. |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 139 | raise ex |
| 140 | |
| 141 | |
| 142 | class HttpMockWithNonRetriableErrors(object): |
| 143 | def __init__(self, num_errors, success_json, success_data): |
| 144 | self.num_errors = num_errors |
| 145 | self.success_json = success_json |
| 146 | self.success_data = success_data |
| 147 | |
| 148 | def request(self, *args, **kwargs): |
| 149 | if not self.num_errors: |
| 150 | return httplib2.Response(self.success_json), self.success_data |
| 151 | else: |
| 152 | self.num_errors -= 1 |
| 153 | ex = socket.error() |
Thomas Bonfort | 88ab76b | 2016-04-19 08:48:53 +0200 | [diff] [blame] | 154 | # set errno to a non-retriable value |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 155 | try: |
| 156 | # For Windows: |
| 157 | ex.errno = socket.errno.WSAECONNREFUSED |
| 158 | except AttributeError: |
| 159 | # For Linux/Mac: |
| 160 | ex.errno = socket.errno.ECONNREFUSED |
| 161 | # Now raise the correct timeout error. |
| 162 | raise ex |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 163 | |
| 164 | |
Joe Gregorio | d0bd388 | 2011-11-22 09:49:47 -0500 | [diff] [blame] | 165 | DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') |
| 166 | |
| 167 | |
| 168 | def datafile(filename): |
| 169 | return os.path.join(DATA_DIR, filename) |
| 170 | |
Joe Gregorio | 6bcbcea | 2011-03-10 15:26:05 -0500 | [diff] [blame] | 171 | class TestUserAgent(unittest.TestCase): |
| 172 | |
| 173 | def test_set_user_agent(self): |
| 174 | http = HttpMockSequence([ |
| 175 | ({'status': '200'}, 'echo_request_headers'), |
| 176 | ]) |
| 177 | |
| 178 | http = set_user_agent(http, "my_app/5.5") |
| 179 | resp, content = http.request("http://example.com") |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 180 | self.assertEqual('my_app/5.5', content['user-agent']) |
Joe Gregorio | 6bcbcea | 2011-03-10 15:26:05 -0500 | [diff] [blame] | 181 | |
| 182 | def test_set_user_agent_nested(self): |
| 183 | http = HttpMockSequence([ |
| 184 | ({'status': '200'}, 'echo_request_headers'), |
| 185 | ]) |
| 186 | |
| 187 | http = set_user_agent(http, "my_app/5.5") |
| 188 | http = set_user_agent(http, "my_library/0.1") |
| 189 | resp, content = http.request("http://example.com") |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 190 | self.assertEqual('my_app/5.5 my_library/0.1', content['user-agent']) |
Joe Gregorio | 6bcbcea | 2011-03-10 15:26:05 -0500 | [diff] [blame] | 191 | |
Joe Gregorio | 910b9b1 | 2012-06-12 09:36:30 -0400 | [diff] [blame] | 192 | |
| 193 | class TestMediaUpload(unittest.TestCase): |
| 194 | |
Nam T. Nguyen | dc13631 | 2015-12-01 10:18:56 -0800 | [diff] [blame] | 195 | def test_media_file_upload_mimetype_detection(self): |
| 196 | upload = MediaFileUpload(datafile('small.png')) |
| 197 | self.assertEqual('image/png', upload.mimetype()) |
| 198 | |
| 199 | upload = MediaFileUpload(datafile('empty')) |
| 200 | self.assertEqual('application/octet-stream', upload.mimetype()) |
| 201 | |
Joe Gregorio | d0bd388 | 2011-11-22 09:49:47 -0500 | [diff] [blame] | 202 | def test_media_file_upload_to_from_json(self): |
| 203 | upload = MediaFileUpload( |
| 204 | datafile('small.png'), chunksize=500, resumable=True) |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 205 | self.assertEqual('image/png', upload.mimetype()) |
| 206 | self.assertEqual(190, upload.size()) |
| 207 | self.assertEqual(True, upload.resumable()) |
| 208 | self.assertEqual(500, upload.chunksize()) |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 209 | self.assertEqual(b'PNG', upload.getbytes(1, 3)) |
Joe Gregorio | d0bd388 | 2011-11-22 09:49:47 -0500 | [diff] [blame] | 210 | |
| 211 | json = upload.to_json() |
| 212 | new_upload = MediaUpload.new_from_json(json) |
| 213 | |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 214 | self.assertEqual('image/png', new_upload.mimetype()) |
| 215 | self.assertEqual(190, new_upload.size()) |
| 216 | self.assertEqual(True, new_upload.resumable()) |
| 217 | self.assertEqual(500, new_upload.chunksize()) |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 218 | self.assertEqual(b'PNG', new_upload.getbytes(1, 3)) |
Joe Gregorio | d0bd388 | 2011-11-22 09:49:47 -0500 | [diff] [blame] | 219 | |
Joe Gregorio | c80ac9d | 2012-08-21 14:09:09 -0400 | [diff] [blame] | 220 | def test_media_file_upload_raises_on_invalid_chunksize(self): |
| 221 | self.assertRaises(InvalidChunkSizeError, MediaFileUpload, |
| 222 | datafile('small.png'), mimetype='image/png', chunksize=-2, |
| 223 | resumable=True) |
| 224 | |
Ali Afshar | 1cb6b67 | 2012-03-12 08:46:14 -0400 | [diff] [blame] | 225 | def test_media_inmemory_upload(self): |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 226 | media = MediaInMemoryUpload(b'abcdef', mimetype='text/plain', chunksize=10, |
Ali Afshar | 1cb6b67 | 2012-03-12 08:46:14 -0400 | [diff] [blame] | 227 | resumable=True) |
| 228 | self.assertEqual('text/plain', media.mimetype()) |
| 229 | self.assertEqual(10, media.chunksize()) |
| 230 | self.assertTrue(media.resumable()) |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 231 | self.assertEqual(b'bc', media.getbytes(1, 2)) |
Ali Afshar | 1cb6b67 | 2012-03-12 08:46:14 -0400 | [diff] [blame] | 232 | self.assertEqual(6, media.size()) |
| 233 | |
Joe Gregorio | d0bd388 | 2011-11-22 09:49:47 -0500 | [diff] [blame] | 234 | def test_http_request_to_from_json(self): |
| 235 | |
| 236 | def _postproc(*kwargs): |
| 237 | pass |
| 238 | |
Igor Maravić | 2243529 | 2017-01-19 22:28:22 +0100 | [diff] [blame] | 239 | http = build_http() |
Joe Gregorio | d0bd388 | 2011-11-22 09:49:47 -0500 | [diff] [blame] | 240 | media_upload = MediaFileUpload( |
| 241 | datafile('small.png'), chunksize=500, resumable=True) |
| 242 | req = HttpRequest( |
| 243 | http, |
| 244 | _postproc, |
| 245 | 'http://example.com', |
| 246 | method='POST', |
| 247 | body='{}', |
| 248 | headers={'content-type': 'multipart/related; boundary="---flubber"'}, |
| 249 | methodId='foo', |
| 250 | resumable=media_upload) |
| 251 | |
| 252 | json = req.to_json() |
| 253 | new_req = HttpRequest.from_json(json, http, _postproc) |
| 254 | |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 255 | self.assertEqual({'content-type': |
| 256 | 'multipart/related; boundary="---flubber"'}, |
| 257 | new_req.headers) |
| 258 | self.assertEqual('http://example.com', new_req.uri) |
| 259 | self.assertEqual('{}', new_req.body) |
| 260 | self.assertEqual(http, new_req.http) |
| 261 | self.assertEqual(media_upload.to_json(), new_req.resumable.to_json()) |
Joe Gregorio | d0bd388 | 2011-11-22 09:49:47 -0500 | [diff] [blame] | 262 | |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 263 | self.assertEqual(random.random, new_req._rand) |
| 264 | self.assertEqual(time.sleep, new_req._sleep) |
| 265 | |
Joe Gregorio | 910b9b1 | 2012-06-12 09:36:30 -0400 | [diff] [blame] | 266 | |
| 267 | class TestMediaIoBaseUpload(unittest.TestCase): |
| 268 | |
| 269 | def test_media_io_base_upload_from_file_io(self): |
Pat Ferate | ed9affd | 2015-03-03 16:03:15 -0800 | [diff] [blame] | 270 | fd = FileIO(datafile('small.png'), 'r') |
| 271 | upload = MediaIoBaseUpload( |
| 272 | fd=fd, mimetype='image/png', chunksize=500, resumable=True) |
| 273 | self.assertEqual('image/png', upload.mimetype()) |
| 274 | self.assertEqual(190, upload.size()) |
| 275 | self.assertEqual(True, upload.resumable()) |
| 276 | self.assertEqual(500, upload.chunksize()) |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 277 | self.assertEqual(b'PNG', upload.getbytes(1, 3)) |
Joe Gregorio | 910b9b1 | 2012-06-12 09:36:30 -0400 | [diff] [blame] | 278 | |
| 279 | def test_media_io_base_upload_from_file_object(self): |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 280 | f = open(datafile('small.png'), 'rb') |
Joe Gregorio | 910b9b1 | 2012-06-12 09:36:30 -0400 | [diff] [blame] | 281 | upload = MediaIoBaseUpload( |
Joe Gregorio | 4a2c29f | 2012-07-12 12:52:47 -0400 | [diff] [blame] | 282 | fd=f, mimetype='image/png', chunksize=500, resumable=True) |
Joe Gregorio | 910b9b1 | 2012-06-12 09:36:30 -0400 | [diff] [blame] | 283 | self.assertEqual('image/png', upload.mimetype()) |
| 284 | self.assertEqual(190, upload.size()) |
| 285 | self.assertEqual(True, upload.resumable()) |
| 286 | self.assertEqual(500, upload.chunksize()) |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 287 | self.assertEqual(b'PNG', upload.getbytes(1, 3)) |
Joe Gregorio | 910b9b1 | 2012-06-12 09:36:30 -0400 | [diff] [blame] | 288 | f.close() |
| 289 | |
| 290 | def test_media_io_base_upload_serializable(self): |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 291 | f = open(datafile('small.png'), 'rb') |
Joe Gregorio | 4a2c29f | 2012-07-12 12:52:47 -0400 | [diff] [blame] | 292 | upload = MediaIoBaseUpload(fd=f, mimetype='image/png') |
Joe Gregorio | 910b9b1 | 2012-06-12 09:36:30 -0400 | [diff] [blame] | 293 | |
| 294 | try: |
| 295 | json = upload.to_json() |
| 296 | self.fail('MediaIoBaseUpload should not be serializable.') |
| 297 | except NotImplementedError: |
| 298 | pass |
| 299 | |
Pat Ferate | c605087 | 2015-03-03 18:24:59 -0800 | [diff] [blame] | 300 | @unittest.skipIf(PY3, 'Strings and Bytes are different types') |
Joe Gregorio | 910b9b1 | 2012-06-12 09:36:30 -0400 | [diff] [blame] | 301 | def test_media_io_base_upload_from_string_io(self): |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 302 | f = open(datafile('small.png'), 'rb') |
Pat Ferate | ed9affd | 2015-03-03 16:03:15 -0800 | [diff] [blame] | 303 | fd = StringIO(f.read()) |
Joe Gregorio | 910b9b1 | 2012-06-12 09:36:30 -0400 | [diff] [blame] | 304 | f.close() |
| 305 | |
| 306 | upload = MediaIoBaseUpload( |
Joe Gregorio | 4a2c29f | 2012-07-12 12:52:47 -0400 | [diff] [blame] | 307 | fd=fd, mimetype='image/png', chunksize=500, resumable=True) |
Joe Gregorio | 910b9b1 | 2012-06-12 09:36:30 -0400 | [diff] [blame] | 308 | self.assertEqual('image/png', upload.mimetype()) |
Joe Gregorio | c80ac9d | 2012-08-21 14:09:09 -0400 | [diff] [blame] | 309 | self.assertEqual(190, upload.size()) |
Joe Gregorio | 910b9b1 | 2012-06-12 09:36:30 -0400 | [diff] [blame] | 310 | self.assertEqual(True, upload.resumable()) |
| 311 | self.assertEqual(500, upload.chunksize()) |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 312 | self.assertEqual(b'PNG', upload.getbytes(1, 3)) |
Joe Gregorio | 910b9b1 | 2012-06-12 09:36:30 -0400 | [diff] [blame] | 313 | f.close() |
| 314 | |
| 315 | def test_media_io_base_upload_from_bytes(self): |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 316 | f = open(datafile('small.png'), 'rb') |
Pat Ferate | ed9affd | 2015-03-03 16:03:15 -0800 | [diff] [blame] | 317 | fd = BytesIO(f.read()) |
| 318 | upload = MediaIoBaseUpload( |
| 319 | fd=fd, mimetype='image/png', chunksize=500, resumable=True) |
| 320 | self.assertEqual('image/png', upload.mimetype()) |
| 321 | self.assertEqual(190, upload.size()) |
| 322 | self.assertEqual(True, upload.resumable()) |
| 323 | self.assertEqual(500, upload.chunksize()) |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 324 | self.assertEqual(b'PNG', upload.getbytes(1, 3)) |
Joe Gregorio | 910b9b1 | 2012-06-12 09:36:30 -0400 | [diff] [blame] | 325 | |
Joe Gregorio | c80ac9d | 2012-08-21 14:09:09 -0400 | [diff] [blame] | 326 | def test_media_io_base_upload_raises_on_invalid_chunksize(self): |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 327 | f = open(datafile('small.png'), 'rb') |
Pat Ferate | ed9affd | 2015-03-03 16:03:15 -0800 | [diff] [blame] | 328 | fd = BytesIO(f.read()) |
| 329 | self.assertRaises(InvalidChunkSizeError, MediaIoBaseUpload, |
| 330 | fd, 'image/png', chunksize=-2, resumable=True) |
Joe Gregorio | c80ac9d | 2012-08-21 14:09:09 -0400 | [diff] [blame] | 331 | |
| 332 | def test_media_io_base_upload_streamable(self): |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 333 | fd = BytesIO(b'stuff') |
Pat Ferate | ed9affd | 2015-03-03 16:03:15 -0800 | [diff] [blame] | 334 | upload = MediaIoBaseUpload( |
| 335 | fd=fd, mimetype='image/png', chunksize=500, resumable=True) |
| 336 | self.assertEqual(True, upload.has_stream()) |
| 337 | self.assertEqual(fd, upload.stream()) |
Joe Gregorio | c80ac9d | 2012-08-21 14:09:09 -0400 | [diff] [blame] | 338 | |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 339 | def test_media_io_base_next_chunk_retries(self): |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 340 | f = open(datafile('small.png'), 'rb') |
Pat Ferate | ed9affd | 2015-03-03 16:03:15 -0800 | [diff] [blame] | 341 | fd = BytesIO(f.read()) |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 342 | upload = MediaIoBaseUpload( |
| 343 | fd=fd, mimetype='image/png', chunksize=500, resumable=True) |
| 344 | |
Zhihao Yuan | cc6d398 | 2016-07-27 11:40:45 -0500 | [diff] [blame] | 345 | # Simulate errors for both the request that creates the resumable upload |
| 346 | # and the upload itself. |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 347 | http = HttpMockSequence([ |
| 348 | ({'status': '500'}, ''), |
| 349 | ({'status': '500'}, ''), |
| 350 | ({'status': '503'}, ''), |
| 351 | ({'status': '200', 'location': 'location'}, ''), |
Zhihao Yuan | cc6d398 | 2016-07-27 11:40:45 -0500 | [diff] [blame] | 352 | ({'status': '403'}, USER_RATE_LIMIT_EXCEEDED_RESPONSE), |
| 353 | ({'status': '403'}, RATE_LIMIT_EXCEEDED_RESPONSE), |
| 354 | ({'status': '429'}, ''), |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 355 | ({'status': '200'}, '{}'), |
| 356 | ]) |
| 357 | |
| 358 | model = JsonModel() |
| 359 | uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar' |
| 360 | method = u'POST' |
| 361 | request = HttpRequest( |
| 362 | http, |
| 363 | model.response, |
| 364 | uri, |
| 365 | method=method, |
| 366 | headers={}, |
| 367 | resumable=upload) |
| 368 | |
| 369 | sleeptimes = [] |
| 370 | request._sleep = lambda x: sleeptimes.append(x) |
| 371 | request._rand = lambda: 10 |
| 372 | |
| 373 | request.execute(num_retries=3) |
| 374 | self.assertEqual([20, 40, 80, 20, 40, 80], sleeptimes) |
| 375 | |
Zhihao Yuan | cc6d398 | 2016-07-27 11:40:45 -0500 | [diff] [blame] | 376 | def test_media_io_base_next_chunk_no_retry_403_not_configured(self): |
| 377 | fd = BytesIO(b"i am png") |
| 378 | upload = MediaIoBaseUpload( |
| 379 | fd=fd, mimetype='image/png', chunksize=500, resumable=True) |
| 380 | |
| 381 | http = HttpMockSequence([ |
| 382 | ({'status': '403'}, NOT_CONFIGURED_RESPONSE), |
| 383 | ({'status': '200'}, '{}') |
| 384 | ]) |
| 385 | |
| 386 | model = JsonModel() |
| 387 | uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar' |
| 388 | method = u'POST' |
| 389 | request = HttpRequest( |
| 390 | http, |
| 391 | model.response, |
| 392 | uri, |
| 393 | method=method, |
| 394 | headers={}, |
| 395 | resumable=upload) |
| 396 | |
| 397 | request._rand = lambda: 1.0 |
| 398 | request._sleep = mock.MagicMock() |
| 399 | |
| 400 | with self.assertRaises(HttpError): |
| 401 | request.execute(num_retries=3) |
| 402 | request._sleep.assert_not_called() |
| 403 | |
Joe Gregorio | 910b9b1 | 2012-06-12 09:36:30 -0400 | [diff] [blame] | 404 | |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 405 | class TestMediaIoBaseDownload(unittest.TestCase): |
| 406 | |
| 407 | def setUp(self): |
| 408 | http = HttpMock(datafile('zoo.json'), {'status': '200'}) |
Joe Gregorio | 68a8cfe | 2012-08-03 16:17:40 -0400 | [diff] [blame] | 409 | zoo = build('zoo', 'v1', http=http) |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 410 | self.request = zoo.animals().get_media(name='Lion') |
Pat Ferate | ed9affd | 2015-03-03 16:03:15 -0800 | [diff] [blame] | 411 | self.fd = BytesIO() |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 412 | |
| 413 | def test_media_io_base_download(self): |
| 414 | self.request.http = HttpMockSequence([ |
| 415 | ({'status': '200', |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 416 | 'content-range': '0-2/5'}, b'123'), |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 417 | ({'status': '200', |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 418 | 'content-range': '3-4/5'}, b'45'), |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 419 | ]) |
Joe Gregorio | 97ef1cc | 2013-06-13 14:47:10 -0400 | [diff] [blame] | 420 | self.assertEqual(True, self.request.http.follow_redirects) |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 421 | |
| 422 | download = MediaIoBaseDownload( |
Joe Gregorio | 4a2c29f | 2012-07-12 12:52:47 -0400 | [diff] [blame] | 423 | fd=self.fd, request=self.request, chunksize=3) |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 424 | |
Joe Gregorio | 4a2c29f | 2012-07-12 12:52:47 -0400 | [diff] [blame] | 425 | self.assertEqual(self.fd, download._fd) |
| 426 | self.assertEqual(3, download._chunksize) |
| 427 | self.assertEqual(0, download._progress) |
| 428 | self.assertEqual(None, download._total_size) |
| 429 | self.assertEqual(False, download._done) |
| 430 | self.assertEqual(self.request.uri, download._uri) |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 431 | |
| 432 | status, done = download.next_chunk() |
| 433 | |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 434 | self.assertEqual(self.fd.getvalue(), b'123') |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 435 | self.assertEqual(False, done) |
Joe Gregorio | 4a2c29f | 2012-07-12 12:52:47 -0400 | [diff] [blame] | 436 | self.assertEqual(3, download._progress) |
| 437 | self.assertEqual(5, download._total_size) |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 438 | self.assertEqual(3, status.resumable_progress) |
| 439 | |
| 440 | status, done = download.next_chunk() |
| 441 | |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 442 | self.assertEqual(self.fd.getvalue(), b'12345') |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 443 | self.assertEqual(True, done) |
Joe Gregorio | 4a2c29f | 2012-07-12 12:52:47 -0400 | [diff] [blame] | 444 | self.assertEqual(5, download._progress) |
| 445 | self.assertEqual(5, download._total_size) |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 446 | |
| 447 | def test_media_io_base_download_handle_redirects(self): |
| 448 | self.request.http = HttpMockSequence([ |
Joe Gregorio | 238feb7 | 2013-06-19 13:15:31 -0400 | [diff] [blame] | 449 | ({'status': '200', |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 450 | 'content-location': 'https://secure.example.net/lion'}, b''), |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 451 | ({'status': '200', |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 452 | 'content-range': '0-2/5'}, b'abc'), |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 453 | ]) |
| 454 | |
| 455 | download = MediaIoBaseDownload( |
Joe Gregorio | 4a2c29f | 2012-07-12 12:52:47 -0400 | [diff] [blame] | 456 | fd=self.fd, request=self.request, chunksize=3) |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 457 | |
| 458 | status, done = download.next_chunk() |
| 459 | |
Joe Gregorio | 4a2c29f | 2012-07-12 12:52:47 -0400 | [diff] [blame] | 460 | self.assertEqual('https://secure.example.net/lion', download._uri) |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 461 | |
| 462 | def test_media_io_base_download_handle_4xx(self): |
| 463 | self.request.http = HttpMockSequence([ |
| 464 | ({'status': '400'}, ''), |
| 465 | ]) |
| 466 | |
| 467 | download = MediaIoBaseDownload( |
Joe Gregorio | 4a2c29f | 2012-07-12 12:52:47 -0400 | [diff] [blame] | 468 | fd=self.fd, request=self.request, chunksize=3) |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 469 | |
| 470 | try: |
| 471 | status, done = download.next_chunk() |
| 472 | self.fail('Should raise an exception') |
| 473 | except HttpError: |
| 474 | pass |
| 475 | |
| 476 | # Even after raising an exception we can pick up where we left off. |
| 477 | self.request.http = HttpMockSequence([ |
| 478 | ({'status': '200', |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 479 | 'content-range': '0-2/5'}, b'123'), |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 480 | ]) |
| 481 | |
| 482 | status, done = download.next_chunk() |
| 483 | |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 484 | self.assertEqual(self.fd.getvalue(), b'123') |
Joe Gregorio | 708388c | 2012-06-15 13:43:04 -0400 | [diff] [blame] | 485 | |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 486 | def test_media_io_base_download_retries_connection_errors(self): |
| 487 | self.request.http = HttpMockWithErrors( |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 488 | 3, {'status': '200', 'content-range': '0-2/3'}, b'123') |
| 489 | |
| 490 | download = MediaIoBaseDownload( |
| 491 | fd=self.fd, request=self.request, chunksize=3) |
| 492 | download._sleep = lambda _x: 0 # do nothing |
| 493 | download._rand = lambda: 10 |
| 494 | |
| 495 | status, done = download.next_chunk(num_retries=3) |
| 496 | |
| 497 | self.assertEqual(self.fd.getvalue(), b'123') |
| 498 | self.assertEqual(True, done) |
| 499 | |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 500 | def test_media_io_base_download_retries_5xx(self): |
| 501 | self.request.http = HttpMockSequence([ |
| 502 | ({'status': '500'}, ''), |
| 503 | ({'status': '500'}, ''), |
| 504 | ({'status': '500'}, ''), |
| 505 | ({'status': '200', |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 506 | 'content-range': '0-2/5'}, b'123'), |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 507 | ({'status': '503'}, ''), |
| 508 | ({'status': '503'}, ''), |
| 509 | ({'status': '503'}, ''), |
| 510 | ({'status': '200', |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 511 | 'content-range': '3-4/5'}, b'45'), |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 512 | ]) |
| 513 | |
| 514 | download = MediaIoBaseDownload( |
| 515 | fd=self.fd, request=self.request, chunksize=3) |
| 516 | |
| 517 | self.assertEqual(self.fd, download._fd) |
| 518 | self.assertEqual(3, download._chunksize) |
| 519 | self.assertEqual(0, download._progress) |
| 520 | self.assertEqual(None, download._total_size) |
| 521 | self.assertEqual(False, download._done) |
| 522 | self.assertEqual(self.request.uri, download._uri) |
| 523 | |
| 524 | # Set time.sleep and random.random stubs. |
| 525 | sleeptimes = [] |
| 526 | download._sleep = lambda x: sleeptimes.append(x) |
| 527 | download._rand = lambda: 10 |
| 528 | |
| 529 | status, done = download.next_chunk(num_retries=3) |
| 530 | |
| 531 | # Check for exponential backoff using the rand function above. |
| 532 | self.assertEqual([20, 40, 80], sleeptimes) |
| 533 | |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 534 | self.assertEqual(self.fd.getvalue(), b'123') |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 535 | self.assertEqual(False, done) |
| 536 | self.assertEqual(3, download._progress) |
| 537 | self.assertEqual(5, download._total_size) |
| 538 | self.assertEqual(3, status.resumable_progress) |
| 539 | |
| 540 | # Reset time.sleep stub. |
| 541 | del sleeptimes[0:len(sleeptimes)] |
| 542 | |
| 543 | status, done = download.next_chunk(num_retries=3) |
| 544 | |
| 545 | # Check for exponential backoff using the rand function above. |
| 546 | self.assertEqual([20, 40, 80], sleeptimes) |
| 547 | |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 548 | self.assertEqual(self.fd.getvalue(), b'12345') |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 549 | self.assertEqual(True, done) |
| 550 | self.assertEqual(5, download._progress) |
| 551 | self.assertEqual(5, download._total_size) |
| 552 | |
andrewnester | a4a44cf | 2017-03-31 16:09:31 +0300 | [diff] [blame^] | 553 | def test_media_io_base_download_empty_file(self): |
| 554 | self.request.http = HttpMockSequence([ |
| 555 | ({'status': '200', |
| 556 | 'content-range': '0-0/0'}, b''), |
| 557 | ]) |
| 558 | |
| 559 | download = MediaIoBaseDownload( |
| 560 | fd=self.fd, request=self.request, chunksize=3) |
| 561 | |
| 562 | self.assertEqual(self.fd, download._fd) |
| 563 | self.assertEqual(0, download._progress) |
| 564 | self.assertEqual(None, download._total_size) |
| 565 | self.assertEqual(False, download._done) |
| 566 | self.assertEqual(self.request.uri, download._uri) |
| 567 | |
| 568 | status, done = download.next_chunk() |
| 569 | |
| 570 | self.assertEqual(True, done) |
| 571 | self.assertEqual(0, download._progress) |
| 572 | self.assertEqual(0, download._total_size) |
| 573 | self.assertEqual(0, status.progress()) |
| 574 | |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 575 | EXPECTED = """POST /someapi/v1/collection/?foo=bar HTTP/1.1 |
| 576 | Content-Type: application/json |
| 577 | MIME-Version: 1.0 |
Joe Gregorio | 5d1171b | 2012-01-05 10:48:24 -0500 | [diff] [blame] | 578 | Host: www.googleapis.com |
| 579 | content-length: 2\r\n\r\n{}""" |
| 580 | |
| 581 | |
| 582 | NO_BODY_EXPECTED = """POST /someapi/v1/collection/?foo=bar HTTP/1.1 |
| 583 | Content-Type: application/json |
| 584 | MIME-Version: 1.0 |
| 585 | Host: www.googleapis.com |
| 586 | content-length: 0\r\n\r\n""" |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 587 | |
Pepper Lebeck-Jobe | 30ecbd4 | 2015-06-12 11:45:57 -0400 | [diff] [blame] | 588 | NO_BODY_EXPECTED_GET = """GET /someapi/v1/collection/?foo=bar HTTP/1.1 |
| 589 | Content-Type: application/json |
| 590 | MIME-Version: 1.0 |
| 591 | Host: www.googleapis.com\r\n\r\n""" |
| 592 | |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 593 | |
| 594 | RESPONSE = """HTTP/1.1 200 OK |
Joe Gregorio | 20b54fb | 2012-07-26 09:59:35 -0400 | [diff] [blame] | 595 | Content-Type: application/json |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 596 | Content-Length: 14 |
| 597 | ETag: "etag/pony"\r\n\r\n{"answer": 42}""" |
| 598 | |
| 599 | |
INADA Naoki | 0915761 | 2015-03-25 01:51:03 +0900 | [diff] [blame] | 600 | BATCH_RESPONSE = b"""--batch_foobarbaz |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 601 | Content-Type: application/http |
| 602 | Content-Transfer-Encoding: binary |
| 603 | Content-ID: <randomness+1> |
| 604 | |
| 605 | HTTP/1.1 200 OK |
Joe Gregorio | 20b54fb | 2012-07-26 09:59:35 -0400 | [diff] [blame] | 606 | Content-Type: application/json |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 607 | Content-Length: 14 |
| 608 | ETag: "etag/pony"\r\n\r\n{"foo": 42} |
| 609 | |
| 610 | --batch_foobarbaz |
| 611 | Content-Type: application/http |
| 612 | Content-Transfer-Encoding: binary |
| 613 | Content-ID: <randomness+2> |
| 614 | |
| 615 | HTTP/1.1 200 OK |
Joe Gregorio | 20b54fb | 2012-07-26 09:59:35 -0400 | [diff] [blame] | 616 | Content-Type: application/json |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 617 | Content-Length: 14 |
| 618 | ETag: "etag/sheep"\r\n\r\n{"baz": "qux"} |
| 619 | --batch_foobarbaz--""" |
| 620 | |
Joe Gregorio | 5d1171b | 2012-01-05 10:48:24 -0500 | [diff] [blame] | 621 | |
INADA Naoki | 0915761 | 2015-03-25 01:51:03 +0900 | [diff] [blame] | 622 | BATCH_ERROR_RESPONSE = b"""--batch_foobarbaz |
Joe Gregorio | 3fb9367 | 2012-07-25 11:31:11 -0400 | [diff] [blame] | 623 | Content-Type: application/http |
| 624 | Content-Transfer-Encoding: binary |
| 625 | Content-ID: <randomness+1> |
| 626 | |
| 627 | HTTP/1.1 200 OK |
Joe Gregorio | 20b54fb | 2012-07-26 09:59:35 -0400 | [diff] [blame] | 628 | Content-Type: application/json |
Joe Gregorio | 3fb9367 | 2012-07-25 11:31:11 -0400 | [diff] [blame] | 629 | Content-Length: 14 |
| 630 | ETag: "etag/pony"\r\n\r\n{"foo": 42} |
| 631 | |
| 632 | --batch_foobarbaz |
| 633 | Content-Type: application/http |
| 634 | Content-Transfer-Encoding: binary |
| 635 | Content-ID: <randomness+2> |
| 636 | |
| 637 | HTTP/1.1 403 Access Not Configured |
Joe Gregorio | 20b54fb | 2012-07-26 09:59:35 -0400 | [diff] [blame] | 638 | Content-Type: application/json |
| 639 | Content-Length: 245 |
| 640 | ETag: "etag/sheep"\r\n\r\n{ |
Joe Gregorio | 3fb9367 | 2012-07-25 11:31:11 -0400 | [diff] [blame] | 641 | "error": { |
| 642 | "errors": [ |
| 643 | { |
| 644 | "domain": "usageLimits", |
| 645 | "reason": "accessNotConfigured", |
| 646 | "message": "Access Not Configured", |
| 647 | "debugInfo": "QuotaState: BLOCKED" |
| 648 | } |
| 649 | ], |
| 650 | "code": 403, |
| 651 | "message": "Access Not Configured" |
| 652 | } |
| 653 | } |
| 654 | |
| 655 | --batch_foobarbaz--""" |
| 656 | |
| 657 | |
INADA Naoki | 0915761 | 2015-03-25 01:51:03 +0900 | [diff] [blame] | 658 | BATCH_RESPONSE_WITH_401 = b"""--batch_foobarbaz |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 659 | Content-Type: application/http |
| 660 | Content-Transfer-Encoding: binary |
| 661 | Content-ID: <randomness+1> |
| 662 | |
Joe Gregorio | c752e33 | 2012-07-11 14:43:52 -0400 | [diff] [blame] | 663 | HTTP/1.1 401 Authorization Required |
Joe Gregorio | 20b54fb | 2012-07-26 09:59:35 -0400 | [diff] [blame] | 664 | Content-Type: application/json |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 665 | Content-Length: 14 |
| 666 | ETag: "etag/pony"\r\n\r\n{"error": {"message": |
| 667 | "Authorizaton failed."}} |
| 668 | |
| 669 | --batch_foobarbaz |
| 670 | Content-Type: application/http |
| 671 | Content-Transfer-Encoding: binary |
| 672 | Content-ID: <randomness+2> |
| 673 | |
| 674 | HTTP/1.1 200 OK |
Joe Gregorio | 20b54fb | 2012-07-26 09:59:35 -0400 | [diff] [blame] | 675 | Content-Type: application/json |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 676 | Content-Length: 14 |
| 677 | ETag: "etag/sheep"\r\n\r\n{"baz": "qux"} |
| 678 | --batch_foobarbaz--""" |
| 679 | |
| 680 | |
INADA Naoki | 0915761 | 2015-03-25 01:51:03 +0900 | [diff] [blame] | 681 | BATCH_SINGLE_RESPONSE = b"""--batch_foobarbaz |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 682 | Content-Type: application/http |
| 683 | Content-Transfer-Encoding: binary |
| 684 | Content-ID: <randomness+1> |
| 685 | |
| 686 | HTTP/1.1 200 OK |
Joe Gregorio | 20b54fb | 2012-07-26 09:59:35 -0400 | [diff] [blame] | 687 | Content-Type: application/json |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 688 | Content-Length: 14 |
| 689 | ETag: "etag/pony"\r\n\r\n{"foo": 42} |
| 690 | --batch_foobarbaz--""" |
| 691 | |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 692 | |
| 693 | USER_RATE_LIMIT_EXCEEDED_RESPONSE = """{ |
| 694 | "error": { |
| 695 | "errors": [ |
| 696 | { |
| 697 | "domain": "usageLimits", |
| 698 | "reason": "userRateLimitExceeded", |
| 699 | "message": "User Rate Limit Exceeded" |
| 700 | } |
| 701 | ], |
| 702 | "code": 403, |
| 703 | "message": "User Rate Limit Exceeded" |
| 704 | } |
| 705 | }""" |
| 706 | |
| 707 | |
| 708 | RATE_LIMIT_EXCEEDED_RESPONSE = """{ |
| 709 | "error": { |
| 710 | "errors": [ |
| 711 | { |
| 712 | "domain": "usageLimits", |
| 713 | "reason": "rateLimitExceeded", |
| 714 | "message": "Rate Limit Exceeded" |
| 715 | } |
| 716 | ], |
| 717 | "code": 403, |
| 718 | "message": "Rate Limit Exceeded" |
| 719 | } |
| 720 | }""" |
| 721 | |
| 722 | |
| 723 | NOT_CONFIGURED_RESPONSE = """{ |
| 724 | "error": { |
| 725 | "errors": [ |
| 726 | { |
| 727 | "domain": "usageLimits", |
| 728 | "reason": "accessNotConfigured", |
| 729 | "message": "Access Not Configured" |
| 730 | } |
| 731 | ], |
| 732 | "code": 403, |
| 733 | "message": "Access Not Configured" |
| 734 | } |
| 735 | }""" |
| 736 | |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 737 | class Callbacks(object): |
| 738 | def __init__(self): |
| 739 | self.responses = {} |
| 740 | self.exceptions = {} |
| 741 | |
| 742 | def f(self, request_id, response, exception): |
| 743 | self.responses[request_id] = response |
| 744 | self.exceptions[request_id] = exception |
| 745 | |
| 746 | |
Joe Gregorio | 83f2ee6 | 2012-12-06 15:25:54 -0500 | [diff] [blame] | 747 | class TestHttpRequest(unittest.TestCase): |
| 748 | def test_unicode(self): |
| 749 | http = HttpMock(datafile('zoo.json'), headers={'status': '200'}) |
| 750 | model = JsonModel() |
| 751 | uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' |
| 752 | method = u'POST' |
| 753 | request = HttpRequest( |
| 754 | http, |
| 755 | model.response, |
| 756 | uri, |
| 757 | method=method, |
| 758 | body=u'{}', |
| 759 | headers={'content-type': 'application/json'}) |
| 760 | request.execute() |
| 761 | self.assertEqual(uri, http.uri) |
| 762 | self.assertEqual(str, type(http.uri)) |
| 763 | self.assertEqual(method, http.method) |
| 764 | self.assertEqual(str, type(http.method)) |
| 765 | |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 766 | def test_no_retry_connection_errors(self): |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 767 | model = JsonModel() |
| 768 | request = HttpRequest( |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 769 | HttpMockWithNonRetriableErrors(1, {'status': '200'}, '{"foo": "bar"}'), |
| 770 | model.response, |
| 771 | u'https://www.example.com/json_api_endpoint') |
| 772 | request._sleep = lambda _x: 0 # do nothing |
| 773 | request._rand = lambda: 10 |
| 774 | with self.assertRaises(socket.error): |
| 775 | response = request.execute(num_retries=3) |
| 776 | |
| 777 | |
| 778 | def test_retry_connection_errors_non_resumable(self): |
| 779 | model = JsonModel() |
| 780 | request = HttpRequest( |
| 781 | HttpMockWithErrors(3, {'status': '200'}, '{"foo": "bar"}'), |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 782 | model.response, |
| 783 | u'https://www.example.com/json_api_endpoint') |
| 784 | request._sleep = lambda _x: 0 # do nothing |
| 785 | request._rand = lambda: 10 |
| 786 | response = request.execute(num_retries=3) |
| 787 | self.assertEqual({u'foo': u'bar'}, response) |
| 788 | |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 789 | def test_retry_connection_errors_resumable(self): |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 790 | with open(datafile('small.png'), 'rb') as small_png_file: |
| 791 | small_png_fd = BytesIO(small_png_file.read()) |
| 792 | upload = MediaIoBaseUpload(fd=small_png_fd, mimetype='image/png', |
| 793 | chunksize=500, resumable=True) |
| 794 | model = JsonModel() |
| 795 | |
| 796 | request = HttpRequest( |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 797 | HttpMockWithErrors( |
Sergiy Byelozyorov | 703c92c | 2015-12-21 23:27:48 +0100 | [diff] [blame] | 798 | 3, {'status': '200', 'location': 'location'}, '{"foo": "bar"}'), |
| 799 | model.response, |
| 800 | u'https://www.example.com/file_upload', |
| 801 | method='POST', |
| 802 | resumable=upload) |
| 803 | request._sleep = lambda _x: 0 # do nothing |
| 804 | request._rand = lambda: 10 |
| 805 | response = request.execute(num_retries=3) |
| 806 | self.assertEqual({u'foo': u'bar'}, response) |
| 807 | |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 808 | def test_retry(self): |
| 809 | num_retries = 5 |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 810 | resp_seq = [({'status': '500'}, '')] * (num_retries - 3) |
| 811 | resp_seq.append(({'status': '403'}, RATE_LIMIT_EXCEEDED_RESPONSE)) |
| 812 | resp_seq.append(({'status': '403'}, USER_RATE_LIMIT_EXCEEDED_RESPONSE)) |
| 813 | resp_seq.append(({'status': '429'}, '')) |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 814 | resp_seq.append(({'status': '200'}, '{}')) |
| 815 | |
| 816 | http = HttpMockSequence(resp_seq) |
| 817 | model = JsonModel() |
| 818 | uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' |
| 819 | method = u'POST' |
| 820 | request = HttpRequest( |
| 821 | http, |
| 822 | model.response, |
| 823 | uri, |
| 824 | method=method, |
| 825 | body=u'{}', |
| 826 | headers={'content-type': 'application/json'}) |
| 827 | |
| 828 | sleeptimes = [] |
| 829 | request._sleep = lambda x: sleeptimes.append(x) |
| 830 | request._rand = lambda: 10 |
| 831 | |
| 832 | request.execute(num_retries=num_retries) |
| 833 | |
| 834 | self.assertEqual(num_retries, len(sleeptimes)) |
INADA Naoki | d898a37 | 2015-03-04 03:52:46 +0900 | [diff] [blame] | 835 | for retry_num in range(num_retries): |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 836 | self.assertEqual(10 * 2**(retry_num + 1), sleeptimes[retry_num]) |
| 837 | |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 838 | def test_no_retry_succeeds(self): |
| 839 | num_retries = 5 |
| 840 | resp_seq = [({'status': '200'}, '{}')] * (num_retries) |
| 841 | |
| 842 | http = HttpMockSequence(resp_seq) |
| 843 | model = JsonModel() |
| 844 | uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' |
| 845 | method = u'POST' |
| 846 | request = HttpRequest( |
| 847 | http, |
| 848 | model.response, |
| 849 | uri, |
| 850 | method=method, |
| 851 | body=u'{}', |
| 852 | headers={'content-type': 'application/json'}) |
| 853 | |
| 854 | sleeptimes = [] |
| 855 | request._sleep = lambda x: sleeptimes.append(x) |
| 856 | request._rand = lambda: 10 |
| 857 | |
| 858 | request.execute(num_retries=num_retries) |
| 859 | |
| 860 | self.assertEqual(0, len(sleeptimes)) |
| 861 | |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 862 | def test_no_retry_fails_fast(self): |
| 863 | http = HttpMockSequence([ |
| 864 | ({'status': '500'}, ''), |
| 865 | ({'status': '200'}, '{}') |
| 866 | ]) |
| 867 | model = JsonModel() |
| 868 | uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' |
| 869 | method = u'POST' |
| 870 | request = HttpRequest( |
| 871 | http, |
| 872 | model.response, |
| 873 | uri, |
| 874 | method=method, |
| 875 | body=u'{}', |
| 876 | headers={'content-type': 'application/json'}) |
| 877 | |
| 878 | request._rand = lambda: 1.0 |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 879 | request._sleep = mock.MagicMock() |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 880 | |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 881 | with self.assertRaises(HttpError): |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 882 | request.execute() |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 883 | request._sleep.assert_not_called() |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 884 | |
eesheesh | c6425a0 | 2016-02-12 15:07:06 +0000 | [diff] [blame] | 885 | def test_no_retry_403_not_configured_fails_fast(self): |
| 886 | http = HttpMockSequence([ |
| 887 | ({'status': '403'}, NOT_CONFIGURED_RESPONSE), |
| 888 | ({'status': '200'}, '{}') |
| 889 | ]) |
| 890 | model = JsonModel() |
| 891 | uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' |
| 892 | method = u'POST' |
| 893 | request = HttpRequest( |
| 894 | http, |
| 895 | model.response, |
| 896 | uri, |
| 897 | method=method, |
| 898 | body=u'{}', |
| 899 | headers={'content-type': 'application/json'}) |
| 900 | |
| 901 | request._rand = lambda: 1.0 |
| 902 | request._sleep = mock.MagicMock() |
| 903 | |
| 904 | with self.assertRaises(HttpError): |
| 905 | request.execute() |
| 906 | request._sleep.assert_not_called() |
| 907 | |
| 908 | def test_no_retry_403_fails_fast(self): |
| 909 | http = HttpMockSequence([ |
| 910 | ({'status': '403'}, ''), |
| 911 | ({'status': '200'}, '{}') |
| 912 | ]) |
| 913 | model = JsonModel() |
| 914 | uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' |
| 915 | method = u'POST' |
| 916 | request = HttpRequest( |
| 917 | http, |
| 918 | model.response, |
| 919 | uri, |
| 920 | method=method, |
| 921 | body=u'{}', |
| 922 | headers={'content-type': 'application/json'}) |
| 923 | |
| 924 | request._rand = lambda: 1.0 |
| 925 | request._sleep = mock.MagicMock() |
| 926 | |
| 927 | with self.assertRaises(HttpError): |
| 928 | request.execute() |
| 929 | request._sleep.assert_not_called() |
| 930 | |
| 931 | def test_no_retry_401_fails_fast(self): |
| 932 | http = HttpMockSequence([ |
| 933 | ({'status': '401'}, ''), |
| 934 | ({'status': '200'}, '{}') |
| 935 | ]) |
| 936 | model = JsonModel() |
| 937 | uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' |
| 938 | method = u'POST' |
| 939 | request = HttpRequest( |
| 940 | http, |
| 941 | model.response, |
| 942 | uri, |
| 943 | method=method, |
| 944 | body=u'{}', |
| 945 | headers={'content-type': 'application/json'}) |
| 946 | |
| 947 | request._rand = lambda: 1.0 |
| 948 | request._sleep = mock.MagicMock() |
| 949 | |
| 950 | with self.assertRaises(HttpError): |
| 951 | request.execute() |
| 952 | request._sleep.assert_not_called() |
Joe Gregorio | 83f2ee6 | 2012-12-06 15:25:54 -0500 | [diff] [blame] | 953 | |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 954 | class TestBatch(unittest.TestCase): |
| 955 | |
| 956 | def setUp(self): |
| 957 | model = JsonModel() |
| 958 | self.request1 = HttpRequest( |
| 959 | None, |
| 960 | model.response, |
| 961 | 'https://www.googleapis.com/someapi/v1/collection/?foo=bar', |
| 962 | method='POST', |
| 963 | body='{}', |
| 964 | headers={'content-type': 'application/json'}) |
| 965 | |
| 966 | self.request2 = HttpRequest( |
| 967 | None, |
| 968 | model.response, |
| 969 | 'https://www.googleapis.com/someapi/v1/collection/?foo=bar', |
Joe Gregorio | 5d1171b | 2012-01-05 10:48:24 -0500 | [diff] [blame] | 970 | method='GET', |
| 971 | body='', |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 972 | headers={'content-type': 'application/json'}) |
| 973 | |
| 974 | |
| 975 | def test_id_to_from_content_id_header(self): |
| 976 | batch = BatchHttpRequest() |
| 977 | self.assertEquals('12', batch._header_to_id(batch._id_to_header('12'))) |
| 978 | |
| 979 | def test_invalid_content_id_header(self): |
| 980 | batch = BatchHttpRequest() |
| 981 | self.assertRaises(BatchError, batch._header_to_id, '[foo+x]') |
| 982 | self.assertRaises(BatchError, batch._header_to_id, 'foo+1') |
| 983 | self.assertRaises(BatchError, batch._header_to_id, '<foo>') |
| 984 | |
| 985 | def test_serialize_request(self): |
| 986 | batch = BatchHttpRequest() |
| 987 | request = HttpRequest( |
| 988 | None, |
| 989 | None, |
| 990 | 'https://www.googleapis.com/someapi/v1/collection/?foo=bar', |
| 991 | method='POST', |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 992 | body=u'{}', |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 993 | headers={'content-type': 'application/json'}, |
| 994 | methodId=None, |
| 995 | resumable=None) |
| 996 | s = batch._serialize_request(request).splitlines() |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 997 | self.assertEqual(EXPECTED.splitlines(), s) |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 998 | |
Joe Gregorio | dd81382 | 2012-01-25 10:32:47 -0500 | [diff] [blame] | 999 | def test_serialize_request_media_body(self): |
| 1000 | batch = BatchHttpRequest() |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 1001 | f = open(datafile('small.png'), 'rb') |
Joe Gregorio | dd81382 | 2012-01-25 10:32:47 -0500 | [diff] [blame] | 1002 | body = f.read() |
| 1003 | f.close() |
| 1004 | |
| 1005 | request = HttpRequest( |
| 1006 | None, |
| 1007 | None, |
| 1008 | 'https://www.googleapis.com/someapi/v1/collection/?foo=bar', |
| 1009 | method='POST', |
| 1010 | body=body, |
| 1011 | headers={'content-type': 'application/json'}, |
| 1012 | methodId=None, |
| 1013 | resumable=None) |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 1014 | # Just testing it shouldn't raise an exception. |
Joe Gregorio | dd81382 | 2012-01-25 10:32:47 -0500 | [diff] [blame] | 1015 | s = batch._serialize_request(request).splitlines() |
| 1016 | |
Joe Gregorio | 5d1171b | 2012-01-05 10:48:24 -0500 | [diff] [blame] | 1017 | def test_serialize_request_no_body(self): |
| 1018 | batch = BatchHttpRequest() |
| 1019 | request = HttpRequest( |
| 1020 | None, |
| 1021 | None, |
| 1022 | 'https://www.googleapis.com/someapi/v1/collection/?foo=bar', |
| 1023 | method='POST', |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 1024 | body=b'', |
Joe Gregorio | 5d1171b | 2012-01-05 10:48:24 -0500 | [diff] [blame] | 1025 | headers={'content-type': 'application/json'}, |
| 1026 | methodId=None, |
| 1027 | resumable=None) |
| 1028 | s = batch._serialize_request(request).splitlines() |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 1029 | self.assertEqual(NO_BODY_EXPECTED.splitlines(), s) |
Joe Gregorio | 5d1171b | 2012-01-05 10:48:24 -0500 | [diff] [blame] | 1030 | |
Pepper Lebeck-Jobe | 30ecbd4 | 2015-06-12 11:45:57 -0400 | [diff] [blame] | 1031 | def test_serialize_get_request_no_body(self): |
| 1032 | batch = BatchHttpRequest() |
| 1033 | request = HttpRequest( |
| 1034 | None, |
| 1035 | None, |
| 1036 | 'https://www.googleapis.com/someapi/v1/collection/?foo=bar', |
| 1037 | method='GET', |
| 1038 | body=None, |
| 1039 | headers={'content-type': 'application/json'}, |
| 1040 | methodId=None, |
| 1041 | resumable=None) |
| 1042 | s = batch._serialize_request(request).splitlines() |
| 1043 | self.assertEqual(NO_BODY_EXPECTED_GET.splitlines(), s) |
| 1044 | |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 1045 | def test_deserialize_response(self): |
| 1046 | batch = BatchHttpRequest() |
| 1047 | resp, content = batch._deserialize_response(RESPONSE) |
| 1048 | |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 1049 | self.assertEqual(200, resp.status) |
| 1050 | self.assertEqual('OK', resp.reason) |
| 1051 | self.assertEqual(11, resp.version) |
| 1052 | self.assertEqual('{"answer": 42}', content) |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 1053 | |
| 1054 | def test_new_id(self): |
| 1055 | batch = BatchHttpRequest() |
| 1056 | |
| 1057 | id_ = batch._new_id() |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 1058 | self.assertEqual('1', id_) |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 1059 | |
| 1060 | id_ = batch._new_id() |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 1061 | self.assertEqual('2', id_) |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 1062 | |
| 1063 | batch.add(self.request1, request_id='3') |
| 1064 | |
| 1065 | id_ = batch._new_id() |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 1066 | self.assertEqual('4', id_) |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 1067 | |
| 1068 | def test_add(self): |
| 1069 | batch = BatchHttpRequest() |
| 1070 | batch.add(self.request1, request_id='1') |
| 1071 | self.assertRaises(KeyError, batch.add, self.request1, request_id='1') |
| 1072 | |
| 1073 | def test_add_fail_for_resumable(self): |
| 1074 | batch = BatchHttpRequest() |
| 1075 | |
| 1076 | upload = MediaFileUpload( |
| 1077 | datafile('small.png'), chunksize=500, resumable=True) |
| 1078 | self.request1.resumable = upload |
ittus | 5f00cad | 2016-10-15 10:32:40 +0800 | [diff] [blame] | 1079 | with self.assertRaises(BatchError) as batch_error: |
| 1080 | batch.add(self.request1, request_id='1') |
| 1081 | str(batch_error.exception) |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 1082 | |
Mohamed Zenadi | 1b5350d | 2015-07-30 11:52:39 +0200 | [diff] [blame] | 1083 | def test_execute_empty_batch_no_http(self): |
| 1084 | batch = BatchHttpRequest() |
| 1085 | ret = batch.execute() |
| 1086 | self.assertEqual(None, ret) |
| 1087 | |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 1088 | def test_execute(self): |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 1089 | batch = BatchHttpRequest() |
| 1090 | callbacks = Callbacks() |
| 1091 | |
| 1092 | batch.add(self.request1, callback=callbacks.f) |
| 1093 | batch.add(self.request2, callback=callbacks.f) |
| 1094 | http = HttpMockSequence([ |
| 1095 | ({'status': '200', |
| 1096 | 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, |
| 1097 | BATCH_RESPONSE), |
| 1098 | ]) |
Joe Gregorio | 68a8cfe | 2012-08-03 16:17:40 -0400 | [diff] [blame] | 1099 | batch.execute(http=http) |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 1100 | self.assertEqual({'foo': 42}, callbacks.responses['1']) |
| 1101 | self.assertEqual(None, callbacks.exceptions['1']) |
| 1102 | self.assertEqual({'baz': 'qux'}, callbacks.responses['2']) |
| 1103 | self.assertEqual(None, callbacks.exceptions['2']) |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 1104 | |
Joe Gregorio | 5d1171b | 2012-01-05 10:48:24 -0500 | [diff] [blame] | 1105 | def test_execute_request_body(self): |
| 1106 | batch = BatchHttpRequest() |
| 1107 | |
| 1108 | batch.add(self.request1) |
| 1109 | batch.add(self.request2) |
| 1110 | http = HttpMockSequence([ |
| 1111 | ({'status': '200', |
| 1112 | 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, |
| 1113 | 'echo_request_body'), |
| 1114 | ]) |
| 1115 | try: |
Joe Gregorio | 68a8cfe | 2012-08-03 16:17:40 -0400 | [diff] [blame] | 1116 | batch.execute(http=http) |
Joe Gregorio | 5d1171b | 2012-01-05 10:48:24 -0500 | [diff] [blame] | 1117 | self.fail('Should raise exception') |
INADA Naoki | c1505df | 2014-08-20 15:19:53 +0900 | [diff] [blame] | 1118 | except BatchError as e: |
Joe Gregorio | 5d1171b | 2012-01-05 10:48:24 -0500 | [diff] [blame] | 1119 | boundary, _ = e.content.split(None, 1) |
| 1120 | self.assertEqual('--', boundary[:2]) |
| 1121 | parts = e.content.split(boundary) |
| 1122 | self.assertEqual(4, len(parts)) |
| 1123 | self.assertEqual('', parts[0]) |
Craig Citro | 4282aa3 | 2014-06-21 00:33:39 -0700 | [diff] [blame] | 1124 | self.assertEqual('--', parts[3].rstrip()) |
Joe Gregorio | 5d1171b | 2012-01-05 10:48:24 -0500 | [diff] [blame] | 1125 | header = parts[1].splitlines()[1] |
| 1126 | self.assertEqual('Content-Type: application/http', header) |
| 1127 | |
Gabriel Garcia | 23174be | 2016-05-25 17:28:07 +0200 | [diff] [blame] | 1128 | def test_execute_initial_refresh_oauth2(self): |
| 1129 | batch = BatchHttpRequest() |
| 1130 | callbacks = Callbacks() |
| 1131 | cred = MockCredentials('Foo') |
| 1132 | |
| 1133 | # Pretend this is a OAuth2Credentials object |
| 1134 | cred.access_token = None |
| 1135 | |
| 1136 | http = HttpMockSequence([ |
| 1137 | ({'status': '200', |
| 1138 | 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, |
| 1139 | BATCH_SINGLE_RESPONSE), |
| 1140 | ]) |
| 1141 | |
| 1142 | cred.authorize(http) |
| 1143 | |
| 1144 | batch.add(self.request1, callback=callbacks.f) |
| 1145 | batch.execute(http=http) |
| 1146 | |
| 1147 | self.assertEqual({'foo': 42}, callbacks.responses['1']) |
| 1148 | self.assertIsNone(callbacks.exceptions['1']) |
| 1149 | |
| 1150 | self.assertEqual(1, cred._refreshed) |
| 1151 | |
| 1152 | self.assertEqual(1, cred._authorized) |
| 1153 | |
| 1154 | self.assertEqual(1, cred._applied) |
| 1155 | |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 1156 | def test_execute_refresh_and_retry_on_401(self): |
| 1157 | batch = BatchHttpRequest() |
| 1158 | callbacks = Callbacks() |
| 1159 | cred_1 = MockCredentials('Foo') |
| 1160 | cred_2 = MockCredentials('Bar') |
| 1161 | |
| 1162 | http = HttpMockSequence([ |
| 1163 | ({'status': '200', |
| 1164 | 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, |
| 1165 | BATCH_RESPONSE_WITH_401), |
| 1166 | ({'status': '200', |
| 1167 | 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, |
| 1168 | BATCH_SINGLE_RESPONSE), |
| 1169 | ]) |
| 1170 | |
| 1171 | creds_http_1 = HttpMockSequence([]) |
| 1172 | cred_1.authorize(creds_http_1) |
| 1173 | |
| 1174 | creds_http_2 = HttpMockSequence([]) |
| 1175 | cred_2.authorize(creds_http_2) |
| 1176 | |
| 1177 | self.request1.http = creds_http_1 |
| 1178 | self.request2.http = creds_http_2 |
| 1179 | |
| 1180 | batch.add(self.request1, callback=callbacks.f) |
| 1181 | batch.add(self.request2, callback=callbacks.f) |
Joe Gregorio | 68a8cfe | 2012-08-03 16:17:40 -0400 | [diff] [blame] | 1182 | batch.execute(http=http) |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 1183 | |
| 1184 | self.assertEqual({'foo': 42}, callbacks.responses['1']) |
| 1185 | self.assertEqual(None, callbacks.exceptions['1']) |
| 1186 | self.assertEqual({'baz': 'qux'}, callbacks.responses['2']) |
| 1187 | self.assertEqual(None, callbacks.exceptions['2']) |
| 1188 | |
| 1189 | self.assertEqual(1, cred_1._refreshed) |
| 1190 | self.assertEqual(0, cred_2._refreshed) |
| 1191 | |
| 1192 | self.assertEqual(1, cred_1._authorized) |
| 1193 | self.assertEqual(1, cred_2._authorized) |
| 1194 | |
| 1195 | self.assertEqual(1, cred_2._applied) |
| 1196 | self.assertEqual(2, cred_1._applied) |
| 1197 | |
| 1198 | def test_http_errors_passed_to_callback(self): |
| 1199 | batch = BatchHttpRequest() |
| 1200 | callbacks = Callbacks() |
| 1201 | cred_1 = MockCredentials('Foo') |
| 1202 | cred_2 = MockCredentials('Bar') |
| 1203 | |
| 1204 | http = HttpMockSequence([ |
| 1205 | ({'status': '200', |
| 1206 | 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, |
| 1207 | BATCH_RESPONSE_WITH_401), |
| 1208 | ({'status': '200', |
| 1209 | 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, |
| 1210 | BATCH_RESPONSE_WITH_401), |
| 1211 | ]) |
| 1212 | |
| 1213 | creds_http_1 = HttpMockSequence([]) |
| 1214 | cred_1.authorize(creds_http_1) |
| 1215 | |
| 1216 | creds_http_2 = HttpMockSequence([]) |
| 1217 | cred_2.authorize(creds_http_2) |
| 1218 | |
| 1219 | self.request1.http = creds_http_1 |
| 1220 | self.request2.http = creds_http_2 |
| 1221 | |
| 1222 | batch.add(self.request1, callback=callbacks.f) |
| 1223 | batch.add(self.request2, callback=callbacks.f) |
Joe Gregorio | 68a8cfe | 2012-08-03 16:17:40 -0400 | [diff] [blame] | 1224 | batch.execute(http=http) |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 1225 | |
| 1226 | self.assertEqual(None, callbacks.responses['1']) |
| 1227 | self.assertEqual(401, callbacks.exceptions['1'].resp.status) |
Joe Gregorio | c752e33 | 2012-07-11 14:43:52 -0400 | [diff] [blame] | 1228 | self.assertEqual( |
| 1229 | 'Authorization Required', callbacks.exceptions['1'].resp.reason) |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 1230 | self.assertEqual({u'baz': u'qux'}, callbacks.responses['2']) |
| 1231 | self.assertEqual(None, callbacks.exceptions['2']) |
| 1232 | |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 1233 | def test_execute_global_callback(self): |
Joe Gregorio | 66f5752 | 2011-11-30 11:00:00 -0500 | [diff] [blame] | 1234 | callbacks = Callbacks() |
| 1235 | batch = BatchHttpRequest(callback=callbacks.f) |
| 1236 | |
| 1237 | batch.add(self.request1) |
| 1238 | batch.add(self.request2) |
| 1239 | http = HttpMockSequence([ |
| 1240 | ({'status': '200', |
| 1241 | 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, |
| 1242 | BATCH_RESPONSE), |
| 1243 | ]) |
Joe Gregorio | 68a8cfe | 2012-08-03 16:17:40 -0400 | [diff] [blame] | 1244 | batch.execute(http=http) |
Joe Gregorio | 654f4a2 | 2012-02-09 14:15:44 -0500 | [diff] [blame] | 1245 | self.assertEqual({'foo': 42}, callbacks.responses['1']) |
| 1246 | self.assertEqual({'baz': 'qux'}, callbacks.responses['2']) |
Joe Gregorio | d0bd388 | 2011-11-22 09:49:47 -0500 | [diff] [blame] | 1247 | |
Joe Gregorio | 20b54fb | 2012-07-26 09:59:35 -0400 | [diff] [blame] | 1248 | def test_execute_batch_http_error(self): |
Joe Gregorio | 3fb9367 | 2012-07-25 11:31:11 -0400 | [diff] [blame] | 1249 | callbacks = Callbacks() |
| 1250 | batch = BatchHttpRequest(callback=callbacks.f) |
| 1251 | |
| 1252 | batch.add(self.request1) |
| 1253 | batch.add(self.request2) |
| 1254 | http = HttpMockSequence([ |
| 1255 | ({'status': '200', |
| 1256 | 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'}, |
| 1257 | BATCH_ERROR_RESPONSE), |
| 1258 | ]) |
Joe Gregorio | 68a8cfe | 2012-08-03 16:17:40 -0400 | [diff] [blame] | 1259 | batch.execute(http=http) |
Joe Gregorio | 3fb9367 | 2012-07-25 11:31:11 -0400 | [diff] [blame] | 1260 | self.assertEqual({'foo': 42}, callbacks.responses['1']) |
| 1261 | expected = ('<HttpError 403 when requesting ' |
| 1262 | 'https://www.googleapis.com/someapi/v1/collection/?foo=bar returned ' |
| 1263 | '"Access Not Configured">') |
| 1264 | self.assertEqual(expected, str(callbacks.exceptions['2'])) |
Ali Afshar | 6f11ea1 | 2012-02-07 10:32:14 -0500 | [diff] [blame] | 1265 | |
Joe Gregorio | 5c120db | 2012-08-23 09:13:55 -0400 | [diff] [blame] | 1266 | |
Joe Gregorio | ba5c790 | 2012-08-03 12:48:16 -0400 | [diff] [blame] | 1267 | class TestRequestUriTooLong(unittest.TestCase): |
| 1268 | |
| 1269 | def test_turn_get_into_post(self): |
| 1270 | |
| 1271 | def _postproc(resp, content): |
| 1272 | return content |
| 1273 | |
| 1274 | http = HttpMockSequence([ |
| 1275 | ({'status': '200'}, |
| 1276 | 'echo_request_body'), |
| 1277 | ({'status': '200'}, |
| 1278 | 'echo_request_headers'), |
| 1279 | ]) |
| 1280 | |
| 1281 | # Send a long query parameter. |
| 1282 | query = { |
| 1283 | 'q': 'a' * MAX_URI_LENGTH + '?&' |
| 1284 | } |
| 1285 | req = HttpRequest( |
| 1286 | http, |
| 1287 | _postproc, |
Pat Ferate | d5b61bd | 2015-03-03 16:04:11 -0800 | [diff] [blame] | 1288 | 'http://example.com?' + urlencode(query), |
Joe Gregorio | ba5c790 | 2012-08-03 12:48:16 -0400 | [diff] [blame] | 1289 | method='GET', |
| 1290 | body=None, |
| 1291 | headers={}, |
| 1292 | methodId='foo', |
| 1293 | resumable=None) |
| 1294 | |
| 1295 | # Query parameters should be sent in the body. |
| 1296 | response = req.execute() |
INADA Naoki | 0915761 | 2015-03-25 01:51:03 +0900 | [diff] [blame] | 1297 | self.assertEqual(b'q=' + b'a' * MAX_URI_LENGTH + b'%3F%26', response) |
Joe Gregorio | ba5c790 | 2012-08-03 12:48:16 -0400 | [diff] [blame] | 1298 | |
| 1299 | # Extra headers should be set. |
| 1300 | response = req.execute() |
| 1301 | self.assertEqual('GET', response['x-http-method-override']) |
| 1302 | self.assertEqual(str(MAX_URI_LENGTH + 8), response['content-length']) |
| 1303 | self.assertEqual( |
| 1304 | 'application/x-www-form-urlencoded', response['content-type']) |
| 1305 | |
Joe Gregorio | 5c120db | 2012-08-23 09:13:55 -0400 | [diff] [blame] | 1306 | |
| 1307 | class TestStreamSlice(unittest.TestCase): |
| 1308 | """Test _StreamSlice.""" |
| 1309 | |
| 1310 | def setUp(self): |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 1311 | self.stream = BytesIO(b'0123456789') |
Joe Gregorio | 5c120db | 2012-08-23 09:13:55 -0400 | [diff] [blame] | 1312 | |
| 1313 | def test_read(self): |
| 1314 | s = _StreamSlice(self.stream, 0, 4) |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 1315 | self.assertEqual(b'', s.read(0)) |
| 1316 | self.assertEqual(b'0', s.read(1)) |
| 1317 | self.assertEqual(b'123', s.read()) |
Joe Gregorio | 5c120db | 2012-08-23 09:13:55 -0400 | [diff] [blame] | 1318 | |
| 1319 | def test_read_too_much(self): |
| 1320 | s = _StreamSlice(self.stream, 1, 4) |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 1321 | self.assertEqual(b'1234', s.read(6)) |
Joe Gregorio | 5c120db | 2012-08-23 09:13:55 -0400 | [diff] [blame] | 1322 | |
| 1323 | def test_read_all(self): |
| 1324 | s = _StreamSlice(self.stream, 2, 1) |
Pat Ferate | 2b14022 | 2015-03-03 18:05:11 -0800 | [diff] [blame] | 1325 | self.assertEqual(b'2', s.read(-1)) |
Joe Gregorio | 5c120db | 2012-08-23 09:13:55 -0400 | [diff] [blame] | 1326 | |
Ali Afshar | 164f37e | 2013-01-07 14:05:45 -0800 | [diff] [blame] | 1327 | |
| 1328 | class TestResponseCallback(unittest.TestCase): |
| 1329 | """Test adding callbacks to responses.""" |
| 1330 | |
| 1331 | def test_ensure_response_callback(self): |
| 1332 | m = JsonModel() |
| 1333 | request = HttpRequest( |
| 1334 | None, |
| 1335 | m.response, |
| 1336 | 'https://www.googleapis.com/someapi/v1/collection/?foo=bar', |
| 1337 | method='POST', |
| 1338 | body='{}', |
| 1339 | headers={'content-type': 'application/json'}) |
| 1340 | h = HttpMockSequence([ ({'status': 200}, '{}')]) |
| 1341 | responses = [] |
| 1342 | def _on_response(resp, responses=responses): |
| 1343 | responses.append(resp) |
| 1344 | request.add_response_callback(_on_response) |
| 1345 | request.execute(http=h) |
| 1346 | self.assertEqual(1, len(responses)) |
| 1347 | |
| 1348 | |
Craig Gurnik | 8e55b76 | 2015-01-20 15:00:10 -0500 | [diff] [blame] | 1349 | class TestHttpMock(unittest.TestCase): |
| 1350 | def test_default_response_headers(self): |
| 1351 | http = HttpMock(datafile('zoo.json')) |
| 1352 | resp, content = http.request("http://example.com") |
| 1353 | self.assertEqual(resp.status, 200) |
| 1354 | |
Alan Briolat | 26b0100 | 2015-08-14 00:13:57 +0100 | [diff] [blame] | 1355 | def test_error_response(self): |
| 1356 | http = HttpMock(datafile('bad_request.json'), {'status': '400'}) |
| 1357 | model = JsonModel() |
| 1358 | request = HttpRequest( |
| 1359 | http, |
| 1360 | model.response, |
| 1361 | 'https://www.googleapis.com/someapi/v1/collection/?foo=bar', |
| 1362 | method='GET', |
| 1363 | headers={}) |
| 1364 | self.assertRaises(HttpError, request.execute) |
| 1365 | |
Craig Gurnik | 8e55b76 | 2015-01-20 15:00:10 -0500 | [diff] [blame] | 1366 | |
Igor Maravić | 2243529 | 2017-01-19 22:28:22 +0100 | [diff] [blame] | 1367 | class TestHttpBuild(unittest.TestCase): |
| 1368 | original_socket_default_timeout = None |
| 1369 | |
| 1370 | @classmethod |
| 1371 | def setUpClass(cls): |
| 1372 | cls.original_socket_default_timeout = socket.getdefaulttimeout() |
| 1373 | |
| 1374 | @classmethod |
| 1375 | def tearDownClass(cls): |
| 1376 | socket.setdefaulttimeout(cls.original_socket_default_timeout) |
| 1377 | |
| 1378 | def test_build_http_sets_default_timeout_if_none_specified(self): |
| 1379 | socket.setdefaulttimeout(None) |
| 1380 | http = build_http() |
| 1381 | self.assertIsInstance(http.timeout, int) |
| 1382 | self.assertGreater(http.timeout, 0) |
| 1383 | |
| 1384 | def test_build_http_default_timeout_can_be_overridden(self): |
| 1385 | socket.setdefaulttimeout(1.5) |
| 1386 | http = build_http() |
| 1387 | self.assertAlmostEqual(http.timeout, 1.5, delta=0.001) |
| 1388 | |
| 1389 | def test_build_http_default_timeout_can_be_set_to_zero(self): |
| 1390 | socket.setdefaulttimeout(0) |
| 1391 | http = build_http() |
| 1392 | self.assertEquals(http.timeout, 0) |
| 1393 | |
| 1394 | |
Joe Gregorio | 6bcbcea | 2011-03-10 15:26:05 -0500 | [diff] [blame] | 1395 | if __name__ == '__main__': |
Joe Gregorio | 9086bd3 | 2013-06-14 16:32:05 -0400 | [diff] [blame] | 1396 | logging.getLogger().setLevel(logging.ERROR) |
Joe Gregorio | 6bcbcea | 2011-03-10 15:26:05 -0500 | [diff] [blame] | 1397 | unittest.main() |