blob: ce27e2e0926bdc3b3fc275a10aa188234c1e3092 [file] [log] [blame]
Craig Citro15744b12015-03-02 13:34:32 -08001#!/usr/bin/env python
Joe Gregorio6bcbcea2011-03-10 15:26:05 -05002#
Craig Citro751b7fb2014-09-23 11:20:38 -07003# Copyright 2014 Google Inc. All Rights Reserved.
Joe Gregorio6bcbcea2011-03-10 15:26:05 -05004#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Http tests
18
John Asmuth864311d2014-04-24 15:46:08 -040019Unit tests for the googleapiclient.http.
Joe Gregorio6bcbcea2011-03-10 15:26:05 -050020"""
INADA Naokid898a372015-03-04 03:52:46 +090021from __future__ import absolute_import
22from six.moves import range
Joe Gregorio6bcbcea2011-03-10 15:26:05 -050023
Bu Sun Kim66bb32c2019-10-30 10:11:58 -070024__author__ = "jcgregorio@google.com (Joe Gregorio)"
Joe Gregorio6bcbcea2011-03-10 15:26:05 -050025
Pat Feratec6050872015-03-03 18:24:59 -080026from six import PY3
Pat Ferateed9affd2015-03-03 16:03:15 -080027from six import BytesIO, StringIO
28from io import FileIO
Pat Ferated5b61bd2015-03-03 16:04:11 -080029from six.moves.urllib.parse import urlencode
Pat Ferateed9affd2015-03-03 16:03:15 -080030
Joe Gregorio7cbceab2011-06-27 10:46:54 -040031# Do not remove the httplib2 import
Chris McDonough0dc81bf2018-07-19 11:19:58 -040032import json
Joe Gregorio7cbceab2011-06-27 10:46:54 -040033import httplib2
Xiaofei Wang20b67582019-07-17 11:16:53 -070034import io
Joe Gregorio9086bd32013-06-14 16:32:05 -040035import logging
eesheeshc6425a02016-02-12 15:07:06 +000036import mock
Joe Gregoriod0bd3882011-11-22 09:49:47 -050037import os
Pat Ferate497a90f2015-03-09 09:52:54 -070038import unittest2 as unittest
Joe Gregorio9086bd32013-06-14 16:32:05 -040039import random
eesheeshc6425a02016-02-12 15:07:06 +000040import socket
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +010041import ssl
Joe Gregorio9086bd32013-06-14 16:32:05 -040042import time
Joe Gregorio6bcbcea2011-03-10 15:26:05 -050043
John Asmuth864311d2014-04-24 15:46:08 -040044from googleapiclient.discovery import build
45from googleapiclient.errors import BatchError
46from googleapiclient.errors import HttpError
47from googleapiclient.errors import InvalidChunkSizeError
Igor Maravić22435292017-01-19 22:28:22 +010048from googleapiclient.http import build_http
John Asmuth864311d2014-04-24 15:46:08 -040049from googleapiclient.http import BatchHttpRequest
50from googleapiclient.http import HttpMock
51from googleapiclient.http import HttpMockSequence
52from googleapiclient.http import HttpRequest
53from googleapiclient.http import MAX_URI_LENGTH
54from googleapiclient.http import MediaFileUpload
55from googleapiclient.http import MediaInMemoryUpload
56from googleapiclient.http import MediaIoBaseDownload
57from googleapiclient.http import MediaIoBaseUpload
58from googleapiclient.http import MediaUpload
59from googleapiclient.http import _StreamSlice
60from googleapiclient.http import set_user_agent
61from googleapiclient.model import JsonModel
Joe Gregorio654f4a22012-02-09 14:15:44 -050062from oauth2client.client import Credentials
63
64
65class MockCredentials(Credentials):
Bu Sun Kim66bb32c2019-10-30 10:11:58 -070066 """Mock class for all Credentials objects."""
Jon Wayne Parrottd3a5cf42017-06-19 17:55:04 -070067
Bu Sun Kim66bb32c2019-10-30 10:11:58 -070068 def __init__(self, bearer_token, expired=False):
69 super(MockCredentials, self).__init__()
70 self._authorized = 0
71 self._refreshed = 0
72 self._applied = 0
73 self._bearer_token = bearer_token
74 self._access_token_expired = expired
Jon Wayne Parrott20e61352018-01-18 09:16:37 -080075
Bu Sun Kim66bb32c2019-10-30 10:11:58 -070076 @property
77 def access_token(self):
78 return self._bearer_token
Joe Gregorio654f4a22012-02-09 14:15:44 -050079
Bu Sun Kim66bb32c2019-10-30 10:11:58 -070080 @property
81 def access_token_expired(self):
82 return self._access_token_expired
Joe Gregorio654f4a22012-02-09 14:15:44 -050083
Bu Sun Kim66bb32c2019-10-30 10:11:58 -070084 def authorize(self, http):
85 self._authorized += 1
Joe Gregorio654f4a22012-02-09 14:15:44 -050086
Bu Sun Kim66bb32c2019-10-30 10:11:58 -070087 request_orig = http.request
Joe Gregorio654f4a22012-02-09 14:15:44 -050088
Bu Sun Kim66bb32c2019-10-30 10:11:58 -070089 # The closure that will replace 'httplib2.Http.request'.
90 def new_request(
91 uri,
92 method="GET",
93 body=None,
94 headers=None,
95 redirections=httplib2.DEFAULT_MAX_REDIRECTS,
96 connection_type=None,
97 ):
98 # Modify the request headers to add the appropriate
99 # Authorization header.
100 if headers is None:
101 headers = {}
102 self.apply(headers)
Joe Gregorio654f4a22012-02-09 14:15:44 -0500103
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700104 resp, content = request_orig(
105 uri, method, body, headers, redirections, connection_type
106 )
Joe Gregorio654f4a22012-02-09 14:15:44 -0500107
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700108 return resp, content
Joe Gregorio654f4a22012-02-09 14:15:44 -0500109
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700110 # Replace the request method with our own closure.
111 http.request = new_request
Joe Gregorio654f4a22012-02-09 14:15:44 -0500112
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700113 # Set credentials as a property of the request method.
114 setattr(http.request, "credentials", self)
Joe Gregorio654f4a22012-02-09 14:15:44 -0500115
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700116 return http
Joe Gregorio654f4a22012-02-09 14:15:44 -0500117
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700118 def refresh(self, http):
119 self._refreshed += 1
120
121 def apply(self, headers):
122 self._applied += 1
123 headers["authorization"] = self._bearer_token + " " + str(self._refreshed)
Joe Gregorio6bcbcea2011-03-10 15:26:05 -0500124
125
eesheeshc6425a02016-02-12 15:07:06 +0000126class HttpMockWithErrors(object):
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700127 def __init__(self, num_errors, success_json, success_data):
128 self.num_errors = num_errors
129 self.success_json = success_json
130 self.success_data = success_data
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100131
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700132 def request(self, *args, **kwargs):
133 if not self.num_errors:
134 return httplib2.Response(self.success_json), self.success_data
Thomas Bonfort88ab76b2016-04-19 08:48:53 +0200135 else:
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700136 self.num_errors -= 1
137 if self.num_errors == 1: # initial == 2
138 raise ssl.SSLError()
139 if self.num_errors == 3: # initial == 4
140 raise httplib2.ServerNotFoundError()
141 else: # initial != 2,4
142 if self.num_errors == 2:
143 # first try a broken pipe error (#218)
144 ex = socket.error()
145 ex.errno = socket.errno.EPIPE
146 else:
147 # Initialize the timeout error code to the platform's error code.
148 try:
149 # For Windows:
150 ex = socket.error()
151 ex.errno = socket.errno.WSAETIMEDOUT
152 except AttributeError:
153 # For Linux/Mac:
154 if PY3:
155 ex = socket.timeout()
156 else:
157 ex = socket.error()
158 ex.errno = socket.errno.ETIMEDOUT
159 # Now raise the correct error.
160 raise ex
eesheeshc6425a02016-02-12 15:07:06 +0000161
162
163class HttpMockWithNonRetriableErrors(object):
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700164 def __init__(self, num_errors, success_json, success_data):
165 self.num_errors = num_errors
166 self.success_json = success_json
167 self.success_data = success_data
eesheeshc6425a02016-02-12 15:07:06 +0000168
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700169 def request(self, *args, **kwargs):
170 if not self.num_errors:
171 return httplib2.Response(self.success_json), self.success_data
172 else:
173 self.num_errors -= 1
174 ex = socket.error()
175 # set errno to a non-retriable value
176 try:
177 # For Windows:
178 ex.errno = socket.errno.WSAECONNREFUSED
179 except AttributeError:
180 # For Linux/Mac:
181 ex.errno = socket.errno.ECONNREFUSED
182 # Now raise the correct timeout error.
183 raise ex
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100184
185
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700186DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500187
188
189def datafile(filename):
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700190 return os.path.join(DATA_DIR, filename)
191
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500192
Ondrej Medekb86bfc92018-01-19 18:53:50 +0100193def _postproc_none(*kwargs):
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700194 pass
Ondrej Medekb86bfc92018-01-19 18:53:50 +0100195
196
Joe Gregorio6bcbcea2011-03-10 15:26:05 -0500197class TestUserAgent(unittest.TestCase):
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700198 def test_set_user_agent(self):
199 http = HttpMockSequence([({"status": "200"}, "echo_request_headers")])
Joe Gregorio6bcbcea2011-03-10 15:26:05 -0500200
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700201 http = set_user_agent(http, "my_app/5.5")
202 resp, content = http.request("http://example.com")
203 self.assertEqual("my_app/5.5", content["user-agent"])
Joe Gregorio6bcbcea2011-03-10 15:26:05 -0500204
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700205 def test_set_user_agent_nested(self):
206 http = HttpMockSequence([({"status": "200"}, "echo_request_headers")])
Joe Gregorio6bcbcea2011-03-10 15:26:05 -0500207
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700208 http = set_user_agent(http, "my_app/5.5")
209 http = set_user_agent(http, "my_library/0.1")
210 resp, content = http.request("http://example.com")
211 self.assertEqual("my_app/5.5 my_library/0.1", content["user-agent"])
Joe Gregorio6bcbcea2011-03-10 15:26:05 -0500212
Joe Gregorio910b9b12012-06-12 09:36:30 -0400213
214class TestMediaUpload(unittest.TestCase):
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700215 def test_media_file_upload_closes_fd_in___del__(self):
216 file_desc = mock.Mock(spec=io.TextIOWrapper)
217 opener = mock.mock_open(file_desc)
218 if PY3:
219 with mock.patch("builtins.open", return_value=opener):
220 upload = MediaFileUpload(datafile("test_close"), mimetype="text/plain")
221 else:
222 with mock.patch("__builtin__.open", return_value=opener):
223 upload = MediaFileUpload(datafile("test_close"), mimetype="text/plain")
224 self.assertIs(upload.stream(), file_desc)
225 del upload
226 file_desc.close.assert_called_once_with()
Joe Gregorio910b9b12012-06-12 09:36:30 -0400227
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700228 def test_media_file_upload_mimetype_detection(self):
229 upload = MediaFileUpload(datafile("small.png"))
230 self.assertEqual("image/png", upload.mimetype())
Xiaofei Wang20b67582019-07-17 11:16:53 -0700231
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700232 upload = MediaFileUpload(datafile("empty"))
233 self.assertEqual("application/octet-stream", upload.mimetype())
Nam T. Nguyendc136312015-12-01 10:18:56 -0800234
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700235 def test_media_file_upload_to_from_json(self):
236 upload = MediaFileUpload(datafile("small.png"), chunksize=500, resumable=True)
237 self.assertEqual("image/png", upload.mimetype())
238 self.assertEqual(190, upload.size())
239 self.assertEqual(True, upload.resumable())
240 self.assertEqual(500, upload.chunksize())
241 self.assertEqual(b"PNG", upload.getbytes(1, 3))
Nam T. Nguyendc136312015-12-01 10:18:56 -0800242
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700243 json = upload.to_json()
244 new_upload = MediaUpload.new_from_json(json)
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500245
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700246 self.assertEqual("image/png", new_upload.mimetype())
247 self.assertEqual(190, new_upload.size())
248 self.assertEqual(True, new_upload.resumable())
249 self.assertEqual(500, new_upload.chunksize())
250 self.assertEqual(b"PNG", new_upload.getbytes(1, 3))
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500251
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700252 def test_media_file_upload_raises_on_invalid_chunksize(self):
253 self.assertRaises(
254 InvalidChunkSizeError,
255 MediaFileUpload,
256 datafile("small.png"),
257 mimetype="image/png",
258 chunksize=-2,
259 resumable=True,
260 )
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500261
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700262 def test_media_inmemory_upload(self):
263 media = MediaInMemoryUpload(
264 b"abcdef", mimetype="text/plain", chunksize=10, resumable=True
265 )
266 self.assertEqual("text/plain", media.mimetype())
267 self.assertEqual(10, media.chunksize())
268 self.assertTrue(media.resumable())
269 self.assertEqual(b"bc", media.getbytes(1, 2))
270 self.assertEqual(6, media.size())
Joe Gregorioc80ac9d2012-08-21 14:09:09 -0400271
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700272 def test_http_request_to_from_json(self):
273 http = build_http()
274 media_upload = MediaFileUpload(
275 datafile("small.png"), chunksize=500, resumable=True
276 )
277 req = HttpRequest(
278 http,
279 _postproc_none,
280 "http://example.com",
281 method="POST",
282 body="{}",
283 headers={"content-type": 'multipart/related; boundary="---flubber"'},
284 methodId="foo",
285 resumable=media_upload,
286 )
Ali Afshar1cb6b672012-03-12 08:46:14 -0400287
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700288 json = req.to_json()
289 new_req = HttpRequest.from_json(json, http, _postproc_none)
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500290
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700291 self.assertEqual(
292 {"content-type": 'multipart/related; boundary="---flubber"'},
293 new_req.headers,
294 )
295 self.assertEqual("http://example.com", new_req.uri)
296 self.assertEqual("{}", new_req.body)
297 self.assertEqual(http, new_req.http)
298 self.assertEqual(media_upload.to_json(), new_req.resumable.to_json())
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500299
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700300 self.assertEqual(random.random, new_req._rand)
301 self.assertEqual(time.sleep, new_req._sleep)
Joe Gregorio9086bd32013-06-14 16:32:05 -0400302
Joe Gregorio910b9b12012-06-12 09:36:30 -0400303
304class TestMediaIoBaseUpload(unittest.TestCase):
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700305 def test_media_io_base_upload_from_file_io(self):
306 fd = FileIO(datafile("small.png"), "r")
307 upload = MediaIoBaseUpload(
308 fd=fd, mimetype="image/png", chunksize=500, resumable=True
309 )
310 self.assertEqual("image/png", upload.mimetype())
311 self.assertEqual(190, upload.size())
312 self.assertEqual(True, upload.resumable())
313 self.assertEqual(500, upload.chunksize())
314 self.assertEqual(b"PNG", upload.getbytes(1, 3))
Joe Gregorio910b9b12012-06-12 09:36:30 -0400315
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700316 def test_media_io_base_upload_from_file_object(self):
317 f = open(datafile("small.png"), "rb")
318 upload = MediaIoBaseUpload(
319 fd=f, mimetype="image/png", chunksize=500, resumable=True
320 )
321 self.assertEqual("image/png", upload.mimetype())
322 self.assertEqual(190, upload.size())
323 self.assertEqual(True, upload.resumable())
324 self.assertEqual(500, upload.chunksize())
325 self.assertEqual(b"PNG", upload.getbytes(1, 3))
326 f.close()
Joe Gregorio910b9b12012-06-12 09:36:30 -0400327
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700328 def test_media_io_base_upload_serializable(self):
329 f = open(datafile("small.png"), "rb")
330 upload = MediaIoBaseUpload(fd=f, mimetype="image/png")
Joe Gregorio910b9b12012-06-12 09:36:30 -0400331
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700332 try:
333 json = upload.to_json()
334 self.fail("MediaIoBaseUpload should not be serializable.")
335 except NotImplementedError:
336 pass
Joe Gregorio910b9b12012-06-12 09:36:30 -0400337
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700338 @unittest.skipIf(PY3, "Strings and Bytes are different types")
339 def test_media_io_base_upload_from_string_io(self):
340 f = open(datafile("small.png"), "rb")
341 fd = StringIO(f.read())
342 f.close()
Joe Gregorio910b9b12012-06-12 09:36:30 -0400343
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700344 upload = MediaIoBaseUpload(
345 fd=fd, mimetype="image/png", chunksize=500, resumable=True
346 )
347 self.assertEqual("image/png", upload.mimetype())
348 self.assertEqual(190, upload.size())
349 self.assertEqual(True, upload.resumable())
350 self.assertEqual(500, upload.chunksize())
351 self.assertEqual(b"PNG", upload.getbytes(1, 3))
352 f.close()
Joe Gregorio910b9b12012-06-12 09:36:30 -0400353
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700354 def test_media_io_base_upload_from_bytes(self):
355 f = open(datafile("small.png"), "rb")
356 fd = BytesIO(f.read())
357 upload = MediaIoBaseUpload(
358 fd=fd, mimetype="image/png", chunksize=500, resumable=True
359 )
360 self.assertEqual("image/png", upload.mimetype())
361 self.assertEqual(190, upload.size())
362 self.assertEqual(True, upload.resumable())
363 self.assertEqual(500, upload.chunksize())
364 self.assertEqual(b"PNG", upload.getbytes(1, 3))
Joe Gregorio910b9b12012-06-12 09:36:30 -0400365
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700366 def test_media_io_base_upload_raises_on_invalid_chunksize(self):
367 f = open(datafile("small.png"), "rb")
368 fd = BytesIO(f.read())
369 self.assertRaises(
370 InvalidChunkSizeError,
371 MediaIoBaseUpload,
372 fd,
373 "image/png",
374 chunksize=-2,
375 resumable=True,
376 )
Joe Gregorio910b9b12012-06-12 09:36:30 -0400377
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700378 def test_media_io_base_upload_streamable(self):
379 fd = BytesIO(b"stuff")
380 upload = MediaIoBaseUpload(
381 fd=fd, mimetype="image/png", chunksize=500, resumable=True
382 )
383 self.assertEqual(True, upload.has_stream())
384 self.assertEqual(fd, upload.stream())
Joe Gregorioc80ac9d2012-08-21 14:09:09 -0400385
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700386 def test_media_io_base_next_chunk_retries(self):
387 f = open(datafile("small.png"), "rb")
388 fd = BytesIO(f.read())
389 upload = MediaIoBaseUpload(
390 fd=fd, mimetype="image/png", chunksize=500, resumable=True
391 )
Joe Gregorioc80ac9d2012-08-21 14:09:09 -0400392
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700393 # Simulate errors for both the request that creates the resumable upload
394 # and the upload itself.
395 http = HttpMockSequence(
396 [
397 ({"status": "500"}, ""),
398 ({"status": "500"}, ""),
399 ({"status": "503"}, ""),
400 ({"status": "200", "location": "location"}, ""),
401 ({"status": "403"}, USER_RATE_LIMIT_EXCEEDED_RESPONSE),
402 ({"status": "403"}, RATE_LIMIT_EXCEEDED_RESPONSE),
403 ({"status": "429"}, ""),
404 ({"status": "200"}, "{}"),
405 ]
406 )
Joe Gregorio9086bd32013-06-14 16:32:05 -0400407
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700408 model = JsonModel()
409 uri = u"https://www.googleapis.com/someapi/v1/upload/?foo=bar"
410 method = u"POST"
411 request = HttpRequest(
412 http, model.response, uri, method=method, headers={}, resumable=upload
413 )
Joe Gregorio9086bd32013-06-14 16:32:05 -0400414
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700415 sleeptimes = []
416 request._sleep = lambda x: sleeptimes.append(x)
417 request._rand = lambda: 10
Joe Gregorio9086bd32013-06-14 16:32:05 -0400418
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700419 request.execute(num_retries=3)
420 self.assertEqual([20, 40, 80, 20, 40, 80], sleeptimes)
Joe Gregorio9086bd32013-06-14 16:32:05 -0400421
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700422 def test_media_io_base_next_chunk_no_retry_403_not_configured(self):
423 fd = BytesIO(b"i am png")
424 upload = MediaIoBaseUpload(
425 fd=fd, mimetype="image/png", chunksize=500, resumable=True
426 )
Joe Gregorio9086bd32013-06-14 16:32:05 -0400427
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700428 http = HttpMockSequence(
429 [({"status": "403"}, NOT_CONFIGURED_RESPONSE), ({"status": "200"}, "{}")]
430 )
Zhihao Yuancc6d3982016-07-27 11:40:45 -0500431
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700432 model = JsonModel()
433 uri = u"https://www.googleapis.com/someapi/v1/upload/?foo=bar"
434 method = u"POST"
435 request = HttpRequest(
436 http, model.response, uri, method=method, headers={}, resumable=upload
437 )
Zhihao Yuancc6d3982016-07-27 11:40:45 -0500438
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700439 request._rand = lambda: 1.0
440 request._sleep = mock.MagicMock()
Zhihao Yuancc6d3982016-07-27 11:40:45 -0500441
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700442 with self.assertRaises(HttpError):
443 request.execute(num_retries=3)
444 request._sleep.assert_not_called()
Zhihao Yuancc6d3982016-07-27 11:40:45 -0500445
Joe Gregorio910b9b12012-06-12 09:36:30 -0400446
Joe Gregorio708388c2012-06-15 13:43:04 -0400447class TestMediaIoBaseDownload(unittest.TestCase):
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700448 def setUp(self):
449 http = HttpMock(datafile("zoo.json"), {"status": "200"})
450 zoo = build("zoo", "v1", http=http)
451 self.request = zoo.animals().get_media(name="Lion")
452 self.fd = BytesIO()
Joe Gregorio708388c2012-06-15 13:43:04 -0400453
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700454 def test_media_io_base_download(self):
455 self.request.http = HttpMockSequence(
456 [
457 ({"status": "200", "content-range": "0-2/5"}, b"123"),
458 ({"status": "200", "content-range": "3-4/5"}, b"45"),
459 ]
460 )
461 self.assertEqual(True, self.request.http.follow_redirects)
Joe Gregorio708388c2012-06-15 13:43:04 -0400462
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700463 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3)
Joe Gregorio708388c2012-06-15 13:43:04 -0400464
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700465 self.assertEqual(self.fd, download._fd)
466 self.assertEqual(3, download._chunksize)
467 self.assertEqual(0, download._progress)
468 self.assertEqual(None, download._total_size)
469 self.assertEqual(False, download._done)
470 self.assertEqual(self.request.uri, download._uri)
Joe Gregorio708388c2012-06-15 13:43:04 -0400471
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700472 status, done = download.next_chunk()
Joe Gregorio708388c2012-06-15 13:43:04 -0400473
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700474 self.assertEqual(self.fd.getvalue(), b"123")
475 self.assertEqual(False, done)
476 self.assertEqual(3, download._progress)
477 self.assertEqual(5, download._total_size)
478 self.assertEqual(3, status.resumable_progress)
Joe Gregorio708388c2012-06-15 13:43:04 -0400479
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700480 status, done = download.next_chunk()
Joe Gregorio708388c2012-06-15 13:43:04 -0400481
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700482 self.assertEqual(self.fd.getvalue(), b"12345")
483 self.assertEqual(True, done)
484 self.assertEqual(5, download._progress)
485 self.assertEqual(5, download._total_size)
Joe Gregorio708388c2012-06-15 13:43:04 -0400486
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700487 def test_media_io_base_download_custom_request_headers(self):
488 self.request.http = HttpMockSequence(
489 [
490 (
491 {"status": "200", "content-range": "0-2/5"},
492 "echo_request_headers_as_json",
493 ),
494 (
495 {"status": "200", "content-range": "3-4/5"},
496 "echo_request_headers_as_json",
497 ),
498 ]
499 )
500 self.assertEqual(True, self.request.http.follow_redirects)
Joe Gregorio708388c2012-06-15 13:43:04 -0400501
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700502 self.request.headers["Cache-Control"] = "no-store"
Chris McDonough0dc81bf2018-07-19 11:19:58 -0400503
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700504 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3)
Chris McDonough0dc81bf2018-07-19 11:19:58 -0400505
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700506 self.assertEqual(download._headers.get("Cache-Control"), "no-store")
Chris McDonough0dc81bf2018-07-19 11:19:58 -0400507
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700508 status, done = download.next_chunk()
Chris McDonough0dc81bf2018-07-19 11:19:58 -0400509
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700510 result = json.loads(self.fd.getvalue().decode("utf-8"))
Chris McDonough0dc81bf2018-07-19 11:19:58 -0400511
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700512 # assert that that the header we added to the original request is
513 # sent up to the server on each call to next_chunk
Chris McDonough0dc81bf2018-07-19 11:19:58 -0400514
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700515 self.assertEqual(result.get("Cache-Control"), "no-store")
Chris McDonough0dc81bf2018-07-19 11:19:58 -0400516
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700517 download._fd = self.fd = BytesIO()
518 status, done = download.next_chunk()
Chris McDonough0dc81bf2018-07-19 11:19:58 -0400519
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700520 result = json.loads(self.fd.getvalue().decode("utf-8"))
521 self.assertEqual(result.get("Cache-Control"), "no-store")
Chris McDonough0dc81bf2018-07-19 11:19:58 -0400522
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700523 def test_media_io_base_download_handle_redirects(self):
524 self.request.http = HttpMockSequence(
525 [
526 (
527 {
528 "status": "200",
529 "content-location": "https://secure.example.net/lion",
530 },
531 b"",
532 ),
533 ({"status": "200", "content-range": "0-2/5"}, b"abc"),
534 ]
535 )
Chris McDonough0dc81bf2018-07-19 11:19:58 -0400536
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700537 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3)
Joe Gregorio708388c2012-06-15 13:43:04 -0400538
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700539 status, done = download.next_chunk()
Joe Gregorio708388c2012-06-15 13:43:04 -0400540
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700541 self.assertEqual("https://secure.example.net/lion", download._uri)
Joe Gregorio708388c2012-06-15 13:43:04 -0400542
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700543 def test_media_io_base_download_handle_4xx(self):
544 self.request.http = HttpMockSequence([({"status": "400"}, "")])
Joe Gregorio708388c2012-06-15 13:43:04 -0400545
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700546 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3)
Joe Gregorio708388c2012-06-15 13:43:04 -0400547
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700548 try:
549 status, done = download.next_chunk()
550 self.fail("Should raise an exception")
551 except HttpError:
552 pass
Joe Gregorio708388c2012-06-15 13:43:04 -0400553
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700554 # Even after raising an exception we can pick up where we left off.
555 self.request.http = HttpMockSequence(
556 [({"status": "200", "content-range": "0-2/5"}, b"123")]
557 )
Joe Gregorio708388c2012-06-15 13:43:04 -0400558
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700559 status, done = download.next_chunk()
Joe Gregorio708388c2012-06-15 13:43:04 -0400560
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700561 self.assertEqual(self.fd.getvalue(), b"123")
Joe Gregorio708388c2012-06-15 13:43:04 -0400562
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700563 def test_media_io_base_download_retries_connection_errors(self):
564 self.request.http = HttpMockWithErrors(
565 4, {"status": "200", "content-range": "0-2/3"}, b"123"
566 )
Joe Gregorio708388c2012-06-15 13:43:04 -0400567
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700568 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3)
569 download._sleep = lambda _x: 0 # do nothing
570 download._rand = lambda: 10
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100571
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700572 status, done = download.next_chunk(num_retries=4)
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100573
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700574 self.assertEqual(self.fd.getvalue(), b"123")
575 self.assertEqual(True, done)
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100576
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700577 def test_media_io_base_download_retries_5xx(self):
578 self.request.http = HttpMockSequence(
579 [
580 ({"status": "500"}, ""),
581 ({"status": "500"}, ""),
582 ({"status": "500"}, ""),
583 ({"status": "200", "content-range": "0-2/5"}, b"123"),
584 ({"status": "503"}, ""),
585 ({"status": "503"}, ""),
586 ({"status": "503"}, ""),
587 ({"status": "200", "content-range": "3-4/5"}, b"45"),
588 ]
589 )
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100590
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700591 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3)
Joe Gregorio9086bd32013-06-14 16:32:05 -0400592
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700593 self.assertEqual(self.fd, download._fd)
594 self.assertEqual(3, download._chunksize)
595 self.assertEqual(0, download._progress)
596 self.assertEqual(None, download._total_size)
597 self.assertEqual(False, download._done)
598 self.assertEqual(self.request.uri, download._uri)
Joe Gregorio9086bd32013-06-14 16:32:05 -0400599
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700600 # Set time.sleep and random.random stubs.
601 sleeptimes = []
602 download._sleep = lambda x: sleeptimes.append(x)
603 download._rand = lambda: 10
Joe Gregorio9086bd32013-06-14 16:32:05 -0400604
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700605 status, done = download.next_chunk(num_retries=3)
Joe Gregorio9086bd32013-06-14 16:32:05 -0400606
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700607 # Check for exponential backoff using the rand function above.
608 self.assertEqual([20, 40, 80], sleeptimes)
Joe Gregorio9086bd32013-06-14 16:32:05 -0400609
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700610 self.assertEqual(self.fd.getvalue(), b"123")
611 self.assertEqual(False, done)
612 self.assertEqual(3, download._progress)
613 self.assertEqual(5, download._total_size)
614 self.assertEqual(3, status.resumable_progress)
Joe Gregorio9086bd32013-06-14 16:32:05 -0400615
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700616 # Reset time.sleep stub.
617 del sleeptimes[0 : len(sleeptimes)]
Joe Gregorio9086bd32013-06-14 16:32:05 -0400618
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700619 status, done = download.next_chunk(num_retries=3)
Joe Gregorio9086bd32013-06-14 16:32:05 -0400620
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700621 # Check for exponential backoff using the rand function above.
622 self.assertEqual([20, 40, 80], sleeptimes)
Joe Gregorio9086bd32013-06-14 16:32:05 -0400623
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700624 self.assertEqual(self.fd.getvalue(), b"12345")
625 self.assertEqual(True, done)
626 self.assertEqual(5, download._progress)
627 self.assertEqual(5, download._total_size)
Joe Gregorio9086bd32013-06-14 16:32:05 -0400628
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700629 def test_media_io_base_download_empty_file(self):
630 self.request.http = HttpMockSequence(
631 [({"status": "200", "content-range": "0-0/0"}, b"")]
632 )
Joe Gregorio9086bd32013-06-14 16:32:05 -0400633
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700634 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3)
andrewnestera4a44cf2017-03-31 16:09:31 +0300635
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700636 self.assertEqual(self.fd, download._fd)
637 self.assertEqual(0, download._progress)
638 self.assertEqual(None, download._total_size)
639 self.assertEqual(False, download._done)
640 self.assertEqual(self.request.uri, download._uri)
andrewnestera4a44cf2017-03-31 16:09:31 +0300641
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700642 status, done = download.next_chunk()
andrewnestera4a44cf2017-03-31 16:09:31 +0300643
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700644 self.assertEqual(True, done)
645 self.assertEqual(0, download._progress)
646 self.assertEqual(0, download._total_size)
647 self.assertEqual(0, status.progress())
andrewnestera4a44cf2017-03-31 16:09:31 +0300648
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700649 def test_media_io_base_download_unknown_media_size(self):
650 self.request.http = HttpMockSequence([({"status": "200"}, b"123")])
andrewnestera4a44cf2017-03-31 16:09:31 +0300651
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700652 download = MediaIoBaseDownload(fd=self.fd, request=self.request, chunksize=3)
Daniel44067782018-01-16 23:17:56 +0100653
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700654 self.assertEqual(self.fd, download._fd)
655 self.assertEqual(0, download._progress)
656 self.assertEqual(None, download._total_size)
657 self.assertEqual(False, download._done)
658 self.assertEqual(self.request.uri, download._uri)
Daniel44067782018-01-16 23:17:56 +0100659
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700660 status, done = download.next_chunk()
Daniel44067782018-01-16 23:17:56 +0100661
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700662 self.assertEqual(self.fd.getvalue(), b"123")
663 self.assertEqual(True, done)
664 self.assertEqual(3, download._progress)
665 self.assertEqual(None, download._total_size)
666 self.assertEqual(0, status.progress())
Daniel44067782018-01-16 23:17:56 +0100667
668
Joe Gregorio66f57522011-11-30 11:00:00 -0500669EXPECTED = """POST /someapi/v1/collection/?foo=bar HTTP/1.1
670Content-Type: application/json
671MIME-Version: 1.0
Joe Gregorio5d1171b2012-01-05 10:48:24 -0500672Host: www.googleapis.com
673content-length: 2\r\n\r\n{}"""
674
675
676NO_BODY_EXPECTED = """POST /someapi/v1/collection/?foo=bar HTTP/1.1
677Content-Type: application/json
678MIME-Version: 1.0
679Host: www.googleapis.com
680content-length: 0\r\n\r\n"""
Joe Gregorio66f57522011-11-30 11:00:00 -0500681
Pepper Lebeck-Jobe30ecbd42015-06-12 11:45:57 -0400682NO_BODY_EXPECTED_GET = """GET /someapi/v1/collection/?foo=bar HTTP/1.1
683Content-Type: application/json
684MIME-Version: 1.0
685Host: www.googleapis.com\r\n\r\n"""
686
Joe Gregorio66f57522011-11-30 11:00:00 -0500687
688RESPONSE = """HTTP/1.1 200 OK
Joe Gregorio20b54fb2012-07-26 09:59:35 -0400689Content-Type: application/json
Joe Gregorio66f57522011-11-30 11:00:00 -0500690Content-Length: 14
691ETag: "etag/pony"\r\n\r\n{"answer": 42}"""
692
693
INADA Naoki09157612015-03-25 01:51:03 +0900694BATCH_RESPONSE = b"""--batch_foobarbaz
Joe Gregorio66f57522011-11-30 11:00:00 -0500695Content-Type: application/http
696Content-Transfer-Encoding: binary
Chris McDonough3cf5e602018-07-18 16:18:38 -0400697Content-ID: <randomness + 1>
Joe Gregorio66f57522011-11-30 11:00:00 -0500698
699HTTP/1.1 200 OK
Joe Gregorio20b54fb2012-07-26 09:59:35 -0400700Content-Type: application/json
Joe Gregorio66f57522011-11-30 11:00:00 -0500701Content-Length: 14
702ETag: "etag/pony"\r\n\r\n{"foo": 42}
703
704--batch_foobarbaz
705Content-Type: application/http
706Content-Transfer-Encoding: binary
Chris McDonough3cf5e602018-07-18 16:18:38 -0400707Content-ID: <randomness + 2>
Joe Gregorio66f57522011-11-30 11:00:00 -0500708
709HTTP/1.1 200 OK
Joe Gregorio20b54fb2012-07-26 09:59:35 -0400710Content-Type: application/json
Joe Gregorio66f57522011-11-30 11:00:00 -0500711Content-Length: 14
712ETag: "etag/sheep"\r\n\r\n{"baz": "qux"}
713--batch_foobarbaz--"""
714
Joe Gregorio5d1171b2012-01-05 10:48:24 -0500715
INADA Naoki09157612015-03-25 01:51:03 +0900716BATCH_ERROR_RESPONSE = b"""--batch_foobarbaz
Joe Gregorio3fb93672012-07-25 11:31:11 -0400717Content-Type: application/http
718Content-Transfer-Encoding: binary
Chris McDonough3cf5e602018-07-18 16:18:38 -0400719Content-ID: <randomness + 1>
Joe Gregorio3fb93672012-07-25 11:31:11 -0400720
721HTTP/1.1 200 OK
Joe Gregorio20b54fb2012-07-26 09:59:35 -0400722Content-Type: application/json
Joe Gregorio3fb93672012-07-25 11:31:11 -0400723Content-Length: 14
724ETag: "etag/pony"\r\n\r\n{"foo": 42}
725
726--batch_foobarbaz
727Content-Type: application/http
728Content-Transfer-Encoding: binary
Chris McDonough3cf5e602018-07-18 16:18:38 -0400729Content-ID: <randomness + 2>
Joe Gregorio3fb93672012-07-25 11:31:11 -0400730
731HTTP/1.1 403 Access Not Configured
Joe Gregorio20b54fb2012-07-26 09:59:35 -0400732Content-Type: application/json
733Content-Length: 245
734ETag: "etag/sheep"\r\n\r\n{
Joe Gregorio3fb93672012-07-25 11:31:11 -0400735 "error": {
736 "errors": [
737 {
738 "domain": "usageLimits",
739 "reason": "accessNotConfigured",
740 "message": "Access Not Configured",
741 "debugInfo": "QuotaState: BLOCKED"
742 }
743 ],
744 "code": 403,
745 "message": "Access Not Configured"
746 }
747}
748
749--batch_foobarbaz--"""
750
751
INADA Naoki09157612015-03-25 01:51:03 +0900752BATCH_RESPONSE_WITH_401 = b"""--batch_foobarbaz
Joe Gregorio654f4a22012-02-09 14:15:44 -0500753Content-Type: application/http
754Content-Transfer-Encoding: binary
Chris McDonough3cf5e602018-07-18 16:18:38 -0400755Content-ID: <randomness + 1>
Joe Gregorio654f4a22012-02-09 14:15:44 -0500756
Joe Gregorioc752e332012-07-11 14:43:52 -0400757HTTP/1.1 401 Authorization Required
Joe Gregorio20b54fb2012-07-26 09:59:35 -0400758Content-Type: application/json
Joe Gregorio654f4a22012-02-09 14:15:44 -0500759Content-Length: 14
760ETag: "etag/pony"\r\n\r\n{"error": {"message":
761 "Authorizaton failed."}}
762
763--batch_foobarbaz
764Content-Type: application/http
765Content-Transfer-Encoding: binary
Chris McDonough3cf5e602018-07-18 16:18:38 -0400766Content-ID: <randomness + 2>
Joe Gregorio654f4a22012-02-09 14:15:44 -0500767
768HTTP/1.1 200 OK
Joe Gregorio20b54fb2012-07-26 09:59:35 -0400769Content-Type: application/json
Joe Gregorio654f4a22012-02-09 14:15:44 -0500770Content-Length: 14
771ETag: "etag/sheep"\r\n\r\n{"baz": "qux"}
772--batch_foobarbaz--"""
773
774
INADA Naoki09157612015-03-25 01:51:03 +0900775BATCH_SINGLE_RESPONSE = b"""--batch_foobarbaz
Joe Gregorio654f4a22012-02-09 14:15:44 -0500776Content-Type: application/http
777Content-Transfer-Encoding: binary
Chris McDonough3cf5e602018-07-18 16:18:38 -0400778Content-ID: <randomness + 1>
Joe Gregorio654f4a22012-02-09 14:15:44 -0500779
780HTTP/1.1 200 OK
Joe Gregorio20b54fb2012-07-26 09:59:35 -0400781Content-Type: application/json
Joe Gregorio654f4a22012-02-09 14:15:44 -0500782Content-Length: 14
783ETag: "etag/pony"\r\n\r\n{"foo": 42}
784--batch_foobarbaz--"""
785
eesheeshc6425a02016-02-12 15:07:06 +0000786
787USER_RATE_LIMIT_EXCEEDED_RESPONSE = """{
788 "error": {
789 "errors": [
790 {
791 "domain": "usageLimits",
792 "reason": "userRateLimitExceeded",
793 "message": "User Rate Limit Exceeded"
794 }
795 ],
796 "code": 403,
797 "message": "User Rate Limit Exceeded"
798 }
799}"""
800
801
802RATE_LIMIT_EXCEEDED_RESPONSE = """{
803 "error": {
804 "errors": [
805 {
806 "domain": "usageLimits",
807 "reason": "rateLimitExceeded",
808 "message": "Rate Limit Exceeded"
809 }
810 ],
811 "code": 403,
812 "message": "Rate Limit Exceeded"
813 }
814}"""
815
816
817NOT_CONFIGURED_RESPONSE = """{
818 "error": {
819 "errors": [
820 {
821 "domain": "usageLimits",
822 "reason": "accessNotConfigured",
823 "message": "Access Not Configured"
824 }
825 ],
826 "code": 403,
827 "message": "Access Not Configured"
828 }
829}"""
830
Nilayan Bhattacharya90ffb852017-12-05 15:30:32 -0800831LIST_NOT_CONFIGURED_RESPONSE = """[
832 "error": {
833 "errors": [
834 {
835 "domain": "usageLimits",
836 "reason": "accessNotConfigured",
837 "message": "Access Not Configured"
838 }
839 ],
840 "code": 403,
841 "message": "Access Not Configured"
842 }
843]"""
844
Joe Gregorio654f4a22012-02-09 14:15:44 -0500845
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700846class Callbacks(object):
847 def __init__(self):
848 self.responses = {}
849 self.exceptions = {}
850
851 def f(self, request_id, response, exception):
852 self.responses[request_id] = response
853 self.exceptions[request_id] = exception
Joe Gregorio654f4a22012-02-09 14:15:44 -0500854
855
Joe Gregorio83f2ee62012-12-06 15:25:54 -0500856class TestHttpRequest(unittest.TestCase):
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700857 def test_unicode(self):
858 http = HttpMock(datafile("zoo.json"), headers={"status": "200"})
859 model = JsonModel()
860 uri = u"https://www.googleapis.com/someapi/v1/collection/?foo=bar"
861 method = u"POST"
862 request = HttpRequest(
863 http,
864 model.response,
865 uri,
866 method=method,
867 body=u"{}",
868 headers={"content-type": "application/json"},
869 )
870 request.execute()
871 self.assertEqual(uri, http.uri)
872 self.assertEqual(str, type(http.uri))
873 self.assertEqual(method, http.method)
874 self.assertEqual(str, type(http.method))
Joe Gregorio83f2ee62012-12-06 15:25:54 -0500875
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700876 def test_empty_content_type(self):
877 """Test for #284"""
878 http = HttpMock(None, headers={"status": 200})
879 uri = u"https://www.googleapis.com/someapi/v1/upload/?foo=bar"
880 method = u"POST"
881 request = HttpRequest(
882 http, _postproc_none, uri, method=method, headers={"content-type": ""}
883 )
884 request.execute()
885 self.assertEqual("", http.headers.get("content-type"))
Xiaofei Wang20b67582019-07-17 11:16:53 -0700886
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700887 def test_no_retry_connection_errors(self):
888 model = JsonModel()
889 request = HttpRequest(
890 HttpMockWithNonRetriableErrors(1, {"status": "200"}, '{"foo": "bar"}'),
891 model.response,
892 u"https://www.example.com/json_api_endpoint",
893 )
894 request._sleep = lambda _x: 0 # do nothing
895 request._rand = lambda: 10
896 with self.assertRaises(socket.error):
897 response = request.execute(num_retries=3)
eesheeshc6425a02016-02-12 15:07:06 +0000898
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700899 def test_retry_connection_errors_non_resumable(self):
900 model = JsonModel()
901 request = HttpRequest(
902 HttpMockWithErrors(4, {"status": "200"}, '{"foo": "bar"}'),
903 model.response,
904 u"https://www.example.com/json_api_endpoint",
905 )
906 request._sleep = lambda _x: 0 # do nothing
907 request._rand = lambda: 10
908 response = request.execute(num_retries=4)
909 self.assertEqual({u"foo": u"bar"}, response)
eesheeshc6425a02016-02-12 15:07:06 +0000910
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700911 def test_retry_connection_errors_resumable(self):
912 with open(datafile("small.png"), "rb") as small_png_file:
913 small_png_fd = BytesIO(small_png_file.read())
914 upload = MediaIoBaseUpload(
915 fd=small_png_fd, mimetype="image/png", chunksize=500, resumable=True
916 )
917 model = JsonModel()
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100918
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700919 request = HttpRequest(
920 HttpMockWithErrors(
921 4, {"status": "200", "location": "location"}, '{"foo": "bar"}'
922 ),
923 model.response,
924 u"https://www.example.com/file_upload",
925 method="POST",
926 resumable=upload,
927 )
928 request._sleep = lambda _x: 0 # do nothing
929 request._rand = lambda: 10
930 response = request.execute(num_retries=4)
931 self.assertEqual({u"foo": u"bar"}, response)
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100932
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700933 def test_retry(self):
934 num_retries = 5
935 resp_seq = [({"status": "500"}, "")] * (num_retries - 3)
936 resp_seq.append(({"status": "403"}, RATE_LIMIT_EXCEEDED_RESPONSE))
937 resp_seq.append(({"status": "403"}, USER_RATE_LIMIT_EXCEEDED_RESPONSE))
938 resp_seq.append(({"status": "429"}, ""))
939 resp_seq.append(({"status": "200"}, "{}"))
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100940
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700941 http = HttpMockSequence(resp_seq)
942 model = JsonModel()
943 uri = u"https://www.googleapis.com/someapi/v1/collection/?foo=bar"
944 method = u"POST"
945 request = HttpRequest(
946 http,
947 model.response,
948 uri,
949 method=method,
950 body=u"{}",
951 headers={"content-type": "application/json"},
952 )
Joe Gregorio9086bd32013-06-14 16:32:05 -0400953
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700954 sleeptimes = []
955 request._sleep = lambda x: sleeptimes.append(x)
956 request._rand = lambda: 10
Joe Gregorio9086bd32013-06-14 16:32:05 -0400957
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700958 request.execute(num_retries=num_retries)
Joe Gregorio9086bd32013-06-14 16:32:05 -0400959
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700960 self.assertEqual(num_retries, len(sleeptimes))
961 for retry_num in range(num_retries):
962 self.assertEqual(10 * 2 ** (retry_num + 1), sleeptimes[retry_num])
Joe Gregorio9086bd32013-06-14 16:32:05 -0400963
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700964 def test_no_retry_succeeds(self):
965 num_retries = 5
966 resp_seq = [({"status": "200"}, "{}")] * (num_retries)
Joe Gregorio9086bd32013-06-14 16:32:05 -0400967
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700968 http = HttpMockSequence(resp_seq)
969 model = JsonModel()
970 uri = u"https://www.googleapis.com/someapi/v1/collection/?foo=bar"
971 method = u"POST"
972 request = HttpRequest(
973 http,
974 model.response,
975 uri,
976 method=method,
977 body=u"{}",
978 headers={"content-type": "application/json"},
979 )
eesheeshc6425a02016-02-12 15:07:06 +0000980
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700981 sleeptimes = []
982 request._sleep = lambda x: sleeptimes.append(x)
983 request._rand = lambda: 10
eesheeshc6425a02016-02-12 15:07:06 +0000984
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700985 request.execute(num_retries=num_retries)
eesheeshc6425a02016-02-12 15:07:06 +0000986
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700987 self.assertEqual(0, len(sleeptimes))
eesheeshc6425a02016-02-12 15:07:06 +0000988
Bu Sun Kim66bb32c2019-10-30 10:11:58 -0700989 def test_no_retry_fails_fast(self):
990 http = HttpMockSequence([({"status": "500"}, ""), ({"status": "200"}, "{}")])
991 model = JsonModel()
992 uri = u"https://www.googleapis.com/someapi/v1/collection/?foo=bar"
993 method = u"POST"
994 request = HttpRequest(
995 http,
996 model.response,
997 uri,
998 method=method,
999 body=u"{}",
1000 headers={"content-type": "application/json"},
1001 )
eesheeshc6425a02016-02-12 15:07:06 +00001002
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001003 request._rand = lambda: 1.0
1004 request._sleep = mock.MagicMock()
Joe Gregorio9086bd32013-06-14 16:32:05 -04001005
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001006 with self.assertRaises(HttpError):
1007 request.execute()
1008 request._sleep.assert_not_called()
Joe Gregorio9086bd32013-06-14 16:32:05 -04001009
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001010 def test_no_retry_403_not_configured_fails_fast(self):
1011 http = HttpMockSequence(
1012 [({"status": "403"}, NOT_CONFIGURED_RESPONSE), ({"status": "200"}, "{}")]
1013 )
1014 model = JsonModel()
1015 uri = u"https://www.googleapis.com/someapi/v1/collection/?foo=bar"
1016 method = u"POST"
1017 request = HttpRequest(
1018 http,
1019 model.response,
1020 uri,
1021 method=method,
1022 body=u"{}",
1023 headers={"content-type": "application/json"},
1024 )
Joe Gregorio9086bd32013-06-14 16:32:05 -04001025
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001026 request._rand = lambda: 1.0
1027 request._sleep = mock.MagicMock()
eesheeshc6425a02016-02-12 15:07:06 +00001028
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001029 with self.assertRaises(HttpError):
1030 request.execute()
1031 request._sleep.assert_not_called()
eesheeshc6425a02016-02-12 15:07:06 +00001032
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001033 def test_no_retry_403_fails_fast(self):
1034 http = HttpMockSequence([({"status": "403"}, ""), ({"status": "200"}, "{}")])
1035 model = JsonModel()
1036 uri = u"https://www.googleapis.com/someapi/v1/collection/?foo=bar"
1037 method = u"POST"
1038 request = HttpRequest(
1039 http,
1040 model.response,
1041 uri,
1042 method=method,
1043 body=u"{}",
1044 headers={"content-type": "application/json"},
1045 )
eesheeshc6425a02016-02-12 15:07:06 +00001046
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001047 request._rand = lambda: 1.0
1048 request._sleep = mock.MagicMock()
eesheeshc6425a02016-02-12 15:07:06 +00001049
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001050 with self.assertRaises(HttpError):
1051 request.execute()
1052 request._sleep.assert_not_called()
eesheeshc6425a02016-02-12 15:07:06 +00001053
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001054 def test_no_retry_401_fails_fast(self):
1055 http = HttpMockSequence([({"status": "401"}, ""), ({"status": "200"}, "{}")])
1056 model = JsonModel()
1057 uri = u"https://www.googleapis.com/someapi/v1/collection/?foo=bar"
1058 method = u"POST"
1059 request = HttpRequest(
1060 http,
1061 model.response,
1062 uri,
1063 method=method,
1064 body=u"{}",
1065 headers={"content-type": "application/json"},
1066 )
eesheeshc6425a02016-02-12 15:07:06 +00001067
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001068 request._rand = lambda: 1.0
1069 request._sleep = mock.MagicMock()
eesheeshc6425a02016-02-12 15:07:06 +00001070
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001071 with self.assertRaises(HttpError):
1072 request.execute()
1073 request._sleep.assert_not_called()
eesheeshc6425a02016-02-12 15:07:06 +00001074
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001075 def test_no_retry_403_list_fails(self):
1076 http = HttpMockSequence(
1077 [
1078 ({"status": "403"}, LIST_NOT_CONFIGURED_RESPONSE),
1079 ({"status": "200"}, "{}"),
1080 ]
1081 )
1082 model = JsonModel()
1083 uri = u"https://www.googleapis.com/someapi/v1/collection/?foo=bar"
1084 method = u"POST"
1085 request = HttpRequest(
1086 http,
1087 model.response,
1088 uri,
1089 method=method,
1090 body=u"{}",
1091 headers={"content-type": "application/json"},
1092 )
Joe Gregorio83f2ee62012-12-06 15:25:54 -05001093
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001094 request._rand = lambda: 1.0
1095 request._sleep = mock.MagicMock()
Nilayan Bhattacharya90ffb852017-12-05 15:30:32 -08001096
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001097 with self.assertRaises(HttpError):
1098 request.execute()
1099 request._sleep.assert_not_called()
Nilayan Bhattacharya90ffb852017-12-05 15:30:32 -08001100
Nilayan Bhattacharya90ffb852017-12-05 15:30:32 -08001101
Joe Gregorio66f57522011-11-30 11:00:00 -05001102class TestBatch(unittest.TestCase):
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001103 def setUp(self):
1104 model = JsonModel()
1105 self.request1 = HttpRequest(
1106 None,
1107 model.response,
1108 "https://www.googleapis.com/someapi/v1/collection/?foo=bar",
1109 method="POST",
1110 body="{}",
1111 headers={"content-type": "application/json"},
1112 )
Joe Gregorio66f57522011-11-30 11:00:00 -05001113
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001114 self.request2 = HttpRequest(
1115 None,
1116 model.response,
1117 "https://www.googleapis.com/someapi/v1/collection/?foo=bar",
1118 method="GET",
1119 body="",
1120 headers={"content-type": "application/json"},
1121 )
Joe Gregorio66f57522011-11-30 11:00:00 -05001122
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001123 def test_id_to_from_content_id_header(self):
1124 batch = BatchHttpRequest()
Bu Sun Kim1cf3cbc2020-03-12 12:38:23 -07001125 self.assertEqual("12", batch._header_to_id(batch._id_to_header("12")))
Joe Gregorio66f57522011-11-30 11:00:00 -05001126
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001127 def test_invalid_content_id_header(self):
1128 batch = BatchHttpRequest()
1129 self.assertRaises(BatchError, batch._header_to_id, "[foo+x]")
1130 self.assertRaises(BatchError, batch._header_to_id, "foo+1")
1131 self.assertRaises(BatchError, batch._header_to_id, "<foo>")
Joe Gregorio66f57522011-11-30 11:00:00 -05001132
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001133 def test_serialize_request(self):
1134 batch = BatchHttpRequest()
1135 request = HttpRequest(
1136 None,
1137 None,
1138 "https://www.googleapis.com/someapi/v1/collection/?foo=bar",
1139 method="POST",
1140 body=u"{}",
1141 headers={"content-type": "application/json"},
1142 methodId=None,
1143 resumable=None,
1144 )
1145 s = batch._serialize_request(request).splitlines()
1146 self.assertEqual(EXPECTED.splitlines(), s)
Joe Gregorio66f57522011-11-30 11:00:00 -05001147
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001148 def test_serialize_request_media_body(self):
1149 batch = BatchHttpRequest()
1150 f = open(datafile("small.png"), "rb")
1151 body = f.read()
1152 f.close()
Joe Gregorio66f57522011-11-30 11:00:00 -05001153
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001154 request = HttpRequest(
1155 None,
1156 None,
1157 "https://www.googleapis.com/someapi/v1/collection/?foo=bar",
1158 method="POST",
1159 body=body,
1160 headers={"content-type": "application/json"},
1161 methodId=None,
1162 resumable=None,
1163 )
1164 # Just testing it shouldn't raise an exception.
1165 s = batch._serialize_request(request).splitlines()
Joe Gregorio66f57522011-11-30 11:00:00 -05001166
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001167 def test_serialize_request_no_body(self):
1168 batch = BatchHttpRequest()
1169 request = HttpRequest(
1170 None,
1171 None,
1172 "https://www.googleapis.com/someapi/v1/collection/?foo=bar",
1173 method="POST",
1174 body=b"",
1175 headers={"content-type": "application/json"},
1176 methodId=None,
1177 resumable=None,
1178 )
1179 s = batch._serialize_request(request).splitlines()
1180 self.assertEqual(NO_BODY_EXPECTED.splitlines(), s)
Joe Gregoriodd813822012-01-25 10:32:47 -05001181
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001182 def test_serialize_get_request_no_body(self):
1183 batch = BatchHttpRequest()
1184 request = HttpRequest(
1185 None,
1186 None,
1187 "https://www.googleapis.com/someapi/v1/collection/?foo=bar",
1188 method="GET",
1189 body=None,
1190 headers={"content-type": "application/json"},
1191 methodId=None,
1192 resumable=None,
1193 )
1194 s = batch._serialize_request(request).splitlines()
1195 self.assertEqual(NO_BODY_EXPECTED_GET.splitlines(), s)
Joe Gregoriodd813822012-01-25 10:32:47 -05001196
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001197 def test_deserialize_response(self):
1198 batch = BatchHttpRequest()
1199 resp, content = batch._deserialize_response(RESPONSE)
Joe Gregorio5d1171b2012-01-05 10:48:24 -05001200
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001201 self.assertEqual(200, resp.status)
1202 self.assertEqual("OK", resp.reason)
1203 self.assertEqual(11, resp.version)
1204 self.assertEqual('{"answer": 42}', content)
Pepper Lebeck-Jobe30ecbd42015-06-12 11:45:57 -04001205
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001206 def test_new_id(self):
1207 batch = BatchHttpRequest()
Joe Gregorio66f57522011-11-30 11:00:00 -05001208
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001209 id_ = batch._new_id()
1210 self.assertEqual("1", id_)
Joe Gregorio66f57522011-11-30 11:00:00 -05001211
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001212 id_ = batch._new_id()
1213 self.assertEqual("2", id_)
Joe Gregorio66f57522011-11-30 11:00:00 -05001214
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001215 batch.add(self.request1, request_id="3")
Joe Gregorio66f57522011-11-30 11:00:00 -05001216
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001217 id_ = batch._new_id()
1218 self.assertEqual("4", id_)
Joe Gregorio66f57522011-11-30 11:00:00 -05001219
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001220 def test_add(self):
1221 batch = BatchHttpRequest()
1222 batch.add(self.request1, request_id="1")
1223 self.assertRaises(KeyError, batch.add, self.request1, request_id="1")
Joe Gregorio66f57522011-11-30 11:00:00 -05001224
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001225 def test_add_fail_for_over_limit(self):
1226 from googleapiclient.http import MAX_BATCH_LIMIT
Joe Gregorio66f57522011-11-30 11:00:00 -05001227
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001228 batch = BatchHttpRequest()
1229 for i in range(0, MAX_BATCH_LIMIT):
1230 batch.add(
1231 HttpRequest(
1232 None,
1233 None,
1234 "https://www.googleapis.com/someapi/v1/collection/?foo=bar",
1235 method="POST",
1236 body="{}",
1237 headers={"content-type": "application/json"},
1238 )
Chris McDonough3cf5e602018-07-18 16:18:38 -04001239 )
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001240 self.assertRaises(BatchError, batch.add, self.request1)
Chris McDonough3cf5e602018-07-18 16:18:38 -04001241
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001242 def test_add_fail_for_resumable(self):
1243 batch = BatchHttpRequest()
Gabriel Garcia23174be2016-05-25 17:28:07 +02001244
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001245 upload = MediaFileUpload(datafile("small.png"), chunksize=500, resumable=True)
1246 self.request1.resumable = upload
1247 with self.assertRaises(BatchError) as batch_error:
1248 batch.add(self.request1, request_id="1")
1249 str(batch_error.exception)
Gabriel Garcia23174be2016-05-25 17:28:07 +02001250
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001251 def test_execute_empty_batch_no_http(self):
1252 batch = BatchHttpRequest()
1253 ret = batch.execute()
1254 self.assertEqual(None, ret)
Gabriel Garcia23174be2016-05-25 17:28:07 +02001255
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001256 def test_execute(self):
1257 batch = BatchHttpRequest()
1258 callbacks = Callbacks()
Gabriel Garcia23174be2016-05-25 17:28:07 +02001259
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001260 batch.add(self.request1, callback=callbacks.f)
1261 batch.add(self.request2, callback=callbacks.f)
1262 http = HttpMockSequence(
1263 [
1264 (
1265 {
1266 "status": "200",
1267 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"',
1268 },
1269 BATCH_RESPONSE,
1270 )
1271 ]
1272 )
1273 batch.execute(http=http)
1274 self.assertEqual({"foo": 42}, callbacks.responses["1"])
1275 self.assertEqual(None, callbacks.exceptions["1"])
1276 self.assertEqual({"baz": "qux"}, callbacks.responses["2"])
1277 self.assertEqual(None, callbacks.exceptions["2"])
Gabriel Garcia23174be2016-05-25 17:28:07 +02001278
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001279 def test_execute_request_body(self):
1280 batch = BatchHttpRequest()
Gabriel Garcia23174be2016-05-25 17:28:07 +02001281
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001282 batch.add(self.request1)
1283 batch.add(self.request2)
1284 http = HttpMockSequence(
1285 [
1286 (
1287 {
1288 "status": "200",
1289 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"',
1290 },
1291 "echo_request_body",
1292 )
1293 ]
1294 )
1295 try:
1296 batch.execute(http=http)
1297 self.fail("Should raise exception")
1298 except BatchError as e:
1299 boundary, _ = e.content.split(None, 1)
1300 self.assertEqual("--", boundary[:2])
1301 parts = e.content.split(boundary)
1302 self.assertEqual(4, len(parts))
1303 self.assertEqual("", parts[0])
1304 self.assertEqual("--", parts[3].rstrip())
1305 header = parts[1].splitlines()[1]
1306 self.assertEqual("Content-Type: application/http", header)
Gabriel Garcia23174be2016-05-25 17:28:07 +02001307
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001308 def test_execute_request_body_with_custom_long_request_ids(self):
1309 batch = BatchHttpRequest()
Gabriel Garcia23174be2016-05-25 17:28:07 +02001310
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001311 batch.add(self.request1, request_id="abc" * 20)
1312 batch.add(self.request2, request_id="def" * 20)
1313 http = HttpMockSequence(
1314 [
1315 (
1316 {
1317 "status": "200",
1318 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"',
1319 },
1320 "echo_request_body",
1321 )
1322 ]
1323 )
1324 try:
1325 batch.execute(http=http)
1326 self.fail("Should raise exception")
1327 except BatchError as e:
1328 boundary, _ = e.content.split(None, 1)
1329 self.assertEqual("--", boundary[:2])
1330 parts = e.content.split(boundary)
1331 self.assertEqual(4, len(parts))
1332 self.assertEqual("", parts[0])
1333 self.assertEqual("--", parts[3].rstrip())
1334 for partindex, request_id in ((1, "abc" * 20), (2, "def" * 20)):
1335 lines = parts[partindex].splitlines()
1336 for n, line in enumerate(lines):
1337 if line.startswith("Content-ID:"):
1338 # assert correct header folding
1339 self.assertTrue(line.endswith("+"), line)
1340 header_continuation = lines[n + 1]
1341 self.assertEqual(
1342 header_continuation,
1343 " %s>" % request_id,
1344 header_continuation,
1345 )
Joe Gregorio654f4a22012-02-09 14:15:44 -05001346
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001347 def test_execute_initial_refresh_oauth2(self):
1348 batch = BatchHttpRequest()
1349 callbacks = Callbacks()
1350 cred = MockCredentials("Foo", expired=True)
Joe Gregorio654f4a22012-02-09 14:15:44 -05001351
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001352 http = HttpMockSequence(
1353 [
1354 (
1355 {
1356 "status": "200",
1357 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"',
1358 },
1359 BATCH_SINGLE_RESPONSE,
1360 )
1361 ]
1362 )
Joe Gregorio654f4a22012-02-09 14:15:44 -05001363
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001364 cred.authorize(http)
Joe Gregorio654f4a22012-02-09 14:15:44 -05001365
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001366 batch.add(self.request1, callback=callbacks.f)
1367 batch.execute(http=http)
Joe Gregorio654f4a22012-02-09 14:15:44 -05001368
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001369 self.assertEqual({"foo": 42}, callbacks.responses["1"])
1370 self.assertIsNone(callbacks.exceptions["1"])
Joe Gregorio654f4a22012-02-09 14:15:44 -05001371
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001372 self.assertEqual(1, cred._refreshed)
Joe Gregorio654f4a22012-02-09 14:15:44 -05001373
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001374 self.assertEqual(1, cred._authorized)
Joe Gregorio654f4a22012-02-09 14:15:44 -05001375
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001376 self.assertEqual(1, cred._applied)
Joe Gregorio654f4a22012-02-09 14:15:44 -05001377
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001378 def test_execute_refresh_and_retry_on_401(self):
1379 batch = BatchHttpRequest()
1380 callbacks = Callbacks()
1381 cred_1 = MockCredentials("Foo")
1382 cred_2 = MockCredentials("Bar")
Joe Gregorio654f4a22012-02-09 14:15:44 -05001383
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001384 http = HttpMockSequence(
1385 [
1386 (
1387 {
1388 "status": "200",
1389 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"',
1390 },
1391 BATCH_RESPONSE_WITH_401,
1392 ),
1393 (
1394 {
1395 "status": "200",
1396 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"',
1397 },
1398 BATCH_SINGLE_RESPONSE,
1399 ),
1400 ]
1401 )
Joe Gregorio654f4a22012-02-09 14:15:44 -05001402
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001403 creds_http_1 = HttpMockSequence([])
1404 cred_1.authorize(creds_http_1)
Joe Gregorio654f4a22012-02-09 14:15:44 -05001405
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001406 creds_http_2 = HttpMockSequence([])
1407 cred_2.authorize(creds_http_2)
Joe Gregorio654f4a22012-02-09 14:15:44 -05001408
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001409 self.request1.http = creds_http_1
1410 self.request2.http = creds_http_2
Joe Gregorio654f4a22012-02-09 14:15:44 -05001411
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001412 batch.add(self.request1, callback=callbacks.f)
1413 batch.add(self.request2, callback=callbacks.f)
1414 batch.execute(http=http)
Joe Gregorio654f4a22012-02-09 14:15:44 -05001415
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001416 self.assertEqual({"foo": 42}, callbacks.responses["1"])
1417 self.assertEqual(None, callbacks.exceptions["1"])
1418 self.assertEqual({"baz": "qux"}, callbacks.responses["2"])
1419 self.assertEqual(None, callbacks.exceptions["2"])
Joe Gregorio654f4a22012-02-09 14:15:44 -05001420
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001421 self.assertEqual(1, cred_1._refreshed)
1422 self.assertEqual(0, cred_2._refreshed)
Joe Gregorio654f4a22012-02-09 14:15:44 -05001423
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001424 self.assertEqual(1, cred_1._authorized)
1425 self.assertEqual(1, cred_2._authorized)
Joe Gregorio66f57522011-11-30 11:00:00 -05001426
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001427 self.assertEqual(1, cred_2._applied)
1428 self.assertEqual(2, cred_1._applied)
Joe Gregoriod0bd3882011-11-22 09:49:47 -05001429
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001430 def test_http_errors_passed_to_callback(self):
1431 batch = BatchHttpRequest()
1432 callbacks = Callbacks()
1433 cred_1 = MockCredentials("Foo")
1434 cred_2 = MockCredentials("Bar")
Joe Gregorio3fb93672012-07-25 11:31:11 -04001435
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001436 http = HttpMockSequence(
1437 [
1438 (
1439 {
1440 "status": "200",
1441 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"',
1442 },
1443 BATCH_RESPONSE_WITH_401,
1444 ),
1445 (
1446 {
1447 "status": "200",
1448 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"',
1449 },
1450 BATCH_RESPONSE_WITH_401,
1451 ),
1452 ]
1453 )
1454
1455 creds_http_1 = HttpMockSequence([])
1456 cred_1.authorize(creds_http_1)
1457
1458 creds_http_2 = HttpMockSequence([])
1459 cred_2.authorize(creds_http_2)
1460
1461 self.request1.http = creds_http_1
1462 self.request2.http = creds_http_2
1463
1464 batch.add(self.request1, callback=callbacks.f)
1465 batch.add(self.request2, callback=callbacks.f)
1466 batch.execute(http=http)
1467
1468 self.assertEqual(None, callbacks.responses["1"])
1469 self.assertEqual(401, callbacks.exceptions["1"].resp.status)
1470 self.assertEqual(
1471 "Authorization Required", callbacks.exceptions["1"].resp.reason
1472 )
1473 self.assertEqual({u"baz": u"qux"}, callbacks.responses["2"])
1474 self.assertEqual(None, callbacks.exceptions["2"])
1475
1476 def test_execute_global_callback(self):
1477 callbacks = Callbacks()
1478 batch = BatchHttpRequest(callback=callbacks.f)
1479
1480 batch.add(self.request1)
1481 batch.add(self.request2)
1482 http = HttpMockSequence(
1483 [
1484 (
1485 {
1486 "status": "200",
1487 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"',
1488 },
1489 BATCH_RESPONSE,
1490 )
1491 ]
1492 )
1493 batch.execute(http=http)
1494 self.assertEqual({"foo": 42}, callbacks.responses["1"])
1495 self.assertEqual({"baz": "qux"}, callbacks.responses["2"])
1496
1497 def test_execute_batch_http_error(self):
1498 callbacks = Callbacks()
1499 batch = BatchHttpRequest(callback=callbacks.f)
1500
1501 batch.add(self.request1)
1502 batch.add(self.request2)
1503 http = HttpMockSequence(
1504 [
1505 (
1506 {
1507 "status": "200",
1508 "content-type": 'multipart/mixed; boundary="batch_foobarbaz"',
1509 },
1510 BATCH_ERROR_RESPONSE,
1511 )
1512 ]
1513 )
1514 batch.execute(http=http)
1515 self.assertEqual({"foo": 42}, callbacks.responses["1"])
1516 expected = (
1517 "<HttpError 403 when requesting "
1518 "https://www.googleapis.com/someapi/v1/collection/?foo=bar returned "
1519 '"Access Not Configured">'
1520 )
1521 self.assertEqual(expected, str(callbacks.exceptions["2"]))
Ali Afshar6f11ea12012-02-07 10:32:14 -05001522
Joe Gregorio5c120db2012-08-23 09:13:55 -04001523
Joe Gregorioba5c7902012-08-03 12:48:16 -04001524class TestRequestUriTooLong(unittest.TestCase):
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001525 def test_turn_get_into_post(self):
1526 def _postproc(resp, content):
1527 return content
Joe Gregorioba5c7902012-08-03 12:48:16 -04001528
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001529 http = HttpMockSequence(
1530 [
1531 ({"status": "200"}, "echo_request_body"),
1532 ({"status": "200"}, "echo_request_headers"),
1533 ]
1534 )
Joe Gregorioba5c7902012-08-03 12:48:16 -04001535
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001536 # Send a long query parameter.
1537 query = {"q": "a" * MAX_URI_LENGTH + "?&"}
1538 req = HttpRequest(
1539 http,
1540 _postproc,
1541 "http://example.com?" + urlencode(query),
1542 method="GET",
1543 body=None,
1544 headers={},
1545 methodId="foo",
1546 resumable=None,
1547 )
Joe Gregorioba5c7902012-08-03 12:48:16 -04001548
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001549 # Query parameters should be sent in the body.
1550 response = req.execute()
1551 self.assertEqual(b"q=" + b"a" * MAX_URI_LENGTH + b"%3F%26", response)
Joe Gregorioba5c7902012-08-03 12:48:16 -04001552
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001553 # Extra headers should be set.
1554 response = req.execute()
1555 self.assertEqual("GET", response["x-http-method-override"])
1556 self.assertEqual(str(MAX_URI_LENGTH + 8), response["content-length"])
1557 self.assertEqual("application/x-www-form-urlencoded", response["content-type"])
Joe Gregorioba5c7902012-08-03 12:48:16 -04001558
Joe Gregorio5c120db2012-08-23 09:13:55 -04001559
1560class TestStreamSlice(unittest.TestCase):
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001561 """Test _StreamSlice."""
Joe Gregorio5c120db2012-08-23 09:13:55 -04001562
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001563 def setUp(self):
1564 self.stream = BytesIO(b"0123456789")
Joe Gregorio5c120db2012-08-23 09:13:55 -04001565
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001566 def test_read(self):
1567 s = _StreamSlice(self.stream, 0, 4)
1568 self.assertEqual(b"", s.read(0))
1569 self.assertEqual(b"0", s.read(1))
1570 self.assertEqual(b"123", s.read())
Joe Gregorio5c120db2012-08-23 09:13:55 -04001571
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001572 def test_read_too_much(self):
1573 s = _StreamSlice(self.stream, 1, 4)
1574 self.assertEqual(b"1234", s.read(6))
Joe Gregorio5c120db2012-08-23 09:13:55 -04001575
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001576 def test_read_all(self):
1577 s = _StreamSlice(self.stream, 2, 1)
1578 self.assertEqual(b"2", s.read(-1))
Joe Gregorio5c120db2012-08-23 09:13:55 -04001579
Ali Afshar164f37e2013-01-07 14:05:45 -08001580
1581class TestResponseCallback(unittest.TestCase):
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001582 """Test adding callbacks to responses."""
Ali Afshar164f37e2013-01-07 14:05:45 -08001583
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001584 def test_ensure_response_callback(self):
1585 m = JsonModel()
1586 request = HttpRequest(
1587 None,
1588 m.response,
1589 "https://www.googleapis.com/someapi/v1/collection/?foo=bar",
1590 method="POST",
1591 body="{}",
1592 headers={"content-type": "application/json"},
1593 )
1594 h = HttpMockSequence([({"status": 200}, "{}")])
1595 responses = []
1596
1597 def _on_response(resp, responses=responses):
1598 responses.append(resp)
1599
1600 request.add_response_callback(_on_response)
1601 request.execute(http=h)
1602 self.assertEqual(1, len(responses))
Ali Afshar164f37e2013-01-07 14:05:45 -08001603
1604
Craig Gurnik8e55b762015-01-20 15:00:10 -05001605class TestHttpMock(unittest.TestCase):
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001606 def test_default_response_headers(self):
1607 http = HttpMock(datafile("zoo.json"))
1608 resp, content = http.request("http://example.com")
1609 self.assertEqual(resp.status, 200)
Craig Gurnik8e55b762015-01-20 15:00:10 -05001610
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001611 def test_error_response(self):
1612 http = HttpMock(datafile("bad_request.json"), {"status": "400"})
1613 model = JsonModel()
1614 request = HttpRequest(
1615 http,
1616 model.response,
1617 "https://www.googleapis.com/someapi/v1/collection/?foo=bar",
1618 method="GET",
1619 headers={},
1620 )
1621 self.assertRaises(HttpError, request.execute)
Alan Briolat26b01002015-08-14 00:13:57 +01001622
Craig Gurnik8e55b762015-01-20 15:00:10 -05001623
Igor Maravić22435292017-01-19 22:28:22 +01001624class TestHttpBuild(unittest.TestCase):
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001625 original_socket_default_timeout = None
Igor Maravić22435292017-01-19 22:28:22 +01001626
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001627 @classmethod
1628 def setUpClass(cls):
1629 cls.original_socket_default_timeout = socket.getdefaulttimeout()
Igor Maravić22435292017-01-19 22:28:22 +01001630
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001631 @classmethod
1632 def tearDownClass(cls):
1633 socket.setdefaulttimeout(cls.original_socket_default_timeout)
Igor Maravić22435292017-01-19 22:28:22 +01001634
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001635 def test_build_http_sets_default_timeout_if_none_specified(self):
1636 socket.setdefaulttimeout(None)
1637 http = build_http()
1638 self.assertIsInstance(http.timeout, int)
1639 self.assertGreater(http.timeout, 0)
Igor Maravić22435292017-01-19 22:28:22 +01001640
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001641 def test_build_http_default_timeout_can_be_overridden(self):
1642 socket.setdefaulttimeout(1.5)
1643 http = build_http()
1644 self.assertAlmostEqual(http.timeout, 1.5, delta=0.001)
Igor Maravić22435292017-01-19 22:28:22 +01001645
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001646 def test_build_http_default_timeout_can_be_set_to_zero(self):
1647 socket.setdefaulttimeout(0)
1648 http = build_http()
Bu Sun Kim1cf3cbc2020-03-12 12:38:23 -07001649 self.assertEqual(http.timeout, 0)
Bu Sun Kimb3b773f2020-03-11 12:58:16 -07001650
1651 def test_build_http_default_308_is_excluded_as_redirect(self):
1652 http = build_http()
1653 self.assertTrue(308 not in http.redirect_codes)
Igor Maravić22435292017-01-19 22:28:22 +01001654
1655
Bu Sun Kim66bb32c2019-10-30 10:11:58 -07001656if __name__ == "__main__":
1657 logging.getLogger().setLevel(logging.ERROR)
1658 unittest.main()