blob: 865d8470378a3cd88eca8cc835912fca70c40c01 [file] [log] [blame]
Craig Citro15744b12015-03-02 13:34:32 -08001#!/usr/bin/env python
Joe Gregorio6bcbcea2011-03-10 15:26:05 -05002#
Craig Citro751b7fb2014-09-23 11:20:38 -07003# Copyright 2014 Google Inc. All Rights Reserved.
Joe Gregorio6bcbcea2011-03-10 15:26:05 -05004#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Http tests
18
John Asmuth864311d2014-04-24 15:46:08 -040019Unit tests for the googleapiclient.http.
Joe Gregorio6bcbcea2011-03-10 15:26:05 -050020"""
INADA Naokid898a372015-03-04 03:52:46 +090021from __future__ import absolute_import
22from six.moves import range
Joe Gregorio6bcbcea2011-03-10 15:26:05 -050023
24__author__ = 'jcgregorio@google.com (Joe Gregorio)'
25
Pat Feratec6050872015-03-03 18:24:59 -080026from six import PY3
Pat Ferateed9affd2015-03-03 16:03:15 -080027from six import BytesIO, StringIO
28from io import FileIO
Pat Ferated5b61bd2015-03-03 16:04:11 -080029from six.moves.urllib.parse import urlencode
Pat Ferateed9affd2015-03-03 16:03:15 -080030
Joe Gregorio7cbceab2011-06-27 10:46:54 -040031# Do not remove the httplib2 import
32import httplib2
Joe Gregorio9086bd32013-06-14 16:32:05 -040033import logging
eesheeshc6425a02016-02-12 15:07:06 +000034import mock
Joe Gregoriod0bd3882011-11-22 09:49:47 -050035import os
Pat Ferate497a90f2015-03-09 09:52:54 -070036import unittest2 as unittest
Joe Gregorio9086bd32013-06-14 16:32:05 -040037import random
eesheeshc6425a02016-02-12 15:07:06 +000038import socket
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +010039import ssl
Joe Gregorio9086bd32013-06-14 16:32:05 -040040import time
Joe Gregorio6bcbcea2011-03-10 15:26:05 -050041
John Asmuth864311d2014-04-24 15:46:08 -040042from googleapiclient.discovery import build
43from googleapiclient.errors import BatchError
44from googleapiclient.errors import HttpError
45from googleapiclient.errors import InvalidChunkSizeError
46from googleapiclient.http import BatchHttpRequest
47from googleapiclient.http import HttpMock
48from googleapiclient.http import HttpMockSequence
49from googleapiclient.http import HttpRequest
50from googleapiclient.http import MAX_URI_LENGTH
51from googleapiclient.http import MediaFileUpload
52from googleapiclient.http import MediaInMemoryUpload
53from googleapiclient.http import MediaIoBaseDownload
54from googleapiclient.http import MediaIoBaseUpload
55from googleapiclient.http import MediaUpload
56from googleapiclient.http import _StreamSlice
57from googleapiclient.http import set_user_agent
58from googleapiclient.model import JsonModel
Joe Gregorio654f4a22012-02-09 14:15:44 -050059from oauth2client.client import Credentials
60
61
62class MockCredentials(Credentials):
63 """Mock class for all Credentials objects."""
64 def __init__(self, bearer_token):
65 super(MockCredentials, self).__init__()
66 self._authorized = 0
67 self._refreshed = 0
68 self._applied = 0
69 self._bearer_token = bearer_token
70
71 def authorize(self, http):
72 self._authorized += 1
73
74 request_orig = http.request
75
76 # The closure that will replace 'httplib2.Http.request'.
77 def new_request(uri, method='GET', body=None, headers=None,
78 redirections=httplib2.DEFAULT_MAX_REDIRECTS,
79 connection_type=None):
80 # Modify the request headers to add the appropriate
81 # Authorization header.
82 if headers is None:
83 headers = {}
84 self.apply(headers)
85
86 resp, content = request_orig(uri, method, body, headers,
87 redirections, connection_type)
88
89 return resp, content
90
91 # Replace the request method with our own closure.
92 http.request = new_request
93
94 # Set credentials as a property of the request method.
95 setattr(http.request, 'credentials', self)
96
97 return http
98
99 def refresh(self, http):
100 self._refreshed += 1
101
102 def apply(self, headers):
103 self._applied += 1
104 headers['authorization'] = self._bearer_token + ' ' + str(self._refreshed)
Joe Gregorio6bcbcea2011-03-10 15:26:05 -0500105
106
eesheeshc6425a02016-02-12 15:07:06 +0000107class HttpMockWithErrors(object):
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100108 def __init__(self, num_errors, success_json, success_data):
109 self.num_errors = num_errors
110 self.success_json = success_json
111 self.success_data = success_data
112
113 def request(self, *args, **kwargs):
114 if not self.num_errors:
115 return httplib2.Response(self.success_json), self.success_data
116 else:
117 self.num_errors -= 1
eesheeshc6425a02016-02-12 15:07:06 +0000118 if self.num_errors == 1:
119 raise ssl.SSLError()
120 else:
121 if PY3:
122 ex = TimeoutError()
123 else:
124 ex = socket.error()
Thomas Bonfort88ab76b2016-04-19 08:48:53 +0200125
126 if self.num_errors == 2:
127 #first try a broken pipe error (#218)
128 ex.errno = socket.errno.EPIPE
129 else:
130 # Initialize the timeout error code to the platform's error code.
131 try:
132 # For Windows:
133 ex.errno = socket.errno.WSAETIMEDOUT
134 except AttributeError:
135 # For Linux/Mac:
136 ex.errno = socket.errno.ETIMEDOUT
137 # Now raise the correct error.
eesheeshc6425a02016-02-12 15:07:06 +0000138 raise ex
139
140
141class HttpMockWithNonRetriableErrors(object):
142 def __init__(self, num_errors, success_json, success_data):
143 self.num_errors = num_errors
144 self.success_json = success_json
145 self.success_data = success_data
146
147 def request(self, *args, **kwargs):
148 if not self.num_errors:
149 return httplib2.Response(self.success_json), self.success_data
150 else:
151 self.num_errors -= 1
152 ex = socket.error()
Thomas Bonfort88ab76b2016-04-19 08:48:53 +0200153 # set errno to a non-retriable value
eesheeshc6425a02016-02-12 15:07:06 +0000154 try:
155 # For Windows:
156 ex.errno = socket.errno.WSAECONNREFUSED
157 except AttributeError:
158 # For Linux/Mac:
159 ex.errno = socket.errno.ECONNREFUSED
160 # Now raise the correct timeout error.
161 raise ex
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100162
163
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500164DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')
165
166
167def datafile(filename):
168 return os.path.join(DATA_DIR, filename)
169
Joe Gregorio6bcbcea2011-03-10 15:26:05 -0500170class TestUserAgent(unittest.TestCase):
171
172 def test_set_user_agent(self):
173 http = HttpMockSequence([
174 ({'status': '200'}, 'echo_request_headers'),
175 ])
176
177 http = set_user_agent(http, "my_app/5.5")
178 resp, content = http.request("http://example.com")
Joe Gregorio654f4a22012-02-09 14:15:44 -0500179 self.assertEqual('my_app/5.5', content['user-agent'])
Joe Gregorio6bcbcea2011-03-10 15:26:05 -0500180
181 def test_set_user_agent_nested(self):
182 http = HttpMockSequence([
183 ({'status': '200'}, 'echo_request_headers'),
184 ])
185
186 http = set_user_agent(http, "my_app/5.5")
187 http = set_user_agent(http, "my_library/0.1")
188 resp, content = http.request("http://example.com")
Joe Gregorio654f4a22012-02-09 14:15:44 -0500189 self.assertEqual('my_app/5.5 my_library/0.1', content['user-agent'])
Joe Gregorio6bcbcea2011-03-10 15:26:05 -0500190
Joe Gregorio910b9b12012-06-12 09:36:30 -0400191
192class TestMediaUpload(unittest.TestCase):
193
Nam T. Nguyendc136312015-12-01 10:18:56 -0800194 def test_media_file_upload_mimetype_detection(self):
195 upload = MediaFileUpload(datafile('small.png'))
196 self.assertEqual('image/png', upload.mimetype())
197
198 upload = MediaFileUpload(datafile('empty'))
199 self.assertEqual('application/octet-stream', upload.mimetype())
200
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500201 def test_media_file_upload_to_from_json(self):
202 upload = MediaFileUpload(
203 datafile('small.png'), chunksize=500, resumable=True)
Joe Gregorio654f4a22012-02-09 14:15:44 -0500204 self.assertEqual('image/png', upload.mimetype())
205 self.assertEqual(190, upload.size())
206 self.assertEqual(True, upload.resumable())
207 self.assertEqual(500, upload.chunksize())
Pat Ferate2b140222015-03-03 18:05:11 -0800208 self.assertEqual(b'PNG', upload.getbytes(1, 3))
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500209
210 json = upload.to_json()
211 new_upload = MediaUpload.new_from_json(json)
212
Joe Gregorio654f4a22012-02-09 14:15:44 -0500213 self.assertEqual('image/png', new_upload.mimetype())
214 self.assertEqual(190, new_upload.size())
215 self.assertEqual(True, new_upload.resumable())
216 self.assertEqual(500, new_upload.chunksize())
Pat Ferate2b140222015-03-03 18:05:11 -0800217 self.assertEqual(b'PNG', new_upload.getbytes(1, 3))
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500218
Joe Gregorioc80ac9d2012-08-21 14:09:09 -0400219 def test_media_file_upload_raises_on_invalid_chunksize(self):
220 self.assertRaises(InvalidChunkSizeError, MediaFileUpload,
221 datafile('small.png'), mimetype='image/png', chunksize=-2,
222 resumable=True)
223
Ali Afshar1cb6b672012-03-12 08:46:14 -0400224 def test_media_inmemory_upload(self):
Pat Ferate2b140222015-03-03 18:05:11 -0800225 media = MediaInMemoryUpload(b'abcdef', mimetype='text/plain', chunksize=10,
Ali Afshar1cb6b672012-03-12 08:46:14 -0400226 resumable=True)
227 self.assertEqual('text/plain', media.mimetype())
228 self.assertEqual(10, media.chunksize())
229 self.assertTrue(media.resumable())
Pat Ferate2b140222015-03-03 18:05:11 -0800230 self.assertEqual(b'bc', media.getbytes(1, 2))
Ali Afshar1cb6b672012-03-12 08:46:14 -0400231 self.assertEqual(6, media.size())
232
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500233 def test_http_request_to_from_json(self):
234
235 def _postproc(*kwargs):
236 pass
237
238 http = httplib2.Http()
239 media_upload = MediaFileUpload(
240 datafile('small.png'), chunksize=500, resumable=True)
241 req = HttpRequest(
242 http,
243 _postproc,
244 'http://example.com',
245 method='POST',
246 body='{}',
247 headers={'content-type': 'multipart/related; boundary="---flubber"'},
248 methodId='foo',
249 resumable=media_upload)
250
251 json = req.to_json()
252 new_req = HttpRequest.from_json(json, http, _postproc)
253
Joe Gregorio654f4a22012-02-09 14:15:44 -0500254 self.assertEqual({'content-type':
255 'multipart/related; boundary="---flubber"'},
256 new_req.headers)
257 self.assertEqual('http://example.com', new_req.uri)
258 self.assertEqual('{}', new_req.body)
259 self.assertEqual(http, new_req.http)
260 self.assertEqual(media_upload.to_json(), new_req.resumable.to_json())
Joe Gregoriod0bd3882011-11-22 09:49:47 -0500261
Joe Gregorio9086bd32013-06-14 16:32:05 -0400262 self.assertEqual(random.random, new_req._rand)
263 self.assertEqual(time.sleep, new_req._sleep)
264
Joe Gregorio910b9b12012-06-12 09:36:30 -0400265
266class TestMediaIoBaseUpload(unittest.TestCase):
267
268 def test_media_io_base_upload_from_file_io(self):
Pat Ferateed9affd2015-03-03 16:03:15 -0800269 fd = FileIO(datafile('small.png'), 'r')
270 upload = MediaIoBaseUpload(
271 fd=fd, mimetype='image/png', chunksize=500, resumable=True)
272 self.assertEqual('image/png', upload.mimetype())
273 self.assertEqual(190, upload.size())
274 self.assertEqual(True, upload.resumable())
275 self.assertEqual(500, upload.chunksize())
Pat Ferate2b140222015-03-03 18:05:11 -0800276 self.assertEqual(b'PNG', upload.getbytes(1, 3))
Joe Gregorio910b9b12012-06-12 09:36:30 -0400277
278 def test_media_io_base_upload_from_file_object(self):
Pat Ferate2b140222015-03-03 18:05:11 -0800279 f = open(datafile('small.png'), 'rb')
Joe Gregorio910b9b12012-06-12 09:36:30 -0400280 upload = MediaIoBaseUpload(
Joe Gregorio4a2c29f2012-07-12 12:52:47 -0400281 fd=f, mimetype='image/png', chunksize=500, resumable=True)
Joe Gregorio910b9b12012-06-12 09:36:30 -0400282 self.assertEqual('image/png', upload.mimetype())
283 self.assertEqual(190, upload.size())
284 self.assertEqual(True, upload.resumable())
285 self.assertEqual(500, upload.chunksize())
Pat Ferate2b140222015-03-03 18:05:11 -0800286 self.assertEqual(b'PNG', upload.getbytes(1, 3))
Joe Gregorio910b9b12012-06-12 09:36:30 -0400287 f.close()
288
289 def test_media_io_base_upload_serializable(self):
Pat Ferate2b140222015-03-03 18:05:11 -0800290 f = open(datafile('small.png'), 'rb')
Joe Gregorio4a2c29f2012-07-12 12:52:47 -0400291 upload = MediaIoBaseUpload(fd=f, mimetype='image/png')
Joe Gregorio910b9b12012-06-12 09:36:30 -0400292
293 try:
294 json = upload.to_json()
295 self.fail('MediaIoBaseUpload should not be serializable.')
296 except NotImplementedError:
297 pass
298
Pat Feratec6050872015-03-03 18:24:59 -0800299 @unittest.skipIf(PY3, 'Strings and Bytes are different types')
Joe Gregorio910b9b12012-06-12 09:36:30 -0400300 def test_media_io_base_upload_from_string_io(self):
Pat Ferate2b140222015-03-03 18:05:11 -0800301 f = open(datafile('small.png'), 'rb')
Pat Ferateed9affd2015-03-03 16:03:15 -0800302 fd = StringIO(f.read())
Joe Gregorio910b9b12012-06-12 09:36:30 -0400303 f.close()
304
305 upload = MediaIoBaseUpload(
Joe Gregorio4a2c29f2012-07-12 12:52:47 -0400306 fd=fd, mimetype='image/png', chunksize=500, resumable=True)
Joe Gregorio910b9b12012-06-12 09:36:30 -0400307 self.assertEqual('image/png', upload.mimetype())
Joe Gregorioc80ac9d2012-08-21 14:09:09 -0400308 self.assertEqual(190, upload.size())
Joe Gregorio910b9b12012-06-12 09:36:30 -0400309 self.assertEqual(True, upload.resumable())
310 self.assertEqual(500, upload.chunksize())
Pat Ferate2b140222015-03-03 18:05:11 -0800311 self.assertEqual(b'PNG', upload.getbytes(1, 3))
Joe Gregorio910b9b12012-06-12 09:36:30 -0400312 f.close()
313
314 def test_media_io_base_upload_from_bytes(self):
Pat Ferate2b140222015-03-03 18:05:11 -0800315 f = open(datafile('small.png'), 'rb')
Pat Ferateed9affd2015-03-03 16:03:15 -0800316 fd = BytesIO(f.read())
317 upload = MediaIoBaseUpload(
318 fd=fd, mimetype='image/png', chunksize=500, resumable=True)
319 self.assertEqual('image/png', upload.mimetype())
320 self.assertEqual(190, upload.size())
321 self.assertEqual(True, upload.resumable())
322 self.assertEqual(500, upload.chunksize())
Pat Ferate2b140222015-03-03 18:05:11 -0800323 self.assertEqual(b'PNG', upload.getbytes(1, 3))
Joe Gregorio910b9b12012-06-12 09:36:30 -0400324
Joe Gregorioc80ac9d2012-08-21 14:09:09 -0400325 def test_media_io_base_upload_raises_on_invalid_chunksize(self):
Pat Ferate2b140222015-03-03 18:05:11 -0800326 f = open(datafile('small.png'), 'rb')
Pat Ferateed9affd2015-03-03 16:03:15 -0800327 fd = BytesIO(f.read())
328 self.assertRaises(InvalidChunkSizeError, MediaIoBaseUpload,
329 fd, 'image/png', chunksize=-2, resumable=True)
Joe Gregorioc80ac9d2012-08-21 14:09:09 -0400330
331 def test_media_io_base_upload_streamable(self):
Pat Ferate2b140222015-03-03 18:05:11 -0800332 fd = BytesIO(b'stuff')
Pat Ferateed9affd2015-03-03 16:03:15 -0800333 upload = MediaIoBaseUpload(
334 fd=fd, mimetype='image/png', chunksize=500, resumable=True)
335 self.assertEqual(True, upload.has_stream())
336 self.assertEqual(fd, upload.stream())
Joe Gregorioc80ac9d2012-08-21 14:09:09 -0400337
Joe Gregorio9086bd32013-06-14 16:32:05 -0400338 def test_media_io_base_next_chunk_retries(self):
Pat Ferate2b140222015-03-03 18:05:11 -0800339 f = open(datafile('small.png'), 'rb')
Pat Ferateed9affd2015-03-03 16:03:15 -0800340 fd = BytesIO(f.read())
Joe Gregorio9086bd32013-06-14 16:32:05 -0400341 upload = MediaIoBaseUpload(
342 fd=fd, mimetype='image/png', chunksize=500, resumable=True)
343
Zhihao Yuancc6d3982016-07-27 11:40:45 -0500344 # Simulate errors for both the request that creates the resumable upload
345 # and the upload itself.
Joe Gregorio9086bd32013-06-14 16:32:05 -0400346 http = HttpMockSequence([
347 ({'status': '500'}, ''),
348 ({'status': '500'}, ''),
349 ({'status': '503'}, ''),
350 ({'status': '200', 'location': 'location'}, ''),
Zhihao Yuancc6d3982016-07-27 11:40:45 -0500351 ({'status': '403'}, USER_RATE_LIMIT_EXCEEDED_RESPONSE),
352 ({'status': '403'}, RATE_LIMIT_EXCEEDED_RESPONSE),
353 ({'status': '429'}, ''),
Joe Gregorio9086bd32013-06-14 16:32:05 -0400354 ({'status': '200'}, '{}'),
355 ])
356
357 model = JsonModel()
358 uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar'
359 method = u'POST'
360 request = HttpRequest(
361 http,
362 model.response,
363 uri,
364 method=method,
365 headers={},
366 resumable=upload)
367
368 sleeptimes = []
369 request._sleep = lambda x: sleeptimes.append(x)
370 request._rand = lambda: 10
371
372 request.execute(num_retries=3)
373 self.assertEqual([20, 40, 80, 20, 40, 80], sleeptimes)
374
Zhihao Yuancc6d3982016-07-27 11:40:45 -0500375 def test_media_io_base_next_chunk_no_retry_403_not_configured(self):
376 fd = BytesIO(b"i am png")
377 upload = MediaIoBaseUpload(
378 fd=fd, mimetype='image/png', chunksize=500, resumable=True)
379
380 http = HttpMockSequence([
381 ({'status': '403'}, NOT_CONFIGURED_RESPONSE),
382 ({'status': '200'}, '{}')
383 ])
384
385 model = JsonModel()
386 uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar'
387 method = u'POST'
388 request = HttpRequest(
389 http,
390 model.response,
391 uri,
392 method=method,
393 headers={},
394 resumable=upload)
395
396 request._rand = lambda: 1.0
397 request._sleep = mock.MagicMock()
398
399 with self.assertRaises(HttpError):
400 request.execute(num_retries=3)
401 request._sleep.assert_not_called()
402
Joe Gregorio910b9b12012-06-12 09:36:30 -0400403
Joe Gregorio708388c2012-06-15 13:43:04 -0400404class TestMediaIoBaseDownload(unittest.TestCase):
405
406 def setUp(self):
407 http = HttpMock(datafile('zoo.json'), {'status': '200'})
Joe Gregorio68a8cfe2012-08-03 16:17:40 -0400408 zoo = build('zoo', 'v1', http=http)
Joe Gregorio708388c2012-06-15 13:43:04 -0400409 self.request = zoo.animals().get_media(name='Lion')
Pat Ferateed9affd2015-03-03 16:03:15 -0800410 self.fd = BytesIO()
Joe Gregorio708388c2012-06-15 13:43:04 -0400411
412 def test_media_io_base_download(self):
413 self.request.http = HttpMockSequence([
414 ({'status': '200',
Pat Ferate2b140222015-03-03 18:05:11 -0800415 'content-range': '0-2/5'}, b'123'),
Joe Gregorio708388c2012-06-15 13:43:04 -0400416 ({'status': '200',
Pat Ferate2b140222015-03-03 18:05:11 -0800417 'content-range': '3-4/5'}, b'45'),
Joe Gregorio708388c2012-06-15 13:43:04 -0400418 ])
Joe Gregorio97ef1cc2013-06-13 14:47:10 -0400419 self.assertEqual(True, self.request.http.follow_redirects)
Joe Gregorio708388c2012-06-15 13:43:04 -0400420
421 download = MediaIoBaseDownload(
Joe Gregorio4a2c29f2012-07-12 12:52:47 -0400422 fd=self.fd, request=self.request, chunksize=3)
Joe Gregorio708388c2012-06-15 13:43:04 -0400423
Joe Gregorio4a2c29f2012-07-12 12:52:47 -0400424 self.assertEqual(self.fd, download._fd)
425 self.assertEqual(3, download._chunksize)
426 self.assertEqual(0, download._progress)
427 self.assertEqual(None, download._total_size)
428 self.assertEqual(False, download._done)
429 self.assertEqual(self.request.uri, download._uri)
Joe Gregorio708388c2012-06-15 13:43:04 -0400430
431 status, done = download.next_chunk()
432
Pat Ferate2b140222015-03-03 18:05:11 -0800433 self.assertEqual(self.fd.getvalue(), b'123')
Joe Gregorio708388c2012-06-15 13:43:04 -0400434 self.assertEqual(False, done)
Joe Gregorio4a2c29f2012-07-12 12:52:47 -0400435 self.assertEqual(3, download._progress)
436 self.assertEqual(5, download._total_size)
Joe Gregorio708388c2012-06-15 13:43:04 -0400437 self.assertEqual(3, status.resumable_progress)
438
439 status, done = download.next_chunk()
440
Pat Ferate2b140222015-03-03 18:05:11 -0800441 self.assertEqual(self.fd.getvalue(), b'12345')
Joe Gregorio708388c2012-06-15 13:43:04 -0400442 self.assertEqual(True, done)
Joe Gregorio4a2c29f2012-07-12 12:52:47 -0400443 self.assertEqual(5, download._progress)
444 self.assertEqual(5, download._total_size)
Joe Gregorio708388c2012-06-15 13:43:04 -0400445
446 def test_media_io_base_download_handle_redirects(self):
447 self.request.http = HttpMockSequence([
Joe Gregorio238feb72013-06-19 13:15:31 -0400448 ({'status': '200',
Pat Ferate2b140222015-03-03 18:05:11 -0800449 'content-location': 'https://secure.example.net/lion'}, b''),
Joe Gregorio708388c2012-06-15 13:43:04 -0400450 ({'status': '200',
Pat Ferate2b140222015-03-03 18:05:11 -0800451 'content-range': '0-2/5'}, b'abc'),
Joe Gregorio708388c2012-06-15 13:43:04 -0400452 ])
453
454 download = MediaIoBaseDownload(
Joe Gregorio4a2c29f2012-07-12 12:52:47 -0400455 fd=self.fd, request=self.request, chunksize=3)
Joe Gregorio708388c2012-06-15 13:43:04 -0400456
457 status, done = download.next_chunk()
458
Joe Gregorio4a2c29f2012-07-12 12:52:47 -0400459 self.assertEqual('https://secure.example.net/lion', download._uri)
Joe Gregorio708388c2012-06-15 13:43:04 -0400460
461 def test_media_io_base_download_handle_4xx(self):
462 self.request.http = HttpMockSequence([
463 ({'status': '400'}, ''),
464 ])
465
466 download = MediaIoBaseDownload(
Joe Gregorio4a2c29f2012-07-12 12:52:47 -0400467 fd=self.fd, request=self.request, chunksize=3)
Joe Gregorio708388c2012-06-15 13:43:04 -0400468
469 try:
470 status, done = download.next_chunk()
471 self.fail('Should raise an exception')
472 except HttpError:
473 pass
474
475 # Even after raising an exception we can pick up where we left off.
476 self.request.http = HttpMockSequence([
477 ({'status': '200',
Pat Ferate2b140222015-03-03 18:05:11 -0800478 'content-range': '0-2/5'}, b'123'),
Joe Gregorio708388c2012-06-15 13:43:04 -0400479 ])
480
481 status, done = download.next_chunk()
482
Pat Ferate2b140222015-03-03 18:05:11 -0800483 self.assertEqual(self.fd.getvalue(), b'123')
Joe Gregorio708388c2012-06-15 13:43:04 -0400484
eesheeshc6425a02016-02-12 15:07:06 +0000485 def test_media_io_base_download_retries_connection_errors(self):
486 self.request.http = HttpMockWithErrors(
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100487 3, {'status': '200', 'content-range': '0-2/3'}, b'123')
488
489 download = MediaIoBaseDownload(
490 fd=self.fd, request=self.request, chunksize=3)
491 download._sleep = lambda _x: 0 # do nothing
492 download._rand = lambda: 10
493
494 status, done = download.next_chunk(num_retries=3)
495
496 self.assertEqual(self.fd.getvalue(), b'123')
497 self.assertEqual(True, done)
498
Joe Gregorio9086bd32013-06-14 16:32:05 -0400499 def test_media_io_base_download_retries_5xx(self):
500 self.request.http = HttpMockSequence([
501 ({'status': '500'}, ''),
502 ({'status': '500'}, ''),
503 ({'status': '500'}, ''),
504 ({'status': '200',
Pat Ferate2b140222015-03-03 18:05:11 -0800505 'content-range': '0-2/5'}, b'123'),
Joe Gregorio9086bd32013-06-14 16:32:05 -0400506 ({'status': '503'}, ''),
507 ({'status': '503'}, ''),
508 ({'status': '503'}, ''),
509 ({'status': '200',
Pat Ferate2b140222015-03-03 18:05:11 -0800510 'content-range': '3-4/5'}, b'45'),
Joe Gregorio9086bd32013-06-14 16:32:05 -0400511 ])
512
513 download = MediaIoBaseDownload(
514 fd=self.fd, request=self.request, chunksize=3)
515
516 self.assertEqual(self.fd, download._fd)
517 self.assertEqual(3, download._chunksize)
518 self.assertEqual(0, download._progress)
519 self.assertEqual(None, download._total_size)
520 self.assertEqual(False, download._done)
521 self.assertEqual(self.request.uri, download._uri)
522
523 # Set time.sleep and random.random stubs.
524 sleeptimes = []
525 download._sleep = lambda x: sleeptimes.append(x)
526 download._rand = lambda: 10
527
528 status, done = download.next_chunk(num_retries=3)
529
530 # Check for exponential backoff using the rand function above.
531 self.assertEqual([20, 40, 80], sleeptimes)
532
Pat Ferate2b140222015-03-03 18:05:11 -0800533 self.assertEqual(self.fd.getvalue(), b'123')
Joe Gregorio9086bd32013-06-14 16:32:05 -0400534 self.assertEqual(False, done)
535 self.assertEqual(3, download._progress)
536 self.assertEqual(5, download._total_size)
537 self.assertEqual(3, status.resumable_progress)
538
539 # Reset time.sleep stub.
540 del sleeptimes[0:len(sleeptimes)]
541
542 status, done = download.next_chunk(num_retries=3)
543
544 # Check for exponential backoff using the rand function above.
545 self.assertEqual([20, 40, 80], sleeptimes)
546
Pat Ferate2b140222015-03-03 18:05:11 -0800547 self.assertEqual(self.fd.getvalue(), b'12345')
Joe Gregorio9086bd32013-06-14 16:32:05 -0400548 self.assertEqual(True, done)
549 self.assertEqual(5, download._progress)
550 self.assertEqual(5, download._total_size)
551
Joe Gregorio66f57522011-11-30 11:00:00 -0500552EXPECTED = """POST /someapi/v1/collection/?foo=bar HTTP/1.1
553Content-Type: application/json
554MIME-Version: 1.0
Joe Gregorio5d1171b2012-01-05 10:48:24 -0500555Host: www.googleapis.com
556content-length: 2\r\n\r\n{}"""
557
558
559NO_BODY_EXPECTED = """POST /someapi/v1/collection/?foo=bar HTTP/1.1
560Content-Type: application/json
561MIME-Version: 1.0
562Host: www.googleapis.com
563content-length: 0\r\n\r\n"""
Joe Gregorio66f57522011-11-30 11:00:00 -0500564
Pepper Lebeck-Jobe30ecbd42015-06-12 11:45:57 -0400565NO_BODY_EXPECTED_GET = """GET /someapi/v1/collection/?foo=bar HTTP/1.1
566Content-Type: application/json
567MIME-Version: 1.0
568Host: www.googleapis.com\r\n\r\n"""
569
Joe Gregorio66f57522011-11-30 11:00:00 -0500570
571RESPONSE = """HTTP/1.1 200 OK
Joe Gregorio20b54fb2012-07-26 09:59:35 -0400572Content-Type: application/json
Joe Gregorio66f57522011-11-30 11:00:00 -0500573Content-Length: 14
574ETag: "etag/pony"\r\n\r\n{"answer": 42}"""
575
576
INADA Naoki09157612015-03-25 01:51:03 +0900577BATCH_RESPONSE = b"""--batch_foobarbaz
Joe Gregorio66f57522011-11-30 11:00:00 -0500578Content-Type: application/http
579Content-Transfer-Encoding: binary
580Content-ID: <randomness+1>
581
582HTTP/1.1 200 OK
Joe Gregorio20b54fb2012-07-26 09:59:35 -0400583Content-Type: application/json
Joe Gregorio66f57522011-11-30 11:00:00 -0500584Content-Length: 14
585ETag: "etag/pony"\r\n\r\n{"foo": 42}
586
587--batch_foobarbaz
588Content-Type: application/http
589Content-Transfer-Encoding: binary
590Content-ID: <randomness+2>
591
592HTTP/1.1 200 OK
Joe Gregorio20b54fb2012-07-26 09:59:35 -0400593Content-Type: application/json
Joe Gregorio66f57522011-11-30 11:00:00 -0500594Content-Length: 14
595ETag: "etag/sheep"\r\n\r\n{"baz": "qux"}
596--batch_foobarbaz--"""
597
Joe Gregorio5d1171b2012-01-05 10:48:24 -0500598
INADA Naoki09157612015-03-25 01:51:03 +0900599BATCH_ERROR_RESPONSE = b"""--batch_foobarbaz
Joe Gregorio3fb93672012-07-25 11:31:11 -0400600Content-Type: application/http
601Content-Transfer-Encoding: binary
602Content-ID: <randomness+1>
603
604HTTP/1.1 200 OK
Joe Gregorio20b54fb2012-07-26 09:59:35 -0400605Content-Type: application/json
Joe Gregorio3fb93672012-07-25 11:31:11 -0400606Content-Length: 14
607ETag: "etag/pony"\r\n\r\n{"foo": 42}
608
609--batch_foobarbaz
610Content-Type: application/http
611Content-Transfer-Encoding: binary
612Content-ID: <randomness+2>
613
614HTTP/1.1 403 Access Not Configured
Joe Gregorio20b54fb2012-07-26 09:59:35 -0400615Content-Type: application/json
616Content-Length: 245
617ETag: "etag/sheep"\r\n\r\n{
Joe Gregorio3fb93672012-07-25 11:31:11 -0400618 "error": {
619 "errors": [
620 {
621 "domain": "usageLimits",
622 "reason": "accessNotConfigured",
623 "message": "Access Not Configured",
624 "debugInfo": "QuotaState: BLOCKED"
625 }
626 ],
627 "code": 403,
628 "message": "Access Not Configured"
629 }
630}
631
632--batch_foobarbaz--"""
633
634
INADA Naoki09157612015-03-25 01:51:03 +0900635BATCH_RESPONSE_WITH_401 = b"""--batch_foobarbaz
Joe Gregorio654f4a22012-02-09 14:15:44 -0500636Content-Type: application/http
637Content-Transfer-Encoding: binary
638Content-ID: <randomness+1>
639
Joe Gregorioc752e332012-07-11 14:43:52 -0400640HTTP/1.1 401 Authorization Required
Joe Gregorio20b54fb2012-07-26 09:59:35 -0400641Content-Type: application/json
Joe Gregorio654f4a22012-02-09 14:15:44 -0500642Content-Length: 14
643ETag: "etag/pony"\r\n\r\n{"error": {"message":
644 "Authorizaton failed."}}
645
646--batch_foobarbaz
647Content-Type: application/http
648Content-Transfer-Encoding: binary
649Content-ID: <randomness+2>
650
651HTTP/1.1 200 OK
Joe Gregorio20b54fb2012-07-26 09:59:35 -0400652Content-Type: application/json
Joe Gregorio654f4a22012-02-09 14:15:44 -0500653Content-Length: 14
654ETag: "etag/sheep"\r\n\r\n{"baz": "qux"}
655--batch_foobarbaz--"""
656
657
INADA Naoki09157612015-03-25 01:51:03 +0900658BATCH_SINGLE_RESPONSE = b"""--batch_foobarbaz
Joe Gregorio654f4a22012-02-09 14:15:44 -0500659Content-Type: application/http
660Content-Transfer-Encoding: binary
661Content-ID: <randomness+1>
662
663HTTP/1.1 200 OK
Joe Gregorio20b54fb2012-07-26 09:59:35 -0400664Content-Type: application/json
Joe Gregorio654f4a22012-02-09 14:15:44 -0500665Content-Length: 14
666ETag: "etag/pony"\r\n\r\n{"foo": 42}
667--batch_foobarbaz--"""
668
eesheeshc6425a02016-02-12 15:07:06 +0000669
670USER_RATE_LIMIT_EXCEEDED_RESPONSE = """{
671 "error": {
672 "errors": [
673 {
674 "domain": "usageLimits",
675 "reason": "userRateLimitExceeded",
676 "message": "User Rate Limit Exceeded"
677 }
678 ],
679 "code": 403,
680 "message": "User Rate Limit Exceeded"
681 }
682}"""
683
684
685RATE_LIMIT_EXCEEDED_RESPONSE = """{
686 "error": {
687 "errors": [
688 {
689 "domain": "usageLimits",
690 "reason": "rateLimitExceeded",
691 "message": "Rate Limit Exceeded"
692 }
693 ],
694 "code": 403,
695 "message": "Rate Limit Exceeded"
696 }
697}"""
698
699
700NOT_CONFIGURED_RESPONSE = """{
701 "error": {
702 "errors": [
703 {
704 "domain": "usageLimits",
705 "reason": "accessNotConfigured",
706 "message": "Access Not Configured"
707 }
708 ],
709 "code": 403,
710 "message": "Access Not Configured"
711 }
712}"""
713
Joe Gregorio654f4a22012-02-09 14:15:44 -0500714class Callbacks(object):
715 def __init__(self):
716 self.responses = {}
717 self.exceptions = {}
718
719 def f(self, request_id, response, exception):
720 self.responses[request_id] = response
721 self.exceptions[request_id] = exception
722
723
Joe Gregorio83f2ee62012-12-06 15:25:54 -0500724class TestHttpRequest(unittest.TestCase):
725 def test_unicode(self):
726 http = HttpMock(datafile('zoo.json'), headers={'status': '200'})
727 model = JsonModel()
728 uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
729 method = u'POST'
730 request = HttpRequest(
731 http,
732 model.response,
733 uri,
734 method=method,
735 body=u'{}',
736 headers={'content-type': 'application/json'})
737 request.execute()
738 self.assertEqual(uri, http.uri)
739 self.assertEqual(str, type(http.uri))
740 self.assertEqual(method, http.method)
741 self.assertEqual(str, type(http.method))
742
eesheeshc6425a02016-02-12 15:07:06 +0000743 def test_no_retry_connection_errors(self):
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100744 model = JsonModel()
745 request = HttpRequest(
eesheeshc6425a02016-02-12 15:07:06 +0000746 HttpMockWithNonRetriableErrors(1, {'status': '200'}, '{"foo": "bar"}'),
747 model.response,
748 u'https://www.example.com/json_api_endpoint')
749 request._sleep = lambda _x: 0 # do nothing
750 request._rand = lambda: 10
751 with self.assertRaises(socket.error):
752 response = request.execute(num_retries=3)
753
754
755 def test_retry_connection_errors_non_resumable(self):
756 model = JsonModel()
757 request = HttpRequest(
758 HttpMockWithErrors(3, {'status': '200'}, '{"foo": "bar"}'),
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100759 model.response,
760 u'https://www.example.com/json_api_endpoint')
761 request._sleep = lambda _x: 0 # do nothing
762 request._rand = lambda: 10
763 response = request.execute(num_retries=3)
764 self.assertEqual({u'foo': u'bar'}, response)
765
eesheeshc6425a02016-02-12 15:07:06 +0000766 def test_retry_connection_errors_resumable(self):
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100767 with open(datafile('small.png'), 'rb') as small_png_file:
768 small_png_fd = BytesIO(small_png_file.read())
769 upload = MediaIoBaseUpload(fd=small_png_fd, mimetype='image/png',
770 chunksize=500, resumable=True)
771 model = JsonModel()
772
773 request = HttpRequest(
eesheeshc6425a02016-02-12 15:07:06 +0000774 HttpMockWithErrors(
Sergiy Byelozyorov703c92c2015-12-21 23:27:48 +0100775 3, {'status': '200', 'location': 'location'}, '{"foo": "bar"}'),
776 model.response,
777 u'https://www.example.com/file_upload',
778 method='POST',
779 resumable=upload)
780 request._sleep = lambda _x: 0 # do nothing
781 request._rand = lambda: 10
782 response = request.execute(num_retries=3)
783 self.assertEqual({u'foo': u'bar'}, response)
784
Joe Gregorio9086bd32013-06-14 16:32:05 -0400785 def test_retry(self):
786 num_retries = 5
eesheeshc6425a02016-02-12 15:07:06 +0000787 resp_seq = [({'status': '500'}, '')] * (num_retries - 3)
788 resp_seq.append(({'status': '403'}, RATE_LIMIT_EXCEEDED_RESPONSE))
789 resp_seq.append(({'status': '403'}, USER_RATE_LIMIT_EXCEEDED_RESPONSE))
790 resp_seq.append(({'status': '429'}, ''))
Joe Gregorio9086bd32013-06-14 16:32:05 -0400791 resp_seq.append(({'status': '200'}, '{}'))
792
793 http = HttpMockSequence(resp_seq)
794 model = JsonModel()
795 uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
796 method = u'POST'
797 request = HttpRequest(
798 http,
799 model.response,
800 uri,
801 method=method,
802 body=u'{}',
803 headers={'content-type': 'application/json'})
804
805 sleeptimes = []
806 request._sleep = lambda x: sleeptimes.append(x)
807 request._rand = lambda: 10
808
809 request.execute(num_retries=num_retries)
810
811 self.assertEqual(num_retries, len(sleeptimes))
INADA Naokid898a372015-03-04 03:52:46 +0900812 for retry_num in range(num_retries):
Joe Gregorio9086bd32013-06-14 16:32:05 -0400813 self.assertEqual(10 * 2**(retry_num + 1), sleeptimes[retry_num])
814
eesheeshc6425a02016-02-12 15:07:06 +0000815 def test_no_retry_succeeds(self):
816 num_retries = 5
817 resp_seq = [({'status': '200'}, '{}')] * (num_retries)
818
819 http = HttpMockSequence(resp_seq)
820 model = JsonModel()
821 uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
822 method = u'POST'
823 request = HttpRequest(
824 http,
825 model.response,
826 uri,
827 method=method,
828 body=u'{}',
829 headers={'content-type': 'application/json'})
830
831 sleeptimes = []
832 request._sleep = lambda x: sleeptimes.append(x)
833 request._rand = lambda: 10
834
835 request.execute(num_retries=num_retries)
836
837 self.assertEqual(0, len(sleeptimes))
838
Joe Gregorio9086bd32013-06-14 16:32:05 -0400839 def test_no_retry_fails_fast(self):
840 http = HttpMockSequence([
841 ({'status': '500'}, ''),
842 ({'status': '200'}, '{}')
843 ])
844 model = JsonModel()
845 uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
846 method = u'POST'
847 request = HttpRequest(
848 http,
849 model.response,
850 uri,
851 method=method,
852 body=u'{}',
853 headers={'content-type': 'application/json'})
854
855 request._rand = lambda: 1.0
eesheeshc6425a02016-02-12 15:07:06 +0000856 request._sleep = mock.MagicMock()
Joe Gregorio9086bd32013-06-14 16:32:05 -0400857
eesheeshc6425a02016-02-12 15:07:06 +0000858 with self.assertRaises(HttpError):
Joe Gregorio9086bd32013-06-14 16:32:05 -0400859 request.execute()
eesheeshc6425a02016-02-12 15:07:06 +0000860 request._sleep.assert_not_called()
Joe Gregorio9086bd32013-06-14 16:32:05 -0400861
eesheeshc6425a02016-02-12 15:07:06 +0000862 def test_no_retry_403_not_configured_fails_fast(self):
863 http = HttpMockSequence([
864 ({'status': '403'}, NOT_CONFIGURED_RESPONSE),
865 ({'status': '200'}, '{}')
866 ])
867 model = JsonModel()
868 uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
869 method = u'POST'
870 request = HttpRequest(
871 http,
872 model.response,
873 uri,
874 method=method,
875 body=u'{}',
876 headers={'content-type': 'application/json'})
877
878 request._rand = lambda: 1.0
879 request._sleep = mock.MagicMock()
880
881 with self.assertRaises(HttpError):
882 request.execute()
883 request._sleep.assert_not_called()
884
885 def test_no_retry_403_fails_fast(self):
886 http = HttpMockSequence([
887 ({'status': '403'}, ''),
888 ({'status': '200'}, '{}')
889 ])
890 model = JsonModel()
891 uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
892 method = u'POST'
893 request = HttpRequest(
894 http,
895 model.response,
896 uri,
897 method=method,
898 body=u'{}',
899 headers={'content-type': 'application/json'})
900
901 request._rand = lambda: 1.0
902 request._sleep = mock.MagicMock()
903
904 with self.assertRaises(HttpError):
905 request.execute()
906 request._sleep.assert_not_called()
907
908 def test_no_retry_401_fails_fast(self):
909 http = HttpMockSequence([
910 ({'status': '401'}, ''),
911 ({'status': '200'}, '{}')
912 ])
913 model = JsonModel()
914 uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
915 method = u'POST'
916 request = HttpRequest(
917 http,
918 model.response,
919 uri,
920 method=method,
921 body=u'{}',
922 headers={'content-type': 'application/json'})
923
924 request._rand = lambda: 1.0
925 request._sleep = mock.MagicMock()
926
927 with self.assertRaises(HttpError):
928 request.execute()
929 request._sleep.assert_not_called()
Joe Gregorio83f2ee62012-12-06 15:25:54 -0500930
Joe Gregorio66f57522011-11-30 11:00:00 -0500931class TestBatch(unittest.TestCase):
932
933 def setUp(self):
934 model = JsonModel()
935 self.request1 = HttpRequest(
936 None,
937 model.response,
938 'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
939 method='POST',
940 body='{}',
941 headers={'content-type': 'application/json'})
942
943 self.request2 = HttpRequest(
944 None,
945 model.response,
946 'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
Joe Gregorio5d1171b2012-01-05 10:48:24 -0500947 method='GET',
948 body='',
Joe Gregorio66f57522011-11-30 11:00:00 -0500949 headers={'content-type': 'application/json'})
950
951
952 def test_id_to_from_content_id_header(self):
953 batch = BatchHttpRequest()
954 self.assertEquals('12', batch._header_to_id(batch._id_to_header('12')))
955
956 def test_invalid_content_id_header(self):
957 batch = BatchHttpRequest()
958 self.assertRaises(BatchError, batch._header_to_id, '[foo+x]')
959 self.assertRaises(BatchError, batch._header_to_id, 'foo+1')
960 self.assertRaises(BatchError, batch._header_to_id, '<foo>')
961
962 def test_serialize_request(self):
963 batch = BatchHttpRequest()
964 request = HttpRequest(
965 None,
966 None,
967 'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
968 method='POST',
Pat Ferate2b140222015-03-03 18:05:11 -0800969 body=u'{}',
Joe Gregorio66f57522011-11-30 11:00:00 -0500970 headers={'content-type': 'application/json'},
971 methodId=None,
972 resumable=None)
973 s = batch._serialize_request(request).splitlines()
Joe Gregorio654f4a22012-02-09 14:15:44 -0500974 self.assertEqual(EXPECTED.splitlines(), s)
Joe Gregorio66f57522011-11-30 11:00:00 -0500975
Joe Gregoriodd813822012-01-25 10:32:47 -0500976 def test_serialize_request_media_body(self):
977 batch = BatchHttpRequest()
Pat Ferate2b140222015-03-03 18:05:11 -0800978 f = open(datafile('small.png'), 'rb')
Joe Gregoriodd813822012-01-25 10:32:47 -0500979 body = f.read()
980 f.close()
981
982 request = HttpRequest(
983 None,
984 None,
985 'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
986 method='POST',
987 body=body,
988 headers={'content-type': 'application/json'},
989 methodId=None,
990 resumable=None)
Joe Gregorio654f4a22012-02-09 14:15:44 -0500991 # Just testing it shouldn't raise an exception.
Joe Gregoriodd813822012-01-25 10:32:47 -0500992 s = batch._serialize_request(request).splitlines()
993
Joe Gregorio5d1171b2012-01-05 10:48:24 -0500994 def test_serialize_request_no_body(self):
995 batch = BatchHttpRequest()
996 request = HttpRequest(
997 None,
998 None,
999 'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
1000 method='POST',
Pat Ferate2b140222015-03-03 18:05:11 -08001001 body=b'',
Joe Gregorio5d1171b2012-01-05 10:48:24 -05001002 headers={'content-type': 'application/json'},
1003 methodId=None,
1004 resumable=None)
1005 s = batch._serialize_request(request).splitlines()
Joe Gregorio654f4a22012-02-09 14:15:44 -05001006 self.assertEqual(NO_BODY_EXPECTED.splitlines(), s)
Joe Gregorio5d1171b2012-01-05 10:48:24 -05001007
Pepper Lebeck-Jobe30ecbd42015-06-12 11:45:57 -04001008 def test_serialize_get_request_no_body(self):
1009 batch = BatchHttpRequest()
1010 request = HttpRequest(
1011 None,
1012 None,
1013 'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
1014 method='GET',
1015 body=None,
1016 headers={'content-type': 'application/json'},
1017 methodId=None,
1018 resumable=None)
1019 s = batch._serialize_request(request).splitlines()
1020 self.assertEqual(NO_BODY_EXPECTED_GET.splitlines(), s)
1021
Joe Gregorio66f57522011-11-30 11:00:00 -05001022 def test_deserialize_response(self):
1023 batch = BatchHttpRequest()
1024 resp, content = batch._deserialize_response(RESPONSE)
1025
Joe Gregorio654f4a22012-02-09 14:15:44 -05001026 self.assertEqual(200, resp.status)
1027 self.assertEqual('OK', resp.reason)
1028 self.assertEqual(11, resp.version)
1029 self.assertEqual('{"answer": 42}', content)
Joe Gregorio66f57522011-11-30 11:00:00 -05001030
1031 def test_new_id(self):
1032 batch = BatchHttpRequest()
1033
1034 id_ = batch._new_id()
Joe Gregorio654f4a22012-02-09 14:15:44 -05001035 self.assertEqual('1', id_)
Joe Gregorio66f57522011-11-30 11:00:00 -05001036
1037 id_ = batch._new_id()
Joe Gregorio654f4a22012-02-09 14:15:44 -05001038 self.assertEqual('2', id_)
Joe Gregorio66f57522011-11-30 11:00:00 -05001039
1040 batch.add(self.request1, request_id='3')
1041
1042 id_ = batch._new_id()
Joe Gregorio654f4a22012-02-09 14:15:44 -05001043 self.assertEqual('4', id_)
Joe Gregorio66f57522011-11-30 11:00:00 -05001044
1045 def test_add(self):
1046 batch = BatchHttpRequest()
1047 batch.add(self.request1, request_id='1')
1048 self.assertRaises(KeyError, batch.add, self.request1, request_id='1')
1049
1050 def test_add_fail_for_resumable(self):
1051 batch = BatchHttpRequest()
1052
1053 upload = MediaFileUpload(
1054 datafile('small.png'), chunksize=500, resumable=True)
1055 self.request1.resumable = upload
1056 self.assertRaises(BatchError, batch.add, self.request1, request_id='1')
1057
Mohamed Zenadi1b5350d2015-07-30 11:52:39 +02001058 def test_execute_empty_batch_no_http(self):
1059 batch = BatchHttpRequest()
1060 ret = batch.execute()
1061 self.assertEqual(None, ret)
1062
Joe Gregorio66f57522011-11-30 11:00:00 -05001063 def test_execute(self):
Joe Gregorio66f57522011-11-30 11:00:00 -05001064 batch = BatchHttpRequest()
1065 callbacks = Callbacks()
1066
1067 batch.add(self.request1, callback=callbacks.f)
1068 batch.add(self.request2, callback=callbacks.f)
1069 http = HttpMockSequence([
1070 ({'status': '200',
1071 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'},
1072 BATCH_RESPONSE),
1073 ])
Joe Gregorio68a8cfe2012-08-03 16:17:40 -04001074 batch.execute(http=http)
Joe Gregorio654f4a22012-02-09 14:15:44 -05001075 self.assertEqual({'foo': 42}, callbacks.responses['1'])
1076 self.assertEqual(None, callbacks.exceptions['1'])
1077 self.assertEqual({'baz': 'qux'}, callbacks.responses['2'])
1078 self.assertEqual(None, callbacks.exceptions['2'])
Joe Gregorio66f57522011-11-30 11:00:00 -05001079
Joe Gregorio5d1171b2012-01-05 10:48:24 -05001080 def test_execute_request_body(self):
1081 batch = BatchHttpRequest()
1082
1083 batch.add(self.request1)
1084 batch.add(self.request2)
1085 http = HttpMockSequence([
1086 ({'status': '200',
1087 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'},
1088 'echo_request_body'),
1089 ])
1090 try:
Joe Gregorio68a8cfe2012-08-03 16:17:40 -04001091 batch.execute(http=http)
Joe Gregorio5d1171b2012-01-05 10:48:24 -05001092 self.fail('Should raise exception')
INADA Naokic1505df2014-08-20 15:19:53 +09001093 except BatchError as e:
Joe Gregorio5d1171b2012-01-05 10:48:24 -05001094 boundary, _ = e.content.split(None, 1)
1095 self.assertEqual('--', boundary[:2])
1096 parts = e.content.split(boundary)
1097 self.assertEqual(4, len(parts))
1098 self.assertEqual('', parts[0])
Craig Citro4282aa32014-06-21 00:33:39 -07001099 self.assertEqual('--', parts[3].rstrip())
Joe Gregorio5d1171b2012-01-05 10:48:24 -05001100 header = parts[1].splitlines()[1]
1101 self.assertEqual('Content-Type: application/http', header)
1102
Gabriel Garcia23174be2016-05-25 17:28:07 +02001103 def test_execute_initial_refresh_oauth2(self):
1104 batch = BatchHttpRequest()
1105 callbacks = Callbacks()
1106 cred = MockCredentials('Foo')
1107
1108 # Pretend this is a OAuth2Credentials object
1109 cred.access_token = None
1110
1111 http = HttpMockSequence([
1112 ({'status': '200',
1113 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'},
1114 BATCH_SINGLE_RESPONSE),
1115 ])
1116
1117 cred.authorize(http)
1118
1119 batch.add(self.request1, callback=callbacks.f)
1120 batch.execute(http=http)
1121
1122 self.assertEqual({'foo': 42}, callbacks.responses['1'])
1123 self.assertIsNone(callbacks.exceptions['1'])
1124
1125 self.assertEqual(1, cred._refreshed)
1126
1127 self.assertEqual(1, cred._authorized)
1128
1129 self.assertEqual(1, cred._applied)
1130
Joe Gregorio654f4a22012-02-09 14:15:44 -05001131 def test_execute_refresh_and_retry_on_401(self):
1132 batch = BatchHttpRequest()
1133 callbacks = Callbacks()
1134 cred_1 = MockCredentials('Foo')
1135 cred_2 = MockCredentials('Bar')
1136
1137 http = HttpMockSequence([
1138 ({'status': '200',
1139 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'},
1140 BATCH_RESPONSE_WITH_401),
1141 ({'status': '200',
1142 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'},
1143 BATCH_SINGLE_RESPONSE),
1144 ])
1145
1146 creds_http_1 = HttpMockSequence([])
1147 cred_1.authorize(creds_http_1)
1148
1149 creds_http_2 = HttpMockSequence([])
1150 cred_2.authorize(creds_http_2)
1151
1152 self.request1.http = creds_http_1
1153 self.request2.http = creds_http_2
1154
1155 batch.add(self.request1, callback=callbacks.f)
1156 batch.add(self.request2, callback=callbacks.f)
Joe Gregorio68a8cfe2012-08-03 16:17:40 -04001157 batch.execute(http=http)
Joe Gregorio654f4a22012-02-09 14:15:44 -05001158
1159 self.assertEqual({'foo': 42}, callbacks.responses['1'])
1160 self.assertEqual(None, callbacks.exceptions['1'])
1161 self.assertEqual({'baz': 'qux'}, callbacks.responses['2'])
1162 self.assertEqual(None, callbacks.exceptions['2'])
1163
1164 self.assertEqual(1, cred_1._refreshed)
1165 self.assertEqual(0, cred_2._refreshed)
1166
1167 self.assertEqual(1, cred_1._authorized)
1168 self.assertEqual(1, cred_2._authorized)
1169
1170 self.assertEqual(1, cred_2._applied)
1171 self.assertEqual(2, cred_1._applied)
1172
1173 def test_http_errors_passed_to_callback(self):
1174 batch = BatchHttpRequest()
1175 callbacks = Callbacks()
1176 cred_1 = MockCredentials('Foo')
1177 cred_2 = MockCredentials('Bar')
1178
1179 http = HttpMockSequence([
1180 ({'status': '200',
1181 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'},
1182 BATCH_RESPONSE_WITH_401),
1183 ({'status': '200',
1184 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'},
1185 BATCH_RESPONSE_WITH_401),
1186 ])
1187
1188 creds_http_1 = HttpMockSequence([])
1189 cred_1.authorize(creds_http_1)
1190
1191 creds_http_2 = HttpMockSequence([])
1192 cred_2.authorize(creds_http_2)
1193
1194 self.request1.http = creds_http_1
1195 self.request2.http = creds_http_2
1196
1197 batch.add(self.request1, callback=callbacks.f)
1198 batch.add(self.request2, callback=callbacks.f)
Joe Gregorio68a8cfe2012-08-03 16:17:40 -04001199 batch.execute(http=http)
Joe Gregorio654f4a22012-02-09 14:15:44 -05001200
1201 self.assertEqual(None, callbacks.responses['1'])
1202 self.assertEqual(401, callbacks.exceptions['1'].resp.status)
Joe Gregorioc752e332012-07-11 14:43:52 -04001203 self.assertEqual(
1204 'Authorization Required', callbacks.exceptions['1'].resp.reason)
Joe Gregorio654f4a22012-02-09 14:15:44 -05001205 self.assertEqual({u'baz': u'qux'}, callbacks.responses['2'])
1206 self.assertEqual(None, callbacks.exceptions['2'])
1207
Joe Gregorio66f57522011-11-30 11:00:00 -05001208 def test_execute_global_callback(self):
Joe Gregorio66f57522011-11-30 11:00:00 -05001209 callbacks = Callbacks()
1210 batch = BatchHttpRequest(callback=callbacks.f)
1211
1212 batch.add(self.request1)
1213 batch.add(self.request2)
1214 http = HttpMockSequence([
1215 ({'status': '200',
1216 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'},
1217 BATCH_RESPONSE),
1218 ])
Joe Gregorio68a8cfe2012-08-03 16:17:40 -04001219 batch.execute(http=http)
Joe Gregorio654f4a22012-02-09 14:15:44 -05001220 self.assertEqual({'foo': 42}, callbacks.responses['1'])
1221 self.assertEqual({'baz': 'qux'}, callbacks.responses['2'])
Joe Gregoriod0bd3882011-11-22 09:49:47 -05001222
Joe Gregorio20b54fb2012-07-26 09:59:35 -04001223 def test_execute_batch_http_error(self):
Joe Gregorio3fb93672012-07-25 11:31:11 -04001224 callbacks = Callbacks()
1225 batch = BatchHttpRequest(callback=callbacks.f)
1226
1227 batch.add(self.request1)
1228 batch.add(self.request2)
1229 http = HttpMockSequence([
1230 ({'status': '200',
1231 'content-type': 'multipart/mixed; boundary="batch_foobarbaz"'},
1232 BATCH_ERROR_RESPONSE),
1233 ])
Joe Gregorio68a8cfe2012-08-03 16:17:40 -04001234 batch.execute(http=http)
Joe Gregorio3fb93672012-07-25 11:31:11 -04001235 self.assertEqual({'foo': 42}, callbacks.responses['1'])
1236 expected = ('<HttpError 403 when requesting '
1237 'https://www.googleapis.com/someapi/v1/collection/?foo=bar returned '
1238 '"Access Not Configured">')
1239 self.assertEqual(expected, str(callbacks.exceptions['2']))
Ali Afshar6f11ea12012-02-07 10:32:14 -05001240
Joe Gregorio5c120db2012-08-23 09:13:55 -04001241
Joe Gregorioba5c7902012-08-03 12:48:16 -04001242class TestRequestUriTooLong(unittest.TestCase):
1243
1244 def test_turn_get_into_post(self):
1245
1246 def _postproc(resp, content):
1247 return content
1248
1249 http = HttpMockSequence([
1250 ({'status': '200'},
1251 'echo_request_body'),
1252 ({'status': '200'},
1253 'echo_request_headers'),
1254 ])
1255
1256 # Send a long query parameter.
1257 query = {
1258 'q': 'a' * MAX_URI_LENGTH + '?&'
1259 }
1260 req = HttpRequest(
1261 http,
1262 _postproc,
Pat Ferated5b61bd2015-03-03 16:04:11 -08001263 'http://example.com?' + urlencode(query),
Joe Gregorioba5c7902012-08-03 12:48:16 -04001264 method='GET',
1265 body=None,
1266 headers={},
1267 methodId='foo',
1268 resumable=None)
1269
1270 # Query parameters should be sent in the body.
1271 response = req.execute()
INADA Naoki09157612015-03-25 01:51:03 +09001272 self.assertEqual(b'q=' + b'a' * MAX_URI_LENGTH + b'%3F%26', response)
Joe Gregorioba5c7902012-08-03 12:48:16 -04001273
1274 # Extra headers should be set.
1275 response = req.execute()
1276 self.assertEqual('GET', response['x-http-method-override'])
1277 self.assertEqual(str(MAX_URI_LENGTH + 8), response['content-length'])
1278 self.assertEqual(
1279 'application/x-www-form-urlencoded', response['content-type'])
1280
Joe Gregorio5c120db2012-08-23 09:13:55 -04001281
1282class TestStreamSlice(unittest.TestCase):
1283 """Test _StreamSlice."""
1284
1285 def setUp(self):
Pat Ferate2b140222015-03-03 18:05:11 -08001286 self.stream = BytesIO(b'0123456789')
Joe Gregorio5c120db2012-08-23 09:13:55 -04001287
1288 def test_read(self):
1289 s = _StreamSlice(self.stream, 0, 4)
Pat Ferate2b140222015-03-03 18:05:11 -08001290 self.assertEqual(b'', s.read(0))
1291 self.assertEqual(b'0', s.read(1))
1292 self.assertEqual(b'123', s.read())
Joe Gregorio5c120db2012-08-23 09:13:55 -04001293
1294 def test_read_too_much(self):
1295 s = _StreamSlice(self.stream, 1, 4)
Pat Ferate2b140222015-03-03 18:05:11 -08001296 self.assertEqual(b'1234', s.read(6))
Joe Gregorio5c120db2012-08-23 09:13:55 -04001297
1298 def test_read_all(self):
1299 s = _StreamSlice(self.stream, 2, 1)
Pat Ferate2b140222015-03-03 18:05:11 -08001300 self.assertEqual(b'2', s.read(-1))
Joe Gregorio5c120db2012-08-23 09:13:55 -04001301
Ali Afshar164f37e2013-01-07 14:05:45 -08001302
1303class TestResponseCallback(unittest.TestCase):
1304 """Test adding callbacks to responses."""
1305
1306 def test_ensure_response_callback(self):
1307 m = JsonModel()
1308 request = HttpRequest(
1309 None,
1310 m.response,
1311 'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
1312 method='POST',
1313 body='{}',
1314 headers={'content-type': 'application/json'})
1315 h = HttpMockSequence([ ({'status': 200}, '{}')])
1316 responses = []
1317 def _on_response(resp, responses=responses):
1318 responses.append(resp)
1319 request.add_response_callback(_on_response)
1320 request.execute(http=h)
1321 self.assertEqual(1, len(responses))
1322
1323
Craig Gurnik8e55b762015-01-20 15:00:10 -05001324class TestHttpMock(unittest.TestCase):
1325 def test_default_response_headers(self):
1326 http = HttpMock(datafile('zoo.json'))
1327 resp, content = http.request("http://example.com")
1328 self.assertEqual(resp.status, 200)
1329
Alan Briolat26b01002015-08-14 00:13:57 +01001330 def test_error_response(self):
1331 http = HttpMock(datafile('bad_request.json'), {'status': '400'})
1332 model = JsonModel()
1333 request = HttpRequest(
1334 http,
1335 model.response,
1336 'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
1337 method='GET',
1338 headers={})
1339 self.assertRaises(HttpError, request.execute)
1340
Craig Gurnik8e55b762015-01-20 15:00:10 -05001341
Joe Gregorio6bcbcea2011-03-10 15:26:05 -05001342if __name__ == '__main__':
Joe Gregorio9086bd32013-06-14 16:32:05 -04001343 logging.getLogger().setLevel(logging.ERROR)
Joe Gregorio6bcbcea2011-03-10 15:26:05 -05001344 unittest.main()