Fix resumable upload protocol. Reviewed in http://codereview.appspot.com/5572075/
diff --git a/apiclient/discovery.py b/apiclient/discovery.py
index a2c81c6..a0ca930 100644
--- a/apiclient/discovery.py
+++ b/apiclient/discovery.py
@@ -461,7 +461,7 @@
multipart_boundary = ''
if media_filename:
- # Convert a simple filename into a MediaUpload object.
+ # Ensure we end up with a valid MediaUpload object.
if isinstance(media_filename, basestring):
(media_mime_type, encoding) = mimetypes.guess_type(media_filename)
if media_mime_type is None:
@@ -474,9 +474,6 @@
else:
raise TypeError('media_filename must be str or MediaUpload.')
- if media_upload.resumable():
- resumable = media_upload
-
# Check the maxSize
if maxSize > 0 and media_upload.size() > maxSize:
raise MediaUploadSizeError("Media larger than: %s" % maxSize)
@@ -488,69 +485,39 @@
expanded_url = uritemplate.expand(mediaPathUrl, params)
url = urlparse.urljoin(self._baseUrl, expanded_url + query)
- if body is None:
- # This is a simple media upload
- headers['content-type'] = media_upload.mimetype()
- expanded_url = uritemplate.expand(mediaResumablePathUrl, params)
- if not media_upload.resumable():
- body = media_upload.getbytes(0, media_upload.size())
+ if media_upload.resumable():
+ # This is all we need to do for resumable, if the body exists it gets
+ # sent in the first request, otherwise an empty body is sent.
+ resumable = media_upload
else:
- # This is a multipart/related upload.
- msgRoot = MIMEMultipart('related')
- # msgRoot should not write out it's own headers
- setattr(msgRoot, '_write_headers', lambda self: None)
-
- # attach the body as one part
- msg = MIMENonMultipart(*headers['content-type'].split('/'))
- msg.set_payload(body)
- msgRoot.attach(msg)
-
- # attach the media as the second part
- msg = MIMENonMultipart(*media_upload.mimetype().split('/'))
- msg['Content-Transfer-Encoding'] = 'binary'
-
- if media_upload.resumable():
- # This is a multipart resumable upload, where a multipart payload
- # looks like this:
- #
- # --===============1678050750164843052==
- # Content-Type: application/json
- # MIME-Version: 1.0
- #
- # {'foo': 'bar'}
- # --===============1678050750164843052==
- # Content-Type: image/png
- # MIME-Version: 1.0
- # Content-Transfer-Encoding: binary
- #
- # <BINARY STUFF>
- # --===============1678050750164843052==--
- #
- # In the case of resumable multipart media uploads, the <BINARY
- # STUFF> is large and will be spread across multiple PUTs. What we
- # do here is compose the multipart message with a random payload in
- # place of <BINARY STUFF> and then split the resulting content into
- # two pieces, text before <BINARY STUFF> and text after <BINARY
- # STUFF>. The text after <BINARY STUFF> is the multipart boundary.
- # In apiclient.http the HttpRequest will send the text before
- # <BINARY STUFF>, then send the actual binary media in chunks, and
- # then will send the multipart delimeter.
-
- payload = hex(random.getrandbits(300))
- msg.set_payload(payload)
- msgRoot.attach(msg)
- body = msgRoot.as_string()
- body, _ = body.split(payload)
- resumable = media_upload
+ # A non-resumable upload
+ if body is None:
+ # This is a simple media upload
+ headers['content-type'] = media_upload.mimetype()
+ body = media_upload.getbytes(0, media_upload.size())
else:
+ # This is a multipart/related upload.
+ msgRoot = MIMEMultipart('related')
+ # msgRoot should not write out it's own headers
+ setattr(msgRoot, '_write_headers', lambda self: None)
+
+ # attach the body as one part
+ msg = MIMENonMultipart(*headers['content-type'].split('/'))
+ msg.set_payload(body)
+ msgRoot.attach(msg)
+
+ # attach the media as the second part
+ msg = MIMENonMultipart(*media_upload.mimetype().split('/'))
+ msg['Content-Transfer-Encoding'] = 'binary'
+
payload = media_upload.getbytes(0, media_upload.size())
msg.set_payload(payload)
msgRoot.attach(msg)
body = msgRoot.as_string()
- multipart_boundary = msgRoot.get_boundary()
- headers['content-type'] = ('multipart/related; '
- 'boundary="%s"') % multipart_boundary
+ multipart_boundary = msgRoot.get_boundary()
+ headers['content-type'] = ('multipart/related; '
+ 'boundary="%s"') % multipart_boundary
logging.info('URL being requested: %s' % url)
return self._requestBuilder(self._http,
diff --git a/apiclient/http.py b/apiclient/http.py
index c8a8cd4..8e15e11 100644
--- a/apiclient/http.py
+++ b/apiclient/http.py
@@ -151,7 +151,7 @@
media_body=media).execute()
"""
- def __init__(self, filename, mimetype=None, chunksize=150000, resumable=False):
+ def __init__(self, filename, mimetype=None, chunksize=256*1024, resumable=False):
"""Constructor.
Args:
@@ -252,11 +252,8 @@
major, minor, params = mimeparse.parse_mime_type(
headers.get('content-type', 'application/json'))
- # Terminating multipart boundary get a trailing '--' appended.
- self.multipart_boundary = params.get('boundary', '').strip('"') + '--'
-
- # If this was a multipart resumable, the size of the non-media part.
- self.multipart_size = 0
+ # The size of the non-media part of the request.
+ self.body_size = len(self.body or '')
# The resumable URI to send chunks to.
self.resumable_uri = None
@@ -264,18 +261,6 @@
# The bytes that have been uploaded.
self.resumable_progress = 0
- self.total_size = 0
-
- if resumable is not None:
- if self.body is not None:
- self.multipart_size = len(self.body)
- else:
- self.multipart_size = 0
- self.total_size = (
- self.resumable.size() +
- self.multipart_size +
- len(self.multipart_boundary))
-
def execute(self, http=None):
"""Execute the request.
@@ -340,28 +325,23 @@
start_headers = copy.copy(self.headers)
start_headers['X-Upload-Content-Type'] = self.resumable.mimetype()
start_headers['X-Upload-Content-Length'] = str(self.resumable.size())
- start_headers['Content-Length'] = '0'
+ start_headers['content-length'] = str(self.body_size)
+
resp, content = http.request(self.uri, self.method,
- body="",
+ body=self.body,
headers=start_headers)
if resp.status == 200 and 'location' in resp:
self.resumable_uri = resp['location']
else:
raise ResumableUploadError("Failed to retrieve starting URI.")
- if self.body:
- begin = 0
- data = self.body
- else:
- begin = self.resumable_progress - self.multipart_size
- data = self.resumable.getbytes(begin, self.resumable.chunksize())
- # Tack on the multipart/related boundary if we are at the end of the file.
- if begin + self.resumable.chunksize() >= self.resumable.size():
- data += self.multipart_boundary
+ data = self.resumable.getbytes(self.resumable_progress,
+ self.resumable.chunksize())
+
headers = {
'Content-Range': 'bytes %d-%d/%d' % (
self.resumable_progress, self.resumable_progress + len(data) - 1,
- self.total_size),
+ self.resumable.size()),
}
resp, content = http.request(self.resumable_uri, 'PUT',
body=data,
@@ -371,14 +351,13 @@
elif resp.status == 308:
# A "308 Resume Incomplete" indicates we are not done.
self.resumable_progress = int(resp['range'].split('-')[1]) + 1
- if self.resumable_progress >= self.multipart_size:
- self.body = None
if 'location' in resp:
self.resumable_uri = resp['location']
else:
raise HttpError(resp, content, self.uri)
- return MediaUploadProgress(self.resumable_progress, self.total_size), None
+ return (MediaUploadProgress(self.resumable_progress, self.resumable.size()),
+ None)
def to_json(self):
"""Returns a JSON representation of the HttpRequest."""