1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """Client for discovery based APIs
16
17 A client library for Google's discovery based APIs.
18 """
19
20 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
21 __all__ = [
22 'build',
23 'build_from_document'
24 'fix_method_name',
25 'key2param'
26 ]
27
28 import copy
29 import httplib2
30 import logging
31 import os
32 import re
33 import uritemplate
34 import urllib
35 import urlparse
36 import mimeparse
37 import mimetypes
38
39 try:
40 from urlparse import parse_qsl
41 except ImportError:
42 from cgi import parse_qsl
43
44 from apiclient.errors import HttpError
45 from apiclient.errors import InvalidJsonError
46 from apiclient.errors import MediaUploadSizeError
47 from apiclient.errors import UnacceptableMimeTypeError
48 from apiclient.errors import UnknownApiNameOrVersion
49 from apiclient.errors import UnknownFileType
50 from apiclient.http import HttpRequest
51 from apiclient.http import MediaFileUpload
52 from apiclient.http import MediaUpload
53 from apiclient.model import JsonModel
54 from apiclient.model import MediaModel
55 from apiclient.model import RawModel
56 from apiclient.schema import Schemas
57 from email.mime.multipart import MIMEMultipart
58 from email.mime.nonmultipart import MIMENonMultipart
59 from oauth2client.util import positional
60 from oauth2client.anyjson import simplejson
61
62
63 httplib2.RETRIES = 1
64
65 logger = logging.getLogger(__name__)
66
67 URITEMPLATE = re.compile('{[^}]*}')
68 VARNAME = re.compile('[a-zA-Z0-9_-]+')
69 DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
70 '{api}/{apiVersion}/rest')
71 DEFAULT_METHOD_DOC = 'A description of how to use this function'
72
73
74 STACK_QUERY_PARAMETERS = ['trace', 'pp', 'userip', 'strict']
75
76
77 RESERVED_WORDS = ['and', 'assert', 'break', 'class', 'continue', 'def', 'del',
78 'elif', 'else', 'except', 'exec', 'finally', 'for', 'from',
79 'global', 'if', 'import', 'in', 'is', 'lambda', 'not', 'or',
80 'pass', 'print', 'raise', 'return', 'try', 'while', 'body']
84 """Fix method names to avoid reserved word conflicts.
85
86 Args:
87 name: string, method name.
88
89 Returns:
90 The name with a '_' prefixed if the name is a reserved word.
91 """
92 if name in RESERVED_WORDS:
93 return name + '_'
94 else:
95 return name
96
99 """Adds a query parameter to a url.
100
101 Replaces the current value if it already exists in the URL.
102
103 Args:
104 url: string, url to add the query parameter to.
105 name: string, query parameter name.
106 value: string, query parameter value.
107
108 Returns:
109 Updated query parameter. Does not update the url if value is None.
110 """
111 if value is None:
112 return url
113 else:
114 parsed = list(urlparse.urlparse(url))
115 q = dict(parse_qsl(parsed[4]))
116 q[name] = value
117 parsed[4] = urllib.urlencode(q)
118 return urlparse.urlunparse(parsed)
119
122 """Converts key names into parameter names.
123
124 For example, converting "max-results" -> "max_results"
125
126 Args:
127 key: string, the method key name.
128
129 Returns:
130 A safe method name based on the key name.
131 """
132 result = []
133 key = list(key)
134 if not key[0].isalpha():
135 result.append('x')
136 for c in key:
137 if c.isalnum():
138 result.append(c)
139 else:
140 result.append('_')
141
142 return ''.join(result)
143
144
145 @positional(2)
146 -def build(serviceName,
147 version,
148 http=None,
149 discoveryServiceUrl=DISCOVERY_URI,
150 developerKey=None,
151 model=None,
152 requestBuilder=HttpRequest):
153 """Construct a Resource for interacting with an API.
154
155 Construct a Resource object for interacting with an API. The serviceName and
156 version are the names from the Discovery service.
157
158 Args:
159 serviceName: string, name of the service.
160 version: string, the version of the service.
161 http: httplib2.Http, An instance of httplib2.Http or something that acts
162 like it that HTTP requests will be made through.
163 discoveryServiceUrl: string, a URI Template that points to the location of
164 the discovery service. It should have two parameters {api} and
165 {apiVersion} that when filled in produce an absolute URI to the discovery
166 document for that service.
167 developerKey: string, key obtained from
168 https://code.google.com/apis/console.
169 model: apiclient.Model, converts to and from the wire format.
170 requestBuilder: apiclient.http.HttpRequest, encapsulator for an HTTP
171 request.
172
173 Returns:
174 A Resource object with methods for interacting with the service.
175 """
176 params = {
177 'api': serviceName,
178 'apiVersion': version
179 }
180
181 if http is None:
182 http = httplib2.Http()
183
184 requested_url = uritemplate.expand(discoveryServiceUrl, params)
185
186
187
188
189
190 if 'REMOTE_ADDR' in os.environ:
191 requested_url = _add_query_parameter(requested_url, 'userIp',
192 os.environ['REMOTE_ADDR'])
193 logger.info('URL being requested: %s' % requested_url)
194
195 resp, content = http.request(requested_url)
196
197 if resp.status == 404:
198 raise UnknownApiNameOrVersion("name: %s version: %s" % (serviceName,
199 version))
200 if resp.status >= 400:
201 raise HttpError(resp, content, uri=requested_url)
202
203 try:
204 service = simplejson.loads(content)
205 except ValueError, e:
206 logger.error('Failed to parse as JSON: ' + content)
207 raise InvalidJsonError()
208
209 return build_from_document(content, base=discoveryServiceUrl, http=http,
210 developerKey=developerKey, model=model, requestBuilder=requestBuilder)
211
212
213 @positional(1)
214 -def build_from_document(
215 service,
216 base=None,
217 future=None,
218 http=None,
219 developerKey=None,
220 model=None,
221 requestBuilder=HttpRequest):
222 """Create a Resource for interacting with an API.
223
224 Same as `build()`, but constructs the Resource object from a discovery
225 document that is it given, as opposed to retrieving one over HTTP.
226
227 Args:
228 service: string or object, the JSON discovery document describing the API.
229 The value passed in may either be the JSON string or the deserialized
230 JSON.
231 base: string, base URI for all HTTP requests, usually the discovery URI.
232 This parameter is no longer used as rootUrl and servicePath are included
233 within the discovery document. (deprecated)
234 future: string, discovery document with future capabilities (deprecated).
235 http: httplib2.Http, An instance of httplib2.Http or something that acts
236 like it that HTTP requests will be made through.
237 developerKey: string, Key for controlling API usage, generated
238 from the API Console.
239 model: Model class instance that serializes and de-serializes requests and
240 responses.
241 requestBuilder: Takes an http request and packages it up to be executed.
242
243 Returns:
244 A Resource object with methods for interacting with the service.
245 """
246
247
248 future = {}
249
250 if isinstance(service, basestring):
251 service = simplejson.loads(service)
252 base = urlparse.urljoin(service['rootUrl'], service['servicePath'])
253 schema = Schemas(service)
254
255 if model is None:
256 features = service.get('features', [])
257 model = JsonModel('dataWrapper' in features)
258 return Resource(http=http, baseUrl=base, model=model,
259 developerKey=developerKey, requestBuilder=requestBuilder,
260 resourceDesc=service, rootDesc=service, schema=schema)
261
262
263 -def _cast(value, schema_type):
264 """Convert value to a string based on JSON Schema type.
265
266 See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on
267 JSON Schema.
268
269 Args:
270 value: any, the value to convert
271 schema_type: string, the type that value should be interpreted as
272
273 Returns:
274 A string representation of 'value' based on the schema_type.
275 """
276 if schema_type == 'string':
277 if type(value) == type('') or type(value) == type(u''):
278 return value
279 else:
280 return str(value)
281 elif schema_type == 'integer':
282 return str(int(value))
283 elif schema_type == 'number':
284 return str(float(value))
285 elif schema_type == 'boolean':
286 return str(bool(value)).lower()
287 else:
288 if type(value) == type('') or type(value) == type(u''):
289 return value
290 else:
291 return str(value)
292
293
294 MULTIPLIERS = {
295 "KB": 2 ** 10,
296 "MB": 2 ** 20,
297 "GB": 2 ** 30,
298 "TB": 2 ** 40,
299 }
319
320
321 -def createMethod(methodName, methodDesc, rootDesc, schema):
322 """Creates a method for attaching to a Resource.
323
324 Args:
325 methodName: string, name of the method to use.
326 methodDesc: object, fragment of deserialized discovery document that
327 describes the method.
328 rootDesc: object, the entire deserialized discovery document.
329 schema: object, mapping of schema names to schema descriptions.
330 """
331 methodName = fix_method_name(methodName)
332 pathUrl = methodDesc['path']
333 httpMethod = methodDesc['httpMethod']
334 methodId = methodDesc['id']
335
336 mediaPathUrl = None
337 accept = []
338 maxSize = 0
339 if 'mediaUpload' in methodDesc:
340 mediaUpload = methodDesc['mediaUpload']
341 mediaPathUrl = (rootDesc['rootUrl'] + 'upload/' + rootDesc['servicePath']
342 + pathUrl)
343 accept = mediaUpload['accept']
344 maxSize = _media_size_to_long(mediaUpload.get('maxSize', ''))
345
346 if 'parameters' not in methodDesc:
347 methodDesc['parameters'] = {}
348
349
350 for name, desc in rootDesc.get('parameters', {}).iteritems():
351 methodDesc['parameters'][name] = desc
352
353
354 for name in STACK_QUERY_PARAMETERS:
355 methodDesc['parameters'][name] = {
356 'type': 'string',
357 'location': 'query'
358 }
359
360 if httpMethod in ['PUT', 'POST', 'PATCH'] and 'request' in methodDesc:
361 methodDesc['parameters']['body'] = {
362 'description': 'The request body.',
363 'type': 'object',
364 'required': True,
365 }
366 if 'request' in methodDesc:
367 methodDesc['parameters']['body'].update(methodDesc['request'])
368 else:
369 methodDesc['parameters']['body']['type'] = 'object'
370 if 'mediaUpload' in methodDesc:
371 methodDesc['parameters']['media_body'] = {
372 'description':
373 'The filename of the media request body, or an instance of a '
374 'MediaUpload object.',
375 'type': 'string',
376 'required': False,
377 }
378 if 'body' in methodDesc['parameters']:
379 methodDesc['parameters']['body']['required'] = False
380
381 argmap = {}
382 required_params = []
383 repeated_params = []
384 pattern_params = {}
385 query_params = []
386 path_params = {}
387 param_type = {}
388 enum_params = {}
389
390 if 'parameters' in methodDesc:
391 for arg, desc in methodDesc['parameters'].iteritems():
392 param = key2param(arg)
393 argmap[param] = arg
394
395 if desc.get('pattern', ''):
396 pattern_params[param] = desc['pattern']
397 if desc.get('enum', ''):
398 enum_params[param] = desc['enum']
399 if desc.get('required', False):
400 required_params.append(param)
401 if desc.get('repeated', False):
402 repeated_params.append(param)
403 if desc.get('location') == 'query':
404 query_params.append(param)
405 if desc.get('location') == 'path':
406 path_params[param] = param
407 param_type[param] = desc.get('type', 'string')
408
409 for match in URITEMPLATE.finditer(pathUrl):
410 for namematch in VARNAME.finditer(match.group(0)):
411 name = key2param(namematch.group(0))
412 path_params[name] = name
413 if name in query_params:
414 query_params.remove(name)
415
416 def method(self, **kwargs):
417
418
419 for name in kwargs.iterkeys():
420 if name not in argmap:
421 raise TypeError('Got an unexpected keyword argument "%s"' % name)
422
423
424 keys = kwargs.keys()
425 for name in keys:
426 if kwargs[name] is None:
427 del kwargs[name]
428
429 for name in required_params:
430 if name not in kwargs:
431 raise TypeError('Missing required parameter "%s"' % name)
432
433 for name, regex in pattern_params.iteritems():
434 if name in kwargs:
435 if isinstance(kwargs[name], basestring):
436 pvalues = [kwargs[name]]
437 else:
438 pvalues = kwargs[name]
439 for pvalue in pvalues:
440 if re.match(regex, pvalue) is None:
441 raise TypeError(
442 'Parameter "%s" value "%s" does not match the pattern "%s"' %
443 (name, pvalue, regex))
444
445 for name, enums in enum_params.iteritems():
446 if name in kwargs:
447
448
449
450 if (name in repeated_params and
451 not isinstance(kwargs[name], basestring)):
452 values = kwargs[name]
453 else:
454 values = [kwargs[name]]
455 for value in values:
456 if value not in enums:
457 raise TypeError(
458 'Parameter "%s" value "%s" is not an allowed value in "%s"' %
459 (name, value, str(enums)))
460
461 actual_query_params = {}
462 actual_path_params = {}
463 for key, value in kwargs.iteritems():
464 to_type = param_type.get(key, 'string')
465
466 if key in repeated_params and type(value) == type([]):
467 cast_value = [_cast(x, to_type) for x in value]
468 else:
469 cast_value = _cast(value, to_type)
470 if key in query_params:
471 actual_query_params[argmap[key]] = cast_value
472 if key in path_params:
473 actual_path_params[argmap[key]] = cast_value
474 body_value = kwargs.get('body', None)
475 media_filename = kwargs.get('media_body', None)
476
477 if self._developerKey:
478 actual_query_params['key'] = self._developerKey
479
480 model = self._model
481 if methodName.endswith('_media'):
482 model = MediaModel()
483 elif 'response' not in methodDesc:
484 model = RawModel()
485
486 headers = {}
487 headers, params, query, body = model.request(headers,
488 actual_path_params, actual_query_params, body_value)
489
490 expanded_url = uritemplate.expand(pathUrl, params)
491 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
492
493 resumable = None
494 multipart_boundary = ''
495
496 if media_filename:
497
498 if isinstance(media_filename, basestring):
499 (media_mime_type, encoding) = mimetypes.guess_type(media_filename)
500 if media_mime_type is None:
501 raise UnknownFileType(media_filename)
502 if not mimeparse.best_match([media_mime_type], ','.join(accept)):
503 raise UnacceptableMimeTypeError(media_mime_type)
504 media_upload = MediaFileUpload(media_filename,
505 mimetype=media_mime_type)
506 elif isinstance(media_filename, MediaUpload):
507 media_upload = media_filename
508 else:
509 raise TypeError('media_filename must be str or MediaUpload.')
510
511
512 if maxSize > 0 and media_upload.size() > maxSize:
513 raise MediaUploadSizeError("Media larger than: %s" % maxSize)
514
515
516 expanded_url = uritemplate.expand(mediaPathUrl, params)
517 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
518 if media_upload.resumable():
519 url = _add_query_parameter(url, 'uploadType', 'resumable')
520
521 if media_upload.resumable():
522
523
524 resumable = media_upload
525 else:
526
527 if body is None:
528
529 headers['content-type'] = media_upload.mimetype()
530 body = media_upload.getbytes(0, media_upload.size())
531 url = _add_query_parameter(url, 'uploadType', 'media')
532 else:
533
534 msgRoot = MIMEMultipart('related')
535
536 setattr(msgRoot, '_write_headers', lambda self: None)
537
538
539 msg = MIMENonMultipart(*headers['content-type'].split('/'))
540 msg.set_payload(body)
541 msgRoot.attach(msg)
542
543
544 msg = MIMENonMultipart(*media_upload.mimetype().split('/'))
545 msg['Content-Transfer-Encoding'] = 'binary'
546
547 payload = media_upload.getbytes(0, media_upload.size())
548 msg.set_payload(payload)
549 msgRoot.attach(msg)
550 body = msgRoot.as_string()
551
552 multipart_boundary = msgRoot.get_boundary()
553 headers['content-type'] = ('multipart/related; '
554 'boundary="%s"') % multipart_boundary
555 url = _add_query_parameter(url, 'uploadType', 'multipart')
556
557 logger.info('URL being requested: %s' % url)
558 return self._requestBuilder(self._http,
559 model.response,
560 url,
561 method=httpMethod,
562 body=body,
563 headers=headers,
564 methodId=methodId,
565 resumable=resumable)
566
567 docs = [methodDesc.get('description', DEFAULT_METHOD_DOC), '\n\n']
568 if len(argmap) > 0:
569 docs.append('Args:\n')
570
571
572 skip_parameters = rootDesc.get('parameters', {}).keys()
573 skip_parameters.extend(STACK_QUERY_PARAMETERS)
574
575 all_args = argmap.keys()
576 args_ordered = [key2param(s) for s in methodDesc.get('parameterOrder', [])]
577
578
579 if 'body' in all_args:
580 args_ordered.append('body')
581
582 for name in all_args:
583 if name not in args_ordered:
584 args_ordered.append(name)
585
586 for arg in args_ordered:
587 if arg in skip_parameters:
588 continue
589
590 repeated = ''
591 if arg in repeated_params:
592 repeated = ' (repeated)'
593 required = ''
594 if arg in required_params:
595 required = ' (required)'
596 paramdesc = methodDesc['parameters'][argmap[arg]]
597 paramdoc = paramdesc.get('description', 'A parameter')
598 if '$ref' in paramdesc:
599 docs.append(
600 (' %s: object, %s%s%s\n The object takes the'
601 ' form of:\n\n%s\n\n') % (arg, paramdoc, required, repeated,
602 schema.prettyPrintByName(paramdesc['$ref'])))
603 else:
604 paramtype = paramdesc.get('type', 'string')
605 docs.append(' %s: %s, %s%s%s\n' % (arg, paramtype, paramdoc, required,
606 repeated))
607 enum = paramdesc.get('enum', [])
608 enumDesc = paramdesc.get('enumDescriptions', [])
609 if enum and enumDesc:
610 docs.append(' Allowed values\n')
611 for (name, desc) in zip(enum, enumDesc):
612 docs.append(' %s - %s\n' % (name, desc))
613 if 'response' in methodDesc:
614 if methodName.endswith('_media'):
615 docs.append('\nReturns:\n The media object as a string.\n\n ')
616 else:
617 docs.append('\nReturns:\n An object of the form:\n\n ')
618 docs.append(schema.prettyPrintSchema(methodDesc['response']))
619
620 setattr(method, '__doc__', ''.join(docs))
621 return (methodName, method)
622
625 """Creates any _next methods for attaching to a Resource.
626
627 The _next methods allow for easy iteration through list() responses.
628
629 Args:
630 methodName: string, name of the method to use.
631 """
632 methodName = fix_method_name(methodName)
633
634 def methodNext(self, previous_request, previous_response):
635 """Retrieves the next page of results.
636
637 Args:
638 previous_request: The request for the previous page. (required)
639 previous_response: The response from the request for the previous page. (required)
640
641 Returns:
642 A request object that you can call 'execute()' on to request the next
643 page. Returns None if there are no more items in the collection.
644 """
645
646
647
648 if 'nextPageToken' not in previous_response:
649 return None
650
651 request = copy.copy(previous_request)
652
653 pageToken = previous_response['nextPageToken']
654 parsed = list(urlparse.urlparse(request.uri))
655 q = parse_qsl(parsed[4])
656
657
658 newq = [(key, value) for (key, value) in q if key != 'pageToken']
659 newq.append(('pageToken', pageToken))
660 parsed[4] = urllib.urlencode(newq)
661 uri = urlparse.urlunparse(parsed)
662
663 request.uri = uri
664
665 logger.info('URL being requested: %s' % uri)
666
667 return request
668
669 return (methodName, methodNext)
670
673 """A class for interacting with a resource."""
674
675 - def __init__(self, http, baseUrl, model, requestBuilder, developerKey,
676 resourceDesc, rootDesc, schema):
677 """Build a Resource from the API description.
678
679 Args:
680 http: httplib2.Http, Object to make http requests with.
681 baseUrl: string, base URL for the API. All requests are relative to this
682 URI.
683 model: apiclient.Model, converts to and from the wire format.
684 requestBuilder: class or callable that instantiates an
685 apiclient.HttpRequest object.
686 developerKey: string, key obtained from
687 https://code.google.com/apis/console
688 resourceDesc: object, section of deserialized discovery document that
689 describes a resource. Note that the top level discovery document
690 is considered a resource.
691 rootDesc: object, the entire deserialized discovery document.
692 schema: object, mapping of schema names to schema descriptions.
693 """
694 self._dynamic_attrs = []
695
696 self._http = http
697 self._baseUrl = baseUrl
698 self._model = model
699 self._developerKey = developerKey
700 self._requestBuilder = requestBuilder
701 self._resourceDesc = resourceDesc
702 self._rootDesc = rootDesc
703 self._schema = schema
704
705 self._set_service_methods()
706
708 """Sets an instance attribute and tracks it in a list of dynamic attributes.
709
710 Args:
711 attr_name: string; The name of the attribute to be set
712 value: The value being set on the object and tracked in the dynamic cache.
713 """
714 self._dynamic_attrs.append(attr_name)
715 self.__dict__[attr_name] = value
716
718 """Trim the state down to something that can be pickled.
719
720 Uses the fact that the instance variable _dynamic_attrs holds attrs that
721 will be wiped and restored on pickle serialization.
722 """
723 state_dict = copy.copy(self.__dict__)
724 for dynamic_attr in self._dynamic_attrs:
725 del state_dict[dynamic_attr]
726 del state_dict['_dynamic_attrs']
727 return state_dict
728
730 """Reconstitute the state of the object from being pickled.
731
732 Uses the fact that the instance variable _dynamic_attrs holds attrs that
733 will be wiped and restored on pickle serialization.
734 """
735 self.__dict__.update(state)
736 self._dynamic_attrs = []
737 self._set_service_methods()
738
743
745
746 if 'methods' in resourceDesc:
747 for methodName, methodDesc in resourceDesc['methods'].iteritems():
748 fixedMethodName, method = createMethod(
749 methodName, methodDesc, rootDesc, schema)
750 self._set_dynamic_attr(fixedMethodName,
751 method.__get__(self, self.__class__))
752
753
754 if methodDesc.get('supportsMediaDownload', False):
755 fixedMethodName, method = createMethod(
756 methodName + '_media', methodDesc, rootDesc, schema)
757 self._set_dynamic_attr(fixedMethodName,
758 method.__get__(self, self.__class__))
759
761
762 if 'resources' in resourceDesc:
763
764 def createResourceMethod(methodName, methodDesc):
765 """Create a method on the Resource to access a nested Resource.
766
767 Args:
768 methodName: string, name of the method to use.
769 methodDesc: object, fragment of deserialized discovery document that
770 describes the method.
771 """
772 methodName = fix_method_name(methodName)
773
774 def methodResource(self):
775 return Resource(http=self._http, baseUrl=self._baseUrl,
776 model=self._model, developerKey=self._developerKey,
777 requestBuilder=self._requestBuilder,
778 resourceDesc=methodDesc, rootDesc=rootDesc,
779 schema=schema)
780
781 setattr(methodResource, '__doc__', 'A collection resource.')
782 setattr(methodResource, '__is_resource__', True)
783
784 return (methodName, methodResource)
785
786 for methodName, methodDesc in resourceDesc['resources'].iteritems():
787 fixedMethodName, method = createResourceMethod(methodName, methodDesc)
788 self._set_dynamic_attr(fixedMethodName,
789 method.__get__(self, self.__class__))
790
792
793
794
795 if 'methods' in resourceDesc:
796 for methodName, methodDesc in resourceDesc['methods'].iteritems():
797 if 'response' in methodDesc:
798 responseSchema = methodDesc['response']
799 if '$ref' in responseSchema:
800 responseSchema = schema.get(responseSchema['$ref'])
801 hasNextPageToken = 'nextPageToken' in responseSchema.get('properties',
802 {})
803 hasPageToken = 'pageToken' in methodDesc.get('parameters', {})
804 if hasNextPageToken and hasPageToken:
805 fixedMethodName, method = createNextMethod(methodName + '_next')
806 self._set_dynamic_attr(fixedMethodName,
807 method.__get__(self, self.__class__))
808