1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """Client for discovery based APIs
16
17 A client library for Google's discovery based APIs.
18 """
19
20 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
21 __all__ = [
22 'build',
23 'build_from_document'
24 'fix_method_name',
25 'key2param'
26 ]
27
28 import copy
29 import httplib2
30 import logging
31 import os
32 import random
33 import re
34 import uritemplate
35 import urllib
36 import urlparse
37 import mimeparse
38 import mimetypes
39
40 try:
41 from urlparse import parse_qsl
42 except ImportError:
43 from cgi import parse_qsl
44
45 from apiclient.errors import HttpError
46 from apiclient.errors import InvalidJsonError
47 from apiclient.errors import MediaUploadSizeError
48 from apiclient.errors import UnacceptableMimeTypeError
49 from apiclient.errors import UnknownApiNameOrVersion
50 from apiclient.errors import UnknownLinkType
51 from apiclient.http import HttpRequest
52 from apiclient.http import MediaFileUpload
53 from apiclient.http import MediaUpload
54 from apiclient.model import JsonModel
55 from apiclient.model import MediaModel
56 from apiclient.model import RawModel
57 from apiclient.schema import Schemas
58 from email.mime.multipart import MIMEMultipart
59 from email.mime.nonmultipart import MIMENonMultipart
60 from oauth2client.anyjson import simplejson
61
62 logger = logging.getLogger(__name__)
63
64 URITEMPLATE = re.compile('{[^}]*}')
65 VARNAME = re.compile('[a-zA-Z0-9_-]+')
66 DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
67 '{api}/{apiVersion}/rest')
68 DEFAULT_METHOD_DOC = 'A description of how to use this function'
69
70
71 STACK_QUERY_PARAMETERS = ['trace', 'pp', 'userip', 'strict']
72
73
74 RESERVED_WORDS = ['and', 'assert', 'break', 'class', 'continue', 'def', 'del',
75 'elif', 'else', 'except', 'exec', 'finally', 'for', 'from',
76 'global', 'if', 'import', 'in', 'is', 'lambda', 'not', 'or',
77 'pass', 'print', 'raise', 'return', 'try', 'while', 'body']
78
79
81 """Fix method names to avoid reserved word conflicts.
82
83 Args:
84 name: string, method name.
85
86 Returns:
87 The name with a '_' prefixed if the name is a reserved word.
88 """
89 if name in RESERVED_WORDS:
90 return name + '_'
91 else:
92 return name
93
94
96 """Adds a query parameter to a url.
97
98 Replaces the current value if it already exists in the URL.
99
100 Args:
101 url: string, url to add the query parameter to.
102 name: string, query parameter name.
103 value: string, query parameter value.
104
105 Returns:
106 Updated query parameter. Does not update the url if value is None.
107 """
108 if value is None:
109 return url
110 else:
111 parsed = list(urlparse.urlparse(url))
112 q = dict(parse_qsl(parsed[4]))
113 q[name] = value
114 parsed[4] = urllib.urlencode(q)
115 return urlparse.urlunparse(parsed)
116
117
119 """Converts key names into parameter names.
120
121 For example, converting "max-results" -> "max_results"
122
123 Args:
124 key: string, the method key name.
125
126 Returns:
127 A safe method name based on the key name.
128 """
129 result = []
130 key = list(key)
131 if not key[0].isalpha():
132 result.append('x')
133 for c in key:
134 if c.isalnum():
135 result.append(c)
136 else:
137 result.append('_')
138
139 return ''.join(result)
140
141
142 -def build(serviceName,
143 version,
144 http=None,
145 discoveryServiceUrl=DISCOVERY_URI,
146 developerKey=None,
147 model=None,
148 requestBuilder=HttpRequest):
149 """Construct a Resource for interacting with an API.
150
151 Construct a Resource object for interacting with an API. The serviceName and
152 version are the names from the Discovery service.
153
154 Args:
155 serviceName: string, name of the service.
156 version: string, the version of the service.
157 http: httplib2.Http, An instance of httplib2.Http or something that acts
158 like it that HTTP requests will be made through.
159 discoveryServiceUrl: string, a URI Template that points to the location of
160 the discovery service. It should have two parameters {api} and
161 {apiVersion} that when filled in produce an absolute URI to the discovery
162 document for that service.
163 developerKey: string, key obtained from
164 https://code.google.com/apis/console.
165 model: apiclient.Model, converts to and from the wire format.
166 requestBuilder: apiclient.http.HttpRequest, encapsulator for an HTTP
167 request.
168
169 Returns:
170 A Resource object with methods for interacting with the service.
171 """
172 params = {
173 'api': serviceName,
174 'apiVersion': version
175 }
176
177 if http is None:
178 http = httplib2.Http()
179
180 requested_url = uritemplate.expand(discoveryServiceUrl, params)
181
182
183
184
185
186 if 'REMOTE_ADDR' in os.environ:
187 requested_url = _add_query_parameter(requested_url, 'userIp',
188 os.environ['REMOTE_ADDR'])
189 logger.info('URL being requested: %s' % requested_url)
190
191 resp, content = http.request(requested_url)
192
193 if resp.status == 404:
194 raise UnknownApiNameOrVersion("name: %s version: %s" % (serviceName,
195 version))
196 if resp.status >= 400:
197 raise HttpError(resp, content, requested_url)
198
199 try:
200 service = simplejson.loads(content)
201 except ValueError, e:
202 logger.error('Failed to parse as JSON: ' + content)
203 raise InvalidJsonError()
204
205 return build_from_document(content, discoveryServiceUrl, http=http,
206 developerKey=developerKey, model=model, requestBuilder=requestBuilder)
207
208
209 -def build_from_document(
210 service,
211 base,
212 future=None,
213 http=None,
214 developerKey=None,
215 model=None,
216 requestBuilder=HttpRequest):
217 """Create a Resource for interacting with an API.
218
219 Same as `build()`, but constructs the Resource object from a discovery
220 document that is it given, as opposed to retrieving one over HTTP.
221
222 Args:
223 service: string, discovery document.
224 base: string, base URI for all HTTP requests, usually the discovery URI.
225 future: string, discovery document with future capabilities (deprecated).
226 http: httplib2.Http, An instance of httplib2.Http or something that acts
227 like it that HTTP requests will be made through.
228 developerKey: string, Key for controlling API usage, generated
229 from the API Console.
230 model: Model class instance that serializes and de-serializes requests and
231 responses.
232 requestBuilder: Takes an http request and packages it up to be executed.
233
234 Returns:
235 A Resource object with methods for interacting with the service.
236 """
237
238
239 future = {}
240
241 service = simplejson.loads(service)
242 base = urlparse.urljoin(base, service['basePath'])
243 schema = Schemas(service)
244
245 if model is None:
246 features = service.get('features', [])
247 model = JsonModel('dataWrapper' in features)
248 resource = _createResource(http, base, model, requestBuilder, developerKey,
249 service, service, schema)
250
251 return resource
252
253
254 -def _cast(value, schema_type):
255 """Convert value to a string based on JSON Schema type.
256
257 See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on
258 JSON Schema.
259
260 Args:
261 value: any, the value to convert
262 schema_type: string, the type that value should be interpreted as
263
264 Returns:
265 A string representation of 'value' based on the schema_type.
266 """
267 if schema_type == 'string':
268 if type(value) == type('') or type(value) == type(u''):
269 return value
270 else:
271 return str(value)
272 elif schema_type == 'integer':
273 return str(int(value))
274 elif schema_type == 'number':
275 return str(float(value))
276 elif schema_type == 'boolean':
277 return str(bool(value)).lower()
278 else:
279 if type(value) == type('') or type(value) == type(u''):
280 return value
281 else:
282 return str(value)
283
284
285 MULTIPLIERS = {
286 "KB": 2 ** 10,
287 "MB": 2 ** 20,
288 "GB": 2 ** 30,
289 "TB": 2 ** 40,
290 }
291
292
310
311
312 -def _createResource(http, baseUrl, model, requestBuilder,
313 developerKey, resourceDesc, rootDesc, schema):
314 """Build a Resource from the API description.
315
316 Args:
317 http: httplib2.Http, Object to make http requests with.
318 baseUrl: string, base URL for the API. All requests are relative to this
319 URI.
320 model: apiclient.Model, converts to and from the wire format.
321 requestBuilder: class or callable that instantiates an
322 apiclient.HttpRequest object.
323 developerKey: string, key obtained from
324 https://code.google.com/apis/console
325 resourceDesc: object, section of deserialized discovery document that
326 describes a resource. Note that the top level discovery document
327 is considered a resource.
328 rootDesc: object, the entire deserialized discovery document.
329 schema: object, mapping of schema names to schema descriptions.
330
331 Returns:
332 An instance of Resource with all the methods attached for interacting with
333 that resource.
334 """
335
336 class Resource(object):
337 """A class for interacting with a resource."""
338
339 def __init__(self):
340 self._http = http
341 self._baseUrl = baseUrl
342 self._model = model
343 self._developerKey = developerKey
344 self._requestBuilder = requestBuilder
345
346 def createMethod(theclass, methodName, methodDesc, rootDesc):
347 """Creates a method for attaching to a Resource.
348
349 Args:
350 theclass: type, the class to attach methods to.
351 methodName: string, name of the method to use.
352 methodDesc: object, fragment of deserialized discovery document that
353 describes the method.
354 rootDesc: object, the entire deserialized discovery document.
355 """
356 methodName = fix_method_name(methodName)
357 pathUrl = methodDesc['path']
358 httpMethod = methodDesc['httpMethod']
359 methodId = methodDesc['id']
360
361 mediaPathUrl = None
362 accept = []
363 maxSize = 0
364 if 'mediaUpload' in methodDesc:
365 mediaUpload = methodDesc['mediaUpload']
366
367 parsed = list(urlparse.urlparse(baseUrl))
368 basePath = parsed[2]
369 mediaPathUrl = '/upload' + basePath + pathUrl
370 accept = mediaUpload['accept']
371 maxSize = _media_size_to_long(mediaUpload.get('maxSize', ''))
372
373 if 'parameters' not in methodDesc:
374 methodDesc['parameters'] = {}
375
376
377 for name, desc in rootDesc.get('parameters', {}).iteritems():
378 methodDesc['parameters'][name] = desc
379
380
381 for name in STACK_QUERY_PARAMETERS:
382 methodDesc['parameters'][name] = {
383 'type': 'string',
384 'location': 'query'
385 }
386
387 if httpMethod in ['PUT', 'POST', 'PATCH'] and 'request' in methodDesc:
388 methodDesc['parameters']['body'] = {
389 'description': 'The request body.',
390 'type': 'object',
391 'required': True,
392 }
393 if 'request' in methodDesc:
394 methodDesc['parameters']['body'].update(methodDesc['request'])
395 else:
396 methodDesc['parameters']['body']['type'] = 'object'
397 if 'mediaUpload' in methodDesc:
398 methodDesc['parameters']['media_body'] = {
399 'description':
400 'The filename of the media request body, or an instance of a '
401 'MediaUpload object.',
402 'type': 'string',
403 'required': False,
404 }
405 if 'body' in methodDesc['parameters']:
406 methodDesc['parameters']['body']['required'] = False
407
408 argmap = {}
409 required_params = []
410 repeated_params = []
411 pattern_params = {}
412 query_params = []
413 path_params = {}
414 param_type = {}
415 enum_params = {}
416
417
418 if 'parameters' in methodDesc:
419 for arg, desc in methodDesc['parameters'].iteritems():
420 param = key2param(arg)
421 argmap[param] = arg
422
423 if desc.get('pattern', ''):
424 pattern_params[param] = desc['pattern']
425 if desc.get('enum', ''):
426 enum_params[param] = desc['enum']
427 if desc.get('required', False):
428 required_params.append(param)
429 if desc.get('repeated', False):
430 repeated_params.append(param)
431 if desc.get('location') == 'query':
432 query_params.append(param)
433 if desc.get('location') == 'path':
434 path_params[param] = param
435 param_type[param] = desc.get('type', 'string')
436
437 for match in URITEMPLATE.finditer(pathUrl):
438 for namematch in VARNAME.finditer(match.group(0)):
439 name = key2param(namematch.group(0))
440 path_params[name] = name
441 if name in query_params:
442 query_params.remove(name)
443
444 def method(self, **kwargs):
445
446
447 for name in kwargs.iterkeys():
448 if name not in argmap:
449 raise TypeError('Got an unexpected keyword argument "%s"' % name)
450
451
452 keys = kwargs.keys()
453 for name in keys:
454 if kwargs[name] is None:
455 del kwargs[name]
456
457 for name in required_params:
458 if name not in kwargs:
459 raise TypeError('Missing required parameter "%s"' % name)
460
461 for name, regex in pattern_params.iteritems():
462 if name in kwargs:
463 if isinstance(kwargs[name], basestring):
464 pvalues = [kwargs[name]]
465 else:
466 pvalues = kwargs[name]
467 for pvalue in pvalues:
468 if re.match(regex, pvalue) is None:
469 raise TypeError(
470 'Parameter "%s" value "%s" does not match the pattern "%s"' %
471 (name, pvalue, regex))
472
473 for name, enums in enum_params.iteritems():
474 if name in kwargs:
475
476
477
478 if (name in repeated_params and
479 not isinstance(kwargs[name], basestring)):
480 values = kwargs[name]
481 else:
482 values = [kwargs[name]]
483 for value in values:
484 if value not in enums:
485 raise TypeError(
486 'Parameter "%s" value "%s" is not an allowed value in "%s"' %
487 (name, value, str(enums)))
488
489 actual_query_params = {}
490 actual_path_params = {}
491 for key, value in kwargs.iteritems():
492 to_type = param_type.get(key, 'string')
493
494 if key in repeated_params and type(value) == type([]):
495 cast_value = [_cast(x, to_type) for x in value]
496 else:
497 cast_value = _cast(value, to_type)
498 if key in query_params:
499 actual_query_params[argmap[key]] = cast_value
500 if key in path_params:
501 actual_path_params[argmap[key]] = cast_value
502 body_value = kwargs.get('body', None)
503 media_filename = kwargs.get('media_body', None)
504
505 if self._developerKey:
506 actual_query_params['key'] = self._developerKey
507
508 model = self._model
509
510 if methodName.endswith('_media'):
511 model = MediaModel()
512 elif 'response' not in methodDesc:
513 model = RawModel()
514
515 headers = {}
516 headers, params, query, body = model.request(headers,
517 actual_path_params, actual_query_params, body_value)
518
519 expanded_url = uritemplate.expand(pathUrl, params)
520 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
521
522 resumable = None
523 multipart_boundary = ''
524
525 if media_filename:
526
527 if isinstance(media_filename, basestring):
528 (media_mime_type, encoding) = mimetypes.guess_type(media_filename)
529 if media_mime_type is None:
530 raise UnknownFileType(media_filename)
531 if not mimeparse.best_match([media_mime_type], ','.join(accept)):
532 raise UnacceptableMimeTypeError(media_mime_type)
533 media_upload = MediaFileUpload(media_filename, media_mime_type)
534 elif isinstance(media_filename, MediaUpload):
535 media_upload = media_filename
536 else:
537 raise TypeError('media_filename must be str or MediaUpload.')
538
539
540 if maxSize > 0 and media_upload.size() > maxSize:
541 raise MediaUploadSizeError("Media larger than: %s" % maxSize)
542
543
544 expanded_url = uritemplate.expand(mediaPathUrl, params)
545 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
546 if media_upload.resumable():
547 url = _add_query_parameter(url, 'uploadType', 'resumable')
548
549 if media_upload.resumable():
550
551
552 resumable = media_upload
553 else:
554
555 if body is None:
556
557 headers['content-type'] = media_upload.mimetype()
558 body = media_upload.getbytes(0, media_upload.size())
559 url = _add_query_parameter(url, 'uploadType', 'media')
560 else:
561
562 msgRoot = MIMEMultipart('related')
563
564 setattr(msgRoot, '_write_headers', lambda self: None)
565
566
567 msg = MIMENonMultipart(*headers['content-type'].split('/'))
568 msg.set_payload(body)
569 msgRoot.attach(msg)
570
571
572 msg = MIMENonMultipart(*media_upload.mimetype().split('/'))
573 msg['Content-Transfer-Encoding'] = 'binary'
574
575 payload = media_upload.getbytes(0, media_upload.size())
576 msg.set_payload(payload)
577 msgRoot.attach(msg)
578 body = msgRoot.as_string()
579
580 multipart_boundary = msgRoot.get_boundary()
581 headers['content-type'] = ('multipart/related; '
582 'boundary="%s"') % multipart_boundary
583 url = _add_query_parameter(url, 'uploadType', 'multipart')
584
585 logger.info('URL being requested: %s' % url)
586 return self._requestBuilder(self._http,
587 model.response,
588 url,
589 method=httpMethod,
590 body=body,
591 headers=headers,
592 methodId=methodId,
593 resumable=resumable)
594
595 docs = [methodDesc.get('description', DEFAULT_METHOD_DOC), '\n\n']
596 if len(argmap) > 0:
597 docs.append('Args:\n')
598
599
600 skip_parameters = rootDesc.get('parameters', {}).keys()
601 skip_parameters.extend(STACK_QUERY_PARAMETERS)
602
603 all_args = argmap.keys()
604 args_ordered = [key2param(s) for s in methodDesc.get('parameterOrder', [])]
605
606
607 if 'body' in all_args:
608 args_ordered.append('body')
609
610 for name in all_args:
611 if name not in args_ordered:
612 args_ordered.append(name)
613
614 for arg in args_ordered:
615 if arg in skip_parameters:
616 continue
617
618 repeated = ''
619 if arg in repeated_params:
620 repeated = ' (repeated)'
621 required = ''
622 if arg in required_params:
623 required = ' (required)'
624 paramdesc = methodDesc['parameters'][argmap[arg]]
625 paramdoc = paramdesc.get('description', 'A parameter')
626 if '$ref' in paramdesc:
627 docs.append(
628 (' %s: object, %s%s%s\n The object takes the'
629 ' form of:\n\n%s\n\n') % (arg, paramdoc, required, repeated,
630 schema.prettyPrintByName(paramdesc['$ref'])))
631 else:
632 paramtype = paramdesc.get('type', 'string')
633 docs.append(' %s: %s, %s%s%s\n' % (arg, paramtype, paramdoc, required,
634 repeated))
635 enum = paramdesc.get('enum', [])
636 enumDesc = paramdesc.get('enumDescriptions', [])
637 if enum and enumDesc:
638 docs.append(' Allowed values\n')
639 for (name, desc) in zip(enum, enumDesc):
640 docs.append(' %s - %s\n' % (name, desc))
641 if 'response' in methodDesc:
642 if methodName.endswith('_media'):
643 docs.append('\nReturns:\n The media object as a string.\n\n ')
644 else:
645 docs.append('\nReturns:\n An object of the form:\n\n ')
646 docs.append(schema.prettyPrintSchema(methodDesc['response']))
647
648 setattr(method, '__doc__', ''.join(docs))
649 setattr(theclass, methodName, method)
650
651 def createNextMethod(theclass, methodName, methodDesc, rootDesc):
652 """Creates any _next methods for attaching to a Resource.
653
654 The _next methods allow for easy iteration through list() responses.
655
656 Args:
657 theclass: type, the class to attach methods to.
658 methodName: string, name of the method to use.
659 methodDesc: object, fragment of deserialized discovery document that
660 describes the method.
661 rootDesc: object, the entire deserialized discovery document.
662 """
663 methodName = fix_method_name(methodName)
664 methodId = methodDesc['id'] + '.next'
665
666 def methodNext(self, previous_request, previous_response):
667 """Retrieves the next page of results.
668
669 Args:
670 previous_request: The request for the previous page. (required)
671 previous_response: The response from the request for the previous page. (required)
672
673 Returns:
674 A request object that you can call 'execute()' on to request the next
675 page. Returns None if there are no more items in the collection.
676 """
677
678
679
680 if 'nextPageToken' not in previous_response:
681 return None
682
683 request = copy.copy(previous_request)
684
685 pageToken = previous_response['nextPageToken']
686 parsed = list(urlparse.urlparse(request.uri))
687 q = parse_qsl(parsed[4])
688
689
690 newq = [(key, value) for (key, value) in q if key != 'pageToken']
691 newq.append(('pageToken', pageToken))
692 parsed[4] = urllib.urlencode(newq)
693 uri = urlparse.urlunparse(parsed)
694
695 request.uri = uri
696
697 logger.info('URL being requested: %s' % uri)
698
699 return request
700
701 setattr(theclass, methodName, methodNext)
702
703
704 if 'methods' in resourceDesc:
705 for methodName, methodDesc in resourceDesc['methods'].iteritems():
706 createMethod(Resource, methodName, methodDesc, rootDesc)
707
708
709 if methodDesc.get('supportsMediaDownload', False):
710 createMethod(Resource, methodName + '_media', methodDesc, rootDesc)
711
712
713 if 'resources' in resourceDesc:
714
715 def createResourceMethod(theclass, methodName, methodDesc, rootDesc):
716 """Create a method on the Resource to access a nested Resource.
717
718 Args:
719 theclass: type, the class to attach methods to.
720 methodName: string, name of the method to use.
721 methodDesc: object, fragment of deserialized discovery document that
722 describes the method.
723 rootDesc: object, the entire deserialized discovery document.
724 """
725 methodName = fix_method_name(methodName)
726
727 def methodResource(self):
728 return _createResource(self._http, self._baseUrl, self._model,
729 self._requestBuilder, self._developerKey,
730 methodDesc, rootDesc, schema)
731
732 setattr(methodResource, '__doc__', 'A collection resource.')
733 setattr(methodResource, '__is_resource__', True)
734 setattr(theclass, methodName, methodResource)
735
736 for methodName, methodDesc in resourceDesc['resources'].iteritems():
737 createResourceMethod(Resource, methodName, methodDesc, rootDesc)
738
739
740
741
742 if 'methods' in resourceDesc:
743 for methodName, methodDesc in resourceDesc['methods'].iteritems():
744 if 'response' in methodDesc:
745 responseSchema = methodDesc['response']
746 if '$ref' in responseSchema:
747 responseSchema = schema.get(responseSchema['$ref'])
748 hasNextPageToken = 'nextPageToken' in responseSchema.get('properties',
749 {})
750 hasPageToken = 'pageToken' in methodDesc.get('parameters', {})
751 if hasNextPageToken and hasPageToken:
752 createNextMethod(Resource, methodName + '_next',
753 resourceDesc['methods'][methodName],
754 methodName)
755
756 return Resource()
757