1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """Client for discovery based APIs
16
17 A client library for Google's discovery based APIs.
18 """
19
20 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
21 __all__ = [
22 'build',
23 'build_from_document'
24 'fix_method_name',
25 'key2param'
26 ]
27
28 import copy
29 import httplib2
30 import logging
31 import os
32 import random
33 import re
34 import uritemplate
35 import urllib
36 import urlparse
37 import mimeparse
38 import mimetypes
39
40 try:
41 from urlparse import parse_qsl
42 except ImportError:
43 from cgi import parse_qsl
44
45 from apiclient.errors import HttpError
46 from apiclient.errors import InvalidJsonError
47 from apiclient.errors import MediaUploadSizeError
48 from apiclient.errors import UnacceptableMimeTypeError
49 from apiclient.errors import UnknownApiNameOrVersion
50 from apiclient.errors import UnknownLinkType
51 from apiclient.http import HttpRequest
52 from apiclient.http import MediaFileUpload
53 from apiclient.http import MediaUpload
54 from apiclient.model import JsonModel
55 from apiclient.model import MediaModel
56 from apiclient.model import RawModel
57 from apiclient.schema import Schemas
58 from email.mime.multipart import MIMEMultipart
59 from email.mime.nonmultipart import MIMENonMultipart
60 from oauth2client import util
61 from oauth2client.anyjson import simplejson
62
63 logger = logging.getLogger(__name__)
64
65 URITEMPLATE = re.compile('{[^}]*}')
66 VARNAME = re.compile('[a-zA-Z0-9_-]+')
67 DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
68 '{api}/{apiVersion}/rest')
69 DEFAULT_METHOD_DOC = 'A description of how to use this function'
70
71
72 STACK_QUERY_PARAMETERS = ['trace', 'pp', 'userip', 'strict']
73
74
75 RESERVED_WORDS = ['and', 'assert', 'break', 'class', 'continue', 'def', 'del',
76 'elif', 'else', 'except', 'exec', 'finally', 'for', 'from',
77 'global', 'if', 'import', 'in', 'is', 'lambda', 'not', 'or',
78 'pass', 'print', 'raise', 'return', 'try', 'while', 'body']
82 """Fix method names to avoid reserved word conflicts.
83
84 Args:
85 name: string, method name.
86
87 Returns:
88 The name with a '_' prefixed if the name is a reserved word.
89 """
90 if name in RESERVED_WORDS:
91 return name + '_'
92 else:
93 return name
94
97 """Adds a query parameter to a url.
98
99 Replaces the current value if it already exists in the URL.
100
101 Args:
102 url: string, url to add the query parameter to.
103 name: string, query parameter name.
104 value: string, query parameter value.
105
106 Returns:
107 Updated query parameter. Does not update the url if value is None.
108 """
109 if value is None:
110 return url
111 else:
112 parsed = list(urlparse.urlparse(url))
113 q = dict(parse_qsl(parsed[4]))
114 q[name] = value
115 parsed[4] = urllib.urlencode(q)
116 return urlparse.urlunparse(parsed)
117
120 """Converts key names into parameter names.
121
122 For example, converting "max-results" -> "max_results"
123
124 Args:
125 key: string, the method key name.
126
127 Returns:
128 A safe method name based on the key name.
129 """
130 result = []
131 key = list(key)
132 if not key[0].isalpha():
133 result.append('x')
134 for c in key:
135 if c.isalnum():
136 result.append(c)
137 else:
138 result.append('_')
139
140 return ''.join(result)
141
151 """Construct a Resource for interacting with an API.
152
153 Construct a Resource object for interacting with an API. The serviceName and
154 version are the names from the Discovery service.
155
156 Args:
157 serviceName: string, name of the service.
158 version: string, the version of the service.
159 http: httplib2.Http, An instance of httplib2.Http or something that acts
160 like it that HTTP requests will be made through.
161 discoveryServiceUrl: string, a URI Template that points to the location of
162 the discovery service. It should have two parameters {api} and
163 {apiVersion} that when filled in produce an absolute URI to the discovery
164 document for that service.
165 developerKey: string, key obtained from
166 https://code.google.com/apis/console.
167 model: apiclient.Model, converts to and from the wire format.
168 requestBuilder: apiclient.http.HttpRequest, encapsulator for an HTTP
169 request.
170
171 Returns:
172 A Resource object with methods for interacting with the service.
173 """
174 params = {
175 'api': serviceName,
176 'apiVersion': version
177 }
178
179 if http is None:
180 http = httplib2.Http()
181
182 requested_url = uritemplate.expand(discoveryServiceUrl, params)
183
184
185
186
187
188 if 'REMOTE_ADDR' in os.environ:
189 requested_url = _add_query_parameter(requested_url, 'userIp',
190 os.environ['REMOTE_ADDR'])
191 logger.info('URL being requested: %s' % requested_url)
192
193 resp, content = http.request(requested_url)
194
195 if resp.status == 404:
196 raise UnknownApiNameOrVersion("name: %s version: %s" % (serviceName,
197 version))
198 if resp.status >= 400:
199 raise HttpError(resp, content, uri=requested_url)
200
201 try:
202 service = simplejson.loads(content)
203 except ValueError, e:
204 logger.error('Failed to parse as JSON: ' + content)
205 raise InvalidJsonError()
206
207 return build_from_document(content, base=discoveryServiceUrl, http=http,
208 developerKey=developerKey, model=model, requestBuilder=requestBuilder)
209
220 """Create a Resource for interacting with an API.
221
222 Same as `build()`, but constructs the Resource object from a discovery
223 document that is it given, as opposed to retrieving one over HTTP.
224
225 Args:
226 service: string, discovery document.
227 base: string, base URI for all HTTP requests, usually the discovery URI.
228 This parameter is no longer used as rootUrl and servicePath are included
229 within the discovery document. (deprecated)
230 future: string, discovery document with future capabilities (deprecated).
231 http: httplib2.Http, An instance of httplib2.Http or something that acts
232 like it that HTTP requests will be made through.
233 developerKey: string, Key for controlling API usage, generated
234 from the API Console.
235 model: Model class instance that serializes and de-serializes requests and
236 responses.
237 requestBuilder: Takes an http request and packages it up to be executed.
238
239 Returns:
240 A Resource object with methods for interacting with the service.
241 """
242
243
244 future = {}
245
246 service = simplejson.loads(service)
247 base = urlparse.urljoin(service['rootUrl'], service['servicePath'])
248 schema = Schemas(service)
249
250 if model is None:
251 features = service.get('features', [])
252 model = JsonModel('dataWrapper' in features)
253 resource = _createResource(http, base, model, requestBuilder, developerKey,
254 service, service, schema)
255
256 return resource
257
258
259 -def _cast(value, schema_type):
260 """Convert value to a string based on JSON Schema type.
261
262 See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on
263 JSON Schema.
264
265 Args:
266 value: any, the value to convert
267 schema_type: string, the type that value should be interpreted as
268
269 Returns:
270 A string representation of 'value' based on the schema_type.
271 """
272 if schema_type == 'string':
273 if type(value) == type('') or type(value) == type(u''):
274 return value
275 else:
276 return str(value)
277 elif schema_type == 'integer':
278 return str(int(value))
279 elif schema_type == 'number':
280 return str(float(value))
281 elif schema_type == 'boolean':
282 return str(bool(value)).lower()
283 else:
284 if type(value) == type('') or type(value) == type(u''):
285 return value
286 else:
287 return str(value)
288
289
290 MULTIPLIERS = {
291 "KB": 2 ** 10,
292 "MB": 2 ** 20,
293 "GB": 2 ** 30,
294 "TB": 2 ** 40,
295 }
315
316
317 -def _createResource(http, baseUrl, model, requestBuilder,
318 developerKey, resourceDesc, rootDesc, schema):
319 """Build a Resource from the API description.
320
321 Args:
322 http: httplib2.Http, Object to make http requests with.
323 baseUrl: string, base URL for the API. All requests are relative to this
324 URI.
325 model: apiclient.Model, converts to and from the wire format.
326 requestBuilder: class or callable that instantiates an
327 apiclient.HttpRequest object.
328 developerKey: string, key obtained from
329 https://code.google.com/apis/console
330 resourceDesc: object, section of deserialized discovery document that
331 describes a resource. Note that the top level discovery document
332 is considered a resource.
333 rootDesc: object, the entire deserialized discovery document.
334 schema: object, mapping of schema names to schema descriptions.
335
336 Returns:
337 An instance of Resource with all the methods attached for interacting with
338 that resource.
339 """
340
341 class Resource(object):
342 """A class for interacting with a resource."""
343
344 def __init__(self):
345 self._http = http
346 self._baseUrl = baseUrl
347 self._model = model
348 self._developerKey = developerKey
349 self._requestBuilder = requestBuilder
350
351 def createMethod(theclass, methodName, methodDesc, rootDesc):
352 """Creates a method for attaching to a Resource.
353
354 Args:
355 theclass: type, the class to attach methods to.
356 methodName: string, name of the method to use.
357 methodDesc: object, fragment of deserialized discovery document that
358 describes the method.
359 rootDesc: object, the entire deserialized discovery document.
360 """
361 methodName = fix_method_name(methodName)
362 pathUrl = methodDesc['path']
363 httpMethod = methodDesc['httpMethod']
364 methodId = methodDesc['id']
365
366 mediaPathUrl = None
367 accept = []
368 maxSize = 0
369 if 'mediaUpload' in methodDesc:
370 mediaUpload = methodDesc['mediaUpload']
371 mediaPathUrl = (rootDesc['rootUrl'] + 'upload/' + rootDesc['servicePath']
372 + pathUrl)
373 accept = mediaUpload['accept']
374 maxSize = _media_size_to_long(mediaUpload.get('maxSize', ''))
375
376 if 'parameters' not in methodDesc:
377 methodDesc['parameters'] = {}
378
379
380 for name, desc in rootDesc.get('parameters', {}).iteritems():
381 methodDesc['parameters'][name] = desc
382
383
384 for name in STACK_QUERY_PARAMETERS:
385 methodDesc['parameters'][name] = {
386 'type': 'string',
387 'location': 'query'
388 }
389
390 if httpMethod in ['PUT', 'POST', 'PATCH'] and 'request' in methodDesc:
391 methodDesc['parameters']['body'] = {
392 'description': 'The request body.',
393 'type': 'object',
394 'required': True,
395 }
396 if 'request' in methodDesc:
397 methodDesc['parameters']['body'].update(methodDesc['request'])
398 else:
399 methodDesc['parameters']['body']['type'] = 'object'
400 if 'mediaUpload' in methodDesc:
401 methodDesc['parameters']['media_body'] = {
402 'description':
403 'The filename of the media request body, or an instance of a '
404 'MediaUpload object.',
405 'type': 'string',
406 'required': False,
407 }
408 if 'body' in methodDesc['parameters']:
409 methodDesc['parameters']['body']['required'] = False
410
411 argmap = {}
412 required_params = []
413 repeated_params = []
414 pattern_params = {}
415 query_params = []
416 path_params = {}
417 param_type = {}
418 enum_params = {}
419
420
421 if 'parameters' in methodDesc:
422 for arg, desc in methodDesc['parameters'].iteritems():
423 param = key2param(arg)
424 argmap[param] = arg
425
426 if desc.get('pattern', ''):
427 pattern_params[param] = desc['pattern']
428 if desc.get('enum', ''):
429 enum_params[param] = desc['enum']
430 if desc.get('required', False):
431 required_params.append(param)
432 if desc.get('repeated', False):
433 repeated_params.append(param)
434 if desc.get('location') == 'query':
435 query_params.append(param)
436 if desc.get('location') == 'path':
437 path_params[param] = param
438 param_type[param] = desc.get('type', 'string')
439
440 for match in URITEMPLATE.finditer(pathUrl):
441 for namematch in VARNAME.finditer(match.group(0)):
442 name = key2param(namematch.group(0))
443 path_params[name] = name
444 if name in query_params:
445 query_params.remove(name)
446
447 def method(self, **kwargs):
448
449
450 for name in kwargs.iterkeys():
451 if name not in argmap:
452 raise TypeError('Got an unexpected keyword argument "%s"' % name)
453
454
455 keys = kwargs.keys()
456 for name in keys:
457 if kwargs[name] is None:
458 del kwargs[name]
459
460 for name in required_params:
461 if name not in kwargs:
462 raise TypeError('Missing required parameter "%s"' % name)
463
464 for name, regex in pattern_params.iteritems():
465 if name in kwargs:
466 if isinstance(kwargs[name], basestring):
467 pvalues = [kwargs[name]]
468 else:
469 pvalues = kwargs[name]
470 for pvalue in pvalues:
471 if re.match(regex, pvalue) is None:
472 raise TypeError(
473 'Parameter "%s" value "%s" does not match the pattern "%s"' %
474 (name, pvalue, regex))
475
476 for name, enums in enum_params.iteritems():
477 if name in kwargs:
478
479
480
481 if (name in repeated_params and
482 not isinstance(kwargs[name], basestring)):
483 values = kwargs[name]
484 else:
485 values = [kwargs[name]]
486 for value in values:
487 if value not in enums:
488 raise TypeError(
489 'Parameter "%s" value "%s" is not an allowed value in "%s"' %
490 (name, value, str(enums)))
491
492 actual_query_params = {}
493 actual_path_params = {}
494 for key, value in kwargs.iteritems():
495 to_type = param_type.get(key, 'string')
496
497 if key in repeated_params and type(value) == type([]):
498 cast_value = [_cast(x, to_type) for x in value]
499 else:
500 cast_value = _cast(value, to_type)
501 if key in query_params:
502 actual_query_params[argmap[key]] = cast_value
503 if key in path_params:
504 actual_path_params[argmap[key]] = cast_value
505 body_value = kwargs.get('body', None)
506 media_filename = kwargs.get('media_body', None)
507
508 if self._developerKey:
509 actual_query_params['key'] = self._developerKey
510
511 model = self._model
512 if methodName.endswith('_media'):
513 model = MediaModel()
514 elif 'response' not in methodDesc:
515 model = RawModel()
516
517 headers = {}
518 headers, params, query, body = model.request(headers,
519 actual_path_params, actual_query_params, body_value)
520
521 expanded_url = uritemplate.expand(pathUrl, params)
522 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
523
524 resumable = None
525 multipart_boundary = ''
526
527 if media_filename:
528
529 if isinstance(media_filename, basestring):
530 (media_mime_type, encoding) = mimetypes.guess_type(media_filename)
531 if media_mime_type is None:
532 raise UnknownFileType(media_filename)
533 if not mimeparse.best_match([media_mime_type], ','.join(accept)):
534 raise UnacceptableMimeTypeError(media_mime_type)
535 media_upload = MediaFileUpload(media_filename,
536 mimetype=media_mime_type)
537 elif isinstance(media_filename, MediaUpload):
538 media_upload = media_filename
539 else:
540 raise TypeError('media_filename must be str or MediaUpload.')
541
542
543 if maxSize > 0 and media_upload.size() > maxSize:
544 raise MediaUploadSizeError("Media larger than: %s" % maxSize)
545
546
547 expanded_url = uritemplate.expand(mediaPathUrl, params)
548 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
549 if media_upload.resumable():
550 url = _add_query_parameter(url, 'uploadType', 'resumable')
551
552 if media_upload.resumable():
553
554
555 resumable = media_upload
556 else:
557
558 if body is None:
559
560 headers['content-type'] = media_upload.mimetype()
561 body = media_upload.getbytes(0, media_upload.size())
562 url = _add_query_parameter(url, 'uploadType', 'media')
563 else:
564
565 msgRoot = MIMEMultipart('related')
566
567 setattr(msgRoot, '_write_headers', lambda self: None)
568
569
570 msg = MIMENonMultipart(*headers['content-type'].split('/'))
571 msg.set_payload(body)
572 msgRoot.attach(msg)
573
574
575 msg = MIMENonMultipart(*media_upload.mimetype().split('/'))
576 msg['Content-Transfer-Encoding'] = 'binary'
577
578 payload = media_upload.getbytes(0, media_upload.size())
579 msg.set_payload(payload)
580 msgRoot.attach(msg)
581 body = msgRoot.as_string()
582
583 multipart_boundary = msgRoot.get_boundary()
584 headers['content-type'] = ('multipart/related; '
585 'boundary="%s"') % multipart_boundary
586 url = _add_query_parameter(url, 'uploadType', 'multipart')
587
588 logger.info('URL being requested: %s' % url)
589 return self._requestBuilder(self._http,
590 model.response,
591 url,
592 method=httpMethod,
593 body=body,
594 headers=headers,
595 methodId=methodId,
596 resumable=resumable)
597
598 docs = [methodDesc.get('description', DEFAULT_METHOD_DOC), '\n\n']
599 if len(argmap) > 0:
600 docs.append('Args:\n')
601
602
603 skip_parameters = rootDesc.get('parameters', {}).keys()
604 skip_parameters.extend(STACK_QUERY_PARAMETERS)
605
606 all_args = argmap.keys()
607 args_ordered = [key2param(s) for s in methodDesc.get('parameterOrder', [])]
608
609
610 if 'body' in all_args:
611 args_ordered.append('body')
612
613 for name in all_args:
614 if name not in args_ordered:
615 args_ordered.append(name)
616
617 for arg in args_ordered:
618 if arg in skip_parameters:
619 continue
620
621 repeated = ''
622 if arg in repeated_params:
623 repeated = ' (repeated)'
624 required = ''
625 if arg in required_params:
626 required = ' (required)'
627 paramdesc = methodDesc['parameters'][argmap[arg]]
628 paramdoc = paramdesc.get('description', 'A parameter')
629 if '$ref' in paramdesc:
630 docs.append(
631 (' %s: object, %s%s%s\n The object takes the'
632 ' form of:\n\n%s\n\n') % (arg, paramdoc, required, repeated,
633 schema.prettyPrintByName(paramdesc['$ref'])))
634 else:
635 paramtype = paramdesc.get('type', 'string')
636 docs.append(' %s: %s, %s%s%s\n' % (arg, paramtype, paramdoc, required,
637 repeated))
638 enum = paramdesc.get('enum', [])
639 enumDesc = paramdesc.get('enumDescriptions', [])
640 if enum and enumDesc:
641 docs.append(' Allowed values\n')
642 for (name, desc) in zip(enum, enumDesc):
643 docs.append(' %s - %s\n' % (name, desc))
644 if 'response' in methodDesc:
645 if methodName.endswith('_media'):
646 docs.append('\nReturns:\n The media object as a string.\n\n ')
647 else:
648 docs.append('\nReturns:\n An object of the form:\n\n ')
649 docs.append(schema.prettyPrintSchema(methodDesc['response']))
650
651 setattr(method, '__doc__', ''.join(docs))
652 setattr(theclass, methodName, method)
653
654 def createNextMethod(theclass, methodName, methodDesc, rootDesc):
655 """Creates any _next methods for attaching to a Resource.
656
657 The _next methods allow for easy iteration through list() responses.
658
659 Args:
660 theclass: type, the class to attach methods to.
661 methodName: string, name of the method to use.
662 methodDesc: object, fragment of deserialized discovery document that
663 describes the method.
664 rootDesc: object, the entire deserialized discovery document.
665 """
666 methodName = fix_method_name(methodName)
667 methodId = methodDesc['id'] + '.next'
668
669 def methodNext(self, previous_request, previous_response):
670 """Retrieves the next page of results.
671
672 Args:
673 previous_request: The request for the previous page. (required)
674 previous_response: The response from the request for the previous page. (required)
675
676 Returns:
677 A request object that you can call 'execute()' on to request the next
678 page. Returns None if there are no more items in the collection.
679 """
680
681
682
683 if 'nextPageToken' not in previous_response:
684 return None
685
686 request = copy.copy(previous_request)
687
688 pageToken = previous_response['nextPageToken']
689 parsed = list(urlparse.urlparse(request.uri))
690 q = parse_qsl(parsed[4])
691
692
693 newq = [(key, value) for (key, value) in q if key != 'pageToken']
694 newq.append(('pageToken', pageToken))
695 parsed[4] = urllib.urlencode(newq)
696 uri = urlparse.urlunparse(parsed)
697
698 request.uri = uri
699
700 logger.info('URL being requested: %s' % uri)
701
702 return request
703
704 setattr(theclass, methodName, methodNext)
705
706
707 if 'methods' in resourceDesc:
708 for methodName, methodDesc in resourceDesc['methods'].iteritems():
709 createMethod(Resource, methodName, methodDesc, rootDesc)
710
711
712 if methodDesc.get('supportsMediaDownload', False):
713 createMethod(Resource, methodName + '_media', methodDesc, rootDesc)
714
715
716 if 'resources' in resourceDesc:
717
718 def createResourceMethod(theclass, methodName, methodDesc, rootDesc):
719 """Create a method on the Resource to access a nested Resource.
720
721 Args:
722 theclass: type, the class to attach methods to.
723 methodName: string, name of the method to use.
724 methodDesc: object, fragment of deserialized discovery document that
725 describes the method.
726 rootDesc: object, the entire deserialized discovery document.
727 """
728 methodName = fix_method_name(methodName)
729
730 def methodResource(self):
731 return _createResource(self._http, self._baseUrl, self._model,
732 self._requestBuilder, self._developerKey,
733 methodDesc, rootDesc, schema)
734
735 setattr(methodResource, '__doc__', 'A collection resource.')
736 setattr(methodResource, '__is_resource__', True)
737 setattr(theclass, methodName, methodResource)
738
739 for methodName, methodDesc in resourceDesc['resources'].iteritems():
740 createResourceMethod(Resource, methodName, methodDesc, rootDesc)
741
742
743
744
745 if 'methods' in resourceDesc:
746 for methodName, methodDesc in resourceDesc['methods'].iteritems():
747 if 'response' in methodDesc:
748 responseSchema = methodDesc['response']
749 if '$ref' in responseSchema:
750 responseSchema = schema.get(responseSchema['$ref'])
751 hasNextPageToken = 'nextPageToken' in responseSchema.get('properties',
752 {})
753 hasPageToken = 'pageToken' in methodDesc.get('parameters', {})
754 if hasNextPageToken and hasPageToken:
755 createNextMethod(Resource, methodName + '_next',
756 resourceDesc['methods'][methodName],
757 methodName)
758
759 return Resource()
760