1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """Client for discovery based APIs
16
17 A client library for Google's discovery based APIs.
18 """
19
20 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
21 __all__ = [
22 'build',
23 'build_from_document'
24 'fix_method_name',
25 'key2param'
26 ]
27
28 import copy
29 import httplib2
30 import logging
31 import os
32 import random
33 import re
34 import uritemplate
35 import urllib
36 import urlparse
37 import mimeparse
38 import mimetypes
39
40 try:
41 from urlparse import parse_qsl
42 except ImportError:
43 from cgi import parse_qsl
44
45 from apiclient.errors import HttpError
46 from apiclient.errors import InvalidJsonError
47 from apiclient.errors import MediaUploadSizeError
48 from apiclient.errors import UnacceptableMimeTypeError
49 from apiclient.errors import UnknownApiNameOrVersion
50 from apiclient.errors import UnknownLinkType
51 from apiclient.http import HttpRequest
52 from apiclient.http import MediaFileUpload
53 from apiclient.http import MediaUpload
54 from apiclient.model import JsonModel
55 from apiclient.model import MediaModel
56 from apiclient.model import RawModel
57 from apiclient.schema import Schemas
58 from email.mime.multipart import MIMEMultipart
59 from email.mime.nonmultipart import MIMENonMultipart
60 from oauth2client.anyjson import simplejson
61
62 logger = logging.getLogger(__name__)
63
64 URITEMPLATE = re.compile('{[^}]*}')
65 VARNAME = re.compile('[a-zA-Z0-9_-]+')
66 DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
67 '{api}/{apiVersion}/rest')
68 DEFAULT_METHOD_DOC = 'A description of how to use this function'
69
70
71 STACK_QUERY_PARAMETERS = ['trace', 'pp', 'userip', 'strict']
72
73
74 RESERVED_WORDS = ['and', 'assert', 'break', 'class', 'continue', 'def', 'del',
75 'elif', 'else', 'except', 'exec', 'finally', 'for', 'from',
76 'global', 'if', 'import', 'in', 'is', 'lambda', 'not', 'or',
77 'pass', 'print', 'raise', 'return', 'try', 'while', 'body']
78
79
81 """Fix method names to avoid reserved word conflicts.
82
83 Args:
84 name: string, method name.
85
86 Returns:
87 The name with a '_' prefixed if the name is a reserved word.
88 """
89 if name in RESERVED_WORDS:
90 return name + '_'
91 else:
92 return name
93
94
96 """Adds a query parameter to a url.
97
98 Replaces the current value if it already exists in the URL.
99
100 Args:
101 url: string, url to add the query parameter to.
102 name: string, query parameter name.
103 value: string, query parameter value.
104
105 Returns:
106 Updated query parameter. Does not update the url if value is None.
107 """
108 if value is None:
109 return url
110 else:
111 parsed = list(urlparse.urlparse(url))
112 q = dict(parse_qsl(parsed[4]))
113 q[name] = value
114 parsed[4] = urllib.urlencode(q)
115 return urlparse.urlunparse(parsed)
116
117
119 """Converts key names into parameter names.
120
121 For example, converting "max-results" -> "max_results"
122
123 Args:
124 key: string, the method key name.
125
126 Returns:
127 A safe method name based on the key name.
128 """
129 result = []
130 key = list(key)
131 if not key[0].isalpha():
132 result.append('x')
133 for c in key:
134 if c.isalnum():
135 result.append(c)
136 else:
137 result.append('_')
138
139 return ''.join(result)
140
141
142 -def build(serviceName,
143 version,
144 http=None,
145 discoveryServiceUrl=DISCOVERY_URI,
146 developerKey=None,
147 model=None,
148 requestBuilder=HttpRequest):
149 """Construct a Resource for interacting with an API.
150
151 Construct a Resource object for interacting with an API. The serviceName and
152 version are the names from the Discovery service.
153
154 Args:
155 serviceName: string, name of the service.
156 version: string, the version of the service.
157 http: httplib2.Http, An instance of httplib2.Http or something that acts
158 like it that HTTP requests will be made through.
159 discoveryServiceUrl: string, a URI Template that points to the location of
160 the discovery service. It should have two parameters {api} and
161 {apiVersion} that when filled in produce an absolute URI to the discovery
162 document for that service.
163 developerKey: string, key obtained from
164 https://code.google.com/apis/console.
165 model: apiclient.Model, converts to and from the wire format.
166 requestBuilder: apiclient.http.HttpRequest, encapsulator for an HTTP
167 request.
168
169 Returns:
170 A Resource object with methods for interacting with the service.
171 """
172 params = {
173 'api': serviceName,
174 'apiVersion': version
175 }
176
177 if http is None:
178 http = httplib2.Http()
179
180 requested_url = uritemplate.expand(discoveryServiceUrl, params)
181
182
183
184
185
186 if 'REMOTE_ADDR' in os.environ:
187 requested_url = _add_query_parameter(requested_url, 'userIp',
188 os.environ['REMOTE_ADDR'])
189 logger.info('URL being requested: %s' % requested_url)
190
191 resp, content = http.request(requested_url)
192
193 if resp.status == 404:
194 raise UnknownApiNameOrVersion("name: %s version: %s" % (serviceName,
195 version))
196 if resp.status >= 400:
197 raise HttpError(resp, content, requested_url)
198
199 try:
200 service = simplejson.loads(content)
201 except ValueError, e:
202 logger.error('Failed to parse as JSON: ' + content)
203 raise InvalidJsonError()
204
205 return build_from_document(content, discoveryServiceUrl, http=http,
206 developerKey=developerKey, model=model, requestBuilder=requestBuilder)
207
208
209 -def build_from_document(
210 service,
211 base=None,
212 future=None,
213 http=None,
214 developerKey=None,
215 model=None,
216 requestBuilder=HttpRequest):
217 """Create a Resource for interacting with an API.
218
219 Same as `build()`, but constructs the Resource object from a discovery
220 document that is it given, as opposed to retrieving one over HTTP.
221
222 Args:
223 service: string, discovery document.
224 base: string, base URI for all HTTP requests, usually the discovery URI.
225 This parameter is no longer used as rootUrl and servicePath are included
226 within the discovery document. (deprecated)
227 future: string, discovery document with future capabilities (deprecated).
228 http: httplib2.Http, An instance of httplib2.Http or something that acts
229 like it that HTTP requests will be made through.
230 developerKey: string, Key for controlling API usage, generated
231 from the API Console.
232 model: Model class instance that serializes and de-serializes requests and
233 responses.
234 requestBuilder: Takes an http request and packages it up to be executed.
235
236 Returns:
237 A Resource object with methods for interacting with the service.
238 """
239
240
241 future = {}
242
243 service = simplejson.loads(service)
244 base = urlparse.urljoin(service['rootUrl'], service['servicePath'])
245 schema = Schemas(service)
246
247 if model is None:
248 features = service.get('features', [])
249 model = JsonModel('dataWrapper' in features)
250 resource = _createResource(http, base, model, requestBuilder, developerKey,
251 service, service, schema)
252
253 return resource
254
255
256 -def _cast(value, schema_type):
257 """Convert value to a string based on JSON Schema type.
258
259 See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on
260 JSON Schema.
261
262 Args:
263 value: any, the value to convert
264 schema_type: string, the type that value should be interpreted as
265
266 Returns:
267 A string representation of 'value' based on the schema_type.
268 """
269 if schema_type == 'string':
270 if type(value) == type('') or type(value) == type(u''):
271 return value
272 else:
273 return str(value)
274 elif schema_type == 'integer':
275 return str(int(value))
276 elif schema_type == 'number':
277 return str(float(value))
278 elif schema_type == 'boolean':
279 return str(bool(value)).lower()
280 else:
281 if type(value) == type('') or type(value) == type(u''):
282 return value
283 else:
284 return str(value)
285
286
287 MULTIPLIERS = {
288 "KB": 2 ** 10,
289 "MB": 2 ** 20,
290 "GB": 2 ** 30,
291 "TB": 2 ** 40,
292 }
293
294
312
313
314 -def _createResource(http, baseUrl, model, requestBuilder,
315 developerKey, resourceDesc, rootDesc, schema):
316 """Build a Resource from the API description.
317
318 Args:
319 http: httplib2.Http, Object to make http requests with.
320 baseUrl: string, base URL for the API. All requests are relative to this
321 URI.
322 model: apiclient.Model, converts to and from the wire format.
323 requestBuilder: class or callable that instantiates an
324 apiclient.HttpRequest object.
325 developerKey: string, key obtained from
326 https://code.google.com/apis/console
327 resourceDesc: object, section of deserialized discovery document that
328 describes a resource. Note that the top level discovery document
329 is considered a resource.
330 rootDesc: object, the entire deserialized discovery document.
331 schema: object, mapping of schema names to schema descriptions.
332
333 Returns:
334 An instance of Resource with all the methods attached for interacting with
335 that resource.
336 """
337
338 class Resource(object):
339 """A class for interacting with a resource."""
340
341 def __init__(self):
342 self._http = http
343 self._baseUrl = baseUrl
344 self._model = model
345 self._developerKey = developerKey
346 self._requestBuilder = requestBuilder
347
348 def createMethod(theclass, methodName, methodDesc, rootDesc):
349 """Creates a method for attaching to a Resource.
350
351 Args:
352 theclass: type, the class to attach methods to.
353 methodName: string, name of the method to use.
354 methodDesc: object, fragment of deserialized discovery document that
355 describes the method.
356 rootDesc: object, the entire deserialized discovery document.
357 """
358 methodName = fix_method_name(methodName)
359 pathUrl = methodDesc['path']
360 httpMethod = methodDesc['httpMethod']
361 methodId = methodDesc['id']
362
363 mediaPathUrl = None
364 accept = []
365 maxSize = 0
366 if 'mediaUpload' in methodDesc:
367 mediaUpload = methodDesc['mediaUpload']
368 mediaPathUrl = (rootDesc['rootUrl'] + 'upload/' + rootDesc['servicePath']
369 + pathUrl)
370 accept = mediaUpload['accept']
371 maxSize = _media_size_to_long(mediaUpload.get('maxSize', ''))
372
373 if 'parameters' not in methodDesc:
374 methodDesc['parameters'] = {}
375
376
377 for name, desc in rootDesc.get('parameters', {}).iteritems():
378 methodDesc['parameters'][name] = desc
379
380
381 for name in STACK_QUERY_PARAMETERS:
382 methodDesc['parameters'][name] = {
383 'type': 'string',
384 'location': 'query'
385 }
386
387 if httpMethod in ['PUT', 'POST', 'PATCH'] and 'request' in methodDesc:
388 methodDesc['parameters']['body'] = {
389 'description': 'The request body.',
390 'type': 'object',
391 'required': True,
392 }
393 if 'request' in methodDesc:
394 methodDesc['parameters']['body'].update(methodDesc['request'])
395 else:
396 methodDesc['parameters']['body']['type'] = 'object'
397 if 'mediaUpload' in methodDesc:
398 methodDesc['parameters']['media_body'] = {
399 'description':
400 'The filename of the media request body, or an instance of a '
401 'MediaUpload object.',
402 'type': 'string',
403 'required': False,
404 }
405 if 'body' in methodDesc['parameters']:
406 methodDesc['parameters']['body']['required'] = False
407
408 argmap = {}
409 required_params = []
410 repeated_params = []
411 pattern_params = {}
412 query_params = []
413 path_params = {}
414 param_type = {}
415 enum_params = {}
416
417
418 if 'parameters' in methodDesc:
419 for arg, desc in methodDesc['parameters'].iteritems():
420 param = key2param(arg)
421 argmap[param] = arg
422
423 if desc.get('pattern', ''):
424 pattern_params[param] = desc['pattern']
425 if desc.get('enum', ''):
426 enum_params[param] = desc['enum']
427 if desc.get('required', False):
428 required_params.append(param)
429 if desc.get('repeated', False):
430 repeated_params.append(param)
431 if desc.get('location') == 'query':
432 query_params.append(param)
433 if desc.get('location') == 'path':
434 path_params[param] = param
435 param_type[param] = desc.get('type', 'string')
436
437 for match in URITEMPLATE.finditer(pathUrl):
438 for namematch in VARNAME.finditer(match.group(0)):
439 name = key2param(namematch.group(0))
440 path_params[name] = name
441 if name in query_params:
442 query_params.remove(name)
443
444 def method(self, **kwargs):
445
446
447 for name in kwargs.iterkeys():
448 if name not in argmap:
449 raise TypeError('Got an unexpected keyword argument "%s"' % name)
450
451
452 keys = kwargs.keys()
453 for name in keys:
454 if kwargs[name] is None:
455 del kwargs[name]
456
457 for name in required_params:
458 if name not in kwargs:
459 raise TypeError('Missing required parameter "%s"' % name)
460
461 for name, regex in pattern_params.iteritems():
462 if name in kwargs:
463 if isinstance(kwargs[name], basestring):
464 pvalues = [kwargs[name]]
465 else:
466 pvalues = kwargs[name]
467 for pvalue in pvalues:
468 if re.match(regex, pvalue) is None:
469 raise TypeError(
470 'Parameter "%s" value "%s" does not match the pattern "%s"' %
471 (name, pvalue, regex))
472
473 for name, enums in enum_params.iteritems():
474 if name in kwargs:
475
476
477
478 if (name in repeated_params and
479 not isinstance(kwargs[name], basestring)):
480 values = kwargs[name]
481 else:
482 values = [kwargs[name]]
483 for value in values:
484 if value not in enums:
485 raise TypeError(
486 'Parameter "%s" value "%s" is not an allowed value in "%s"' %
487 (name, value, str(enums)))
488
489 actual_query_params = {}
490 actual_path_params = {}
491 for key, value in kwargs.iteritems():
492 to_type = param_type.get(key, 'string')
493
494 if key in repeated_params and type(value) == type([]):
495 cast_value = [_cast(x, to_type) for x in value]
496 else:
497 cast_value = _cast(value, to_type)
498 if key in query_params:
499 actual_query_params[argmap[key]] = cast_value
500 if key in path_params:
501 actual_path_params[argmap[key]] = cast_value
502 body_value = kwargs.get('body', None)
503 media_filename = kwargs.get('media_body', None)
504
505 if self._developerKey:
506 actual_query_params['key'] = self._developerKey
507
508 model = self._model
509 if methodName.endswith('_media'):
510 model = MediaModel()
511 elif 'response' not in methodDesc:
512 model = RawModel()
513
514 headers = {}
515 headers, params, query, body = model.request(headers,
516 actual_path_params, actual_query_params, body_value)
517
518 expanded_url = uritemplate.expand(pathUrl, params)
519 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
520
521 resumable = None
522 multipart_boundary = ''
523
524 if media_filename:
525
526 if isinstance(media_filename, basestring):
527 (media_mime_type, encoding) = mimetypes.guess_type(media_filename)
528 if media_mime_type is None:
529 raise UnknownFileType(media_filename)
530 if not mimeparse.best_match([media_mime_type], ','.join(accept)):
531 raise UnacceptableMimeTypeError(media_mime_type)
532 media_upload = MediaFileUpload(media_filename, media_mime_type)
533 elif isinstance(media_filename, MediaUpload):
534 media_upload = media_filename
535 else:
536 raise TypeError('media_filename must be str or MediaUpload.')
537
538
539 if maxSize > 0 and media_upload.size() > maxSize:
540 raise MediaUploadSizeError("Media larger than: %s" % maxSize)
541
542
543 expanded_url = uritemplate.expand(mediaPathUrl, params)
544 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
545 if media_upload.resumable():
546 url = _add_query_parameter(url, 'uploadType', 'resumable')
547
548 if media_upload.resumable():
549
550
551 resumable = media_upload
552 else:
553
554 if body is None:
555
556 headers['content-type'] = media_upload.mimetype()
557 body = media_upload.getbytes(0, media_upload.size())
558 url = _add_query_parameter(url, 'uploadType', 'media')
559 else:
560
561 msgRoot = MIMEMultipart('related')
562
563 setattr(msgRoot, '_write_headers', lambda self: None)
564
565
566 msg = MIMENonMultipart(*headers['content-type'].split('/'))
567 msg.set_payload(body)
568 msgRoot.attach(msg)
569
570
571 msg = MIMENonMultipart(*media_upload.mimetype().split('/'))
572 msg['Content-Transfer-Encoding'] = 'binary'
573
574 payload = media_upload.getbytes(0, media_upload.size())
575 msg.set_payload(payload)
576 msgRoot.attach(msg)
577 body = msgRoot.as_string()
578
579 multipart_boundary = msgRoot.get_boundary()
580 headers['content-type'] = ('multipart/related; '
581 'boundary="%s"') % multipart_boundary
582 url = _add_query_parameter(url, 'uploadType', 'multipart')
583
584 logger.info('URL being requested: %s' % url)
585 return self._requestBuilder(self._http,
586 model.response,
587 url,
588 method=httpMethod,
589 body=body,
590 headers=headers,
591 methodId=methodId,
592 resumable=resumable)
593
594 docs = [methodDesc.get('description', DEFAULT_METHOD_DOC), '\n\n']
595 if len(argmap) > 0:
596 docs.append('Args:\n')
597
598
599 skip_parameters = rootDesc.get('parameters', {}).keys()
600 skip_parameters.extend(STACK_QUERY_PARAMETERS)
601
602 all_args = argmap.keys()
603 args_ordered = [key2param(s) for s in methodDesc.get('parameterOrder', [])]
604
605
606 if 'body' in all_args:
607 args_ordered.append('body')
608
609 for name in all_args:
610 if name not in args_ordered:
611 args_ordered.append(name)
612
613 for arg in args_ordered:
614 if arg in skip_parameters:
615 continue
616
617 repeated = ''
618 if arg in repeated_params:
619 repeated = ' (repeated)'
620 required = ''
621 if arg in required_params:
622 required = ' (required)'
623 paramdesc = methodDesc['parameters'][argmap[arg]]
624 paramdoc = paramdesc.get('description', 'A parameter')
625 if '$ref' in paramdesc:
626 docs.append(
627 (' %s: object, %s%s%s\n The object takes the'
628 ' form of:\n\n%s\n\n') % (arg, paramdoc, required, repeated,
629 schema.prettyPrintByName(paramdesc['$ref'])))
630 else:
631 paramtype = paramdesc.get('type', 'string')
632 docs.append(' %s: %s, %s%s%s\n' % (arg, paramtype, paramdoc, required,
633 repeated))
634 enum = paramdesc.get('enum', [])
635 enumDesc = paramdesc.get('enumDescriptions', [])
636 if enum and enumDesc:
637 docs.append(' Allowed values\n')
638 for (name, desc) in zip(enum, enumDesc):
639 docs.append(' %s - %s\n' % (name, desc))
640 if 'response' in methodDesc:
641 if methodName.endswith('_media'):
642 docs.append('\nReturns:\n The media object as a string.\n\n ')
643 else:
644 docs.append('\nReturns:\n An object of the form:\n\n ')
645 docs.append(schema.prettyPrintSchema(methodDesc['response']))
646
647 setattr(method, '__doc__', ''.join(docs))
648 setattr(theclass, methodName, method)
649
650 def createNextMethod(theclass, methodName, methodDesc, rootDesc):
651 """Creates any _next methods for attaching to a Resource.
652
653 The _next methods allow for easy iteration through list() responses.
654
655 Args:
656 theclass: type, the class to attach methods to.
657 methodName: string, name of the method to use.
658 methodDesc: object, fragment of deserialized discovery document that
659 describes the method.
660 rootDesc: object, the entire deserialized discovery document.
661 """
662 methodName = fix_method_name(methodName)
663 methodId = methodDesc['id'] + '.next'
664
665 def methodNext(self, previous_request, previous_response):
666 """Retrieves the next page of results.
667
668 Args:
669 previous_request: The request for the previous page. (required)
670 previous_response: The response from the request for the previous page. (required)
671
672 Returns:
673 A request object that you can call 'execute()' on to request the next
674 page. Returns None if there are no more items in the collection.
675 """
676
677
678
679 if 'nextPageToken' not in previous_response:
680 return None
681
682 request = copy.copy(previous_request)
683
684 pageToken = previous_response['nextPageToken']
685 parsed = list(urlparse.urlparse(request.uri))
686 q = parse_qsl(parsed[4])
687
688
689 newq = [(key, value) for (key, value) in q if key != 'pageToken']
690 newq.append(('pageToken', pageToken))
691 parsed[4] = urllib.urlencode(newq)
692 uri = urlparse.urlunparse(parsed)
693
694 request.uri = uri
695
696 logger.info('URL being requested: %s' % uri)
697
698 return request
699
700 setattr(theclass, methodName, methodNext)
701
702
703 if 'methods' in resourceDesc:
704 for methodName, methodDesc in resourceDesc['methods'].iteritems():
705 createMethod(Resource, methodName, methodDesc, rootDesc)
706
707
708 if methodDesc.get('supportsMediaDownload', False):
709 createMethod(Resource, methodName + '_media', methodDesc, rootDesc)
710
711
712 if 'resources' in resourceDesc:
713
714 def createResourceMethod(theclass, methodName, methodDesc, rootDesc):
715 """Create a method on the Resource to access a nested Resource.
716
717 Args:
718 theclass: type, the class to attach methods to.
719 methodName: string, name of the method to use.
720 methodDesc: object, fragment of deserialized discovery document that
721 describes the method.
722 rootDesc: object, the entire deserialized discovery document.
723 """
724 methodName = fix_method_name(methodName)
725
726 def methodResource(self):
727 return _createResource(self._http, self._baseUrl, self._model,
728 self._requestBuilder, self._developerKey,
729 methodDesc, rootDesc, schema)
730
731 setattr(methodResource, '__doc__', 'A collection resource.')
732 setattr(methodResource, '__is_resource__', True)
733 setattr(theclass, methodName, methodResource)
734
735 for methodName, methodDesc in resourceDesc['resources'].iteritems():
736 createResourceMethod(Resource, methodName, methodDesc, rootDesc)
737
738
739
740
741 if 'methods' in resourceDesc:
742 for methodName, methodDesc in resourceDesc['methods'].iteritems():
743 if 'response' in methodDesc:
744 responseSchema = methodDesc['response']
745 if '$ref' in responseSchema:
746 responseSchema = schema.get(responseSchema['$ref'])
747 hasNextPageToken = 'nextPageToken' in responseSchema.get('properties',
748 {})
749 hasPageToken = 'pageToken' in methodDesc.get('parameters', {})
750 if hasNextPageToken and hasPageToken:
751 createNextMethod(Resource, methodName + '_next',
752 resourceDesc['methods'][methodName],
753 methodName)
754
755 return Resource()
756