1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """Client for discovery based APIs
16
17 A client library for Google's discovery based APIs.
18 """
19
20 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
21 __all__ = [
22 'build',
23 'build_from_document'
24 'fix_method_name',
25 'key2param'
26 ]
27
28 import copy
29 import httplib2
30 import logging
31 import os
32 import random
33 import re
34 import uritemplate
35 import urllib
36 import urlparse
37 import mimeparse
38 import mimetypes
39
40 try:
41 from urlparse import parse_qsl
42 except ImportError:
43 from cgi import parse_qsl
44
45 from apiclient.errors import HttpError
46 from apiclient.errors import InvalidJsonError
47 from apiclient.errors import MediaUploadSizeError
48 from apiclient.errors import UnacceptableMimeTypeError
49 from apiclient.errors import UnknownApiNameOrVersion
50 from apiclient.errors import UnknownLinkType
51 from apiclient.http import HttpRequest
52 from apiclient.http import MediaFileUpload
53 from apiclient.http import MediaUpload
54 from apiclient.model import JsonModel
55 from apiclient.model import RawModel
56 from apiclient.schema import Schemas
57 from email.mime.multipart import MIMEMultipart
58 from email.mime.nonmultipart import MIMENonMultipart
59 from oauth2client.anyjson import simplejson
60
61 logger = logging.getLogger(__name__)
62
63 URITEMPLATE = re.compile('{[^}]*}')
64 VARNAME = re.compile('[a-zA-Z0-9_-]+')
65 DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
66 '{api}/{apiVersion}/rest')
67 DEFAULT_METHOD_DOC = 'A description of how to use this function'
68
69
70 STACK_QUERY_PARAMETERS = ['trace', 'pp', 'userip', 'strict']
71
72
73 RESERVED_WORDS = ['and', 'assert', 'break', 'class', 'continue', 'def', 'del',
74 'elif', 'else', 'except', 'exec', 'finally', 'for', 'from',
75 'global', 'if', 'import', 'in', 'is', 'lambda', 'not', 'or',
76 'pass', 'print', 'raise', 'return', 'try', 'while' ]
77
78
80 """Fix method names to avoid reserved word conflicts.
81
82 Args:
83 name: string, method name.
84
85 Returns:
86 The name with a '_' prefixed if the name is a reserved word.
87 """
88 if name in RESERVED_WORDS:
89 return name + '_'
90 else:
91 return name
92
93
95 """Adds a query parameter to a url.
96
97 Replaces the current value if it already exists in the URL.
98
99 Args:
100 url: string, url to add the query parameter to.
101 name: string, query parameter name.
102 value: string, query parameter value.
103
104 Returns:
105 Updated query parameter. Does not update the url if value is None.
106 """
107 if value is None:
108 return url
109 else:
110 parsed = list(urlparse.urlparse(url))
111 q = dict(parse_qsl(parsed[4]))
112 q[name] = value
113 parsed[4] = urllib.urlencode(q)
114 return urlparse.urlunparse(parsed)
115
116
118 """Converts key names into parameter names.
119
120 For example, converting "max-results" -> "max_results"
121
122 Args:
123 key: string, the method key name.
124
125 Returns:
126 A safe method name based on the key name.
127 """
128 result = []
129 key = list(key)
130 if not key[0].isalpha():
131 result.append('x')
132 for c in key:
133 if c.isalnum():
134 result.append(c)
135 else:
136 result.append('_')
137
138 return ''.join(result)
139
140
141 -def build(serviceName,
142 version,
143 http=None,
144 discoveryServiceUrl=DISCOVERY_URI,
145 developerKey=None,
146 model=None,
147 requestBuilder=HttpRequest):
148 """Construct a Resource for interacting with an API.
149
150 Construct a Resource object for interacting with an API. The serviceName and
151 version are the names from the Discovery service.
152
153 Args:
154 serviceName: string, name of the service.
155 version: string, the version of the service.
156 http: httplib2.Http, An instance of httplib2.Http or something that acts
157 like it that HTTP requests will be made through.
158 discoveryServiceUrl: string, a URI Template that points to the location of
159 the discovery service. It should have two parameters {api} and
160 {apiVersion} that when filled in produce an absolute URI to the discovery
161 document for that service.
162 developerKey: string, key obtained from
163 https://code.google.com/apis/console.
164 model: apiclient.Model, converts to and from the wire format.
165 requestBuilder: apiclient.http.HttpRequest, encapsulator for an HTTP
166 request.
167
168 Returns:
169 A Resource object with methods for interacting with the service.
170 """
171 params = {
172 'api': serviceName,
173 'apiVersion': version
174 }
175
176 if http is None:
177 http = httplib2.Http()
178
179 requested_url = uritemplate.expand(discoveryServiceUrl, params)
180
181
182
183
184
185 if 'REMOTE_ADDR' in os.environ:
186 requested_url = _add_query_parameter(requested_url, 'userIp',
187 os.environ['REMOTE_ADDR'])
188 logger.info('URL being requested: %s' % requested_url)
189
190 resp, content = http.request(requested_url)
191
192 if resp.status == 404:
193 raise UnknownApiNameOrVersion("name: %s version: %s" % (serviceName,
194 version))
195 if resp.status >= 400:
196 raise HttpError(resp, content, requested_url)
197
198 try:
199 service = simplejson.loads(content)
200 except ValueError, e:
201 logger.error('Failed to parse as JSON: ' + content)
202 raise InvalidJsonError()
203
204 return build_from_document(content, discoveryServiceUrl, http=http,
205 developerKey=developerKey, model=model, requestBuilder=requestBuilder)
206
207
208 -def build_from_document(
209 service,
210 base,
211 future=None,
212 http=None,
213 developerKey=None,
214 model=None,
215 requestBuilder=HttpRequest):
216 """Create a Resource for interacting with an API.
217
218 Same as `build()`, but constructs the Resource object from a discovery
219 document that is it given, as opposed to retrieving one over HTTP.
220
221 Args:
222 service: string, discovery document.
223 base: string, base URI for all HTTP requests, usually the discovery URI.
224 future: string, discovery document with future capabilities (deprecated).
225 http: httplib2.Http, An instance of httplib2.Http or something that acts
226 like it that HTTP requests will be made through.
227 developerKey: string, Key for controlling API usage, generated
228 from the API Console.
229 model: Model class instance that serializes and de-serializes requests and
230 responses.
231 requestBuilder: Takes an http request and packages it up to be executed.
232
233 Returns:
234 A Resource object with methods for interacting with the service.
235 """
236
237
238 future = {}
239
240 service = simplejson.loads(service)
241 base = urlparse.urljoin(base, service['basePath'])
242 schema = Schemas(service)
243
244 if model is None:
245 features = service.get('features', [])
246 model = JsonModel('dataWrapper' in features)
247 resource = createResource(http, base, model, requestBuilder, developerKey,
248 service, service, schema)
249
250 return resource
251
252
253 -def _cast(value, schema_type):
254 """Convert value to a string based on JSON Schema type.
255
256 See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on
257 JSON Schema.
258
259 Args:
260 value: any, the value to convert
261 schema_type: string, the type that value should be interpreted as
262
263 Returns:
264 A string representation of 'value' based on the schema_type.
265 """
266 if schema_type == 'string':
267 if type(value) == type('') or type(value) == type(u''):
268 return value
269 else:
270 if value is None:
271 raise ValueError('String parameters can not be None.')
272 return str(value)
273 elif schema_type == 'integer':
274 return str(int(value))
275 elif schema_type == 'number':
276 return str(float(value))
277 elif schema_type == 'boolean':
278 return str(bool(value)).lower()
279 else:
280 if type(value) == type('') or type(value) == type(u''):
281 return value
282 else:
283 return str(value)
284
285
286 MULTIPLIERS = {
287 "KB": 2 ** 10,
288 "MB": 2 ** 20,
289 "GB": 2 ** 30,
290 "TB": 2 ** 40,
291 }
292
293
311
312
313 -def createResource(http, baseUrl, model, requestBuilder,
314 developerKey, resourceDesc, rootDesc, schema):
315 """Build a Resource from the API description.
316
317 Args:
318 http: httplib2.Http, Object to make http requests with.
319 baseUrl: string, base URL for the API. All requests are relative to this
320 URI.
321 model: apiclient.Model, converts to and from the wire format.
322 requestBuilder: class or callable that instantiates an
323 apiclient.HttpRequest object.
324 developerKey: string, key obtained from
325 https://code.google.com/apis/console
326 resourceDesc: object, section of deserialized discovery document that
327 describes a resource. Note that the top level discovery document
328 is considered a resource.
329 rootDesc: object, the entire deserialized discovery document.
330 schema: object, mapping of schema names to schema descriptions.
331
332 Returns:
333 An instance of Resource with all the methods attached for interacting with
334 that resource.
335 """
336
337 class Resource(object):
338 """A class for interacting with a resource."""
339
340 def __init__(self):
341 self._http = http
342 self._baseUrl = baseUrl
343 self._model = model
344 self._developerKey = developerKey
345 self._requestBuilder = requestBuilder
346
347 def createMethod(theclass, methodName, methodDesc, rootDesc):
348 """Creates a method for attaching to a Resource.
349
350 Args:
351 theclass: type, the class to attach methods to.
352 methodName: string, name of the method to use.
353 methodDesc: object, fragment of deserialized discovery document that
354 describes the method.
355 rootDesc: object, the entire deserialized discovery document.
356 """
357 methodName = fix_method_name(methodName)
358 pathUrl = methodDesc['path']
359 httpMethod = methodDesc['httpMethod']
360 methodId = methodDesc['id']
361
362 mediaPathUrl = None
363 accept = []
364 maxSize = 0
365 if 'mediaUpload' in methodDesc:
366 mediaUpload = methodDesc['mediaUpload']
367
368 parsed = list(urlparse.urlparse(baseUrl))
369 basePath = parsed[2]
370 mediaPathUrl = '/upload' + basePath + pathUrl
371 accept = mediaUpload['accept']
372 maxSize = _media_size_to_long(mediaUpload.get('maxSize', ''))
373
374 if 'parameters' not in methodDesc:
375 methodDesc['parameters'] = {}
376
377
378 for name, desc in rootDesc.get('parameters', {}).iteritems():
379 methodDesc['parameters'][name] = desc
380
381
382 for name in STACK_QUERY_PARAMETERS:
383 methodDesc['parameters'][name] = {
384 'type': 'string',
385 'location': 'query'
386 }
387
388 if httpMethod in ['PUT', 'POST', 'PATCH'] and 'request' in methodDesc:
389 methodDesc['parameters']['body'] = {
390 'description': 'The request body.',
391 'type': 'object',
392 'required': True,
393 }
394 if 'request' in methodDesc:
395 methodDesc['parameters']['body'].update(methodDesc['request'])
396 else:
397 methodDesc['parameters']['body']['type'] = 'object'
398 if 'mediaUpload' in methodDesc:
399 methodDesc['parameters']['media_body'] = {
400 'description': 'The filename of the media request body.',
401 'type': 'string',
402 'required': False,
403 }
404 if 'body' in methodDesc['parameters']:
405 methodDesc['parameters']['body']['required'] = False
406
407 argmap = {}
408 required_params = []
409 repeated_params = []
410 pattern_params = {}
411 query_params = []
412 path_params = {}
413 param_type = {}
414 enum_params = {}
415
416
417 if 'parameters' in methodDesc:
418 for arg, desc in methodDesc['parameters'].iteritems():
419 param = key2param(arg)
420 argmap[param] = arg
421
422 if desc.get('pattern', ''):
423 pattern_params[param] = desc['pattern']
424 if desc.get('enum', ''):
425 enum_params[param] = desc['enum']
426 if desc.get('required', False):
427 required_params.append(param)
428 if desc.get('repeated', False):
429 repeated_params.append(param)
430 if desc.get('location') == 'query':
431 query_params.append(param)
432 if desc.get('location') == 'path':
433 path_params[param] = param
434 param_type[param] = desc.get('type', 'string')
435
436 for match in URITEMPLATE.finditer(pathUrl):
437 for namematch in VARNAME.finditer(match.group(0)):
438 name = key2param(namematch.group(0))
439 path_params[name] = name
440 if name in query_params:
441 query_params.remove(name)
442
443 def method(self, **kwargs):
444
445 for name in kwargs.iterkeys():
446 if name not in argmap:
447 raise TypeError('Got an unexpected keyword argument "%s"' % name)
448
449 for name in required_params:
450 if name not in kwargs:
451 raise TypeError('Missing required parameter "%s"' % name)
452
453 for name, regex in pattern_params.iteritems():
454 if name in kwargs:
455 if isinstance(kwargs[name], basestring):
456 pvalues = [kwargs[name]]
457 else:
458 pvalues = kwargs[name]
459 for pvalue in pvalues:
460 if re.match(regex, pvalue) is None:
461 raise TypeError(
462 'Parameter "%s" value "%s" does not match the pattern "%s"' %
463 (name, pvalue, regex))
464
465 for name, enums in enum_params.iteritems():
466 if name in kwargs:
467
468
469
470 if (name in repeated_params and
471 not isinstance(kwargs[name], basestring)):
472 values = kwargs[name]
473 else:
474 values = [kwargs[name]]
475 for value in values:
476 if value not in enums:
477 raise TypeError(
478 'Parameter "%s" value "%s" is not an allowed value in "%s"' %
479 (name, value, str(enums)))
480
481 actual_query_params = {}
482 actual_path_params = {}
483 for key, value in kwargs.iteritems():
484 to_type = param_type.get(key, 'string')
485
486 if key in repeated_params and type(value) == type([]):
487 cast_value = [_cast(x, to_type) for x in value]
488 else:
489 cast_value = _cast(value, to_type)
490 if key in query_params:
491 actual_query_params[argmap[key]] = cast_value
492 if key in path_params:
493 actual_path_params[argmap[key]] = cast_value
494 body_value = kwargs.get('body', None)
495 media_filename = kwargs.get('media_body', None)
496
497 if self._developerKey:
498 actual_query_params['key'] = self._developerKey
499
500 model = self._model
501
502 if 'response' not in methodDesc:
503 model = RawModel()
504
505 headers = {}
506 headers, params, query, body = model.request(headers,
507 actual_path_params, actual_query_params, body_value)
508
509 expanded_url = uritemplate.expand(pathUrl, params)
510 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
511
512 resumable = None
513 multipart_boundary = ''
514
515 if media_filename:
516
517 if isinstance(media_filename, basestring):
518 (media_mime_type, encoding) = mimetypes.guess_type(media_filename)
519 if media_mime_type is None:
520 raise UnknownFileType(media_filename)
521 if not mimeparse.best_match([media_mime_type], ','.join(accept)):
522 raise UnacceptableMimeTypeError(media_mime_type)
523 media_upload = MediaFileUpload(media_filename, media_mime_type)
524 elif isinstance(media_filename, MediaUpload):
525 media_upload = media_filename
526 else:
527 raise TypeError('media_filename must be str or MediaUpload.')
528
529
530 if maxSize > 0 and media_upload.size() > maxSize:
531 raise MediaUploadSizeError("Media larger than: %s" % maxSize)
532
533
534 expanded_url = uritemplate.expand(mediaPathUrl, params)
535 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
536 if media_upload.resumable():
537 url = _add_query_parameter(url, 'uploadType', 'resumable')
538
539 if media_upload.resumable():
540
541
542 resumable = media_upload
543 else:
544
545 if body is None:
546
547 headers['content-type'] = media_upload.mimetype()
548 body = media_upload.getbytes(0, media_upload.size())
549 url = _add_query_parameter(url, 'uploadType', 'media')
550 else:
551
552 msgRoot = MIMEMultipart('related')
553
554 setattr(msgRoot, '_write_headers', lambda self: None)
555
556
557 msg = MIMENonMultipart(*headers['content-type'].split('/'))
558 msg.set_payload(body)
559 msgRoot.attach(msg)
560
561
562 msg = MIMENonMultipart(*media_upload.mimetype().split('/'))
563 msg['Content-Transfer-Encoding'] = 'binary'
564
565 payload = media_upload.getbytes(0, media_upload.size())
566 msg.set_payload(payload)
567 msgRoot.attach(msg)
568 body = msgRoot.as_string()
569
570 multipart_boundary = msgRoot.get_boundary()
571 headers['content-type'] = ('multipart/related; '
572 'boundary="%s"') % multipart_boundary
573 url = _add_query_parameter(url, 'uploadType', 'multipart')
574
575 logger.info('URL being requested: %s' % url)
576 return self._requestBuilder(self._http,
577 model.response,
578 url,
579 method=httpMethod,
580 body=body,
581 headers=headers,
582 methodId=methodId,
583 resumable=resumable)
584
585 docs = [methodDesc.get('description', DEFAULT_METHOD_DOC), '\n\n']
586 if len(argmap) > 0:
587 docs.append('Args:\n')
588
589
590 skip_parameters = rootDesc.get('parameters', {}).keys()
591 skip_parameters.append(STACK_QUERY_PARAMETERS)
592
593 for arg in argmap.iterkeys():
594 if arg in skip_parameters:
595 continue
596
597 repeated = ''
598 if arg in repeated_params:
599 repeated = ' (repeated)'
600 required = ''
601 if arg in required_params:
602 required = ' (required)'
603 paramdesc = methodDesc['parameters'][argmap[arg]]
604 paramdoc = paramdesc.get('description', 'A parameter')
605 if '$ref' in paramdesc:
606 docs.append(
607 (' %s: object, %s%s%s\n The object takes the'
608 ' form of:\n\n%s\n\n') % (arg, paramdoc, required, repeated,
609 schema.prettyPrintByName(paramdesc['$ref'])))
610 else:
611 paramtype = paramdesc.get('type', 'string')
612 docs.append(' %s: %s, %s%s%s\n' % (arg, paramtype, paramdoc, required,
613 repeated))
614 enum = paramdesc.get('enum', [])
615 enumDesc = paramdesc.get('enumDescriptions', [])
616 if enum and enumDesc:
617 docs.append(' Allowed values\n')
618 for (name, desc) in zip(enum, enumDesc):
619 docs.append(' %s - %s\n' % (name, desc))
620 if 'response' in methodDesc:
621 docs.append('\nReturns:\n An object of the form\n\n ')
622 docs.append(schema.prettyPrintSchema(methodDesc['response']))
623
624 setattr(method, '__doc__', ''.join(docs))
625 setattr(theclass, methodName, method)
626
627 def createNextMethod(theclass, methodName, methodDesc, rootDesc):
628 """Creates any _next methods for attaching to a Resource.
629
630 The _next methods allow for easy iteration through list() responses.
631
632 Args:
633 theclass: type, the class to attach methods to.
634 methodName: string, name of the method to use.
635 methodDesc: object, fragment of deserialized discovery document that
636 describes the method.
637 rootDesc: object, the entire deserialized discovery document.
638 """
639 methodName = fix_method_name(methodName)
640 methodId = methodDesc['id'] + '.next'
641
642 def methodNext(self, previous_request, previous_response):
643 """Retrieves the next page of results.
644
645 Args:
646 previous_request: The request for the previous page.
647 previous_response: The response from the request for the previous page.
648
649 Returns:
650 A request object that you can call 'execute()' on to request the next
651 page. Returns None if there are no more items in the collection.
652 """
653
654
655
656 if 'nextPageToken' not in previous_response:
657 return None
658
659 request = copy.copy(previous_request)
660
661 pageToken = previous_response['nextPageToken']
662 parsed = list(urlparse.urlparse(request.uri))
663 q = parse_qsl(parsed[4])
664
665
666 newq = [(key, value) for (key, value) in q if key != 'pageToken']
667 newq.append(('pageToken', pageToken))
668 parsed[4] = urllib.urlencode(newq)
669 uri = urlparse.urlunparse(parsed)
670
671 request.uri = uri
672
673 logger.info('URL being requested: %s' % uri)
674
675 return request
676
677 setattr(theclass, methodName, methodNext)
678
679
680 if 'methods' in resourceDesc:
681 for methodName, methodDesc in resourceDesc['methods'].iteritems():
682 createMethod(Resource, methodName, methodDesc, rootDesc)
683
684
685 if 'resources' in resourceDesc:
686
687 def createResourceMethod(theclass, methodName, methodDesc, rootDesc):
688 """Create a method on the Resource to access a nested Resource.
689
690 Args:
691 theclass: type, the class to attach methods to.
692 methodName: string, name of the method to use.
693 methodDesc: object, fragment of deserialized discovery document that
694 describes the method.
695 rootDesc: object, the entire deserialized discovery document.
696 """
697 methodName = fix_method_name(methodName)
698
699 def methodResource(self):
700 return createResource(self._http, self._baseUrl, self._model,
701 self._requestBuilder, self._developerKey,
702 methodDesc, rootDesc, schema)
703
704 setattr(methodResource, '__doc__', 'A collection resource.')
705 setattr(methodResource, '__is_resource__', True)
706 setattr(theclass, methodName, methodResource)
707
708 for methodName, methodDesc in resourceDesc['resources'].iteritems():
709 createResourceMethod(Resource, methodName, methodDesc, rootDesc)
710
711
712
713
714 if 'methods' in resourceDesc:
715 for methodName, methodDesc in resourceDesc['methods'].iteritems():
716 if 'response' in methodDesc:
717 responseSchema = methodDesc['response']
718 if '$ref' in responseSchema:
719 responseSchema = schema.get(responseSchema['$ref'])
720 hasNextPageToken = 'nextPageToken' in responseSchema.get('properties',
721 {})
722 hasPageToken = 'pageToken' in methodDesc.get('parameters', {})
723 if hasNextPageToken and hasPageToken:
724 createNextMethod(Resource, methodName + '_next',
725 resourceDesc['methods'][methodName],
726 methodName)
727
728 return Resource()
729