1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """Client for discovery based APIs
16
17 A client library for Google's discovery based APIs.
18 """
19
20 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
21 __all__ = [
22 'build',
23 'build_from_document'
24 'fix_method_name',
25 'key2param'
26 ]
27
28 import copy
29 import httplib2
30 import logging
31 import os
32 import random
33 import re
34 import uritemplate
35 import urllib
36 import urlparse
37 import mimeparse
38 import mimetypes
39
40 try:
41 from urlparse import parse_qsl
42 except ImportError:
43 from cgi import parse_qsl
44
45 from apiclient.errors import HttpError
46 from apiclient.errors import InvalidJsonError
47 from apiclient.errors import MediaUploadSizeError
48 from apiclient.errors import UnacceptableMimeTypeError
49 from apiclient.errors import UnknownApiNameOrVersion
50 from apiclient.errors import UnknownLinkType
51 from apiclient.http import HttpRequest
52 from apiclient.http import MediaFileUpload
53 from apiclient.http import MediaUpload
54 from apiclient.model import JsonModel
55 from apiclient.model import MediaModel
56 from apiclient.model import RawModel
57 from apiclient.schema import Schemas
58 from email.mime.multipart import MIMEMultipart
59 from email.mime.nonmultipart import MIMENonMultipart
60 from oauth2client.anyjson import simplejson
61
62 logger = logging.getLogger(__name__)
63
64 URITEMPLATE = re.compile('{[^}]*}')
65 VARNAME = re.compile('[a-zA-Z0-9_-]+')
66 DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
67 '{api}/{apiVersion}/rest')
68 DEFAULT_METHOD_DOC = 'A description of how to use this function'
69
70
71 STACK_QUERY_PARAMETERS = ['trace', 'pp', 'userip', 'strict']
72
73
74 RESERVED_WORDS = ['and', 'assert', 'break', 'class', 'continue', 'def', 'del',
75 'elif', 'else', 'except', 'exec', 'finally', 'for', 'from',
76 'global', 'if', 'import', 'in', 'is', 'lambda', 'not', 'or',
77 'pass', 'print', 'raise', 'return', 'try', 'while' ]
78
79
81 """Fix method names to avoid reserved word conflicts.
82
83 Args:
84 name: string, method name.
85
86 Returns:
87 The name with a '_' prefixed if the name is a reserved word.
88 """
89 if name in RESERVED_WORDS:
90 return name + '_'
91 else:
92 return name
93
94
96 """Adds a query parameter to a url.
97
98 Replaces the current value if it already exists in the URL.
99
100 Args:
101 url: string, url to add the query parameter to.
102 name: string, query parameter name.
103 value: string, query parameter value.
104
105 Returns:
106 Updated query parameter. Does not update the url if value is None.
107 """
108 if value is None:
109 return url
110 else:
111 parsed = list(urlparse.urlparse(url))
112 q = dict(parse_qsl(parsed[4]))
113 q[name] = value
114 parsed[4] = urllib.urlencode(q)
115 return urlparse.urlunparse(parsed)
116
117
119 """Converts key names into parameter names.
120
121 For example, converting "max-results" -> "max_results"
122
123 Args:
124 key: string, the method key name.
125
126 Returns:
127 A safe method name based on the key name.
128 """
129 result = []
130 key = list(key)
131 if not key[0].isalpha():
132 result.append('x')
133 for c in key:
134 if c.isalnum():
135 result.append(c)
136 else:
137 result.append('_')
138
139 return ''.join(result)
140
141
142 -def build(serviceName,
143 version,
144 http=None,
145 discoveryServiceUrl=DISCOVERY_URI,
146 developerKey=None,
147 model=None,
148 requestBuilder=HttpRequest):
149 """Construct a Resource for interacting with an API.
150
151 Construct a Resource object for interacting with an API. The serviceName and
152 version are the names from the Discovery service.
153
154 Args:
155 serviceName: string, name of the service.
156 version: string, the version of the service.
157 http: httplib2.Http, An instance of httplib2.Http or something that acts
158 like it that HTTP requests will be made through.
159 discoveryServiceUrl: string, a URI Template that points to the location of
160 the discovery service. It should have two parameters {api} and
161 {apiVersion} that when filled in produce an absolute URI to the discovery
162 document for that service.
163 developerKey: string, key obtained from
164 https://code.google.com/apis/console.
165 model: apiclient.Model, converts to and from the wire format.
166 requestBuilder: apiclient.http.HttpRequest, encapsulator for an HTTP
167 request.
168
169 Returns:
170 A Resource object with methods for interacting with the service.
171 """
172 params = {
173 'api': serviceName,
174 'apiVersion': version
175 }
176
177 if http is None:
178 http = httplib2.Http()
179
180 requested_url = uritemplate.expand(discoveryServiceUrl, params)
181
182
183
184
185
186 if 'REMOTE_ADDR' in os.environ:
187 requested_url = _add_query_parameter(requested_url, 'userIp',
188 os.environ['REMOTE_ADDR'])
189 logger.info('URL being requested: %s' % requested_url)
190
191 resp, content = http.request(requested_url)
192
193 if resp.status == 404:
194 raise UnknownApiNameOrVersion("name: %s version: %s" % (serviceName,
195 version))
196 if resp.status >= 400:
197 raise HttpError(resp, content, requested_url)
198
199 try:
200 service = simplejson.loads(content)
201 except ValueError, e:
202 logger.error('Failed to parse as JSON: ' + content)
203 raise InvalidJsonError()
204
205 return build_from_document(content, discoveryServiceUrl, http=http,
206 developerKey=developerKey, model=model, requestBuilder=requestBuilder)
207
208
209 -def build_from_document(
210 service,
211 base,
212 future=None,
213 http=None,
214 developerKey=None,
215 model=None,
216 requestBuilder=HttpRequest):
217 """Create a Resource for interacting with an API.
218
219 Same as `build()`, but constructs the Resource object from a discovery
220 document that is it given, as opposed to retrieving one over HTTP.
221
222 Args:
223 service: string, discovery document.
224 base: string, base URI for all HTTP requests, usually the discovery URI.
225 future: string, discovery document with future capabilities (deprecated).
226 http: httplib2.Http, An instance of httplib2.Http or something that acts
227 like it that HTTP requests will be made through.
228 developerKey: string, Key for controlling API usage, generated
229 from the API Console.
230 model: Model class instance that serializes and de-serializes requests and
231 responses.
232 requestBuilder: Takes an http request and packages it up to be executed.
233
234 Returns:
235 A Resource object with methods for interacting with the service.
236 """
237
238
239 future = {}
240
241 service = simplejson.loads(service)
242 base = urlparse.urljoin(base, service['basePath'])
243 schema = Schemas(service)
244
245 if model is None:
246 features = service.get('features', [])
247 model = JsonModel('dataWrapper' in features)
248 resource = _createResource(http, base, model, requestBuilder, developerKey,
249 service, service, schema)
250
251 return resource
252
253
254 -def _cast(value, schema_type):
255 """Convert value to a string based on JSON Schema type.
256
257 See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on
258 JSON Schema.
259
260 Args:
261 value: any, the value to convert
262 schema_type: string, the type that value should be interpreted as
263
264 Returns:
265 A string representation of 'value' based on the schema_type.
266 """
267 if schema_type == 'string':
268 if type(value) == type('') or type(value) == type(u''):
269 return value
270 else:
271 if value is None:
272 raise ValueError('String parameters can not be None.')
273 return str(value)
274 elif schema_type == 'integer':
275 return str(int(value))
276 elif schema_type == 'number':
277 return str(float(value))
278 elif schema_type == 'boolean':
279 return str(bool(value)).lower()
280 else:
281 if type(value) == type('') or type(value) == type(u''):
282 return value
283 else:
284 return str(value)
285
286
287 MULTIPLIERS = {
288 "KB": 2 ** 10,
289 "MB": 2 ** 20,
290 "GB": 2 ** 30,
291 "TB": 2 ** 40,
292 }
293
294
312
313
314 -def _createResource(http, baseUrl, model, requestBuilder,
315 developerKey, resourceDesc, rootDesc, schema):
316 """Build a Resource from the API description.
317
318 Args:
319 http: httplib2.Http, Object to make http requests with.
320 baseUrl: string, base URL for the API. All requests are relative to this
321 URI.
322 model: apiclient.Model, converts to and from the wire format.
323 requestBuilder: class or callable that instantiates an
324 apiclient.HttpRequest object.
325 developerKey: string, key obtained from
326 https://code.google.com/apis/console
327 resourceDesc: object, section of deserialized discovery document that
328 describes a resource. Note that the top level discovery document
329 is considered a resource.
330 rootDesc: object, the entire deserialized discovery document.
331 schema: object, mapping of schema names to schema descriptions.
332
333 Returns:
334 An instance of Resource with all the methods attached for interacting with
335 that resource.
336 """
337
338 class Resource(object):
339 """A class for interacting with a resource."""
340
341 def __init__(self):
342 self._http = http
343 self._baseUrl = baseUrl
344 self._model = model
345 self._developerKey = developerKey
346 self._requestBuilder = requestBuilder
347
348 def createMethod(theclass, methodName, methodDesc, rootDesc):
349 """Creates a method for attaching to a Resource.
350
351 Args:
352 theclass: type, the class to attach methods to.
353 methodName: string, name of the method to use.
354 methodDesc: object, fragment of deserialized discovery document that
355 describes the method.
356 rootDesc: object, the entire deserialized discovery document.
357 """
358 methodName = fix_method_name(methodName)
359 pathUrl = methodDesc['path']
360 httpMethod = methodDesc['httpMethod']
361 methodId = methodDesc['id']
362
363 mediaPathUrl = None
364 accept = []
365 maxSize = 0
366 if 'mediaUpload' in methodDesc:
367 mediaUpload = methodDesc['mediaUpload']
368
369 parsed = list(urlparse.urlparse(baseUrl))
370 basePath = parsed[2]
371 mediaPathUrl = '/upload' + basePath + pathUrl
372 accept = mediaUpload['accept']
373 maxSize = _media_size_to_long(mediaUpload.get('maxSize', ''))
374
375 if 'parameters' not in methodDesc:
376 methodDesc['parameters'] = {}
377
378
379 for name, desc in rootDesc.get('parameters', {}).iteritems():
380 methodDesc['parameters'][name] = desc
381
382
383 for name in STACK_QUERY_PARAMETERS:
384 methodDesc['parameters'][name] = {
385 'type': 'string',
386 'location': 'query'
387 }
388
389 if httpMethod in ['PUT', 'POST', 'PATCH'] and 'request' in methodDesc:
390 methodDesc['parameters']['body'] = {
391 'description': 'The request body.',
392 'type': 'object',
393 'required': True,
394 }
395 if 'request' in methodDesc:
396 methodDesc['parameters']['body'].update(methodDesc['request'])
397 else:
398 methodDesc['parameters']['body']['type'] = 'object'
399 if 'mediaUpload' in methodDesc:
400 methodDesc['parameters']['media_body'] = {
401 'description': 'The filename of the media request body.',
402 'type': 'string',
403 'required': False,
404 }
405 if 'body' in methodDesc['parameters']:
406 methodDesc['parameters']['body']['required'] = False
407
408 argmap = {}
409 required_params = []
410 repeated_params = []
411 pattern_params = {}
412 query_params = []
413 path_params = {}
414 param_type = {}
415 enum_params = {}
416
417
418 if 'parameters' in methodDesc:
419 for arg, desc in methodDesc['parameters'].iteritems():
420 param = key2param(arg)
421 argmap[param] = arg
422
423 if desc.get('pattern', ''):
424 pattern_params[param] = desc['pattern']
425 if desc.get('enum', ''):
426 enum_params[param] = desc['enum']
427 if desc.get('required', False):
428 required_params.append(param)
429 if desc.get('repeated', False):
430 repeated_params.append(param)
431 if desc.get('location') == 'query':
432 query_params.append(param)
433 if desc.get('location') == 'path':
434 path_params[param] = param
435 param_type[param] = desc.get('type', 'string')
436
437 for match in URITEMPLATE.finditer(pathUrl):
438 for namematch in VARNAME.finditer(match.group(0)):
439 name = key2param(namematch.group(0))
440 path_params[name] = name
441 if name in query_params:
442 query_params.remove(name)
443
444 def method(self, **kwargs):
445
446 for name in kwargs.iterkeys():
447 if name not in argmap:
448 raise TypeError('Got an unexpected keyword argument "%s"' % name)
449
450 for name in required_params:
451 if name not in kwargs:
452 raise TypeError('Missing required parameter "%s"' % name)
453
454 for name, regex in pattern_params.iteritems():
455 if name in kwargs:
456 if isinstance(kwargs[name], basestring):
457 pvalues = [kwargs[name]]
458 else:
459 pvalues = kwargs[name]
460 for pvalue in pvalues:
461 if re.match(regex, pvalue) is None:
462 raise TypeError(
463 'Parameter "%s" value "%s" does not match the pattern "%s"' %
464 (name, pvalue, regex))
465
466 for name, enums in enum_params.iteritems():
467 if name in kwargs:
468
469
470
471 if (name in repeated_params and
472 not isinstance(kwargs[name], basestring)):
473 values = kwargs[name]
474 else:
475 values = [kwargs[name]]
476 for value in values:
477 if value not in enums:
478 raise TypeError(
479 'Parameter "%s" value "%s" is not an allowed value in "%s"' %
480 (name, value, str(enums)))
481
482 actual_query_params = {}
483 actual_path_params = {}
484 for key, value in kwargs.iteritems():
485 to_type = param_type.get(key, 'string')
486
487 if key in repeated_params and type(value) == type([]):
488 cast_value = [_cast(x, to_type) for x in value]
489 else:
490 cast_value = _cast(value, to_type)
491 if key in query_params:
492 actual_query_params[argmap[key]] = cast_value
493 if key in path_params:
494 actual_path_params[argmap[key]] = cast_value
495 body_value = kwargs.get('body', None)
496 media_filename = kwargs.get('media_body', None)
497
498 if self._developerKey:
499 actual_query_params['key'] = self._developerKey
500
501 model = self._model
502
503 if methodName.endswith('_media'):
504 model = MediaModel()
505 elif 'response' not in methodDesc:
506 model = RawModel()
507
508 headers = {}
509 headers, params, query, body = model.request(headers,
510 actual_path_params, actual_query_params, body_value)
511
512 expanded_url = uritemplate.expand(pathUrl, params)
513 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
514
515 resumable = None
516 multipart_boundary = ''
517
518 if media_filename:
519
520 if isinstance(media_filename, basestring):
521 (media_mime_type, encoding) = mimetypes.guess_type(media_filename)
522 if media_mime_type is None:
523 raise UnknownFileType(media_filename)
524 if not mimeparse.best_match([media_mime_type], ','.join(accept)):
525 raise UnacceptableMimeTypeError(media_mime_type)
526 media_upload = MediaFileUpload(media_filename, media_mime_type)
527 elif isinstance(media_filename, MediaUpload):
528 media_upload = media_filename
529 else:
530 raise TypeError('media_filename must be str or MediaUpload.')
531
532
533 if maxSize > 0 and media_upload.size() > maxSize:
534 raise MediaUploadSizeError("Media larger than: %s" % maxSize)
535
536
537 expanded_url = uritemplate.expand(mediaPathUrl, params)
538 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
539 if media_upload.resumable():
540 url = _add_query_parameter(url, 'uploadType', 'resumable')
541
542 if media_upload.resumable():
543
544
545 resumable = media_upload
546 else:
547
548 if body is None:
549
550 headers['content-type'] = media_upload.mimetype()
551 body = media_upload.getbytes(0, media_upload.size())
552 url = _add_query_parameter(url, 'uploadType', 'media')
553 else:
554
555 msgRoot = MIMEMultipart('related')
556
557 setattr(msgRoot, '_write_headers', lambda self: None)
558
559
560 msg = MIMENonMultipart(*headers['content-type'].split('/'))
561 msg.set_payload(body)
562 msgRoot.attach(msg)
563
564
565 msg = MIMENonMultipart(*media_upload.mimetype().split('/'))
566 msg['Content-Transfer-Encoding'] = 'binary'
567
568 payload = media_upload.getbytes(0, media_upload.size())
569 msg.set_payload(payload)
570 msgRoot.attach(msg)
571 body = msgRoot.as_string()
572
573 multipart_boundary = msgRoot.get_boundary()
574 headers['content-type'] = ('multipart/related; '
575 'boundary="%s"') % multipart_boundary
576 url = _add_query_parameter(url, 'uploadType', 'multipart')
577
578 logger.info('URL being requested: %s' % url)
579 return self._requestBuilder(self._http,
580 model.response,
581 url,
582 method=httpMethod,
583 body=body,
584 headers=headers,
585 methodId=methodId,
586 resumable=resumable)
587
588 docs = [methodDesc.get('description', DEFAULT_METHOD_DOC), '\n\n']
589 if len(argmap) > 0:
590 docs.append('Args:\n')
591
592
593 skip_parameters = rootDesc.get('parameters', {}).keys()
594 skip_parameters.append(STACK_QUERY_PARAMETERS)
595
596 for arg in argmap.iterkeys():
597 if arg in skip_parameters:
598 continue
599
600 repeated = ''
601 if arg in repeated_params:
602 repeated = ' (repeated)'
603 required = ''
604 if arg in required_params:
605 required = ' (required)'
606 paramdesc = methodDesc['parameters'][argmap[arg]]
607 paramdoc = paramdesc.get('description', 'A parameter')
608 if '$ref' in paramdesc:
609 docs.append(
610 (' %s: object, %s%s%s\n The object takes the'
611 ' form of:\n\n%s\n\n') % (arg, paramdoc, required, repeated,
612 schema.prettyPrintByName(paramdesc['$ref'])))
613 else:
614 paramtype = paramdesc.get('type', 'string')
615 docs.append(' %s: %s, %s%s%s\n' % (arg, paramtype, paramdoc, required,
616 repeated))
617 enum = paramdesc.get('enum', [])
618 enumDesc = paramdesc.get('enumDescriptions', [])
619 if enum and enumDesc:
620 docs.append(' Allowed values\n')
621 for (name, desc) in zip(enum, enumDesc):
622 docs.append(' %s - %s\n' % (name, desc))
623 if 'response' in methodDesc:
624 if methodName.endswith('_media'):
625 docs.append('\nReturns:\n The media object as a string.\n\n ')
626 else:
627 docs.append('\nReturns:\n An object of the form:\n\n ')
628 docs.append(schema.prettyPrintSchema(methodDesc['response']))
629
630 setattr(method, '__doc__', ''.join(docs))
631 setattr(theclass, methodName, method)
632
633 def createNextMethod(theclass, methodName, methodDesc, rootDesc):
634 """Creates any _next methods for attaching to a Resource.
635
636 The _next methods allow for easy iteration through list() responses.
637
638 Args:
639 theclass: type, the class to attach methods to.
640 methodName: string, name of the method to use.
641 methodDesc: object, fragment of deserialized discovery document that
642 describes the method.
643 rootDesc: object, the entire deserialized discovery document.
644 """
645 methodName = fix_method_name(methodName)
646 methodId = methodDesc['id'] + '.next'
647
648 def methodNext(self, previous_request, previous_response):
649 """Retrieves the next page of results.
650
651 Args:
652 previous_request: The request for the previous page.
653 previous_response: The response from the request for the previous page.
654
655 Returns:
656 A request object that you can call 'execute()' on to request the next
657 page. Returns None if there are no more items in the collection.
658 """
659
660
661
662 if 'nextPageToken' not in previous_response:
663 return None
664
665 request = copy.copy(previous_request)
666
667 pageToken = previous_response['nextPageToken']
668 parsed = list(urlparse.urlparse(request.uri))
669 q = parse_qsl(parsed[4])
670
671
672 newq = [(key, value) for (key, value) in q if key != 'pageToken']
673 newq.append(('pageToken', pageToken))
674 parsed[4] = urllib.urlencode(newq)
675 uri = urlparse.urlunparse(parsed)
676
677 request.uri = uri
678
679 logger.info('URL being requested: %s' % uri)
680
681 return request
682
683 setattr(theclass, methodName, methodNext)
684
685
686 if 'methods' in resourceDesc:
687 for methodName, methodDesc in resourceDesc['methods'].iteritems():
688 createMethod(Resource, methodName, methodDesc, rootDesc)
689
690
691 if methodDesc.get('supportsMediaDownload', False):
692 createMethod(Resource, methodName + '_media', methodDesc, rootDesc)
693
694
695 if 'resources' in resourceDesc:
696
697 def createResourceMethod(theclass, methodName, methodDesc, rootDesc):
698 """Create a method on the Resource to access a nested Resource.
699
700 Args:
701 theclass: type, the class to attach methods to.
702 methodName: string, name of the method to use.
703 methodDesc: object, fragment of deserialized discovery document that
704 describes the method.
705 rootDesc: object, the entire deserialized discovery document.
706 """
707 methodName = fix_method_name(methodName)
708
709 def methodResource(self):
710 return _createResource(self._http, self._baseUrl, self._model,
711 self._requestBuilder, self._developerKey,
712 methodDesc, rootDesc, schema)
713
714 setattr(methodResource, '__doc__', 'A collection resource.')
715 setattr(methodResource, '__is_resource__', True)
716 setattr(theclass, methodName, methodResource)
717
718 for methodName, methodDesc in resourceDesc['resources'].iteritems():
719 createResourceMethod(Resource, methodName, methodDesc, rootDesc)
720
721
722
723
724 if 'methods' in resourceDesc:
725 for methodName, methodDesc in resourceDesc['methods'].iteritems():
726 if 'response' in methodDesc:
727 responseSchema = methodDesc['response']
728 if '$ref' in responseSchema:
729 responseSchema = schema.get(responseSchema['$ref'])
730 hasNextPageToken = 'nextPageToken' in responseSchema.get('properties',
731 {})
732 hasPageToken = 'pageToken' in methodDesc.get('parameters', {})
733 if hasNextPageToken and hasPageToken:
734 createNextMethod(Resource, methodName + '_next',
735 resourceDesc['methods'][methodName],
736 methodName)
737
738 return Resource()
739