1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """Client for discovery based APIs
16
17 A client library for Google's discovery based APIs.
18 """
19
20 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
21 __all__ = [
22 'build',
23 'build_from_document'
24 'fix_method_name',
25 'key2param'
26 ]
27
28 import copy
29 import httplib2
30 import logging
31 import os
32 import random
33 import re
34 import uritemplate
35 import urllib
36 import urlparse
37 import mimeparse
38 import mimetypes
39
40 try:
41 from urlparse import parse_qsl
42 except ImportError:
43 from cgi import parse_qsl
44
45 from apiclient.errors import HttpError
46 from apiclient.errors import InvalidJsonError
47 from apiclient.errors import MediaUploadSizeError
48 from apiclient.errors import UnacceptableMimeTypeError
49 from apiclient.errors import UnknownApiNameOrVersion
50 from apiclient.errors import UnknownLinkType
51 from apiclient.http import HttpRequest
52 from apiclient.http import MediaFileUpload
53 from apiclient.http import MediaUpload
54 from apiclient.model import JsonModel
55 from apiclient.model import MediaModel
56 from apiclient.model import RawModel
57 from apiclient.schema import Schemas
58 from email.mime.multipart import MIMEMultipart
59 from email.mime.nonmultipart import MIMENonMultipart
60 from oauth2client.anyjson import simplejson
61
62 logger = logging.getLogger(__name__)
63
64 URITEMPLATE = re.compile('{[^}]*}')
65 VARNAME = re.compile('[a-zA-Z0-9_-]+')
66 DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
67 '{api}/{apiVersion}/rest')
68 DEFAULT_METHOD_DOC = 'A description of how to use this function'
69
70
71 STACK_QUERY_PARAMETERS = ['trace', 'pp', 'userip', 'strict']
72
73
74 RESERVED_WORDS = ['and', 'assert', 'break', 'class', 'continue', 'def', 'del',
75 'elif', 'else', 'except', 'exec', 'finally', 'for', 'from',
76 'global', 'if', 'import', 'in', 'is', 'lambda', 'not', 'or',
77 'pass', 'print', 'raise', 'return', 'try', 'while' ]
78
79
81 """Fix method names to avoid reserved word conflicts.
82
83 Args:
84 name: string, method name.
85
86 Returns:
87 The name with a '_' prefixed if the name is a reserved word.
88 """
89 if name in RESERVED_WORDS:
90 return name + '_'
91 else:
92 return name
93
94
96 """Adds a query parameter to a url.
97
98 Replaces the current value if it already exists in the URL.
99
100 Args:
101 url: string, url to add the query parameter to.
102 name: string, query parameter name.
103 value: string, query parameter value.
104
105 Returns:
106 Updated query parameter. Does not update the url if value is None.
107 """
108 if value is None:
109 return url
110 else:
111 parsed = list(urlparse.urlparse(url))
112 q = dict(parse_qsl(parsed[4]))
113 q[name] = value
114 parsed[4] = urllib.urlencode(q)
115 return urlparse.urlunparse(parsed)
116
117
119 """Converts key names into parameter names.
120
121 For example, converting "max-results" -> "max_results"
122
123 Args:
124 key: string, the method key name.
125
126 Returns:
127 A safe method name based on the key name.
128 """
129 result = []
130 key = list(key)
131 if not key[0].isalpha():
132 result.append('x')
133 for c in key:
134 if c.isalnum():
135 result.append(c)
136 else:
137 result.append('_')
138
139 return ''.join(result)
140
141
142 -def build(serviceName,
143 version,
144 http=None,
145 discoveryServiceUrl=DISCOVERY_URI,
146 developerKey=None,
147 model=None,
148 requestBuilder=HttpRequest):
149 """Construct a Resource for interacting with an API.
150
151 Construct a Resource object for interacting with an API. The serviceName and
152 version are the names from the Discovery service.
153
154 Args:
155 serviceName: string, name of the service.
156 version: string, the version of the service.
157 http: httplib2.Http, An instance of httplib2.Http or something that acts
158 like it that HTTP requests will be made through.
159 discoveryServiceUrl: string, a URI Template that points to the location of
160 the discovery service. It should have two parameters {api} and
161 {apiVersion} that when filled in produce an absolute URI to the discovery
162 document for that service.
163 developerKey: string, key obtained from
164 https://code.google.com/apis/console.
165 model: apiclient.Model, converts to and from the wire format.
166 requestBuilder: apiclient.http.HttpRequest, encapsulator for an HTTP
167 request.
168
169 Returns:
170 A Resource object with methods for interacting with the service.
171 """
172 params = {
173 'api': serviceName,
174 'apiVersion': version
175 }
176
177 if http is None:
178 http = httplib2.Http()
179
180 requested_url = uritemplate.expand(discoveryServiceUrl, params)
181
182
183
184
185
186 if 'REMOTE_ADDR' in os.environ:
187 requested_url = _add_query_parameter(requested_url, 'userIp',
188 os.environ['REMOTE_ADDR'])
189 logger.info('URL being requested: %s' % requested_url)
190
191 resp, content = http.request(requested_url)
192
193 if resp.status == 404:
194 raise UnknownApiNameOrVersion("name: %s version: %s" % (serviceName,
195 version))
196 if resp.status >= 400:
197 raise HttpError(resp, content, requested_url)
198
199 try:
200 service = simplejson.loads(content)
201 except ValueError, e:
202 logger.error('Failed to parse as JSON: ' + content)
203 raise InvalidJsonError()
204
205 return build_from_document(content, discoveryServiceUrl, http=http,
206 developerKey=developerKey, model=model, requestBuilder=requestBuilder)
207
208
209 -def build_from_document(
210 service,
211 base,
212 future=None,
213 http=None,
214 developerKey=None,
215 model=None,
216 requestBuilder=HttpRequest):
217 """Create a Resource for interacting with an API.
218
219 Same as `build()`, but constructs the Resource object from a discovery
220 document that is it given, as opposed to retrieving one over HTTP.
221
222 Args:
223 service: string, discovery document.
224 base: string, base URI for all HTTP requests, usually the discovery URI.
225 future: string, discovery document with future capabilities (deprecated).
226 http: httplib2.Http, An instance of httplib2.Http or something that acts
227 like it that HTTP requests will be made through.
228 developerKey: string, Key for controlling API usage, generated
229 from the API Console.
230 model: Model class instance that serializes and de-serializes requests and
231 responses.
232 requestBuilder: Takes an http request and packages it up to be executed.
233
234 Returns:
235 A Resource object with methods for interacting with the service.
236 """
237
238
239 future = {}
240
241 service = simplejson.loads(service)
242 base = urlparse.urljoin(base, service['basePath'])
243 schema = Schemas(service)
244
245 if model is None:
246 features = service.get('features', [])
247 model = JsonModel('dataWrapper' in features)
248 resource = _createResource(http, base, model, requestBuilder, developerKey,
249 service, service, schema)
250
251 return resource
252
253
254 -def _cast(value, schema_type):
255 """Convert value to a string based on JSON Schema type.
256
257 See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on
258 JSON Schema.
259
260 Args:
261 value: any, the value to convert
262 schema_type: string, the type that value should be interpreted as
263
264 Returns:
265 A string representation of 'value' based on the schema_type.
266 """
267 if schema_type == 'string':
268 if type(value) == type('') or type(value) == type(u''):
269 return value
270 else:
271 return str(value)
272 elif schema_type == 'integer':
273 return str(int(value))
274 elif schema_type == 'number':
275 return str(float(value))
276 elif schema_type == 'boolean':
277 return str(bool(value)).lower()
278 else:
279 if type(value) == type('') or type(value) == type(u''):
280 return value
281 else:
282 return str(value)
283
284
285 MULTIPLIERS = {
286 "KB": 2 ** 10,
287 "MB": 2 ** 20,
288 "GB": 2 ** 30,
289 "TB": 2 ** 40,
290 }
291
292
310
311
312 -def _createResource(http, baseUrl, model, requestBuilder,
313 developerKey, resourceDesc, rootDesc, schema):
314 """Build a Resource from the API description.
315
316 Args:
317 http: httplib2.Http, Object to make http requests with.
318 baseUrl: string, base URL for the API. All requests are relative to this
319 URI.
320 model: apiclient.Model, converts to and from the wire format.
321 requestBuilder: class or callable that instantiates an
322 apiclient.HttpRequest object.
323 developerKey: string, key obtained from
324 https://code.google.com/apis/console
325 resourceDesc: object, section of deserialized discovery document that
326 describes a resource. Note that the top level discovery document
327 is considered a resource.
328 rootDesc: object, the entire deserialized discovery document.
329 schema: object, mapping of schema names to schema descriptions.
330
331 Returns:
332 An instance of Resource with all the methods attached for interacting with
333 that resource.
334 """
335
336 class Resource(object):
337 """A class for interacting with a resource."""
338
339 def __init__(self):
340 self._http = http
341 self._baseUrl = baseUrl
342 self._model = model
343 self._developerKey = developerKey
344 self._requestBuilder = requestBuilder
345
346 def createMethod(theclass, methodName, methodDesc, rootDesc):
347 """Creates a method for attaching to a Resource.
348
349 Args:
350 theclass: type, the class to attach methods to.
351 methodName: string, name of the method to use.
352 methodDesc: object, fragment of deserialized discovery document that
353 describes the method.
354 rootDesc: object, the entire deserialized discovery document.
355 """
356 methodName = fix_method_name(methodName)
357 pathUrl = methodDesc['path']
358 httpMethod = methodDesc['httpMethod']
359 methodId = methodDesc['id']
360
361 mediaPathUrl = None
362 accept = []
363 maxSize = 0
364 if 'mediaUpload' in methodDesc:
365 mediaUpload = methodDesc['mediaUpload']
366
367 parsed = list(urlparse.urlparse(baseUrl))
368 basePath = parsed[2]
369 mediaPathUrl = '/upload' + basePath + pathUrl
370 accept = mediaUpload['accept']
371 maxSize = _media_size_to_long(mediaUpload.get('maxSize', ''))
372
373 if 'parameters' not in methodDesc:
374 methodDesc['parameters'] = {}
375
376
377 for name, desc in rootDesc.get('parameters', {}).iteritems():
378 methodDesc['parameters'][name] = desc
379
380
381 for name in STACK_QUERY_PARAMETERS:
382 methodDesc['parameters'][name] = {
383 'type': 'string',
384 'location': 'query'
385 }
386
387 if httpMethod in ['PUT', 'POST', 'PATCH'] and 'request' in methodDesc:
388 methodDesc['parameters']['body'] = {
389 'description': 'The request body.',
390 'type': 'object',
391 'required': True,
392 }
393 if 'request' in methodDesc:
394 methodDesc['parameters']['body'].update(methodDesc['request'])
395 else:
396 methodDesc['parameters']['body']['type'] = 'object'
397 if 'mediaUpload' in methodDesc:
398 methodDesc['parameters']['media_body'] = {
399 'description': 'The filename of the media request body.',
400 'type': 'string',
401 'required': False,
402 }
403 if 'body' in methodDesc['parameters']:
404 methodDesc['parameters']['body']['required'] = False
405
406 argmap = {}
407 required_params = []
408 repeated_params = []
409 pattern_params = {}
410 query_params = []
411 path_params = {}
412 param_type = {}
413 enum_params = {}
414
415
416 if 'parameters' in methodDesc:
417 for arg, desc in methodDesc['parameters'].iteritems():
418 param = key2param(arg)
419 argmap[param] = arg
420
421 if desc.get('pattern', ''):
422 pattern_params[param] = desc['pattern']
423 if desc.get('enum', ''):
424 enum_params[param] = desc['enum']
425 if desc.get('required', False):
426 required_params.append(param)
427 if desc.get('repeated', False):
428 repeated_params.append(param)
429 if desc.get('location') == 'query':
430 query_params.append(param)
431 if desc.get('location') == 'path':
432 path_params[param] = param
433 param_type[param] = desc.get('type', 'string')
434
435 for match in URITEMPLATE.finditer(pathUrl):
436 for namematch in VARNAME.finditer(match.group(0)):
437 name = key2param(namematch.group(0))
438 path_params[name] = name
439 if name in query_params:
440 query_params.remove(name)
441
442 def method(self, **kwargs):
443
444
445 for name in kwargs.iterkeys():
446 if name not in argmap:
447 raise TypeError('Got an unexpected keyword argument "%s"' % name)
448
449
450 keys = kwargs.keys()
451 for name in keys:
452 if kwargs[name] is None:
453 del kwargs[name]
454
455 for name in required_params:
456 if name not in kwargs:
457 raise TypeError('Missing required parameter "%s"' % name)
458
459 for name, regex in pattern_params.iteritems():
460 if name in kwargs:
461 if isinstance(kwargs[name], basestring):
462 pvalues = [kwargs[name]]
463 else:
464 pvalues = kwargs[name]
465 for pvalue in pvalues:
466 if re.match(regex, pvalue) is None:
467 raise TypeError(
468 'Parameter "%s" value "%s" does not match the pattern "%s"' %
469 (name, pvalue, regex))
470
471 for name, enums in enum_params.iteritems():
472 if name in kwargs:
473
474
475
476 if (name in repeated_params and
477 not isinstance(kwargs[name], basestring)):
478 values = kwargs[name]
479 else:
480 values = [kwargs[name]]
481 for value in values:
482 if value not in enums:
483 raise TypeError(
484 'Parameter "%s" value "%s" is not an allowed value in "%s"' %
485 (name, value, str(enums)))
486
487 actual_query_params = {}
488 actual_path_params = {}
489 for key, value in kwargs.iteritems():
490 to_type = param_type.get(key, 'string')
491
492 if key in repeated_params and type(value) == type([]):
493 cast_value = [_cast(x, to_type) for x in value]
494 else:
495 cast_value = _cast(value, to_type)
496 if key in query_params:
497 actual_query_params[argmap[key]] = cast_value
498 if key in path_params:
499 actual_path_params[argmap[key]] = cast_value
500 body_value = kwargs.get('body', None)
501 media_filename = kwargs.get('media_body', None)
502
503 if self._developerKey:
504 actual_query_params['key'] = self._developerKey
505
506 model = self._model
507
508 if methodName.endswith('_media'):
509 model = MediaModel()
510 elif 'response' not in methodDesc:
511 model = RawModel()
512
513 headers = {}
514 headers, params, query, body = model.request(headers,
515 actual_path_params, actual_query_params, body_value)
516
517 expanded_url = uritemplate.expand(pathUrl, params)
518 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
519
520 resumable = None
521 multipart_boundary = ''
522
523 if media_filename:
524
525 if isinstance(media_filename, basestring):
526 (media_mime_type, encoding) = mimetypes.guess_type(media_filename)
527 if media_mime_type is None:
528 raise UnknownFileType(media_filename)
529 if not mimeparse.best_match([media_mime_type], ','.join(accept)):
530 raise UnacceptableMimeTypeError(media_mime_type)
531 media_upload = MediaFileUpload(media_filename, media_mime_type)
532 elif isinstance(media_filename, MediaUpload):
533 media_upload = media_filename
534 else:
535 raise TypeError('media_filename must be str or MediaUpload.')
536
537
538 if maxSize > 0 and media_upload.size() > maxSize:
539 raise MediaUploadSizeError("Media larger than: %s" % maxSize)
540
541
542 expanded_url = uritemplate.expand(mediaPathUrl, params)
543 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
544 if media_upload.resumable():
545 url = _add_query_parameter(url, 'uploadType', 'resumable')
546
547 if media_upload.resumable():
548
549
550 resumable = media_upload
551 else:
552
553 if body is None:
554
555 headers['content-type'] = media_upload.mimetype()
556 body = media_upload.getbytes(0, media_upload.size())
557 url = _add_query_parameter(url, 'uploadType', 'media')
558 else:
559
560 msgRoot = MIMEMultipart('related')
561
562 setattr(msgRoot, '_write_headers', lambda self: None)
563
564
565 msg = MIMENonMultipart(*headers['content-type'].split('/'))
566 msg.set_payload(body)
567 msgRoot.attach(msg)
568
569
570 msg = MIMENonMultipart(*media_upload.mimetype().split('/'))
571 msg['Content-Transfer-Encoding'] = 'binary'
572
573 payload = media_upload.getbytes(0, media_upload.size())
574 msg.set_payload(payload)
575 msgRoot.attach(msg)
576 body = msgRoot.as_string()
577
578 multipart_boundary = msgRoot.get_boundary()
579 headers['content-type'] = ('multipart/related; '
580 'boundary="%s"') % multipart_boundary
581 url = _add_query_parameter(url, 'uploadType', 'multipart')
582
583 logger.info('URL being requested: %s' % url)
584 return self._requestBuilder(self._http,
585 model.response,
586 url,
587 method=httpMethod,
588 body=body,
589 headers=headers,
590 methodId=methodId,
591 resumable=resumable)
592
593 docs = [methodDesc.get('description', DEFAULT_METHOD_DOC), '\n\n']
594 if len(argmap) > 0:
595 docs.append('Args:\n')
596
597
598 skip_parameters = rootDesc.get('parameters', {}).keys()
599 skip_parameters.append(STACK_QUERY_PARAMETERS)
600
601 for arg in argmap.iterkeys():
602 if arg in skip_parameters:
603 continue
604
605 repeated = ''
606 if arg in repeated_params:
607 repeated = ' (repeated)'
608 required = ''
609 if arg in required_params:
610 required = ' (required)'
611 paramdesc = methodDesc['parameters'][argmap[arg]]
612 paramdoc = paramdesc.get('description', 'A parameter')
613 if '$ref' in paramdesc:
614 docs.append(
615 (' %s: object, %s%s%s\n The object takes the'
616 ' form of:\n\n%s\n\n') % (arg, paramdoc, required, repeated,
617 schema.prettyPrintByName(paramdesc['$ref'])))
618 else:
619 paramtype = paramdesc.get('type', 'string')
620 docs.append(' %s: %s, %s%s%s\n' % (arg, paramtype, paramdoc, required,
621 repeated))
622 enum = paramdesc.get('enum', [])
623 enumDesc = paramdesc.get('enumDescriptions', [])
624 if enum and enumDesc:
625 docs.append(' Allowed values\n')
626 for (name, desc) in zip(enum, enumDesc):
627 docs.append(' %s - %s\n' % (name, desc))
628 if 'response' in methodDesc:
629 if methodName.endswith('_media'):
630 docs.append('\nReturns:\n The media object as a string.\n\n ')
631 else:
632 docs.append('\nReturns:\n An object of the form:\n\n ')
633 docs.append(schema.prettyPrintSchema(methodDesc['response']))
634
635 setattr(method, '__doc__', ''.join(docs))
636 setattr(theclass, methodName, method)
637
638 def createNextMethod(theclass, methodName, methodDesc, rootDesc):
639 """Creates any _next methods for attaching to a Resource.
640
641 The _next methods allow for easy iteration through list() responses.
642
643 Args:
644 theclass: type, the class to attach methods to.
645 methodName: string, name of the method to use.
646 methodDesc: object, fragment of deserialized discovery document that
647 describes the method.
648 rootDesc: object, the entire deserialized discovery document.
649 """
650 methodName = fix_method_name(methodName)
651 methodId = methodDesc['id'] + '.next'
652
653 def methodNext(self, previous_request, previous_response):
654 """Retrieves the next page of results.
655
656 Args:
657 previous_request: The request for the previous page.
658 previous_response: The response from the request for the previous page.
659
660 Returns:
661 A request object that you can call 'execute()' on to request the next
662 page. Returns None if there are no more items in the collection.
663 """
664
665
666
667 if 'nextPageToken' not in previous_response:
668 return None
669
670 request = copy.copy(previous_request)
671
672 pageToken = previous_response['nextPageToken']
673 parsed = list(urlparse.urlparse(request.uri))
674 q = parse_qsl(parsed[4])
675
676
677 newq = [(key, value) for (key, value) in q if key != 'pageToken']
678 newq.append(('pageToken', pageToken))
679 parsed[4] = urllib.urlencode(newq)
680 uri = urlparse.urlunparse(parsed)
681
682 request.uri = uri
683
684 logger.info('URL being requested: %s' % uri)
685
686 return request
687
688 setattr(theclass, methodName, methodNext)
689
690
691 if 'methods' in resourceDesc:
692 for methodName, methodDesc in resourceDesc['methods'].iteritems():
693 createMethod(Resource, methodName, methodDesc, rootDesc)
694
695
696 if methodDesc.get('supportsMediaDownload', False):
697 createMethod(Resource, methodName + '_media', methodDesc, rootDesc)
698
699
700 if 'resources' in resourceDesc:
701
702 def createResourceMethod(theclass, methodName, methodDesc, rootDesc):
703 """Create a method on the Resource to access a nested Resource.
704
705 Args:
706 theclass: type, the class to attach methods to.
707 methodName: string, name of the method to use.
708 methodDesc: object, fragment of deserialized discovery document that
709 describes the method.
710 rootDesc: object, the entire deserialized discovery document.
711 """
712 methodName = fix_method_name(methodName)
713
714 def methodResource(self):
715 return _createResource(self._http, self._baseUrl, self._model,
716 self._requestBuilder, self._developerKey,
717 methodDesc, rootDesc, schema)
718
719 setattr(methodResource, '__doc__', 'A collection resource.')
720 setattr(methodResource, '__is_resource__', True)
721 setattr(theclass, methodName, methodResource)
722
723 for methodName, methodDesc in resourceDesc['resources'].iteritems():
724 createResourceMethod(Resource, methodName, methodDesc, rootDesc)
725
726
727
728
729 if 'methods' in resourceDesc:
730 for methodName, methodDesc in resourceDesc['methods'].iteritems():
731 if 'response' in methodDesc:
732 responseSchema = methodDesc['response']
733 if '$ref' in responseSchema:
734 responseSchema = schema.get(responseSchema['$ref'])
735 hasNextPageToken = 'nextPageToken' in responseSchema.get('properties',
736 {})
737 hasPageToken = 'pageToken' in methodDesc.get('parameters', {})
738 if hasNextPageToken and hasPageToken:
739 createNextMethod(Resource, methodName + '_next',
740 resourceDesc['methods'][methodName],
741 methodName)
742
743 return Resource()
744