1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """Client for discovery based APIs
16
17 A client library for Google's discovery based APIs.
18 """
19
20 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
21 __all__ = [
22 'build',
23 'build_from_document'
24 'fix_method_name',
25 'key2param'
26 ]
27
28 import copy
29 import httplib2
30 import logging
31 import os
32 import random
33 import re
34 import uritemplate
35 import urllib
36 import urlparse
37 import mimeparse
38 import mimetypes
39
40 try:
41 from urlparse import parse_qsl
42 except ImportError:
43 from cgi import parse_qsl
44
45 from apiclient.errors import HttpError
46 from apiclient.errors import InvalidJsonError
47 from apiclient.errors import MediaUploadSizeError
48 from apiclient.errors import UnacceptableMimeTypeError
49 from apiclient.errors import UnknownApiNameOrVersion
50 from apiclient.errors import UnknownLinkType
51 from apiclient.http import HttpRequest
52 from apiclient.http import MediaFileUpload
53 from apiclient.http import MediaUpload
54 from apiclient.model import JsonModel
55 from apiclient.model import RawModel
56 from apiclient.schema import Schemas
57 from email.mime.multipart import MIMEMultipart
58 from email.mime.nonmultipart import MIMENonMultipart
59 from oauth2client.anyjson import simplejson
60
61 logger = logging.getLogger(__name__)
62
63 URITEMPLATE = re.compile('{[^}]*}')
64 VARNAME = re.compile('[a-zA-Z0-9_-]+')
65 DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
66 '{api}/{apiVersion}/rest')
67 DEFAULT_METHOD_DOC = 'A description of how to use this function'
68
69
70 STACK_QUERY_PARAMETERS = ['trace', 'pp', 'userip', 'strict']
71
72
73 RESERVED_WORDS = ['and', 'assert', 'break', 'class', 'continue', 'def', 'del',
74 'elif', 'else', 'except', 'exec', 'finally', 'for', 'from',
75 'global', 'if', 'import', 'in', 'is', 'lambda', 'not', 'or',
76 'pass', 'print', 'raise', 'return', 'try', 'while' ]
77
78
80 """Fix method names to avoid reserved word conflicts.
81
82 Args:
83 name: string, method name.
84
85 Returns:
86 The name with a '_' prefixed if the name is a reserved word.
87 """
88 if name in RESERVED_WORDS:
89 return name + '_'
90 else:
91 return name
92
93
95 """Adds a query parameter to a url.
96
97 Replaces the current value if it already exists in the URL.
98
99 Args:
100 url: string, url to add the query parameter to.
101 name: string, query parameter name.
102 value: string, query parameter value.
103
104 Returns:
105 Updated query parameter. Does not update the url if value is None.
106 """
107 if value is None:
108 return url
109 else:
110 parsed = list(urlparse.urlparse(url))
111 q = dict(parse_qsl(parsed[4]))
112 q[name] = value
113 parsed[4] = urllib.urlencode(q)
114 return urlparse.urlunparse(parsed)
115
116
118 """Converts key names into parameter names.
119
120 For example, converting "max-results" -> "max_results"
121
122 Args:
123 key: string, the method key name.
124
125 Returns:
126 A safe method name based on the key name.
127 """
128 result = []
129 key = list(key)
130 if not key[0].isalpha():
131 result.append('x')
132 for c in key:
133 if c.isalnum():
134 result.append(c)
135 else:
136 result.append('_')
137
138 return ''.join(result)
139
140
141 -def build(serviceName,
142 version,
143 http=None,
144 discoveryServiceUrl=DISCOVERY_URI,
145 developerKey=None,
146 model=None,
147 requestBuilder=HttpRequest):
148 """Construct a Resource for interacting with an API.
149
150 Construct a Resource object for interacting with an API. The serviceName and
151 version are the names from the Discovery service.
152
153 Args:
154 serviceName: string, name of the service.
155 version: string, the version of the service.
156 http: httplib2.Http, An instance of httplib2.Http or something that acts
157 like it that HTTP requests will be made through.
158 discoveryServiceUrl: string, a URI Template that points to the location of
159 the discovery service. It should have two parameters {api} and
160 {apiVersion} that when filled in produce an absolute URI to the discovery
161 document for that service.
162 developerKey: string, key obtained from
163 https://code.google.com/apis/console.
164 model: apiclient.Model, converts to and from the wire format.
165 requestBuilder: apiclient.http.HttpRequest, encapsulator for an HTTP
166 request.
167
168 Returns:
169 A Resource object with methods for interacting with the service.
170 """
171 params = {
172 'api': serviceName,
173 'apiVersion': version
174 }
175
176 if http is None:
177 http = httplib2.Http()
178
179 requested_url = uritemplate.expand(discoveryServiceUrl, params)
180
181
182
183
184
185 if 'REMOTE_ADDR' in os.environ:
186 requested_url = _add_query_parameter(requested_url, 'userIp',
187 os.environ['REMOTE_ADDR'])
188 logger.info('URL being requested: %s' % requested_url)
189
190 resp, content = http.request(requested_url)
191
192 if resp.status == 404:
193 raise UnknownApiNameOrVersion("name: %s version: %s" % (serviceName,
194 version))
195 if resp.status >= 400:
196 raise HttpError(resp, content, requested_url)
197
198 try:
199 service = simplejson.loads(content)
200 except ValueError, e:
201 logger.error('Failed to parse as JSON: ' + content)
202 raise InvalidJsonError()
203
204 return build_from_document(content, discoveryServiceUrl, http=http,
205 developerKey=developerKey, model=model, requestBuilder=requestBuilder)
206
207
208 -def build_from_document(
209 service,
210 base,
211 future=None,
212 http=None,
213 developerKey=None,
214 model=None,
215 requestBuilder=HttpRequest):
216 """Create a Resource for interacting with an API.
217
218 Same as `build()`, but constructs the Resource object from a discovery
219 document that is it given, as opposed to retrieving one over HTTP.
220
221 Args:
222 service: string, discovery document.
223 base: string, base URI for all HTTP requests, usually the discovery URI.
224 future: string, discovery document with future capabilities (deprecated).
225 http: httplib2.Http, An instance of httplib2.Http or something that acts
226 like it that HTTP requests will be made through.
227 developerKey: string, Key for controlling API usage, generated
228 from the API Console.
229 model: Model class instance that serializes and de-serializes requests and
230 responses.
231 requestBuilder: Takes an http request and packages it up to be executed.
232
233 Returns:
234 A Resource object with methods for interacting with the service.
235 """
236
237
238 future = {}
239
240 service = simplejson.loads(service)
241 base = urlparse.urljoin(base, service['basePath'])
242 schema = Schemas(service)
243
244 if model is None:
245 features = service.get('features', [])
246 model = JsonModel('dataWrapper' in features)
247 resource = createResource(http, base, model, requestBuilder, developerKey,
248 service, service, schema)
249
250 return resource
251
252
253 -def _cast(value, schema_type):
254 """Convert value to a string based on JSON Schema type.
255
256 See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on
257 JSON Schema.
258
259 Args:
260 value: any, the value to convert
261 schema_type: string, the type that value should be interpreted as
262
263 Returns:
264 A string representation of 'value' based on the schema_type.
265 """
266 if schema_type == 'string':
267 if type(value) == type('') or type(value) == type(u''):
268 return value
269 else:
270 return str(value)
271 elif schema_type == 'integer':
272 return str(int(value))
273 elif schema_type == 'number':
274 return str(float(value))
275 elif schema_type == 'boolean':
276 return str(bool(value)).lower()
277 else:
278 if type(value) == type('') or type(value) == type(u''):
279 return value
280 else:
281 return str(value)
282
283
284 MULTIPLIERS = {
285 "KB": 2 ** 10,
286 "MB": 2 ** 20,
287 "GB": 2 ** 30,
288 "TB": 2 ** 40,
289 }
290
291
309
310
311 -def createResource(http, baseUrl, model, requestBuilder,
312 developerKey, resourceDesc, rootDesc, schema):
313 """Build a Resource from the API description.
314
315 Args:
316 http: httplib2.Http, Object to make http requests with.
317 baseUrl: string, base URL for the API. All requests are relative to this
318 URI.
319 model: apiclient.Model, converts to and from the wire format.
320 requestBuilder: class or callable that instantiates an
321 apiclient.HttpRequest object.
322 developerKey: string, key obtained from
323 https://code.google.com/apis/console
324 resourceDesc: object, section of deserialized discovery document that
325 describes a resource. Note that the top level discovery document
326 is considered a resource.
327 rootDesc: object, the entire deserialized discovery document.
328 schema: object, mapping of schema names to schema descriptions.
329
330 Returns:
331 An instance of Resource with all the methods attached for interacting with
332 that resource.
333 """
334
335 class Resource(object):
336 """A class for interacting with a resource."""
337
338 def __init__(self):
339 self._http = http
340 self._baseUrl = baseUrl
341 self._model = model
342 self._developerKey = developerKey
343 self._requestBuilder = requestBuilder
344
345 def createMethod(theclass, methodName, methodDesc, rootDesc):
346 """Creates a method for attaching to a Resource.
347
348 Args:
349 theclass: type, the class to attach methods to.
350 methodName: string, name of the method to use.
351 methodDesc: object, fragment of deserialized discovery document that
352 describes the method.
353 rootDesc: object, the entire deserialized discovery document.
354 """
355 methodName = fix_method_name(methodName)
356 pathUrl = methodDesc['path']
357 httpMethod = methodDesc['httpMethod']
358 methodId = methodDesc['id']
359
360 mediaPathUrl = None
361 accept = []
362 maxSize = 0
363 if 'mediaUpload' in methodDesc:
364 mediaUpload = methodDesc['mediaUpload']
365
366 parsed = list(urlparse.urlparse(baseUrl))
367 basePath = parsed[2]
368 mediaPathUrl = '/upload' + basePath + pathUrl
369 accept = mediaUpload['accept']
370 maxSize = _media_size_to_long(mediaUpload.get('maxSize', ''))
371
372 if 'parameters' not in methodDesc:
373 methodDesc['parameters'] = {}
374
375
376 for name, desc in rootDesc.get('parameters', {}).iteritems():
377 methodDesc['parameters'][name] = desc
378
379
380 for name in STACK_QUERY_PARAMETERS:
381 methodDesc['parameters'][name] = {
382 'type': 'string',
383 'location': 'query'
384 }
385
386 if httpMethod in ['PUT', 'POST', 'PATCH'] and 'request' in methodDesc:
387 methodDesc['parameters']['body'] = {
388 'description': 'The request body.',
389 'type': 'object',
390 'required': True,
391 }
392 if 'request' in methodDesc:
393 methodDesc['parameters']['body'].update(methodDesc['request'])
394 else:
395 methodDesc['parameters']['body']['type'] = 'object'
396 if 'mediaUpload' in methodDesc:
397 methodDesc['parameters']['media_body'] = {
398 'description': 'The filename of the media request body.',
399 'type': 'string',
400 'required': False,
401 }
402 if 'body' in methodDesc['parameters']:
403 methodDesc['parameters']['body']['required'] = False
404
405 argmap = {}
406 required_params = []
407 repeated_params = []
408 pattern_params = {}
409 query_params = []
410 path_params = {}
411 param_type = {}
412 enum_params = {}
413
414
415 if 'parameters' in methodDesc:
416 for arg, desc in methodDesc['parameters'].iteritems():
417 param = key2param(arg)
418 argmap[param] = arg
419
420 if desc.get('pattern', ''):
421 pattern_params[param] = desc['pattern']
422 if desc.get('enum', ''):
423 enum_params[param] = desc['enum']
424 if desc.get('required', False):
425 required_params.append(param)
426 if desc.get('repeated', False):
427 repeated_params.append(param)
428 if desc.get('location') == 'query':
429 query_params.append(param)
430 if desc.get('location') == 'path':
431 path_params[param] = param
432 param_type[param] = desc.get('type', 'string')
433
434 for match in URITEMPLATE.finditer(pathUrl):
435 for namematch in VARNAME.finditer(match.group(0)):
436 name = key2param(namematch.group(0))
437 path_params[name] = name
438 if name in query_params:
439 query_params.remove(name)
440
441 def method(self, **kwargs):
442
443 for name in kwargs.iterkeys():
444 if name not in argmap:
445 raise TypeError('Got an unexpected keyword argument "%s"' % name)
446
447 for name in required_params:
448 if name not in kwargs:
449 raise TypeError('Missing required parameter "%s"' % name)
450
451 for name, regex in pattern_params.iteritems():
452 if name in kwargs:
453 if isinstance(kwargs[name], basestring):
454 pvalues = [kwargs[name]]
455 else:
456 pvalues = kwargs[name]
457 for pvalue in pvalues:
458 if re.match(regex, pvalue) is None:
459 raise TypeError(
460 'Parameter "%s" value "%s" does not match the pattern "%s"' %
461 (name, pvalue, regex))
462
463 for name, enums in enum_params.iteritems():
464 if name in kwargs:
465
466
467
468 if (name in repeated_params and
469 not isinstance(kwargs[name], basestring)):
470 values = kwargs[name]
471 else:
472 values = [kwargs[name]]
473 for value in values:
474 if value not in enums:
475 raise TypeError(
476 'Parameter "%s" value "%s" is not an allowed value in "%s"' %
477 (name, value, str(enums)))
478
479 actual_query_params = {}
480 actual_path_params = {}
481 for key, value in kwargs.iteritems():
482 to_type = param_type.get(key, 'string')
483
484 if key in repeated_params and type(value) == type([]):
485 cast_value = [_cast(x, to_type) for x in value]
486 else:
487 cast_value = _cast(value, to_type)
488 if key in query_params:
489 actual_query_params[argmap[key]] = cast_value
490 if key in path_params:
491 actual_path_params[argmap[key]] = cast_value
492 body_value = kwargs.get('body', None)
493 media_filename = kwargs.get('media_body', None)
494
495 if self._developerKey:
496 actual_query_params['key'] = self._developerKey
497
498 model = self._model
499
500 if 'response' not in methodDesc:
501 model = RawModel()
502
503 headers = {}
504 headers, params, query, body = model.request(headers,
505 actual_path_params, actual_query_params, body_value)
506
507 expanded_url = uritemplate.expand(pathUrl, params)
508 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
509
510 resumable = None
511 multipart_boundary = ''
512
513 if media_filename:
514
515 if isinstance(media_filename, basestring):
516 (media_mime_type, encoding) = mimetypes.guess_type(media_filename)
517 if media_mime_type is None:
518 raise UnknownFileType(media_filename)
519 if not mimeparse.best_match([media_mime_type], ','.join(accept)):
520 raise UnacceptableMimeTypeError(media_mime_type)
521 media_upload = MediaFileUpload(media_filename, media_mime_type)
522 elif isinstance(media_filename, MediaUpload):
523 media_upload = media_filename
524 else:
525 raise TypeError('media_filename must be str or MediaUpload.')
526
527
528 if maxSize > 0 and media_upload.size() > maxSize:
529 raise MediaUploadSizeError("Media larger than: %s" % maxSize)
530
531
532 expanded_url = uritemplate.expand(mediaPathUrl, params)
533 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
534 if media_upload.resumable():
535 url = _add_query_parameter(url, 'uploadType', 'resumable')
536
537 if media_upload.resumable():
538
539
540 resumable = media_upload
541 else:
542
543 if body is None:
544
545 headers['content-type'] = media_upload.mimetype()
546 body = media_upload.getbytes(0, media_upload.size())
547 url = _add_query_parameter(url, 'uploadType', 'media')
548 else:
549
550 msgRoot = MIMEMultipart('related')
551
552 setattr(msgRoot, '_write_headers', lambda self: None)
553
554
555 msg = MIMENonMultipart(*headers['content-type'].split('/'))
556 msg.set_payload(body)
557 msgRoot.attach(msg)
558
559
560 msg = MIMENonMultipart(*media_upload.mimetype().split('/'))
561 msg['Content-Transfer-Encoding'] = 'binary'
562
563 payload = media_upload.getbytes(0, media_upload.size())
564 msg.set_payload(payload)
565 msgRoot.attach(msg)
566 body = msgRoot.as_string()
567
568 multipart_boundary = msgRoot.get_boundary()
569 headers['content-type'] = ('multipart/related; '
570 'boundary="%s"') % multipart_boundary
571 url = _add_query_parameter(url, 'uploadType', 'multipart')
572
573 logger.info('URL being requested: %s' % url)
574 return self._requestBuilder(self._http,
575 model.response,
576 url,
577 method=httpMethod,
578 body=body,
579 headers=headers,
580 methodId=methodId,
581 resumable=resumable)
582
583 docs = [methodDesc.get('description', DEFAULT_METHOD_DOC), '\n\n']
584 if len(argmap) > 0:
585 docs.append('Args:\n')
586
587
588 skip_parameters = rootDesc.get('parameters', {}).keys()
589 skip_parameters.append(STACK_QUERY_PARAMETERS)
590
591 for arg in argmap.iterkeys():
592 if arg in skip_parameters:
593 continue
594
595 repeated = ''
596 if arg in repeated_params:
597 repeated = ' (repeated)'
598 required = ''
599 if arg in required_params:
600 required = ' (required)'
601 paramdesc = methodDesc['parameters'][argmap[arg]]
602 paramdoc = paramdesc.get('description', 'A parameter')
603 if '$ref' in paramdesc:
604 docs.append(
605 (' %s: object, %s%s%s\n The object takes the'
606 ' form of:\n\n%s\n\n') % (arg, paramdoc, required, repeated,
607 schema.prettyPrintByName(paramdesc['$ref'])))
608 else:
609 paramtype = paramdesc.get('type', 'string')
610 docs.append(' %s: %s, %s%s%s\n' % (arg, paramtype, paramdoc, required,
611 repeated))
612 enum = paramdesc.get('enum', [])
613 enumDesc = paramdesc.get('enumDescriptions', [])
614 if enum and enumDesc:
615 docs.append(' Allowed values\n')
616 for (name, desc) in zip(enum, enumDesc):
617 docs.append(' %s - %s\n' % (name, desc))
618 if 'response' in methodDesc:
619 docs.append('\nReturns:\n An object of the form\n\n ')
620 docs.append(schema.prettyPrintSchema(methodDesc['response']))
621
622 setattr(method, '__doc__', ''.join(docs))
623 setattr(theclass, methodName, method)
624
625 def createNextMethod(theclass, methodName, methodDesc, rootDesc):
626 """Creates any _next methods for attaching to a Resource.
627
628 The _next methods allow for easy iteration through list() responses.
629
630 Args:
631 theclass: type, the class to attach methods to.
632 methodName: string, name of the method to use.
633 methodDesc: object, fragment of deserialized discovery document that
634 describes the method.
635 rootDesc: object, the entire deserialized discovery document.
636 """
637 methodName = fix_method_name(methodName)
638 methodId = methodDesc['id'] + '.next'
639
640 def methodNext(self, previous_request, previous_response):
641 """Retrieves the next page of results.
642
643 Args:
644 previous_request: The request for the previous page.
645 previous_response: The response from the request for the previous page.
646
647 Returns:
648 A request object that you can call 'execute()' on to request the next
649 page. Returns None if there are no more items in the collection.
650 """
651
652
653
654 if 'nextPageToken' not in previous_response:
655 return None
656
657 request = copy.copy(previous_request)
658
659 pageToken = previous_response['nextPageToken']
660 parsed = list(urlparse.urlparse(request.uri))
661 q = parse_qsl(parsed[4])
662
663
664 newq = [(key, value) for (key, value) in q if key != 'pageToken']
665 newq.append(('pageToken', pageToken))
666 parsed[4] = urllib.urlencode(newq)
667 uri = urlparse.urlunparse(parsed)
668
669 request.uri = uri
670
671 logger.info('URL being requested: %s' % uri)
672
673 return request
674
675 setattr(theclass, methodName, methodNext)
676
677
678 if 'methods' in resourceDesc:
679 for methodName, methodDesc in resourceDesc['methods'].iteritems():
680 createMethod(Resource, methodName, methodDesc, rootDesc)
681
682
683 if 'resources' in resourceDesc:
684
685 def createResourceMethod(theclass, methodName, methodDesc, rootDesc):
686 """Create a method on the Resource to access a nested Resource.
687
688 Args:
689 theclass: type, the class to attach methods to.
690 methodName: string, name of the method to use.
691 methodDesc: object, fragment of deserialized discovery document that
692 describes the method.
693 rootDesc: object, the entire deserialized discovery document.
694 """
695 methodName = fix_method_name(methodName)
696
697 def methodResource(self):
698 return createResource(self._http, self._baseUrl, self._model,
699 self._requestBuilder, self._developerKey,
700 methodDesc, rootDesc, schema)
701
702 setattr(methodResource, '__doc__', 'A collection resource.')
703 setattr(methodResource, '__is_resource__', True)
704 setattr(theclass, methodName, methodResource)
705
706 for methodName, methodDesc in resourceDesc['resources'].iteritems():
707 createResourceMethod(Resource, methodName, methodDesc, rootDesc)
708
709
710
711
712 if 'methods' in resourceDesc:
713 for methodName, methodDesc in resourceDesc['methods'].iteritems():
714 if 'response' in methodDesc:
715 responseSchema = methodDesc['response']
716 if '$ref' in responseSchema:
717 responseSchema = schema.get(responseSchema['$ref'])
718 hasNextPageToken = 'nextPageToken' in responseSchema.get('properties',
719 {})
720 hasPageToken = 'pageToken' in methodDesc.get('parameters', {})
721 if hasNextPageToken and hasPageToken:
722 createNextMethod(Resource, methodName + '_next',
723 resourceDesc['methods'][methodName],
724 methodName)
725
726 return Resource()
727