1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """Client for discovery based APIs
16
17 A client library for Google's discovery based APIs.
18 """
19
20 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
21 __all__ = [
22 'build', 'build_from_document'
23 ]
24
25 import copy
26 import httplib2
27 import logging
28 import os
29 import random
30 import re
31 import uritemplate
32 import urllib
33 import urlparse
34 import mimeparse
35 import mimetypes
36
37 try:
38 from urlparse import parse_qsl
39 except ImportError:
40 from cgi import parse_qsl
41
42 from apiclient.errors import HttpError
43 from apiclient.errors import InvalidJsonError
44 from apiclient.errors import MediaUploadSizeError
45 from apiclient.errors import UnacceptableMimeTypeError
46 from apiclient.errors import UnknownApiNameOrVersion
47 from apiclient.errors import UnknownLinkType
48 from apiclient.http import HttpRequest
49 from apiclient.http import MediaFileUpload
50 from apiclient.http import MediaUpload
51 from apiclient.model import JsonModel
52 from apiclient.model import RawModel
53 from apiclient.schema import Schemas
54 from email.mime.multipart import MIMEMultipart
55 from email.mime.nonmultipart import MIMENonMultipart
56 from oauth2client.anyjson import simplejson
57
58 logger = logging.getLogger(__name__)
59
60 URITEMPLATE = re.compile('{[^}]*}')
61 VARNAME = re.compile('[a-zA-Z0-9_-]+')
62 DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
63 '{api}/{apiVersion}/rest')
64 DEFAULT_METHOD_DOC = 'A description of how to use this function'
65
66
67 STACK_QUERY_PARAMETERS = ['trace', 'fields', 'pp', 'prettyPrint', 'userIp',
68 'userip', 'strict']
69
70 RESERVED_WORDS = ['and', 'assert', 'break', 'class', 'continue', 'def', 'del',
71 'elif', 'else', 'except', 'exec', 'finally', 'for', 'from',
72 'global', 'if', 'import', 'in', 'is', 'lambda', 'not', 'or',
73 'pass', 'print', 'raise', 'return', 'try', 'while' ]
74
75
77 if name in RESERVED_WORDS:
78 return name + '_'
79 else:
80 return name
81
82
86
87
89 """Adds a query parameter to a url.
90
91 Replaces the current value if it already exists in the URL.
92
93 Args:
94 url: string, url to add the query parameter to.
95 name: string, query parameter name.
96 value: string, query parameter value.
97
98 Returns:
99 Updated query parameter. Does not update the url if value is None.
100 """
101 if value is None:
102 return url
103 else:
104 parsed = list(urlparse.urlparse(url))
105 q = dict(parse_qsl(parsed[4]))
106 q[name] = value
107 parsed[4] = urllib.urlencode(q)
108 return urlparse.urlunparse(parsed)
109
110
112 """Converts key names into parameter names.
113
114 For example, converting "max-results" -> "max_results"
115 """
116 result = []
117 key = list(key)
118 if not key[0].isalpha():
119 result.append('x')
120 for c in key:
121 if c.isalnum():
122 result.append(c)
123 else:
124 result.append('_')
125
126 return ''.join(result)
127
128
129 -def build(serviceName,
130 version,
131 http=None,
132 discoveryServiceUrl=DISCOVERY_URI,
133 developerKey=None,
134 model=None,
135 requestBuilder=HttpRequest):
136 """Construct a Resource for interacting with an API.
137
138 Construct a Resource object for interacting with
139 an API. The serviceName and version are the
140 names from the Discovery service.
141
142 Args:
143 serviceName: string, name of the service
144 version: string, the version of the service
145 http: httplib2.Http, An instance of httplib2.Http or something that acts
146 like it that HTTP requests will be made through.
147 discoveryServiceUrl: string, a URI Template that points to
148 the location of the discovery service. It should have two
149 parameters {api} and {apiVersion} that when filled in
150 produce an absolute URI to the discovery document for
151 that service.
152 developerKey: string, key obtained
153 from https://code.google.com/apis/console
154 model: apiclient.Model, converts to and from the wire format
155 requestBuilder: apiclient.http.HttpRequest, encapsulator for
156 an HTTP request
157
158 Returns:
159 A Resource object with methods for interacting with
160 the service.
161 """
162 params = {
163 'api': serviceName,
164 'apiVersion': version
165 }
166
167 if http is None:
168 http = httplib2.Http()
169
170 requested_url = uritemplate.expand(discoveryServiceUrl, params)
171
172
173
174
175
176 if 'REMOTE_ADDR' in os.environ:
177 requested_url = _add_query_parameter(requested_url, 'userIp',
178 os.environ['REMOTE_ADDR'])
179 logger.info('URL being requested: %s' % requested_url)
180
181 resp, content = http.request(requested_url)
182
183 if resp.status == 404:
184 raise UnknownApiNameOrVersion("name: %s version: %s" % (serviceName,
185 version))
186 if resp.status >= 400:
187 raise HttpError(resp, content, requested_url)
188
189 try:
190 service = simplejson.loads(content)
191 except ValueError, e:
192 logger.error('Failed to parse as JSON: ' + content)
193 raise InvalidJsonError()
194
195 filename = os.path.join(os.path.dirname(__file__), 'contrib',
196 serviceName, 'future.json')
197 try:
198 f = file(filename, 'r')
199 future = f.read()
200 f.close()
201 except IOError:
202 future = None
203
204 return build_from_document(content, discoveryServiceUrl, future,
205 http, developerKey, model, requestBuilder)
206
207
208 -def build_from_document(
209 service,
210 base,
211 future=None,
212 http=None,
213 developerKey=None,
214 model=None,
215 requestBuilder=HttpRequest):
216 """Create a Resource for interacting with an API.
217
218 Same as `build()`, but constructs the Resource object
219 from a discovery document that is it given, as opposed to
220 retrieving one over HTTP.
221
222 Args:
223 service: string, discovery document
224 base: string, base URI for all HTTP requests, usually the discovery URI
225 future: string, discovery document with future capabilities
226 auth_discovery: dict, information about the authentication the API supports
227 http: httplib2.Http, An instance of httplib2.Http or something that acts
228 like it that HTTP requests will be made through.
229 developerKey: string, Key for controlling API usage, generated
230 from the API Console.
231 model: Model class instance that serializes and
232 de-serializes requests and responses.
233 requestBuilder: Takes an http request and packages it up to be executed.
234
235 Returns:
236 A Resource object with methods for interacting with
237 the service.
238 """
239
240 service = simplejson.loads(service)
241 base = urlparse.urljoin(base, service['basePath'])
242 if future:
243 future = simplejson.loads(future)
244 auth_discovery = future.get('auth', {})
245 else:
246 future = {}
247 auth_discovery = {}
248 schema = Schemas(service)
249
250 if model is None:
251 features = service.get('features', [])
252 model = JsonModel('dataWrapper' in features)
253 resource = createResource(http, base, model, requestBuilder, developerKey,
254 service, future, schema)
255
256 def auth_method():
257 """Discovery information about the authentication the API uses."""
258 return auth_discovery
259
260 setattr(resource, 'auth_discovery', auth_method)
261
262 return resource
263
264
265 -def _cast(value, schema_type):
266 """Convert value to a string based on JSON Schema type.
267
268 See http://tools.ietf.org/html/draft-zyp-json-schema-03 for more details on
269 JSON Schema.
270
271 Args:
272 value: any, the value to convert
273 schema_type: string, the type that value should be interpreted as
274
275 Returns:
276 A string representation of 'value' based on the schema_type.
277 """
278 if schema_type == 'string':
279 if type(value) == type('') or type(value) == type(u''):
280 return value
281 else:
282 return str(value)
283 elif schema_type == 'integer':
284 return str(int(value))
285 elif schema_type == 'number':
286 return str(float(value))
287 elif schema_type == 'boolean':
288 return str(bool(value)).lower()
289 else:
290 if type(value) == type('') or type(value) == type(u''):
291 return value
292 else:
293 return str(value)
294
295 MULTIPLIERS = {
296 "KB": 2 ** 10,
297 "MB": 2 ** 20,
298 "GB": 2 ** 30,
299 "TB": 2 ** 40,
300 }
301
302
313
314
315 -def createResource(http, baseUrl, model, requestBuilder,
316 developerKey, resourceDesc, futureDesc, schema):
317
318 class Resource(object):
319 """A class for interacting with a resource."""
320
321 def __init__(self):
322 self._http = http
323 self._baseUrl = baseUrl
324 self._model = model
325 self._developerKey = developerKey
326 self._requestBuilder = requestBuilder
327
328 def createMethod(theclass, methodName, methodDesc, futureDesc):
329 methodName = _fix_method_name(methodName)
330 pathUrl = methodDesc['path']
331 httpMethod = methodDesc['httpMethod']
332 methodId = methodDesc['id']
333
334 mediaPathUrl = None
335 accept = []
336 maxSize = 0
337 if 'mediaUpload' in methodDesc:
338 mediaUpload = methodDesc['mediaUpload']
339
340 parsed = list(urlparse.urlparse(baseUrl))
341 basePath = parsed[2]
342 mediaPathUrl = '/upload' + basePath + pathUrl
343 accept = mediaUpload['accept']
344 maxSize = _media_size_to_long(mediaUpload.get('maxSize', ''))
345
346 if 'parameters' not in methodDesc:
347 methodDesc['parameters'] = {}
348 for name in STACK_QUERY_PARAMETERS:
349 methodDesc['parameters'][name] = {
350 'type': 'string',
351 'location': 'query'
352 }
353
354 if httpMethod in ['PUT', 'POST', 'PATCH'] and 'request' in methodDesc:
355 methodDesc['parameters']['body'] = {
356 'description': 'The request body.',
357 'type': 'object',
358 'required': True,
359 }
360 if 'request' in methodDesc:
361 methodDesc['parameters']['body'].update(methodDesc['request'])
362 else:
363 methodDesc['parameters']['body']['type'] = 'object'
364 if 'mediaUpload' in methodDesc:
365 methodDesc['parameters']['media_body'] = {
366 'description': 'The filename of the media request body.',
367 'type': 'string',
368 'required': False,
369 }
370 if 'body' in methodDesc['parameters']:
371 methodDesc['parameters']['body']['required'] = False
372
373 argmap = {}
374 required_params = []
375 repeated_params = []
376 pattern_params = {}
377 query_params = []
378 path_params = {}
379 param_type = {}
380 enum_params = {}
381
382
383 if 'parameters' in methodDesc:
384 for arg, desc in methodDesc['parameters'].iteritems():
385 param = key2param(arg)
386 argmap[param] = arg
387
388 if desc.get('pattern', ''):
389 pattern_params[param] = desc['pattern']
390 if desc.get('enum', ''):
391 enum_params[param] = desc['enum']
392 if desc.get('required', False):
393 required_params.append(param)
394 if desc.get('repeated', False):
395 repeated_params.append(param)
396 if desc.get('location') == 'query':
397 query_params.append(param)
398 if desc.get('location') == 'path':
399 path_params[param] = param
400 param_type[param] = desc.get('type', 'string')
401
402 for match in URITEMPLATE.finditer(pathUrl):
403 for namematch in VARNAME.finditer(match.group(0)):
404 name = key2param(namematch.group(0))
405 path_params[name] = name
406 if name in query_params:
407 query_params.remove(name)
408
409 def method(self, **kwargs):
410 for name in kwargs.iterkeys():
411 if name not in argmap:
412 raise TypeError('Got an unexpected keyword argument "%s"' % name)
413
414 for name in required_params:
415 if name not in kwargs:
416 raise TypeError('Missing required parameter "%s"' % name)
417
418 for name, regex in pattern_params.iteritems():
419 if name in kwargs:
420 if isinstance(kwargs[name], basestring):
421 pvalues = [kwargs[name]]
422 else:
423 pvalues = kwargs[name]
424 for pvalue in pvalues:
425 if re.match(regex, pvalue) is None:
426 raise TypeError(
427 'Parameter "%s" value "%s" does not match the pattern "%s"' %
428 (name, pvalue, regex))
429
430 for name, enums in enum_params.iteritems():
431 if name in kwargs:
432
433
434
435 if (name in repeated_params and
436 not isinstance(kwargs[name], basestring)):
437 values = kwargs[name]
438 else:
439 values = [kwargs[name]]
440 for value in values:
441 if value not in enums:
442 raise TypeError(
443 'Parameter "%s" value "%s" is not an allowed value in "%s"' %
444 (name, value, str(enums)))
445
446 actual_query_params = {}
447 actual_path_params = {}
448 for key, value in kwargs.iteritems():
449 to_type = param_type.get(key, 'string')
450
451 if key in repeated_params and type(value) == type([]):
452 cast_value = [_cast(x, to_type) for x in value]
453 else:
454 cast_value = _cast(value, to_type)
455 if key in query_params:
456 actual_query_params[argmap[key]] = cast_value
457 if key in path_params:
458 actual_path_params[argmap[key]] = cast_value
459 body_value = kwargs.get('body', None)
460 media_filename = kwargs.get('media_body', None)
461
462 if self._developerKey:
463 actual_query_params['key'] = self._developerKey
464
465 model = self._model
466
467 if 'response' not in methodDesc:
468 model = RawModel()
469
470 headers = {}
471 headers, params, query, body = model.request(headers,
472 actual_path_params, actual_query_params, body_value)
473
474 expanded_url = uritemplate.expand(pathUrl, params)
475 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
476
477 resumable = None
478 multipart_boundary = ''
479
480 if media_filename:
481
482 if isinstance(media_filename, basestring):
483 (media_mime_type, encoding) = mimetypes.guess_type(media_filename)
484 if media_mime_type is None:
485 raise UnknownFileType(media_filename)
486 if not mimeparse.best_match([media_mime_type], ','.join(accept)):
487 raise UnacceptableMimeTypeError(media_mime_type)
488 media_upload = MediaFileUpload(media_filename, media_mime_type)
489 elif isinstance(media_filename, MediaUpload):
490 media_upload = media_filename
491 else:
492 raise TypeError('media_filename must be str or MediaUpload.')
493
494
495 if maxSize > 0 and media_upload.size() > maxSize:
496 raise MediaUploadSizeError("Media larger than: %s" % maxSize)
497
498
499 expanded_url = uritemplate.expand(mediaPathUrl, params)
500 url = urlparse.urljoin(self._baseUrl, expanded_url + query)
501 if media_upload.resumable():
502 url = _add_query_parameter(url, 'uploadType', 'resumable')
503
504 if media_upload.resumable():
505
506
507 resumable = media_upload
508 else:
509
510 if body is None:
511
512 headers['content-type'] = media_upload.mimetype()
513 body = media_upload.getbytes(0, media_upload.size())
514 url = _add_query_parameter(url, 'uploadType', 'media')
515 else:
516
517 msgRoot = MIMEMultipart('related')
518
519 setattr(msgRoot, '_write_headers', lambda self: None)
520
521
522 msg = MIMENonMultipart(*headers['content-type'].split('/'))
523 msg.set_payload(body)
524 msgRoot.attach(msg)
525
526
527 msg = MIMENonMultipart(*media_upload.mimetype().split('/'))
528 msg['Content-Transfer-Encoding'] = 'binary'
529
530 payload = media_upload.getbytes(0, media_upload.size())
531 msg.set_payload(payload)
532 msgRoot.attach(msg)
533 body = msgRoot.as_string()
534
535 multipart_boundary = msgRoot.get_boundary()
536 headers['content-type'] = ('multipart/related; '
537 'boundary="%s"') % multipart_boundary
538 url = _add_query_parameter(url, 'uploadType', 'multipart')
539
540 logger.info('URL being requested: %s' % url)
541 return self._requestBuilder(self._http,
542 model.response,
543 url,
544 method=httpMethod,
545 body=body,
546 headers=headers,
547 methodId=methodId,
548 resumable=resumable)
549
550 docs = [methodDesc.get('description', DEFAULT_METHOD_DOC), '\n\n']
551 if len(argmap) > 0:
552 docs.append('Args:\n')
553 for arg in argmap.iterkeys():
554 if arg in STACK_QUERY_PARAMETERS:
555 continue
556 repeated = ''
557 if arg in repeated_params:
558 repeated = ' (repeated)'
559 required = ''
560 if arg in required_params:
561 required = ' (required)'
562 paramdesc = methodDesc['parameters'][argmap[arg]]
563 paramdoc = paramdesc.get('description', 'A parameter')
564 if '$ref' in paramdesc:
565 docs.append(
566 (' %s: object, %s%s%s\n The object takes the'
567 ' form of:\n\n%s\n\n') % (arg, paramdoc, required, repeated,
568 schema.prettyPrintByName(paramdesc['$ref'])))
569 else:
570 paramtype = paramdesc.get('type', 'string')
571 docs.append(' %s: %s, %s%s%s\n' % (arg, paramtype, paramdoc, required,
572 repeated))
573 enum = paramdesc.get('enum', [])
574 enumDesc = paramdesc.get('enumDescriptions', [])
575 if enum and enumDesc:
576 docs.append(' Allowed values\n')
577 for (name, desc) in zip(enum, enumDesc):
578 docs.append(' %s - %s\n' % (name, desc))
579 if 'response' in methodDesc:
580 docs.append('\nReturns:\n An object of the form\n\n ')
581 docs.append(schema.prettyPrintSchema(methodDesc['response']))
582
583 setattr(method, '__doc__', ''.join(docs))
584 setattr(theclass, methodName, method)
585
586 def createNextMethodFromFuture(theclass, methodName, methodDesc, futureDesc):
587 """ This is a legacy method, as only Buzz and Moderator use the future.json
588 functionality for generating _next methods. It will be kept around as long
589 as those API versions are around, but no new APIs should depend upon it.
590 """
591 methodName = _fix_method_name(methodName)
592 methodId = methodDesc['id'] + '.next'
593
594 def methodNext(self, previous):
595 """Retrieve the next page of results.
596
597 Takes a single argument, 'body', which is the results
598 from the last call, and returns the next set of items
599 in the collection.
600
601 Returns:
602 None if there are no more items in the collection.
603 """
604 if futureDesc['type'] != 'uri':
605 raise UnknownLinkType(futureDesc['type'])
606
607 try:
608 p = previous
609 for key in futureDesc['location']:
610 p = p[key]
611 url = p
612 except (KeyError, TypeError):
613 return None
614
615 url = _add_query_parameter(url, 'key', self._developerKey)
616
617 headers = {}
618 headers, params, query, body = self._model.request(headers, {}, {}, None)
619
620 logger.info('URL being requested: %s' % url)
621 resp, content = self._http.request(url, method='GET', headers=headers)
622
623 return self._requestBuilder(self._http,
624 self._model.response,
625 url,
626 method='GET',
627 headers=headers,
628 methodId=methodId)
629
630 setattr(theclass, methodName, methodNext)
631
632 def createNextMethod(theclass, methodName, methodDesc, futureDesc):
633 methodName = _fix_method_name(methodName)
634 methodId = methodDesc['id'] + '.next'
635
636 def methodNext(self, previous_request, previous_response):
637 """Retrieves the next page of results.
638
639 Args:
640 previous_request: The request for the previous page.
641 previous_response: The response from the request for the previous page.
642
643 Returns:
644 A request object that you can call 'execute()' on to request the next
645 page. Returns None if there are no more items in the collection.
646 """
647
648
649
650 if 'nextPageToken' not in previous_response:
651 return None
652
653 request = copy.copy(previous_request)
654
655 pageToken = previous_response['nextPageToken']
656 parsed = list(urlparse.urlparse(request.uri))
657 q = parse_qsl(parsed[4])
658
659
660 newq = [(key, value) for (key, value) in q if key != 'pageToken']
661 newq.append(('pageToken', pageToken))
662 parsed[4] = urllib.urlencode(newq)
663 uri = urlparse.urlunparse(parsed)
664
665 request.uri = uri
666
667 logger.info('URL being requested: %s' % uri)
668
669 return request
670
671 setattr(theclass, methodName, methodNext)
672
673
674 if 'methods' in resourceDesc:
675 for methodName, methodDesc in resourceDesc['methods'].iteritems():
676 if futureDesc:
677 future = futureDesc['methods'].get(methodName, {})
678 else:
679 future = None
680 createMethod(Resource, methodName, methodDesc, future)
681
682
683 if 'resources' in resourceDesc:
684
685 def createResourceMethod(theclass, methodName, methodDesc, futureDesc):
686 methodName = _fix_method_name(methodName)
687
688 def methodResource(self):
689 return createResource(self._http, self._baseUrl, self._model,
690 self._requestBuilder, self._developerKey,
691 methodDesc, futureDesc, schema)
692
693 setattr(methodResource, '__doc__', 'A collection resource.')
694 setattr(methodResource, '__is_resource__', True)
695 setattr(theclass, methodName, methodResource)
696
697 for methodName, methodDesc in resourceDesc['resources'].iteritems():
698 if futureDesc and 'resources' in futureDesc:
699 future = futureDesc['resources'].get(methodName, {})
700 else:
701 future = {}
702 createResourceMethod(Resource, methodName, methodDesc, future)
703
704
705 if futureDesc and 'methods' in futureDesc:
706 for methodName, methodDesc in futureDesc['methods'].iteritems():
707 if 'next' in methodDesc and methodName in resourceDesc['methods']:
708 createNextMethodFromFuture(Resource, methodName + '_next',
709 resourceDesc['methods'][methodName],
710 methodDesc['next'])
711
712
713
714 if 'methods' in resourceDesc:
715 for methodName, methodDesc in resourceDesc['methods'].iteritems():
716 if 'response' in methodDesc:
717 responseSchema = methodDesc['response']
718 if '$ref' in responseSchema:
719 responseSchema = schema.get(responseSchema['$ref'])
720 hasNextPageToken = 'nextPageToken' in responseSchema.get('properties',
721 {})
722 hasPageToken = 'pageToken' in methodDesc.get('parameters', {})
723 if hasNextPageToken and hasPageToken:
724 createNextMethod(Resource, methodName + '_next',
725 resourceDesc['methods'][methodName],
726 methodName)
727
728 return Resource()
729