Various changes to support further high-level automation efforts.
* added a RESTful interface for TKO. right now there's only a single, simple resource for accessing test attributes.
* extended the REST server library in a few ways, most notably to support
* querying on keyvals, with something like ?has_keyval=mykey=myvalue&...
* operators, delimited by a colon, like ?hostname:in=host1,host2,host3
* loading relationships over many items efficiently (see InstanceEntry.prepare_for_full_representation()). this is used to fill in keyvals when requesting a job listing, but it can (and should) be used in other places, such as listing labels for a host collection.
* loading a collection with inlined full representations, by passing full_representations=true
* added various features to the AFE RESTful interface as necessary.
* various fixes to the rest_client library, most notably
* changed HTTP client in rest_client.py to use DI rather than singleton, easing testability. the same should be done for _get_request_headers(), to be honest.
* better support for query params, including accepting a MultiValueDict and supporting URIs that already have query args
* basic support for redirects
* builtin support for requesting a full collection (get_full()), when clients explicitly expect the result not to be paged. i'm still considering alternative approaches to this -- it may make sense to have something like this be the default, and have clients set a default page size limit rather than passing it every time.
* minor change to mock.py to provide better debugging output.
Signed-off-by: Steve Howard <showard@google.com>
git-svn-id: http://test.kernel.org/svn/autotest/trunk@4438 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/frontend/shared/rest_client.py b/frontend/shared/rest_client.py
index b5b83af..9c4f5d5 100644
--- a/frontend/shared/rest_client.py
+++ b/frontend/shared/rest_client.py
@@ -1,11 +1,10 @@
-import logging, pprint, re, urllib, getpass, urlparse
+import copy, getpass, logging, pprint, re, urllib, urlparse
import httplib2
-from django.utils import simplejson
+from django.utils import datastructures, simplejson
from autotest_lib.frontend.afe import rpc_client_lib
from autotest_lib.client.common_lib import utils
-_http = httplib2.Http()
_request_headers = {}
@@ -59,7 +58,8 @@
class Resource(object):
- def __init__(self, representation_dict):
+ def __init__(self, representation_dict, http):
+ self._http = http
assert 'href' in representation_dict
for key, value in representation_dict.iteritems():
setattr(self, str(key), value)
@@ -75,8 +75,10 @@
@classmethod
- def load(cls, uri):
- directory = cls({'href': uri})
+ def load(cls, uri, http=None):
+ if not http:
+ http = httplib2.Http()
+ directory = cls({'href': uri}, http)
return directory.get()
@@ -88,7 +90,7 @@
converted_dict = dict((key, self._read_representation(sub_value))
for key, sub_value in value.iteritems())
if 'href' in converted_dict:
- return type(self)(converted_dict)
+ return type(self)(converted_dict, http=self._http)
return converted_dict
return value
@@ -113,11 +115,14 @@
def _do_request(self, method, uri, query_parameters, encoded_body):
+ uri_parts = [uri]
if query_parameters:
- query_string = '?' + urllib.urlencode(query_parameters)
- else:
- query_string = ''
- full_uri = uri + query_string
+ if '?' in uri:
+ uri_parts += '&'
+ else:
+ uri_parts += '?'
+ uri_parts += urllib.urlencode(query_parameters, doseq=True)
+ full_uri = ''.join(uri_parts)
if encoded_body:
entity_body = simplejson.dumps(encoded_body)
@@ -131,7 +136,7 @@
site_verify = utils.import_site_function(
__file__, 'autotest_lib.frontend.shared.site_rest_client',
'site_verify_response', _site_verify_response_default)
- headers, response_body = _http.request(
+ headers, response_body = self._http.request(
full_uri, method, body=entity_body,
headers=_get_request_headers(uri))
if not site_verify(headers, response_body):
@@ -155,7 +160,8 @@
encoded_body)
if 300 <= response.status < 400: # redirection
- raise NotImplementedError(str(response)) # TODO
+ return self._do_request(method, response.headers['location'],
+ query_parameters, encoded_body)
if 400 <= response.status < 500:
raise ClientError(str(response))
if 500 <= response.status < 600:
@@ -165,19 +171,59 @@
def _stringify_query_parameter(self, value):
if isinstance(value, (list, tuple)):
- return ','.join(value)
+ return ','.join(self._stringify_query_parameter(item)
+ for item in value)
return str(value)
- def get(self, **query_parameters):
- string_parameters = dict((key, self._stringify_query_parameter(value))
- for key, value in query_parameters.iteritems()
- if value is not None)
- response = self._request('GET', query_parameters=string_parameters)
+ def _iterlists(self, mapping):
+ """This effectively lets us treat dicts as MultiValueDicts."""
+ if hasattr(mapping, 'iterlists'): # mapping is already a MultiValueDict
+ return mapping.iterlists()
+ return ((key, (value,)) for key, value in mapping.iteritems())
+
+
+ def get(self, query_parameters=None, **kwarg_query_parameters):
+ """
+ @param query_parameters: a dict or MultiValueDict
+ """
+ query_parameters = copy.copy(query_parameters) # avoid mutating original
+ if query_parameters is None:
+ query_parameters = {}
+ query_parameters.update(kwarg_query_parameters)
+
+ string_parameters = datastructures.MultiValueDict()
+ for key, values in self._iterlists(query_parameters):
+ string_parameters.setlist(
+ key, [self._stringify_query_parameter(value)
+ for value in values])
+
+ response = self._request('GET',
+ query_parameters=string_parameters.lists())
assert response.status == 200
return self._read_representation(response.decoded_body())
+ def get_full(self, results_limit, query_parameters=None,
+ **kwarg_query_parameters):
+ """
+ Like get() for collections, when the full collection is expected.
+
+ @param results_limit: maxmimum number of results to allow
+ @raises ClientError if there are more than results_limit results.
+ """
+ result = self.get(query_parameters=query_parameters,
+ items_per_page=results_limit,
+ **kwarg_query_parameters)
+ if result.total_results > results_limit:
+ raise ClientError(
+ 'Too many results (%s > %s) for request %s (%s %s)'
+ % (result.total_results, results_limit, self.href,
+ query_parameters, kwarg_query_parameters))
+ return result
+
+
+
def put(self):
response = self._request('PUT', encoded_body=self._representation())
assert response.status == 200