Add new files logic to devserver and clean up AU E2E test to use it.
There's a couple nested things here:
1) Pipe files through the autotest-side of the devserver logic. I've gone ahead
and added 5 unittests to test this new workflow with various combos of
artifacts. The main thing here is the optional pass of archive_url + files to
the stage_artifacts methods. I've also added a convenience function to return
the url to any staged file on the devserver if you know the filename (which you
would if you are using files= API).
2) Use the new logic in AU E2E API and figure out the right URL's to pass to
stage artifacts from. This was a bit of refactoring to get right.
In order to allow me to serve any artifact I had to change a bit of of the
source version installation logic to allow me to update from any static url
(rather than one that must end in update.gz). This actually makes the
OmahaDevserver much more flexible and more Omaha-y.
BUG=chromium:267896
TEST=Unittests + tested with both an nmo + npo full run.
Change-Id: Ic490c7c0e7d76eb8c0416accefbe113b9c0bba32
Reviewed-on: https://gerrit.chromium.org/gerrit/65732
Reviewed-by: Chris Sosa <sosa@chromium.org>
Tested-by: Chris Sosa <sosa@chromium.org>
Commit-Queue: Chris Sosa <sosa@chromium.org>
diff --git a/client/common_lib/cros/autoupdater.py b/client/common_lib/cros/autoupdater.py
index 78eaaa1..7109b34 100644
--- a/client/common_lib/cros/autoupdater.py
+++ b/client/common_lib/cros/autoupdater.py
@@ -224,7 +224,8 @@
(self.host.hostname, str(e)))
- def _update_root(self):
+ def update_rootfs(self):
+ """Updates the rootfs partition only."""
logging.info('Updating root partition...')
# Run update_engine using the specified URL.
@@ -309,7 +310,7 @@
try:
updaters = [
- multiprocessing.process.Process(target=self._update_root),
+ multiprocessing.process.Process(target=self.update_rootfs),
multiprocessing.process.Process(target=self.update_stateful)
]
if not update_root:
diff --git a/client/common_lib/cros/dev_server.py b/client/common_lib/cros/dev_server.py
index 99d3cc7..0500410 100644
--- a/client/common_lib/cros/dev_server.py
+++ b/client/common_lib/cros/dev_server.py
@@ -6,7 +6,6 @@
import httplib
import json
import logging
-import os
import urllib2
import HTMLParser
import cStringIO
@@ -138,16 +137,16 @@
@param timeout_min: How long to wait in minutes before deciding the
the devserver is not up (float).
"""
- server_name = re.sub(r':\d+$', '', devserver.lstrip('http://'))
+ server_name = re.sub(r':\d+$', '', devserver.lstrip('http://'))
# statsd treats |.| as path separator.
- server_name = server_name.replace('.', '_')
+ server_name = server_name.replace('.', '_')
call = DevServer._build_call(devserver, 'check_health')
@remote_devserver_call(timeout_min=timeout_min)
def make_call():
"""Inner method that makes the call."""
return utils.urlopen_socket_timeout(call,
- timeout=timeout_min*60).read()
+ timeout=timeout_min * 60).read()
try:
result_dict = json.load(cStringIO.StringIO(make_call()))
@@ -162,7 +161,8 @@
elif (free_disk < DevServer._MIN_FREE_DISK_SPACE_GB):
logging.error('Devserver check_health failed. Free disk space '
'is low. Only %dGB is available.', free_disk)
- stats.Counter(server_name +'.devserver_not_healthy').increment()
+ stats.Counter(server_name +
+ '.devserver_not_healthy').increment()
return False
# This counter indicates the load of a devserver. By comparing the
@@ -172,7 +172,7 @@
return True
except Exception as e:
logging.error('Devserver call failed: "%s", timeout: %s seconds,'
- ' Error: %s', call, timeout_min*60, str(e))
+ ' Error: %s', call, timeout_min * 60, e)
stats.Counter(server_name + '.devserver_not_healthy').increment()
return False
@@ -342,16 +342,18 @@
self.nton_payload = nton_payload
- def wait_for_artifacts_staged(self, archive_url, artifacts=''):
+ def wait_for_artifacts_staged(self, archive_url, artifacts='', files=''):
"""Polling devserver.is_staged until all artifacts are staged.
@param archive_url: Google Storage URL for the build.
@param artifacts: Comma separated list of artifacts to download.
+ @param files: Comma separated list of files to download.
@return: True if all artifacts are staged in devserver.
"""
call = self.build_call('is_staged',
archive_url=archive_url,
- artifacts=artifacts)
+ artifacts=artifacts,
+ files=files)
def all_staged():
"""Call devserver.is_staged rpc to check if all files are staged.
@@ -365,20 +367,22 @@
except IOError:
return False
- site_utils.poll_for_condition(all_staged,
- exception=site_utils.TimeoutError(),
- timeout=sys.maxint,
- sleep_interval=_ARTIFACT_STAGE_POLLING_INTERVAL)
+ site_utils.poll_for_condition(
+ all_staged,
+ exception=site_utils.TimeoutError(),
+ timeout=sys.maxint,
+ sleep_interval=_ARTIFACT_STAGE_POLLING_INTERVAL)
return True
- def call_and_wait(self, call_name, archive_url, artifacts,
+ def call_and_wait(self, call_name, archive_url, artifacts, files,
error_message, expected_response='Success'):
"""Helper method to make a urlopen call, and wait for artifacts staged.
@param call_name: name of devserver rpc call.
@param archive_url: Google Storage URL for the build..
@param artifacts: Comma separated list of artifacts to download.
+ @param files: Comma separated list of files to download.
@param expected_response: Expected response from rpc, default to
|Success|. If it's set to None, do not compare
the actual response. Any response is consider
@@ -393,6 +397,7 @@
call = self.build_call(call_name,
archive_url=archive_url,
artifacts=artifacts,
+ files=files,
async=True)
try:
response = urllib2.urlopen(call).read()
@@ -405,12 +410,13 @@
if expected_response and not response == expected_response:
raise DevServerException(error_message)
- self.wait_for_artifacts_staged(archive_url, artifacts)
+ self.wait_for_artifacts_staged(archive_url, artifacts, files)
return response
@remote_devserver_call()
- def stage_artifacts(self, image, artifacts):
+ def stage_artifacts(self, image, artifacts=None, files=None,
+ archive_url=None):
"""Tell the devserver to download and stage |artifacts| from |image|.
This is the main call point for staging any specific artifacts for a
@@ -422,17 +428,28 @@
@param image: the image to fetch and stage.
@param artifacts: A list of artifacts.
+ @param files: A list of files to stage.
+ @param archive_url: Optional parameter that has the archive_url to stage
+ this artifact from. Default is specified in autotest config +
+ image.
@raise DevServerException upon any return code that's not HTTP OK.
+ @raise DevServerException upon any return code that's not HTTP OK.
"""
- archive_url = _get_image_storage_server() + image
- artifacts = ','.join(artifacts)
- error_message = ("staging artifacts %s for %s failed;"
+ assert artifacts or files, 'Must specify something to stage.'
+ if not archive_url:
+ archive_url = _get_image_storage_server() + image
+
+ artifacts_arg = ','.join(artifacts) if artifacts else ''
+ files_arg = ','.join(files) if files else ''
+ error_message = ("staging %s for %s failed;"
"HTTP OK not accompanied by 'Success'." %
- (' '.join(artifacts), image))
+ ('artifacts=%s files=%s ' % (artifacts_arg, files_arg),
+ image))
self.call_and_wait(call_name='stage',
archive_url=archive_url,
- artifacts=artifacts,
+ artifacts=artifacts_arg,
+ files=files_arg,
error_message=error_message)
@@ -456,13 +473,14 @@
@raise DevServerException upon any return code that's not HTTP OK.
"""
- archive_url=_get_image_storage_server() + image
+ archive_url = _get_image_storage_server() + image
artifacts = _ARTIFACTS_TO_BE_STAGED_FOR_IMAGE
error_message = ("trigger_download for %s failed;"
"HTTP OK not accompanied by 'Success'." % image)
response = self.call_and_wait(call_name='stage',
archive_url=archive_url,
artifacts=artifacts,
+ files='',
error_message=error_message)
was_successful = response == 'Success'
if was_successful and synchronous:
@@ -480,11 +498,12 @@
@returns path on the devserver that telemetry is installed to.
"""
- archive_url=_get_image_storage_server() + build
+ archive_url = _get_image_storage_server() + build
artifacts = _ARTIFACTS_TO_BE_STAGED_FOR_IMAGE_WITH_AUTOTEST
response = self.call_and_wait(call_name='setup_telemetry',
archive_url=archive_url,
artifacts=artifacts,
+ files='',
error_message=None,
expected_response=None)
return response
@@ -502,13 +521,14 @@
@param image: the image to fetch and stage.
@raise DevServerException upon any return code that's not HTTP OK.
"""
- archive_url=_get_image_storage_server() + image
+ archive_url = _get_image_storage_server() + image
artifacts = _ARTIFACTS_TO_BE_STAGED_FOR_IMAGE_WITH_AUTOTEST
error_message = ("finish_download for %s failed;"
"HTTP OK not accompanied by 'Success'." % image)
self.call_and_wait(call_name='stage',
archive_url=archive_url,
artifacts=artifacts,
+ files='',
error_message=error_message)
@@ -530,27 +550,12 @@
url_pattern = CONFIG.get_config_value('CROS', 'image_url_pattern',
type=str)
return (url_pattern % (self.url(), image)).replace(
- 'update', 'static')
+ 'update', 'static')
- def get_delta_payload_url(self, payload_type, image):
- """Returns a URL to a staged delta payload.
-
- @param payload_type: either 'mton' or 'nton'
- @param image: the image that was fetched.
-
- @return A fully qualified URL that can be used for downloading the
- payload.
-
- @raise DevServerException if payload type argument is invalid.
-
- """
- if payload_type not in ('mton', 'nton'):
- raise DevServerException('invalid delta payload type: %s' %
- payload_type)
- version = os.path.basename(image)
- base_url = self._get_image_url(image)
- return base_url + '/au/%s_%s/update.gz' % (version, payload_type)
+ def get_staged_file_url(self, filename, image):
+ """Returns the url of a staged file for this image on the devserver."""
+ return '/'.join([self._get_image_url(image), filename])
def get_full_payload_url(self, image):
diff --git a/client/common_lib/cros/dev_server_unittest.py b/client/common_lib/cros/dev_server_unittest.py
index 2673c7c..c32a463 100644
--- a/client/common_lib/cros/dev_server_unittest.py
+++ b/client/common_lib/cros/dev_server_unittest.py
@@ -365,3 +365,78 @@
call = self.crash_server.build_call('symbolicate_dump')
self.assertTrue(call.startswith(self._CRASH_HOST))
+
+ def _stageTestHelper(self, artifacts=[], files=[], archive_url=None):
+ """Helper to test combos of files/artifacts/urls with stage call."""
+ expected_archive_url = archive_url
+ if not archive_url:
+ expected_archive_url = 'gs://my_default_url'
+ self.mox.StubOutWithMock(dev_server, '_get_image_storage_server')
+ dev_server._get_image_storage_server().AndReturn(
+ 'gs://my_default_url')
+ name = 'fake/image'
+ else:
+ # This is embedded in the archive_url. Not needed.
+ name = ''
+
+ to_return = StringIO.StringIO('Success')
+ urllib2.urlopen(mox.And(mox.StrContains(expected_archive_url),
+ mox.StrContains(name),
+ mox.StrContains('artifacts=%s' %
+ ','.join(artifacts)),
+ mox.StrContains('files=%s' % ','.join(files)),
+ mox.StrContains('stage?'))).AndReturn(to_return)
+ to_return = StringIO.StringIO('True')
+ urllib2.urlopen(mox.And(mox.StrContains(expected_archive_url),
+ mox.StrContains(name),
+ mox.StrContains('artifacts=%s' %
+ ','.join(artifacts)),
+ mox.StrContains('files=%s' % ','.join(files)),
+ mox.StrContains('is_staged'))).AndReturn(
+ to_return)
+
+ self.mox.ReplayAll()
+ self.dev_server.stage_artifacts(name, artifacts, files, archive_url)
+ self.mox.VerifyAll()
+
+
+ def testStageArtifactsBasic(self):
+ """Basic functionality to stage artifacts (similar to trigger_download).
+ """
+ self._stageTestHelper(artifacts=['full_payload', 'stateful'])
+
+
+ def testStageArtifactsBasicWithFiles(self):
+ """Basic functionality to stage artifacts (similar to trigger_download).
+ """
+ self._stageTestHelper(artifacts=['full_payload', 'stateful'],
+ files=['taco_bell.coupon'])
+
+
+ def testStageArtifactsOnlyFiles(self):
+ """Test staging of only file artifacts."""
+ self._stageTestHelper(files=['tasty_taco_bell.coupon'])
+
+
+ def testStageWithArchiveURL(self):
+ """Basic functionality to stage artifacts (similar to trigger_download).
+ """
+ self._stageTestHelper(files=['tasty_taco_bell.coupon'],
+ archive_url='gs://tacos_galore/my/dir')
+
+
+ def testStagedFileUrl(self):
+ """Sanity tests that the staged file url looks right."""
+ devserver_label = 'x86-mario-release/R30-1234.0.0'
+ url = self.dev_server.get_staged_file_url('stateful.tgz',
+ devserver_label)
+ expected_url = '/'.join([self._HOST, 'static', devserver_label,
+ 'stateful.tgz'])
+ self.assertEquals(url, expected_url)
+
+ devserver_label = 'something_crazy/that/you_MIGHT/hate'
+ url = self.dev_server.get_staged_file_url('chromiumos_image.bin',
+ devserver_label)
+ expected_url = '/'.join([self._HOST, 'static', devserver_label,
+ 'chromiumos_image.bin'])
+ self.assertEquals(url, expected_url)