Re-land: Autotest: rewrite machine_install to be robust to devserver flakiness.
This CL both simplifies machine_install as well as adds protections
when devservers are flaky to provisioning.
1) I simplified machine_install's repair logic by having it in one place
rather than scattered throughout machine_install and using the same
pre-condition state for all machine_install's (rather than a heavier one just
for repair).
2) Added a loop around the actual full update. Basically, I've added a #
of attempts for update. If an update fails, I will run a health check against
its devserver and re-attempt on another devserver if it's failed. This includes
re-staging the image on the other devserver as well as updating from it.
BUG=chromium:470581,chromium:471294
TEST=Moblab + Dynamic Suite + Instrumented Code to fail Autoupdate once to
check fallback + unittests in autoupdater.py + pyflakes/pylint on all
affected files.
Change-Id: I5b73a41db32ec832202f6f09f51604b62640b2b6
Reviewed-on: https://chromium-review.googlesource.com/267121
Reviewed-by: Chris Sosa <sosa@chromium.org>
Commit-Queue: Chris Sosa <sosa@chromium.org>
Tested-by: Chris Sosa <sosa@chromium.org>
diff --git a/client/common_lib/cros/autoupdater.py b/client/common_lib/cros/autoupdater.py
index 5dd930b..2bc8ce2 100644
--- a/client/common_lib/cros/autoupdater.py
+++ b/client/common_lib/cros/autoupdater.py
@@ -150,13 +150,11 @@
def reset_update_engine(self):
- """Restarts the update-engine service."""
+ """Resets the host to prepare for a clean update regardless of state."""
self._run('rm -f %s' % UPDATED_MARKER)
- try:
- self._run('initctl stop update-engine')
- except error.AutoservRunError:
- logging.warning('Stopping update-engine service failed. Already dead?')
- self._run('initctl start update-engine')
+ self._run('stop ui || true')
+ self._run('stop update-engine || true')
+ self._run('start update-engine')
if self.check_update_status() != UPDATER_IDLE:
raise ChromiumOSError('%s is not in an installable state' %
@@ -352,14 +350,11 @@
def update_rootfs(self):
- """Updates the rootfs partition only."""
- logging.info('Updating root partition...')
-
- # Run update_engine using the specified URL.
+ """Run the standard command to force an update."""
try:
autoupdate_cmd = '%s --update --omaha_url=%s 2>&1' % (
UPDATER_BIN, self.update_url)
- self._run(autoupdate_cmd, timeout=900)
+ self._run(autoupdate_cmd, timeout=1200)
except error.AutoservRunError:
list_image_dir_contents(self.update_url)
update_error = RootFSUpdateError('update-engine failed on %s' %
@@ -407,21 +402,12 @@
raise e
- def run_update(self, force_update, update_root=True):
+ def run_update(self, update_root=True):
"""Update the DUT with image of specific version.
- @param force_update: True to update DUT even if it's running the same
- version already.
- @param update_root: True to force a kernel update. If it's False and
- force_update is True, stateful update will be used to clean up
- the DUT.
-
+ @param update_root: True to force a rootfs update.
"""
booted_version = self.host.get_release_version()
- if (self.check_version() and not force_update):
- logging.info('System is already up to date. Skipping update.')
- return False
-
if self.update_version:
logging.info('Updating from version %s to %s.',
booted_version, self.update_version)
@@ -463,7 +449,6 @@
raise update_error
logging.info('Update complete.')
- return True
except:
# Collect update engine logs in the event of failure.
if self.host.job: