Download official & trybot images; treat as local images for tests.
Currently we cannot get md5sums for official & trybot images, which
causes multiple problems: Crosperf re-images the DUT every time it
runs a test for an image, even if the image is already on the DUT;
if crosperf is running an experiment with multiple official or trybot
images, it can get confused about which results belong to which image;
caching does not work properly without md5sums.
To fix all of these problems, this CL changes Crosperf to download the
official or trybot images into the /tmp directory in the chroot, and from
there it can treat them as local images (getting md5sums etc). In order
to download them, it first has to translate the xbuddy syntax (which can
contain aliases such as 'lumpy/dev-latest') into the actual image name
(e.g. lumpy-release/R36-5727.0.0). This translation is done by a new
script, translate_xbuddy.py, which must be run from the toolchain-utils
directory inside the chroot.
BUG=356279,356474,356476
TEST=Tested with official and trybot images, using both correct
designations and xbuddy aliases; tested with the image missing from
the chroot and with it already in /tmp; tested with and without caching.
Also tested with the experiment file included in the 3 bugs.
Change-Id: I917b5520604b7d4851db2c8123165e81b866da2b
Reviewed-on: https://chrome-internal-review.googlesource.com/159465
Reviewed-by: Yunlian Jiang <yunlian@google.com>
Commit-Queue: Caroline Tice <cmtice@google.com>
Tested-by: Caroline Tice <cmtice@google.com>
diff --git a/crosperf/download_images.py b/crosperf/download_images.py
new file mode 100644
index 0000000..ac8fd3a
--- /dev/null
+++ b/crosperf/download_images.py
@@ -0,0 +1,52 @@
+#!/usr/bin/python
+
+# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+
+from utils import command_executer
+
+class ImageDownloader(object):
+
+ def __init__(self, logger_to_use=None, log_level="verbose"):
+ self._logger = logger_to_use
+ self.log_level = log_level
+ self._ce = command_executer.GetCommandExecuter(self._logger,
+ log_level = self.log_level)
+
+ def Run(self, chromeos_root, xbuddy_label):
+ # Get the translation of the xbuddy_label into the real Google Storage
+ # image name.
+ command = ("cd ~/trunk/src/third_party/toolchain-utils/crosperf; "
+ "python translate_xbuddy.py '%s'" % xbuddy_label)
+ retval, build_id_tuple_str, _ = self._ce.ChrootRunCommand(chromeos_root,
+ command, True)
+ build_id_tuple = eval(build_id_tuple_str)
+ build_id = build_id_tuple[0]
+
+ # Make sure the directory for downloading the image exists.
+ download_path = os.path.join(chromeos_root, "chroot/tmp",
+ build_id)
+ image_path = os.path.join(download_path, "chromiumos_test_image.bin")
+ if not os.path.exists(download_path):
+ command = "mkdir -p %s" % download_path
+ self._ce.RunCommand (command)
+
+ # Check to see if the image has already been downloaded. If not,
+ # download the image.
+ retval = 0
+ if not os.path.exists(image_path):
+ command = ("gsutil cp gs://chromeos-image-archive/%s"
+ "/chromiumos_test_image.tar.xz /tmp/%s" % (build_id,
+ build_id))
+
+ retval = self._ce.ChrootRunCommand(chromeos_root, command)
+
+ # Uncompress and untar the downloaded image.
+ command = ("cd /tmp/%s ;unxz chromiumos_test_image.tar.xz; "
+ "tar -xvf chromiumos_test_image.tar" % build_id)
+ retval = self._ce.ChrootRunCommand(chromeos_root, command)
+
+ return retval, image_path
diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py
index 0050da5..e10bc26 100644
--- a/crosperf/experiment_factory.py
+++ b/crosperf/experiment_factory.py
@@ -190,10 +190,11 @@
label_name = label_settings.name
board = label_settings.GetField("board")
image = label_settings.GetField("chromeos_image")
+ chromeos_root = label_settings.GetField("chromeos_root")
if image == "":
build = label_settings.GetField("build")
- image = label_settings.GetXbuddyPath (build, board)
- chromeos_root = label_settings.GetField("chromeos_root")
+ image = label_settings.GetXbuddyPath (build, board, chromeos_root,
+ log_level)
my_remote = label_settings.GetField("remote")
new_remote = []
for i in my_remote:
diff --git a/crosperf/experiment_file.py b/crosperf/experiment_file.py
index 0a3c3b9..3cb46dc 100644
--- a/crosperf/experiment_file.py
+++ b/crosperf/experiment_file.py
@@ -153,8 +153,12 @@
if real_file != field.GetString():
res += "\t#actual_image: %s\n" % real_file
if field.name == "build":
+ chromeos_root_field = settings.fields["chromeos_root"]
+ if chromeos_root_field:
+ chromeos_root = chromeos_root_field.GetString()
value = field.GetString()
- xbuddy_path = settings.GetXbuddyPath (value, board)
+ xbuddy_path = settings.GetXbuddyPath (value, board, chromeos_root,
+ "quiet")
res += "\t#actual_image: %s\n" % xbuddy_path
res += "}\n\n"
diff --git a/crosperf/settings.py b/crosperf/settings.py
index 0afc8e3..0e079bb 100644
--- a/crosperf/settings.py
+++ b/crosperf/settings.py
@@ -2,6 +2,8 @@
# Copyright 2011 Google Inc. All Rights Reserved.
+from utils import logger
+from download_images import *
class Settings(object):
"""Class representing settings (a set of fields) from an experiment file."""
@@ -61,10 +63,15 @@
if not self.fields[name].assigned and self.fields[name].required:
raise Exception("Field %s is invalid." % name)
- def GetXbuddyPath(self, path_str, board):
- prefix = "xbuddy://remote"
+ def GetXbuddyPath(self, path_str, board, chromeos_root, log_level):
+ prefix = "remote"
+ l = logger.GetLogger()
if path_str.find("trybot") < 0 and path_str.find(board) < 0:
xbuddy_path = "%s/%s/%s" % (prefix, board, path_str)
else:
xbuddy_path = "%s/%s" % (prefix, path_str)
- return xbuddy_path
+ image_downloader = ImageDownloader(l, log_level)
+ retval, image_path = image_downloader.Run(chromeos_root, xbuddy_path)
+ if retval != 0:
+ raise Exception("Unable to find/download xbuddy image.")
+ return image_path
diff --git a/crosperf/translate_xbuddy.py b/crosperf/translate_xbuddy.py
new file mode 100644
index 0000000..8a55927
--- /dev/null
+++ b/crosperf/translate_xbuddy.py
@@ -0,0 +1,30 @@
+#!/usr/bin/python
+
+import os
+import sys
+
+if '/mnt/host/source/src/third_party/toolchain-utils/crosperf' in sys.path:
+ dev_path = os.path.expanduser('~/trunk/src/platform/dev')
+ sys.path.append(dev_path)
+else:
+ print ('This script can only be run from inside a ChromeOS chroot. Please '
+ 'enter your chroot, go to ~/src/third_party/toolchain-utils/crosperf'
+ ' and try again.')
+ sys.exit(0)
+
+import build_util
+import xbuddy
+
+def Main(xbuddy_string):
+ if not os.path.exists('./xbuddy_config.ini'):
+ config_path = os.path.expanduser('~/trunk/src/platform/dev/'
+ 'xbuddy_config.ini')
+ os.symlink (config_path, './xbuddy_config.ini')
+ x = xbuddy.XBuddy(manage_builds=False, static_dir='/tmp/devserver/static')
+ build_id = x.Translate(os.path.split(xbuddy_string))
+ return build_id
+
+if __name__ == "__main__":
+ build_id = Main(sys.argv[1])
+ print build_id
+ sys.exit(0)
diff --git a/image_chromeos.py b/image_chromeos.py
index 97d1534..c6b9b02 100755
--- a/image_chromeos.py
+++ b/image_chromeos.py
@@ -176,13 +176,19 @@
real_src_dir = os.path.join(os.path.realpath(options.chromeos_root),
"src")
+ real_chroot_dir = os.path.join(os.path.realpath(options.chromeos_root),
+ "chroot")
if local_image:
if located_image.find(real_src_dir) != 0:
- raise Exception("Located image: %s not in chromeos_root: %s" %
- (located_image, options.chromeos_root))
- chroot_image = os.path.join(
- "..",
- located_image[len(real_src_dir):].lstrip("/"))
+ if located_image.find(real_chroot_dir) != 0:
+ raise Exception("Located image: %s not in chromeos_root: %s" %
+ (located_image, options.chromeos_root))
+ else:
+ chroot_image = located_image[len(real_chroot_dir):]
+ else:
+ chroot_image = os.path.join(
+ "..",
+ located_image[len(real_src_dir):].lstrip("/"))
# Check to see if cros flash is in the chroot or not.
use_cros_flash = CheckForCrosFlash (options.chromeos_root,
diff --git a/utils/tabulator.py b/utils/tabulator.py
index 9709105..0a2eae6 100644
--- a/utils/tabulator.py
+++ b/utils/tabulator.py
@@ -770,10 +770,16 @@
def GenerateCellTable(self, table_type):
row_index = 0
+ all_failed = False
for row in self._table[1:]:
# It does not make sense to put retval in the summary table.
if str(row[0]) == "retval" and table_type == "summary":
+ # Check to see if any runs passed, and update all_failed.
+ all_failed = True
+ for values in row[1:]:
+ if 0 in values:
+ all_failed = False
continue
key = Cell()
key.string_value = str(row[0])
@@ -802,6 +808,26 @@
self._out_table.append(out_row)
row_index += 1
+ # If this is a summary table, and the only row in it is 'retval', and
+ # all the test runs failed, we need to a 'Results' row to the output
+ # table.
+ if table_type == "summary" and all_failed and len(self._table) == 2:
+ labels_row = self._table[0]
+ key = Cell()
+ key.string_value = "Results"
+ out_row = [key]
+ baseline = None
+ for value in labels_row[1:]:
+ for column in self._columns:
+ cell = Cell()
+ cell.name = key.string_value
+ column.result.Compute(cell, ["Fail"], baseline)
+ column.fmt.Compute(cell)
+ out_row.append(cell)
+ if not row_index:
+ self._table_columns.append(column)
+ self._out_table.append(out_row)
+
def AddColumnName(self):
"""Generate Column name at the top of table."""
key = Cell()