Add --logging_level option to crosperf.
Add --logging_level={verbose, average, quiet} option to crosperf,
optionally reducing the amount of logging output. "verbose" gives
the same output Crosperf always gave; "quiet" gives minimal output, and
"average" is in the middle. The default for now is "verbose". Doing this
also involved adding logging levels to the command_executer, and to
image_chromeos (again, with the default being "verbose").
BUG=None
TEST=Tested all three levels of logging, with and without having to
reimage the DUT. It all worked properly.
Change-Id: Icf8c9a6831fe25202adcb624c6c7e1d1a7ac25a5
Reviewed-on: https://chrome-internal-review.googlesource.com/156883
Reviewed-by: Caroline Tice <cmtice@google.com>
Commit-Queue: Caroline Tice <cmtice@google.com>
Tested-by: Caroline Tice <cmtice@google.com>
diff --git a/crosperf/benchmark_run.py b/crosperf/benchmark_run.py
index fa40997..7e61543 100644
--- a/crosperf/benchmark_run.py
+++ b/crosperf/benchmark_run.py
@@ -35,10 +35,12 @@
cache_conditions,
machine_manager,
logger_to_use,
+ log_level,
share_users):
threading.Thread.__init__(self)
self.name = name
self._logger = logger_to_use
+ self.log_level = log_level
self.benchmark = benchmark
self.iteration = iteration
self.label = label
@@ -47,7 +49,7 @@
self.retval = None
self.run_completed = False
self.machine_manager = machine_manager
- self.suite_runner = SuiteRunner(self._logger)
+ self.suite_runner = SuiteRunner(self._logger, self.log_level)
self.machine = None
self.cache_conditions = cache_conditions
self.runs_complete = 0
@@ -55,7 +57,8 @@
self.failure_reason = ""
self.test_args = benchmark.test_args
self.profiler_args = self._GetExtraAutotestArgs()
- self._ce = command_executer.GetCommandExecuter(self._logger)
+ self._ce = command_executer.GetCommandExecuter(self._logger,
+ log_level=self.log_level)
self.timeline = timeline.Timeline()
self.timeline.Record(STATUS_PENDING)
self.share_users = share_users
@@ -74,6 +77,7 @@
self.label.board,
self.cache_conditions,
self._logger,
+ self.log_level,
self.label,
self.share_users,
self.benchmark.suite,
@@ -185,6 +189,7 @@
self.profiler_args)
self.run_completed = True
return Result.CreateFromRun(self._logger,
+ self.log_level,
self.label,
out,
err,
@@ -214,6 +219,7 @@
self.label.board,
self.cache_conditions,
self._logger,
+ self.log_level,
self.label,
self.share_users,
self.benchmark.suite,
diff --git a/crosperf/experiment.py b/crosperf/experiment.py
index ec0b20e..aa56455 100644
--- a/crosperf/experiment.py
+++ b/crosperf/experiment.py
@@ -24,7 +24,7 @@
def __init__(self, name, remote, working_directory,
chromeos_root, cache_conditions, labels, benchmarks,
experiment_file, email_to, acquire_timeout, log_dir,
- share_users, results_directory):
+ log_level, share_users, results_directory):
self.name = name
self.working_directory = working_directory
self.remote = remote
@@ -38,6 +38,7 @@
else:
self.results_directory = misc.CanonicalizePath(results_directory)
self.log_dir = log_dir
+ self.log_level = log_level
self.labels = labels
self.benchmarks = benchmarks
self.num_complete = 0
@@ -55,9 +56,11 @@
"the image path.")
if test_flag.GetTestMode():
- self.machine_manager = MockMachineManager(chromeos_root, acquire_timeout)
+ self.machine_manager = MockMachineManager(chromeos_root, acquire_timeout,
+ log_level)
else:
- self.machine_manager = MachineManager(chromeos_root, acquire_timeout)
+ self.machine_manager = MachineManager(chromeos_root, acquire_timeout,
+ log_level)
self.l = logger.GetLogger(log_dir)
for machine in remote:
@@ -81,7 +84,8 @@
full_name = "%s_%s_%s" % (label.name, benchmark.name, iteration)
logger_to_use = logger.Logger(self.log_dir,
"run.%s" % (full_name),
- True)
+ True,
+ self.log_level)
benchmark_run = BenchmarkRun(benchmark_run_name,
benchmark,
label,
@@ -89,6 +93,7 @@
self.cache_conditions,
self.machine_manager,
logger_to_use,
+ self.log_level,
self.share_users)
benchmark_runs.append(benchmark_run)
diff --git a/crosperf/experiment_factory.py b/crosperf/experiment_factory.py
index 86118a3..ac9e6ef 100644
--- a/crosperf/experiment_factory.py
+++ b/crosperf/experiment_factory.py
@@ -141,6 +141,9 @@
suite = benchmark_settings.GetField("suite")
use_test_that = benchmark_settings.GetField("use_test_that")
show_all_results = benchmark_settings.GetField("show_all_results")
+ log_level = benchmark_settings.GetField("logging_level")
+ if log_level not in ("quiet", "average", "verbose"):
+ log_level = "verbose"
if suite == 'telemetry_Crosperf':
if test_name == 'all_perfv2':
@@ -226,7 +229,8 @@
working_directory, chromeos_root,
cache_conditions, labels, benchmarks,
experiment_file.Canonicalize(),
- email, acquire_timeout, log_dir, share_users,
+ email, acquire_timeout, log_dir, log_level,
+ share_users,
results_dir)
return experiment
diff --git a/crosperf/experiment_runner.py b/crosperf/experiment_runner.py
index 9212ba5..002fae3 100644
--- a/crosperf/experiment_runner.py
+++ b/crosperf/experiment_runner.py
@@ -29,21 +29,39 @@
self.l = logger.GetLogger(experiment.log_dir)
self._ce = command_executer.GetCommandExecuter(self.l)
self._terminated = False
+ if experiment.log_level != "verbose":
+ self.STATUS_TIME_DELAY = 10
def _Run(self, experiment):
status = ExperimentStatus(experiment)
experiment.Run()
last_status_time = 0
+ last_status_string = ""
try:
+ if experiment.log_level == "quiet":
+ self.l.LogStartDots()
while not experiment.IsComplete():
if last_status_time + self.STATUS_TIME_DELAY < time.time():
last_status_time = time.time()
border = "=============================="
- self.l.LogOutput(border)
- self.l.LogOutput(status.GetProgressString())
- self.l.LogOutput(status.GetStatusString())
- logger.GetLogger().LogOutput(border)
+ if experiment.log_level == "verbose":
+ self.l.LogOutput(border)
+ self.l.LogOutput(status.GetProgressString())
+ self.l.LogOutput(status.GetStatusString())
+ self.l.LogOutput(border)
+ else:
+ current_status_string = status.GetStatusString()
+ if (current_status_string != last_status_string):
+ self.l.LogEndDots()
+ self.l.LogOutput(border)
+ self.l.LogOutput(current_status_string)
+ self.l.LogOutput(border)
+ last_status_string = current_status_string
+ else:
+ self.l.LogAppendDot()
time.sleep(self.THREAD_MONITOR_DELAY)
+ if experiment.log_level != "verbose":
+ self.l.LogEndDots()
except KeyboardInterrupt:
self._terminated = True
self.l.LogError("Ctrl-c pressed. Cleaning up...")
diff --git a/crosperf/experiment_status.py b/crosperf/experiment_status.py
index a9ff8ed..f855ca6 100644
--- a/crosperf/experiment_status.py
+++ b/crosperf/experiment_status.py
@@ -16,6 +16,7 @@
self.num_total = len(self.experiment.benchmark_runs)
self.completed = 0
self.new_job_start_time = time.time()
+ self.log_level = experiment.log_level
def _GetProgressBar(self, num_complete, num_total):
ret = "Done: %s%%" % int(100.0 * num_complete / num_total)
@@ -96,8 +97,9 @@
(key, self._GetNamesAndIterations(val)))
result = "Thread Status:\n%s" % "\n".join(status_strings)
- # Add the machine manager status.
- result += "\n" + self.experiment.machine_manager.AsString() + "\n"
+ if self.experiment.log_level == "verbose":
+ # Add the machine manager status.
+ result += "\n" + self.experiment.machine_manager.AsString() + "\n"
return result
@@ -107,5 +109,8 @@
for benchmark_run in benchmark_runs:
t_last = benchmark_run.timeline.GetLastEventTime()
elapsed = str(datetime.timedelta(seconds=int(t-t_last)))
- strings.append("'{0}' {1}".format(benchmark_run.name, elapsed))
+ if self.experiment.log_level == "verbose":
+ strings.append("'{0}' {1}".format(benchmark_run.name, elapsed))
+ else:
+ strings.append("'{0}'".format(benchmark_run.name))
return " %s (%s)" % (len(strings), ", ".join(strings))
diff --git a/crosperf/image_checksummer.py b/crosperf/image_checksummer.py
index eeb4466..2f6694d 100644
--- a/crosperf/image_checksummer.py
+++ b/crosperf/image_checksummer.py
@@ -11,10 +11,11 @@
class ImageChecksummer(object):
class PerImageChecksummer(object):
- def __init__(self, label):
+ def __init__(self, label, log_level):
self._lock = threading.Lock()
self.label = label
self._checksum = None
+ self.log_level = log_level
def Checksum(self):
with self._lock:
@@ -26,7 +27,8 @@
raise Exception("Called Checksum on non-local image!")
if self.label.chromeos_image:
if os.path.exists(self.label.chromeos_image):
- self._checksum = FileUtils().Md5File(self.label.chromeos_image)
+ self._checksum = FileUtils().Md5File(self.label.chromeos_image,
+ log_level=self.log_level)
logger.GetLogger().LogOutput("Computed checksum is "
": %s" % self._checksum)
if not self._checksum:
@@ -50,13 +52,14 @@
*args, **kwargs)
return cls._instance
- def Checksum(self, label):
+ def Checksum(self, label, log_level):
if label.image_type != "local":
raise Exception("Attempt to call Checksum on non-local image.")
with self._lock:
if label.name not in self._per_image_checksummers:
self._per_image_checksummers[label.name] = (ImageChecksummer.
- PerImageChecksummer(label))
+ PerImageChecksummer(label,
+ log_level))
checksummer = self._per_image_checksummers[label.name]
try:
diff --git a/crosperf/machine_manager.py b/crosperf/machine_manager.py
index 51e1055..0271df4 100644
--- a/crosperf/machine_manager.py
+++ b/crosperf/machine_manager.py
@@ -24,7 +24,7 @@
class CrosMachine(object):
- def __init__(self, name, chromeos_root):
+ def __init__(self, name, chromeos_root, log_level):
self.name = name
self.image = None
self.checksum = None
@@ -32,6 +32,7 @@
self.released_time = time.time()
self.test_run = None
self.chromeos_root = chromeos_root
+ self.log_level = log_level
if not self.IsReachable():
self.machine_checksum = None
return
@@ -43,7 +44,7 @@
self.machine_id_checksum = self._GetMD5Checksum(self.machine_id)
def IsReachable(self):
- ce = command_executer.GetCommandExecuter()
+ ce = command_executer.GetCommandExecuter(log_level=self.log_level)
command = "ls"
ret = ce.CrosRunCommand(command,
machine=self.name,
@@ -86,7 +87,7 @@
def _GetMemoryInfo(self):
#TODO yunlian: when the machine in rebooting, it will not return
#meminfo, the assert does not catch it either
- ce = command_executer.GetCommandExecuter()
+ ce = command_executer.GetCommandExecuter(log_level=self.log_level)
command = "cat /proc/meminfo"
ret, self.meminfo, _ = ce.CrosRunCommand(
command, return_output=True,
@@ -101,7 +102,7 @@
return 0
def _GetCPUInfo(self):
- ce = command_executer.GetCommandExecuter()
+ ce = command_executer.GetCommandExecuter(log_level=self.log_level)
command = "cat /proc/cpuinfo"
ret, self.cpuinfo, _ = ce.CrosRunCommand(
command, return_output=True,
@@ -125,7 +126,7 @@
return ""
def _GetMachineID(self):
- ce = command_executer.GetCommandExecuter()
+ ce = command_executer.GetCommandExecuter(log_level=self.log_level)
command = "dump_vpd_log --full --stdout"
ret, if_out, _ = ce.CrosRunCommand(
command, return_output=True,
@@ -161,7 +162,7 @@
class MachineManager(object):
- def __init__(self, chromeos_root, acquire_timeout):
+ def __init__(self, chromeos_root, acquire_timeout, log_level):
self._lock = threading.RLock()
self._all_machines = []
self._machines = []
@@ -171,6 +172,7 @@
self.machine_checksum = {}
self.machine_checksum_string = {}
self.acquire_timeout = acquire_timeout
+ self.log_level = log_level
if os.path.isdir(lock_machine.Machine.LOCKS_DIR):
self.no_lock = False
@@ -181,7 +183,7 @@
def ImageMachine(self, machine, label):
if label.image_type == "local":
- checksum = ImageChecksummer().Checksum(label)
+ checksum = ImageChecksummer().Checksum(label, self.log_level)
elif label.image_type == "trybot":
checksum = machine._GetMD5Checksum(label.chromeos_image)
else:
@@ -196,20 +198,34 @@
"--chromeos_root=%s" % chromeos_root,
"--image=%s" % label.chromeos_image,
"--image_args=%s" % label.image_args,
- "--remote=%s" % machine.name]
+ "--remote=%s" % machine.name,
+ "--logging_level=%s" % self.log_level]
if label.board:
image_chromeos_args.append("--board=%s" % label.board)
# Currently can't image two machines at once.
# So have to serialized on this lock.
- ce = command_executer.GetCommandExecuter()
+ if self.log_level != "verbose":
+ ce = command_executer.GetCommandExecuter(log_level="average")
+ else:
+ ce = command_executer.GetCommandExecuter()
with self.image_lock:
+ if self.log_level != "verbose":
+ logger.GetLogger().LogOutput("Pushing image onto machine.")
+ logger.GetLogger().LogOutput("CMD : python %s "
+ % " ".join(image_chromeos_args))
retval = ce.RunCommand(" ".join(["python"] + image_chromeos_args))
if retval:
cmd ="reboot && exit"
+ if self.log_level != "verbose":
+ logger.GetLogger().LogOutput("reboot & exit.")
ce.CrosRunCommand(cmd, machine=machine.name,
chromeos_root=self.chromeos_root)
time.sleep(60)
+ if self.log_level != "verbose":
+ logger.GetLogger().LogOutput("Pushing image onto machine.")
+ logger.GetLogger().LogOutput("CMD : python %s "
+ % " ".join(image_chromeos_args))
retval = ce.RunCommand(" ".join(["python"] + image_chromeos_args))
if retval:
raise Exception("Could not image machine: '%s'." % machine.name)
@@ -244,7 +260,7 @@
locked = lock_machine.Machine(cros_machine.name).Lock(True, sys.argv[0])
if locked:
self._machines.append(cros_machine)
- ce = command_executer.GetCommandExecuter()
+ ce = command_executer.GetCommandExecuter(log_level=self.log_level)
command = "cat %s" % CHECKSUM_FILE
ret, out, _ = ce.CrosRunCommand(
command, return_output=True, chromeos_root=self.chromeos_root,
@@ -259,7 +275,12 @@
with self._lock:
for m in self._all_machines:
assert m.name != machine_name, "Tried to double-add %s" % machine_name
- cm = CrosMachine(machine_name, self.chromeos_root)
+ if self.log_level != "verbose":
+ logger.GetLogger().LogOutput("Setting up remote access to %s"
+ % machine_name)
+ logger.GetLogger().LogOutput("Checking machine characteristics for %s"
+ % machine_name)
+ cm = CrosMachine(machine_name, self.chromeos_root, self.log_level)
if cm.machine_checksum:
self._all_machines.append(cm)
@@ -278,7 +299,7 @@
def AcquireMachine(self, chromeos_image, label):
if label.image_type == "local":
- image_checksum = ImageChecksummer().Checksum(label)
+ image_checksum = ImageChecksummer().Checksum(label, self.log_level)
elif label.image_type == "trybot":
image_checksum = hashlib.md5(chromeos_image).hexdigest()
else:
@@ -423,7 +444,7 @@
class MockCrosMachine(CrosMachine):
- def __init__(self, name, chromeos_root):
+ def __init__(self, name, chromeos_root, log_level):
self.name = name
self.image = None
self.checksum = None
@@ -434,12 +455,14 @@
self.checksum_string = re.sub("\d", "", name)
#In test, we assume "lumpy1", "lumpy2" are the same machine.
self.machine_checksum = self._GetMD5Checksum(self.checksum_string)
+ self.log_level = log_level
class MockMachineManager(MachineManager):
- def __init__(self, chromeos_root, acquire_timeout):
- super(MockMachineManager, self).__init__(chromeos_root, acquire_timeout)
+ def __init__(self, chromeos_root, acquire_timeout, log_level):
+ super(MockMachineManager, self).__init__(chromeos_root, acquire_timeout,
+ log_level)
def _TryToLockMachine(self, cros_machine):
self._machines.append(cros_machine)
@@ -449,7 +472,7 @@
with self._lock:
for m in self._all_machines:
assert m.name != machine_name, "Tried to double-add %s" % machine_name
- cm = MockCrosMachine(machine_name, self.chromeos_root)
+ cm = MockCrosMachine(machine_name, self.chromeos_root, self.log_level)
assert cm.machine_checksum, ("Could not find checksum for machine %s" %
machine_name)
self._all_machines.append(cm)
diff --git a/crosperf/results_cache.py b/crosperf/results_cache.py
index 2402c2e..fcb86ae 100644
--- a/crosperf/results_cache.py
+++ b/crosperf/results_cache.py
@@ -35,13 +35,15 @@
perf.report, etc. The key generation is handled by the ResultsCache class.
"""
- def __init__(self, logger, label):
+ def __init__(self, logger, label, log_level):
self._chromeos_root = label.chromeos_root
self._logger = logger
- self._ce = command_executer.GetCommandExecuter(self._logger)
+ self._ce = command_executer.GetCommandExecuter(self._logger,
+ log_level=log_level)
self._temp_dir = None
self.label = label
self.results_dir = None
+ self.log_level = log_level
self.perf_data_files = []
self.perf_report_files = []
@@ -386,22 +388,22 @@
(temp_dir, cache_dir))
@classmethod
- def CreateFromRun(cls, logger, label, out, err, retval, show_all, test,
- suite="pyauto"):
+ def CreateFromRun(cls, logger, log_level, label, out, err, retval, show_all,
+ test, suite="pyauto"):
if suite == "telemetry":
result = TelemetryResult(logger, label)
else:
- result = cls(logger, label)
+ result = cls(logger, label, log_level)
result._PopulateFromRun(out, err, retval, show_all, test, suite)
return result
@classmethod
- def CreateFromCacheHit(cls, logger, label, cache_dir, show_all, test,
- suite="pyauto"):
+ def CreateFromCacheHit(cls, logger, log_level, label, cache_dir,
+ show_all, test, suite="pyauto"):
if suite == "telemetry":
result = TelemetryResult(logger, label)
else:
- result = cls(logger, label)
+ result = cls(logger, label, log_level)
try:
result._PopulateFromCacheDir(cache_dir, show_all, test, suite)
@@ -498,7 +500,8 @@
def Init(self, chromeos_image, chromeos_root, test_name, iteration,
test_args, profiler_args, machine_manager, board, cache_conditions,
- logger_to_use, label, share_users, suite, show_all_results):
+ logger_to_use, log_level, label, share_users, suite,
+ show_all_results):
self.chromeos_image = chromeos_image
self.chromeos_root = chromeos_root
self.test_name = test_name
@@ -509,10 +512,12 @@
self.cache_conditions = cache_conditions
self.machine_manager = machine_manager
self._logger = logger_to_use
- self._ce = command_executer.GetCommandExecuter(self._logger)
+ self._ce = command_executer.GetCommandExecuter(self._logger,
+ log_level=log_level)
self.label = label
self.share_users = share_users
self.suite = suite
+ self.log_level = log_level
self.show_all = show_all_results
def _GetCacheDirForRead(self):
@@ -556,7 +561,7 @@
elif self.label.image_type == "official":
checksum = "*"
else:
- checksum = ImageChecksummer().Checksum(self.label)
+ checksum = ImageChecksummer().Checksum(self.label, self.log_level)
if read and CacheConditions.IMAGE_PATH_MATCH not in self.cache_conditions:
image_path_checksum = "*"
@@ -599,6 +604,7 @@
self._logger.LogOutput("Trying to read from cache dir: %s" % cache_dir)
result = Result.CreateFromCacheHit(self._logger,
+ self.log_level,
self.label,
cache_dir,
self.show_all,
diff --git a/crosperf/settings_factory.py b/crosperf/settings_factory.py
index 6be3481..b827cc4 100644
--- a/crosperf/settings_factory.py
+++ b/crosperf/settings_factory.py
@@ -42,6 +42,9 @@
"command record or stat followed by arguments."))
self.AddField(TextField("suite", default="pyauto",
description="The type of the benchmark"))
+ self.AddField(TextField("logging_level", default="verbose",
+ description="The level of logging desired. "
+ "Options are 'quiet', 'average', and 'verbose'."))
self.AddField(BooleanField("use_test_that", default=True,
description="Whether to use the"
" new test_that script for running the test."))
@@ -127,6 +130,9 @@
"contains a src/scripts directory. Defaults to "
"the chromeos checkout which contains the "
"chromeos_image."))
+ self.AddField(TextField("logging_level", default="verbose",
+ description="The level of logging desired. "
+ "Options are 'quiet', 'average', and 'verbose'."))
self.AddField(BooleanField("key_results_only", default=True,
description="Whether only show the key results"
"of pyautoperf"))
diff --git a/crosperf/suite_runner.py b/crosperf/suite_runner.py
index 087f64d..00e6e51 100644
--- a/crosperf/suite_runner.py
+++ b/crosperf/suite_runner.py
@@ -43,9 +43,11 @@
""" This defines the interface from crosperf to test script.
"""
- def __init__(self, logger_to_use=None):
+ def __init__(self, logger_to_use=None, log_level="verbose"):
self._logger = logger_to_use
- self._ce = command_executer.GetCommandExecuter(self._logger)
+ self.log_level = log_level
+ self._ce = command_executer.GetCommandExecuter(self._logger,
+ log_level=self.log_level)
self._ct = command_executer.CommandTerminator()
def Run(self, machine, label, benchmark, test_args, profiler_args):
@@ -99,6 +101,9 @@
body="echo %s > $f" % highest_freq)
change_perf_gov = BASH_FOR.format(list=CPUFREQ_DIRS + "scaling_governor",
body="echo performance > $f")
+ if self.log_level == "average":
+ self._logger.LogOutput("Pinning governor execution frequencies for %s"
+ % machine_name)
ret = self._ce.CrosRunCommand(" && ".join(("set -e ",
change_max_freq,
change_min_freq,
@@ -135,6 +140,9 @@
command = ("./run_remote_tests.sh --use_emerged --remote=%s %s %s" %
(machine, options, benchmark.test_name))
+ if self.log_level != "verbose":
+ self._logger.LogOutput("Running test.")
+ self._logger.LogOutput("CMD: %s" % command)
return self._ce.ChrootRunCommand(label.chromeos_root,
command,
True,
@@ -159,6 +167,9 @@
command = ("%s %s %s %s" %
(TEST_THAT_PATH, options, machine, benchmark.test_name))
+ if self.log_level != "verbose":
+ self._logger.LogOutput("Running test.")
+ self._logger.LogOutput("CMD: %s" % command)
return self._ce.ChrootRunCommand(label.chromeos_root,
command,
True,
@@ -197,6 +208,9 @@
benchmark.test_name,
profiler_args,
machine))
+ if self.log_level != "verbose":
+ self._logger.LogOutput("Running test.")
+ self._logger.LogOutput("CMD: %s" % cmd)
return self._ce.ChrootRunCommand (label.chromeos_root,
cmd,
return_output=True,
@@ -221,6 +235,9 @@
rsa_key,
benchmark.test_name,
benchmark.test_args))
+ if self.log_level != "verbose":
+ self._logger.LogOutput("Running test.")
+ self._logger.LogOutput("CMD: %s" % cmd)
return self._ce.RunCommand(cmd, return_output=True,
print_to_console=False)
diff --git a/image_chromeos.py b/image_chromeos.py
index 001f4a6..06328e2 100755
--- a/image_chromeos.py
+++ b/image_chromeos.py
@@ -33,8 +33,8 @@
sys.exit(0)
-def CheckForCrosFlash (chromeos_root, remote):
- cmd_executer = command_executer.GetCommandExecuter()
+def CheckForCrosFlash(chromeos_root, remote, log_level):
+ cmd_executer = command_executer.GetCommandExecuter(log_level=log_level)
chroot_has_cros_flash = False
remote_has_cherrypy = False
@@ -48,7 +48,8 @@
# Check to see if remote machine has cherrypy.
keypath = os.path.join (os.path.realpath(chromeos_root),
- "src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa")
+ "src/scripts/mod_for_test_scripts/ssh_keys/"
+ "testing_rsa")
command = ("ssh -i %s -o StrictHostKeyChecking=no -o CheckHostIP=no "
"-o BatchMode=yes root@%s \"python -c 'import cherrypy'\" " %
@@ -62,10 +63,6 @@
def DoImage(argv):
"""Build ChromeOS."""
- # Common initializations
- cmd_executer = command_executer.GetCommandExecuter()
- l = logger.GetLogger()
-
parser = optparse.OptionParser()
parser.add_option("-c", "--chromeos_root", dest="chromeos_root",
help="Target directory for ChromeOS installation.")
@@ -79,6 +76,10 @@
action="store_true",
default=False,
help="Force an image even if it is non-test.")
+ parser.add_option("-l", "--logging_level", dest="log_level",
+ default="verbose",
+ help="Amount of logging to be used. Valid levels are "
+ "'quiet', 'average', and 'verbose'.")
parser.add_option("-a",
"--image_args",
dest="image_args")
@@ -86,6 +87,15 @@
options = parser.parse_args(argv[1:])[0]
+ if not options.log_level in command_executer.LOG_LEVEL:
+ Usage(parser, "--logging_level must be 'quiet', 'average' or 'verbose'")
+ else:
+ log_level = options.log_level
+
+ # Common initializations
+ cmd_executer = command_executer.GetCommandExecuter(log_level=log_level)
+ l = logger.GetLogger()
+
if options.chromeos_root is None:
Usage(parser, "--chromeos_root must be set")
@@ -123,7 +133,7 @@
local_image = False
if image.find("xbuddy://") < 0:
local_image = True
- image_checksum = FileUtils().Md5File(image)
+ image_checksum = FileUtils().Md5File(image, log_level=log_level)
command = "cat " + checksum_file
retval, device_checksum, err = cmd_executer.CrosRunCommand(command,
@@ -146,7 +156,7 @@
l.LogOutput("Checksums do not match. Re-imaging...")
is_test_image = IsImageModdedForTest(options.chromeos_root,
- located_image)
+ located_image, log_level)
if not is_test_image and not options.force:
logger.GetLogger().LogFatal("Have to pass --force to image a non-test "
@@ -176,7 +186,7 @@
# Check to see if cros flash is in the chroot or not.
use_cros_flash = CheckForCrosFlash (options.chromeos_root,
- options.remote)
+ options.remote, log_level)
if use_cros_flash:
# Use 'cros flash'
@@ -212,15 +222,25 @@
# Workaround for crosbug.com/35684.
os.chmod(misc.GetChromeOSKeyFile(options.chromeos_root), 0600)
+ if log_level == "quiet":
+ l.LogOutput("CMD : %s" % command)
+ elif log_level == "average":
+ cmd_executer.SetLogLevel("verbose");
retval = cmd_executer.ChrootRunCommand(options.chromeos_root,
command, command_timeout=600)
retries = 0
while retval != 0 and retries < 2:
retries += 1
+ if log_level == "quiet":
+ l.LogOutput("Imaging failed. Retry # %d." % retries)
+ l.LogOutput("CMD : %s" % command)
retval = cmd_executer.ChrootRunCommand(options.chromeos_root,
command, command_timeout=600)
+ if log_level == "average":
+ cmd_executer.SetLogLevel(log_level)
+
if found == False:
temp_dir = os.path.dirname(located_image)
l.LogOutput("Deleting temp image dir: %s" % temp_dir)
@@ -230,7 +250,8 @@
# Unfortunately cros_image_to_target.py sometimes returns early when the
# machine isn't fully up yet.
- retval = EnsureMachineUp(options.chromeos_root, options.remote)
+ retval = EnsureMachineUp(options.chromeos_root, options.remote,
+ log_level)
# If this is a non-local image, then the retval returned from
# EnsureMachineUp is the one that will be returned by this function;
@@ -241,6 +262,8 @@
retval = 1
if local_image:
+ if log_level == "average":
+ l.LogOutput("Verifying image.")
command = "echo %s > %s && chmod -w %s" % (image_checksum,
checksum_file,
checksum_file)
@@ -251,10 +274,11 @@
successfully_imaged = VerifyChromeChecksum(options.chromeos_root,
image,
- options.remote)
+ options.remote, log_level)
logger.GetLogger().LogFatalIf(not successfully_imaged,
"Image verification failed!")
- TryRemountPartitionAsRW(options.chromeos_root, options.remote)
+ TryRemountPartitionAsRW(options.chromeos_root, options.remote,
+ log_level)
else:
l.LogOutput("Checksums match. Skipping reimage")
return retval
@@ -282,7 +306,8 @@
if filecmp.cmp(potential_image, image):
l.LogOutput("Found matching image %s in chromeos_root." % potential_image)
return [True, potential_image]
- # We did not find an image. Copy it in the src dir and return the copied file.
+ # We did not find an image. Copy it in the src dir and return the copied
+ # file.
if board is None:
board = ""
base_dir = ("%s/src/build/images/%s" %
@@ -311,8 +336,9 @@
return mount_command
-def MountImage(chromeos_root, image, rootfs_mp, stateful_mp, unmount=False):
- cmd_executer = command_executer.GetCommandExecuter()
+def MountImage(chromeos_root, image, rootfs_mp, stateful_mp, log_level,
+ unmount=False):
+ cmd_executer = command_executer.GetCommandExecuter(log_level=log_level)
command = GetImageMountCommand(chromeos_root, image, rootfs_mp, stateful_mp)
if unmount:
command = "%s --unmount" % command
@@ -321,25 +347,30 @@
return retval
-def IsImageModdedForTest(chromeos_root, image):
+def IsImageModdedForTest(chromeos_root, image, log_level):
+ if log_level != "verbose":
+ log_level = "quiet"
rootfs_mp = tempfile.mkdtemp()
stateful_mp = tempfile.mkdtemp()
- MountImage(chromeos_root, image, rootfs_mp, stateful_mp)
+ MountImage(chromeos_root, image, rootfs_mp, stateful_mp, log_level)
lsb_release_file = os.path.join(rootfs_mp, "etc/lsb-release")
lsb_release_contents = open(lsb_release_file).read()
is_test_image = re.search("test", lsb_release_contents, re.IGNORECASE)
- MountImage(chromeos_root, image, rootfs_mp, stateful_mp, unmount=True)
+ MountImage(chromeos_root, image, rootfs_mp, stateful_mp, log_level,
+ unmount=True)
return is_test_image
-def VerifyChromeChecksum(chromeos_root, image, remote):
- cmd_executer = command_executer.GetCommandExecuter()
+def VerifyChromeChecksum(chromeos_root, image, remote, log_level):
+ cmd_executer = command_executer.GetCommandExecuter(log_level=log_level)
rootfs_mp = tempfile.mkdtemp()
stateful_mp = tempfile.mkdtemp()
- MountImage(chromeos_root, image, rootfs_mp, stateful_mp)
+ MountImage(chromeos_root, image, rootfs_mp, stateful_mp, log_level)
image_chrome_checksum = FileUtils().Md5File("%s/opt/google/chrome/chrome" %
- rootfs_mp)
- MountImage(chromeos_root, image, rootfs_mp, stateful_mp, unmount=True)
+ rootfs_mp,
+ log_level=log_level)
+ MountImage(chromeos_root, image, rootfs_mp, stateful_mp, log_level,
+ unmount=True)
command = "md5sum /opt/google/chrome/chrome"
[r, o, e] = cmd_executer.CrosRunCommand(command,
@@ -354,9 +385,9 @@
# Remount partition as writable.
# TODO: auto-detect if an image is built using --noenable_rootfs_verification.
-def TryRemountPartitionAsRW(chromeos_root, remote):
+def TryRemountPartitionAsRW(chromeos_root, remote, log_level):
l = logger.GetLogger()
- cmd_executer = command_executer.GetCommandExecuter()
+ cmd_executer = command_executer.GetCommandExecuter(log_level=log_level)
command = "sudo mount -o remount,rw /"
retval = cmd_executer.CrosRunCommand(\
command, chromeos_root=chromeos_root, machine=remote, terminated_timeout=10)
@@ -370,9 +401,9 @@
l.LogOutput("Re-mounted partition as writable.")
-def EnsureMachineUp(chromeos_root, remote):
+def EnsureMachineUp(chromeos_root, remote, log_level):
l = logger.GetLogger()
- cmd_executer = command_executer.GetCommandExecuter()
+ cmd_executer = command_executer.GetCommandExecuter(log_level=log_level)
timeout = 600
magic = "abcdefghijklmnopqrstuvwxyz"
command = "echo %s" % magic
diff --git a/utils/command_executer.py b/utils/command_executer.py
index a4dca64..137348b 100644
--- a/utils/command_executer.py
+++ b/utils/command_executer.py
@@ -16,6 +16,7 @@
mock_default = False
+LOG_LEVEL=("quiet", "average", "verbose")
def InitCommandExecuter(mock=False):
global mock_default
@@ -23,21 +24,28 @@
mock_default = mock
-def GetCommandExecuter(logger_to_set=None, mock=False):
+def GetCommandExecuter(logger_to_set=None, mock=False, log_level="verbose"):
# If the default is a mock executer, always return one.
if mock_default or mock:
return MockCommandExecuter(logger_to_set)
else:
- return CommandExecuter(logger_to_set)
+ return CommandExecuter(log_level, logger_to_set)
class CommandExecuter:
- def __init__(self, logger_to_set=None):
+ def __init__(self, log_level, logger_to_set=None):
+ self.log_level = log_level
if logger_to_set is not None:
self.logger = logger_to_set
else:
self.logger = logger.GetLogger()
+ def GetLogLevel(self):
+ return self.log_level
+
+ def SetLogLevel(self, log_level):
+ self.log_level = log_level
+
def RunCommand(self, cmd, return_output=False, machine=None,
username=None, command_terminator=None,
command_timeout=None,
@@ -47,7 +55,11 @@
cmd = str(cmd)
- self.logger.LogCmd(cmd, machine, username, print_to_console)
+ if self.log_level == "quiet":
+ print_to_console=False
+
+ if self.log_level == "verbose":
+ self.logger.LogCmd(cmd, machine, username, print_to_console)
if command_terminator and command_terminator.IsTerminated():
self.logger.LogError("Command was terminated!", print_to_console)
if return_output:
@@ -170,7 +182,11 @@
terminated_timeout=10,
print_to_console=True):
"""Run a command on a chromeos box"""
- self.logger.LogCmd(cmd, print_to_console)
+
+ if self.log_level != "verbose":
+ print_to_console=False
+
+ self.logger.LogCmd(cmd, print_to_console=print_to_console)
self.logger.LogFatalIf(not machine, "No machine provided!")
self.logger.LogFatalIf(not chromeos_root, "chromeos_root not given!")
chromeos_root = os.path.expanduser(chromeos_root)
@@ -213,7 +229,11 @@
terminated_timeout=10,
print_to_console=True,
cros_sdk_options=""):
- self.logger.LogCmd(command, print_to_console)
+
+ if self.log_level != "verbose":
+ print_to_console = False
+
+ self.logger.LogCmd(command, print_to_console=print_to_console)
handle, command_file = tempfile.mkstemp(dir=os.path.join(chromeos_root,
"src/scripts"),
diff --git a/utils/file_utils.py b/utils/file_utils.py
index 2480933..86a3172 100644
--- a/utils/file_utils.py
+++ b/utils/file_utils.py
@@ -28,9 +28,9 @@
**kwargs)
return cls._instance
- def Md5File(self, filename, block_size=2 ** 10):
+ def Md5File(self, filename, log_level="verbose", block_size=2 ** 10):
command = "md5sum %s" % filename
- ce = command_executer.GetCommandExecuter()
+ ce = command_executer.GetCommandExecuter(log_level=log_level)
ret, out, err = ce.RunCommand(command, return_output=True)
if ret:
raise Exception("Could not run md5sum on: %s" % filename)
diff --git a/utils/logger.py b/utils/logger.py
index 4f826ad..984283c 100644
--- a/utils/logger.py
+++ b/utils/logger.py
@@ -95,6 +95,25 @@
if flush:
fd.flush()
+ def LogStartDots(self, print_to_console=True):
+ term_fd = self._GetStdout(print_to_console)
+ if (term_fd):
+ term_fd.flush()
+ term_fd.write(". ")
+ term_fd.flush()
+
+ def LogAppendDot(self, print_to_console=True):
+ term_fd = self._GetStdout(print_to_console)
+ if (term_fd):
+ term_fd.write(". ")
+ term_fd.flush()
+
+ def LogEndDots(self, print_to_console=True):
+ term_fd = self._GetStdout(print_to_console)
+ if (term_fd):
+ term_fd.write("\n")
+ term_fd.flush()
+
def _LogMsg(self, file_fd, term_fd, msg, flush=True):
if file_fd:
self._WriteTo(file_fd, msg, flush)
diff --git a/utils/tabulator.py b/utils/tabulator.py
index 6fec427..b2cc7d9 100644
--- a/utils/tabulator.py
+++ b/utils/tabulator.py
@@ -371,7 +371,10 @@
class CoeffVarResult(NumericalResult):
def _ComputeFloat(self, cell, values, baseline_values):
- noise = numpy.abs(numpy.std(values)/numpy.mean(values))
+ if numpy.mean(values) != 0.0:
+ noise = numpy.abs(numpy.std(values)/numpy.mean(values))
+ else:
+ noise = 0.0
cell.value = noise