Update the lab autotest code to send chart-json format data to
the chromeperf dashboard.
BUG=chromium:589868
TEST=Tested client side and server side tests locally, and unit tests.
Change-Id: Ief821a2d524313ef6328bf3749760e4df906e630
Reviewed-on: https://chromium-review.googlesource.com/331795
Commit-Ready: Keith Haddow <haddowk@chromium.org>
Tested-by: Keith Haddow <haddowk@chromium.org>
Reviewed-by: Keith Haddow <haddowk@chromium.org>
diff --git a/server/cros/telemetry_runner.py b/server/cros/telemetry_runner.py
index 30c44a5..2adb15b 100644
--- a/server/cros/telemetry_runner.py
+++ b/server/cros/telemetry_runner.py
@@ -2,12 +2,8 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
-import json
import logging
-import math
import os
-import pprint
-import re
import StringIO
from autotest_lib.client.common_lib import error, utils
@@ -16,8 +12,6 @@
TELEMETRY_RUN_BENCHMARKS_SCRIPT = 'tools/perf/run_benchmark'
-TELEMETRY_RUN_CROS_TESTS_SCRIPT = 'chrome/test/telemetry/run_cros_tests'
-TELEMETRY_RUN_GPU_TESTS_SCRIPT = 'content/test/gpu/run_gpu_test.py'
TELEMETRY_RUN_TESTS_SCRIPT = 'tools/telemetry/run_tests'
TELEMETRY_TIMEOUT_MINS = 120
@@ -26,17 +20,6 @@
WARNING_STATUS = 'WARNING'
FAILED_STATUS = 'FAILED'
-# Regex for the RESULT output lines understood by chrome buildbot.
-# Keep in sync with
-# chromium/tools/build/scripts/slave/performance_log_processor.py.
-RESULTS_REGEX = re.compile(r'(?P<IMPORTANT>\*)?RESULT '
- r'(?P<GRAPH>[^:]*): (?P<TRACE>[^=]*)= '
- r'(?P<VALUE>[\{\[]?[-\d\., ]+[\}\]]?)('
- r' ?(?P<UNITS>.+))?')
-HISTOGRAM_REGEX = re.compile(r'(?P<IMPORTANT>\*)?HISTOGRAM '
- r'(?P<GRAPH>[^:]*): (?P<TRACE>[^=]*)= '
- r'(?P<VALUE_JSON>{.*})(?P<UNITS>.+)?')
-
class TelemetryResult(object):
"""Class to represent the results of a telemetry run.
@@ -58,185 +41,11 @@
else:
self.status = FAILED_STATUS
- # A list of perf values, e.g.
- # [{'graph': 'graphA', 'trace': 'page_load_time',
- # 'units': 'secs', 'value':0.5}, ...]
- self.perf_data = []
self._stdout = stdout
self._stderr = stderr
self.output = '\n'.join([stdout, stderr])
- def _cleanup_perf_string(self, str):
- """Clean up a perf-related string by removing illegal characters.
-
- Perf keys stored in the chromeOS database may contain only letters,
- numbers, underscores, periods, and dashes. Transform an inputted
- string so that any illegal characters are replaced by underscores.
-
- @param str: The perf string to clean up.
-
- @return The cleaned-up perf string.
- """
- return re.sub(r'[^\w.-]', '_', str)
-
-
- def _cleanup_units_string(self, units):
- """Cleanup a units string.
-
- Given a string representing units for a perf measurement, clean it up
- by replacing certain illegal characters with meaningful alternatives.
- Any other illegal characters should then be replaced with underscores.
-
- Examples:
- count/time -> count_per_time
- % -> percent
- units! --> units_
- score (bigger is better) -> score__bigger_is_better_
- score (runs/s) -> score__runs_per_s_
-
- @param units: The units string to clean up.
-
- @return The cleaned-up units string.
- """
- if '%' in units:
- units = units.replace('%', 'percent')
- if '/' in units:
- units = units.replace('/','_per_')
- return self._cleanup_perf_string(units)
-
-
- def parse_benchmark_results(self):
- """Parse the results of a telemetry benchmark run.
-
- Stdout has the output in RESULT block format below.
-
- The lines of interest start with the substring "RESULT". These are
- specially-formatted perf data lines that are interpreted by chrome
- builbot (when the Telemetry tests run for chrome desktop) and are
- parsed to extract perf data that can then be displayed on a perf
- dashboard. This format is documented in the docstring of class
- GraphingLogProcessor in this file in the chrome tree:
-
- chromium/tools/build/scripts/slave/process_log_utils.py
-
- Example RESULT output lines:
- RESULT average_commit_time_by_url: http___www.ebay.com= 8.86528 ms
- RESULT CodeLoad: CodeLoad= 6343 score (bigger is better)
- RESULT ai-astar: ai-astar= [614,527,523,471,530,523,577,625,614,538] ms
-
- Currently for chromeOS, we can only associate a single perf key (string)
- with a perf value. That string can only contain letters, numbers,
- dashes, periods, and underscores, as defined by write_keyval() in:
-
- chromeos/src/third_party/autotest/files/client/common_lib/
- base_utils.py
-
- We therefore parse each RESULT line, clean up the strings to remove any
- illegal characters not accepted by chromeOS, and construct a perf key
- string based on the parsed components of the RESULT line (with each
- component separated by a special delimiter). We prefix the perf key
- with the substring "TELEMETRY" to identify it as a telemetry-formatted
- perf key.
-
- Stderr has the format of Warnings/Tracebacks. There is always a default
- warning of the display enviornment setting, followed by warnings of
- page timeouts or a traceback.
-
- If there are any other warnings we flag the test as warning. If there
- is a traceback we consider this test a failure.
- """
- if not self._stdout:
- # Nothing in stdout implies a test failure.
- logging.error('No stdout, test failed.')
- self.status = FAILED_STATUS
- return
-
- stdout_lines = self._stdout.splitlines()
- for line in stdout_lines:
- results_match = RESULTS_REGEX.search(line)
- histogram_match = HISTOGRAM_REGEX.search(line)
- if results_match:
- self._process_results_line(results_match)
- elif histogram_match:
- self._process_histogram_line(histogram_match)
-
- pp = pprint.PrettyPrinter(indent=2)
- logging.debug('Perf values: %s', pp.pformat(self.perf_data))
-
- if self.status is SUCCESS_STATUS:
- return
-
- # Otherwise check if simply a Warning occurred or a Failure,
- # i.e. a Traceback is listed.
- self.status = WARNING_STATUS
- for line in self._stderr.splitlines():
- if line.startswith('Traceback'):
- self.status = FAILED_STATUS
-
- def _process_results_line(self, line_match):
- """Processes a line that matches the standard RESULT line format.
-
- Args:
- line_match: A MatchObject as returned by re.search.
- """
- match_dict = line_match.groupdict()
- graph_name = self._cleanup_perf_string(match_dict['GRAPH'].strip())
- trace_name = self._cleanup_perf_string(match_dict['TRACE'].strip())
- units = self._cleanup_units_string(
- (match_dict['UNITS'] or 'units').strip())
- value = match_dict['VALUE'].strip()
- unused_important = match_dict['IMPORTANT'] or False # Unused now.
-
- if value.startswith('['):
- # A list of values, e.g., "[12,15,8,7,16]". Extract just the
- # numbers, compute the average and use that. In this example,
- # we'd get 12+15+8+7+16 / 5 --> 11.6.
- value_list = [float(x) for x in value.strip('[],').split(',')]
- value = float(sum(value_list)) / len(value_list)
- elif value.startswith('{'):
- # A single value along with a standard deviation, e.g.,
- # "{34.2,2.15}". Extract just the value itself and use that.
- # In this example, we'd get 34.2.
- value_list = [float(x) for x in value.strip('{},').split(',')]
- value = value_list[0] # Position 0 is the value.
- elif re.search('^\d+$', value):
- value = int(value)
- else:
- value = float(value)
-
- self.perf_data.append({'graph':graph_name, 'trace': trace_name,
- 'units': units, 'value': value})
-
- def _process_histogram_line(self, line_match):
- """Processes a line that matches the HISTOGRAM line format.
-
- Args:
- line_match: A MatchObject as returned by re.search.
- """
- match_dict = line_match.groupdict()
- graph_name = self._cleanup_perf_string(match_dict['GRAPH'].strip())
- trace_name = self._cleanup_perf_string(match_dict['TRACE'].strip())
- units = self._cleanup_units_string(
- (match_dict['UNITS'] or 'units').strip())
- histogram_json = match_dict['VALUE_JSON'].strip()
- unused_important = match_dict['IMPORTANT'] or False # Unused now.
- histogram_data = json.loads(histogram_json)
-
- # Compute geometric mean
- count = 0
- sum_of_logs = 0
- for bucket in histogram_data['buckets']:
- mean = (bucket['low'] + bucket['high']) / 2.0
- if mean > 0:
- sum_of_logs += math.log(mean) * bucket['count']
- count += bucket['count']
-
- value = math.exp(sum_of_logs / count) if count > 0 else 0.0
-
- self.perf_data.append({'graph':graph_name, 'trace': trace_name,
- 'units': units, 'value': value})
-
class TelemetryRunner(object):
"""Class responsible for telemetry for a given build.
@@ -331,8 +140,8 @@
"""
telemetry_cmd = []
if self._devserver:
- devserver_hostname = self._devserver.url().split(
- 'http://')[1].split(':')[0]
+ devserver_hostname = dev_server.DevServer.get_server_name(
+ self._devserver.url())
telemetry_cmd.extend(['ssh', devserver_hostname])
telemetry_cmd.extend(
@@ -340,11 +149,61 @@
script,
'--verbose',
'--browser=cros-chrome',
+ '--output-format=chartjson',
+ '--output-dir=%s' % self._telemetry_path,
'--remote=%s' % self._host.hostname])
telemetry_cmd.extend(args)
telemetry_cmd.append(test_or_benchmark)
- return telemetry_cmd
+ return ' '.join(telemetry_cmd)
+
+
+ def _scp_telemetry_results_cmd(self, perf_results_dir):
+ """Build command to copy the telemetry results from the devserver.
+
+ @param perf_results_dir: directory path where test output is to be
+ collected.
+ @returns SCP command to copy the results json to the specified directory.
+ """
+ scp_cmd = []
+ if self._devserver and perf_results_dir:
+ devserver_hostname = dev_server.DevServer.get_server_name(
+ self._devserver.url())
+ scp_cmd.extend(['scp',
+ '%s:%s/results-chart.json' % (
+ devserver_hostname, self._telemetry_path),
+ perf_results_dir])
+
+ return ' '.join(scp_cmd)
+
+
+ def _run_cmd(self, cmd):
+ """Execute an command in a external shell and capture the output.
+
+ @param cmd: String of is a valid shell command.
+
+ @returns The standard out, standard error and the integer exit code of
+ the executed command.
+ """
+ logging.debug('Running: %s', cmd)
+
+ output = StringIO.StringIO()
+ error_output = StringIO.StringIO()
+ exit_code = 0
+ try:
+ result = utils.run(cmd, stdout_tee=output,
+ stderr_tee=error_output,
+ timeout=TELEMETRY_TIMEOUT_MINS*60)
+ exit_code = result.exit_status
+ except error.CmdError as e:
+ logging.debug('Error occurred executing.')
+ exit_code = e.result_obj.exit_status
+
+ stdout = output.getvalue()
+ stderr = error_output.getvalue()
+ logging.debug('Completed with exit code: %d.\nstdout:%s\n'
+ 'stderr:%s', exit_code, stdout, stderr)
+ return stdout, stderr, exit_code
def _run_telemetry(self, script, test_or_benchmark, *args):
@@ -365,33 +224,26 @@
telemetry_cmd = self._get_telemetry_cmd(script,
test_or_benchmark,
*args)
- logging.debug('Running Telemetry: %s', ' '.join(telemetry_cmd))
+ logging.debug('Running Telemetry: %s', telemetry_cmd)
- output = StringIO.StringIO()
- error_output = StringIO.StringIO()
- exit_code = 0
- try:
- result = utils.run(' '.join(telemetry_cmd), stdout_tee=output,
- stderr_tee=error_output,
- timeout=TELEMETRY_TIMEOUT_MINS*60)
- exit_code = result.exit_status
- except error.CmdError as e:
- # Telemetry returned a return code of not 0; for benchmarks this
- # can be due to a timeout on one of the pages of the page set and
- # we may still have data on the rest. For a test however this
- # indicates failure.
- logging.debug('Error occurred executing telemetry.')
- exit_code = e.result_obj.exit_status
-
- stdout = output.getvalue()
- stderr = error_output.getvalue()
- logging.debug('Telemetry completed with exit code: %d.\nstdout:%s\n'
- 'stderr:%s', exit_code, stdout, stderr)
+ stdout, stderr, exit_code = self._run_cmd(telemetry_cmd)
return TelemetryResult(exit_code=exit_code, stdout=stdout,
stderr=stderr)
+ def _run_scp(self, perf_results_dir):
+ """Runs telemetry on a dut.
+
+ @param perf_results_dir: The local directory that results are being
+ collected.
+ """
+ scp_cmd = self._scp_telemetry_results_cmd(perf_results_dir)
+ logging.debug('Retrieving Results: %s', scp_cmd)
+
+ self._run_cmd(scp_cmd)
+
+
def _run_test(self, script, test, *args):
"""Runs a telemetry test on a dut.
@@ -425,56 +277,6 @@
return self._run_test(TELEMETRY_RUN_TESTS_SCRIPT, test, *args)
- def run_cros_telemetry_test(self, test, *args):
- """Runs a cros specific telemetry test on a dut.
-
- @param test: Telemetry test we want to run.
- @param args: additional list of arguments to pass to the telemetry
- execution script.
-
- @returns A TelemetryResult instance with the results of this telemetry
- execution.
- """
- return self._run_test(TELEMETRY_RUN_CROS_TESTS_SCRIPT, test, *args)
-
-
- def run_gpu_test(self, test, *args):
- """Runs a gpu test on a dut.
-
- @param test: Gpu test we want to run.
- @param args: additional list of arguments to pass to the telemetry
- execution script.
-
- @returns A TelemetryResult instance with the results of this telemetry
- execution.
- """
- return self._run_test(TELEMETRY_RUN_GPU_TESTS_SCRIPT, test, *args)
-
-
- @staticmethod
- def _output_perf_value(perf_value_writer, perf_data):
- """Output perf values to result dir.
-
- The perf values will be output to the result dir and
- be subsequently uploaded to perf dashboard.
-
- @param perf_value_writer: Should be an instance with the function
- output_perf_value(), if None, no perf value
- will be written. Typically this will be the
- job object from an autotest test.
- @param perf_data: A list of perf values, each value is
- a dictionary that looks like
- {'graph':'GraphA', 'trace':'metric1',
- 'units':'secs', 'value':0.5}
- """
- for perf_value in perf_data:
- perf_value_writer.output_perf_value(
- description=perf_value['trace'],
- value=perf_value['value'],
- units=perf_value['units'],
- graph=perf_value['graph'])
-
-
def run_telemetry_benchmark(self, benchmark, perf_value_writer=None,
*args):
"""Runs a telemetry benchmark on a dut.
@@ -494,10 +296,6 @@
telemetry_script = os.path.join(self._telemetry_path,
TELEMETRY_RUN_BENCHMARKS_SCRIPT)
result = self._run_telemetry(telemetry_script, benchmark, *args)
- result.parse_benchmark_results()
-
- if perf_value_writer:
- self._output_perf_value(perf_value_writer, result.perf_data)
if result.status is WARNING_STATUS:
raise error.TestWarn('Telemetry Benchmark: %s'
@@ -505,5 +303,6 @@
if result.status is FAILED_STATUS:
raise error.TestFail('Telemetry Benchmark: %s'
' failed to run.' % benchmark)
-
+ if perf_value_writer:
+ self._run_scp(perf_value_writer.resultsdir)
return result