Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python2 |
| 2 | # |
| 3 | # Copyright 2017 The Chromium OS Authors. All rights reserved. |
| 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
| 6 | # |
| 7 | # pylint: disable=cros-logging-import |
| 8 | """Transforms skia benchmark results to ones that crosperf can understand.""" |
| 9 | |
| 10 | from __future__ import print_function |
| 11 | |
| 12 | import itertools |
| 13 | import logging |
| 14 | import json |
| 15 | import sys |
| 16 | |
| 17 | # Turn the logging level to INFO before importing other autotest |
| 18 | # code, to avoid having failed import logging messages confuse the |
| 19 | # test_droid user. |
| 20 | logging.basicConfig(level=logging.INFO) |
| 21 | |
| 22 | # All of the results we care about, by name. |
| 23 | # Each of these *must* end in _ns, _us, _ms, or _s, since all the metrics we |
| 24 | # collect (so far) are related to time, and we alter the results based on the |
| 25 | # suffix of these strings (so we don't have 0.000421ms per sample, for example) |
| 26 | _RESULT_RENAMES = { |
| 27 | 'memset32_100000_640_480_nonrendering': 'memset_time_ms', |
| 28 | 'path_equality_50%_640_480_nonrendering': 'path_equality_ns', |
| 29 | 'sort_qsort_backward_640_480_nonrendering': 'qsort_us' |
| 30 | } |
| 31 | |
| 32 | |
| 33 | def _GetFamiliarName(name): |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 34 | r = _RESULT_RENAMES[name] |
| 35 | return r if r else name |
Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 36 | |
| 37 | |
| 38 | def _IsResultInteresting(name): |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 39 | return name in _RESULT_RENAMES |
Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 40 | |
| 41 | |
| 42 | def _GetTimeMultiplier(label_name): |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 43 | """Given a time (in milliseconds), normalize it to what label_name expects. |
Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 44 | |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 45 | "What label_name expects" meaning "we pattern match against the last few |
| 46 | non-space chars in label_name." |
Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 47 | |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 48 | This expects the time unit to be separated from anything else by '_'. |
| 49 | """ |
| 50 | ms_mul = 1000 * 1000. |
| 51 | endings = [('_ns', 1), ('_us', 1000), |
| 52 | ('_ms', ms_mul), ('_s', ms_mul * 1000)] |
| 53 | for end, mul in endings: |
| 54 | if label_name.endswith(end): |
| 55 | return ms_mul / mul |
| 56 | raise ValueError('Unknown ending in "%s"; expecting one of %s' % |
| 57 | (label_name, [end for end, _ in endings])) |
Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 58 | |
| 59 | |
| 60 | def _GetTimeDenom(ms): |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 61 | """Given a list of times (in milliseconds), find a sane time unit for them. |
Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 62 | |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 63 | Returns the unit name, and `ms` normalized to that time unit. |
Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 64 | |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 65 | >>> _GetTimeDenom([1, 2, 3]) |
| 66 | ('ms', [1.0, 2.0, 3.0]) |
| 67 | >>> _GetTimeDenom([.1, .2, .3]) |
| 68 | ('us', [100.0, 200.0, 300.0]) |
| 69 | """ |
Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 70 | |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 71 | ms_mul = 1000 * 1000 |
| 72 | units = [('us', 1000), ('ms', ms_mul), ('s', ms_mul * 1000)] |
| 73 | for name, mul in reversed(units): |
| 74 | normalized = [float(t) * ms_mul / mul for t in ms] |
| 75 | average = sum(normalized) / len(normalized) |
| 76 | if all(n > 0.1 for n in normalized) and average >= 1: |
| 77 | return name, normalized |
Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 78 | |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 79 | normalized = [float(t) * ms_mul for t in ms] |
| 80 | return 'ns', normalized |
Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 81 | |
| 82 | |
| 83 | def _TransformBenchmarks(raw_benchmarks): |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 84 | # We get {"results": {"bench_name": Results}} |
| 85 | # where |
| 86 | # Results = {"config_name": {"samples": [float], etc.}} |
| 87 | # |
| 88 | # We want {"data": {"skia": [[BenchmarkData]]}, |
| 89 | # "platforms": ["platform1, ..."]} |
| 90 | # where |
| 91 | # BenchmarkData = {"bench_name": bench_samples[N], ..., "retval": 0} |
| 92 | # |
| 93 | # Note that retval is awkward -- crosperf's JSON reporter reports the result |
| 94 | # as a failure if it's not there. Everything else treats it like a |
| 95 | # statistic... |
| 96 | benchmarks = raw_benchmarks['results'] |
| 97 | results = [] |
| 98 | for bench_name, bench_result in benchmarks.iteritems(): |
| 99 | try: |
| 100 | for cfg_name, keyvals in bench_result.iteritems(): |
| 101 | # Some benchmarks won't have timing data (either it won't exist |
| 102 | # at all, or it'll be empty); skip them. |
| 103 | samples = keyvals.get('samples') |
| 104 | if not samples: |
| 105 | continue |
Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 106 | |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 107 | bench_name = '%s_%s' % (bench_name, cfg_name) |
| 108 | if not _IsResultInteresting(bench_name): |
| 109 | continue |
Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 110 | |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 111 | friendly_name = _GetFamiliarName(bench_name) |
| 112 | if len(results) < len(samples): |
| 113 | results.extend({ |
| 114 | 'retval': 0 |
| 115 | } for _ in xrange(len(samples) - len(results))) |
Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 116 | |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 117 | time_mul = _GetTimeMultiplier(friendly_name) |
| 118 | for sample, app in itertools.izip(samples, results): |
| 119 | assert friendly_name not in app |
| 120 | app[friendly_name] = sample * time_mul |
| 121 | except (KeyError, ValueError) as e: |
| 122 | logging.error('While converting "%s" (key: %s): %s', |
| 123 | bench_result, bench_name, e.message) |
| 124 | raise |
Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 125 | |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 126 | # Realistically, [results] should be multiple results, where each entry in |
| 127 | # the list is the result for a different label. Because we only deal with |
| 128 | # one label at the moment, we need to wrap it in its own list. |
| 129 | return results |
Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 130 | |
| 131 | |
| 132 | if __name__ == '__main__': |
| 133 | |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 134 | def _GetUserFile(argv): |
| 135 | if not argv or argv[0] == '-': |
| 136 | return sys.stdin |
| 137 | return open(argv[0]) |
Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 138 | |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 139 | def _Main(): |
| 140 | with _GetUserFile(sys.argv[1:]) as in_file: |
| 141 | obj = json.load(in_file) |
| 142 | output = _TransformBenchmarks(obj) |
| 143 | json.dump(output, sys.stdout) |
Zhizhou Yang | e598690 | 2017-08-10 17:37:53 -0700 | [diff] [blame] | 144 | |
Zhizhou Yang | 6236292 | 2017-08-30 16:04:36 -0700 | [diff] [blame] | 145 | _Main() |