Lalit Maganti | df50009 | 2020-07-16 23:56:24 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 2 | # Copyright (C) 2018 The Android Open Source Project |
| 3 | # |
| 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
| 7 | # |
| 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | # |
| 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
| 15 | |
Florian Mayer | cdc44b3 | 2020-06-29 17:18:29 +0100 | [diff] [blame] | 16 | from __future__ import absolute_import |
| 17 | from __future__ import division |
| 18 | from __future__ import print_function |
| 19 | |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 20 | import argparse |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 21 | import datetime |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 22 | import difflib |
| 23 | import glob |
Lalit Maganti | dfe69ca | 2018-10-30 12:18:23 +0000 | [diff] [blame] | 24 | import importlib |
Lalit Maganti | 46a0a53 | 2019-07-15 15:09:22 +0100 | [diff] [blame] | 25 | import json |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 26 | import os |
Lalit Maganti | 4be82e1 | 2019-12-05 15:33:09 +0000 | [diff] [blame] | 27 | import re |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 28 | import subprocess |
| 29 | import sys |
Lalit Maganti | dfe69ca | 2018-10-30 12:18:23 +0000 | [diff] [blame] | 30 | import tempfile |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 31 | |
Lalit Maganti | c8d1d2c | 2019-08-12 16:10:45 +0100 | [diff] [blame] | 32 | from itertools import chain |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 33 | from google.protobuf import reflection, text_format |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 34 | |
Lalit Maganti | 5eacf42 | 2020-06-30 13:51:23 +0100 | [diff] [blame] | 35 | from proto_utils import create_message_factory, serialize_textproto_trace, serialize_python_trace |
| 36 | |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 37 | ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
Florian Mayer | cee13fa | 2021-01-05 19:05:23 +0000 | [diff] [blame] | 38 | ENV = { |
| 39 | 'PERFETTO_BINARY_PATH': os.path.join(ROOT_DIR, 'test', 'data'), |
| 40 | } |
| 41 | if sys.platform.startswith('linux'): |
| 42 | ENV['PATH'] = os.path.join(ROOT_DIR, 'buildtools', 'linux64', 'clang', 'bin') |
| 43 | elif sys.platform.startswith('dawin'): |
| 44 | # Sadly, on macOS we need to check out the Android deps to get |
| 45 | # llvm symbolizer. |
| 46 | ENV['PATH'] = os.path.join(ROOT_DIR, 'buildtools', 'ndk', 'toolchains', |
| 47 | 'llvm', 'prebuilt', 'darwin-x86_64', 'bin') |
| 48 | elif sys.platform.startswith('win32'): |
| 49 | ENV['PATH'] = os.path.join(ROOT_DIR, 'buildtools', 'win', 'clang', 'bin') |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 50 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 51 | |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 52 | class Test(object): |
| 53 | |
Lalit Maganti | 0ec47c6 | 2020-02-12 16:24:08 +0000 | [diff] [blame] | 54 | def __init__(self, type, trace_path, query_path_or_metric, expected_path): |
| 55 | self.type = type |
Lalit Maganti | 1642e1d | 2020-02-12 16:23:38 +0000 | [diff] [blame] | 56 | self.trace_path = trace_path |
| 57 | self.query_path_or_metric = query_path_or_metric |
| 58 | self.expected_path = expected_path |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 59 | |
| 60 | |
Lalit Maganti | c8d1d2c | 2019-08-12 16:10:45 +0100 | [diff] [blame] | 61 | class PerfResult(object): |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 62 | |
Lalit Maganti | 0ec47c6 | 2020-02-12 16:24:08 +0000 | [diff] [blame] | 63 | def __init__(self, test_type, trace_path, query_path_or_metric, |
| 64 | ingest_time_ns_str, real_time_ns_str): |
| 65 | self.test_type = test_type |
Lalit Maganti | 1642e1d | 2020-02-12 16:23:38 +0000 | [diff] [blame] | 66 | self.trace_path = trace_path |
| 67 | self.query_path_or_metric = query_path_or_metric |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 68 | self.ingest_time_ns = int(ingest_time_ns_str) |
| 69 | self.real_time_ns = int(real_time_ns_str) |
Lalit Maganti | c8d1d2c | 2019-08-12 16:10:45 +0100 | [diff] [blame] | 70 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 71 | |
Lalit Maganti | 04a450a | 2019-07-03 16:04:19 +0100 | [diff] [blame] | 72 | class TestResult(object): |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 73 | |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 74 | def __init__(self, test_type, input_name, trace, cmd, expected, actual, |
Lalit Maganti | b6fd42a | 2020-03-31 14:34:36 +0100 | [diff] [blame] | 75 | stderr, exit_code): |
Lalit Maganti | 04a450a | 2019-07-03 16:04:19 +0100 | [diff] [blame] | 76 | self.test_type = test_type |
Lalit Maganti | 637589a | 2019-07-04 17:25:29 +0100 | [diff] [blame] | 77 | self.input_name = input_name |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 78 | self.trace = trace |
Lalit Maganti | 04a450a | 2019-07-03 16:04:19 +0100 | [diff] [blame] | 79 | self.cmd = cmd |
| 80 | self.expected = expected |
| 81 | self.actual = actual |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 82 | self.stderr = stderr |
Lalit Maganti | b6fd42a | 2020-03-31 14:34:36 +0100 | [diff] [blame] | 83 | self.exit_code = exit_code |
Lalit Maganti | 04a450a | 2019-07-03 16:04:19 +0100 | [diff] [blame] | 84 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 85 | |
Deepanjan Roy | 5d56d4c | 2021-02-10 10:53:36 -0500 | [diff] [blame] | 86 | def create_metrics_message_factory(metrics_descriptor_paths): |
| 87 | return create_message_factory(metrics_descriptor_paths, |
Lalit Maganti | a951d3d | 2020-07-16 23:57:00 +0100 | [diff] [blame] | 88 | 'perfetto.protos.TraceMetrics') |
| 89 | |
| 90 | |
| 91 | def write_diff(expected, actual): |
| 92 | expected_lines = expected.splitlines(True) |
| 93 | actual_lines = actual.splitlines(True) |
| 94 | diff = difflib.unified_diff( |
| 95 | expected_lines, actual_lines, fromfile='expected', tofile='actual') |
| 96 | for line in diff: |
| 97 | sys.stderr.write(line) |
| 98 | |
| 99 | |
Lalit Maganti | 04a450a | 2019-07-03 16:04:19 +0100 | [diff] [blame] | 100 | def run_metrics_test(trace_processor_path, gen_trace_path, metric, |
Lalit Maganti | 75869e6 | 2019-07-12 16:55:38 +0100 | [diff] [blame] | 101 | expected_path, perf_path, metrics_message_factory): |
Lalit Maganti | 46a0a53 | 2019-07-15 15:09:22 +0100 | [diff] [blame] | 102 | with open(expected_path, 'r') as expected_file: |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 103 | expected = expected_file.read() |
| 104 | |
Lalit Maganti | 25863f7 | 2019-08-28 22:14:25 +0100 | [diff] [blame] | 105 | json_output = os.path.basename(expected_path).endswith('.json.out') |
Lalit Maganti | 58e2492 | 2019-05-30 18:56:14 +0100 | [diff] [blame] | 106 | cmd = [ |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 107 | trace_processor_path, |
| 108 | '--run-metrics', |
| 109 | metric, |
| 110 | '--metrics-output=%s' % ('json' if json_output else 'binary'), |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 111 | '--perf-file', |
| 112 | perf_path, |
Primiano Tucci | 4656ba1 | 2021-01-05 19:55:15 +0100 | [diff] [blame] | 113 | gen_trace_path, |
Lalit Maganti | 58e2492 | 2019-05-30 18:56:14 +0100 | [diff] [blame] | 114 | ] |
Florian Mayer | cee13fa | 2021-01-05 19:05:23 +0000 | [diff] [blame] | 115 | tp = subprocess.Popen( |
| 116 | cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=ENV) |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 117 | (stdout, stderr) = tp.communicate() |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 118 | |
Lalit Maganti | 25863f7 | 2019-08-28 22:14:25 +0100 | [diff] [blame] | 119 | if json_output: |
| 120 | expected_text = expected |
Florian Mayer | cdc44b3 | 2020-06-29 17:18:29 +0100 | [diff] [blame] | 121 | actual_text = stdout.decode('utf8') |
Lalit Maganti | 25863f7 | 2019-08-28 22:14:25 +0100 | [diff] [blame] | 122 | else: |
| 123 | # Expected will be in text proto format and we'll need to parse it to a real |
| 124 | # proto. |
| 125 | expected_message = metrics_message_factory() |
| 126 | text_format.Merge(expected, expected_message) |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 127 | |
Lalit Maganti | 25863f7 | 2019-08-28 22:14:25 +0100 | [diff] [blame] | 128 | # Actual will be the raw bytes of the proto and we'll need to parse it into |
| 129 | # a message. |
| 130 | actual_message = metrics_message_factory() |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 131 | actual_message.ParseFromString(stdout) |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 132 | |
Lalit Maganti | 25863f7 | 2019-08-28 22:14:25 +0100 | [diff] [blame] | 133 | # Convert both back to text format. |
| 134 | expected_text = text_format.MessageToString(expected_message) |
| 135 | actual_text = text_format.MessageToString(actual_message) |
Lalit Maganti | 770167a | 2019-05-08 14:32:50 +0100 | [diff] [blame] | 136 | |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 137 | return TestResult('metric', metric, gen_trace_path, cmd, expected_text, |
Florian Mayer | cdc44b3 | 2020-06-29 17:18:29 +0100 | [diff] [blame] | 138 | actual_text, stderr.decode('utf8'), tp.returncode) |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 139 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 140 | |
| 141 | def run_query_test(trace_processor_path, gen_trace_path, query_path, |
| 142 | expected_path, perf_path): |
Lalit Maganti | 46a0a53 | 2019-07-15 15:09:22 +0100 | [diff] [blame] | 143 | with open(expected_path, 'r') as expected_file: |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 144 | expected = expected_file.read() |
| 145 | |
Lalit Maganti | 75869e6 | 2019-07-12 16:55:38 +0100 | [diff] [blame] | 146 | cmd = [ |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 147 | trace_processor_path, |
| 148 | '-q', |
| 149 | query_path, |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 150 | '--perf-file', |
| 151 | perf_path, |
Primiano Tucci | 4656ba1 | 2021-01-05 19:55:15 +0100 | [diff] [blame] | 152 | gen_trace_path, |
Lalit Maganti | 75869e6 | 2019-07-12 16:55:38 +0100 | [diff] [blame] | 153 | ] |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 154 | |
Florian Mayer | cee13fa | 2021-01-05 19:05:23 +0000 | [diff] [blame] | 155 | tp = subprocess.Popen( |
| 156 | cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=ENV) |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 157 | (stdout, stderr) = tp.communicate() |
Florian Mayer | cdc44b3 | 2020-06-29 17:18:29 +0100 | [diff] [blame] | 158 | return TestResult('query', query_path, gen_trace_path, cmd, expected, |
| 159 | stdout.decode('utf8'), stderr.decode('utf8'), tp.returncode) |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 160 | |
| 161 | |
Lalit Maganti | 0ec47c6 | 2020-02-12 16:24:08 +0000 | [diff] [blame] | 162 | def run_all_tests(trace_processor, trace_descriptor_path, |
Andrew Shulaev | 94d1bf2 | 2021-01-15 15:30:42 +0000 | [diff] [blame] | 163 | extension_descriptor_paths, metrics_message_factory, tests, |
| 164 | keep_input, rebase): |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 165 | perf_data = [] |
| 166 | test_failure = 0 |
Dan Elphick | e603bf7 | 2020-12-21 16:01:06 +0000 | [diff] [blame] | 167 | rebased = 0 |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 168 | for test in tests: |
Lalit Maganti | 1642e1d | 2020-02-12 16:23:38 +0000 | [diff] [blame] | 169 | trace_path = test.trace_path |
| 170 | expected_path = test.expected_path |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 171 | if not os.path.exists(trace_path): |
| 172 | sys.stderr.write('Trace file not found {}\n'.format(trace_path)) |
| 173 | test_failure += 1 |
| 174 | continue |
| 175 | elif not os.path.exists(expected_path): |
| 176 | sys.stderr.write('Expected file not found {}\n'.format(expected_path)) |
| 177 | test_failure += 1 |
| 178 | continue |
| 179 | |
Lalit Maganti | df50009 | 2020-07-16 23:56:24 +0100 | [diff] [blame] | 180 | is_generated_trace = trace_path.endswith('.py') or trace_path.endswith( |
| 181 | '.textproto') |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 182 | if trace_path.endswith('.py'): |
Eric Seckler | fbd9aed | 2020-03-10 18:07:38 +0000 | [diff] [blame] | 183 | gen_trace_file = tempfile.NamedTemporaryFile(delete=False) |
Lalit Maganti | 5eacf42 | 2020-06-30 13:51:23 +0100 | [diff] [blame] | 184 | serialize_python_trace(trace_descriptor_path, trace_path, gen_trace_file) |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 185 | gen_trace_path = os.path.realpath(gen_trace_file.name) |
| 186 | elif trace_path.endswith('.textproto'): |
Eric Seckler | fbd9aed | 2020-03-10 18:07:38 +0000 | [diff] [blame] | 187 | gen_trace_file = tempfile.NamedTemporaryFile(delete=False) |
Andrew Shulaev | 94d1bf2 | 2021-01-15 15:30:42 +0000 | [diff] [blame] | 188 | serialize_textproto_trace(trace_descriptor_path, |
| 189 | extension_descriptor_paths, trace_path, |
Lalit Maganti | 5eacf42 | 2020-06-30 13:51:23 +0100 | [diff] [blame] | 190 | gen_trace_file) |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 191 | gen_trace_path = os.path.realpath(gen_trace_file.name) |
| 192 | else: |
| 193 | gen_trace_file = None |
| 194 | gen_trace_path = trace_path |
| 195 | |
Primiano Tucci | 4656ba1 | 2021-01-05 19:55:15 +0100 | [diff] [blame] | 196 | # We can't use delete=True here. When using that on Windwows, the resulting |
| 197 | # file is opened in exclusive mode (in turn that's a subtle side-effect of |
| 198 | # the underlying CreateFile(FILE_ATTRIBUTE_TEMPORARY)) and TP fails to open |
| 199 | # the passed path. |
| 200 | tmp_perf_file = tempfile.NamedTemporaryFile(delete=False) |
| 201 | sys.stderr.write('[ RUN ] {} {}\n'.format( |
| 202 | os.path.basename(test.query_path_or_metric), |
| 203 | os.path.basename(trace_path))) |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 204 | |
Primiano Tucci | 4656ba1 | 2021-01-05 19:55:15 +0100 | [diff] [blame] | 205 | tmp_perf_path = tmp_perf_file.name |
| 206 | if test.type == 'queries': |
| 207 | query_path = test.query_path_or_metric |
Lalit Maganti | 1642e1d | 2020-02-12 16:23:38 +0000 | [diff] [blame] | 208 | |
Primiano Tucci | 4656ba1 | 2021-01-05 19:55:15 +0100 | [diff] [blame] | 209 | if not os.path.exists(test.query_path_or_metric): |
| 210 | print('Query file not found {}'.format(query_path)) |
| 211 | test_failure += 1 |
| 212 | continue |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 213 | |
Primiano Tucci | 4656ba1 | 2021-01-05 19:55:15 +0100 | [diff] [blame] | 214 | result = run_query_test(trace_processor, gen_trace_path, query_path, |
| 215 | expected_path, tmp_perf_path) |
| 216 | elif test.type == 'metrics': |
| 217 | result = run_metrics_test(trace_processor, gen_trace_path, |
| 218 | test.query_path_or_metric, expected_path, |
| 219 | tmp_perf_path, metrics_message_factory) |
| 220 | else: |
| 221 | assert False |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 222 | |
Primiano Tucci | 4656ba1 | 2021-01-05 19:55:15 +0100 | [diff] [blame] | 223 | perf_lines = [line.decode('utf8') for line in tmp_perf_file.readlines()] |
| 224 | tmp_perf_file.close() |
| 225 | os.remove(tmp_perf_file.name) |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 226 | |
| 227 | if gen_trace_file: |
Eric Seckler | fbd9aed | 2020-03-10 18:07:38 +0000 | [diff] [blame] | 228 | if keep_input: |
Lalit Maganti | df50009 | 2020-07-16 23:56:24 +0100 | [diff] [blame] | 229 | sys.stderr.write( |
| 230 | "Saving generated input trace: {}\n".format(gen_trace_path)) |
Eric Seckler | fbd9aed | 2020-03-10 18:07:38 +0000 | [diff] [blame] | 231 | else: |
| 232 | gen_trace_file.close() |
| 233 | os.remove(gen_trace_path) |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 234 | |
Lalit Maganti | df50009 | 2020-07-16 23:56:24 +0100 | [diff] [blame] | 235 | def write_cmdlines(): |
| 236 | if is_generated_trace: |
| 237 | sys.stderr.write( |
| 238 | 'Command to generate trace:\n' |
| 239 | 'tools/serialize_test_trace.py --descriptor {} {} > {}\n'.format( |
| 240 | os.path.relpath(trace_descriptor_path, ROOT_DIR), |
Lalit Maganti | a951d3d | 2020-07-16 23:57:00 +0100 | [diff] [blame] | 241 | os.path.relpath(trace_path, ROOT_DIR), |
Lalit Maganti | df50009 | 2020-07-16 23:56:24 +0100 | [diff] [blame] | 242 | os.path.relpath(gen_trace_path, ROOT_DIR))) |
| 243 | sys.stderr.write('Command line:\n{}\n'.format(' '.join(result.cmd))) |
| 244 | |
Primiano Tucci | 4656ba1 | 2021-01-05 19:55:15 +0100 | [diff] [blame] | 245 | contents_equal = ( |
| 246 | result.expected.replace('\r\n', |
| 247 | '\n') == result.actual.replace('\r\n', '\n')) |
| 248 | if result.exit_code != 0 or not contents_equal: |
Lalit Maganti | b6fd42a | 2020-03-31 14:34:36 +0100 | [diff] [blame] | 249 | sys.stderr.write(result.stderr) |
| 250 | |
| 251 | if result.exit_code == 0: |
| 252 | sys.stderr.write( |
| 253 | 'Expected did not match actual for trace {} and {} {}\n'.format( |
| 254 | trace_path, result.test_type, result.input_name)) |
| 255 | sys.stderr.write('Expected file: {}\n'.format(expected_path)) |
Lalit Maganti | df50009 | 2020-07-16 23:56:24 +0100 | [diff] [blame] | 256 | write_cmdlines() |
Lalit Maganti | b6fd42a | 2020-03-31 14:34:36 +0100 | [diff] [blame] | 257 | write_diff(result.expected, result.actual) |
| 258 | else: |
Lalit Maganti | df50009 | 2020-07-16 23:56:24 +0100 | [diff] [blame] | 259 | write_cmdlines() |
Lalit Maganti | b6fd42a | 2020-03-31 14:34:36 +0100 | [diff] [blame] | 260 | |
| 261 | sys.stderr.write('[ FAIL ] {} {}\n'.format( |
| 262 | os.path.basename(test.query_path_or_metric), |
| 263 | os.path.basename(trace_path))) |
| 264 | |
Dan Elphick | e603bf7 | 2020-12-21 16:01:06 +0000 | [diff] [blame] | 265 | if rebase: |
| 266 | if result.exit_code == 0: |
| 267 | sys.stderr.write('Rebasing {}\n'.format(expected_path)) |
| 268 | with open(expected_path, 'w') as f: |
| 269 | f.write(result.actual) |
| 270 | rebase += 1 |
| 271 | else: |
| 272 | sys.stderr.write( |
| 273 | 'Rebase failed for {} as query failed\n'.format(expected_path)) |
| 274 | |
Lalit Maganti | b6fd42a | 2020-03-31 14:34:36 +0100 | [diff] [blame] | 275 | test_failure += 1 |
| 276 | else: |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 277 | assert len(perf_lines) == 1 |
| 278 | perf_numbers = perf_lines[0].split(',') |
| 279 | |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 280 | assert len(perf_numbers) == 2 |
Lalit Maganti | 0ec47c6 | 2020-02-12 16:24:08 +0000 | [diff] [blame] | 281 | perf_result = PerfResult(test.type, trace_path, test.query_path_or_metric, |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 282 | perf_numbers[0], perf_numbers[1]) |
| 283 | perf_data.append(perf_result) |
| 284 | |
| 285 | sys.stderr.write( |
| 286 | '[ OK ] {} {} (ingest: {} ms, query: {} ms)\n'.format( |
Lalit Maganti | 1642e1d | 2020-02-12 16:23:38 +0000 | [diff] [blame] | 287 | os.path.basename(test.query_path_or_metric), |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 288 | os.path.basename(trace_path), |
| 289 | perf_result.ingest_time_ns / 1000000, |
| 290 | perf_result.real_time_ns / 1000000)) |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 291 | |
Dan Elphick | e603bf7 | 2020-12-21 16:01:06 +0000 | [diff] [blame] | 292 | return test_failure, perf_data, rebased |
Lalit Maganti | dfe69ca | 2018-10-30 12:18:23 +0000 | [diff] [blame] | 293 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 294 | |
Lalit Maganti | a951d3d | 2020-07-16 23:57:00 +0100 | [diff] [blame] | 295 | def read_all_tests_from_index(index_path, query_metric_pattern, trace_pattern): |
Lalit Maganti | 244200e | 2020-07-15 11:39:07 +0100 | [diff] [blame] | 296 | index_dir = os.path.dirname(index_path) |
Lalit Maganti | 0ec47c6 | 2020-02-12 16:24:08 +0000 | [diff] [blame] | 297 | |
Lalit Maganti | 244200e | 2020-07-15 11:39:07 +0100 | [diff] [blame] | 298 | with open(index_path, 'r') as index_file: |
| 299 | index_lines = index_file.readlines() |
Lalit Maganti | 0ec47c6 | 2020-02-12 16:24:08 +0000 | [diff] [blame] | 300 | |
| 301 | tests = [] |
| 302 | for line in index_lines: |
| 303 | stripped = line.strip() |
| 304 | if stripped.startswith('#'): |
| 305 | continue |
| 306 | elif not stripped: |
| 307 | continue |
| 308 | |
| 309 | [trace_fname, query_fname_or_metric, expected_fname] = stripped.split(' ') |
| 310 | if not query_metric_pattern.match(os.path.basename(query_fname_or_metric)): |
| 311 | continue |
| 312 | |
| 313 | if not trace_pattern.match(os.path.basename(trace_fname)): |
| 314 | continue |
| 315 | |
| 316 | trace_path = os.path.abspath(os.path.join(index_dir, trace_fname)) |
| 317 | expected_path = os.path.abspath(os.path.join(index_dir, expected_fname)) |
| 318 | |
Lalit Maganti | a951d3d | 2020-07-16 23:57:00 +0100 | [diff] [blame] | 319 | if query_fname_or_metric.endswith('.sql'): |
| 320 | test_type = 'queries' |
Lalit Maganti | 0ec47c6 | 2020-02-12 16:24:08 +0000 | [diff] [blame] | 321 | query_path_or_metric = os.path.abspath( |
| 322 | os.path.join(index_dir, query_fname_or_metric)) |
Lalit Maganti | a951d3d | 2020-07-16 23:57:00 +0100 | [diff] [blame] | 323 | else: |
| 324 | test_type = 'metrics' |
| 325 | query_path_or_metric = query_fname_or_metric |
Lalit Maganti | 0ec47c6 | 2020-02-12 16:24:08 +0000 | [diff] [blame] | 326 | |
| 327 | tests.append( |
| 328 | Test(test_type, trace_path, query_path_or_metric, expected_path)) |
Lalit Maganti | 244200e | 2020-07-15 11:39:07 +0100 | [diff] [blame] | 329 | return tests |
Lalit Maganti | 0ec47c6 | 2020-02-12 16:24:08 +0000 | [diff] [blame] | 330 | |
Lalit Maganti | 244200e | 2020-07-15 11:39:07 +0100 | [diff] [blame] | 331 | |
Lalit Maganti | a951d3d | 2020-07-16 23:57:00 +0100 | [diff] [blame] | 332 | def read_all_tests(query_metric_pattern, trace_pattern): |
| 333 | include_index_dir = os.path.join(ROOT_DIR, 'test', 'trace_processor') |
| 334 | include_index = os.path.join(include_index_dir, 'include_index') |
Lalit Maganti | 244200e | 2020-07-15 11:39:07 +0100 | [diff] [blame] | 335 | tests = [] |
| 336 | with open(include_index, 'r') as include_file: |
| 337 | for index_relpath in include_file.readlines(): |
| 338 | index_path = os.path.join(include_index_dir, index_relpath.strip()) |
| 339 | tests.extend( |
Lalit Maganti | a951d3d | 2020-07-16 23:57:00 +0100 | [diff] [blame] | 340 | read_all_tests_from_index(index_path, query_metric_pattern, |
Lalit Maganti | 244200e | 2020-07-15 11:39:07 +0100 | [diff] [blame] | 341 | trace_pattern)) |
Lalit Maganti | 0ec47c6 | 2020-02-12 16:24:08 +0000 | [diff] [blame] | 342 | return tests |
| 343 | |
| 344 | |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 345 | def main(): |
| 346 | parser = argparse.ArgumentParser() |
Lalit Maganti | 0ec47c6 | 2020-02-12 16:24:08 +0000 | [diff] [blame] | 347 | parser.add_argument('--test-type', type=str, default='all') |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 348 | parser.add_argument('--trace-descriptor', type=str) |
| 349 | parser.add_argument('--metrics-descriptor', type=str) |
Lalit Maganti | 75869e6 | 2019-07-12 16:55:38 +0100 | [diff] [blame] | 350 | parser.add_argument('--perf-file', type=str) |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 351 | parser.add_argument( |
Lalit Maganti | 4be82e1 | 2019-12-05 15:33:09 +0000 | [diff] [blame] | 352 | '--query-metric-filter', |
| 353 | default='.*', |
| 354 | type=str, |
| 355 | help= |
| 356 | 'Filter the name of query files or metrics to diff test (regex syntax)') |
| 357 | parser.add_argument( |
| 358 | '--trace-filter', |
| 359 | default='.*', |
| 360 | type=str, |
| 361 | help='Filter the name of trace files to diff test (regex syntax)') |
| 362 | parser.add_argument( |
Eric Seckler | fbd9aed | 2020-03-10 18:07:38 +0000 | [diff] [blame] | 363 | '--keep-input', |
| 364 | action='store_true', |
| 365 | help='Save the (generated) input pb file for debugging') |
| 366 | parser.add_argument( |
Dan Elphick | e603bf7 | 2020-12-21 16:01:06 +0000 | [diff] [blame] | 367 | '--rebase', |
| 368 | action='store_true', |
| 369 | help='Update the expected output file with the actual result') |
| 370 | parser.add_argument( |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 371 | 'trace_processor', type=str, help='location of trace processor binary') |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 372 | args = parser.parse_args() |
| 373 | |
Lalit Maganti | 0ec47c6 | 2020-02-12 16:24:08 +0000 | [diff] [blame] | 374 | query_metric_pattern = re.compile(args.query_metric_filter) |
| 375 | trace_pattern = re.compile(args.trace_filter) |
| 376 | |
Lalit Maganti | a951d3d | 2020-07-16 23:57:00 +0100 | [diff] [blame] | 377 | tests = read_all_tests(query_metric_pattern, trace_pattern) |
Lalit Maganti | 0ec47c6 | 2020-02-12 16:24:08 +0000 | [diff] [blame] | 378 | sys.stderr.write('[==========] Running {} tests.\n'.format(len(tests))) |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 379 | |
Andrew Shulaev | 94d1bf2 | 2021-01-15 15:30:42 +0000 | [diff] [blame] | 380 | out_path = os.path.dirname(args.trace_processor) |
Lalit Maganti | dfe69ca | 2018-10-30 12:18:23 +0000 | [diff] [blame] | 381 | if args.trace_descriptor: |
| 382 | trace_descriptor_path = args.trace_descriptor |
| 383 | else: |
Lalit Maganti | e0986f3 | 2020-09-17 15:35:47 +0100 | [diff] [blame] | 384 | def find_trace_descriptor(parent): |
| 385 | trace_protos_path = os.path.join(parent, 'gen', 'protos', 'perfetto', |
| 386 | 'trace') |
| 387 | return os.path.join(trace_protos_path, 'trace.descriptor') |
| 388 | |
| 389 | trace_descriptor_path = find_trace_descriptor(out_path) |
| 390 | if not os.path.exists(trace_descriptor_path): |
| 391 | trace_descriptor_path = find_trace_descriptor( |
| 392 | os.path.join(out_path, 'gcc_like_host')) |
Lalit Maganti | dfe69ca | 2018-10-30 12:18:23 +0000 | [diff] [blame] | 393 | |
Deepanjan Roy | 5d56d4c | 2021-02-10 10:53:36 -0500 | [diff] [blame] | 394 | |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 395 | if args.metrics_descriptor: |
Deepanjan Roy | 5d56d4c | 2021-02-10 10:53:36 -0500 | [diff] [blame] | 396 | metrics_descriptor_paths = [args.metrics_descriptor] |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 397 | else: |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 398 | metrics_protos_path = os.path.join(out_path, 'gen', 'protos', 'perfetto', |
| 399 | 'metrics') |
Deepanjan Roy | 5d56d4c | 2021-02-10 10:53:36 -0500 | [diff] [blame] | 400 | metrics_descriptor_paths = [ |
| 401 | os.path.join(metrics_protos_path, 'metrics.descriptor'), |
| 402 | os.path.join(metrics_protos_path, 'chrome', |
| 403 | 'all_chrome_metrics.descriptor') |
| 404 | ] |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 405 | |
Andrew Shulaev | 94d1bf2 | 2021-01-15 15:30:42 +0000 | [diff] [blame] | 406 | chrome_extensions = os.path.join(out_path, 'gen', 'protos', 'third_party', |
| 407 | 'chromium', 'chrome_track_event.descriptor') |
| 408 | test_extensions = os.path.join(out_path, 'gen', 'protos', 'perfetto', 'trace', |
| 409 | 'test_extensions.descriptor') |
| 410 | |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 411 | metrics_message_factory = create_metrics_message_factory( |
Deepanjan Roy | 5d56d4c | 2021-02-10 10:53:36 -0500 | [diff] [blame] | 412 | metrics_descriptor_paths) |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 413 | |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 414 | test_run_start = datetime.datetime.now() |
Andrew Shulaev | 94d1bf2 | 2021-01-15 15:30:42 +0000 | [diff] [blame] | 415 | test_failure, perf_data, rebased = run_all_tests( |
| 416 | args.trace_processor, trace_descriptor_path, |
| 417 | [chrome_extensions, test_extensions], metrics_message_factory, tests, |
| 418 | args.keep_input, args.rebase) |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 419 | test_run_end = datetime.datetime.now() |
Hector Dearman | f1f51a3 | 2018-10-22 11:09:41 +0100 | [diff] [blame] | 420 | |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 421 | sys.stderr.write('[==========] {} tests ran. ({} ms total)\n'.format( |
| 422 | len(tests), int((test_run_end - test_run_start).total_seconds() * 1000))) |
| 423 | sys.stderr.write('[ PASSED ] {} tests.\n'.format(len(tests) - test_failure)) |
Dan Elphick | e603bf7 | 2020-12-21 16:01:06 +0000 | [diff] [blame] | 424 | if args.rebase: |
| 425 | sys.stderr.write('{} tests rebased.\n'.format(rebased)) |
Lalit Maganti | 04a450a | 2019-07-03 16:04:19 +0100 | [diff] [blame] | 426 | |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 427 | if test_failure == 0: |
Lalit Maganti | 75869e6 | 2019-07-12 16:55:38 +0100 | [diff] [blame] | 428 | if args.perf_file: |
Lalit Maganti | 0ec47c6 | 2020-02-12 16:24:08 +0000 | [diff] [blame] | 429 | test_dir = os.path.join(ROOT_DIR, 'test') |
| 430 | trace_processor_dir = os.path.join(test_dir, 'trace_processor') |
| 431 | |
| 432 | metrics = [] |
Lalit Maganti | df50009 | 2020-07-16 23:56:24 +0100 | [diff] [blame] | 433 | sorted_data = sorted( |
| 434 | perf_data, |
| 435 | key=lambda x: (x.test_type, x.trace_path, x.query_path_or_metric)) |
| 436 | for perf_args in sorted_data: |
Lalit Maganti | 0ec47c6 | 2020-02-12 16:24:08 +0000 | [diff] [blame] | 437 | trace_short_path = os.path.relpath(perf_args.trace_path, test_dir) |
| 438 | |
| 439 | query_short_path_or_metric = perf_args.query_path_or_metric |
| 440 | if perf_args.test_type == 'queries': |
| 441 | query_short_path_or_metric = os.path.relpath( |
| 442 | perf_args.query_path_or_metric, trace_processor_dir) |
| 443 | |
| 444 | metrics.append({ |
| 445 | 'metric': 'tp_perf_test_ingest_time', |
| 446 | 'value': float(perf_args.ingest_time_ns) / 1.0e9, |
| 447 | 'unit': 's', |
| 448 | 'tags': { |
| 449 | 'test_name': |
| 450 | '{}-{}'.format(trace_short_path, |
| 451 | query_short_path_or_metric), |
| 452 | 'test_type': |
| 453 | perf_args.test_type, |
| 454 | }, |
| 455 | 'labels': {}, |
| 456 | }) |
| 457 | metrics.append({ |
| 458 | 'metric': 'perf_test_real_time', |
| 459 | 'value': float(perf_args.real_time_ns) / 1.0e9, |
| 460 | 'unit': 's', |
| 461 | 'tags': { |
| 462 | 'test_name': |
| 463 | '{}-{}'.format( |
| 464 | os.path.relpath(perf_args.trace_path, test_dir), |
| 465 | query_short_path_or_metric), |
| 466 | 'test_type': |
| 467 | perf_args.test_type, |
| 468 | }, |
| 469 | 'labels': {}, |
| 470 | }) |
| 471 | |
| 472 | output_data = {'metrics': metrics} |
Lalit Maganti | 75869e6 | 2019-07-12 16:55:38 +0100 | [diff] [blame] | 473 | with open(args.perf_file, 'w+') as perf_file: |
Lalit Maganti | 46a0a53 | 2019-07-15 15:09:22 +0100 | [diff] [blame] | 474 | perf_file.write(json.dumps(output_data, indent=2)) |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 475 | return 0 |
| 476 | else: |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 477 | sys.stderr.write('[ FAILED ] {} tests.\n'.format(test_failure)) |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 478 | return 1 |
| 479 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 480 | |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 481 | if __name__ == '__main__': |
| 482 | sys.exit(main()) |