Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright (C) 2018 The Android Open Source Project |
| 3 | # |
| 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
| 7 | # |
| 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | # |
| 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
| 15 | |
| 16 | import argparse |
| 17 | import difflib |
| 18 | import glob |
Lalit Maganti | dfe69ca | 2018-10-30 12:18:23 +0000 | [diff] [blame] | 19 | import importlib |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 20 | import os |
| 21 | import subprocess |
| 22 | import sys |
Lalit Maganti | dfe69ca | 2018-10-30 12:18:23 +0000 | [diff] [blame] | 23 | import tempfile |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 24 | |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 25 | from google.protobuf import descriptor, descriptor_pb2, message_factory |
| 26 | from google.protobuf import reflection, text_format |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 27 | |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 28 | ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 29 | |
| 30 | def create_metrics_message_factory(metrics_descriptor_path): |
| 31 | with open(metrics_descriptor_path, "r") as metrics_descriptor_file: |
| 32 | metrics_descriptor_content = metrics_descriptor_file.read() |
| 33 | |
| 34 | file_desc_set_pb2 = descriptor_pb2.FileDescriptorSet() |
| 35 | file_desc_set_pb2.MergeFromString(metrics_descriptor_content) |
| 36 | |
| 37 | desc_by_path = {} |
| 38 | for f_desc_pb2 in file_desc_set_pb2.file: |
| 39 | f_desc_pb2_encode = f_desc_pb2.SerializeToString() |
| 40 | f_desc = descriptor.FileDescriptor( |
| 41 | name=f_desc_pb2.name, |
| 42 | package=f_desc_pb2.package, |
| 43 | serialized_pb=f_desc_pb2_encode) |
| 44 | |
| 45 | for desc in f_desc.message_types_by_name.values(): |
| 46 | desc_by_path[desc.full_name] = desc |
| 47 | |
| 48 | return message_factory.MessageFactory().GetPrototype( |
| 49 | desc_by_path["perfetto.protos.TraceMetrics"]) |
| 50 | |
| 51 | def write_diff(expected, actual): |
| 52 | expected_lines = expected.splitlines(True) |
| 53 | actual_lines = actual.splitlines(True) |
| 54 | diff = difflib.unified_diff(expected_lines, actual_lines, |
| 55 | fromfile="expected", tofile="actual") |
| 56 | for line in diff: |
| 57 | sys.stderr.write(line) |
| 58 | |
| 59 | def run_metrics_test(trace_processor_path, gen_trace_path, trace_path, metric, |
| 60 | expected_path, trace_descriptor_path, |
| 61 | metrics_message_factory): |
| 62 | with open(expected_path, "r") as expected_file: |
| 63 | expected = expected_file.read() |
| 64 | |
| 65 | cmd = [trace_processor_path, '--run-metrics', metric, gen_trace_path] |
| 66 | actual = subprocess.check_output(cmd) |
| 67 | |
| 68 | # Expected will be in text proto format and we'll need to parse it to a real |
| 69 | # proto. |
| 70 | expected_message = metrics_message_factory() |
| 71 | text_format.Merge(expected, expected_message) |
| 72 | |
| 73 | # Actual will be the raw bytes of the proto and we'll need to parse it into |
| 74 | # a message. |
| 75 | actual_message = metrics_message_factory() |
| 76 | actual_message.ParseFromString(actual) |
| 77 | |
Lalit Maganti | 770167a | 2019-05-08 14:32:50 +0100 | [diff] [blame] | 78 | # Convert both back to text format. |
| 79 | expected_text = text_format.MessageToString(expected_message) |
| 80 | actual_text = text_format.MessageToString(actual_message) |
| 81 | |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 82 | # Do an equality check of the python messages |
Lalit Maganti | 770167a | 2019-05-08 14:32:50 +0100 | [diff] [blame] | 83 | if expected_text == actual_text: |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 84 | return True |
| 85 | |
| 86 | # Write some metadata about the traces. |
| 87 | sys.stderr.write( |
| 88 | "Expected did not match actual for trace {} and metric {}\n" |
| 89 | .format(trace_path, metric)) |
| 90 | sys.stderr.write("Expected file: {}\n".format(expected_path)) |
| 91 | sys.stderr.write("Command line: {}\n".format(' '.join(cmd))) |
| 92 | |
Lalit Maganti | 770167a | 2019-05-08 14:32:50 +0100 | [diff] [blame] | 93 | # Print the diff between the two. |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 94 | write_diff(expected_text, actual_text) |
| 95 | |
| 96 | return False |
| 97 | |
| 98 | def run_query_test(trace_processor_path, gen_trace_path, trace_path, |
| 99 | query_path, expected_path, trace_descriptor_path): |
| 100 | with open(expected_path, "r") as expected_file: |
| 101 | expected = expected_file.read() |
| 102 | |
| 103 | cmd = [trace_processor_path, '-q', query_path, gen_trace_path] |
| 104 | actual = subprocess.check_output(cmd).decode("utf-8") |
| 105 | |
| 106 | if expected == actual: |
| 107 | return True |
| 108 | |
| 109 | # Write some metadata. |
| 110 | sys.stderr.write( |
| 111 | "Expected did not match actual for trace {} and query {}\n" |
| 112 | .format(trace_path, query_path)) |
| 113 | sys.stderr.write("Expected file: {}\n".format(expected_path)) |
| 114 | sys.stderr.write("Command line: {}\n".format(' '.join(cmd))) |
| 115 | |
| 116 | # Write the diff of the two files. |
| 117 | write_diff(expected, actual) |
| 118 | |
| 119 | return False |
Lalit Maganti | dfe69ca | 2018-10-30 12:18:23 +0000 | [diff] [blame] | 120 | |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 121 | def main(): |
| 122 | parser = argparse.ArgumentParser() |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 123 | parser.add_argument("--test-type", type=str, default="queries") |
| 124 | parser.add_argument('--trace-descriptor', type=str) |
| 125 | parser.add_argument('--metrics-descriptor', type=str) |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 126 | parser.add_argument('trace_processor', type=str, |
| 127 | help='location of trace processor binary') |
| 128 | args = parser.parse_args() |
| 129 | |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 130 | if args.test_type == 'queries': |
| 131 | index = os.path.join(ROOT_DIR, "test", "trace_processor", "index") |
| 132 | elif args.test_type == 'metrics': |
| 133 | index = os.path.join(ROOT_DIR, "test", "metrics", "index") |
| 134 | else: |
| 135 | print("Unknown test type {}. Supported: queries, metircs".format( |
| 136 | args.test_type)) |
| 137 | return 1 |
| 138 | |
| 139 | with open(index, 'r') as file: |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 140 | index_lines = file.readlines() |
| 141 | |
Lalit Maganti | dfe69ca | 2018-10-30 12:18:23 +0000 | [diff] [blame] | 142 | if args.trace_descriptor: |
| 143 | trace_descriptor_path = args.trace_descriptor |
| 144 | else: |
| 145 | out_path = os.path.dirname(args.trace_processor) |
Lalit Maganti | fc7b058 | 2019-05-22 23:52:08 +0100 | [diff] [blame] | 146 | trace_protos_path = os.path.join( |
| 147 | out_path, "gcc_like_host", "gen", "protos", "trace") |
Lalit Maganti | dfe69ca | 2018-10-30 12:18:23 +0000 | [diff] [blame] | 148 | trace_descriptor_path = os.path.join(trace_protos_path, "trace.descriptor") |
| 149 | |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 150 | if args.metrics_descriptor: |
| 151 | metrics_descriptor_path = args.metrics_descriptor |
| 152 | else: |
| 153 | out_path = os.path.dirname(args.trace_processor) |
Lalit Maganti | fc7b058 | 2019-05-22 23:52:08 +0100 | [diff] [blame] | 154 | metrics_protos_path = os.path.join( |
| 155 | out_path, "gcc_like_host", "gen", "protos", "perfetto", "metrics") |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 156 | metrics_descriptor_path = os.path.join(metrics_protos_path, |
| 157 | "metrics.descriptor") |
| 158 | |
| 159 | metrics_message_factory = create_metrics_message_factory( |
| 160 | metrics_descriptor_path) |
| 161 | |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 162 | test_failure = 0 |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 163 | index_dir = os.path.dirname(index) |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 164 | for line in index_lines: |
Lalit Maganti | cc67422 | 2019-01-23 17:54:00 +0000 | [diff] [blame] | 165 | stripped = line.strip() |
| 166 | if stripped.startswith('#'): |
| 167 | continue |
Lalit Maganti | 6cf6edb | 2019-02-07 11:09:47 +0000 | [diff] [blame] | 168 | elif not stripped: |
| 169 | continue |
Lalit Maganti | cc67422 | 2019-01-23 17:54:00 +0000 | [diff] [blame] | 170 | |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 171 | [trace_fname, query_fname_or_metric, expected_fname] = stripped.split(' ') |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 172 | |
| 173 | trace_path = os.path.abspath(os.path.join(index_dir, trace_fname)) |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 174 | expected_path = os.path.abspath(os.path.join(index_dir, expected_fname)) |
| 175 | if not os.path.exists(trace_path): |
| 176 | print("Trace file not found {}".format(trace_path)) |
| 177 | return 1 |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 178 | elif not os.path.exists(expected_path): |
| 179 | print("Expected file not found {}".format(expected_path)) |
| 180 | return 1 |
| 181 | |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 182 | if trace_path.endswith('.py'): |
| 183 | gen_trace_file = tempfile.NamedTemporaryFile() |
| 184 | python_cmd = ["python", trace_path, trace_descriptor_path] |
| 185 | subprocess.check_call(python_cmd, stdout=gen_trace_file) |
| 186 | gen_trace_path = os.path.realpath(gen_trace_file.name) |
Lalit Maganti | dfe69ca | 2018-10-30 12:18:23 +0000 | [diff] [blame] | 187 | else: |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 188 | gen_trace_file = None |
| 189 | gen_trace_path = trace_path |
Hector Dearman | f1f51a3 | 2018-10-22 11:09:41 +0100 | [diff] [blame] | 190 | |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 191 | if args.test_type == 'queries': |
| 192 | query_path = os.path.abspath( |
| 193 | os.path.join(index_dir, query_fname_or_metric)) |
| 194 | if not os.path.exists(query_path): |
| 195 | print("Query file not found {}".format(query_path)) |
| 196 | return 1 |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 197 | |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 198 | success = run_query_test(args.trace_processor, gen_trace_path, |
| 199 | trace_path, query_path, expected_path, |
| 200 | trace_descriptor_path) |
| 201 | elif args.test_type == 'metrics': |
| 202 | success = run_metrics_test(args.trace_processor, gen_trace_path, |
| 203 | trace_path, query_fname_or_metric, |
| 204 | expected_path, trace_descriptor_path, |
| 205 | metrics_message_factory) |
| 206 | else: |
| 207 | assert False |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 208 | |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 209 | if gen_trace_file: |
| 210 | gen_trace_file.close() |
| 211 | |
| 212 | if not success: |
| 213 | test_failure += 1 |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 214 | |
| 215 | if test_failure == 0: |
| 216 | print("All tests passed successfully") |
| 217 | return 0 |
| 218 | else: |
| 219 | print("Total failures: {}".format(test_failure)) |
| 220 | return 1 |
| 221 | |
| 222 | if __name__ == '__main__': |
| 223 | sys.exit(main()) |