trace_processor: add support for JSON metric output
This supports the use-case of consumers who do not want to depend on protobuf to
consume the output of the trace processor. Instead, they can use their
favourite JSON parsing library along with considering the proto schema
to read the metric values.
We implement this dep-free as we want to remove jsoncpp support eventually.
Bug: 139461901
Change-Id: I993feaf80d0af2170fbd85d55be780de9fe2a5d0
diff --git a/Android.bp b/Android.bp
index 2af83bc..4bfaa94 100644
--- a/Android.bp
+++ b/Android.bp
@@ -4599,6 +4599,7 @@
"src/trace_processor/metrics/metrics.cc",
"src/trace_processor/process_table.cc",
"src/trace_processor/process_tracker.cc",
+ "src/trace_processor/proto_to_json.cc",
"src/trace_processor/proto_trace_parser.cc",
"src/trace_processor/proto_trace_tokenizer.cc",
"src/trace_processor/raw_table.cc",
diff --git a/BUILD b/BUILD
index e62f1bb..9ac86ea 100644
--- a/BUILD
+++ b/BUILD
@@ -699,6 +699,8 @@
"src/trace_processor/process_tracker.cc",
"src/trace_processor/process_tracker.h",
"src/trace_processor/proto_incremental_state.h",
+ "src/trace_processor/proto_to_json.cc",
+ "src/trace_processor/proto_to_json.h",
"src/trace_processor/proto_trace_parser.cc",
"src/trace_processor/proto_trace_parser.h",
"src/trace_processor/proto_trace_tokenizer.cc",
diff --git a/src/trace_processor/BUILD.gn b/src/trace_processor/BUILD.gn
index abca975..47b1e57 100644
--- a/src/trace_processor/BUILD.gn
+++ b/src/trace_processor/BUILD.gn
@@ -227,6 +227,8 @@
}
sources = [
+ "proto_to_json.cc",
+ "proto_to_json.h",
"trace_processor_shell.cc",
]
}
diff --git a/src/trace_processor/proto_to_json.cc b/src/trace_processor/proto_to_json.cc
new file mode 100644
index 0000000..5b07889
--- /dev/null
+++ b/src/trace_processor/proto_to_json.cc
@@ -0,0 +1,173 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <google/protobuf/message.h>
+
+#include "perfetto/base/logging.h"
+#include "src/trace_processor/proto_to_json.h"
+
+namespace perfetto {
+namespace trace_processor {
+namespace proto_to_json {
+
+namespace {
+
+std::string EscapeJsonString(const std::string& raw) {
+ std::string ret;
+ for (auto it = raw.cbegin(); it != raw.cend(); it++) {
+ switch (*it) {
+ case '\\':
+ ret += "\\\\";
+ break;
+ case '"':
+ ret += "\\\"";
+ break;
+ case '/':
+ ret += "\\/";
+ break;
+ case '\b':
+ ret += "\\b";
+ break;
+ case '\f':
+ ret += "\\f";
+ break;
+ case '\n':
+ ret += "\\n";
+ break;
+ case '\r':
+ ret += "\\r";
+ break;
+ case '\t':
+ ret += "\\t";
+ break;
+ default:
+ ret += *it;
+ break;
+ }
+ }
+ return '"' + ret + '"';
+}
+
+std::string FieldToJson(const google::protobuf::Message& message,
+ const google::protobuf::FieldDescriptor* field_desc,
+ int idx,
+ uint32_t indent) {
+ using google::protobuf::FieldDescriptor;
+
+ const google::protobuf::Reflection* ref = message.GetReflection();
+ bool is_repeated = field_desc->is_repeated();
+ switch (field_desc->cpp_type()) {
+ case FieldDescriptor::CppType::CPPTYPE_BOOL:
+ return std::to_string(is_repeated
+ ? ref->GetRepeatedBool(message, field_desc, idx)
+ : ref->GetBool(message, field_desc));
+ case FieldDescriptor::CppType::CPPTYPE_ENUM:
+ return EscapeJsonString(
+ is_repeated ? ref->GetRepeatedEnum(message, field_desc, idx)->name()
+ : ref->GetEnum(message, field_desc)->name());
+ case FieldDescriptor::CppType::CPPTYPE_FLOAT:
+ return std::to_string(
+ is_repeated
+ ? static_cast<double>(
+ ref->GetRepeatedFloat(message, field_desc, idx))
+ : static_cast<double>(ref->GetFloat(message, field_desc)));
+ case FieldDescriptor::CppType::CPPTYPE_INT32:
+ return std::to_string(
+ is_repeated ? ref->GetRepeatedInt32(message, field_desc, idx)
+ : ref->GetInt32(message, field_desc));
+ case FieldDescriptor::CppType::CPPTYPE_INT64:
+ return std::to_string(
+ is_repeated ? ref->GetRepeatedInt64(message, field_desc, idx)
+ : ref->GetInt64(message, field_desc));
+ case FieldDescriptor::CppType::CPPTYPE_DOUBLE:
+ return std::to_string(
+ is_repeated ? ref->GetRepeatedDouble(message, field_desc, idx)
+ : ref->GetDouble(message, field_desc));
+ case FieldDescriptor::CppType::CPPTYPE_STRING:
+ return EscapeJsonString(
+ is_repeated ? ref->GetRepeatedString(message, field_desc, idx)
+ : ref->GetString(message, field_desc));
+ case FieldDescriptor::CppType::CPPTYPE_UINT32:
+ return std::to_string(
+ is_repeated ? ref->GetRepeatedUInt32(message, field_desc, idx)
+ : ref->GetUInt32(message, field_desc));
+ case FieldDescriptor::CppType::CPPTYPE_UINT64:
+ return std::to_string(
+ is_repeated ? ref->GetRepeatedInt64(message, field_desc, idx)
+ : ref->GetInt64(message, field_desc));
+ case FieldDescriptor::CppType::CPPTYPE_MESSAGE:
+ return MessageToJson(
+ is_repeated ? ref->GetRepeatedMessage(message, field_desc, idx)
+ : ref->GetMessage(message, field_desc),
+ indent);
+ }
+ PERFETTO_FATAL("For GCC");
+}
+
+std::string RepeatedFieldValuesToJson(
+ const google::protobuf::Message& message,
+ const google::protobuf::FieldDescriptor* field_desc,
+ uint32_t indent) {
+ const google::protobuf::Reflection* ref = message.GetReflection();
+ std::string ret;
+ for (int i = 0; i < ref->FieldSize(message, field_desc); ++i) {
+ if (i != 0) {
+ ret += ",";
+ }
+ ret += "\n" + std::string(indent, ' ') +
+ FieldToJson(message, field_desc, i, indent);
+ }
+ return ret;
+}
+
+std::string MessageFieldsToJson(const google::protobuf::Message& message,
+ uint32_t indent) {
+ const google::protobuf::Reflection* ref = message.GetReflection();
+ std::vector<const google::protobuf::FieldDescriptor*> field_descs;
+ ref->ListFields(message, &field_descs);
+
+ std::string ret;
+ uint32_t next_field_idx = 0;
+ for (const google::protobuf::FieldDescriptor* field_desc : field_descs) {
+ if (next_field_idx++ != 0) {
+ ret += ",";
+ }
+ std::string value;
+ if (field_desc->is_repeated()) {
+ value = "[" + RepeatedFieldValuesToJson(message, field_desc, indent + 2) +
+ "\n" + std::string(indent, ' ') + "]";
+ } else {
+ value = FieldToJson(message, field_desc, 0, indent);
+ }
+ const std::string& name = field_desc->is_extension()
+ ? field_desc->full_name()
+ : field_desc->name();
+ ret += "\n" + std::string(indent, ' ') + "\"" + name + "\": " + value;
+ }
+ return ret;
+}
+
+} // namespace
+
+std::string MessageToJson(const google::protobuf::Message& message,
+ uint32_t indent) {
+ return "{" + MessageFieldsToJson(message, indent + 2) + "\n" +
+ std::string(indent, ' ') + "}";
+}
+
+} // namespace proto_to_json
+} // namespace trace_processor
+} // namespace perfetto
diff --git a/src/trace_processor/proto_to_json.h b/src/trace_processor/proto_to_json.h
new file mode 100644
index 0000000..f46f4c1
--- /dev/null
+++ b/src/trace_processor/proto_to_json.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SRC_TRACE_PROCESSOR_PROTO_TO_JSON_H_
+#define SRC_TRACE_PROCESSOR_PROTO_TO_JSON_H_
+
+#include <google/protobuf/message.h>
+
+namespace perfetto {
+namespace trace_processor {
+namespace proto_to_json {
+
+std::string MessageToJson(const google::protobuf::Message& message,
+ uint32_t indent = 0);
+
+} // namespace proto_to_json
+} // namespace trace_processor
+} // namespace perfetto
+
+#endif // SRC_TRACE_PROCESSOR_PROTO_TO_JSON_H_
diff --git a/src/trace_processor/trace_processor_shell.cc b/src/trace_processor/trace_processor_shell.cc
index 5cca3c9..bc3c7e8 100644
--- a/src/trace_processor/trace_processor_shell.cc
+++ b/src/trace_processor/trace_processor_shell.cc
@@ -36,6 +36,7 @@
#include "perfetto/ext/base/string_splitter.h"
#include "perfetto/trace_processor/trace_processor.h"
#include "src/trace_processor/metrics/metrics.descriptor.h"
+#include "src/trace_processor/proto_to_json.h"
#if PERFETTO_BUILDFLAG(PERFETTO_OS_LINUX) || \
PERFETTO_BUILDFLAG(PERFETTO_OS_ANDROID) || \
@@ -334,8 +335,14 @@
return g_tp->ExtendMetricsProto(metric_proto.data(), metric_proto.size());
}
+enum OutputFormat {
+ kBinaryProto,
+ kTextProto,
+ kJson,
+};
+
int RunMetrics(const std::vector<std::string>& metric_names,
- bool metrics_textproto,
+ OutputFormat format,
const google::protobuf::DescriptorPool& pool) {
std::vector<uint8_t> metric_result;
util::Status status = g_tp->ComputeMetric(metric_names, &metric_result);
@@ -343,19 +350,32 @@
PERFETTO_ELOG("Error when computing metrics: %s", status.c_message());
return 1;
}
- if (metrics_textproto) {
- google::protobuf::DynamicMessageFactory factory(&pool);
- auto* descriptor =
- pool.FindMessageTypeByName("perfetto.protos.TraceMetrics");
- std::unique_ptr<google::protobuf::Message> metrics(
- factory.GetPrototype(descriptor)->New());
- metrics->ParseFromArray(metric_result.data(),
- static_cast<int>(metric_result.size()));
- std::string out;
- google::protobuf::TextFormat::PrintToString(*metrics, &out);
- fwrite(out.c_str(), sizeof(char), out.size(), stdout);
- } else {
+ if (format == OutputFormat::kBinaryProto) {
fwrite(metric_result.data(), sizeof(uint8_t), metric_result.size(), stdout);
+ return 0;
+ }
+
+ google::protobuf::DynamicMessageFactory factory(&pool);
+ auto* descriptor = pool.FindMessageTypeByName("perfetto.protos.TraceMetrics");
+ std::unique_ptr<google::protobuf::Message> metrics(
+ factory.GetPrototype(descriptor)->New());
+ metrics->ParseFromArray(metric_result.data(),
+ static_cast<int>(metric_result.size()));
+
+ switch (format) {
+ case OutputFormat::kTextProto: {
+ std::string out;
+ google::protobuf::TextFormat::PrintToString(*metrics, &out);
+ fwrite(out.c_str(), sizeof(char), out.size(), stdout);
+ break;
+ }
+ case OutputFormat::kJson: {
+ auto out = proto_to_json::MessageToJson(*metrics) + "\n";
+ fwrite(out.c_str(), sizeof(char), out.size(), stdout);
+ break;
+ }
+ case OutputFormat::kBinaryProto:
+ PERFETTO_FATAL("Unsupported output format.");
}
return 0;
}
@@ -1024,8 +1044,15 @@
metrics[i] = basename;
}
- bool metrics_textproto = options.metric_output != "binary";
- int ret = RunMetrics(std::move(metrics), metrics_textproto, pool);
+ OutputFormat format;
+ if (options.metric_output == "binary") {
+ format = OutputFormat::kBinaryProto;
+ } else if (options.metric_output == "json") {
+ format = OutputFormat::kJson;
+ } else {
+ format = OutputFormat::kTextProto;
+ }
+ int ret = RunMetrics(std::move(metrics), format, pool);
if (!ret) {
auto t_query = base::GetWallTimeNs() - t_run_start;
ret = MaybePrintPerfFile(options.perf_file_path, t_load, t_query);
diff --git a/test/metrics/index b/test/metrics/index
index eb13d74..f3cc26f 100644
--- a/test/metrics/index
+++ b/test/metrics/index
@@ -19,3 +19,6 @@
android_package_list.py android_package_list android_package_list.out
heap_profile.textproto heap_profile_callsite_stats heap_profile_callsite_stats.out
+
+# Json output
+../data/memory_counters.pb trace_metadata trace_metadata.json.out
\ No newline at end of file
diff --git a/test/metrics/trace_metadata.json.out b/test/metrics/trace_metadata.json.out
new file mode 100644
index 0000000..c961b9e
--- /dev/null
+++ b/test/metrics/trace_metadata.json.out
@@ -0,0 +1,11 @@
+{
+ "trace_metadata": {
+ "error_stats_entry": [
+ {
+ "name": "mismatched_sched_switch_tids",
+ "value": 5
+ }
+ ],
+ "trace_duration_ns": 9519159074
+ }
+}
diff --git a/tools/diff_test_trace_processor.py b/tools/diff_test_trace_processor.py
index 74a39ab..9f1e8d6 100755
--- a/tools/diff_test_trace_processor.py
+++ b/tools/diff_test_trace_processor.py
@@ -95,30 +95,35 @@
with open(expected_path, 'r') as expected_file:
expected = expected_file.read()
+ json_output = os.path.basename(expected_path).endswith('.json.out')
cmd = [
trace_processor_path,
'--run-metrics',
metric,
- '--metrics-output=binary',
+ '--metrics-output=%s' % ('json' if json_output else 'binary'),
gen_trace_path,
'--perf-file',
perf_path,
]
actual = subprocess.check_output(cmd)
- # Expected will be in text proto format and we'll need to parse it to a real
- # proto.
- expected_message = metrics_message_factory()
- text_format.Merge(expected, expected_message)
+ if json_output:
+ expected_text = expected
+ actual_text = actual
+ else:
+ # Expected will be in text proto format and we'll need to parse it to a real
+ # proto.
+ expected_message = metrics_message_factory()
+ text_format.Merge(expected, expected_message)
- # Actual will be the raw bytes of the proto and we'll need to parse it into
- # a message.
- actual_message = metrics_message_factory()
- actual_message.ParseFromString(actual)
+ # Actual will be the raw bytes of the proto and we'll need to parse it into
+ # a message.
+ actual_message = metrics_message_factory()
+ actual_message.ParseFromString(actual)
- # Convert both back to text format.
- expected_text = text_format.MessageToString(expected_message)
- actual_text = text_format.MessageToString(actual_message)
+ # Convert both back to text format.
+ expected_text = text_format.MessageToString(expected_message)
+ actual_text = text_format.MessageToString(actual_message)
return TestResult('metric', metric, cmd, expected_text, actual_text)