Introduce DebugAnnotationParser
Refactor the debug annotation parsing logic into a standalone class,
which later will be used to parse debug annotations for console
interceptor.
R=eseckler@google.com,lalitm@google.com
Change-Id: I6b53706c4b63d455f4df1600b30029b02cc92139
diff --git a/src/trace_processor/util/BUILD.gn b/src/trace_processor/util/BUILD.gn
index 02887ac..31925e8 100644
--- a/src/trace_processor/util/BUILD.gn
+++ b/src/trace_processor/util/BUILD.gn
@@ -45,6 +45,23 @@
]
}
+source_set("trace_blob_view") {
+ sources = [ "trace_blob_view.h" ]
+ deps = [
+ "../../../gn:default_deps",
+ "../../base",
+ ]
+}
+
+source_set("interned_message_view") {
+ sources = [ "interned_message_view.h" ]
+ deps = [
+ "../../../gn:default_deps",
+ "../../base",
+ ]
+ public_deps = [ ":trace_blob_view" ]
+}
+
source_set("descriptors") {
sources = [
"descriptors.cc",
@@ -55,14 +72,16 @@
"../../../gn:default_deps",
"../../../protos/perfetto/common:zero",
"../../../protos/perfetto/trace_processor:zero",
+ "../../base",
"../../protozero",
"../importers:gen_cc_track_event_descriptor",
]
- public_deps = [ "../../base" ]
}
source_set("proto_to_args_parser") {
sources = [
+ "debug_annotation_parser.cc",
+ "debug_annotation_parser.h",
"proto_to_args_parser.cc",
"proto_to_args_parser.h",
]
@@ -70,6 +89,7 @@
"../../../gn:default_deps",
"../../../protos/perfetto/common:zero",
"../../../protos/perfetto/trace/interned_data:zero",
+ "../../../protos/perfetto/trace/track_event:zero",
"../../../protos/perfetto/trace_processor:zero",
"../../protozero",
"../importers:gen_cc_track_event_descriptor",
@@ -77,17 +97,15 @@
public_deps = [
":descriptors",
+ ":interned_message_view",
":util",
"../../base",
-
- # TODO(altimin): Move InternedMessageView and TraceBlobView here and remove
- # this dependency.
- "../importers/common",
]
}
source_set("unittests") {
sources = [
+ "debug_annotation_parser_unittest.cc",
"proto_to_args_parser_unittest.cc",
"protozero_to_text_unittests.cc",
]
@@ -104,9 +122,5 @@
"../../protozero",
"../../protozero:testing_messages_zero",
"../importers:gen_cc_track_event_descriptor",
-
- # TODO(altimin): Move InternedMessageView and TraceBlobView here and remove
- # this dependency.
- "..:storage_minimal",
]
}
diff --git a/src/trace_processor/util/debug_annotation_parser.cc b/src/trace_processor/util/debug_annotation_parser.cc
new file mode 100644
index 0000000..eeaf39f
--- /dev/null
+++ b/src/trace_processor/util/debug_annotation_parser.cc
@@ -0,0 +1,229 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/trace_processor/util/debug_annotation_parser.h"
+#include "perfetto/base/build_config.h"
+#include "protos/perfetto/trace/track_event/debug_annotation.pbzero.h"
+#include "src/trace_processor/util/interned_message_view.h"
+
+namespace perfetto {
+namespace trace_processor {
+namespace util {
+
+namespace {
+
+std::string SanitizeDebugAnnotationName(const std::string& raw_name) {
+ std::string result = raw_name;
+ std::replace(result.begin(), result.end(), '.', '_');
+ std::replace(result.begin(), result.end(), '[', '_');
+ std::replace(result.begin(), result.end(), ']', '_');
+ return result;
+}
+
+bool IsJsonSupported() {
+#if PERFETTO_BUILDFLAG(PERFETTO_TP_JSON)
+ return true;
+#else
+ return false;
+#endif
+}
+
+} // namespace
+
+DebugAnnotationParser::DebugAnnotationParser(ProtoToArgsParser& parser)
+ : proto_to_args_parser_(parser) {}
+
+base::Status DebugAnnotationParser::ParseDebugAnnotationName(
+ protos::pbzero::DebugAnnotation::Decoder& annotation,
+ ProtoToArgsParser::Delegate& delegate,
+ std::string& result) {
+ uint64_t name_iid = annotation.name_iid();
+ if (PERFETTO_LIKELY(name_iid)) {
+ auto* decoder = delegate.GetInternedMessage(
+ protos::pbzero::InternedData::kDebugAnnotationNames, name_iid);
+ if (!decoder)
+ return base::ErrStatus("Debug annotation with invalid name_iid");
+
+ result = SanitizeDebugAnnotationName(decoder->name().ToStdString());
+ } else if (annotation.has_name()) {
+ result = SanitizeDebugAnnotationName(annotation.name().ToStdString());
+ } else {
+ return base::ErrStatus("Debug annotation without name");
+ }
+ return base::OkStatus();
+}
+
+DebugAnnotationParser::ParseResult
+DebugAnnotationParser::ParseDebugAnnotationValue(
+ protos::pbzero::DebugAnnotation::Decoder& annotation,
+ ProtoToArgsParser::Delegate& delegate,
+ const ProtoToArgsParser::Key& context_name) {
+ if (annotation.has_bool_value()) {
+ delegate.AddBoolean(context_name, annotation.bool_value());
+ } else if (annotation.has_uint_value()) {
+ delegate.AddUnsignedInteger(context_name, annotation.uint_value());
+ } else if (annotation.has_int_value()) {
+ delegate.AddInteger(context_name, annotation.int_value());
+ } else if (annotation.has_double_value()) {
+ delegate.AddDouble(context_name, annotation.double_value());
+ } else if (annotation.has_string_value()) {
+ delegate.AddString(context_name, annotation.string_value());
+ } else if (annotation.has_pointer_value()) {
+ delegate.AddPointer(context_name, reinterpret_cast<const void*>(
+ annotation.pointer_value()));
+ } else if (annotation.has_dict_entries()) {
+ bool added_entry = false;
+ for (auto it = annotation.dict_entries(); it; ++it) {
+ protos::pbzero::DebugAnnotation::Decoder key_value(*it);
+ std::string key;
+ base::Status key_parse_result =
+ ParseDebugAnnotationName(key_value, delegate, key);
+ if (!key_parse_result.ok())
+ return {key_parse_result, added_entry};
+
+ auto nested_key = proto_to_args_parser_.EnterDictionary(key);
+ ParseResult value_parse_result =
+ ParseDebugAnnotationValue(key_value, delegate, nested_key.key());
+ added_entry |= value_parse_result.added_entry;
+ if (!value_parse_result.status.ok())
+ return {value_parse_result.status, added_entry};
+ }
+ } else if (annotation.has_array_values()) {
+ size_t index = delegate.GetArrayEntryIndex(context_name.key);
+ bool added_entry = false;
+ for (auto it = annotation.array_values(); it; ++it) {
+ protos::pbzero::DebugAnnotation::Decoder value(*it);
+
+ auto nested_key = proto_to_args_parser_.EnterArray(index);
+ ParseResult value_parse_result =
+ ParseDebugAnnotationValue(value, delegate, nested_key.key());
+
+ // Reset the key here to ensure that we have the correct array key to
+ // increment.
+ nested_key.Reset();
+
+ if (value_parse_result.added_entry) {
+ index = delegate.IncrementArrayEntryIndex(context_name.key);
+ added_entry = true;
+ }
+ if (!value_parse_result.status.ok())
+ return {value_parse_result.status, added_entry};
+ }
+ } else if (annotation.has_legacy_json_value()) {
+ if (!IsJsonSupported())
+ return {base::ErrStatus("Ignoring legacy_json_value (no json support)"),
+ false};
+
+ bool added_entry =
+ delegate.AddJson(context_name, annotation.legacy_json_value());
+ return {base::OkStatus(), added_entry};
+ } else if (annotation.has_nested_value()) {
+ return ParseNestedValueArgs(annotation.nested_value(), context_name,
+ delegate);
+ } else {
+ return {base::OkStatus(), /*added_entry=*/false};
+ }
+
+ return {base::OkStatus(), /*added_entry=*/true};
+}
+
+// static
+base::Status DebugAnnotationParser::Parse(
+ protozero::ConstBytes data,
+ ProtoToArgsParser::Delegate& delegate) {
+ protos::pbzero::DebugAnnotation::Decoder annotation(data);
+
+ std::string name;
+ base::Status name_parse_result =
+ ParseDebugAnnotationName(annotation, delegate, name);
+ if (!name_parse_result.ok())
+ return name_parse_result;
+
+ auto context = proto_to_args_parser_.EnterDictionary(name);
+
+ return ParseDebugAnnotationValue(annotation, delegate, context.key()).status;
+}
+
+DebugAnnotationParser::ParseResult DebugAnnotationParser::ParseNestedValueArgs(
+ protozero::ConstBytes nested_value,
+ const ProtoToArgsParser::Key& context_name,
+ ProtoToArgsParser::Delegate& delegate) {
+ protos::pbzero::DebugAnnotation::NestedValue::Decoder value(nested_value);
+ switch (value.nested_type()) {
+ case protos::pbzero::DebugAnnotation::NestedValue::UNSPECIFIED: {
+ // Leaf value.
+ if (value.has_bool_value()) {
+ delegate.AddBoolean(context_name, value.bool_value());
+ return {base::OkStatus(), true};
+ }
+ if (value.has_int_value()) {
+ delegate.AddInteger(context_name, value.int_value());
+ return {base::OkStatus(), true};
+ }
+ if (value.has_double_value()) {
+ delegate.AddDouble(context_name, value.double_value());
+ return {base::OkStatus(), true};
+ }
+ if (value.has_string_value()) {
+ delegate.AddString(context_name, value.string_value());
+ return {base::OkStatus(), true};
+ }
+ return {base::OkStatus(), false};
+ }
+ case protos::pbzero::DebugAnnotation::NestedValue::DICT: {
+ bool added_entry = false;
+ auto key_it = value.dict_keys();
+ auto value_it = value.dict_values();
+ for (; key_it && value_it; ++key_it, ++value_it) {
+ std::string child_name =
+ SanitizeDebugAnnotationName((*key_it).ToStdString());
+ auto nested_key = proto_to_args_parser_.EnterDictionary(child_name);
+ ParseResult result =
+ ParseNestedValueArgs(*value_it, nested_key.key(), delegate);
+ added_entry |= result.added_entry;
+ if (!result.status.ok())
+ return {result.status, added_entry};
+ }
+ return {base::OkStatus(), true};
+ }
+
+ case protos::pbzero::DebugAnnotation::NestedValue::ARRAY: {
+ std::string array_key = context_name.key;
+ size_t array_index = delegate.GetArrayEntryIndex(context_name.key);
+ bool added_entry = false;
+
+ for (auto value_it = value.array_values(); value_it; ++value_it) {
+ auto nested_key = proto_to_args_parser_.EnterArray(array_index);
+ ParseResult result =
+ ParseNestedValueArgs(*value_it, nested_key.key(), delegate);
+
+ if (result.added_entry) {
+ ++array_index;
+ delegate.IncrementArrayEntryIndex(array_key);
+ added_entry = true;
+ }
+ if (!result.status.ok())
+ return {result.status, added_entry};
+ }
+ return {base::OkStatus(), added_entry};
+ }
+ }
+ return {base::OkStatus(), false};
+}
+
+} // namespace util
+} // namespace trace_processor
+} // namespace perfetto
diff --git a/src/trace_processor/util/debug_annotation_parser.h b/src/trace_processor/util/debug_annotation_parser.h
new file mode 100644
index 0000000..0e68eec
--- /dev/null
+++ b/src/trace_processor/util/debug_annotation_parser.h
@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SRC_TRACE_PROCESSOR_UTIL_DEBUG_ANNOTATION_PARSER_H_
+#define SRC_TRACE_PROCESSOR_UTIL_DEBUG_ANNOTATION_PARSER_H_
+
+#include "protos/perfetto/trace/track_event/debug_annotation.pbzero.h"
+#include "src/trace_processor/util/proto_to_args_parser.h"
+
+namespace perfetto {
+namespace trace_processor {
+namespace util {
+
+// |DebugAnnotationParser| is responsible for parsing DebugAnnotation protos
+// and turning it into key-value arg pairs.
+// |DebugAnnotationParser| is a logical extension of |ProtoToArgsParser|:
+// it uses |ProtoToArgsParser::Delegate| for writing the results and uses
+// |ProtoToArgsParser| to parse arbitrary protos nested inside DebugAnnotation.
+class DebugAnnotationParser {
+ public:
+ explicit DebugAnnotationParser(ProtoToArgsParser& proto_to_args_parser);
+
+ base::Status Parse(protozero::ConstBytes data,
+ ProtoToArgsParser::Delegate& delegate);
+
+ private:
+ struct ParseResult {
+ base::Status status;
+ // True if parsing of the annotation added at least one entry to the
+ // |delegate|.
+ bool added_entry;
+ };
+
+ base::Status ParseDebugAnnotationName(
+ protos::pbzero::DebugAnnotation::Decoder& annotation,
+ ProtoToArgsParser::Delegate& delegate,
+ std::string& result);
+ ParseResult ParseDebugAnnotationValue(
+ protos::pbzero::DebugAnnotation::Decoder& annotation,
+ ProtoToArgsParser::Delegate& delegate,
+ const ProtoToArgsParser::Key& context_name);
+ ParseResult ParseNestedValueArgs(protozero::ConstBytes nested_value,
+ const ProtoToArgsParser::Key& context_name,
+ ProtoToArgsParser::Delegate& delegate);
+
+ ProtoToArgsParser& proto_to_args_parser_;
+};
+
+} // namespace util
+} // namespace trace_processor
+} // namespace perfetto
+
+#endif // SRC_TRACE_PROCESSOR_UTIL_DEBUG_ANNOTATION_PARSER_H_
diff --git a/src/trace_processor/util/debug_annotation_parser_unittest.cc b/src/trace_processor/util/debug_annotation_parser_unittest.cc
new file mode 100644
index 0000000..e3f3380
--- /dev/null
+++ b/src/trace_processor/util/debug_annotation_parser_unittest.cc
@@ -0,0 +1,237 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/trace_processor/util/debug_annotation_parser.h"
+
+#include "perfetto/ext/base/string_view.h"
+#include "perfetto/protozero/scattered_heap_buffer.h"
+#include "protos/perfetto/common/descriptor.pbzero.h"
+#include "protos/perfetto/trace/track_event/debug_annotation.pbzero.h"
+#include "protos/perfetto/trace/track_event/source_location.pbzero.h"
+#include "src/protozero/test/example_proto/test_messages.pbzero.h"
+#include "src/trace_processor/test_messages.descriptor.h"
+#include "src/trace_processor/util/interned_message_view.h"
+#include "src/trace_processor/util/proto_to_args_parser.h"
+#include "src/trace_processor/util/trace_blob_view.h"
+#include "test/gtest_and_gmock.h"
+
+#include <sstream>
+
+namespace perfetto {
+namespace trace_processor {
+namespace util {
+namespace {
+
+base::Status ParseDebugAnnotation(
+ DebugAnnotationParser& parser,
+ protozero::HeapBuffered<protos::pbzero::DebugAnnotation>& msg,
+ ProtoToArgsParser::Delegate& delegate) {
+ std::vector<uint8_t> data = msg.SerializeAsArray();
+ return parser.Parse(protozero::ConstBytes{data.data(), data.size()},
+ delegate);
+}
+
+class DebugAnnotationParserTest : public ::testing::Test,
+ public ProtoToArgsParser::Delegate {
+ protected:
+ DebugAnnotationParserTest() {}
+
+ const std::vector<std::string>& args() const { return args_; }
+
+ private:
+ using Key = ProtoToArgsParser::Key;
+
+ void AddInteger(const Key& key, int64_t value) override {
+ std::stringstream ss;
+ ss << key.flat_key << " " << key.key << " " << value;
+ args_.push_back(ss.str());
+ }
+
+ void AddUnsignedInteger(const Key& key, uint64_t value) override {
+ std::stringstream ss;
+ ss << key.flat_key << " " << key.key << " " << value;
+ args_.push_back(ss.str());
+ }
+
+ void AddString(const Key& key, const protozero::ConstChars& value) override {
+ std::stringstream ss;
+ ss << key.flat_key << " " << key.key << " " << value.ToStdString();
+ args_.push_back(ss.str());
+ }
+
+ void AddDouble(const Key& key, double value) override {
+ std::stringstream ss;
+ ss << key.flat_key << " " << key.key << " " << value;
+ args_.push_back(ss.str());
+ }
+
+ void AddPointer(const Key& key, const void* value) override {
+ std::stringstream ss;
+ ss << key.flat_key << " " << key.key << " " << std::hex
+ << reinterpret_cast<uintptr_t>(value) << std::dec;
+ args_.push_back(ss.str());
+ }
+
+ void AddBoolean(const Key& key, bool value) override {
+ std::stringstream ss;
+ ss << key.flat_key << " " << key.key << " " << (value ? "true" : "false");
+ args_.push_back(ss.str());
+ }
+
+ bool AddJson(const Key& key, const protozero::ConstChars& value) override {
+ std::stringstream ss;
+ ss << key.flat_key << " " << key.key << " " << std::hex
+ << value.ToStdString() << std::dec;
+ args_.push_back(ss.str());
+ return true;
+ }
+
+ size_t GetArrayEntryIndex(const std::string& array_key) final {
+ return array_indices_[array_key];
+ }
+
+ size_t IncrementArrayEntryIndex(const std::string& array_key) final {
+ return ++array_indices_[array_key];
+ }
+
+ InternedMessageView* GetInternedMessageView(uint32_t, uint64_t) override {
+ return nullptr;
+ }
+
+ std::vector<std::string> args_;
+ std::map<std::string, size_t> array_indices_;
+};
+
+// This test checks that in when an array is nested inside a dict which is
+// nested inside an array which is nested inside a dict, flat keys and non-flat
+// keys are parsed correctly.
+TEST_F(DebugAnnotationParserTest, DeeplyNestedDictsAndArrays) {
+ protozero::HeapBuffered<protos::pbzero::DebugAnnotation> msg;
+
+ msg->set_name("root");
+ auto* dict1 = msg->add_dict_entries();
+ dict1->set_name("k1");
+ auto* array1 = dict1->add_array_values();
+ auto* dict2 = array1->add_dict_entries();
+ dict2->set_name("k2");
+ auto* array2 = dict2->add_array_values();
+ array2->set_int_value(42);
+
+ DescriptorPool pool;
+ auto status = pool.AddFromFileDescriptorSet(kTestMessagesDescriptor.data(),
+ kTestMessagesDescriptor.size());
+ EXPECT_TRUE(status.ok()) << "Failed to parse kTestMessagesDescriptor: "
+ << status.message();
+
+ ProtoToArgsParser args_parser(pool);
+ DebugAnnotationParser parser(args_parser);
+
+ status = ParseDebugAnnotation(parser, msg, *this);
+ EXPECT_TRUE(status.ok()) << "DebugAnnotationParser::Parse failed with error: "
+ << status.message();
+
+ EXPECT_THAT(args(), testing::ElementsAre("root.k1.k2 root.k1[0].k2[0] 42"));
+}
+
+// This test checks that array indexes are correctly merged across messages.
+TEST_F(DebugAnnotationParserTest, MergeArrays) {
+ protozero::HeapBuffered<protos::pbzero::DebugAnnotation> msg1;
+ msg1->set_name("root");
+ auto* item1 = msg1->add_array_values();
+ item1->set_int_value(1);
+
+ protozero::HeapBuffered<protos::pbzero::DebugAnnotation> msg2;
+ msg2->set_name("root");
+ auto* item2 = msg1->add_array_values();
+ item2->set_int_value(2);
+
+ DescriptorPool pool;
+ ProtoToArgsParser args_parser(pool);
+ DebugAnnotationParser parser(args_parser);
+
+ base::Status status = ParseDebugAnnotation(parser, msg1, *this);
+ EXPECT_TRUE(status.ok()) << "DebugAnnotationParser::Parse failed with error: "
+ << status.message();
+
+ status = ParseDebugAnnotation(parser, msg2, *this);
+ EXPECT_TRUE(status.ok()) << "DebugAnnotationParser::Parse failed with error: "
+ << status.message();
+
+ EXPECT_THAT(args(), testing::ElementsAre("root root[0] 1", "root root[1] 2"));
+}
+
+// This test checks that nested empty dictionaries / arrays do not cause array
+// index to be incremented.
+TEST_F(DebugAnnotationParserTest, EmptyArrayIndexIsSkipped) {
+ protozero::HeapBuffered<protos::pbzero::DebugAnnotation> msg;
+ msg->set_name("root");
+
+ msg->add_array_values()->set_int_value(1);
+
+ // Empty item.
+ msg->add_array_values();
+
+ msg->add_array_values()->set_int_value(3);
+
+ // Empty dict.
+ msg->add_array_values()->add_dict_entries()->set_name("key1");
+
+ auto* nested_dict_entry = msg->add_array_values()->add_dict_entries();
+ nested_dict_entry->set_name("key2");
+ nested_dict_entry->set_string_value("value");
+
+ msg->add_array_values()->set_int_value(5);
+
+ DescriptorPool pool;
+ ProtoToArgsParser args_parser(pool);
+ DebugAnnotationParser parser(args_parser);
+
+ base::Status status = ParseDebugAnnotation(parser, msg, *this);
+ EXPECT_TRUE(status.ok()) << "DebugAnnotationParser::Parse failed with error: "
+ << status.message();
+
+ EXPECT_THAT(args(), testing::ElementsAre("root root[0] 1", "root root[1] 3",
+ "root.key2 root[3].key2 value",
+ "root root[4] 5"));
+}
+
+TEST_F(DebugAnnotationParserTest, NestedArrays) {
+ protozero::HeapBuffered<protos::pbzero::DebugAnnotation> msg;
+ msg->set_name("root");
+ auto* item1 = msg->add_array_values();
+ item1->add_array_values()->set_int_value(1);
+ item1->add_array_values()->set_int_value(2);
+ auto* item2 = msg->add_array_values();
+ item2->add_array_values()->set_int_value(3);
+ item2->add_array_values()->set_int_value(4);
+
+ DescriptorPool pool;
+ ProtoToArgsParser args_parser(pool);
+ DebugAnnotationParser parser(args_parser);
+
+ base::Status status = ParseDebugAnnotation(parser, msg, *this);
+ EXPECT_TRUE(status.ok()) << "DebugAnnotationParser::Parse failed with error: "
+ << status.message();
+
+ EXPECT_THAT(args(),
+ testing::ElementsAre("root root[0][0] 1", "root root[0][1] 2",
+ "root root[1][0] 3", "root root[1][1] 4"));
+}
+
+} // namespace
+} // namespace util
+} // namespace trace_processor
+} // namespace perfetto
diff --git a/src/trace_processor/util/interned_message_view.h b/src/trace_processor/util/interned_message_view.h
new file mode 100644
index 0000000..7341aea
--- /dev/null
+++ b/src/trace_processor/util/interned_message_view.h
@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SRC_TRACE_PROCESSOR_UTIL_INTERNED_MESSAGE_VIEW_H_
+#define SRC_TRACE_PROCESSOR_UTIL_INTERNED_MESSAGE_VIEW_H_
+
+#include "src/trace_processor/util/trace_blob_view.h"
+
+#include <unordered_map>
+
+namespace perfetto {
+namespace trace_processor {
+
+#if PERFETTO_DCHECK_IS_ON()
+// When called from GetOrCreateDecoder(), should include the stringified name of
+// the MessageType.
+#define PERFETTO_TYPE_IDENTIFIER PERFETTO_DEBUG_FUNCTION_IDENTIFIER()
+#else // PERFETTO_DCHECK_IS_ON()
+#define PERFETTO_TYPE_IDENTIFIER nullptr
+#endif // PERFETTO_DCHECK_IS_ON()
+
+// Entry in an interning index, refers to the interned message.
+class InternedMessageView {
+ public:
+ InternedMessageView(TraceBlobView msg) : message_(std::move(msg)) {}
+
+ InternedMessageView(InternedMessageView&&) = default;
+ InternedMessageView& operator=(InternedMessageView&&) = default;
+
+ // Allow copy by cloning the TraceBlobView. This is required for
+ // UpdateTracePacketDefaults().
+ InternedMessageView(const InternedMessageView& view)
+ : message_(view.message_.slice(0, view.message_.length())) {}
+ InternedMessageView& operator=(const InternedMessageView& view) {
+ this->message_ = view.message_.slice(0, view.message_.length());
+ this->decoder_ = nullptr;
+ this->decoder_type_ = nullptr;
+ this->submessages_.clear();
+ return *this;
+ }
+
+ // Lazily initializes and returns the decoder object for the message. The
+ // decoder is stored in the InternedMessageView to avoid having to parse the
+ // message multiple times.
+ template <typename MessageType>
+ typename MessageType::Decoder* GetOrCreateDecoder() {
+ if (!decoder_) {
+ // Lazy init the decoder and save it away, so that we don't have to
+ // reparse the message every time we access the interning entry.
+ decoder_ = std::unique_ptr<void, std::function<void(void*)>>(
+ new typename MessageType::Decoder(message_.data(), message_.length()),
+ [](void* obj) {
+ delete reinterpret_cast<typename MessageType::Decoder*>(obj);
+ });
+ decoder_type_ = PERFETTO_TYPE_IDENTIFIER;
+ }
+ // Verify that the type of the decoder didn't change.
+ if (PERFETTO_TYPE_IDENTIFIER &&
+ strcmp(decoder_type_,
+ // GCC complains if this arg can be null.
+ PERFETTO_TYPE_IDENTIFIER ? PERFETTO_TYPE_IDENTIFIER : "") != 0) {
+ PERFETTO_FATAL(
+ "Interning entry accessed under different types! previous type: "
+ "%s. new type: %s.",
+ decoder_type_, PERFETTO_DEBUG_FUNCTION_IDENTIFIER());
+ }
+ return reinterpret_cast<typename MessageType::Decoder*>(decoder_.get());
+ }
+
+ // Lookup a submessage of the interned message, which is then itself stored
+ // as InternedMessageView, so that we only need to parse it once. Returns
+ // nullptr if the field isn't set.
+ // TODO(eseckler): Support repeated fields.
+ template <typename MessageType, uint32_t FieldId>
+ InternedMessageView* GetOrCreateSubmessageView() {
+ auto it = submessages_.find(FieldId);
+ if (it != submessages_.end())
+ return it->second.get();
+ auto* decoder = GetOrCreateDecoder<MessageType>();
+ // Calls the at() template method on the decoder.
+ auto field = decoder->template at<FieldId>().as_bytes();
+ if (!field.data)
+ return nullptr;
+ const size_t offset = message_.offset_of(field.data);
+ TraceBlobView submessage = message_.slice(offset, field.size);
+ InternedMessageView* submessage_view =
+ new InternedMessageView(std::move(submessage));
+ submessages_.emplace_hint(
+ it, FieldId, std::unique_ptr<InternedMessageView>(submessage_view));
+ return submessage_view;
+ }
+
+ const TraceBlobView& message() { return message_; }
+
+ private:
+ using SubMessageViewMap =
+ std::unordered_map<uint32_t /*field_id*/,
+ std::unique_ptr<InternedMessageView>>;
+
+ TraceBlobView message_;
+
+ // Stores the decoder for the message_, so that the message does not have to
+ // be re-decoded every time the interned message is looked up. Lazily
+ // initialized in GetOrCreateDecoder(). Since we don't know the type of the
+ // decoder until GetOrCreateDecoder() is called, we store the decoder as a
+ // void* unique_pointer with a destructor function that's supplied in
+ // GetOrCreateDecoder() when the decoder is created.
+ std::unique_ptr<void, std::function<void(void*)>> decoder_;
+
+ // Type identifier for the decoder. Only valid in debug builds and on
+ // supported platforms. Used to verify that GetOrCreateDecoder() is always
+ // called with the same template argument.
+ const char* decoder_type_ = nullptr;
+
+ // Views of submessages of the interned message. Submessages are lazily
+ // added by GetOrCreateSubmessageView(). By storing submessages and their
+ // decoders, we avoid having to decode submessages multiple times if they
+ // looked up often.
+ SubMessageViewMap submessages_;
+};
+
+} // namespace trace_processor
+} // namespace perfetto
+
+#endif // SRC_TRACE_PROCESSOR_UTIL_INTERNED_MESSAGE_VIEW_H_
diff --git a/src/trace_processor/util/proto_to_args_parser.cc b/src/trace_processor/util/proto_to_args_parser.cc
index 6fec35e..8e4eb22 100644
--- a/src/trace_processor/util/proto_to_args_parser.cc
+++ b/src/trace_processor/util/proto_to_args_parser.cc
@@ -63,6 +63,33 @@
: flat_key(fk), key(k) {}
ProtoToArgsParser::Key::~Key() = default;
+ProtoToArgsParser::ScopedNestedKeyContext::ScopedNestedKeyContext(Key& key)
+ : key_(key),
+ old_flat_key_length_(key.flat_key.length()),
+ old_key_length_(key.key.length()) {}
+
+ProtoToArgsParser::ScopedNestedKeyContext::ScopedNestedKeyContext(
+ ProtoToArgsParser::ScopedNestedKeyContext&& other)
+ : key_(other.key_),
+ old_flat_key_length_(other.old_flat_key_length_),
+ old_key_length_(other.old_key_length_) {
+ other.old_flat_key_length_ = base::nullopt;
+ other.old_key_length_ = base::nullopt;
+}
+
+ProtoToArgsParser::ScopedNestedKeyContext::~ScopedNestedKeyContext() {
+ Reset();
+}
+
+void ProtoToArgsParser::ScopedNestedKeyContext::Reset() {
+ if (old_flat_key_length_)
+ key_.flat_key.resize(old_flat_key_length_.value());
+ if (old_key_length_)
+ key_.key.resize(old_key_length_.value());
+ old_flat_key_length_ = base::nullopt;
+ old_key_length_ = base::nullopt;
+}
+
ProtoToArgsParser::Delegate::~Delegate() = default;
ProtoToArgsParser::ProtoToArgsParser(const DescriptorPool& pool) : pool_(pool) {
@@ -238,6 +265,25 @@
}
}
+ProtoToArgsParser::ScopedNestedKeyContext ProtoToArgsParser::EnterArray(
+ size_t index) {
+ auto context = ScopedNestedKeyContext(key_prefix_);
+ key_prefix_.key += "[" + std::to_string(index) + "]";
+ return context;
+}
+
+ProtoToArgsParser::ScopedNestedKeyContext ProtoToArgsParser::EnterDictionary(
+ const std::string& name) {
+ auto context = ScopedNestedKeyContext(key_prefix_);
+ if (!key_prefix_.key.empty())
+ key_prefix_.key += '.';
+ key_prefix_.key += name;
+ if (!key_prefix_.flat_key.empty())
+ key_prefix_.flat_key += '.';
+ key_prefix_.flat_key += name;
+ return context;
+}
+
} // namespace util
} // namespace trace_processor
} // namespace perfetto
diff --git a/src/trace_processor/util/proto_to_args_parser.h b/src/trace_processor/util/proto_to_args_parser.h
index f99179c..9110e91 100644
--- a/src/trace_processor/util/proto_to_args_parser.h
+++ b/src/trace_processor/util/proto_to_args_parser.h
@@ -83,9 +83,13 @@
virtual void AddDouble(const Key& key, double value) = 0;
virtual void AddPointer(const Key& key, const void* value) = 0;
virtual void AddBoolean(const Key& key, bool value) = 0;
- virtual void AddJson(const Key& key,
+ // Returns whether an entry was added or not.
+ virtual bool AddJson(const Key& key,
const protozero::ConstChars& value) = 0;
+ virtual size_t GetArrayEntryIndex(const std::string& array_key) = 0;
+ virtual size_t IncrementArrayEntryIndex(const std::string& array_key) = 0;
+
template <typename FieldMetadata>
typename FieldMetadata::cpp_field_type::Decoder* GetInternedMessage(
protozero::proto_utils::internal::FieldMetadataHelper<FieldMetadata>,
@@ -159,6 +163,37 @@
const std::vector<uint16_t>* allowed_fields,
Delegate& delegate);
+ struct ScopedNestedKeyContext {
+ public:
+ ~ScopedNestedKeyContext();
+ ScopedNestedKeyContext(ScopedNestedKeyContext&&);
+ ScopedNestedKeyContext(const ScopedNestedKeyContext&) = delete;
+ ScopedNestedKeyContext& operator=(const ScopedNestedKeyContext&) = delete;
+
+ const Key& key() const { return key_; }
+
+ // Reset this context, which sets |key_| to the state before the nested
+ // context was created.
+ void Reset();
+
+ private:
+ friend class ProtoToArgsParser;
+
+ ScopedNestedKeyContext(Key& old_value);
+
+ struct ScopedStringAppender;
+
+ Key& key_;
+ base::Optional<size_t> old_flat_key_length_ = base::nullopt;
+ base::Optional<size_t> old_key_length_ = base::nullopt;
+ };
+
+ // These methods can be called from parsing overrides to enter nested
+ // contexts. The contexts are left when the returned scope is destroyed or
+ // reset.
+ ScopedNestedKeyContext EnterDictionary(const std::string& key);
+ ScopedNestedKeyContext EnterArray(size_t index);
+
private:
base::Status ParseField(const FieldDescriptor& field_descriptor,
int repeated_field_number,
diff --git a/src/trace_processor/util/proto_to_args_parser_unittest.cc b/src/trace_processor/util/proto_to_args_parser_unittest.cc
index 89ee945..a8d17f6 100644
--- a/src/trace_processor/util/proto_to_args_parser_unittest.cc
+++ b/src/trace_processor/util/proto_to_args_parser_unittest.cc
@@ -21,9 +21,9 @@
#include "protos/perfetto/common/descriptor.pbzero.h"
#include "protos/perfetto/trace/track_event/source_location.pbzero.h"
#include "src/protozero/test/example_proto/test_messages.pbzero.h"
-#include "src/trace_processor/importers/common/trace_blob_view.h"
-#include "src/trace_processor/importers/proto/packet_sequence_state.h"
#include "src/trace_processor/test_messages.descriptor.h"
+#include "src/trace_processor/util/interned_message_view.h"
+#include "src/trace_processor/util/trace_blob_view.h"
#include "test/gtest_and_gmock.h"
#include <sstream>
@@ -92,13 +92,18 @@
args_.push_back(ss.str());
}
- void AddJson(const Key& key, const protozero::ConstChars& value) override {
+ bool AddJson(const Key& key, const protozero::ConstChars& value) override {
std::stringstream ss;
ss << key.flat_key << " " << key.key << " " << std::hex
<< value.ToStdString() << std::dec;
args_.push_back(ss.str());
+ return true;
}
+ size_t GetArrayEntryIndex(const std::string&) final { return 0; }
+
+ size_t IncrementArrayEntryIndex(const std::string&) final { return 0; }
+
InternedMessageView* GetInternedMessageView(uint32_t field_id,
uint64_t iid) override {
if (field_id != protos::pbzero::InternedData::kSourceLocationsFieldNumber)
diff --git a/src/trace_processor/util/trace_blob_view.h b/src/trace_processor/util/trace_blob_view.h
new file mode 100644
index 0000000..7d9c1a4
--- /dev/null
+++ b/src/trace_processor/util/trace_blob_view.h
@@ -0,0 +1,148 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SRC_TRACE_PROCESSOR_UTIL_TRACE_BLOB_VIEW_H_
+#define SRC_TRACE_PROCESSOR_UTIL_TRACE_BLOB_VIEW_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <limits>
+#include <memory>
+
+#include "perfetto/base/logging.h"
+
+namespace perfetto {
+namespace trace_processor {
+
+// This class is an equivalent of std::string_view for trace binary data.
+// The main difference is that this class has also shared ownership of a portion
+// of the raw trace.
+// The underlying buffer will be freed once all the TraceBlobViews that refer
+// to the same buffer have passed through the pipeline and been parsed.
+class TraceBlobView {
+ public:
+ TraceBlobView(std::unique_ptr<uint8_t[]> buffer, size_t offset, size_t length)
+ : shbuf_(SharedBuf(std::move(buffer))),
+ offset_(static_cast<uint32_t>(offset)),
+ length_(static_cast<uint32_t>(length)) {
+ PERFETTO_DCHECK(offset <= std::numeric_limits<uint32_t>::max());
+ PERFETTO_DCHECK(length <= std::numeric_limits<uint32_t>::max());
+ }
+
+ // Allow std::move().
+ TraceBlobView(TraceBlobView&&) noexcept = default;
+ TraceBlobView& operator=(TraceBlobView&&) = default;
+
+ // Disable implicit copy.
+ TraceBlobView(const TraceBlobView&) = delete;
+ TraceBlobView& operator=(const TraceBlobView&) = delete;
+
+ TraceBlobView slice(size_t offset, size_t length) const {
+ PERFETTO_DCHECK(offset + length <= offset_ + length_);
+ return TraceBlobView(shbuf_, offset, length);
+ }
+
+ bool operator==(const TraceBlobView& rhs) const {
+ return (shbuf_ == rhs.shbuf_) && (offset_ == rhs.offset_) &&
+ (length_ == rhs.length_);
+ }
+ bool operator!=(const TraceBlobView& rhs) const { return !(*this == rhs); }
+
+ inline const uint8_t* data() const { return start() + offset_; }
+
+ size_t offset_of(const uint8_t* data) const {
+ // When a field is size 0, data can be equal to start() + offset_ + length_.
+ PERFETTO_DCHECK(data >= start() && data <= (start() + offset_ + length_));
+ return static_cast<size_t>(data - start());
+ }
+
+ size_t length() const { return length_; }
+ size_t offset() const { return offset_; }
+
+ private:
+ // An equivalent to std::shared_ptr<uint8_t>, with the differnce that:
+ // - Supports array types, available for shared_ptr only in C++17.
+ // - Is not thread safe, which is not needed for our purposes.
+ class SharedBuf {
+ public:
+ explicit SharedBuf(std::unique_ptr<uint8_t[]> mem) {
+ rcbuf_ = new RefCountedBuf(std::move(mem));
+ }
+
+ SharedBuf(const SharedBuf& copy) : rcbuf_(copy.rcbuf_) {
+ PERFETTO_DCHECK(rcbuf_->refcount > 0);
+ rcbuf_->refcount++;
+ }
+
+ ~SharedBuf() {
+ if (!rcbuf_)
+ return;
+ PERFETTO_DCHECK(rcbuf_->refcount > 0);
+ if (--rcbuf_->refcount == 0) {
+ RefCountedBuf* rcbuf = rcbuf_;
+ rcbuf_ = nullptr;
+ delete rcbuf;
+ }
+ }
+
+ SharedBuf(SharedBuf&& other) noexcept {
+ rcbuf_ = other.rcbuf_;
+ other.rcbuf_ = nullptr;
+ }
+
+ SharedBuf& operator=(SharedBuf&& other) {
+ if (this != &other) {
+ // A bit of a ugly but pragmatic pattern to implement move assignment.
+ // First invoke the distructor and then invoke the move constructor
+ // inline via placement-new.
+ this->~SharedBuf();
+ new (this) SharedBuf(std::move(other));
+ }
+ return *this;
+ }
+
+ bool operator==(const SharedBuf& x) const { return x.rcbuf_ == rcbuf_; }
+ bool operator!=(const SharedBuf& x) const { return !(x == *this); }
+ const uint8_t* data() const { return rcbuf_->mem.get(); }
+
+ private:
+ struct RefCountedBuf {
+ explicit RefCountedBuf(std::unique_ptr<uint8_t[]> buf)
+ : refcount(1), mem(std::move(buf)) {}
+ int refcount;
+ std::unique_ptr<uint8_t[]> mem;
+ };
+
+ RefCountedBuf* rcbuf_ = nullptr;
+ };
+
+ inline const uint8_t* start() const { return shbuf_.data(); }
+
+ TraceBlobView(SharedBuf b, size_t o, size_t l)
+ : shbuf_(b),
+ offset_(static_cast<uint32_t>(o)),
+ length_(static_cast<uint32_t>(l)) {}
+
+ SharedBuf shbuf_;
+ uint32_t offset_;
+ uint32_t length_; // Measured from |offset_|, not from |data()|.
+};
+
+} // namespace trace_processor
+} // namespace perfetto
+
+#endif // SRC_TRACE_PROCESSOR_UTIL_TRACE_BLOB_VIEW_H_