Deserialization support for extensions in trace processor
Bug: 156904196
Change-Id: I6dcf8dc9241b0d875434e0608c6351318af948c5
diff --git a/Android.bp b/Android.bp
index bc5ed66..8708851 100644
--- a/Android.bp
+++ b/Android.bp
@@ -4610,6 +4610,7 @@
genrule {
name: "perfetto_protos_perfetto_trace_non_minimal_cpp_gen",
srcs: [
+ "protos/perfetto/trace/extension_descriptor.proto",
"protos/perfetto/trace/test_event.proto",
"protos/perfetto/trace/trace.proto",
"protos/perfetto/trace/trace_packet.proto",
@@ -4621,6 +4622,7 @@
],
cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --plugin=protoc-gen-plugin=$(location perfetto_src_protozero_protoc_plugin_cppgen_plugin) --plugin_out=wrapper_namespace=gen:$(genDir)/external/perfetto/ $(in)",
out: [
+ "external/perfetto/protos/perfetto/trace/extension_descriptor.gen.cc",
"external/perfetto/protos/perfetto/trace/test_event.gen.cc",
"external/perfetto/protos/perfetto/trace/trace.gen.cc",
"external/perfetto/protos/perfetto/trace/trace_packet.gen.cc",
@@ -4632,6 +4634,7 @@
genrule {
name: "perfetto_protos_perfetto_trace_non_minimal_cpp_gen_headers",
srcs: [
+ "protos/perfetto/trace/extension_descriptor.proto",
"protos/perfetto/trace/test_event.proto",
"protos/perfetto/trace/trace.proto",
"protos/perfetto/trace/trace_packet.proto",
@@ -4643,6 +4646,7 @@
],
cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --plugin=protoc-gen-plugin=$(location perfetto_src_protozero_protoc_plugin_cppgen_plugin) --plugin_out=wrapper_namespace=gen:$(genDir)/external/perfetto/ $(in)",
out: [
+ "external/perfetto/protos/perfetto/trace/extension_descriptor.gen.h",
"external/perfetto/protos/perfetto/trace/test_event.gen.h",
"external/perfetto/protos/perfetto/trace/trace.gen.h",
"external/perfetto/protos/perfetto/trace/trace_packet.gen.h",
@@ -4658,6 +4662,7 @@
genrule {
name: "perfetto_protos_perfetto_trace_non_minimal_lite_gen",
srcs: [
+ "protos/perfetto/trace/extension_descriptor.proto",
"protos/perfetto/trace/test_event.proto",
"protos/perfetto/trace/trace.proto",
"protos/perfetto/trace/trace_packet.proto",
@@ -4668,6 +4673,7 @@
],
cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --cpp_out=lite=true:$(genDir)/external/perfetto/ $(in)",
out: [
+ "external/perfetto/protos/perfetto/trace/extension_descriptor.pb.cc",
"external/perfetto/protos/perfetto/trace/test_event.pb.cc",
"external/perfetto/protos/perfetto/trace/trace.pb.cc",
"external/perfetto/protos/perfetto/trace/trace_packet.pb.cc",
@@ -4679,6 +4685,7 @@
genrule {
name: "perfetto_protos_perfetto_trace_non_minimal_lite_gen_headers",
srcs: [
+ "protos/perfetto/trace/extension_descriptor.proto",
"protos/perfetto/trace/test_event.proto",
"protos/perfetto/trace/trace.proto",
"protos/perfetto/trace/trace_packet.proto",
@@ -4689,6 +4696,7 @@
],
cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --cpp_out=lite=true:$(genDir)/external/perfetto/ $(in)",
out: [
+ "external/perfetto/protos/perfetto/trace/extension_descriptor.pb.h",
"external/perfetto/protos/perfetto/trace/test_event.pb.h",
"external/perfetto/protos/perfetto/trace/trace.pb.h",
"external/perfetto/protos/perfetto/trace/trace_packet.pb.h",
@@ -4704,6 +4712,7 @@
genrule {
name: "perfetto_protos_perfetto_trace_non_minimal_zero_gen",
srcs: [
+ "protos/perfetto/trace/extension_descriptor.proto",
"protos/perfetto/trace/test_event.proto",
"protos/perfetto/trace/trace.proto",
"protos/perfetto/trace/trace_packet.proto",
@@ -4715,6 +4724,7 @@
],
cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --plugin=protoc-gen-plugin=$(location protozero_plugin) --plugin_out=wrapper_namespace=pbzero:$(genDir)/external/perfetto/ $(in)",
out: [
+ "external/perfetto/protos/perfetto/trace/extension_descriptor.pbzero.cc",
"external/perfetto/protos/perfetto/trace/test_event.pbzero.cc",
"external/perfetto/protos/perfetto/trace/trace.pbzero.cc",
"external/perfetto/protos/perfetto/trace/trace_packet.pbzero.cc",
@@ -4726,6 +4736,7 @@
genrule {
name: "perfetto_protos_perfetto_trace_non_minimal_zero_gen_headers",
srcs: [
+ "protos/perfetto/trace/extension_descriptor.proto",
"protos/perfetto/trace/test_event.proto",
"protos/perfetto/trace/trace.proto",
"protos/perfetto/trace/trace_packet.proto",
@@ -4737,6 +4748,7 @@
],
cmd: "mkdir -p $(genDir)/external/perfetto/ && $(location aprotoc) --proto_path=external/perfetto --plugin=protoc-gen-plugin=$(location protozero_plugin) --plugin_out=wrapper_namespace=pbzero:$(genDir)/external/perfetto/ $(in)",
out: [
+ "external/perfetto/protos/perfetto/trace/extension_descriptor.pbzero.h",
"external/perfetto/protos/perfetto/trace/test_event.pbzero.h",
"external/perfetto/protos/perfetto/trace/trace.pbzero.h",
"external/perfetto/protos/perfetto/trace/trace_packet.pbzero.h",
diff --git a/BUILD b/BUILD
index 1f74e4b..b3993f6 100644
--- a/BUILD
+++ b/BUILD
@@ -2375,6 +2375,7 @@
perfetto_proto_library(
name = "protos_perfetto_trace_non_minimal_protos",
srcs = [
+ "protos/perfetto/trace/extension_descriptor.proto",
"protos/perfetto/trace/test_event.proto",
"protos/perfetto/trace/trace.proto",
"protos/perfetto/trace/trace_packet.proto",
diff --git a/docs/design-docs/extensions.md b/docs/design-docs/extensions.md
new file mode 100644
index 0000000..be8a466
--- /dev/null
+++ b/docs/design-docs/extensions.md
@@ -0,0 +1,67 @@
+Extensions: adding new types to traces
+======================================
+
+NOTE: **extensions are work-in-progress and are not ready to be used at the
+moment**
+
+Currently, it is not possible to add new types to traces while using Perfetto
+without modifying Perfetto proto message definitions upstream.
+
+This page describes ongoing work to use [protobuf
+extensions](https://developers.google.com/protocol-buffers/docs/overview#extensions)
+in order to make it possible to define new typed messages outside of the
+Perfetto repository.
+
+Protozero support
+-----------------
+
+Perfetto uses its own implementation of code generation for protocol buffer
+messages called [Protozero](/docs/design-docs/protozero.md), which is not a
+full-fledged protobuf implementation. Implementation of extensions is fairly
+limited, and all extensions are supposed to be nested inside a message that is
+used in order to provide the class name for generated code.
+
+For example,
+
+ message MyEvent {
+ extend TrackEvent {
+ optional string custom_string = 1000;
+ }
+ }
+
+Is going to generate a subclass of `TrackEvent` called `MyEvent`, that is going
+to have a new method to set `custom_string` in addition to all other protobuf
+fields defined in `TrackEvent`.
+
+Deserialization
+---------------
+
+When analyzing traces, protos are not used directly and instead are parsed into
+database, which can be queried by SQL. In order to make it possible, Perfetto
+has to know field descriptors (the binary representation of the extended proto
+schema) of the extensions. Currently, the only way to do that is to add an
+[ExtensionDescriptor
+packet](reference/trace-packet-proto.autogen#ExtensionDescriptor). In the
+future, there is going to be a way to specify protobuf extensions at compile
+time in order to be able to avoid this overhead in every single trace.
+
+However, an ability to specify extension descriptors in the trace itself will
+still be useful in order to be able to use a pre-compiled trace processor in the
+UI when adding new typed messages during local development.
+
+Deserialization of protobuf extension are supported only for TrackEvent message
+at the moment, and is implemented in trace processor via ProtoToArgsUtils class.
+The extensions will appear in args table, similar to other trace event
+arguments.
+
+Testing extensions support inside Perfetto
+------------------------------------------
+
+Perfetto trace processor is mostly tested by integration tests, where input
+traces are specified most frequently in textproto format. Textproto format
+supports extensions, but the parser has to be aware of all the extensions used.
+In order to make it possible, all the extensions that are used in integration
+tests have to be specified in the `test_extensions.proto` file. Since this file
+is only used in the testing harness and is parsed by protoc, it does not have to
+adhere to the convention of all extensions being inside a wrapper message, which
+helps with making extension identifiers more concise.
diff --git a/protos/perfetto/trace/BUILD.gn b/protos/perfetto/trace/BUILD.gn
index b644aaf..07ee972 100644
--- a/protos/perfetto/trace/BUILD.gn
+++ b/protos/perfetto/trace/BUILD.gn
@@ -31,6 +31,7 @@
"test_event.proto",
"trace_packet.proto",
"trace.proto",
+ "extension_descriptor.proto",
]
proto_sources_minimal = [
@@ -109,7 +110,7 @@
perfetto_proto_library("descriptor") {
proto_generators = [ "descriptor" ]
generate_descriptor = "trace.descriptor"
- sources = [ "trace.proto" ]
+ sources = [ "perfetto_trace.proto" ]
}
}
diff --git a/protos/perfetto/trace/extension_descriptor.proto b/protos/perfetto/trace/extension_descriptor.proto
new file mode 100644
index 0000000..180e4d9
--- /dev/null
+++ b/protos/perfetto/trace/extension_descriptor.proto
@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto2";
+
+package perfetto.protos;
+
+import "protos/perfetto/common/descriptor.proto";
+
+// This message contains descriptors used to parse extension fields of
+// TrackEvent.
+//
+// See docs/design-docs/extensions.md for more details.
+message ExtensionDescriptor {
+ repeated FileDescriptorProto extension_file = 1;
+}
diff --git a/protos/perfetto/trace/perfetto_trace.proto b/protos/perfetto/trace/perfetto_trace.proto
index c62389f..9782f44 100644
--- a/protos/perfetto/trace/perfetto_trace.proto
+++ b/protos/perfetto/trace/perfetto_trace.proto
@@ -2009,6 +2009,201 @@
// End of protos/perfetto/trace/clock_snapshot.proto
+// Begin of protos/perfetto/common/descriptor.proto
+
+
+// The protocol compiler can output a FileDescriptorSet containing the .proto
+// files it parses.
+message FileDescriptorSet {
+ repeated FileDescriptorProto file = 1;
+}
+
+// Describes a complete .proto file.
+message FileDescriptorProto {
+ // file name, relative to root of source tree
+ optional string name = 1;
+ // e.g. "foo", "foo.bar", etc.
+ optional string package = 2;
+
+ // Names of files imported by this file.
+ repeated string dependency = 3;
+ // Indexes of the public imported files in the dependency list above.
+ repeated int32 public_dependency = 10;
+ // Indexes of the weak imported files in the dependency list.
+ // For Google-internal migration only. Do not use.
+ repeated int32 weak_dependency = 11;
+
+ // All top-level definitions in this file.
+ repeated DescriptorProto message_type = 4;
+ repeated EnumDescriptorProto enum_type = 5;
+ repeated FieldDescriptorProto extension = 7;
+
+ reserved 6;
+ reserved 8;
+ reserved 9;
+ reserved 12;
+}
+
+// Describes a message type.
+message DescriptorProto {
+ optional string name = 1;
+
+ repeated FieldDescriptorProto field = 2;
+ repeated FieldDescriptorProto extension = 6;
+
+ repeated DescriptorProto nested_type = 3;
+ repeated EnumDescriptorProto enum_type = 4;
+
+ reserved 5;
+
+ repeated OneofDescriptorProto oneof_decl = 8;
+
+ reserved 7;
+
+ // Range of reserved tag numbers. Reserved tag numbers may not be used by
+ // fields or extension ranges in the same message. Reserved ranges may
+ // not overlap.
+ message ReservedRange {
+ // Inclusive.
+ optional int32 start = 1;
+ // Exclusive.
+ optional int32 end = 2;
+ }
+ repeated ReservedRange reserved_range = 9;
+ // Reserved field names, which may not be used by fields in the same message.
+ // A given name may only be reserved once.
+ repeated string reserved_name = 10;
+}
+
+// Describes a field within a message.
+message FieldDescriptorProto {
+ enum Type {
+ // 0 is reserved for errors.
+ // Order is weird for historical reasons.
+ TYPE_DOUBLE = 1;
+ TYPE_FLOAT = 2;
+ // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if
+ // negative values are likely.
+ TYPE_INT64 = 3;
+ TYPE_UINT64 = 4;
+ // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if
+ // negative values are likely.
+ TYPE_INT32 = 5;
+ TYPE_FIXED64 = 6;
+ TYPE_FIXED32 = 7;
+ TYPE_BOOL = 8;
+ TYPE_STRING = 9;
+ // Tag-delimited aggregate.
+ // Group type is deprecated and not supported in proto3. However, Proto3
+ // implementations should still be able to parse the group wire format and
+ // treat group fields as unknown fields.
+ TYPE_GROUP = 10;
+ // Length-delimited aggregate.
+ TYPE_MESSAGE = 11;
+
+ // New in version 2.
+ TYPE_BYTES = 12;
+ TYPE_UINT32 = 13;
+ TYPE_ENUM = 14;
+ TYPE_SFIXED32 = 15;
+ TYPE_SFIXED64 = 16;
+ // Uses ZigZag encoding.
+ TYPE_SINT32 = 17;
+ // Uses ZigZag encoding.
+ TYPE_SINT64 = 18;
+ };
+
+ enum Label {
+ // 0 is reserved for errors
+ LABEL_OPTIONAL = 1;
+ LABEL_REQUIRED = 2;
+ LABEL_REPEATED = 3;
+ };
+
+ optional string name = 1;
+ optional int32 number = 3;
+ optional Label label = 4;
+
+ // If type_name is set, this need not be set. If both this and type_name
+ // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP.
+ optional Type type = 5;
+
+ // For message and enum types, this is the name of the type. If the name
+ // starts with a '.', it is fully-qualified. Otherwise, C++-like scoping
+ // rules are used to find the type (i.e. first the nested types within this
+ // message are searched, then within the parent, on up to the root
+ // namespace).
+ optional string type_name = 6;
+
+ // For extensions, this is the name of the type being extended. It is
+ // resolved in the same manner as type_name.
+ optional string extendee = 2;
+
+ // For numeric types, contains the original text representation of the value.
+ // For booleans, "true" or "false".
+ // For strings, contains the default text contents (not escaped in any way).
+ // For bytes, contains the C escaped value. All bytes >= 128 are escaped.
+ // TODO(kenton): Base-64 encode?
+ optional string default_value = 7;
+
+ // If set, gives the index of a oneof in the containing type's oneof_decl
+ // list. This field is a member of that oneof.
+ optional int32 oneof_index = 9;
+
+ reserved 10;
+
+ reserved 8;
+}
+
+// Describes a oneof.
+message OneofDescriptorProto {
+ optional string name = 1;
+ optional OneofOptions options = 2;
+}
+
+// Describes an enum type.
+message EnumDescriptorProto {
+ optional string name = 1;
+
+ repeated EnumValueDescriptorProto value = 2;
+
+ reserved 3;
+ reserved 4;
+
+ // Reserved enum value names, which may not be reused. A given name may only
+ // be reserved once.
+ repeated string reserved_name = 5;
+}
+
+// Describes a value within an enum.
+message EnumValueDescriptorProto {
+ optional string name = 1;
+ optional int32 number = 2;
+
+ reserved 3;
+}
+
+message OneofOptions {
+ reserved 999;
+
+ // Clients can define custom options in extensions of this message. See above.
+ extensions 1000 to max;
+}
+
+// End of protos/perfetto/common/descriptor.proto
+
+// Begin of protos/perfetto/trace/extension_descriptor.proto
+
+// This message contains descriptors used to parse extension fields of
+// TrackEvent.
+//
+// See docs/design-docs/extensions.md for more details.
+message ExtensionDescriptor {
+ repeated FileDescriptorProto extension_file = 1;
+}
+
+// End of protos/perfetto/trace/extension_descriptor.proto
+
// Begin of protos/perfetto/trace/filesystem/inode_file_map.proto
// Represents the mapping between inode numbers in a block device and their path
@@ -5859,6 +6054,21 @@
optional ChromeFrameReporter chrome_frame_reporter = 32;
// New argument types go here :)
+ // Extension range for typed events defined externally.
+ // See docs/design-docs/extensions.md for more details.
+ //
+ // Extension support is work-in-progress, in the future the way to reserve a
+ // subrange for a particular project will be described here and in the design
+ // document linked above.
+ //
+ // Contact perfetto-dev@googlegroups.com if you are interested in a subrange
+ // for your project.
+
+ // Extension range for future use.
+ extensions 1000 to 9899;
+ // Reserved for Perfetto unit and integration tests.
+ extensions 9900 to 10000;
+
// ---------------------------------------------------------------------------
// Deprecated / legacy event fields, which will be removed in the future:
// ---------------------------------------------------------------------------
@@ -7156,7 +7366,7 @@
// See the [Buffers and Dataflow](/docs/concepts/buffers.md) doc for details.
//
// Next reserved id: 13 (up to 15).
-// Next id: 72.
+// Next id: 73.
message TracePacket {
// The timestamp of the TracePacket.
// By default this timestamps refers to the trace clock (CLOCK_BOOTTIME on
@@ -7183,7 +7393,7 @@
TrackEvent track_event = 11;
// IDs up to 15 are reserved. They take only one byte to encode their
- // preamble so should be used for freqeuent events.
+ // preamble so should be used for frequent events.
TraceConfig trace_config = 33;
FtraceStats ftrace_stats = 34;
@@ -7241,6 +7451,13 @@
// sizes) should be less than 512KB.
bytes compressed_packets = 50;
+ // Data sources can extend the trace proto with custom extension protos (see
+ // docs/design-docs/extensions.md). When they do that, the descriptor of
+ // their extension proto descriptor is serialized in this packet. This
+ // allows trace_processor to deserialize extended messages using reflection
+ // even if the extension proto is not checked in the Perfetto repo.
+ ExtensionDescriptor extension_descriptor = 72;
+
// This field is only used for testing.
// In previous versions of this proto this field had the id 268435455
// This caused many problems:
@@ -7342,3 +7559,19 @@
}
// End of protos/perfetto/trace/trace.proto
+
+// Begin of protos/perfetto/trace/test_extensions.proto
+
+// Extensions for TrackEvent used for integration testing. This proto file is
+// compiled to descriptor and is used in tools/diff_test_trace_processor.py.
+//
+// See docs/design-docs/extensions.md for more details.
+message TestExtension {
+ extend TrackEvent {
+ optional string string_extension_for_testing = 9900;
+ repeated int32 int_extension_for_testing = 9901;
+ optional string omitted_extension_for_testing = 9902;
+ }
+}
+
+// End of protos/perfetto/trace/test_extensions.proto
diff --git a/protos/perfetto/trace/test_extensions.proto b/protos/perfetto/trace/test_extensions.proto
new file mode 100644
index 0000000..eae3a7d
--- /dev/null
+++ b/protos/perfetto/trace/test_extensions.proto
@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto2";
+
+import "protos/perfetto/trace/track_event/track_event.proto";
+
+package perfetto.protos;
+
+// Extensions for TrackEvent used for integration testing. This proto file is
+// compiled to descriptor and is used in tools/diff_test_trace_processor.py.
+//
+// See docs/design-docs/extensions.md for more details.
+message TestExtension {
+ extend TrackEvent {
+ optional string string_extension_for_testing = 9900;
+ repeated int32 int_extension_for_testing = 9901;
+ optional string omitted_extension_for_testing = 9902;
+ }
+}
diff --git a/protos/perfetto/trace/trace_packet.proto b/protos/perfetto/trace/trace_packet.proto
index 01a7c5d..431ad4f 100644
--- a/protos/perfetto/trace/trace_packet.proto
+++ b/protos/perfetto/trace/trace_packet.proto
@@ -18,6 +18,7 @@
import "protos/perfetto/common/trace_stats.proto";
import "protos/perfetto/config/trace_config.proto";
+import "protos/perfetto/trace/extension_descriptor.proto";
import "protos/perfetto/trace/android/android_log.proto";
import "protos/perfetto/trace/android/gpu_mem_event.proto";
import "protos/perfetto/trace/android/graphics_frame_event.proto";
@@ -79,7 +80,7 @@
// See the [Buffers and Dataflow](/docs/concepts/buffers.md) doc for details.
//
// Next reserved id: 13 (up to 15).
-// Next id: 72.
+// Next id: 73.
message TracePacket {
// The timestamp of the TracePacket.
// By default this timestamps refers to the trace clock (CLOCK_BOOTTIME on
@@ -106,7 +107,7 @@
TrackEvent track_event = 11;
// IDs up to 15 are reserved. They take only one byte to encode their
- // preamble so should be used for freqeuent events.
+ // preamble so should be used for frequent events.
TraceConfig trace_config = 33;
FtraceStats ftrace_stats = 34;
@@ -164,6 +165,13 @@
// sizes) should be less than 512KB.
bytes compressed_packets = 50;
+ // Data sources can extend the trace proto with custom extension protos (see
+ // docs/design-docs/extensions.md). When they do that, the descriptor of
+ // their extension proto descriptor is serialized in this packet. This
+ // allows trace_processor to deserialize extended messages using reflection
+ // even if the extension proto is not checked in the Perfetto repo.
+ ExtensionDescriptor extension_descriptor = 72;
+
// This field is only used for testing.
// In previous versions of this proto this field had the id 268435455
// This caused many problems:
diff --git a/protos/perfetto/trace/track_event/track_event.proto b/protos/perfetto/trace/track_event/track_event.proto
index 581e5d8..ee0db5f 100644
--- a/protos/perfetto/trace/track_event/track_event.proto
+++ b/protos/perfetto/trace/track_event/track_event.proto
@@ -201,6 +201,21 @@
optional ChromeFrameReporter chrome_frame_reporter = 32;
// New argument types go here :)
+ // Extension range for typed events defined externally.
+ // See docs/design-docs/extensions.md for more details.
+ //
+ // Extension support is work-in-progress, in the future the way to reserve a
+ // subrange for a particular project will be described here and in the design
+ // document linked above.
+ //
+ // Contact perfetto-dev@googlegroups.com if you are interested in a subrange
+ // for your project.
+
+ // Extension range for future use.
+ extensions 1000 to 9899;
+ // Reserved for Perfetto unit and integration tests.
+ extensions 9900 to 10000;
+
// ---------------------------------------------------------------------------
// Deprecated / legacy event fields, which will be removed in the future:
// ---------------------------------------------------------------------------
diff --git a/src/trace_processor/importers/proto/args_table_utils.cc b/src/trace_processor/importers/proto/args_table_utils.cc
index b1dd2e0..2a583ad 100644
--- a/src/trace_processor/importers/proto/args_table_utils.cc
+++ b/src/trace_processor/importers/proto/args_table_utils.cc
@@ -15,6 +15,8 @@
*/
#include "src/trace_processor/importers/proto/args_table_utils.h"
+#include "protos/perfetto/common/descriptor.pbzero.h"
+#include "src/trace_processor/util/descriptors.h"
#include "protos/perfetto/common/descriptor.pbzero.h"
#include "protos/perfetto/trace/interned_data/interned_data.pbzero.h"
@@ -38,6 +40,23 @@
proto_descriptor_array_size);
}
+void ProtoToArgsTable::AddExtensionFileDescriptor(
+ const uint8_t* descriptor_array,
+ size_t descriptor_array_size) {
+ // TODO(ddrone): parse message and enum definitions as well.
+ protos::pbzero::FileDescriptorProto::Decoder file_decoder(
+ descriptor_array, descriptor_array_size);
+ for (auto type = file_decoder.message_type(); type; ++type) {
+ protos::pbzero::DescriptorProto::Decoder message_decoder(*type);
+ for (auto field = message_decoder.extension(); field; ++field) {
+ protos::pbzero::FieldDescriptorProto::Decoder decoder(*field);
+
+ auto descriptor = CreateFieldFromDecoder(decoder);
+ extension_fields_.emplace(descriptor.number(), descriptor);
+ }
+ }
+}
+
util::Status ProtoToArgsTable::InternProtoFieldsIntoArgsTable(
const protozero::ConstBytes& cb,
const std::string& type,
@@ -51,11 +70,38 @@
auto descriptor = pool_.descriptors()[*idx];
+ std::unordered_map<size_t, int> repeated_field_index;
+
protozero::ProtoDecoder decoder(cb);
for (protozero::Field f = decoder.ReadField(); f.valid();
f = decoder.ReadField()) {
auto it = std::find(fields.begin(), fields.end(), f.id());
if (it == fields.end()) {
+ // Unknown field in the base message. Checking whether it is a known
+ // extension field.
+ auto extensions_it = extension_fields_.find(f.id());
+ if (extensions_it == extension_fields_.end()) {
+ continue;
+ }
+
+ auto field = extensions_it->second;
+ if (field.type() ==
+ perfetto::protos::pbzero::FieldDescriptorProto::TYPE_MESSAGE) {
+ return util::Status(
+ "Nested messages in extensions are not yet supported");
+ }
+ ParsingOverrideState state{context_, sequence_state};
+
+ auto status = InternFieldIntoArgsTable(
+ field, repeated_field_index[f.id()], state, inserter, f);
+ if (!status.ok()) {
+ return status;
+ }
+
+ if (field.is_repeated()) {
+ repeated_field_index[f.id()]++;
+ }
+
continue;
}
@@ -86,7 +132,61 @@
flat_key_prefix_.assign(key_prefix);
ParsingOverrideState state{context_, sequence_state};
- return InternProtoIntoArgsTableInternal(cb, type, inserter, state);
+ auto result = InternProtoIntoArgsTableInternal(cb, type, inserter, state);
+
+ key_prefix_.clear();
+ flat_key_prefix_.clear();
+ return result;
+}
+
+util::Status ProtoToArgsTable::InternFieldIntoArgsTable(
+ const FieldDescriptor& field_descriptor,
+ int repeated_field_number,
+ ParsingOverrideState state,
+ ArgsTracker::BoundInserter* inserter,
+ protozero::Field field) {
+ std::string prefix_part = field_descriptor.name();
+ if (field_descriptor.is_repeated()) {
+ std::string number = std::to_string(repeated_field_number);
+ prefix_part.reserve(prefix_part.length() + number.length() + 2);
+ prefix_part.append("[");
+ prefix_part.append(number);
+ prefix_part.append("]");
+ }
+
+ // In the args table we build up message1.message2.field1 as the column
+ // name. This will append the ".field1" suffix to |key_prefix| and then
+ // remove it when it goes out of scope.
+ ScopedStringAppender scoped_prefix(prefix_part, &key_prefix_);
+ ScopedStringAppender scoped_flat_key_prefix(field_descriptor.name(),
+ &flat_key_prefix_);
+
+ // If we have an override parser then use that instead and move onto the
+ // next loop.
+ auto it = FindOverride(key_prefix_);
+ if (it != overrides_.end()) {
+ if (it->second(state, field, inserter)) {
+ return util::OkStatus();
+ }
+ }
+
+ // If this is not a message we can just immediately add the column name and
+ // get the value out of |field|. However if it is a message we need to
+ // recurse into it.
+ if (field_descriptor.type() ==
+ protos::pbzero::FieldDescriptorProto::TYPE_MESSAGE) {
+ return InternProtoIntoArgsTableInternal(
+ field.as_bytes(), field_descriptor.resolved_type_name(), inserter,
+ state);
+ }
+
+ const StringId key_id =
+ state.context->storage->InternString(base::StringView(key_prefix_));
+ const StringId flat_key_id =
+ state.context->storage->InternString(base::StringView(flat_key_prefix_));
+ inserter->AddArg(flat_key_id, key_id,
+ ConvertProtoTypeToVariadic(field_descriptor, field, state));
+ return util::OkStatus();
}
util::Status ProtoToArgsTable::InternProtoIntoArgsTableInternal(
@@ -119,53 +219,12 @@
const auto& field_descriptor =
proto_descriptor.fields()[*opt_field_descriptor_idx];
- std::string prefix_part = field_descriptor.name();
+ InternFieldIntoArgsTable(field_descriptor,
+ repeated_field_index[*opt_field_descriptor_idx],
+ state, inserter, field);
if (field_descriptor.is_repeated()) {
- std::string number =
- std::to_string(repeated_field_index[*opt_field_descriptor_idx]);
- prefix_part.reserve(prefix_part.length() + number.length() + 2);
- prefix_part.append("[");
- prefix_part.append(number);
- prefix_part.append("]");
repeated_field_index[*opt_field_descriptor_idx]++;
}
-
- // In the args table we build up message1.message2.field1 as the column
- // name. This will append the ".field1" suffix to |key_prefix| and then
- // remove it when it goes out of scope.
- ScopedStringAppender scoped_prefix(prefix_part, &key_prefix_);
- ScopedStringAppender scoped_flat_key_prefix(field_descriptor.name(),
- &flat_key_prefix_);
-
- // If we have an override parser then use that instead and move onto the
- // next loop.
- auto it = FindOverride(key_prefix_);
- if (it != overrides_.end()) {
- if (it->second(state, field, inserter)) {
- continue;
- }
- }
-
- // If this is not a message we can just immediately add the column name and
- // get the value out of |field|. However if it is a message we need to
- // recurse into it.
- if (field_descriptor.type() ==
- protos::pbzero::FieldDescriptorProto::TYPE_MESSAGE) {
- auto status = InternProtoIntoArgsTableInternal(
- field.as_bytes(), field_descriptor.resolved_type_name(), inserter,
- state);
- if (!status.ok()) {
- return status;
- }
- } else {
- const StringId key_id =
- state.context->storage->InternString(base::StringView(key_prefix_));
- const StringId flat_key_id = state.context->storage->InternString(
- base::StringView(flat_key_prefix_));
- inserter->AddArg(
- flat_key_id, key_id,
- ConvertProtoTypeToVariadic(field_descriptor, field, state));
- }
}
PERFETTO_DCHECK(decoder.bytes_left() == 0);
return util::OkStatus();
diff --git a/src/trace_processor/importers/proto/args_table_utils.h b/src/trace_processor/importers/proto/args_table_utils.h
index c103a1f..445224d 100644
--- a/src/trace_processor/importers/proto/args_table_utils.h
+++ b/src/trace_processor/importers/proto/args_table_utils.h
@@ -112,6 +112,11 @@
util::Status AddProtoFileDescriptor(const uint8_t* proto_descriptor_array,
size_t proto_descriptor_array_size);
+ // Register root message extension file descriptor. Support for extensions of
+ // nested messages is not available.
+ void AddExtensionFileDescriptor(const uint8_t* descriptor_array,
+ size_t descriptor_array_size);
+
// Given a view of bytes that represent a serialized protozero message of
// |type| we will parse each field into the Args table using RowId |row|,
// adding |key_prefix| in front of each name (can be an empty string if no
@@ -175,6 +180,12 @@
ArgsTracker::BoundInserter* inserter,
ParsingOverrideState state);
+ util::Status InternFieldIntoArgsTable(const FieldDescriptor& field_descriptor,
+ int repeated_field_number,
+ ParsingOverrideState state,
+ ArgsTracker::BoundInserter* inserter,
+ protozero::Field field);
+
using OverrideIterator =
std::vector<std::pair<std::string, ParsingOverride>>::iterator;
OverrideIterator FindOverride(const std::string& field);
@@ -188,6 +199,7 @@
std::string key_prefix_;
std::string flat_key_prefix_;
TraceProcessorContext* context_;
+ std::unordered_map<int, FieldDescriptor> extension_fields_;
};
} // namespace trace_processor
diff --git a/src/trace_processor/importers/proto/proto_trace_parser_unittest.cc b/src/trace_processor/importers/proto/proto_trace_parser_unittest.cc
index cf1056a..6610f78 100644
--- a/src/trace_processor/importers/proto/proto_trace_parser_unittest.cc
+++ b/src/trace_processor/importers/proto/proto_trace_parser_unittest.cc
@@ -28,6 +28,7 @@
#include "src/trace_processor/importers/common/track_tracker.h"
#include "src/trace_processor/importers/default_modules.h"
#include "src/trace_processor/importers/ftrace/sched_event_tracker.h"
+#include "src/trace_processor/importers/proto/args_table_utils.h"
#include "src/trace_processor/importers/proto/metadata_tracker.h"
#include "src/trace_processor/importers/proto/proto_trace_parser.h"
#include "src/trace_processor/importers/proto/stack_profile_tracker.h"
@@ -219,6 +220,7 @@
clock_ = new ClockTracker(&context_);
context_.clock_tracker.reset(clock_);
context_.sorter.reset(new TraceSorter(CreateParser(), 0 /*window size*/));
+ context_.proto_to_args_table_.reset(new ProtoToArgsTable(&context_));
RegisterDefaultModules(&context_);
RegisterAdditionalModules(&context_);
diff --git a/src/trace_processor/importers/proto/proto_trace_tokenizer.cc b/src/trace_processor/importers/proto/proto_trace_tokenizer.cc
index da3baa5..b33379f 100644
--- a/src/trace_processor/importers/proto/proto_trace_tokenizer.cc
+++ b/src/trace_processor/importers/proto/proto_trace_tokenizer.cc
@@ -31,6 +31,7 @@
#include "src/trace_processor/importers/common/track_tracker.h"
#include "src/trace_processor/importers/ftrace/ftrace_module.h"
#include "src/trace_processor/importers/gzip/gzip_utils.h"
+#include "src/trace_processor/importers/proto/args_table_utils.h"
#include "src/trace_processor/importers/proto/packet_sequence_state.h"
#include "src/trace_processor/importers/proto/proto_incremental_state.h"
#include "src/trace_processor/storage/stats.h"
@@ -40,6 +41,7 @@
#include "protos/perfetto/common/builtin_clock.pbzero.h"
#include "protos/perfetto/config/trace_config.pbzero.h"
#include "protos/perfetto/trace/clock_snapshot.pbzero.h"
+#include "protos/perfetto/trace/extension_descriptor.pbzero.h"
#include "protos/perfetto/trace/profiling/profile_common.pbzero.h"
#include "protos/perfetto/trace/trace.pbzero.h"
#include "protos/perfetto/trace/trace_packet.pbzero.h"
@@ -181,6 +183,19 @@
return util::OkStatus();
}
+util::Status ProtoTraceTokenizer::ParseExtensionDescriptor(
+ ConstBytes descriptor) {
+ protos::pbzero::ExtensionDescriptor::Decoder decoder(descriptor.data,
+ descriptor.size);
+
+ for (auto extension = decoder.extension_file(); extension; extension++) {
+ context_->proto_to_args_table_->AddExtensionFileDescriptor(
+ extension->data(), extension->size());
+ }
+
+ return util::OkStatus();
+}
+
util::Status ProtoTraceTokenizer::ParsePacket(TraceBlobView packet) {
protos::pbzero::TracePacket::Decoder decoder(packet.data(), packet.length());
if (PERFETTO_UNLIKELY(decoder.bytes_left()))
@@ -219,6 +234,10 @@
decoder.trusted_packet_sequence_id());
}
+ if (decoder.has_extension_descriptor()) {
+ return ParseExtensionDescriptor(decoder.extension_descriptor());
+ }
+
if (decoder.sequence_flags() &
protos::pbzero::TracePacket::SEQ_NEEDS_INCREMENTAL_STATE) {
if (!seq_id) {
diff --git a/src/trace_processor/importers/proto/proto_trace_tokenizer.h b/src/trace_processor/importers/proto/proto_trace_tokenizer.h
index 62ddb39..1d9a560 100644
--- a/src/trace_processor/importers/proto/proto_trace_tokenizer.h
+++ b/src/trace_processor/importers/proto/proto_trace_tokenizer.h
@@ -79,6 +79,7 @@
incremental_state.reset(new ProtoIncrementalState(context_));
return incremental_state->GetOrCreateStateForPacketSequence(sequence_id);
}
+ util::Status ParseExtensionDescriptor(ConstBytes descriptor);
TraceProcessorContext* context_;
diff --git a/src/trace_processor/importers/proto/track_event.descriptor.h b/src/trace_processor/importers/proto/track_event.descriptor.h
index 635fc1d..3887159 100644
--- a/src/trace_processor/importers/proto/track_event.descriptor.h
+++ b/src/trace_processor/importers/proto/track_event.descriptor.h
@@ -27,14 +27,14 @@
// SHA1(tools/gen_binary_descriptors)
// b70b596b8569e2a2482f869a8eeb49f5ee801530
// SHA1(protos/perfetto/trace/track_event/track_event.proto)
-// 1e4267d78600ca00a8514c5479125c0d6ec6dddd
+// 723097690a042f5be26a438547bd5cca56ed3e5a
// This is the proto TrackEvent encoded as a ProtoFileDescriptor to allow
// for reflection without libprotobuf full/non-lite protos.
namespace perfetto {
-constexpr std::array<uint8_t, 18902> kTrackEventDescriptor{
+constexpr std::array<uint8_t, 18918> kTrackEventDescriptor{
{0x0a, 0x96, 0x08, 0x0a, 0x38, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f,
0x70, 0x65, 0x72, 0x66, 0x65, 0x74, 0x74, 0x6f, 0x2f, 0x74, 0x72, 0x61,
0x63, 0x65, 0x2f, 0x74, 0x72, 0x61, 0x63, 0x6b, 0x5f, 0x65, 0x76, 0x65,
@@ -1310,7 +1310,7 @@
0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e,
0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x68,
0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x61,
- 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x61, 0x73, 0x68, 0x0a, 0x86, 0x1c,
+ 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x61, 0x73, 0x68, 0x0a, 0x96, 0x1c,
0x0a, 0x33, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x70, 0x65, 0x72,
0x66, 0x65, 0x74, 0x74, 0x6f, 0x2f, 0x74, 0x72, 0x61, 0x63, 0x65, 0x2f,
0x74, 0x72, 0x61, 0x63, 0x6b, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2f,
@@ -1367,7 +1367,7 @@
0x74, 0x74, 0x6f, 0x2f, 0x74, 0x72, 0x61, 0x63, 0x65, 0x2f, 0x74, 0x72,
0x61, 0x63, 0x6b, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x2f, 0x63, 0x68,
0x72, 0x6f, 0x6d, 0x65, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x65, 0x76,
- 0x65, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x80, 0x15,
+ 0x65, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x90, 0x15,
0x0a, 0x0a, 0x54, 0x72, 0x61, 0x63, 0x6b, 0x45, 0x76, 0x65, 0x6e, 0x74,
0x12, 0x23, 0x0a, 0x0d, 0x63, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x79,
0x5f, 0x69, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x04, 0x52,
@@ -1586,31 +1586,32 @@
0x10, 0x02, 0x12, 0x10, 0x0a, 0x0c, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49,
0x4e, 0x53, 0x54, 0x41, 0x4e, 0x54, 0x10, 0x03, 0x12, 0x10, 0x0a, 0x0c,
0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x45, 0x52,
- 0x10, 0x04, 0x42, 0x0c, 0x0a, 0x0a, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x66,
- 0x69, 0x65, 0x6c, 0x64, 0x42, 0x0b, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65,
- 0x73, 0x74, 0x61, 0x6d, 0x70, 0x42, 0x0d, 0x0a, 0x0b, 0x74, 0x68, 0x72,
- 0x65, 0x61, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x42, 0x1a, 0x0a, 0x18,
- 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x69, 0x6e, 0x73, 0x74, 0x72,
- 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74,
- 0x22, 0x6e, 0x0a, 0x12, 0x54, 0x72, 0x61, 0x63, 0x6b, 0x45, 0x76, 0x65,
- 0x6e, 0x74, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x1d,
- 0x0a, 0x0a, 0x74, 0x72, 0x61, 0x63, 0x6b, 0x5f, 0x75, 0x75, 0x69, 0x64,
- 0x18, 0x0b, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x74, 0x72, 0x61, 0x63,
- 0x6b, 0x55, 0x75, 0x69, 0x64, 0x12, 0x39, 0x0a, 0x19, 0x65, 0x78, 0x74,
- 0x72, 0x61, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x5f, 0x74,
- 0x72, 0x61, 0x63, 0x6b, 0x5f, 0x75, 0x75, 0x69, 0x64, 0x73, 0x18, 0x1f,
- 0x20, 0x03, 0x28, 0x04, 0x52, 0x16, 0x65, 0x78, 0x74, 0x72, 0x61, 0x43,
- 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x54, 0x72, 0x61, 0x63, 0x6b, 0x55,
- 0x75, 0x69, 0x64, 0x73, 0x22, 0x35, 0x0a, 0x0d, 0x45, 0x76, 0x65, 0x6e,
- 0x74, 0x43, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72, 0x79, 0x12, 0x10, 0x0a,
- 0x03, 0x69, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03,
- 0x69, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
- 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22,
- 0x31, 0x0a, 0x09, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x61, 0x6d, 0x65,
- 0x12, 0x10, 0x0a, 0x03, 0x69, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
- 0x04, 0x52, 0x03, 0x69, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61,
- 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61,
- 0x6d, 0x65}};
+ 0x10, 0x04, 0x2a, 0x06, 0x08, 0xe8, 0x07, 0x10, 0xac, 0x4d, 0x2a, 0x06,
+ 0x08, 0xac, 0x4d, 0x10, 0x91, 0x4e, 0x42, 0x0c, 0x0a, 0x0a, 0x6e, 0x61,
+ 0x6d, 0x65, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x42, 0x0b, 0x0a, 0x09,
+ 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x42, 0x0d, 0x0a,
+ 0x0b, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65,
+ 0x42, 0x1a, 0x0a, 0x18, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x69,
+ 0x6e, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63,
+ 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x6e, 0x0a, 0x12, 0x54, 0x72, 0x61, 0x63,
+ 0x6b, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c,
+ 0x74, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x72, 0x61, 0x63, 0x6b, 0x5f,
+ 0x75, 0x75, 0x69, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09,
+ 0x74, 0x72, 0x61, 0x63, 0x6b, 0x55, 0x75, 0x69, 0x64, 0x12, 0x39, 0x0a,
+ 0x19, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74,
+ 0x65, 0x72, 0x5f, 0x74, 0x72, 0x61, 0x63, 0x6b, 0x5f, 0x75, 0x75, 0x69,
+ 0x64, 0x73, 0x18, 0x1f, 0x20, 0x03, 0x28, 0x04, 0x52, 0x16, 0x65, 0x78,
+ 0x74, 0x72, 0x61, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x54, 0x72,
+ 0x61, 0x63, 0x6b, 0x55, 0x75, 0x69, 0x64, 0x73, 0x22, 0x35, 0x0a, 0x0d,
+ 0x45, 0x76, 0x65, 0x6e, 0x74, 0x43, 0x61, 0x74, 0x65, 0x67, 0x6f, 0x72,
+ 0x79, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01,
+ 0x28, 0x04, 0x52, 0x03, 0x69, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e,
+ 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e,
+ 0x61, 0x6d, 0x65, 0x22, 0x31, 0x0a, 0x09, 0x45, 0x76, 0x65, 0x6e, 0x74,
+ 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x69, 0x64, 0x18,
+ 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x69, 0x69, 0x64, 0x12, 0x12,
+ 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
+ 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65}};
} // namespace perfetto
diff --git a/src/trace_processor/importers/proto/track_event_parser.cc b/src/trace_processor/importers/proto/track_event_parser.cc
index e5dde59..3bb0e6e 100644
--- a/src/trace_processor/importers/proto/track_event_parser.cc
+++ b/src/trace_processor/importers/proto/track_event_parser.cc
@@ -16,6 +16,7 @@
#include "src/trace_processor/importers/proto/track_event_parser.h"
+#include <iostream>
#include <string>
#include "perfetto/base/logging.h"
@@ -830,9 +831,10 @@
log_errors(ParseLogMessage(event_.log_message(), inserter));
}
- log_errors(parser_->proto_to_args_.InternProtoFieldsIntoArgsTable(
- blob_, ".perfetto.protos.TrackEvent", parser_->reflect_fields_,
- inserter, sequence_state_));
+ log_errors(
+ parser_->context_->proto_to_args_table_->InternProtoFieldsIntoArgsTable(
+ blob_, ".perfetto.protos.TrackEvent", parser_->reflect_fields_,
+ inserter, sequence_state_));
if (legacy_passthrough_utid_) {
inserter->AddArg(parser_->legacy_event_passthrough_utid_id_,
@@ -1083,7 +1085,6 @@
TrackEventParser::TrackEventParser(TraceProcessorContext* context)
: context_(context),
- proto_to_args_(context_),
counter_name_thread_time_id_(
context->storage->InternString("thread_time")),
counter_name_thread_instruction_count_id_(
@@ -1208,26 +1209,27 @@
counter_unit_ids_{{kNullStringId, context_->storage->InternString("ns"),
context_->storage->InternString("count"),
context_->storage->InternString("bytes")}} {
- auto status = proto_to_args_.AddProtoFileDescriptor(
+ auto status = context_->proto_to_args_table_->AddProtoFileDescriptor(
kTrackEventDescriptor.data(), kTrackEventDescriptor.size());
+
PERFETTO_DCHECK(status.ok());
// Switch |source_location_iid| into its interned data variant.
- proto_to_args_.AddParsingOverride(
+ context_->proto_to_args_table_->AddParsingOverride(
"begin_impl_frame_args.current_args.source_location_iid",
[](const ProtoToArgsTable::ParsingOverrideState& state,
const protozero::Field& field, BoundInserter* inserter) {
return MaybeParseSourceLocation("begin_impl_frame_args.current_args",
state, field, inserter);
});
- proto_to_args_.AddParsingOverride(
+ context_->proto_to_args_table_->AddParsingOverride(
"begin_impl_frame_args.last_args.source_location_iid",
[](const ProtoToArgsTable::ParsingOverrideState& state,
const protozero::Field& field, BoundInserter* inserter) {
return MaybeParseSourceLocation("begin_impl_frame_args.last_args",
state, field, inserter);
});
- proto_to_args_.AddParsingOverride(
+ context_->proto_to_args_table_->AddParsingOverride(
"begin_frame_observer_state.last_begin_frame_args.source_location_iid",
[](const ProtoToArgsTable::ParsingOverrideState& state,
const protozero::Field& field, BoundInserter* inserter) {
diff --git a/src/trace_processor/importers/proto/track_event_parser.h b/src/trace_processor/importers/proto/track_event_parser.h
index 1eb3cfd..2453eda 100644
--- a/src/trace_processor/importers/proto/track_event_parser.h
+++ b/src/trace_processor/importers/proto/track_event_parser.h
@@ -64,7 +64,6 @@
void ParseCounterDescriptor(TrackId, protozero::ConstBytes);
TraceProcessorContext* context_;
- ProtoToArgsTable proto_to_args_;
const StringId counter_name_thread_time_id_;
const StringId counter_name_thread_instruction_count_id_;
diff --git a/src/trace_processor/trace_processor_storage_impl.cc b/src/trace_processor/trace_processor_storage_impl.cc
index fdd346a..32146b1 100644
--- a/src/trace_processor/trace_processor_storage_impl.cc
+++ b/src/trace_processor/trace_processor_storage_impl.cc
@@ -25,6 +25,7 @@
#include "src/trace_processor/importers/common/slice_tracker.h"
#include "src/trace_processor/importers/common/track_tracker.h"
#include "src/trace_processor/importers/default_modules.h"
+#include "src/trace_processor/importers/proto/args_table_utils.h"
#include "src/trace_processor/importers/proto/heap_profile_tracker.h"
#include "src/trace_processor/importers/proto/metadata_tracker.h"
#include "src/trace_processor/importers/proto/proto_importer_module.h"
@@ -48,6 +49,7 @@
context_.heap_profile_tracker.reset(new HeapProfileTracker(&context_));
context_.metadata_tracker.reset(new MetadataTracker(&context_));
context_.global_args_tracker.reset(new GlobalArgsTracker(&context_));
+ context_.proto_to_args_table_.reset(new ProtoToArgsTable(&context_));
RegisterDefaultModules(&context_);
}
diff --git a/src/trace_processor/types/trace_processor_context.h b/src/trace_processor/types/trace_processor_context.h
index e23a91a..ab9c016 100644
--- a/src/trace_processor/types/trace_processor_context.h
+++ b/src/trace_processor/types/trace_processor_context.h
@@ -45,6 +45,7 @@
class TraceStorage;
class TrackTracker;
class JsonTracker;
+class ProtoToArgsTable;
class TraceProcessorContext {
public:
@@ -100,6 +101,9 @@
std::unique_ptr<TraceParser> json_trace_parser;
std::unique_ptr<TraceParser> fuchsia_trace_parser;
+ // Reflection-based proto parser used to convert TrackEvent fields into SQL.
+ std::unique_ptr<ProtoToArgsTable> proto_to_args_table_;
+
// The module at the index N is registered to handle field id N in
// TracePacket.
std::vector<ProtoImporterModule*> modules_by_field;
diff --git a/src/trace_processor/util/descriptors.cc b/src/trace_processor/util/descriptors.cc
index e75a77b..4c0ecd9 100644
--- a/src/trace_processor/util/descriptors.cc
+++ b/src/trace_processor/util/descriptors.cc
@@ -23,8 +23,6 @@
namespace perfetto {
namespace trace_processor {
-namespace {
-
FieldDescriptor CreateFieldFromDecoder(
const protos::pbzero::FieldDescriptorProto::Decoder& f_decoder) {
using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
@@ -43,8 +41,6 @@
f_decoder.label() == FieldDescriptorProto::LABEL_REPEATED);
}
-} // namespace
-
base::Optional<uint32_t> DescriptorPool::ResolveShortType(
const std::string& parent_path,
const std::string& short_type) {
diff --git a/src/trace_processor/util/descriptors.h b/src/trace_processor/util/descriptors.h
index 59ea48d..ff6ed55 100644
--- a/src/trace_processor/util/descriptors.h
+++ b/src/trace_processor/util/descriptors.h
@@ -24,6 +24,7 @@
#include "perfetto/ext/base/optional.h"
#include "perfetto/trace_processor/basic_types.h"
#include "perfetto/trace_processor/status.h"
+#include "protos/perfetto/common/descriptor.pbzero.h"
namespace protozero {
struct ConstBytes;
@@ -60,6 +61,9 @@
bool is_repeated_;
};
+FieldDescriptor CreateFieldFromDecoder(
+ const protos::pbzero::FieldDescriptorProto::Decoder& f_decoder);
+
class ProtoDescriptor {
public:
enum class Type { kEnum = 0, kMessage = 1 };
diff --git a/test/trace_processor/track_event_typed_args.textproto b/test/trace_processor/track_event_typed_args.textproto
index 67e37b2..9ccb9f0 100644
--- a/test/trace_processor/track_event_typed_args.textproto
+++ b/test/trace_processor/track_event_typed_args.textproto
@@ -13,6 +13,30 @@
}
packet {
trusted_packet_sequence_id: 1
+ timestamp: 0
+ extension_descriptor {
+ extension_file {
+ message_type {
+ extension {
+ name: "string_extension_for_testing"
+ extendee: ".perfetto.protos.TrackEvent"
+ number: 9900
+ type: TYPE_STRING
+ label: LABEL_OPTIONAL
+ }
+ extension {
+ name: "int_extension_for_testing"
+ extendee: ".perfetto.protos.TrackEvent"
+ number: 9901
+ type: TYPE_INT32
+ label: LABEL_REPEATED
+ }
+ }
+ }
+ }
+}
+packet {
+ trusted_packet_sequence_id: 1
timestamp: 1000
track_event {
track_uuid: 1
@@ -70,5 +94,11 @@
}
is_coalesced: true
}
+ [perfetto.protos.TestExtension.string_extension_for_testing]:
+ "an extension string!"
+ [perfetto.protos.TestExtension.int_extension_for_testing]: 42
+ [perfetto.protos.TestExtension.int_extension_for_testing]: 1337
+ [perfetto.protos.TestExtension.omitted_extension_for_testing]:
+ "should be absent from result"
}
}
diff --git a/test/trace_processor/track_event_typed_args_args.out b/test/trace_processor/track_event_typed_args_args.out
index e6d06f7..d898a9a 100644
--- a/test/trace_processor/track_event_typed_args_args.out
+++ b/test/trace_processor/track_event_typed_args_args.out
@@ -8,3 +8,6 @@
4,"chrome_latency_info.component_info.time_us","chrome_latency_info.component_info[1].time_us",928310,"[NULL]"
4,"chrome_latency_info.is_coalesced","chrome_latency_info.is_coalesced",1,"[NULL]"
4,"chrome_latency_info.trace_id","chrome_latency_info.trace_id",7,"[NULL]"
+4,"int_extension_for_testing","int_extension_for_testing[0]",42,"[NULL]"
+4,"int_extension_for_testing","int_extension_for_testing[1]",1337,"[NULL]"
+4,"string_extension_for_testing","string_extension_for_testing","[NULL]","an extension string!"
diff --git a/tools/gen_merged_protos b/tools/gen_merged_protos
index dee1920..15b2ecb 100755
--- a/tools/gen_merged_protos
+++ b/tools/gen_merged_protos
@@ -30,7 +30,10 @@
]
MERGED_CONFIG_PROTO = 'protos/perfetto/config/perfetto_config.proto'
-TRACE_PROTO_ROOTS = CONFIG_PROTO_ROOTS + ['protos/perfetto/trace/trace.proto']
+TRACE_PROTO_ROOTS = CONFIG_PROTO_ROOTS + [
+ 'protos/perfetto/trace/trace.proto',
+ 'protos/perfetto/trace/test_extensions.proto',
+]
MERGED_TRACE_PROTO = 'protos/perfetto/trace/perfetto_trace.proto'
METRICS_PROTOS_ROOTS = ['protos/perfetto/metrics/metrics.proto']