Improve ProtoZero decoders, introduce TypedProtoDecoder
This CL introduces a new class for one-shot proto decoding
(i.e. all fields are parsed before hand) built on top of the
iterator-based ProtoDecoder. This can be used when the proto
schema is known at compile time. The protozero compiler now
generates Decoder classes for each protozero header.
As in the spirit of protozero, decoders are also zero-syscall
zero-allocation, with the exception of decoding protobuf with
a lot (> 999) repeated fields, in which case heap is used as
a fallback.
Also get rid of the mini-reflection support and
ProtoFieldDescriptor. We ended up never using that and instead
rolled a more complex one based on proto for ftrace.
Also fix a bug when dealing with floating point decoding,
as_real() would lead to erroneous results.
Change-Id: If510a474566fbe906a21d1f310578f50b5d707fe
diff --git a/Android.bp b/Android.bp
index e28d21c..87b7509 100644
--- a/Android.bp
+++ b/Android.bp
@@ -71,7 +71,6 @@
"src/protozero/message.cc",
"src/protozero/message_handle.cc",
"src/protozero/proto_decoder.cc",
- "src/protozero/proto_field_descriptor.cc",
"src/protozero/scattered_heap_buffer.cc",
"src/protozero/scattered_stream_null_delegate.cc",
"src/protozero/scattered_stream_writer.cc",
@@ -245,7 +244,6 @@
"src/protozero/message.cc",
"src/protozero/message_handle.cc",
"src/protozero/proto_decoder.cc",
- "src/protozero/proto_field_descriptor.cc",
"src/protozero/scattered_heap_buffer.cc",
"src/protozero/scattered_stream_null_delegate.cc",
"src/protozero/scattered_stream_writer.cc",
@@ -431,7 +429,6 @@
"src/protozero/message.cc",
"src/protozero/message_handle.cc",
"src/protozero/proto_decoder.cc",
- "src/protozero/proto_field_descriptor.cc",
"src/protozero/scattered_heap_buffer.cc",
"src/protozero/scattered_stream_null_delegate.cc",
"src/protozero/scattered_stream_writer.cc",
@@ -608,7 +605,6 @@
"src/protozero/message.cc",
"src/protozero/message_handle.cc",
"src/protozero/proto_decoder.cc",
- "src/protozero/proto_field_descriptor.cc",
"src/protozero/scattered_heap_buffer.cc",
"src/protozero/scattered_stream_null_delegate.cc",
"src/protozero/scattered_stream_writer.cc",
@@ -2612,7 +2608,6 @@
"src/protozero/message.cc",
"src/protozero/message_handle.cc",
"src/protozero/proto_decoder.cc",
- "src/protozero/proto_field_descriptor.cc",
"src/protozero/scattered_heap_buffer.cc",
"src/protozero/scattered_stream_null_delegate.cc",
"src/protozero/scattered_stream_writer.cc",
@@ -2899,7 +2894,6 @@
"src/protozero/message_unittest.cc",
"src/protozero/proto_decoder.cc",
"src/protozero/proto_decoder_unittest.cc",
- "src/protozero/proto_field_descriptor.cc",
"src/protozero/proto_utils_unittest.cc",
"src/protozero/scattered_heap_buffer.cc",
"src/protozero/scattered_stream_null_delegate.cc",
@@ -3089,20 +3083,33 @@
name: "trace_to_text",
srcs: [
":perfetto_protos_perfetto_common_lite_gen",
+ ":perfetto_protos_perfetto_common_zero_gen",
":perfetto_protos_perfetto_config_lite_gen",
+ ":perfetto_protos_perfetto_config_zero_gen",
":perfetto_protos_perfetto_trace_android_lite_gen",
+ ":perfetto_protos_perfetto_trace_android_zero_gen",
":perfetto_protos_perfetto_trace_chrome_lite_gen",
+ ":perfetto_protos_perfetto_trace_chrome_zero_gen",
":perfetto_protos_perfetto_trace_filesystem_lite_gen",
+ ":perfetto_protos_perfetto_trace_filesystem_zero_gen",
":perfetto_protos_perfetto_trace_ftrace_lite_gen",
+ ":perfetto_protos_perfetto_trace_ftrace_zero_gen",
":perfetto_protos_perfetto_trace_interned_data_lite_gen",
+ ":perfetto_protos_perfetto_trace_interned_data_zero_gen",
":perfetto_protos_perfetto_trace_lite_gen",
":perfetto_protos_perfetto_trace_minimal_lite_gen",
":perfetto_protos_perfetto_trace_power_lite_gen",
+ ":perfetto_protos_perfetto_trace_power_zero_gen",
":perfetto_protos_perfetto_trace_processor_lite_gen",
":perfetto_protos_perfetto_trace_profiling_lite_gen",
+ ":perfetto_protos_perfetto_trace_profiling_zero_gen",
":perfetto_protos_perfetto_trace_ps_lite_gen",
+ ":perfetto_protos_perfetto_trace_ps_zero_gen",
":perfetto_protos_perfetto_trace_sys_stats_lite_gen",
+ ":perfetto_protos_perfetto_trace_sys_stats_zero_gen",
":perfetto_protos_perfetto_trace_track_event_lite_gen",
+ ":perfetto_protos_perfetto_trace_track_event_zero_gen",
+ ":perfetto_protos_perfetto_trace_zero_gen",
":perfetto_protos_third_party_pprof_lite_gen",
"src/base/event.cc",
"src/base/file_utils.cc",
@@ -3122,7 +3129,6 @@
"src/protozero/message.cc",
"src/protozero/message_handle.cc",
"src/protozero/proto_decoder.cc",
- "src/protozero/proto_field_descriptor.cc",
"src/protozero/scattered_heap_buffer.cc",
"src/protozero/scattered_stream_null_delegate.cc",
"src/protozero/scattered_stream_writer.cc",
@@ -3183,20 +3189,33 @@
],
generated_headers: [
"perfetto_protos_perfetto_common_lite_gen_headers",
+ "perfetto_protos_perfetto_common_zero_gen_headers",
"perfetto_protos_perfetto_config_lite_gen_headers",
+ "perfetto_protos_perfetto_config_zero_gen_headers",
"perfetto_protos_perfetto_trace_android_lite_gen_headers",
+ "perfetto_protos_perfetto_trace_android_zero_gen_headers",
"perfetto_protos_perfetto_trace_chrome_lite_gen_headers",
+ "perfetto_protos_perfetto_trace_chrome_zero_gen_headers",
"perfetto_protos_perfetto_trace_filesystem_lite_gen_headers",
+ "perfetto_protos_perfetto_trace_filesystem_zero_gen_headers",
"perfetto_protos_perfetto_trace_ftrace_lite_gen_headers",
+ "perfetto_protos_perfetto_trace_ftrace_zero_gen_headers",
"perfetto_protos_perfetto_trace_interned_data_lite_gen_headers",
+ "perfetto_protos_perfetto_trace_interned_data_zero_gen_headers",
"perfetto_protos_perfetto_trace_lite_gen_headers",
"perfetto_protos_perfetto_trace_minimal_lite_gen_headers",
"perfetto_protos_perfetto_trace_power_lite_gen_headers",
+ "perfetto_protos_perfetto_trace_power_zero_gen_headers",
"perfetto_protos_perfetto_trace_processor_lite_gen_headers",
"perfetto_protos_perfetto_trace_profiling_lite_gen_headers",
+ "perfetto_protos_perfetto_trace_profiling_zero_gen_headers",
"perfetto_protos_perfetto_trace_ps_lite_gen_headers",
+ "perfetto_protos_perfetto_trace_ps_zero_gen_headers",
"perfetto_protos_perfetto_trace_sys_stats_lite_gen_headers",
+ "perfetto_protos_perfetto_trace_sys_stats_zero_gen_headers",
"perfetto_protos_perfetto_trace_track_event_lite_gen_headers",
+ "perfetto_protos_perfetto_trace_track_event_zero_gen_headers",
+ "perfetto_protos_perfetto_trace_zero_gen_headers",
"perfetto_protos_third_party_pprof_lite_gen_headers",
],
defaults: [
diff --git a/BUILD.gn b/BUILD.gn
index c54e571..242eb47 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -279,6 +279,7 @@
"src/ipc:buffered_frame_deserializer_fuzzer",
"src/profiling/memory:shared_ring_buffer_fuzzer",
"src/profiling/memory:unwinding_fuzzer",
+ "src/protozero:protozero_decoder_fuzzer",
"src/trace_processor:trace_processor_fuzzer",
"src/traced/probes/ftrace:cpu_reader_fuzzer",
"test:end_to_end_shared_memory_fuzzer",
diff --git a/include/perfetto/base/string_view.h b/include/perfetto/base/string_view.h
index 67f14b0..994264c 100644
--- a/include/perfetto/base/string_view.h
+++ b/include/perfetto/base/string_view.h
@@ -39,6 +39,11 @@
StringView& operator=(const StringView&) = default;
StringView(const char* data, size_t size) : data_(data), size_(size) {}
+ // Allow implicit conversion from any class that has a |data| and |size| field
+ // and has the kConvertibleToStringView trait (e.g., protozero::ConstChars).
+ template <typename T, typename = std::enable_if<T::kConvertibleToStringView>>
+ StringView(const T& x) : StringView(x.data, x.size) {}
+
// Creates a StringView from a null-terminated C string.
// Deliberately not "explicit".
StringView(const char* cstr) : data_(cstr), size_(strlen(cstr)) {}
diff --git a/include/perfetto/protozero/BUILD.gn b/include/perfetto/protozero/BUILD.gn
index fef1c07..170e325 100644
--- a/include/perfetto/protozero/BUILD.gn
+++ b/include/perfetto/protozero/BUILD.gn
@@ -18,10 +18,11 @@
]
sources = [
"contiguous_memory_range.h",
+ "field.h",
"message.h",
"message_handle.h",
"proto_decoder.h",
- "proto_field_descriptor.h",
+ "proto_decoder.h",
"proto_utils.h",
"scattered_heap_buffer.h",
"scattered_stream_null_delegate.h",
diff --git a/include/perfetto/protozero/field.h b/include/perfetto/protozero/field.h
new file mode 100644
index 0000000..cbb48e6
--- /dev/null
+++ b/include/perfetto/protozero/field.h
@@ -0,0 +1,160 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef INCLUDE_PERFETTO_PROTOZERO_FIELD_H_
+#define INCLUDE_PERFETTO_PROTOZERO_FIELD_H_
+
+#include <stdint.h>
+
+#include "perfetto/base/logging.h"
+#include "perfetto/protozero/contiguous_memory_range.h"
+#include "perfetto/protozero/proto_utils.h"
+
+namespace protozero {
+
+struct ConstBytes {
+ const uint8_t* data;
+ size_t size;
+};
+
+struct ConstChars {
+ // Allow implicit conversion to perfetto's base::StringView without depending
+ // on perfetto/base or viceversa.
+ static constexpr bool kConvertibleToStringView = true;
+
+ const char* data;
+ size_t size;
+};
+
+// A protobuf field decoded by the protozero proto decoders. It exposes
+// convenience accessors with minimal debug checks.
+// This class is used both by the iterator-based ProtoDecoder and by the
+// one-shot TypedProtoDecoder.
+// If the field is not valid the accessors consistently return zero-integers or
+// null strings.
+class Field {
+ public:
+ inline bool valid() const { return id_ != 0; }
+ inline uint16_t id() const { return id_; }
+ explicit inline operator bool() const { return valid(); }
+
+ inline proto_utils::ProtoWireType type() const {
+ auto res = static_cast<proto_utils::ProtoWireType>(type_);
+ PERFETTO_DCHECK(res == proto_utils::ProtoWireType::kVarInt ||
+ res == proto_utils::ProtoWireType::kLengthDelimited ||
+ res == proto_utils::ProtoWireType::kFixed32 ||
+ res == proto_utils::ProtoWireType::kFixed64);
+ return res;
+ }
+
+ inline bool as_bool() const {
+ PERFETTO_DCHECK(!valid() || type() == proto_utils::ProtoWireType::kVarInt);
+ return static_cast<bool>(int_value_);
+ }
+
+ inline uint32_t as_uint32() const {
+ PERFETTO_DCHECK(!valid() || type() == proto_utils::ProtoWireType::kVarInt ||
+ type() == proto_utils::ProtoWireType::kFixed32);
+ return static_cast<uint32_t>(int_value_);
+ }
+
+ inline int32_t as_int32() const {
+ PERFETTO_DCHECK(!valid() || type() == proto_utils::ProtoWireType::kVarInt ||
+ type() == proto_utils::ProtoWireType::kFixed32);
+ return static_cast<int32_t>(int_value_);
+ }
+
+ inline uint64_t as_uint64() const {
+ PERFETTO_DCHECK(!valid() || type() == proto_utils::ProtoWireType::kVarInt ||
+ type() == proto_utils::ProtoWireType::kFixed32 ||
+ type() == proto_utils::ProtoWireType::kFixed64);
+ return int_value_;
+ }
+
+ inline int64_t as_int64() const {
+ PERFETTO_DCHECK(!valid() || type() == proto_utils::ProtoWireType::kVarInt ||
+ type() == proto_utils::ProtoWireType::kFixed32 ||
+ type() == proto_utils::ProtoWireType::kFixed64);
+ return static_cast<int64_t>(int_value_);
+ }
+
+ inline float as_float() const {
+ PERFETTO_DCHECK(!valid() || type() == proto_utils::ProtoWireType::kFixed32);
+ float res;
+ uint32_t value32 = static_cast<uint32_t>(int_value_);
+ memcpy(&res, &value32, sizeof(res));
+ return res;
+ }
+
+ inline double as_double() const {
+ PERFETTO_DCHECK(!valid() || type() == proto_utils::ProtoWireType::kFixed64);
+ double res;
+ memcpy(&res, &int_value_, sizeof(res));
+ return res;
+ }
+
+ inline ConstChars as_string() const {
+ PERFETTO_DCHECK(!valid() ||
+ type() == proto_utils::ProtoWireType::kLengthDelimited);
+ return ConstChars{reinterpret_cast<const char*>(data()), size_};
+ }
+
+ inline ConstBytes as_bytes() const {
+ PERFETTO_DCHECK(!valid() ||
+ type() == proto_utils::ProtoWireType::kLengthDelimited);
+ return ConstBytes{data(), size_};
+ }
+
+ inline const uint8_t* data() const {
+ PERFETTO_DCHECK(!valid() ||
+ type() == proto_utils::ProtoWireType::kLengthDelimited);
+ return reinterpret_cast<const uint8_t*>(int_value_);
+ }
+
+ inline size_t size() const {
+ PERFETTO_DCHECK(!valid() ||
+ type() == proto_utils::ProtoWireType::kLengthDelimited);
+ return size_;
+ }
+
+ inline uint64_t raw_int_value() const { return int_value_; }
+
+ inline void initialize(uint16_t id,
+ uint8_t type,
+ uint64_t int_value,
+ uint32_t size) {
+ id_ = id;
+ type_ = type;
+ int_value_ = int_value;
+ size_ = size;
+ }
+
+ private:
+ // Fields are deliberately not initialized to keep the class trivially
+ // constructible. It makes a large perf difference for ProtoDecoder.
+
+ uint64_t int_value_; // In kLengthDelimited this contains the data() addr.
+ uint32_t size_; // Only valid when when type == kLengthDelimited.
+ uint16_t id_; // Proto field ordinal.
+ uint8_t type_; // proto_utils::ProtoWireType.
+};
+
+// The Field struct is used in a lot of perf-sensitive contexts.
+static_assert(sizeof(Field) == 16, "Field struct too big");
+
+} // namespace protozero
+
+#endif // INCLUDE_PERFETTO_PROTOZERO_FIELD_H_
diff --git a/include/perfetto/protozero/proto_decoder.h b/include/perfetto/protozero/proto_decoder.h
index 68b5e1c..8e67bf9 100644
--- a/include/perfetto/protozero/proto_decoder.h
+++ b/include/perfetto/protozero/proto_decoder.h
@@ -18,181 +18,198 @@
#define INCLUDE_PERFETTO_PROTOZERO_PROTO_DECODER_H_
#include <stdint.h>
+#include <array>
#include <memory>
+#include <vector>
#include "perfetto/base/logging.h"
#include "perfetto/base/string_view.h"
+#include "perfetto/protozero/field.h"
#include "perfetto/protozero/proto_utils.h"
namespace protozero {
-// Reads and decodes protobuf messages from a fixed length buffer. This class
-// does not allocate and does no more work than necessary so can be used in
-// performance sensitive contexts.
+// A generic protobuf decoder. Doesn't require any knowledge about the proto
+// schema. It tokenizes fields, retrieves their ID and type and exposes
+// accessors to retrieve its values.
+// It does NOT recurse in nested submessages, instead it just computes their
+// boundaries, recursion is left to the caller.
+// This class is designed to be used in perf-sensitive contexts. It does not
+// allocate and does not perform any proto semantic checks (e.g. repeated /
+// required / optional). It's supposedly safe wrt out-of-bounds memory accesses
+// (see proto_decoder_fuzzer.cc).
+// This class serves also as a building block for TypedProtoDecoder, used when
+// the schema is known at compile time.
class ProtoDecoder {
public:
- using StringView = ::perfetto::base::StringView;
-
- // The field of a protobuf message. |id| == 0 if the tag is not valid (e.g.
- // because the full tag was unable to be read etc.).
- struct Field {
- struct LengthDelimited {
- const uint8_t* data;
- size_t length;
- };
-
- uint32_t id = 0;
- proto_utils::ProtoWireType type;
- union {
- uint64_t int_value;
- LengthDelimited length_limited;
- };
-
- inline bool as_bool() const {
- PERFETTO_DCHECK(type == proto_utils::ProtoWireType::kVarInt);
- return static_cast<bool>(int_value);
- }
-
- inline uint32_t as_uint32() const {
- PERFETTO_DCHECK(type == proto_utils::ProtoWireType::kVarInt ||
- type == proto_utils::ProtoWireType::kFixed32);
- return static_cast<uint32_t>(int_value);
- }
-
- inline int32_t as_int32() const {
- PERFETTO_DCHECK(type == proto_utils::ProtoWireType::kVarInt ||
- type == proto_utils::ProtoWireType::kFixed32);
- return static_cast<int32_t>(int_value);
- }
-
- inline uint64_t as_uint64() const {
- PERFETTO_DCHECK(type == proto_utils::ProtoWireType::kVarInt ||
- type == proto_utils::ProtoWireType::kFixed64);
- return int_value;
- }
-
- inline int64_t as_int64() const {
- PERFETTO_DCHECK(type == proto_utils::ProtoWireType::kVarInt ||
- type == proto_utils::ProtoWireType::kFixed64);
- return static_cast<int64_t>(int_value);
- }
-
- // A relaxed version for when we are storing any int as an int64
- // in the raw events table.
- inline int64_t as_integer() const {
- PERFETTO_DCHECK(type == proto_utils::ProtoWireType::kVarInt ||
- type == proto_utils::ProtoWireType::kFixed64 ||
- type == proto_utils::ProtoWireType::kFixed32);
- return static_cast<int64_t>(int_value);
- }
-
- inline float as_float() const {
- PERFETTO_DCHECK(type == proto_utils::ProtoWireType::kFixed32);
- float res;
- uint32_t value32 = static_cast<uint32_t>(int_value);
- memcpy(&res, &value32, sizeof(res));
- return res;
- }
-
- inline double as_double() const {
- PERFETTO_DCHECK(type == proto_utils::ProtoWireType::kFixed64);
- double res;
- memcpy(&res, &int_value, sizeof(res));
- return res;
- }
-
- // A relaxed version for when we are storing floats and doubles
- // as real in the raw events table.
- inline double as_real() const {
- PERFETTO_DCHECK(type == proto_utils::ProtoWireType::kFixed64 ||
- type == proto_utils::ProtoWireType::kFixed32);
- double res;
- uint64_t value64 = static_cast<uint64_t>(int_value);
- memcpy(&res, &value64, sizeof(res));
- return res;
- }
-
- inline StringView as_string() const {
- PERFETTO_DCHECK(type == proto_utils::ProtoWireType::kLengthDelimited);
- return StringView(reinterpret_cast<const char*>(length_limited.data),
- length_limited.length);
- }
-
- inline const uint8_t* data() const {
- PERFETTO_DCHECK(type == proto_utils::ProtoWireType::kLengthDelimited);
- return length_limited.data;
- }
-
- inline size_t size() const {
- PERFETTO_DCHECK(type == proto_utils::ProtoWireType::kLengthDelimited);
- return static_cast<size_t>(length_limited.length);
- }
- };
-
// Creates a ProtoDecoder using the given |buffer| with size |length| bytes.
- inline ProtoDecoder(const uint8_t* buffer, uint64_t length)
- : buffer_(buffer), length_(length), current_position_(buffer) {}
+ inline ProtoDecoder(const uint8_t* buffer, size_t length)
+ : begin_(buffer), end_(buffer + length), read_ptr_(buffer) {}
- // Reads the next field from the buffer. If the full field cannot be read,
- // the returned struct will have id 0 which is an invalid field id.
+ // Reads the next field from the buffer and advances the read cursor. If a
+ // full field cannot be read, the returned Field will be invalid (i.e.
+ // field.valid() == false).
Field ReadField();
- template <int field_id>
- inline bool FindIntField(uint64_t* field_value) {
- bool res = false;
- for (auto f = ReadField(); f.id != 0; f = ReadField()) {
- if (f.id == field_id) {
- *field_value = f.int_value;
- res = true;
- break;
- }
- }
- Reset();
- return res;
- }
+ // Finds the first field with the given id. Doesn't affect the read cursor.
+ Field FindField(uint32_t field_id);
- template <int field_id>
- inline bool FindStringField(StringView* field_value) {
- bool res = false;
- for (auto f = ReadField(); f.id != 0; f = ReadField()) {
- if (f.id == field_id) {
- *field_value = f.as_string();
- res = true;
- break;
- }
- }
- Reset();
- return res;
- }
+ // Resets the read cursor to the start of the buffer.
+ inline void Reset() { read_ptr_ = begin_; }
- // Returns true if |length_| == |current_position_| - |buffer| and false
- // otherwise.
- inline bool IsEndOfBuffer() {
- PERFETTO_DCHECK(current_position_ >= buffer_);
- return length_ == static_cast<uint64_t>(current_position_ - buffer_);
- }
-
- // Resets the current position to the start of the buffer.
- inline void Reset() { current_position_ = buffer_; }
-
- // Resets to the given position (must be within the buffer).
+ // Resets the read cursor to the given position (must be within the buffer).
inline void Reset(const uint8_t* pos) {
- PERFETTO_DCHECK(pos >= buffer_ && pos < buffer_ + length_);
- current_position_ = pos;
+ PERFETTO_DCHECK(pos >= begin_ && pos < end_);
+ read_ptr_ = pos;
}
- // Return's offset inside the buffer.
- inline uint64_t offset() const {
- return static_cast<uint64_t>(current_position_ - buffer_);
+ // Returns the position of read cursor, relative to the start of the buffer.
+ inline size_t read_offset() const {
+ return static_cast<size_t>(read_ptr_ - begin_);
}
- inline const uint8_t* buffer() const { return buffer_; }
- inline uint64_t length() const { return length_; }
+ inline size_t bytes_left() const {
+ PERFETTO_DCHECK(read_ptr_ <= end_);
+ return static_cast<size_t>(end_ - read_ptr_);
+ }
+
+ const uint8_t* begin() const { return begin_; }
+ const uint8_t* end() const { return end_; }
+
+ protected:
+ const uint8_t* const begin_;
+ const uint8_t* const end_;
+ const uint8_t* read_ptr_ = nullptr;
+};
+
+// An iterator-like class used to iterate through repeated fields. Used by
+// TypedProtoDecoder.
+class RepeatedFieldIterator {
+ public:
+ RepeatedFieldIterator(uint32_t field_id, const Field* begin, const Field* end)
+ : field_id_(field_id), iter_(begin), end_(end) {
+ FindNextMatchingId();
+ }
+
+ inline const Field* operator->() const { return &*iter_; }
+ inline const Field& operator*() const { return *iter_; }
+ inline explicit operator bool() const { return iter_ != end_; }
+
+ RepeatedFieldIterator& operator++() {
+ ++iter_;
+ FindNextMatchingId();
+ return *this;
+ }
private:
- const uint8_t* const buffer_;
- const uint64_t length_; // The outer buffer can be larger than 4GB.
- const uint8_t* current_position_ = nullptr;
+ inline void FindNextMatchingId() {
+ for (; iter_ != end_; ++iter_) {
+ if (iter_->id() == field_id_)
+ break;
+ }
+ }
+
+ uint32_t field_id_;
+ const Field* iter_;
+ const Field* end_;
+};
+
+// This decoder loads all fields upfront, without recursing in nested messages.
+// It is used as a base class for typed decoders generated by the pbzero plugin.
+// The split between TypedProtoDecoderBase and TypedProtoDecoder<> is to have
+// unique definition of functions like ParseAllFields() and ExpandHeapStorage().
+class TypedProtoDecoderBase : public ProtoDecoder {
+ public:
+ // If the field |id| is known at compile time, prefer the templated
+ // specialization at<kFieldNumber>().
+ inline const Field& Get(uint32_t id) {
+ return PERFETTO_LIKELY(id < size_) ? fields_[id] : fields_[0];
+ }
+
+ inline RepeatedFieldIterator GetRepeated(uint32_t field_id) const {
+ return RepeatedFieldIterator(field_id, &fields_[0], &fields_[size_]);
+ }
+
+ protected:
+ TypedProtoDecoderBase(Field* storage,
+ uint32_t size,
+ uint32_t capacity,
+ const uint8_t* buffer,
+ size_t length)
+ : ProtoDecoder(buffer, length),
+ fields_(storage),
+ size_(size),
+ capacity_(capacity) {
+ // The reason why Field needs to be trivially de/constructible is to avoid
+ // implicit initializers on all the ~1000 entries. We need it to initialize
+ // only on the first |max_field_id| fields, the remaining capacity doesn't
+ // require initialization.
+ static_assert(std::is_trivially_constructible<Field>::value &&
+ std::is_trivially_destructible<Field>::value &&
+ std::is_trivial<Field>::value,
+ "Field must be a trivial aggregate type");
+ memset(fields_, 0, sizeof(Field) * size_);
+ }
+
+ void ParseAllFields();
+ void ExpandHeapStorage();
+
+ // Used only in presence of a large number of repeated fields, when the
+ // default on-stack storage is exhausted.
+ std::unique_ptr<Field[]> heap_storage_;
+
+ // Points to the storage, either on-stack (default, provided by the template
+ // specialization) or |heap_storage_| after ExpandHeapStorage() is called, in
+ // case of a large number of repeated fields.
+ Field* fields_;
+
+ // Number of active |fields_| entries. This is initially equal to the highest
+ // number of fields for the message (MAX_FIELD_ID + 1) and can grow up to
+ // |capacity_| in the case of repeated fields.
+ uint32_t size_;
+
+ // Initially equal to kFieldsCapacity of the TypedProtoDecoder
+ // specialization. Can grow when falling back on heap-based storage, in which
+ // case it represents the size (#Fields) of the |heap_storage_| array.
+ uint32_t capacity_;
+};
+
+// Template class instantiated by the auto-generated decoder classes declared in
+// xxx.pbzero.h files.
+template <int MAX_FIELD_ID, bool HAS_REPEATED_FIELDS>
+class TypedProtoDecoder : public TypedProtoDecoderBase {
+ public:
+ TypedProtoDecoder(const uint8_t* buffer, size_t length)
+ : TypedProtoDecoderBase(on_stack_storage_,
+ /*size=*/MAX_FIELD_ID + 1,
+ kCapacity,
+ buffer,
+ length) {
+ static_assert(MAX_FIELD_ID <= kMaxDecoderFieldId, "Field ordinal too high");
+ TypedProtoDecoderBase::ParseAllFields();
+ }
+
+ template <uint32_t FIELD_ID>
+ inline const Field& at() const {
+ static_assert(FIELD_ID <= MAX_FIELD_ID, "FIELD_ID > MAX_FIELD_ID");
+ return fields_[FIELD_ID];
+ }
+
+ private:
+ // In the case of non-repeated fields, this constant defines the highest field
+ // id we are able to decode. This is to limit the on-stack storage.
+ // In the case of repeated fields, this constant defines the max number of
+ // repeated fields that we'll be able to store before falling back on the
+ // heap. Keep this value in sync with the one in protozero_generator.cc.
+ static constexpr size_t kMaxDecoderFieldId = 999;
+
+ // If we the message has no repeated fields we need at most N Field entries
+ // in the on-stack storage, where N is the highest field id.
+ // Otherwise we need some room to store repeated fields.
+ static constexpr size_t kCapacity =
+ 1 + (HAS_REPEATED_FIELDS ? kMaxDecoderFieldId : MAX_FIELD_ID);
+
+ Field on_stack_storage_[kCapacity];
};
} // namespace protozero
diff --git a/include/perfetto/protozero/proto_field_descriptor.h b/include/perfetto/protozero/proto_field_descriptor.h
deleted file mode 100644
index 94df0cb..0000000
--- a/include/perfetto/protozero/proto_field_descriptor.h
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright (C) 2017 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef INCLUDE_PERFETTO_PROTOZERO_PROTO_FIELD_DESCRIPTOR_H_
-#define INCLUDE_PERFETTO_PROTOZERO_PROTO_FIELD_DESCRIPTOR_H_
-
-#include <stdint.h>
-
-#include "perfetto/base/export.h"
-
-namespace protozero {
-
-// Used for minimal reflection support in auto-generated .pbzero.h files.
-class ProtoFieldDescriptor {
- public:
- enum Type {
- TYPE_INVALID = 0,
- TYPE_DOUBLE = 1,
- TYPE_FLOAT = 2,
- TYPE_INT64 = 3,
- TYPE_UINT64 = 4,
- TYPE_INT32 = 5,
- TYPE_FIXED64 = 6,
- TYPE_FIXED32 = 7,
- TYPE_BOOL = 8,
- TYPE_STRING = 9,
- TYPE_MESSAGE = 11,
- // TYPE_GROUP = 10 is not supported.
- TYPE_BYTES = 12,
- TYPE_UINT32 = 13,
- TYPE_ENUM = 14,
- TYPE_SFIXED32 = 15,
- TYPE_SFIXED64 = 16,
- TYPE_SINT32 = 17,
- TYPE_SINT64 = 18,
- };
-
- static PERFETTO_EXPORT const ProtoFieldDescriptor* GetInvalidInstance();
-
- ProtoFieldDescriptor(const char* name,
- Type type,
- uint32_t number,
- bool is_repeated)
- : name_(name), type_(type), number_(number), is_repeated_(is_repeated) {}
-
- const char* name() const { return name_; }
- Type type() const { return type_; }
- uint32_t number() const { return number_; }
- bool is_repeated() const { return is_repeated_; }
- bool is_valid() const { return type_ != Type::TYPE_INVALID; }
-
- private:
- const char* const name_;
- const Type type_;
- const uint32_t number_;
- const bool is_repeated_;
-};
-
-} // namespace protozero
-
-#endif // INCLUDE_PERFETTO_PROTOZERO_PROTO_FIELD_DESCRIPTOR_H_
diff --git a/src/protozero/BUILD.gn b/src/protozero/BUILD.gn
index d320f4f..1197437 100644
--- a/src/protozero/BUILD.gn
+++ b/src/protozero/BUILD.gn
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import("../../gn/fuzzer.gni")
import("../../gn/perfetto.gni")
import("../../gn/proto_library.gni")
import("../../gn/protozero_library.gni")
@@ -30,7 +31,6 @@
"message.cc",
"message_handle.cc",
"proto_decoder.cc",
- "proto_field_descriptor.cc",
"scattered_heap_buffer.cc",
"scattered_stream_null_delegate.cc",
"scattered_stream_writer.cc",
@@ -90,3 +90,13 @@
proto_in_dir = perfetto_root_path
proto_out_dir = perfetto_root_path
}
+
+perfetto_fuzzer_test("protozero_decoder_fuzzer") {
+ sources = [
+ "proto_decoder_fuzzer.cc",
+ ]
+ deps = [
+ ":protozero",
+ "../../gn:default_deps",
+ ]
+}
diff --git a/src/protozero/proto_decoder.cc b/src/protozero/proto_decoder.cc
index ff3aaf1..a78f93e 100644
--- a/src/protozero/proto_decoder.cc
+++ b/src/protozero/proto_decoder.cc
@@ -19,21 +19,29 @@
#include <string.h>
#include "perfetto/base/logging.h"
+#include "perfetto/base/utils.h"
#include "perfetto/protozero/proto_utils.h"
namespace protozero {
using namespace proto_utils;
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-#define BYTE_SWAP_TO_LE32(x) (x)
-#define BYTE_SWAP_TO_LE64(x) (x)
-#else
+#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__
#error Unimplemented for big endian archs.
#endif
-ProtoDecoder::Field ProtoDecoder::ReadField() {
- Field field{};
+namespace {
+
+struct ParseFieldResult {
+ const uint8_t* next;
+ Field field;
+};
+
+// Parses one field and returns the field itself and a pointer to the next
+// field to parse. If parsing fails, the returned |next| == |buffer|.
+PERFETTO_ALWAYS_INLINE ParseFieldResult
+ParseOneField(const uint8_t* const buffer, const uint8_t* const end) {
+ ParseFieldResult res{buffer, Field{}};
// The first byte of a proto field is structured as follows:
// The least 3 significant bits determine the field type.
@@ -41,99 +49,161 @@
// field id continues on the next bytes following the VarInt encoding.
const uint8_t kFieldTypeNumBits = 3;
const uint64_t kFieldTypeMask = (1 << kFieldTypeNumBits) - 1; // 0000 0111;
-
- const uint8_t* end = buffer_ + length_;
- const uint8_t* pos = current_position_;
- PERFETTO_DCHECK(pos >= buffer_);
- PERFETTO_DCHECK(pos <= end);
+ const uint8_t* pos = buffer;
// If we've already hit the end, just return an invalid field.
- if (pos >= end) {
- return field;
- }
+ if (PERFETTO_UNLIKELY(pos >= end))
+ return res;
- uint64_t raw_field_id = 0;
- if (PERFETTO_LIKELY(*pos < 0x80)) {
- raw_field_id = *(pos++); // Fastpath for fields with ID < 32.
+ uint64_t preamble = 0;
+ if (PERFETTO_LIKELY(*pos < 0x80)) { // Fastpath for fields with ID < 32.
+ preamble = *(pos++);
} else {
- pos = ParseVarInt(pos, end, &raw_field_id);
+ pos = ParseVarInt(pos, end, &preamble);
}
- uint32_t field_id = static_cast<uint32_t>(raw_field_id >> kFieldTypeNumBits);
- if (field_id == 0 || pos >= end) {
- return field;
- }
+ uint32_t field_id = static_cast<uint32_t>(preamble >> kFieldTypeNumBits);
+ if (field_id == 0 || pos >= end)
+ return res;
- field.type = static_cast<ProtoWireType>(raw_field_id & kFieldTypeMask);
+ auto field_type = static_cast<uint8_t>(preamble & kFieldTypeMask);
+ const uint8_t* new_pos = pos;
+ uint64_t int_value = 0;
+ uint32_t size = 0;
- const uint8_t* new_pos = nullptr;
- uint64_t field_intvalue = 0;
- switch (field.type) {
- case ProtoWireType::kVarInt: {
- new_pos = ParseVarInt(pos, end, &field.int_value);
+ switch (field_type) {
+ case static_cast<uint8_t>(ProtoWireType::kVarInt): {
+ new_pos = ParseVarInt(pos, end, &int_value);
// new_pos not being greater than pos means ParseVarInt could not fully
// parse the number. This is because we are out of space in the buffer.
// Set the id to zero and return but don't update the offset so a future
// read can read this field.
- if (new_pos == pos) {
- return field;
- }
- pos = new_pos;
+ if (PERFETTO_UNLIKELY(new_pos == pos))
+ return res;
+
break;
}
- case ProtoWireType::kLengthDelimited: {
- new_pos = ParseVarInt(pos, end, &field_intvalue);
- // new_pos not being greater than pos means ParseVarInt could not fully
- // parse the number. This is because we are out of space in the buffer.
- // Alternatively, we may not have space to fully read the length
- // delimited field. Set the id to zero and return but don't update the
- // offset so a future read can read this field.
- // It is safe to static_cast end - new_pos as new_pos will be <= end after
- // ParseVarInt.
- if (new_pos == pos ||
- field_intvalue > static_cast<uint64_t>(end - new_pos)) {
- return field;
- }
+ case static_cast<uint8_t>(ProtoWireType::kLengthDelimited): {
+ uint64_t payload_length;
+ new_pos = ParseVarInt(pos, end, &payload_length);
+ if (PERFETTO_UNLIKELY(new_pos == pos))
+ return res;
+
+ // ParseVarInt guarantees that |new_pos| <= |end| when it succeeds;
+ if (payload_length > static_cast<uint64_t>(end - new_pos))
+ return res;
// If the message is larger than 256 MiB silently skip it.
- if (field_intvalue >= proto_utils::kMaxMessageLength) {
- current_position_ = new_pos + field_intvalue;
- return ReadField();
+ if (PERFETTO_LIKELY(payload_length <= proto_utils::kMaxMessageLength)) {
+ const uintptr_t payload_start = reinterpret_cast<uintptr_t>(new_pos);
+ int_value = payload_start;
+ size = static_cast<uint32_t>(payload_length);
}
- pos = new_pos;
- field.length_limited.data = pos;
- field.length_limited.length = static_cast<size_t>(field_intvalue);
- pos += field_intvalue;
+ new_pos += payload_length;
break;
}
- case ProtoWireType::kFixed64: {
- if (pos + sizeof(uint64_t) > end) {
- return field;
- }
- memcpy(&field_intvalue, pos, sizeof(uint64_t));
- field.int_value = BYTE_SWAP_TO_LE64(field_intvalue);
- pos += sizeof(uint64_t);
+
+ case static_cast<uint8_t>(ProtoWireType::kFixed64): {
+ new_pos = pos + sizeof(uint64_t);
+ if (PERFETTO_UNLIKELY(new_pos > end))
+ return res;
+ memcpy(&int_value, pos, sizeof(uint64_t));
break;
}
- case ProtoWireType::kFixed32: {
- if (pos + sizeof(uint32_t) > end) {
- return field;
- }
- uint32_t tmp;
- memcpy(&tmp, pos, sizeof(uint32_t));
- field.int_value = BYTE_SWAP_TO_LE32(tmp);
- pos += sizeof(uint32_t);
+
+ case static_cast<uint8_t>(ProtoWireType::kFixed32): {
+ new_pos = pos + sizeof(uint32_t);
+ if (PERFETTO_UNLIKELY(new_pos > end))
+ return res;
+ memcpy(&int_value, pos, sizeof(uint32_t));
+ break;
+ }
+
+ default:
+ PERFETTO_DLOG("Invalid proto field type: %u", field_type);
+ return res;
+ }
+
+ if (PERFETTO_UNLIKELY(field_id > std::numeric_limits<uint16_t>::max())) {
+ PERFETTO_DFATAL("Cannot parse proto field ids > 0xFFFF");
+ return res;
+ }
+
+ res.field.initialize(static_cast<uint16_t>(field_id), field_type, int_value,
+ size);
+ res.next = new_pos;
+ return res;
+}
+
+} // namespace
+
+Field ProtoDecoder::FindField(uint32_t field_id) {
+ Field res;
+ auto old_position = read_ptr_;
+ read_ptr_ = begin_;
+ for (auto f = ReadField(); f.valid(); f = ReadField()) {
+ if (f.id() == field_id) {
+ res = f;
break;
}
}
- // Set the field id to make the returned value valid and update the current
- // position in the buffer.
- field.id = field_id;
- current_position_ = pos;
- return field;
+ read_ptr_ = old_position;
+ return res;
+}
+
+PERFETTO_ALWAYS_INLINE
+Field ProtoDecoder::ReadField() {
+ ParseFieldResult res = ParseOneField(read_ptr_, end_);
+ read_ptr_ = res.next;
+ return res.field;
+}
+
+void TypedProtoDecoderBase::ParseAllFields() {
+ const uint8_t* cur = begin_;
+ ParseFieldResult res;
+ for (;;) {
+ res = ParseOneField(cur, end_);
+ if (res.next == cur)
+ break;
+ cur = res.next;
+
+ auto field_id = res.field.id();
+ if (PERFETTO_UNLIKELY(field_id >= size_))
+ continue;
+
+ Field* fld = &fields_[field_id];
+ if (PERFETTO_LIKELY(!fld->valid())) {
+ // This is the first time we see this field.
+ *fld = std::move(res.field);
+ } else {
+ // Repeated field case. Append to the end of the |fields_| vector.
+ // The RepeatedFieldIterator will find first the one at fields_[id] and
+ // then will keep finding the other repeated fields with matching id.
+ if (PERFETTO_UNLIKELY(size_ == capacity_)) {
+ ExpandHeapStorage();
+ PERFETTO_DCHECK(size_ < capacity_);
+ }
+ fields_[size_++] = std::move(res.field);
+ }
+ }
+ read_ptr_ = res.next;
+}
+
+void TypedProtoDecoderBase::ExpandHeapStorage() {
+ uint32_t new_capacity = capacity_ * 2;
+ PERFETTO_CHECK(new_capacity > size_);
+ std::unique_ptr<Field[]> new_storage(new Field[new_capacity]);
+
+ static_assert(std::is_trivially_copyable<Field>::value,
+ "Field must be trivially copyable");
+ memcpy(&new_storage[0], fields_, sizeof(Field) * size_);
+
+ heap_storage_ = std::move(new_storage);
+ fields_ = &heap_storage_[0];
+ capacity_ = new_capacity;
}
} // namespace protozero
diff --git a/src/protozero/proto_decoder_fuzzer.cc b/src/protozero/proto_decoder_fuzzer.cc
new file mode 100644
index 0000000..6a6d216
--- /dev/null
+++ b/src/protozero/proto_decoder_fuzzer.cc
@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "perfetto/base/logging.h"
+#include "perfetto/protozero/proto_decoder.h"
+
+namespace protozero {
+namespace {
+
+int FuzzProtoDecoder(const uint8_t* data, size_t size) {
+ volatile uint64_t value = 0;
+ ProtoDecoder dec(data, size);
+ for (auto field = dec.ReadField(); field.valid(); field = dec.ReadField()) {
+ value += field.raw_int_value();
+ }
+ TypedProtoDecoder<1, 0> typed_decoder_1(data, size);
+ TypedProtoDecoder<999, 0> typed_decoder_2(data, size);
+ return 0;
+}
+
+} // namespace
+} // namespace protozero
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size);
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+ return protozero::FuzzProtoDecoder(data, size);
+}
diff --git a/src/protozero/proto_decoder_unittest.cc b/src/protozero/proto_decoder_unittest.cc
index 8208beb..342d1e7 100644
--- a/src/protozero/proto_decoder_unittest.cc
+++ b/src/protozero/proto_decoder_unittest.cc
@@ -31,7 +31,7 @@
using ::testing::Invoke;
using namespace proto_utils;
-TEST(ProtoDecoder, ReadString) {
+TEST(ProtoDecoderTest, ReadString) {
Message message;
ScatteredHeapBuffer delegate(512, 512);
ScatteredStreamWriter writer(&delegate);
@@ -44,18 +44,17 @@
delegate.AdjustUsedSizeOfCurrentSlice();
auto used_range = delegate.slices()[0].GetUsedRange();
- ProtoDecoder decoder(used_range.begin, used_range.size());
- ProtoDecoder::Field field = decoder.ReadField();
+ TypedProtoDecoder<32, false> decoder(used_range.begin, used_range.size());
- ASSERT_EQ(field.id, 1u);
- ASSERT_EQ(field.type, ProtoWireType::kLengthDelimited);
- ASSERT_EQ(field.length_limited.length, sizeof(kTestString) - 1);
+ const auto& field = decoder.Get(1);
+ ASSERT_EQ(field.type(), ProtoWireType::kLengthDelimited);
+ ASSERT_EQ(field.size(), sizeof(kTestString) - 1);
for (size_t i = 0; i < sizeof(kTestString) - 1; i++) {
- ASSERT_EQ(field.length_limited.data[i], kTestString[i]);
+ ASSERT_EQ(field.data()[i], kTestString[i]);
}
}
-TEST(ProtoDecoder, VeryLargeField) {
+TEST(ProtoDecoderTest, VeryLargeField) {
const uint64_t size = 512 * 1024 * 1024 + 6;
std::unique_ptr<uint8_t, perfetto::base::FreeDeleter> data(
static_cast<uint8_t*>(malloc(size)));
@@ -68,12 +67,14 @@
data.get()[5] = static_cast<unsigned char>('\x02');
ProtoDecoder decoder(data.get(), size);
- ProtoDecoder::Field field = decoder.ReadField();
- ASSERT_EQ(0u, field.id);
- ASSERT_TRUE(decoder.IsEndOfBuffer());
+ Field field = decoder.ReadField();
+ ASSERT_EQ(1u, field.id());
+ ASSERT_EQ(nullptr, field.data());
+ ASSERT_EQ(0u, field.size());
+ ASSERT_EQ(0u, decoder.bytes_left());
}
-TEST(ProtoDecoder, FixedData) {
+TEST(ProtoDecoderTest, FixedData) {
struct FieldExpectation {
const char* encoded;
size_t encoded_size;
@@ -87,43 +88,48 @@
{"\x08\x01", 2, 1, ProtoWireType::kVarInt, 1},
{"\x08\x42", 2, 1, ProtoWireType::kVarInt, 0x42},
{"\xF8\x07\x42", 3, 127, ProtoWireType::kVarInt, 0x42},
- {"\x90\x4D\xFF\xFF\xFF\xFF\x0F", 7, 1234, ProtoWireType::kVarInt,
+ {"\xB8\x3E\xFF\xFF\xFF\xFF\x0F", 7, 999, ProtoWireType::kVarInt,
0xFFFFFFFF},
{"\x7D\x42\x00\x00\x00", 5, 15, ProtoWireType::kFixed32, 0x42},
- {"\x95\x4D\x78\x56\x34\x12", 6, 1234, ProtoWireType::kFixed32,
- 0x12345678},
+ {"\xBD\x3E\x78\x56\x34\x12", 6, 999, ProtoWireType::kFixed32, 0x12345678},
{"\x79\x42\x00\x00\x00\x00\x00\x00\x00", 9, 15, ProtoWireType::kFixed64,
0x42},
- {"\x91\x4D\x08\x07\x06\x05\x04\x03\x02\x01", 10, 1234,
+ {"\xB9\x3E\x08\x07\x06\x05\x04\x03\x02\x01", 10, 999,
ProtoWireType::kFixed64, 0x0102030405060708},
{"\x0A\x00", 2, 1, ProtoWireType::kLengthDelimited, 0},
{"\x0A\x04|abc", 6, 1, ProtoWireType::kLengthDelimited, 4},
- {"\x92\x4D\x04|abc", 7, 1234, ProtoWireType::kLengthDelimited, 4},
- {"\x92\x4D\x83\x01|abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzab"
+ {"\xBA\x3E\x04|abc", 7, 999, ProtoWireType::kLengthDelimited, 4},
+ {"\xBA\x3E\x83\x01|abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzab"
"cdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstu"
"vwx",
- 135, 1234, ProtoWireType::kLengthDelimited, 131},
+ 135, 999, ProtoWireType::kLengthDelimited, 131},
};
for (size_t i = 0; i < perfetto::base::ArraySize(kFieldExpectations); ++i) {
const FieldExpectation& exp = kFieldExpectations[i];
- ProtoDecoder decoder(reinterpret_cast<const uint8_t*>(exp.encoded),
- exp.encoded_size);
+ TypedProtoDecoder<999, 0> decoder(
+ reinterpret_cast<const uint8_t*>(exp.encoded), exp.encoded_size);
- ProtoDecoder::Field field = decoder.ReadField();
- ASSERT_EQ(exp.id, field.id);
- ASSERT_EQ(exp.type, field.type);
+ auto& field = decoder.Get(exp.id);
+ ASSERT_EQ(exp.type, field.type());
- if (field.type == ProtoWireType::kLengthDelimited) {
- ASSERT_EQ(exp.int_value, field.length_limited.length);
+ if (field.type() == ProtoWireType::kLengthDelimited) {
+ ASSERT_EQ(exp.int_value, field.size());
} else {
- ASSERT_EQ(exp.int_value, field.int_value);
+ ASSERT_EQ(exp.int_value, field.as_int64());
// Proto encodes booleans as varints of 0 or 1.
if (exp.int_value == 0 || exp.int_value == 1) {
ASSERT_EQ(exp.int_value, field.as_bool());
}
}
}
+
+ // Test float and doubles decoding.
+ const char buf[] = "\x0d\x00\x00\xa0\x3f\x11\x00\x00\x00\x00\x00\x42\x8f\xc0";
+ TypedProtoDecoder<2, 0> decoder(reinterpret_cast<const uint8_t*>(buf),
+ sizeof(buf));
+ EXPECT_FLOAT_EQ(decoder.Get(1).as_float(), 1.25f);
+ EXPECT_DOUBLE_EQ(decoder.Get(2).as_double(), -1000.25);
}
} // namespace
diff --git a/src/protozero/proto_field_descriptor.cc b/src/protozero/proto_field_descriptor.cc
deleted file mode 100644
index d94dfa9..0000000
--- a/src/protozero/proto_field_descriptor.cc
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "perfetto/protozero/proto_field_descriptor.h"
-
-namespace protozero {
-
-// static
-const ProtoFieldDescriptor* ProtoFieldDescriptor::GetInvalidInstance() {
- static ProtoFieldDescriptor instance("", Type::TYPE_INVALID, 0, false);
- return &instance;
-}
-
-} // namespace protozero
diff --git a/src/protozero/protoc_plugin/protozero_generator.cc b/src/protozero/protoc_plugin/protozero_generator.cc
index b9824c2..19d4c70 100644
--- a/src/protozero/protoc_plugin/protozero_generator.cc
+++ b/src/protozero/protoc_plugin/protozero_generator.cc
@@ -45,6 +45,11 @@
namespace {
+// Keep this value in sync with ProtoDecoder::kMaxDecoderFieldId. If they go out
+// of sync pbzero.h files will stop compiling, hitting the at() static_assert.
+// Not worth an extra dependency.
+constexpr int kMaxDecoderFieldId = 999;
+
void Assert(bool condition) {
if (!condition)
__builtin_trap();
@@ -274,7 +279,7 @@
"#include <stddef.h>\n"
"#include <stdint.h>\n\n"
"#include \"perfetto/base/export.h\"\n"
- "#include \"perfetto/protozero/proto_field_descriptor.h\"\n"
+ "#include \"perfetto/protozero/proto_decoder.h\"\n"
"#include \"perfetto/protozero/message.h\"\n",
"greeting", greeting, "guard", guard);
@@ -459,7 +464,117 @@
action, "inner_class", inner_class);
}
- void GenerateReflectionForMessageFields(const Descriptor* message) {
+ void GenerateDecoder(const Descriptor* message) {
+ int max_field_id = 0;
+ bool has_repeated_fields = false;
+ for (int i = 0; i < message->field_count(); ++i) {
+ const FieldDescriptor* field = message->field(i);
+ if (field->number() > kMaxDecoderFieldId)
+ continue;
+ max_field_id = std::max(max_field_id, field->number());
+ if (field->is_repeated())
+ has_repeated_fields = true;
+ }
+
+ stub_h_->Print(
+ "class Decoder : public "
+ "::protozero::TypedProtoDecoder</*MAX_FIELD_ID=*/$max$, "
+ "/*HAS_REPEATED_FIELDS=*/$rep$> {\n",
+ "max", std::to_string(max_field_id), "rep",
+ has_repeated_fields ? "true" : "false");
+ stub_h_->Print(" public:\n");
+ stub_h_->Indent();
+ stub_h_->Print(
+ "Decoder(const uint8_t* data, size_t len) "
+ ": TypedProtoDecoder(data, len) {}\n");
+
+ for (int i = 0; i < message->field_count(); ++i) {
+ const FieldDescriptor* field = message->field(i);
+ if (field->is_packed()) {
+ Abort("Packed repeated fields are not supported.");
+ return;
+ }
+
+ if (field->number() > max_field_id) {
+ stub_h_->Print("// field $name$ omitted because its id is too high",
+ "name", field->name());
+ continue;
+ }
+ std::string getter;
+ std::string cpp_type;
+ switch (field->type()) {
+ case FieldDescriptor::TYPE_BOOL:
+ getter = "as_bool";
+ cpp_type = "bool";
+ break;
+ case FieldDescriptor::TYPE_SFIXED32:
+ case FieldDescriptor::TYPE_SINT32:
+ case FieldDescriptor::TYPE_INT32:
+ getter = "as_int32";
+ cpp_type = "int32_t";
+ break;
+ case FieldDescriptor::TYPE_SFIXED64:
+ case FieldDescriptor::TYPE_SINT64:
+ case FieldDescriptor::TYPE_INT64:
+ getter = "as_int64";
+ cpp_type = "int64_t";
+ break;
+ case FieldDescriptor::TYPE_FIXED32:
+ case FieldDescriptor::TYPE_UINT32:
+ getter = "as_uint32";
+ cpp_type = "uint32_t";
+ break;
+ case FieldDescriptor::TYPE_FIXED64:
+ case FieldDescriptor::TYPE_UINT64:
+ getter = "as_uint64";
+ cpp_type = "uint64_t";
+ break;
+ case FieldDescriptor::TYPE_FLOAT:
+ getter = "as_float";
+ cpp_type = "float";
+ break;
+ case FieldDescriptor::TYPE_DOUBLE:
+ getter = "as_double";
+ cpp_type = "double";
+ break;
+ case FieldDescriptor::TYPE_ENUM:
+ getter = "as_int32";
+ cpp_type = "int32_t";
+ break;
+ case FieldDescriptor::TYPE_STRING:
+ getter = "as_string";
+ cpp_type = "::protozero::ConstChars";
+ break;
+ case FieldDescriptor::TYPE_MESSAGE:
+ case FieldDescriptor::TYPE_BYTES:
+ getter = "as_bytes";
+ cpp_type = "::protozero::ConstBytes";
+ break;
+ case FieldDescriptor::TYPE_GROUP:
+ continue;
+ }
+
+ stub_h_->Print("bool has_$name$() const { return at<$id$>().valid(); }\n",
+ "name", field->name(), "id",
+ std::to_string(field->number()));
+
+ if (field->is_repeated()) {
+ stub_h_->Print(
+ "::protozero::RepeatedFieldIterator $name$() const { return "
+ "GetRepeated($id$); }\n",
+ "name", field->name(), "id", std::to_string(field->number()));
+ } else {
+ stub_h_->Print(
+ "$cpp_type$ $name$() const { return at<$id$>().$getter$(); }\n",
+ "name", field->name(), "id", std::to_string(field->number()),
+ "cpp_type", cpp_type, "getter", getter);
+ }
+ }
+ stub_h_->Outdent();
+ stub_h_->Print("};\n");
+ }
+
+ void GenerateConstantsForMessageFields(const Descriptor* message) {
const bool has_fields = (message->field_count() > 0);
// Field number constants.
@@ -476,47 +591,6 @@
stub_h_->Outdent();
stub_h_->Print("};\n");
}
-
- // Fields reflection table.
- std::string class_name = GetCppClassName(message);
-
- // Fields reflection getter.
- stub_h_->Print(
- "static const ::protozero::ProtoFieldDescriptor "
- "GetFieldDescriptor(uint32_t field_id) {\n");
- stub_h_->Indent();
- if (has_fields) {
- stub_h_->Print("switch (field_id) {\n");
- stub_h_->Indent();
- for (int i = 0; i < message->field_count(); ++i) {
- const FieldDescriptor* field = message->field(i);
- std::string type_const =
- std::string("TYPE_") + FieldDescriptor::TypeName(field->type());
- UpperString(&type_const);
- stub_h_->Print(
- "case $field$:\n"
- " return {\"$name$\", "
- "::protozero::ProtoFieldDescriptor::Type::$type$, $number$, "
- "$is_repeated$};\n",
- "class", class_name, "field",
- GetFieldNumberConstant(message->field(i)), "id", std::to_string(i),
- "name", field->name(), "type", type_const, "number",
- std::to_string(field->number()), "is_repeated",
- std::to_string(field->is_repeated()));
- }
- stub_h_->Print(
- "default:\n"
- " return "
- "*::protozero::ProtoFieldDescriptor::GetInvalidInstance();\n");
- stub_h_->Outdent();
- stub_h_->Print("}\n");
- } else {
- stub_h_->Print(
- "(void)(field_id);\n"
- "return *::protozero::ProtoFieldDescriptor::GetInvalidInstance();\n");
- }
- stub_h_->Outdent();
- stub_h_->Print("}\n\n");
}
void GenerateMessageDescriptor(const Descriptor* message) {
@@ -526,7 +600,8 @@
"name", GetCppClassName(message));
stub_h_->Indent();
- GenerateReflectionForMessageFields(message);
+ GenerateConstantsForMessageFields(message);
+ GenerateDecoder(message);
// Using statements for nested messages.
for (int i = 0; i < message->nested_type_count(); ++i) {
diff --git a/src/protozero/test/protozero_conformance_unittest.cc b/src/protozero/test/protozero_conformance_unittest.cc
index 50c0a94..034487f 100644
--- a/src/protozero/test/protozero_conformance_unittest.cc
+++ b/src/protozero/test/protozero_conformance_unittest.cc
@@ -163,8 +163,7 @@
EXPECT_LE(0u, sizeof(pbtest::TrickyPublicImport));
}
-TEST(ProtoZeroTest, Reflection) {
- // Tests camel case conversion as well.
+TEST(ProtoZeroTest, FieldNumbers) {
EXPECT_EQ(1, pbtest::CamelCaseFields::kFooBarBazFieldNumber);
EXPECT_EQ(2, pbtest::CamelCaseFields::kBarBazFieldNumber);
EXPECT_EQ(3, pbtest::CamelCaseFields::kMooMooFieldNumber);
@@ -174,16 +173,6 @@
EXPECT_EQ(7, pbtest::CamelCaseFields::kBigBangFieldNumber);
EXPECT_EQ(8, pbtest::CamelCaseFields::kU2FieldNumber);
EXPECT_EQ(9, pbtest::CamelCaseFields::kBangBigFieldNumber);
-
- ProtoFieldDescriptor reflection = pbtest::EveryField::GetFieldDescriptor(
- pbtest::EveryField::kFieldInt32FieldNumber);
- EXPECT_STREQ("field_int32", reflection.name());
- EXPECT_EQ(ProtoFieldDescriptor::Type::TYPE_INT32, reflection.type());
- EXPECT_EQ(1u, reflection.number());
- EXPECT_FALSE(reflection.is_repeated());
- EXPECT_TRUE(reflection.is_valid());
-
- EXPECT_FALSE(pbtest::TransgalacticParcel::GetFieldDescriptor(42).is_valid());
}
} // namespace
diff --git a/src/trace_processor/BUILD.gn b/src/trace_processor/BUILD.gn
index 57592b0..5aa0251 100644
--- a/src/trace_processor/BUILD.gn
+++ b/src/trace_processor/BUILD.gn
@@ -123,12 +123,16 @@
"window_operator_table.cc",
"window_operator_table.h",
]
+
+ # TODO(primiano): remove :lite deps and depend only on protozero targets.
deps = [
"../../buildtools:sqlite",
"../../gn:default_deps",
"../../include/perfetto/traced:sys_stats_counters",
"../../protos/perfetto/trace:lite",
+ "../../protos/perfetto/trace:zero",
"../../protos/perfetto/trace/ftrace:lite",
+ "../../protos/perfetto/trace/ftrace:zero",
"../../protos/perfetto/trace_processor:lite",
"../base",
"../protozero",
diff --git a/src/trace_processor/ftrace_descriptors.h b/src/trace_processor/ftrace_descriptors.h
index bd32847..71e58d0 100644
--- a/src/trace_processor/ftrace_descriptors.h
+++ b/src/trace_processor/ftrace_descriptors.h
@@ -25,6 +25,10 @@
using protozero::proto_utils::ProtoSchemaType;
+// We assume that no ftrace event (e.g. SchedSwitchFtraceEvent) has a proto
+// field which id is >= this.
+static constexpr size_t kMaxFtraceEventFields = 32;
+
// This file is the header for the generated descriptors for all ftrace event
// protos. These descriptors can be used to parse ftrace event protos without
// needing individual parsing logic for every event. (In proto_trace_parser.cc)
@@ -39,7 +43,7 @@
struct MessageDescriptor {
const char* name;
size_t max_field_id;
- FieldDescriptor fields[32];
+ FieldDescriptor fields[kMaxFtraceEventFields];
};
MessageDescriptor* GetMessageDescriptorForId(size_t id);
diff --git a/src/trace_processor/proto_trace_parser.cc b/src/trace_processor/proto_trace_parser.cc
index 7fbeb93..9bcad0f 100644
--- a/src/trace_processor/proto_trace_parser.cc
+++ b/src/trace_processor/proto_trace_parser.cc
@@ -267,8 +267,8 @@
void ProtoTraceParser::ParseTracePacket(int64_t ts, TraceBlobView packet) {
ProtoDecoder decoder(packet.data(), packet.length());
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::TracePacket::kProcessTreeFieldNumber: {
const size_t fld_off = packet.offset_of(fld.data());
ParseProcessTree(packet.slice(fld_off, fld.size()));
@@ -323,13 +323,13 @@
// needs to be handled carefully.
context_->args_tracker->Flush();
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseSysStats(int64_t ts, TraceBlobView stats) {
ProtoDecoder decoder(stats.data(), stats.length());
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::SysStats::kMeminfoFieldNumber: {
const size_t fld_off = stats.offset_of(fld.data());
ParseMemInfo(ts, stats.slice(fld_off, fld.size()));
@@ -384,8 +384,8 @@
ProtoDecoder decoder(irq.data(), irq.length());
uint32_t key = 0;
uint32_t value = 0;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::SysStats::InterruptCount::kIrqFieldNumber:
key = fld.as_uint32();
break;
@@ -403,8 +403,8 @@
ProtoDecoder decoder(mem.data(), mem.length());
uint32_t key = 0;
uint32_t value = 0;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::SysStats::MeminfoValue::kKeyFieldNumber:
key = fld.as_uint32();
break;
@@ -427,8 +427,8 @@
ProtoDecoder decoder(stat.data(), stat.length());
uint32_t key = 0;
uint32_t value = 0;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::SysStats::VmstatValue::kKeyFieldNumber:
key = fld.as_uint32();
break;
@@ -457,9 +457,10 @@
cpu_times.data()[1] < 0x80) {
raw_cpu = cpu_times.data()[1];
} else {
- if (!PERFETTO_LIKELY((
- decoder.FindIntField<protos::SysStats::CpuTimes::kCpuIdFieldNumber>(
- &raw_cpu)))) {
+ if (auto cpu_field =
+ decoder.FindField(protos::SysStats::CpuTimes::kCpuIdFieldNumber)) {
+ raw_cpu = cpu_field.as_uint64();
+ } else {
PERFETTO_ELOG("CPU field not found in CpuTimes");
context_->storage->IncrementStats(stats::invalid_cpu_times);
return;
@@ -467,8 +468,8 @@
}
int64_t cpu = static_cast<int64_t>(raw_cpu);
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::SysStats::CpuTimes::kUserNsFieldNumber: {
value = fld.as_uint32();
context_->event_tracker->PushCounter(ts, value, cpu_times_user_ns_id_,
@@ -520,9 +521,9 @@
void ProtoTraceParser::ParseProcessTree(TraceBlobView pstree) {
ProtoDecoder decoder(pstree.data(), pstree.length());
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
const size_t fld_off = pstree.offset_of(fld.data());
- switch (fld.id) {
+ switch (fld.id()) {
case protos::ProcessTree::kProcessesFieldNumber: {
ParseProcess(pstree.slice(fld_off, fld.size()));
break;
@@ -535,15 +536,15 @@
break;
}
}
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseProcessStats(int64_t ts, TraceBlobView stats) {
ProtoDecoder decoder(stats.data(), stats.length());
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
const size_t fld_off = stats.offset_of(fld.data());
- switch (fld.id) {
+ switch (fld.id()) {
case protos::ProcessStats::kProcessesFieldNumber: {
ParseProcessStatsProcess(ts, stats.slice(fld_off, fld.size()));
break;
@@ -552,7 +553,7 @@
break;
}
}
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseProcessStatsProcess(int64_t ts,
@@ -565,22 +566,22 @@
std::array<int64_t, kProcStatsProcessSize> counter_values{};
std::array<bool, kProcStatsProcessSize> has_counter{};
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::ProcessStats::Process::kPidFieldNumber:
pid = fld.as_uint32();
break;
default: {
- bool is_counter_field = fld.id < has_counter.size() &&
- proc_stats_process_names_[fld.id] != 0;
+ bool is_counter_field = fld.id() < has_counter.size() &&
+ proc_stats_process_names_[fld.id()] != 0;
if (is_counter_field) {
// Memory counters are in KB, keep values in bytes in the trace
// processor.
- counter_values[fld.id] =
- fld.id == protos::ProcessStats::Process::kOomScoreAdjFieldNumber
+ counter_values[fld.id()] =
+ fld.id() == protos::ProcessStats::Process::kOomScoreAdjFieldNumber
? fld.as_int64()
: fld.as_int64() * 1024;
- has_counter[fld.id] = true;
+ has_counter[fld.id()] = true;
} else {
context_->storage->IncrementStats(stats::proc_stat_unknown_counters);
}
@@ -604,15 +605,15 @@
RefType::kRefUpid);
}
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseThread(TraceBlobView thread) {
ProtoDecoder decoder(thread.data(), thread.length());
uint32_t tid = 0;
uint32_t tgid = 0;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::ProcessTree::Thread::kTidFieldNumber:
tid = fld.as_uint32();
break;
@@ -625,7 +626,7 @@
}
context_->process_tracker->UpdateThread(tid, tgid);
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseProcess(TraceBlobView process) {
@@ -635,8 +636,8 @@
uint32_t ppid = 0;
base::StringView process_name;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::ProcessTree::Process::kPidFieldNumber:
pid = fld.as_uint32();
break;
@@ -653,7 +654,7 @@
}
context_->process_tracker->UpdateProcess(pid, ppid, process_name);
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseFtracePacket(uint32_t cpu,
@@ -661,31 +662,32 @@
TraceBlobView ftrace) {
ProtoDecoder decoder(ftrace.data(), ftrace.length());
uint64_t raw_pid = 0;
- if (!PERFETTO_LIKELY(
- (decoder.FindIntField<protos::FtraceEvent::kPidFieldNumber>(
- &raw_pid)))) {
+ if (auto pid_field =
+ decoder.FindField(protos::FtraceEvent::kPidFieldNumber)) {
+ raw_pid = pid_field.as_uint64();
+ } else {
PERFETTO_ELOG("Pid field not found in ftrace packet");
return;
}
uint32_t pid = static_cast<uint32_t>(raw_pid);
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
bool is_metadata_field =
- fld.id == protos::FtraceEvent::kPidFieldNumber ||
- fld.id == protos::FtraceEvent::kTimestampFieldNumber;
+ fld.id() == protos::FtraceEvent::kPidFieldNumber ||
+ fld.id() == protos::FtraceEvent::kTimestampFieldNumber;
if (is_metadata_field)
continue;
const size_t fld_off = ftrace.offset_of(fld.data());
- if (fld.id == protos::FtraceEvent::kGenericFieldNumber) {
+ if (fld.id() == protos::FtraceEvent::kGenericFieldNumber) {
ParseGenericFtrace(timestamp, cpu, pid,
ftrace.slice(fld_off, fld.size()));
- } else if (fld.id != protos::FtraceEvent::kSchedSwitchFieldNumber) {
- ParseTypedFtraceToRaw(fld.id, timestamp, cpu, pid,
+ } else if (fld.id() != protos::FtraceEvent::kSchedSwitchFieldNumber) {
+ ParseTypedFtraceToRaw(fld.id(), timestamp, cpu, pid,
ftrace.slice(fld_off, fld.size()));
}
- switch (fld.id) {
+ switch (fld.id()) {
case protos::FtraceEvent::kSchedSwitchFieldNumber: {
ParseSchedSwitch(cpu, timestamp, ftrace.slice(fld_off, fld.size()));
break;
@@ -766,7 +768,7 @@
// needs to be handled carefully.
context_->args_tracker->Flush();
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseSignalDeliver(int64_t timestamp,
@@ -774,8 +776,8 @@
TraceBlobView view) {
ProtoDecoder decoder(view.data(), view.length());
uint32_t sig = 0;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::SignalDeliverFtraceEvent::kSigFieldNumber:
sig = fld.as_uint32();
break;
@@ -794,8 +796,8 @@
ProtoDecoder decoder(view.data(), view.length());
uint32_t pid = 0;
uint32_t sig = 0;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::SignalGenerateFtraceEvent::kPidFieldNumber:
pid = fld.as_uint32();
break;
@@ -817,8 +819,8 @@
ProtoDecoder decoder(view.data(), view.length());
uint32_t pid = 0;
base::StringView comm;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::LowmemoryKillFtraceEvent::kPidFieldNumber:
pid = fld.as_uint32();
break;
@@ -848,8 +850,8 @@
const auto kRssStatUnknown = static_cast<uint32_t>(rss_members_.size()) - 1;
uint32_t member = kRssStatUnknown;
int64_t size = 0;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::RssStatFtraceEvent::kMemberFieldNumber:
member = fld.as_uint32();
break;
@@ -871,7 +873,7 @@
} else {
context_->storage->IncrementStats(stats::rss_stat_negative_size);
}
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseIonHeapGrowOrShrink(int64_t timestamp,
@@ -883,8 +885,8 @@
int64_t change_bytes = 0;
StringId global_name_id = ion_total_unknown_id_;
StringId change_name_id = ion_change_unknown_id_;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::IonHeapGrowFtraceEvent::kTotalAllocatedFieldNumber:
total_bytes = fld.as_int64();
break;
@@ -917,7 +919,7 @@
utid, RefType::kRefUtid);
context_->event_tracker->PushCounter(timestamp + 1, 0, change_name_id, utid,
RefType::kRefUtid);
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
// We are reusing the same function for ion_heap_grow and ion_heap_shrink.
// It is fine as the arguments are the same, but we need to be sure that the
@@ -937,8 +939,8 @@
uint32_t cpu_affected = 0;
uint32_t new_freq = 0;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::CpuFrequencyFtraceEvent::kCpuIdFieldNumber:
cpu_affected = fld.as_uint32();
break;
@@ -949,7 +951,7 @@
}
context_->event_tracker->PushCounter(timestamp, new_freq, cpu_freq_name_id_,
cpu_affected, RefType::kRefCpuId);
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseCpuIdle(int64_t timestamp, TraceBlobView view) {
@@ -957,8 +959,8 @@
uint32_t cpu_affected = 0;
uint32_t new_state = 0;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::CpuIdleFtraceEvent::kCpuIdFieldNumber:
cpu_affected = fld.as_uint32();
break;
@@ -969,7 +971,7 @@
}
context_->event_tracker->PushCounter(timestamp, new_state, cpu_idle_name_id_,
cpu_affected, RefType::kRefCpuId);
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseSchedSwitch(uint32_t cpu,
@@ -984,8 +986,8 @@
base::StringView next_comm;
uint32_t next_pid = 0;
int32_t next_prio = 0;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::SchedSwitchFtraceEvent::kPrevPidFieldNumber:
prev_pid = fld.as_uint32();
break;
@@ -1014,7 +1016,7 @@
context_->event_tracker->PushSchedSwitch(cpu, timestamp, prev_pid, prev_comm,
prev_prio, prev_state, next_pid,
next_comm, next_prio);
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseSchedWakeup(int64_t timestamp,
@@ -1023,8 +1025,8 @@
base::StringView comm;
uint32_t wakee_pid = 0;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::SchedWakeupFtraceEvent::kCommFieldNumber:
comm = fld.as_string();
break;
@@ -1038,7 +1040,7 @@
context_->process_tracker->UpdateThread(timestamp, wakee_pid, name_id);
context_->storage->mutable_instants()->AddInstantEvent(
timestamp, sched_wakeup_name_id_, 0 /* value */, utid, RefType::kRefUtid);
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseTaskNewTask(int64_t timestamp,
@@ -1049,8 +1051,8 @@
uint32_t new_tid = 0;
StringId new_comm = 0;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::TaskNewtaskFtraceEvent::kCloneFlagsFieldNumber:
clone_flags = fld.as_uint32();
break;
@@ -1088,8 +1090,8 @@
uint32_t tid = 0;
StringId comm = 0;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::TaskRenameFtraceEvent::kPidFieldNumber:
tid = fld.as_uint32();
break;
@@ -1111,8 +1113,8 @@
ProtoDecoder decoder(print.data(), print.length());
base::StringView buf{};
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- if (fld.id == protos::PrintFtraceEvent::kBufFieldNumber) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ if (fld.id() == protos::PrintFtraceEvent::kBufFieldNumber) {
buf = fld.as_string();
break;
}
@@ -1160,13 +1162,13 @@
upid, RefType::kRefUpid);
}
}
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseBatteryCounters(int64_t ts, TraceBlobView battery) {
ProtoDecoder decoder(battery.data(), battery.length());
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::BatteryCounters::kChargeCounterUahFieldNumber:
context_->event_tracker->PushCounter(
ts, fld.as_int64(), batt_charge_id_, 0, RefType::kRefNoRef);
@@ -1188,7 +1190,7 @@
break;
}
}
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseOOMScoreAdjUpdate(int64_t ts,
@@ -1197,8 +1199,8 @@
uint32_t pid = 0;
int16_t oom_adj = 0;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::OomScoreAdjUpdateFtraceEvent::kOomScoreAdjFieldNumber:
// TODO(b/120618641): The int16_t static cast is required because of
// the linked negative varint encoding bug.
@@ -1212,7 +1214,7 @@
break;
}
}
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
UniquePid upid = context_->process_tracker->UpdateProcess(pid);
context_->event_tracker->PushCounter(ts, oom_adj, oom_score_adj_id_, upid,
@@ -1228,8 +1230,8 @@
uint32_t count = 0;
uint32_t max_lat = 0;
uint32_t avg_lat = 0;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::MmEventRecordFtraceEvent::kTypeFieldNumber:
type = fld.as_uint32();
break;
@@ -1262,7 +1264,7 @@
context_->event_tracker->PushCounter(ts, avg_lat, counter_names.avg_lat, utid,
RefType::kRefUtidLookupUpid);
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseSysEvent(int64_t ts,
@@ -1272,8 +1274,8 @@
ProtoDecoder decoder(view.data(), view.length());
uint32_t id = 0;
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::SysEnterFtraceEvent::kIdFieldNumber:
id = fld.as_uint32();
break;
@@ -1315,9 +1317,10 @@
ProtoDecoder decoder(view.data(), view.length());
base::StringView event_name;
- if (!PERFETTO_LIKELY((decoder.FindStringField<
- protos::GenericFtraceEvent::kEventNameFieldNumber>(
- &event_name)))) {
+ if (auto name_field = decoder.FindField(
+ protos::GenericFtraceEvent::kEventNameFieldNumber)) {
+ event_name = name_field.as_string();
+ } else {
PERFETTO_ELOG("Event name not found in generic ftrace packet");
return;
}
@@ -1327,8 +1330,8 @@
RowId row_id = context_->storage->mutable_raw_events()->AddRawEvent(
timestamp, event_id, cpu, utid);
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::GenericFtraceEvent::kFieldFieldNumber:
const size_t fld_off = view.offset_of(fld.data());
ParseGenericFtraceField(row_id, view.slice(fld_off, fld.size()));
@@ -1342,20 +1345,21 @@
ProtoDecoder decoder(view.data(), view.length());
base::StringView field_name;
- if (!PERFETTO_LIKELY((decoder.FindStringField<
- protos::GenericFtraceEvent::Field::kNameFieldNumber>(
- &field_name)))) {
+ if (auto name_field = decoder.FindField(
+ protos::GenericFtraceEvent::Field::kNameFieldNumber)) {
+ field_name = name_field.as_string();
+ } else {
PERFETTO_ELOG("Event name not found in generic ftrace packet");
return;
}
auto field_name_id = context_->storage->InternString(std::move(field_name));
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::GenericFtraceEvent::Field::kIntValue:
case protos::GenericFtraceEvent::Field::kUintValue: {
context_->args_tracker->AddArg(generic_row_id, field_name_id,
field_name_id,
- Variadic::Integer(fld.as_integer()));
+ Variadic::Integer(fld.as_int64()));
break;
}
case protos::GenericFtraceEvent::Field::kStrValue: {
@@ -1384,15 +1388,15 @@
UniqueTid utid = context_->process_tracker->UpdateThread(timestamp, tid, 0);
RowId raw_event_id = context_->storage->mutable_raw_events()->AddRawEvent(
timestamp, message_strings.message_name_id, cpu, utid);
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- if (PERFETTO_UNLIKELY(fld.id >= kFtraceMaxFieldCount)) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ if (PERFETTO_UNLIKELY(fld.id() >= kMaxFtraceEventFields)) {
PERFETTO_DLOG(
- "Skipping ftrace arg - proto field id is too large (%" PRIu32 ")",
- fld.id);
+ "Skipping ftrace arg - proto field id is too large (%" PRIu16 ")",
+ fld.id());
continue;
}
- ProtoSchemaType type = m->fields[fld.id].type;
- StringId name_id = message_strings.field_name_ids[fld.id];
+ ProtoSchemaType type = m->fields[fld.id()].type;
+ StringId name_id = message_strings.field_name_ids[fld.id()];
switch (type) {
case ProtoSchemaType::kUint32:
case ProtoSchemaType::kInt32:
@@ -1407,7 +1411,7 @@
case ProtoSchemaType::kBool:
case ProtoSchemaType::kEnum: {
context_->args_tracker->AddArg(raw_event_id, name_id, name_id,
- Variadic::Integer(fld.as_integer()));
+ Variadic::Integer(fld.as_int64()));
break;
}
case ProtoSchemaType::kString:
@@ -1417,10 +1421,15 @@
Variadic::String(value));
break;
}
- case ProtoSchemaType::kDouble:
- case ProtoSchemaType::kFloat: {
+ case ProtoSchemaType::kDouble: {
context_->args_tracker->AddArg(raw_event_id, name_id, name_id,
- Variadic::Real(fld.as_real()));
+ Variadic::Real(fld.as_double()));
+ break;
+ }
+ case ProtoSchemaType::kFloat: {
+ context_->args_tracker->AddArg(
+ raw_event_id, name_id, name_id,
+ Variadic::Real(static_cast<double>(fld.as_float())));
break;
}
case ProtoSchemaType::kUnknown:
@@ -1440,8 +1449,8 @@
int64_t clock_realtime = 0;
// This loop iterates over the "repeated Clock" entries.
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::ClockSnapshot::kClocksFieldNumber: {
const size_t fld_off = packet.offset_of(fld.data());
auto clk = ParseClockField(packet.slice(fld_off, fld.size()));
@@ -1462,7 +1471,7 @@
break;
}
}
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
// Usually these snapshots come all together.
PERFETTO_DCHECK(clock_boottime > 0 && clock_monotonic > 0 &&
@@ -1495,8 +1504,8 @@
// This loop iterates over the |type| and |timestamp| field of each
// clock snapshot.
- for (auto fld = decoder.ReadField(); fld.id; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.id(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::ClockSnapshot::Clock::kTypeFieldNumber:
type = fld.as_int32();
break;
@@ -1510,8 +1519,8 @@
void ProtoTraceParser::ParseAndroidLogPacket(TraceBlobView packet) {
ProtoDecoder decoder(packet.data(), packet.length());
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::AndroidLogPacket::kEventsFieldNumber: {
const size_t fld_off = packet.offset_of(fld.data());
ParseAndroidLogEvent(packet.slice(fld_off, fld.size()));
@@ -1524,7 +1533,7 @@
}
}
}
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseAndroidLogEvent(TraceBlobView event) {
@@ -1543,8 +1552,8 @@
return sizeof(arg_msg) - static_cast<size_t>(arg_str - arg_msg);
};
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::AndroidLogPacket::LogEvent::kPidFieldNumber:
pid = fld.as_uint32();
break;
@@ -1573,7 +1582,7 @@
break;
}
}
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
if (prio == 0)
prio = protos::AndroidLogPriority::PRIO_INFO;
@@ -1599,8 +1608,8 @@
char** str,
size_t avail) {
ProtoDecoder decoder(arg.data(), arg.length());
- for (auto fld = decoder.ReadField(); fld.id; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.id(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::AndroidLogPacket::LogEvent::Arg::kNameFieldNumber: {
base::StringView name = fld.as_string();
*str += snprintf(*str, avail, " %.*s=", static_cast<int>(name.size()),
@@ -1626,8 +1635,8 @@
void ProtoTraceParser::ParseAndroidLogStats(TraceBlobView packet) {
ProtoDecoder decoder(packet.data(), packet.length());
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::AndroidLogPacket::Stats::kNumFailedFieldNumber:
context_->storage->SetStats(stats::android_log_num_failed,
fld.as_int64());
@@ -1642,15 +1651,15 @@
break;
}
}
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseTraceStats(TraceBlobView packet) {
ProtoDecoder decoder(packet.data(), packet.length());
int buf_num = 0;
auto* storage = context_->storage.get();
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::TraceStats::kProducersConnectedFieldNumber:
storage->SetStats(stats::traced_producers_connected, fld.as_int64());
break;
@@ -1680,8 +1689,9 @@
const size_t fld_off = packet.offset_of(fld.data());
TraceBlobView buf_data = packet.slice(fld_off, fld.size());
ProtoDecoder buf_d(buf_data.data(), buf_data.length());
- for (auto fld2 = buf_d.ReadField(); fld2.id; fld2 = buf_d.ReadField()) {
- switch (fld2.id) {
+ for (auto fld2 = buf_d.ReadField(); fld2.id();
+ fld2 = buf_d.ReadField()) {
+ switch (fld2.id()) {
case protos::TraceStats::BufferStats::kBufferSizeFieldNumber:
storage->SetIndexedStats(stats::traced_buf_buffer_size, buf_num,
fld2.as_int64());
@@ -1764,17 +1774,18 @@
break;
}
}
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseFtraceStats(TraceBlobView packet) {
ProtoDecoder decoder(packet.data(), packet.length());
size_t phase = 0;
auto* storage = context_->storage.get();
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::FtraceStats::kPhaseFieldNumber:
- phase = fld.int_value == protos::FtraceStats_Phase_END_OF_TRACE ? 1 : 0;
+ phase =
+ fld.as_uint64() == protos::FtraceStats_Phase_END_OF_TRACE ? 1 : 0;
// This code relies on the fact that each ftrace_cpu_XXX_end event is
// just after the corresponding ftrace_cpu_XXX_begin event.
@@ -1791,8 +1802,9 @@
TraceBlobView cpu_data = packet.slice(fld_off, fld.size());
ProtoDecoder cpu_d(cpu_data.data(), cpu_data.length());
int cpu_num = -1;
- for (auto fld2 = cpu_d.ReadField(); fld2.id; fld2 = cpu_d.ReadField()) {
- switch (fld2.id) {
+ for (auto fld2 = cpu_d.ReadField(); fld2.id();
+ fld2 = cpu_d.ReadField()) {
+ switch (fld2.id()) {
case protos::FtraceCpuStats::kCpuFieldNumber:
cpu_num = fld2.as_int32();
break;
@@ -1842,20 +1854,20 @@
break;
}
}
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
void ProtoTraceParser::ParseProfilePacket(TraceBlobView packet) {
ProtoDecoder decoder(packet.data(), packet.length());
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
+ for (auto fld = decoder.ReadField(); fld.valid(); fld = decoder.ReadField()) {
+ switch (fld.id()) {
case protos::ProfilePacket::kStringsFieldNumber: {
const size_t fld_off = packet.offset_of(fld.data());
TraceBlobView nestedPacket = packet.slice(fld_off, fld.size());
ProtoDecoder nested(nestedPacket.data(), nestedPacket.length());
- for (auto sub = nested.ReadField(); sub.id != 0;
+ for (auto sub = nested.ReadField(); sub.valid();
sub = nested.ReadField()) {
- switch (sub.id) {
+ switch (sub.id()) {
case protos::ProfilePacket::InternedString::kIdFieldNumber: {
break;
}
@@ -1891,7 +1903,7 @@
}
}
}
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
} // namespace trace_processor
diff --git a/src/trace_processor/proto_trace_parser.h b/src/trace_processor/proto_trace_parser.h
index 9782003..5abb5af 100644
--- a/src/trace_processor/proto_trace_parser.h
+++ b/src/trace_processor/proto_trace_parser.h
@@ -143,11 +143,10 @@
std::vector<StringId> vmstat_strs_id_;
std::vector<StringId> rss_members_;
- static constexpr size_t kFtraceMaxFieldCount = 32;
struct FtraceMessageStrings {
// The string id of name of the event field (e.g. sched_switch's id).
StringId message_name_id = 0;
- std::array<StringId, kFtraceMaxFieldCount> field_name_ids;
+ std::array<StringId, kMaxFtraceEventFields> field_name_ids;
};
std::vector<FtraceMessageStrings> ftrace_message_strings_;
diff --git a/src/trace_processor/proto_trace_tokenizer.cc b/src/trace_processor/proto_trace_tokenizer.cc
index 75322bb..932b240 100644
--- a/src/trace_processor/proto_trace_tokenizer.cc
+++ b/src/trace_processor/proto_trace_tokenizer.cc
@@ -29,8 +29,10 @@
#include "src/trace_processor/trace_sorter.h"
#include "src/trace_processor/trace_storage.h"
-#include "perfetto/trace/trace.pb.h"
-#include "perfetto/trace/trace_packet.pb.h"
+#include "perfetto/trace/ftrace/ftrace_event.pbzero.h"
+#include "perfetto/trace/ftrace/ftrace_event_bundle.pbzero.h"
+#include "perfetto/trace/trace.pbzero.h"
+#include "perfetto/trace/trace_packet.pbzero.h"
namespace perfetto {
namespace trace_processor {
@@ -59,10 +61,10 @@
size -= missing_len;
}
- // At this point we have enough data in partial_buf_ to read at least the
+ // At this point we have enough data in |partial_buf_| to read at least the
// field header and know the size of the next TracePacket.
constexpr uint8_t kTracePacketTag =
- MakeTagLengthDelimited(protos::Trace::kPacketFieldNumber);
+ MakeTagLengthDelimited(protos::pbzero::Trace::kPacketFieldNumber);
const uint8_t* pos = &partial_buf_[0];
uint8_t proto_field_tag = *pos;
uint64_t field_size = 0;
@@ -115,120 +117,70 @@
const uint8_t* start = &owned_buf[0];
const size_t data_off = static_cast<size_t>(data - start);
TraceBlobView whole_buf(std::move(owned_buf), data_off, size);
- ProtoDecoder decoder(data, size);
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- if (fld.id != protos::Trace::kPacketFieldNumber) {
- PERFETTO_ELOG("Non-trace packet field found in root Trace proto");
- continue;
- }
- size_t field_offset = static_cast<size_t>(fld.data() - start);
- ParsePacket(whole_buf.slice(field_offset, fld.size()));
+
+ protos::pbzero::Trace::Decoder decoder(data, size);
+ for (auto it = decoder.packet(); it; ++it) {
+ size_t field_offset = whole_buf.offset_of(it->data());
+ ParsePacket(whole_buf.slice(field_offset, it->size()));
}
- const size_t leftover = static_cast<size_t>(size - decoder.offset());
- if (leftover > 0) {
+ const size_t bytes_left = decoder.bytes_left();
+ if (bytes_left > 0) {
PERFETTO_DCHECK(partial_buf_.empty());
- partial_buf_.insert(partial_buf_.end(), &data[decoder.offset()],
- &data[decoder.offset() + leftover]);
+ partial_buf_.insert(partial_buf_.end(), &data[decoder.read_offset()],
+ &data[decoder.read_offset() + bytes_left]);
}
}
void ProtoTraceTokenizer::ParsePacket(TraceBlobView packet) {
- constexpr auto kTimestampFieldNumber =
- protos::TracePacket::kTimestampFieldNumber;
- ProtoDecoder decoder(packet.data(), packet.length());
- uint64_t raw_timestamp = 0;
- bool timestamp_found = false;
+ protos::pbzero::TracePacket::Decoder decoder(packet.data(), packet.length());
- // Speculate on the fact that the timestamp is often the 1st field of the
- // packet.
- constexpr auto timestampFieldTag = MakeTagVarInt(kTimestampFieldNumber);
- if (PERFETTO_LIKELY(packet.length() > 10 &&
- packet.data()[0] == timestampFieldTag)) {
- // Fastpath.
- const uint8_t* next =
- ParseVarInt(packet.data() + 1, packet.data() + 11, &raw_timestamp);
- timestamp_found = next != packet.data() + 1;
- decoder.Reset(next);
- } else {
- // Slowpath.
- timestamp_found =
- decoder.FindIntField<kTimestampFieldNumber>(&raw_timestamp);
- }
-
- int64_t timestamp =
- timestamp_found ? static_cast<int64_t>(raw_timestamp) : latest_timestamp_;
+ auto timestamp = decoder.has_timestamp()
+ ? static_cast<int64_t>(decoder.timestamp())
+ : latest_timestamp_;
latest_timestamp_ = std::max(timestamp, latest_timestamp_);
- // TODO(primiano): this can be optimized for the ftrace case.
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- if (fld.id == protos::TracePacket::kTrustedUidFieldNumber)
- continue;
-
- if (fld.id == protos::TracePacket::kFtraceEventsFieldNumber) {
- const size_t fld_off = packet.offset_of(fld.data());
- ParseFtraceBundle(packet.slice(fld_off, fld.size()));
- return;
- }
+ if (decoder.has_ftrace_events()) {
+ auto ftrace_field = decoder.ftrace_events();
+ const size_t fld_off = packet.offset_of(ftrace_field.data);
+ ParseFtraceBundle(packet.slice(fld_off, ftrace_field.size));
+ return;
}
// Use parent data and length because we want to parse this again
// later to get the exact type of the packet.
trace_sorter_->PushTracePacket(timestamp, std::move(packet));
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+ PERFETTO_DCHECK(!decoder.bytes_left());
}
PERFETTO_ALWAYS_INLINE
void ProtoTraceTokenizer::ParseFtraceBundle(TraceBlobView bundle) {
- constexpr auto kCpuFieldNumber = protos::FtraceEventBundle::kCpuFieldNumber;
- constexpr auto kCpuFieldTag = MakeTagVarInt(kCpuFieldNumber);
- const uint8_t* data = bundle.data();
- const size_t length = bundle.length();
- ProtoDecoder decoder(data, length);
+ protos::pbzero::FtraceEventBundle::Decoder decoder(bundle.data(),
+ bundle.length());
- // For speed we speculate on the location and size (<128) of the cpu field.
- // In P+ cpu is pushed as the first field.
- // In P cpu is pushed as the 2nd last field.
- uint64_t cpu = 0;
- if (length > 2 && data[0] == kCpuFieldTag && data[1] < 0x80) {
- cpu = data[1];
- } else if (PERFETTO_LIKELY(length > 4 && data[length - 4] == kCpuFieldTag) &&
- data[length - 3] < 0x80) {
- cpu = data[length - 3];
- } else {
- if (!PERFETTO_LIKELY((decoder.FindIntField<kCpuFieldNumber>(&cpu)))) {
- PERFETTO_ELOG("CPU field not found in FtraceEventBundle");
- trace_storage_->IncrementStats(stats::ftrace_bundle_tokenizer_errors);
- return;
- }
- }
-
- if (PERFETTO_UNLIKELY(cpu > base::kMaxCpus)) {
- PERFETTO_ELOG("CPU number larger than kMaxCpus (%" PRIu64 " > %zu)", cpu,
- base::kMaxCpus);
+ if (PERFETTO_UNLIKELY(!decoder.has_cpu())) {
+ PERFETTO_ELOG("CPU field not found in FtraceEventBundle");
+ trace_storage_->IncrementStats(stats::ftrace_bundle_tokenizer_errors);
return;
}
- for (auto fld = decoder.ReadField(); fld.id != 0; fld = decoder.ReadField()) {
- switch (fld.id) {
- case protos::FtraceEventBundle::kEventFieldNumber: {
- const size_t fld_off = bundle.offset_of(fld.data());
- auto cpu_32 = static_cast<uint32_t>(cpu);
- ParseFtraceEvent(cpu_32, bundle.slice(fld_off, fld.size()));
- break;
- }
- default:
- break;
- }
+ uint32_t cpu = decoder.cpu();
+ if (PERFETTO_UNLIKELY(cpu > base::kMaxCpus)) {
+ PERFETTO_ELOG("CPU larger than kMaxCpus (%u > %zu)", cpu, base::kMaxCpus);
+ return;
}
- trace_sorter_->FinalizeFtraceEventBatch(static_cast<uint32_t>(cpu));
- PERFETTO_DCHECK(decoder.IsEndOfBuffer());
+
+ for (auto it = decoder.event(); it; ++it) {
+ size_t off = bundle.offset_of(it->data());
+ ParseFtraceEvent(cpu, bundle.slice(off, it->size()));
+ }
+ trace_sorter_->FinalizeFtraceEventBatch(cpu);
}
PERFETTO_ALWAYS_INLINE
void ProtoTraceTokenizer::ParseFtraceEvent(uint32_t cpu, TraceBlobView event) {
constexpr auto kTimestampFieldNumber =
- protos::FtraceEvent::kTimestampFieldNumber;
+ protos::pbzero::FtraceEvent::kTimestampFieldNumber;
const uint8_t* data = event.data();
const size_t length = event.length();
ProtoDecoder decoder(data, length);
@@ -245,8 +197,10 @@
decoder.Reset(next);
} else {
// Slowpath.
- timestamp_found =
- decoder.FindIntField<kTimestampFieldNumber>(&raw_timestamp);
+ if (auto ts_field = decoder.FindField(kTimestampFieldNumber)) {
+ timestamp_found = true;
+ raw_timestamp = ts_field.as_uint64();
+ }
}
if (PERFETTO_UNLIKELY(!timestamp_found)) {
diff --git a/src/trace_processor/trace_blob_view.h b/src/trace_processor/trace_blob_view.h
index 5901997..46cc424 100644
--- a/src/trace_processor/trace_blob_view.h
+++ b/src/trace_processor/trace_blob_view.h
@@ -70,8 +70,8 @@
return static_cast<size_t>(data - start());
}
- // const std::shared_ptr<uint8_t[]>& buffer() const { return shbuf_; }
size_t length() const { return length_; }
+ size_t offset() const { return offset_; }
private:
// An equivalent to std::shared_ptr<uint8_t>, with the differnce that:
diff --git a/src/trace_processor/trace_sorter.cc b/src/trace_processor/trace_sorter.cc
index 4569941..63cd9b3 100644
--- a/src/trace_processor/trace_sorter.cc
+++ b/src/trace_processor/trace_sorter.cc
@@ -27,6 +27,8 @@
: context_(context), window_size_ns_(window_size_ns) {
const char* env = getenv("TRACE_PROCESSOR_SORT_ONLY");
bypass_next_stage_for_testing_ = env && !strcmp(env, "1");
+ if (bypass_next_stage_for_testing_)
+ PERFETTO_ELOG("TEST MODE: bypassing protobuf parsing stage");
}
void TraceSorter::Queue::Sort() {