trace_processor: split up counters table
This CL splits up the counters table into two tables: counter_definitions
and counter_values. We also add a view to make the change transparent to
current callers so we don't need to change any users of trace processor.
This change should allow for more efficient queries
from the UI which only wants the definitions to be able to create the
tracks. Moreover, even on the counters view, if queries are done on name,
ref and ref type, we can do this a lot more efficiently on the definitions.
For example on a 80MB trace (b/124495829) with just memory counters
(i.e. no sched) and the following query (very similar to those done
both in the UI and in benchmarking metrics code)
select
ts,
lead(ts) over (partition by ref_type order by ts) as ts_end,
value
from counters
where name = 'SwapCached' and ref = 0
we get the following performance numbers:
Old code: 263.902 ms
New code (using the view): 107.088 ms (2.46x speedup)
In query the UI does on startup (and called out in b/124495829):
select
name,
ref,
ref_type
from counters
where ref is not null
group by ref, ref_type, name
we get the following performance numbers:
Old code: 11020.611 ms
New code (using the view): 29076.535 ms (2.63x slow down)
New code (directly querying counter_definitions): 48.554 ms (226x speedup)
A follow up CL will further improve the speed of the join operation in view
and filters in general significantly.
Bug: 124495829
Change-Id: I57cf1be328364d53bd6e65eeb4434d015df86981
diff --git a/src/trace_processor/trace_storage.h b/src/trace_processor/trace_storage.h
index ca942db..d30fc14 100644
--- a/src/trace_processor/trace_storage.h
+++ b/src/trace_processor/trace_storage.h
@@ -52,7 +52,7 @@
enum TableId : uint8_t {
// Intentionally don't have TableId == 0 so that RowId == 0 can refer to an
// invalid row id.
- kCounters = 1,
+ kCounterValues = 1,
kRawEvents = 2,
kInstants = 3,
kSched = 4,
@@ -331,44 +331,78 @@
std::deque<int64_t> parent_stack_ids_;
};
- class Counters {
+ class CounterDefinitions {
public:
- inline size_t AddCounter(int64_t timestamp,
- StringId name_id,
- double value,
- int64_t ref,
- RefType type) {
- timestamps_.emplace_back(timestamp);
+ using Id = uint32_t;
+
+ inline Id AddCounterDefinition(StringId name_id,
+ int64_t ref,
+ RefType type) {
+ base::Hash hash;
+ hash.Update(name_id);
+ hash.Update(ref);
+ hash.Update(type);
+
+ // TODO(lalitm): this is a perf bottleneck and likely we can do something
+ // quite a bit better here.
+ uint64_t digest = hash.digest();
+ auto it = hash_to_row_idx_.find(digest);
+ if (it != hash_to_row_idx_.end())
+ return it->second;
+
name_ids_.emplace_back(name_id);
- values_.emplace_back(value);
refs_.emplace_back(ref);
types_.emplace_back(type);
- arg_set_ids_.emplace_back(kInvalidArgSetId);
- return counter_count() - 1;
+ hash_to_row_idx_.emplace(digest, size() - 1);
+ return size() - 1;
}
- void set_arg_set_id(uint32_t row, ArgSetId id) { arg_set_ids_[row] = id; }
-
- size_t counter_count() const { return timestamps_.size(); }
-
- const std::deque<int64_t>& timestamps() const { return timestamps_; }
+ uint32_t size() const { return static_cast<uint32_t>(name_ids_.size()); }
const std::deque<StringId>& name_ids() const { return name_ids_; }
- const std::deque<double>& values() const { return values_; }
-
const std::deque<int64_t>& refs() const { return refs_; }
const std::deque<RefType>& types() const { return types_; }
+ private:
+ std::deque<StringId> name_ids_;
+ std::deque<int64_t> refs_;
+ std::deque<RefType> types_;
+
+ std::unordered_map<uint64_t, uint32_t> hash_to_row_idx_;
+ };
+
+ class CounterValues {
+ public:
+ inline uint32_t AddCounterValue(CounterDefinitions::Id counter_id,
+ int64_t timestamp,
+ double value) {
+ counter_ids_.emplace_back(counter_id);
+ timestamps_.emplace_back(timestamp);
+ values_.emplace_back(value);
+ arg_set_ids_.emplace_back(kInvalidArgSetId);
+ return size() - 1;
+ }
+
+ void set_arg_set_id(uint32_t row, ArgSetId id) { arg_set_ids_[row] = id; }
+
+ uint32_t size() const { return static_cast<uint32_t>(counter_ids_.size()); }
+
+ const std::deque<CounterDefinitions::Id>& counter_ids() const {
+ return counter_ids_;
+ }
+
+ const std::deque<int64_t>& timestamps() const { return timestamps_; }
+
+ const std::deque<double>& values() const { return values_; }
+
const std::deque<ArgSetId>& arg_set_ids() const { return arg_set_ids_; }
private:
+ std::deque<CounterDefinitions::Id> counter_ids_;
std::deque<int64_t> timestamps_;
- std::deque<StringId> name_ids_;
std::deque<double> values_;
- std::deque<int64_t> refs_;
- std::deque<RefType> types_;
std::deque<ArgSetId> arg_set_ids_;
};
@@ -590,8 +624,15 @@
const NestableSlices& nestable_slices() const { return nestable_slices_; }
NestableSlices* mutable_nestable_slices() { return &nestable_slices_; }
- const Counters& counters() const { return counters_; }
- Counters* mutable_counters() { return &counters_; }
+ const CounterDefinitions& counter_definitions() const {
+ return counter_definitions_;
+ }
+ CounterDefinitions* mutable_counter_definitions() {
+ return &counter_definitions_;
+ }
+
+ const CounterValues& counter_values() const { return counter_values_; }
+ CounterValues* mutable_counter_values() { return &counter_values_; }
const SqlStats& sql_stats() const { return sql_stats_; }
SqlStats* mutable_sql_stats() { return &sql_stats_; }
@@ -660,9 +701,12 @@
// Slices coming from userspace events (e.g. Chromium TRACE_EVENT macros).
NestableSlices nestable_slices_;
- // Counter events from the trace. This includes CPU frequency events as well
- // systrace trace_marker counter events.
- Counters counters_;
+ // The type of counters in the trace. Can be thought of the the "metadata".
+ CounterDefinitions counter_definitions_;
+
+ // The values from the Counter events from the trace. This includes CPU
+ // frequency events as well systrace trace_marker counter events.
+ CounterValues counter_values_;
SqlStats sql_stats_;