blob: 2efaf0b1f2679f1edb378e8af737377e9d44eaf9 [file] [log] [blame]
Ioannis Ilkoseff38f52018-10-29 10:37:55 +00001/*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "src/trace_processor/trace_processor_impl.h"
18
Ioannis Ilkosb8b11102019-01-29 17:56:55 +000019#include <inttypes.h>
Hector Dearmanc0a29092018-11-08 17:34:23 +000020#include <algorithm>
Mikhail Khokhlova8d310d2019-05-07 17:34:21 +010021#include <fstream>
Ioannis Ilkoseff38f52018-10-29 10:37:55 +000022#include <functional>
23
Lalit Maganti77095012019-02-19 13:13:59 +000024#include "perfetto/base/logging.h"
Eric Seckler83dcc8c2019-08-21 12:18:43 +010025#include "perfetto/base/time.h"
Primiano Tucci2c5488f2019-06-01 03:27:28 +010026#include "perfetto/ext/base/string_splitter.h"
27#include "perfetto/ext/base/string_utils.h"
Lalit Maganti26f69bd2019-04-29 18:23:47 +010028#include "perfetto/protozero/scattered_heap_buffer.h"
Primiano Tucci2c761ef2019-01-07 20:20:46 +000029#include "src/trace_processor/android_logs_table.h"
Lalit Maganti6e9c55e2018-11-29 12:00:39 +000030#include "src/trace_processor/args_table.h"
Lalit Maganti4fa7c6c2019-02-06 15:06:36 +000031#include "src/trace_processor/args_tracker.h"
Primiano Tuccia270f012019-01-07 20:01:00 +000032#include "src/trace_processor/clock_tracker.h"
Lalit Maganti8320e6d2019-03-14 18:49:33 +000033#include "src/trace_processor/counter_definitions_table.h"
34#include "src/trace_processor/counter_values_table.h"
Oystein Eftevaag7f64c102019-08-29 10:27:31 -070035#include "src/trace_processor/cpu_profile_stack_sample_table.h"
Ioannis Ilkoseff38f52018-10-29 10:37:55 +000036#include "src/trace_processor/event_tracker.h"
Hector Dearmand8c3d5c2019-08-06 13:28:23 +010037#include "src/trace_processor/forwarding_trace_parser.h"
Florian Mayer25e013e2019-10-01 14:06:15 +010038#include "src/trace_processor/heap_graph_tracker.h"
Lalit Maganti9b2d52b2019-05-07 14:32:15 +010039#include "src/trace_processor/heap_profile_allocation_table.h"
Florian Mayer438b5ab2019-05-02 11:18:06 +010040#include "src/trace_processor/heap_profile_tracker.h"
Eric Seckleracfe35b2019-10-18 15:50:17 +010041#include "src/trace_processor/importers/ftrace/ftrace_module.h"
Eric Secklerd3795132019-10-21 16:43:10 +010042#include "src/trace_processor/importers/ftrace/sched_event_tracker.h"
Eric Secklerab0604c2019-10-25 10:12:07 +010043#include "src/trace_processor/importers/proto/android_probes_module.h"
Eric Seckler63afd792019-10-23 15:05:05 +010044#include "src/trace_processor/importers/proto/graphics_event_module.h"
Eric Seckler87ede882019-10-18 10:48:36 +010045#include "src/trace_processor/importers/proto/proto_importer_module.h"
Eric Seckler137a4672019-10-24 08:51:14 +010046#include "src/trace_processor/importers/proto/proto_trace_tokenizer.h"
Eric Secklerab0604c2019-10-25 10:12:07 +010047#include "src/trace_processor/importers/proto/system_probes_module.h"
Eric Seckler87ede882019-10-18 10:48:36 +010048#include "src/trace_processor/importers/proto/track_event_module.h"
Eric Secklerd8b52082019-10-17 15:58:38 +010049#include "src/trace_processor/importers/systrace/systrace_parser.h"
50#include "src/trace_processor/importers/systrace/systrace_trace_parser.h"
Isabelle Taylorc8c11202018-11-05 11:36:22 +000051#include "src/trace_processor/instants_table.h"
Mikhail Khokhlove466c002019-05-23 13:33:33 +010052#include "src/trace_processor/metadata_table.h"
Ioannis Ilkoseff38f52018-10-29 10:37:55 +000053#include "src/trace_processor/process_table.h"
54#include "src/trace_processor/process_tracker.h"
Isabelle Taylord3f590e2019-01-08 17:21:23 +000055#include "src/trace_processor/raw_table.h"
Ioannis Ilkoseff38f52018-10-29 10:37:55 +000056#include "src/trace_processor/sched_slice_table.h"
57#include "src/trace_processor/slice_table.h"
58#include "src/trace_processor/slice_tracker.h"
Lalit Maganti95693bb2018-11-05 16:20:24 +000059#include "src/trace_processor/span_join_operator_table.h"
Primiano Tucci5cb84f82018-10-31 21:46:36 -070060#include "src/trace_processor/sql_stats_table.h"
Lalit Maganti40697112019-09-04 22:06:42 +010061#include "src/trace_processor/sqlite/db_sqlite_table.h"
Lalit Maganti03d10622019-08-27 14:47:39 +010062#include "src/trace_processor/sqlite/sqlite3_str_split.h"
Lalit Maganti737b3762019-08-26 13:46:37 -070063#include "src/trace_processor/sqlite/sqlite_table.h"
Oystein Eftevaag5419c582019-08-21 13:58:49 -070064#include "src/trace_processor/stack_profile_callsite_table.h"
65#include "src/trace_processor/stack_profile_frame_table.h"
66#include "src/trace_processor/stack_profile_mapping_table.h"
67#include "src/trace_processor/stack_profile_tracker.h"
Lalit Maganti05e8c132018-11-09 18:16:12 +000068#include "src/trace_processor/stats_table.h"
Hector Dearmanf31a7032019-03-29 10:05:09 +000069#include "src/trace_processor/syscall_tracker.h"
Ioannis Ilkoseff38f52018-10-29 10:37:55 +000070#include "src/trace_processor/thread_table.h"
Deepanjan Roy01994ca2019-04-02 11:05:34 -070071#include "src/trace_processor/trace_blob_view.h"
Ioannis Ilkoseff38f52018-10-29 10:37:55 +000072#include "src/trace_processor/trace_sorter.h"
Lalit Maganti4ea78d92019-09-20 20:20:41 +010073#include "src/trace_processor/track_tracker.h"
Mohammad Reza Zakerinasabb06d1852019-09-11 14:41:36 -040074#include "src/trace_processor/vulkan_memory_tracker.h"
Ioannis Ilkoseff38f52018-10-29 10:37:55 +000075#include "src/trace_processor/window_operator_table.h"
76
Eric Secklerc622b8b2019-10-21 09:54:00 +010077#if PERFETTO_BUILDFLAG(PERFETTO_TP_METRICS)
78#include "src/trace_processor/metrics/descriptors.h"
79#include "src/trace_processor/metrics/metrics.descriptor.h"
80#include "src/trace_processor/metrics/metrics.h"
81#include "src/trace_processor/metrics/sql_metrics.h"
82
Primiano Tucci355b8c82019-08-29 08:37:51 +020083#include "protos/perfetto/metrics/android/mem_metric.pbzero.h"
84#include "protos/perfetto/metrics/metrics.pbzero.h"
Eric Secklerc622b8b2019-10-21 09:54:00 +010085#endif // PERFETTO_BUILDFLAG(PERFETTO_TP_METRICS)
Lalit Maganti26f69bd2019-04-29 18:23:47 +010086
Primiano Tucci02c11762019-08-30 00:57:59 +020087#if PERFETTO_BUILDFLAG(PERFETTO_TP_JSON)
Mikhail Khokhlova8d310d2019-05-07 17:34:21 +010088#include "src/trace_processor/export_json.h"
Lalit Maganti17aa2732019-02-08 15:47:26 +000089#endif
90
Mikhail Khokhlov8643d1c2019-06-04 12:02:47 +010091// In Android and Chromium tree builds, we don't have the percentile module.
Lalit Maganti17aa2732019-02-08 15:47:26 +000092// Just don't include it.
Primiano Tucci02c11762019-08-30 00:57:59 +020093#if PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
Ioannis Ilkos178535e2018-11-05 17:32:45 +000094// defined in sqlite_src/ext/misc/percentile.c
95extern "C" int sqlite3_percentile_init(sqlite3* db,
96 char** error,
97 const sqlite3_api_routines* api);
Lalit Maganti17aa2732019-02-08 15:47:26 +000098#endif
Ioannis Ilkos178535e2018-11-05 17:32:45 +000099
Ioannis Ilkosbf54ebb2019-03-08 13:26:14 +0000100namespace perfetto {
101namespace trace_processor {
Ioannis Ilkos178535e2018-11-05 17:32:45 +0000102namespace {
Ioannis Ilkosbf54ebb2019-03-08 13:26:14 +0000103
Ioannis Ilkos7199d552019-03-13 18:35:31 +0000104void InitializeSqlite(sqlite3* db) {
105 char* error = nullptr;
106 sqlite3_exec(db, "PRAGMA temp_store=2", 0, 0, &error);
107 if (error) {
108 PERFETTO_FATAL("Error setting pragma temp_store: %s", error);
109 }
Ioannis Ilkosbf54ebb2019-03-08 13:26:14 +0000110 sqlite3_str_split_init(db);
Lalit Maganti17aa2732019-02-08 15:47:26 +0000111// In Android tree builds, we don't have the percentile module.
112// Just don't include it.
Primiano Tucci02c11762019-08-30 00:57:59 +0200113#if PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
Ioannis Ilkos178535e2018-11-05 17:32:45 +0000114 sqlite3_percentile_init(db, &error, nullptr);
Sami Kyostila33668942018-11-13 16:33:32 +0000115 if (error) {
Ioannis Ilkos178535e2018-11-05 17:32:45 +0000116 PERFETTO_ELOG("Error initializing: %s", error);
Sami Kyostila33668942018-11-13 16:33:32 +0000117 sqlite3_free(error);
118 }
Lalit Maganti17aa2732019-02-08 15:47:26 +0000119#endif
Sami Kyostila33668942018-11-13 16:33:32 +0000120}
121
Lalit Maganti086295b2019-04-04 16:59:19 +0100122void BuildBoundsTable(sqlite3* db, std::pair<int64_t, int64_t> bounds) {
123 char* error = nullptr;
124 sqlite3_exec(db, "DELETE FROM trace_bounds", nullptr, nullptr, &error);
125 if (error) {
126 PERFETTO_ELOG("Error deleting from bounds table: %s", error);
127 sqlite3_free(error);
128 return;
129 }
130
131 char* insert_sql = sqlite3_mprintf("INSERT INTO trace_bounds VALUES(%" PRId64
132 ", %" PRId64 ")",
133 bounds.first, bounds.second);
134
135 sqlite3_exec(db, insert_sql, 0, 0, &error);
136 sqlite3_free(insert_sql);
137 if (error) {
138 PERFETTO_ELOG("Error inserting bounds table: %s", error);
139 sqlite3_free(error);
140 }
141}
142
Sami Kyostila33668942018-11-13 16:33:32 +0000143void CreateBuiltinTables(sqlite3* db) {
144 char* error = nullptr;
145 sqlite3_exec(db, "CREATE TABLE perfetto_tables(name STRING)", 0, 0, &error);
146 if (error) {
147 PERFETTO_ELOG("Error initializing: %s", error);
148 sqlite3_free(error);
Ioannis Ilkos178535e2018-11-05 17:32:45 +0000149 }
Ioannis Ilkosb8b11102019-01-29 17:56:55 +0000150 sqlite3_exec(db,
151 "CREATE TABLE trace_bounds(start_ts BIG INT, end_ts BIG INT)", 0,
152 0, &error);
153 if (error) {
154 PERFETTO_ELOG("Error initializing: %s", error);
155 sqlite3_free(error);
156 }
Ioannis Ilkosb8b11102019-01-29 17:56:55 +0000157
Lalit Maganti086295b2019-04-04 16:59:19 +0100158 // Initialize the bounds table with some data so even before parsing any data,
159 // we still have a valid table.
160 BuildBoundsTable(db, std::make_pair(0, 0));
Ioannis Ilkos178535e2018-11-05 17:32:45 +0000161}
Hector Dearmanc0a29092018-11-08 17:34:23 +0000162
Lalit Maganti8320e6d2019-03-14 18:49:33 +0000163void CreateBuiltinViews(sqlite3* db) {
164 char* error = nullptr;
165 sqlite3_exec(db,
166 "CREATE VIEW counters AS "
167 "SELECT * FROM counter_values "
Lalit Maganti6fbb0ad2019-10-25 14:42:28 +0100168 "INNER JOIN counter_definitions USING(counter_id) "
169 "ORDER BY ts;",
Lalit Maganti8320e6d2019-03-14 18:49:33 +0000170 0, 0, &error);
171 if (error) {
172 PERFETTO_ELOG("Error initializing: %s", error);
173 sqlite3_free(error);
174 }
Eric Seckler972225e2019-04-18 11:07:12 +0100175
176 sqlite3_exec(db,
177 "CREATE VIEW slice AS "
178 "SELECT "
179 " *, "
Lalit Maganti81a02cd2019-07-12 17:05:11 +0100180 " category as cat, "
Eric Seckler972225e2019-04-18 11:07:12 +0100181 " CASE ref_type "
182 " WHEN 'utid' THEN ref "
183 " ELSE NULL "
184 " END AS utid "
185 "FROM internal_slice;",
186 0, 0, &error);
187 if (error) {
188 PERFETTO_ELOG("Error initializing: %s", error);
189 sqlite3_free(error);
190 }
191
Lalit Maganti24b08332019-10-04 12:53:08 +0100192 sqlite3_exec(db,
193 "CREATE VIEW gpu_slice AS "
194 "SELECT "
195 "* "
196 "FROM internal_gpu_slice join internal_slice using(slice_id);",
197 0, 0, &error);
198 if (error) {
199 PERFETTO_ELOG("Error initializing: %s", error);
200 sqlite3_free(error);
201 }
202
Eric Seckler972225e2019-04-18 11:07:12 +0100203 // Legacy view for "slice" table with a deprecated table name.
204 // TODO(eseckler): Remove this view when all users have switched to "slice".
205 sqlite3_exec(db,
206 "CREATE VIEW slices AS "
207 "SELECT * FROM slice;",
208 0, 0, &error);
209 if (error) {
210 PERFETTO_ELOG("Error initializing: %s", error);
211 sqlite3_free(error);
212 }
Lalit Maganti8320e6d2019-03-14 18:49:33 +0000213}
214
Primiano Tucci02c11762019-08-30 00:57:59 +0200215#if PERFETTO_BUILDFLAG(PERFETTO_TP_JSON)
Mikhail Khokhlova8d310d2019-05-07 17:34:21 +0100216void ExportJson(sqlite3_context* ctx, int /*argc*/, sqlite3_value** argv) {
217 TraceStorage* storage = static_cast<TraceStorage*>(sqlite3_user_data(ctx));
Eric Seckler37ad6af2019-10-08 15:51:14 +0100218 FILE* output;
219 if (sqlite3_value_type(argv[0]) == SQLITE_INTEGER) {
220 // Assume input is an FD.
221 output = fdopen(sqlite3_value_int(argv[0]), "w");
222 if (!output) {
223 sqlite3_result_error(ctx, "Couldn't open output file from given FD", -1);
224 return;
225 }
226 } else {
227 const char* filename =
228 reinterpret_cast<const char*>(sqlite3_value_text(argv[0]));
229 output = fopen(filename, "w");
230 if (!output) {
231 sqlite3_result_error(ctx, "Couldn't open output file", -1);
232 return;
233 }
Mikhail Khokhlova8d310d2019-05-07 17:34:21 +0100234 }
235
Eric Seckler7fb1d232019-10-23 14:42:43 +0100236 util::Status result = json::ExportJson(storage, output);
237 if (!result.ok()) {
238 sqlite3_result_error(ctx, result.message().c_str(), -1);
239 return;
Mikhail Khokhlova8d310d2019-05-07 17:34:21 +0100240 }
241}
242
243void CreateJsonExportFunction(TraceStorage* ts, sqlite3* db) {
244 auto ret = sqlite3_create_function_v2(db, "EXPORT_JSON", 1, SQLITE_UTF8, ts,
245 ExportJson, nullptr, nullptr,
246 sqlite_utils::kSqliteStatic);
247 if (ret) {
248 PERFETTO_ELOG("Error initializing EXPORT_JSON");
249 }
250}
251#endif
252
Lalit Maganti062c8f82019-10-03 20:11:29 +0100253void Hash(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
254 base::Hash hash;
255 for (int i = 0; i < argc; ++i) {
256 sqlite3_value* value = argv[i];
257 switch (sqlite3_value_type(value)) {
258 case SQLITE_INTEGER:
259 hash.Update(sqlite3_value_int64(value));
260 break;
261 case SQLITE_TEXT: {
262 const char* ptr =
263 reinterpret_cast<const char*>(sqlite3_value_text(value));
264 hash.Update(ptr, strlen(ptr));
265 break;
266 }
267 default:
268 sqlite3_result_error(ctx, "Unsupported type of arg passed to HASH", -1);
269 return;
270 }
271 }
272 sqlite3_result_int64(ctx, static_cast<int64_t>(hash.digest()));
273}
274
275void CreateHashFunction(sqlite3* db) {
276 auto ret = sqlite3_create_function_v2(
277 db, "HASH", -1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr, &Hash,
278 nullptr, nullptr, nullptr);
279 if (ret) {
280 PERFETTO_ELOG("Error initializing HASH");
281 }
282}
283
Eric Secklerc622b8b2019-10-21 09:54:00 +0100284#if PERFETTO_BUILDFLAG(PERFETTO_TP_METRICS)
Lalit Maganti2553b802019-05-24 13:25:58 +0100285void SetupMetrics(TraceProcessor* tp,
286 sqlite3* db,
Lalit Maganti2553b802019-05-24 13:25:58 +0100287 std::vector<metrics::SqlMetricFile>* sql_metrics) {
Lalit Maganti5f8727d2019-05-24 13:38:50 +0100288 tp->ExtendMetricsProto(kMetricsDescriptor.data(), kMetricsDescriptor.size());
289
Lalit Maganti2553b802019-05-24 13:25:58 +0100290 for (const auto& file_to_sql : metrics::sql_metrics::kFileToSql) {
Lalit Maganti5f8727d2019-05-24 13:38:50 +0100291 tp->RegisterMetric(file_to_sql.path, file_to_sql.sql);
Lalit Maganti2553b802019-05-24 13:25:58 +0100292 }
293
294 {
295 std::unique_ptr<metrics::RunMetricContext> ctx(
296 new metrics::RunMetricContext());
297 ctx->tp = tp;
298 ctx->metrics = sql_metrics;
299 auto ret = sqlite3_create_function_v2(
300 db, "RUN_METRIC", -1, SQLITE_UTF8, ctx.release(), metrics::RunMetric,
301 nullptr, nullptr,
302 [](void* ptr) { delete static_cast<metrics::RunMetricContext*>(ptr); });
303 if (ret)
304 PERFETTO_ELOG("Error initializing RUN_METRIC");
305 }
306
307 {
308 auto ret = sqlite3_create_function_v2(
309 db, "RepeatedField", 1, SQLITE_UTF8, nullptr, nullptr,
310 metrics::RepeatedFieldStep, metrics::RepeatedFieldFinal, nullptr);
311 if (ret)
312 PERFETTO_ELOG("Error initializing RepeatedField");
313 }
314}
Eric Secklerc622b8b2019-10-21 09:54:00 +0100315#endif // PERFETTO_BUILDFLAG(PERFETTO_TP_METRICS)
Lalit Maganti2553b802019-05-24 13:25:58 +0100316
Hector Dearmanc0a29092018-11-08 17:34:23 +0000317} // namespace
318
Lalit Maganti295a8612019-05-21 13:57:42 +0100319TraceProcessorImpl::TraceProcessorImpl(const Config& cfg) {
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000320 sqlite3* db = nullptr;
Mikhail Khokhlov4a911282019-05-31 14:25:07 +0100321 PERFETTO_CHECK(sqlite3_initialize() == SQLITE_OK);
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000322 PERFETTO_CHECK(sqlite3_open(":memory:", &db) == SQLITE_OK);
Ioannis Ilkos7199d552019-03-13 18:35:31 +0000323 InitializeSqlite(db);
Sami Kyostila33668942018-11-13 16:33:32 +0000324 CreateBuiltinTables(db);
Lalit Maganti8320e6d2019-03-14 18:49:33 +0000325 CreateBuiltinViews(db);
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000326 db_.reset(std::move(db));
327
Lalit Maganti295a8612019-05-21 13:57:42 +0100328 context_.config = cfg;
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000329 context_.storage.reset(new TraceStorage());
Lalit Maganti4ea78d92019-09-20 20:20:41 +0100330 context_.track_tracker.reset(new TrackTracker(&context_));
Lalit Maganti4fa7c6c2019-02-06 15:06:36 +0000331 context_.args_tracker.reset(new ArgsTracker(&context_));
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000332 context_.slice_tracker.reset(new SliceTracker(&context_));
333 context_.event_tracker.reset(new EventTracker(&context_));
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000334 context_.process_tracker.reset(new ProcessTracker(&context_));
Hector Dearmanf31a7032019-03-29 10:05:09 +0000335 context_.syscall_tracker.reset(new SyscallTracker(&context_));
Primiano Tuccia270f012019-01-07 20:01:00 +0000336 context_.clock_tracker.reset(new ClockTracker(&context_));
Florian Mayer438b5ab2019-05-02 11:18:06 +0100337 context_.heap_profile_tracker.reset(new HeapProfileTracker(&context_));
Florian Mayer25e013e2019-10-01 14:06:15 +0100338 context_.heap_graph_tracker.reset(new HeapGraphTracker(&context_));
Eric Secklerac2ea612019-10-21 16:44:39 +0100339#if PERFETTO_BUILDFLAG(PERFETTO_TP_FTRACE)
340 context_.sched_tracker.reset(new SchedEventTracker(&context_));
Lalit Magantid33dc272019-06-06 14:51:45 +0100341 context_.systrace_parser.reset(new SystraceParser(&context_));
Eric Secklerac2ea612019-10-21 16:44:39 +0100342#endif // PERFETTO_BUILDFLAG(PERFETTO_TP_FTRACE)
Mohammad Reza Zakerinasabb06d1852019-09-11 14:41:36 -0400343 context_.vulkan_memory_tracker.reset(new VulkanMemoryTracker(&context_));
Eric Seckler87ede882019-10-18 10:48:36 +0100344 context_.ftrace_module.reset(
345 new ProtoImporterModule<FtraceModule>(&context_));
346 context_.track_event_module.reset(
347 new ProtoImporterModule<TrackEventModule>(&context_));
Eric Secklerab0604c2019-10-25 10:12:07 +0100348 context_.systrace_module.reset(
349 new ProtoImporterModule<SystraceProtoModule>(&context_));
350 context_.android_probes_module.reset(
351 new ProtoImporterModule<AndroidProbesModule>(&context_));
Eric Seckler63afd792019-10-23 15:05:05 +0100352 context_.graphics_event_module.reset(
353 new ProtoImporterModule<GraphicsEventModule>(&context_));
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000354
Primiano Tucci02c11762019-08-30 00:57:59 +0200355#if PERFETTO_BUILDFLAG(PERFETTO_TP_JSON)
Mikhail Khokhlova8d310d2019-05-07 17:34:21 +0100356 CreateJsonExportFunction(this->context_.storage.get(), db);
357#endif
Lalit Maganti062c8f82019-10-03 20:11:29 +0100358 CreateHashFunction(db);
Mikhail Khokhlova8d310d2019-05-07 17:34:21 +0100359
Eric Secklerc622b8b2019-10-21 09:54:00 +0100360#if PERFETTO_BUILDFLAG(PERFETTO_TP_METRICS)
Lalit Maganti5f8727d2019-05-24 13:38:50 +0100361 SetupMetrics(this, *db_, &sql_metrics_);
Eric Secklerc622b8b2019-10-21 09:54:00 +0100362#endif // PERFETTO_BUILDFLAG(PERFETTO_TP_METRICS)
Lalit Maganti2553b802019-05-24 13:25:58 +0100363
Lalit Maganti6e9c55e2018-11-29 12:00:39 +0000364 ArgsTable::RegisterTable(*db_, context_.storage.get());
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000365 ProcessTable::RegisterTable(*db_, context_.storage.get());
Eric Secklerac2ea612019-10-21 16:44:39 +0100366#if PERFETTO_BUILDFLAG(PERFETTO_TP_FTRACE)
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000367 SchedSliceTable::RegisterTable(*db_, context_.storage.get());
Eric Secklerac2ea612019-10-21 16:44:39 +0100368#endif // PERFETTO_BUILDFLAG(PERFETTO_TP_FTRACE)
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000369 SliceTable::RegisterTable(*db_, context_.storage.get());
Primiano Tucci5cb84f82018-10-31 21:46:36 -0700370 SqlStatsTable::RegisterTable(*db_, context_.storage.get());
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000371 ThreadTable::RegisterTable(*db_, context_.storage.get());
Lalit Maganti8320e6d2019-03-14 18:49:33 +0000372 CounterDefinitionsTable::RegisterTable(*db_, context_.storage.get());
373 CounterValuesTable::RegisterTable(*db_, context_.storage.get());
Lalit Maganti95693bb2018-11-05 16:20:24 +0000374 SpanJoinOperatorTable::RegisterTable(*db_, context_.storage.get());
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000375 WindowOperatorTable::RegisterTable(*db_, context_.storage.get());
Isabelle Taylorc8c11202018-11-05 11:36:22 +0000376 InstantsTable::RegisterTable(*db_, context_.storage.get());
Lalit Maganti05e8c132018-11-09 18:16:12 +0000377 StatsTable::RegisterTable(*db_, context_.storage.get());
Primiano Tucci2c761ef2019-01-07 20:20:46 +0000378 AndroidLogsTable::RegisterTable(*db_, context_.storage.get());
Isabelle Taylord3f590e2019-01-08 17:21:23 +0000379 RawTable::RegisterTable(*db_, context_.storage.get());
Lalit Maganti9b2d52b2019-05-07 14:32:15 +0100380 HeapProfileAllocationTable::RegisterTable(*db_, context_.storage.get());
Oystein Eftevaag7f64c102019-08-29 10:27:31 -0700381 CpuProfileStackSampleTable::RegisterTable(*db_, context_.storage.get());
Oystein Eftevaag5419c582019-08-21 13:58:49 -0700382 StackProfileCallsiteTable::RegisterTable(*db_, context_.storage.get());
383 StackProfileFrameTable::RegisterTable(*db_, context_.storage.get());
384 StackProfileMappingTable::RegisterTable(*db_, context_.storage.get());
Mikhail Khokhlove466c002019-05-23 13:33:33 +0100385 MetadataTable::RegisterTable(*db_, context_.storage.get());
Lalit Maganti40697112019-09-04 22:06:42 +0100386
387 // New style db-backed tables.
Lalit Maganti786cc922019-09-06 16:04:32 +0100388 const TraceStorage* storage = context_.storage.get();
Lalit Magantiac68e9c2019-09-09 18:12:15 +0100389 DbSqliteTable::RegisterTable(*db_, &storage->track_table(),
390 storage->track_table().table_name());
Lalit Maganti21b21632019-09-28 16:50:23 +0100391 DbSqliteTable::RegisterTable(*db_, &storage->thread_track_table(),
392 storage->thread_track_table().table_name());
Lalit Maganti53bdbb22019-09-25 11:11:18 +0100393 DbSqliteTable::RegisterTable(*db_, &storage->process_track_table(),
394 storage->process_track_table().table_name());
Lalit Maganti786cc922019-09-06 16:04:32 +0100395 DbSqliteTable::RegisterTable(*db_, &storage->gpu_slice_table(),
Lalit Magantiac68e9c2019-09-09 18:12:15 +0100396 storage->gpu_slice_table().table_name());
Lalit Maganti786cc922019-09-06 16:04:32 +0100397 DbSqliteTable::RegisterTable(*db_, &storage->gpu_track_table(),
Lalit Magantiac68e9c2019-09-09 18:12:15 +0100398 storage->gpu_track_table().table_name());
Ioannis Ilkose0b47f52019-09-18 11:14:57 +0100399 DbSqliteTable::RegisterTable(*db_, &storage->symbol_table(),
400 storage->symbol_table().table_name());
Florian Mayer25e013e2019-10-01 14:06:15 +0100401 DbSqliteTable::RegisterTable(*db_, &storage->heap_graph_object_table(),
402 storage->heap_graph_object_table().table_name());
Mohammad Reza Zakerinasabb06d1852019-09-11 14:41:36 -0400403 DbSqliteTable::RegisterTable(
Florian Mayer9c01d162019-10-09 17:55:32 +0100404 *db_, &storage->heap_graph_reference_table(),
405 storage->heap_graph_reference_table().table_name());
406 DbSqliteTable::RegisterTable(
Mohammad Reza Zakerinasabb06d1852019-09-11 14:41:36 -0400407 *db_, &storage->vulkan_memory_allocations_table(),
408 storage->vulkan_memory_allocations_table().table_name());
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000409}
410
Lalit Maganti77095012019-02-19 13:13:59 +0000411TraceProcessorImpl::~TraceProcessorImpl() {
412 for (auto* it : iterators_)
413 it->Reset();
414}
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000415
Lalit Maganti1f067182019-05-09 14:50:05 +0100416util::Status TraceProcessorImpl::Parse(std::unique_ptr<uint8_t[]> data,
417 size_t size) {
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000418 if (size == 0)
Lalit Maganti1f067182019-05-09 14:50:05 +0100419 return util::OkStatus();
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000420 if (unrecoverable_parse_error_)
Lalit Maganti1f067182019-05-09 14:50:05 +0100421 return util::ErrStatus(
422 "Failed unrecoverably while parsing in a previous Parse call");
Hector Dearmand8c3d5c2019-08-06 13:28:23 +0100423 if (!context_.chunk_reader)
424 context_.chunk_reader.reset(new ForwardingTraceParser(&context_));
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000425
Eric Seckler77b52782019-05-02 15:18:57 +0100426 auto scoped_trace = context_.storage->TraceExecutionTimeIntoStats(
427 stats::parse_trace_duration_ns);
Lalit Magantid71a9452019-05-09 15:13:24 +0100428 util::Status status = context_.chunk_reader->Parse(std::move(data), size);
429 unrecoverable_parse_error_ |= !status.ok();
430 return status;
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000431}
432
433void TraceProcessorImpl::NotifyEndOfFile() {
Deepanjan Roy01994ca2019-04-02 11:05:34 -0700434 if (unrecoverable_parse_error_ || !context_.chunk_reader)
435 return;
436
Lalit Magantid54d7522019-05-30 14:36:08 +0100437 if (context_.sorter)
438 context_.sorter->ExtractEventsForced();
Eric Secklerac2ea612019-10-21 16:44:39 +0100439#if PERFETTO_BUILDFLAG(PERFETTO_TP_FTRACE)
Eric Secklerd3795132019-10-21 16:43:10 +0100440 context_.sched_tracker->FlushPendingEvents();
Eric Secklerac2ea612019-10-21 16:44:39 +0100441#endif // PERFETTO_BUILDFLAG(PERFETTO_TP_FTRACE)
Lalit Maganti521d97b2019-04-29 13:47:03 +0100442 context_.event_tracker->FlushPendingEvents();
Eric Seckler87fd11a2019-06-07 15:52:28 +0100443 context_.slice_tracker->FlushPendingSlices();
Ioannis Ilkosb8b11102019-01-29 17:56:55 +0000444 BuildBoundsTable(*db_, context_.storage->GetTraceTimestampBoundsNs());
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000445}
446
Lalit Maganti77095012019-02-19 13:13:59 +0000447TraceProcessor::Iterator TraceProcessorImpl::ExecuteQuery(
Lalit Magantiaac2f652019-04-30 12:16:21 +0100448 const std::string& sql,
449 int64_t time_queued) {
Lalit Maganti77095012019-02-19 13:13:59 +0000450 sqlite3_stmt* raw_stmt;
Lalit Maganti73b1a0a2019-04-08 13:51:14 +0100451 int err = sqlite3_prepare_v2(*db_, sql.c_str(), static_cast<int>(sql.size()),
Lalit Maganti77095012019-02-19 13:13:59 +0000452 &raw_stmt, nullptr);
Lalit Maganti1f067182019-05-09 14:50:05 +0100453 util::Status status;
Lalit Maganti06acc022019-04-03 16:02:15 +0100454 uint32_t col_count = 0;
455 if (err != SQLITE_OK) {
Lalit Maganti7ee6abb2019-05-09 17:57:32 +0100456 status = util::ErrStatus("%s", sqlite3_errmsg(*db_));
Lalit Maganti77095012019-02-19 13:13:59 +0000457 } else {
458 col_count = static_cast<uint32_t>(sqlite3_column_count(raw_stmt));
459 }
Lalit Magantiaac2f652019-04-30 12:16:21 +0100460
461 base::TimeNanos t_start = base::GetWallTimeNs();
462 uint32_t sql_stats_row =
463 context_.storage->mutable_sql_stats()->RecordQueryBegin(sql, time_queued,
464 t_start.count());
465
466 std::unique_ptr<IteratorImpl> impl(new IteratorImpl(
Lalit Maganti1f067182019-05-09 14:50:05 +0100467 this, *db_, ScopedStmt(raw_stmt), col_count, status, sql_stats_row));
Lalit Maganti77095012019-02-19 13:13:59 +0000468 iterators_.emplace_back(impl.get());
469 return TraceProcessor::Iterator(std::move(impl));
470}
471
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000472void TraceProcessorImpl::InterruptQuery() {
473 if (!db_)
474 return;
475 query_interrupted_.store(true);
476 sqlite3_interrupt(db_.get());
477}
478
Eric Secklerc622b8b2019-10-21 09:54:00 +0100479#if PERFETTO_BUILDFLAG(PERFETTO_TP_METRICS)
Lalit Maganti5f8727d2019-05-24 13:38:50 +0100480util::Status TraceProcessorImpl::RegisterMetric(const std::string& path,
481 const std::string& sql) {
Lalit Magantif9d80302019-06-19 12:48:25 +0100482 std::string stripped_sql;
483 for (base::StringSplitter sp(sql, '\n'); sp.Next();) {
484 if (strncmp(sp.cur_token(), "--", 2) != 0) {
485 stripped_sql.append(sp.cur_token());
486 stripped_sql.push_back('\n');
487 }
488 }
489
490 // Check if the metric with the given path already exists and if it does, just
491 // update the SQL associated with it.
492 auto it = std::find_if(
493 sql_metrics_.begin(), sql_metrics_.end(),
494 [&path](const metrics::SqlMetricFile& m) { return m.path == path; });
495 if (it != sql_metrics_.end()) {
496 it->sql = stripped_sql;
497 return util::OkStatus();
498 }
499
Lalit Maganti5f8727d2019-05-24 13:38:50 +0100500 auto sep_idx = path.rfind("/");
501 std::string basename =
502 sep_idx == std::string::npos ? path : path.substr(sep_idx + 1);
503
504 auto sql_idx = basename.rfind(".sql");
505 if (sql_idx == std::string::npos) {
506 return util::ErrStatus("Unable to find .sql extension for metric");
507 }
508 auto no_ext_name = basename.substr(0, sql_idx);
509
510 metrics::SqlMetricFile metric;
511 metric.path = path;
512 metric.proto_field_name = no_ext_name;
513 metric.output_table_name = no_ext_name + "_output";
Lalit Magantica4d5142019-05-28 13:25:47 +0100514 metric.sql = stripped_sql;
Lalit Maganti5f8727d2019-05-24 13:38:50 +0100515 sql_metrics_.emplace_back(metric);
516 return util::OkStatus();
517}
518
519util::Status TraceProcessorImpl::ExtendMetricsProto(const uint8_t* data,
520 size_t size) {
Lalit Maganti996f5442019-06-21 16:12:58 +0100521 util::Status status = pool_.AddFromFileDescriptorSet(data, size);
522 if (!status.ok())
523 return status;
Lalit Maganti5f8727d2019-05-24 13:38:50 +0100524
Lalit Maganti2553b802019-05-24 13:25:58 +0100525 for (const auto& desc : pool_.descriptors()) {
Lalit Maganti62211072019-05-10 14:09:58 +0100526 // Convert the full name (e.g. .perfetto.protos.TraceMetrics.SubMetric)
527 // into a function name of the form (TraceMetrics_SubMetric).
528 auto fn_name = desc.full_name().substr(desc.package_name().size() + 1);
529 std::replace(fn_name.begin(), fn_name.end(), '.', '_');
530
Lalit Magantib182d7e2019-05-10 14:13:06 +0100531 std::unique_ptr<metrics::BuildProtoContext> ctx(
532 new metrics::BuildProtoContext());
533 ctx->tp = this;
Lalit Maganti2553b802019-05-24 13:25:58 +0100534 ctx->pool = &pool_;
Lalit Magantib182d7e2019-05-10 14:13:06 +0100535 ctx->desc = &desc;
536
Lalit Maganti62211072019-05-10 14:09:58 +0100537 auto ret = sqlite3_create_function_v2(
Lalit Magantib182d7e2019-05-10 14:13:06 +0100538 *db_, fn_name.c_str(), -1, SQLITE_UTF8, ctx.release(),
539 metrics::BuildProto, nullptr, nullptr, [](void* ptr) {
540 delete static_cast<metrics::BuildProtoContext*>(ptr);
541 });
Lalit Maganti62211072019-05-10 14:09:58 +0100542 if (ret != SQLITE_OK)
543 return util::ErrStatus("%s", sqlite3_errmsg(*db_));
544 }
Lalit Maganti996f5442019-06-21 16:12:58 +0100545 return util::OkStatus();
546}
Lalit Maganti031ad2b2019-05-14 17:37:14 +0100547
Lalit Maganti996f5442019-06-21 16:12:58 +0100548util::Status TraceProcessorImpl::ComputeMetric(
549 const std::vector<std::string>& metric_names,
550 std::vector<uint8_t>* metrics_proto) {
Lalit Maganti2553b802019-05-24 13:25:58 +0100551 auto opt_idx = pool_.FindDescriptorIdx(".perfetto.protos.TraceMetrics");
Lalit Maganti031ad2b2019-05-14 17:37:14 +0100552 if (!opt_idx.has_value())
553 return util::Status("Root metrics proto descriptor not found");
554
Lalit Maganti2553b802019-05-24 13:25:58 +0100555 const auto& root_descriptor = pool_.descriptors()[opt_idx.value()];
556 return metrics::ComputeMetrics(this, metric_names, sql_metrics_,
557 root_descriptor, metrics_proto);
Lalit Magantid9f86b62019-04-08 11:11:51 +0100558}
Eric Secklerc622b8b2019-10-21 09:54:00 +0100559#endif // PERFETTO_BUILDFLAG(PERFETTO_TP_METRICS)
Lalit Magantid9f86b62019-04-08 11:11:51 +0100560
Lalit Maganti77095012019-02-19 13:13:59 +0000561TraceProcessor::IteratorImpl::IteratorImpl(TraceProcessorImpl* trace_processor,
562 sqlite3* db,
563 ScopedStmt stmt,
564 uint32_t column_count,
Lalit Maganti1f067182019-05-09 14:50:05 +0100565 util::Status status,
Lalit Magantiaac2f652019-04-30 12:16:21 +0100566 uint32_t sql_stats_row)
Lalit Maganti77095012019-02-19 13:13:59 +0000567 : trace_processor_(trace_processor),
568 db_(db),
569 stmt_(std::move(stmt)),
570 column_count_(column_count),
Lalit Maganti1f067182019-05-09 14:50:05 +0100571 status_(status),
Lalit Magantiaac2f652019-04-30 12:16:21 +0100572 sql_stats_row_(sql_stats_row) {}
Lalit Maganti77095012019-02-19 13:13:59 +0000573
574TraceProcessor::IteratorImpl::~IteratorImpl() {
575 if (trace_processor_) {
576 auto* its = &trace_processor_->iterators_;
577 auto it = std::find(its->begin(), its->end(), this);
578 PERFETTO_CHECK(it != its->end());
579 its->erase(it);
Lalit Magantiaac2f652019-04-30 12:16:21 +0100580
581 base::TimeNanos t_end = base::GetWallTimeNs();
582 auto* sql_stats = trace_processor_->context_.storage->mutable_sql_stats();
583 sql_stats->RecordQueryEnd(sql_stats_row_, t_end.count());
Lalit Maganti77095012019-02-19 13:13:59 +0000584 }
585}
586
587void TraceProcessor::IteratorImpl::Reset() {
Lalit Maganti1f067182019-05-09 14:50:05 +0100588 *this = IteratorImpl(nullptr, nullptr, ScopedStmt(), 0,
589 util::ErrStatus("Trace processor was deleted"), 0);
Lalit Magantiaac2f652019-04-30 12:16:21 +0100590}
591
592void TraceProcessor::IteratorImpl::RecordFirstNextInSqlStats() {
593 base::TimeNanos t_first_next = base::GetWallTimeNs();
594 auto* sql_stats = trace_processor_->context_.storage->mutable_sql_stats();
595 sql_stats->RecordQueryFirstNext(sql_stats_row_, t_first_next.count());
Lalit Maganti77095012019-02-19 13:13:59 +0000596}
597
Ioannis Ilkoseff38f52018-10-29 10:37:55 +0000598} // namespace trace_processor
599} // namespace perfetto