Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2018 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #include "src/trace_processor/trace_processor_impl.h" |
| 18 | |
| 19 | #include <sqlite3.h> |
Hector Dearman | c0a2909 | 2018-11-08 17:34:23 +0000 | [diff] [blame] | 20 | #include <algorithm> |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 21 | #include <functional> |
| 22 | |
| 23 | #include "perfetto/base/time.h" |
Primiano Tucci | 2c761ef | 2019-01-07 20:20:46 +0000 | [diff] [blame^] | 24 | #include "src/trace_processor/android_logs_table.h" |
Lalit Maganti | 6e9c55e | 2018-11-29 12:00:39 +0000 | [diff] [blame] | 25 | #include "src/trace_processor/args_table.h" |
Primiano Tucci | a270f01 | 2019-01-07 20:01:00 +0000 | [diff] [blame] | 26 | #include "src/trace_processor/clock_tracker.h" |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 27 | #include "src/trace_processor/counters_table.h" |
| 28 | #include "src/trace_processor/event_tracker.h" |
Isabelle Taylor | c8c1120 | 2018-11-05 11:36:22 +0000 | [diff] [blame] | 29 | #include "src/trace_processor/instants_table.h" |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 30 | #include "src/trace_processor/json_trace_parser.h" |
| 31 | #include "src/trace_processor/process_table.h" |
| 32 | #include "src/trace_processor/process_tracker.h" |
| 33 | #include "src/trace_processor/proto_trace_parser.h" |
| 34 | #include "src/trace_processor/proto_trace_tokenizer.h" |
| 35 | #include "src/trace_processor/sched_slice_table.h" |
| 36 | #include "src/trace_processor/slice_table.h" |
| 37 | #include "src/trace_processor/slice_tracker.h" |
Lalit Maganti | 95693bb | 2018-11-05 16:20:24 +0000 | [diff] [blame] | 38 | #include "src/trace_processor/span_join_operator_table.h" |
Primiano Tucci | 5cb84f8 | 2018-10-31 21:46:36 -0700 | [diff] [blame] | 39 | #include "src/trace_processor/sql_stats_table.h" |
Lalit Maganti | 05e8c13 | 2018-11-09 18:16:12 +0000 | [diff] [blame] | 40 | #include "src/trace_processor/stats_table.h" |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 41 | #include "src/trace_processor/string_table.h" |
| 42 | #include "src/trace_processor/table.h" |
| 43 | #include "src/trace_processor/thread_table.h" |
| 44 | #include "src/trace_processor/trace_sorter.h" |
| 45 | #include "src/trace_processor/window_operator_table.h" |
| 46 | |
| 47 | #include "perfetto/trace_processor/raw_query.pb.h" |
| 48 | |
Ioannis Ilkos | 178535e | 2018-11-05 17:32:45 +0000 | [diff] [blame] | 49 | // defined in sqlite_src/ext/misc/percentile.c |
| 50 | extern "C" int sqlite3_percentile_init(sqlite3* db, |
| 51 | char** error, |
| 52 | const sqlite3_api_routines* api); |
| 53 | |
| 54 | namespace { |
| 55 | void InitializeSqliteModules(sqlite3* db) { |
| 56 | char* error = nullptr; |
| 57 | sqlite3_percentile_init(db, &error, nullptr); |
Sami Kyostila | 3366894 | 2018-11-13 16:33:32 +0000 | [diff] [blame] | 58 | if (error) { |
Ioannis Ilkos | 178535e | 2018-11-05 17:32:45 +0000 | [diff] [blame] | 59 | PERFETTO_ELOG("Error initializing: %s", error); |
Sami Kyostila | 3366894 | 2018-11-13 16:33:32 +0000 | [diff] [blame] | 60 | sqlite3_free(error); |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | void CreateBuiltinTables(sqlite3* db) { |
| 65 | char* error = nullptr; |
| 66 | sqlite3_exec(db, "CREATE TABLE perfetto_tables(name STRING)", 0, 0, &error); |
| 67 | if (error) { |
| 68 | PERFETTO_ELOG("Error initializing: %s", error); |
| 69 | sqlite3_free(error); |
Ioannis Ilkos | 178535e | 2018-11-05 17:32:45 +0000 | [diff] [blame] | 70 | } |
| 71 | } |
| 72 | } // namespace |
| 73 | |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 74 | namespace perfetto { |
| 75 | namespace trace_processor { |
Hector Dearman | c0a2909 | 2018-11-08 17:34:23 +0000 | [diff] [blame] | 76 | namespace { |
| 77 | |
| 78 | bool IsPrefix(const std::string& a, const std::string& b) { |
| 79 | return a.size() <= b.size() && b.substr(0, a.size()) == a; |
| 80 | } |
| 81 | |
| 82 | std::string RemoveWhitespace(const std::string& input) { |
| 83 | std::string str(input); |
| 84 | str.erase(std::remove_if(str.begin(), str.end(), ::isspace), str.end()); |
| 85 | return str; |
| 86 | } |
| 87 | |
| 88 | } // namespace |
| 89 | |
| 90 | TraceType GuessTraceType(const uint8_t* data, size_t size) { |
| 91 | if (size == 0) |
| 92 | return kUnknownTraceType; |
| 93 | std::string start(reinterpret_cast<const char*>(data), |
| 94 | std::min<size_t>(size, 20)); |
| 95 | std::string start_minus_white_space = RemoveWhitespace(start); |
| 96 | if (IsPrefix("{\"traceEvents\":[", start_minus_white_space)) |
| 97 | return kJsonTraceType; |
| 98 | if (IsPrefix("[{", start_minus_white_space)) |
| 99 | return kJsonTraceType; |
| 100 | return kProtoTraceType; |
| 101 | } |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 102 | |
| 103 | TraceProcessorImpl::TraceProcessorImpl(const Config& cfg) { |
| 104 | sqlite3* db = nullptr; |
| 105 | PERFETTO_CHECK(sqlite3_open(":memory:", &db) == SQLITE_OK); |
Ioannis Ilkos | 178535e | 2018-11-05 17:32:45 +0000 | [diff] [blame] | 106 | InitializeSqliteModules(db); |
Sami Kyostila | 3366894 | 2018-11-13 16:33:32 +0000 | [diff] [blame] | 107 | CreateBuiltinTables(db); |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 108 | db_.reset(std::move(db)); |
| 109 | |
| 110 | context_.storage.reset(new TraceStorage()); |
| 111 | context_.slice_tracker.reset(new SliceTracker(&context_)); |
| 112 | context_.event_tracker.reset(new EventTracker(&context_)); |
| 113 | context_.proto_parser.reset(new ProtoTraceParser(&context_)); |
| 114 | context_.process_tracker.reset(new ProcessTracker(&context_)); |
Primiano Tucci | a270f01 | 2019-01-07 20:01:00 +0000 | [diff] [blame] | 115 | context_.clock_tracker.reset(new ClockTracker(&context_)); |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 116 | context_.sorter.reset( |
Lalit Maganti | 85ca4a8 | 2018-12-07 17:28:02 +0000 | [diff] [blame] | 117 | new TraceSorter(&context_, cfg.optimization_mode, |
| 118 | static_cast<int64_t>(cfg.window_size_ns))); |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 119 | |
Lalit Maganti | 6e9c55e | 2018-11-29 12:00:39 +0000 | [diff] [blame] | 120 | ArgsTable::RegisterTable(*db_, context_.storage.get()); |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 121 | ProcessTable::RegisterTable(*db_, context_.storage.get()); |
| 122 | SchedSliceTable::RegisterTable(*db_, context_.storage.get()); |
| 123 | SliceTable::RegisterTable(*db_, context_.storage.get()); |
Primiano Tucci | 5cb84f8 | 2018-10-31 21:46:36 -0700 | [diff] [blame] | 124 | SqlStatsTable::RegisterTable(*db_, context_.storage.get()); |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 125 | StringTable::RegisterTable(*db_, context_.storage.get()); |
| 126 | ThreadTable::RegisterTable(*db_, context_.storage.get()); |
| 127 | CountersTable::RegisterTable(*db_, context_.storage.get()); |
Lalit Maganti | 95693bb | 2018-11-05 16:20:24 +0000 | [diff] [blame] | 128 | SpanJoinOperatorTable::RegisterTable(*db_, context_.storage.get()); |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 129 | WindowOperatorTable::RegisterTable(*db_, context_.storage.get()); |
Isabelle Taylor | c8c1120 | 2018-11-05 11:36:22 +0000 | [diff] [blame] | 130 | InstantsTable::RegisterTable(*db_, context_.storage.get()); |
Lalit Maganti | 05e8c13 | 2018-11-09 18:16:12 +0000 | [diff] [blame] | 131 | StatsTable::RegisterTable(*db_, context_.storage.get()); |
Primiano Tucci | 2c761ef | 2019-01-07 20:20:46 +0000 | [diff] [blame^] | 132 | AndroidLogsTable::RegisterTable(*db_, context_.storage.get()); |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 133 | } |
| 134 | |
| 135 | TraceProcessorImpl::~TraceProcessorImpl() = default; |
| 136 | |
| 137 | bool TraceProcessorImpl::Parse(std::unique_ptr<uint8_t[]> data, size_t size) { |
| 138 | if (size == 0) |
| 139 | return true; |
| 140 | if (unrecoverable_parse_error_) |
| 141 | return false; |
| 142 | |
| 143 | // If this is the first Parse() call, guess the trace type and create the |
| 144 | // appropriate parser. |
| 145 | if (!context_.chunk_reader) { |
Hector Dearman | c0a2909 | 2018-11-08 17:34:23 +0000 | [diff] [blame] | 146 | TraceType trace_type = GuessTraceType(data.get(), size); |
| 147 | switch (trace_type) { |
| 148 | case kJsonTraceType: |
| 149 | PERFETTO_DLOG("Legacy JSON trace detected"); |
| 150 | context_.chunk_reader.reset(new JsonTraceParser(&context_)); |
| 151 | break; |
| 152 | case kProtoTraceType: |
| 153 | context_.chunk_reader.reset(new ProtoTraceTokenizer(&context_)); |
| 154 | break; |
| 155 | case kUnknownTraceType: |
| 156 | return false; |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 157 | } |
| 158 | } |
| 159 | |
| 160 | bool res = context_.chunk_reader->Parse(std::move(data), size); |
| 161 | unrecoverable_parse_error_ |= !res; |
| 162 | return res; |
| 163 | } |
| 164 | |
| 165 | void TraceProcessorImpl::NotifyEndOfFile() { |
| 166 | context_.sorter->FlushEventsForced(); |
| 167 | } |
| 168 | |
| 169 | void TraceProcessorImpl::ExecuteQuery( |
| 170 | const protos::RawQueryArgs& args, |
| 171 | std::function<void(const protos::RawQueryResult&)> callback) { |
| 172 | protos::RawQueryResult proto; |
| 173 | query_interrupted_.store(false, std::memory_order_relaxed); |
| 174 | |
| 175 | base::TimeNanos t_start = base::GetWallTimeNs(); |
Primiano Tucci | 5cb84f8 | 2018-10-31 21:46:36 -0700 | [diff] [blame] | 176 | const std::string& sql = args.sql_query(); |
| 177 | context_.storage->mutable_sql_stats()->RecordQueryBegin( |
Lalit Maganti | 85ca4a8 | 2018-12-07 17:28:02 +0000 | [diff] [blame] | 178 | sql, static_cast<int64_t>(args.time_queued_ns()), t_start.count()); |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 179 | sqlite3_stmt* raw_stmt; |
| 180 | int err = sqlite3_prepare_v2(*db_, sql.c_str(), static_cast<int>(sql.size()), |
| 181 | &raw_stmt, nullptr); |
| 182 | ScopedStmt stmt(raw_stmt); |
Lalit Maganti | b55c884 | 2018-12-13 14:26:08 +0000 | [diff] [blame] | 183 | |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 184 | int col_count = sqlite3_column_count(*stmt); |
| 185 | int row_count = 0; |
Lalit Maganti | b55c884 | 2018-12-13 14:26:08 +0000 | [diff] [blame] | 186 | |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 187 | while (!err) { |
| 188 | int r = sqlite3_step(*stmt); |
| 189 | if (r != SQLITE_ROW) { |
| 190 | if (r != SQLITE_DONE) |
| 191 | err = r; |
| 192 | break; |
| 193 | } |
| 194 | |
Lalit Maganti | b55c884 | 2018-12-13 14:26:08 +0000 | [diff] [blame] | 195 | using ColumnDesc = protos::RawQueryResult::ColumnDesc; |
| 196 | for (int col = 0; col < col_count; col++) { |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 197 | if (row_count == 0) { |
| 198 | // Setup the descriptors. |
| 199 | auto* descriptor = proto.add_column_descriptors(); |
Lalit Maganti | b55c884 | 2018-12-13 14:26:08 +0000 | [diff] [blame] | 200 | descriptor->set_name(sqlite3_column_name(*stmt, col)); |
| 201 | descriptor->set_type(ColumnDesc::UNKNOWN); |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 202 | |
| 203 | // Add an empty column. |
| 204 | proto.add_columns(); |
| 205 | } |
| 206 | |
Lalit Maganti | b55c884 | 2018-12-13 14:26:08 +0000 | [diff] [blame] | 207 | auto* column = proto.mutable_columns(col); |
| 208 | auto* desc = proto.mutable_column_descriptors(col); |
| 209 | auto col_type = sqlite3_column_type(*stmt, col); |
| 210 | if (desc->type() == ColumnDesc::UNKNOWN) { |
| 211 | switch (col_type) { |
| 212 | case SQLITE_INTEGER: |
| 213 | desc->set_type(ColumnDesc::LONG); |
| 214 | break; |
| 215 | case SQLITE_TEXT: |
| 216 | desc->set_type(ColumnDesc::STRING); |
| 217 | break; |
| 218 | case SQLITE_FLOAT: |
| 219 | desc->set_type(ColumnDesc::DOUBLE); |
| 220 | break; |
| 221 | case SQLITE_NULL: |
| 222 | break; |
| 223 | } |
| 224 | } |
| 225 | |
| 226 | // If either the column type is null or we still don't know the type, |
| 227 | // just add null values to all the columns. |
| 228 | if (col_type == SQLITE_NULL || desc->type() == ColumnDesc::UNKNOWN) { |
| 229 | column->add_long_values(0); |
| 230 | column->add_string_values("[NULL]"); |
| 231 | column->add_double_values(0); |
| 232 | column->add_is_nulls(true); |
| 233 | continue; |
| 234 | } |
| 235 | |
| 236 | // Cast the sqlite value to the type of the column. |
| 237 | switch (desc->type()) { |
| 238 | case ColumnDesc::LONG: |
| 239 | column->add_long_values(sqlite3_column_int64(*stmt, col)); |
| 240 | column->add_is_nulls(false); |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 241 | break; |
Lalit Maganti | b55c884 | 2018-12-13 14:26:08 +0000 | [diff] [blame] | 242 | case ColumnDesc::STRING: { |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 243 | const char* str = |
Lalit Maganti | b55c884 | 2018-12-13 14:26:08 +0000 | [diff] [blame] | 244 | reinterpret_cast<const char*>(sqlite3_column_text(*stmt, col)); |
| 245 | column->add_string_values(str); |
| 246 | column->add_is_nulls(false); |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 247 | break; |
| 248 | } |
Lalit Maganti | b55c884 | 2018-12-13 14:26:08 +0000 | [diff] [blame] | 249 | case ColumnDesc::DOUBLE: |
| 250 | column->add_double_values(sqlite3_column_double(*stmt, col)); |
| 251 | column->add_is_nulls(false); |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 252 | break; |
Lalit Maganti | b55c884 | 2018-12-13 14:26:08 +0000 | [diff] [blame] | 253 | case ColumnDesc::UNKNOWN: |
| 254 | PERFETTO_FATAL("Handled in if statement above."); |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 255 | } |
| 256 | } |
| 257 | row_count++; |
| 258 | } |
| 259 | |
| 260 | if (err) { |
| 261 | proto.set_error(sqlite3_errmsg(*db_)); |
| 262 | callback(std::move(proto)); |
| 263 | return; |
| 264 | } |
| 265 | |
| 266 | proto.set_num_records(static_cast<uint64_t>(row_count)); |
| 267 | |
| 268 | if (query_interrupted_.load()) { |
| 269 | PERFETTO_ELOG("SQLite query interrupted"); |
| 270 | query_interrupted_ = false; |
| 271 | } |
| 272 | |
| 273 | base::TimeNanos t_end = base::GetWallTimeNs(); |
Lalit Maganti | 85ca4a8 | 2018-12-07 17:28:02 +0000 | [diff] [blame] | 274 | context_.storage->mutable_sql_stats()->RecordQueryEnd(t_end.count()); |
Ioannis Ilkos | eff38f5 | 2018-10-29 10:37:55 +0000 | [diff] [blame] | 275 | proto.set_execution_time_ns(static_cast<uint64_t>((t_end - t_start).count())); |
| 276 | callback(proto); |
| 277 | } |
| 278 | |
| 279 | void TraceProcessorImpl::InterruptQuery() { |
| 280 | if (!db_) |
| 281 | return; |
| 282 | query_interrupted_.store(true); |
| 283 | sqlite3_interrupt(db_.get()); |
| 284 | } |
| 285 | |
| 286 | } // namespace trace_processor |
| 287 | } // namespace perfetto |