Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2017, The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | #pragma once |
| 17 | |
| 18 | #include "config/ConfigKey.h" |
Chenjie Yu | b038b70 | 2017-12-18 15:15:34 -0800 | [diff] [blame] | 19 | #include "statslog.h" |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 20 | |
Yao Chen | 69f1baf | 2017-11-27 17:25:36 -0800 | [diff] [blame] | 21 | #include <gtest/gtest_prod.h> |
David Chen | d9269e2 | 2017-12-05 13:43:51 -0800 | [diff] [blame] | 22 | #include <log/log_time.h> |
Yao Chen | f6723df | 2018-01-08 15:11:58 -0800 | [diff] [blame] | 23 | #include <list> |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 24 | #include <mutex> |
| 25 | #include <string> |
| 26 | #include <vector> |
| 27 | |
| 28 | namespace android { |
| 29 | namespace os { |
| 30 | namespace statsd { |
| 31 | |
Yao Chen | 20e9e62 | 2018-02-28 11:18:51 -0800 | [diff] [blame] | 32 | struct ConfigStats { |
| 33 | int32_t uid; |
| 34 | int64_t id; |
| 35 | int32_t creation_time_sec; |
| 36 | int32_t deletion_time_sec = 0; |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 37 | int32_t reset_time_sec = 0; |
Yao Chen | 20e9e62 | 2018-02-28 11:18:51 -0800 | [diff] [blame] | 38 | int32_t metric_count; |
| 39 | int32_t condition_count; |
| 40 | int32_t matcher_count; |
| 41 | int32_t alert_count; |
| 42 | bool is_valid; |
| 43 | |
| 44 | std::list<int32_t> broadcast_sent_time_sec; |
Tej Singh | 6ede28b | 2019-01-29 17:06:54 -0800 | [diff] [blame] | 45 | |
| 46 | // Times at which this config is activated. |
| 47 | std::list<int32_t> activation_time_sec; |
| 48 | |
| 49 | // Times at which this config is deactivated. |
| 50 | std::list<int32_t> deactivation_time_sec; |
| 51 | |
Yao Chen | 20e9e62 | 2018-02-28 11:18:51 -0800 | [diff] [blame] | 52 | std::list<int32_t> data_drop_time_sec; |
Chenjie Yu | c3c30c0 | 2018-10-26 09:48:07 -0700 | [diff] [blame] | 53 | // Number of bytes dropped at corresponding time. |
| 54 | std::list<int64_t> data_drop_bytes; |
Yangster-mac | e68f3a5 | 2018-04-04 00:01:43 -0700 | [diff] [blame] | 55 | std::list<std::pair<int32_t, int64_t>> dump_report_stats; |
Yao Chen | 20e9e62 | 2018-02-28 11:18:51 -0800 | [diff] [blame] | 56 | |
| 57 | // Stores how many times a matcher have been matched. The map size is capped by kMaxConfigCount. |
| 58 | std::map<const int64_t, int> matcher_stats; |
| 59 | |
| 60 | // Stores the number of output tuple of condition trackers when it's bigger than |
| 61 | // kDimensionKeySizeSoftLimit. When you see the number is kDimensionKeySizeHardLimit +1, |
| 62 | // it means some data has been dropped. The map size is capped by kMaxConfigCount. |
| 63 | std::map<const int64_t, int> condition_stats; |
| 64 | |
| 65 | // Stores the number of output tuple of metric producers when it's bigger than |
| 66 | // kDimensionKeySizeSoftLimit. When you see the number is kDimensionKeySizeHardLimit +1, |
| 67 | // it means some data has been dropped. The map size is capped by kMaxConfigCount. |
| 68 | std::map<const int64_t, int> metric_stats; |
| 69 | |
Yangster-mac | 306ccc2 | 2018-03-24 15:03:40 -0700 | [diff] [blame] | 70 | // Stores the max number of output tuple of dimensions in condition across dimensions in what |
| 71 | // when it's bigger than kDimensionKeySizeSoftLimit. When you see the number is |
| 72 | // kDimensionKeySizeHardLimit +1, it means some data has been dropped. The map size is capped by |
| 73 | // kMaxConfigCount. |
| 74 | std::map<const int64_t, int> metric_dimension_in_condition_stats; |
| 75 | |
Yao Chen | 20e9e62 | 2018-02-28 11:18:51 -0800 | [diff] [blame] | 76 | // Stores the number of times an anomaly detection alert has been declared. |
| 77 | // The map size is capped by kMaxConfigCount. |
| 78 | std::map<const int64_t, int> alert_stats; |
David Chen | faa1af5 | 2018-03-30 15:14:04 -0700 | [diff] [blame] | 79 | |
| 80 | // Stores the config ID for each sub-config used. |
| 81 | std::list<std::pair<const int64_t, const int32_t>> annotations; |
Yao Chen | 20e9e62 | 2018-02-28 11:18:51 -0800 | [diff] [blame] | 82 | }; |
| 83 | |
| 84 | struct UidMapStats { |
Yao Chen | 20e9e62 | 2018-02-28 11:18:51 -0800 | [diff] [blame] | 85 | int32_t changes; |
| 86 | int32_t bytes_used; |
Yao Chen | 20e9e62 | 2018-02-28 11:18:51 -0800 | [diff] [blame] | 87 | int32_t dropped_changes; |
David Chen | bd12527 | 2018-04-04 19:02:50 -0700 | [diff] [blame] | 88 | int32_t deleted_apps = 0; |
Yao Chen | 20e9e62 | 2018-02-28 11:18:51 -0800 | [diff] [blame] | 89 | }; |
| 90 | |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 91 | // Keeps track of stats of statsd. |
Yao Chen | 20e9e62 | 2018-02-28 11:18:51 -0800 | [diff] [blame] | 92 | // Single instance shared across the process. All public methods are thread safe. |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 93 | class StatsdStats { |
| 94 | public: |
| 95 | static StatsdStats& getInstance(); |
| 96 | ~StatsdStats(){}; |
| 97 | |
Yao Chen | 52b478b | 2018-03-27 10:59:45 -0700 | [diff] [blame] | 98 | const static int kDimensionKeySizeSoftLimit = 500; |
| 99 | const static int kDimensionKeySizeHardLimit = 800; |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 100 | |
Chenjie Yu | c587505 | 2018-03-09 10:13:11 -0800 | [diff] [blame] | 101 | // Per atom dimension key size limit |
| 102 | static const std::map<int, std::pair<size_t, size_t>> kAtomDimensionKeySizeLimitMap; |
| 103 | |
Yao Chen | 52b478b | 2018-03-27 10:59:45 -0700 | [diff] [blame] | 104 | const static int kMaxConfigCountPerUid = 10; |
Bookatz | 1476ef2 | 2018-02-13 12:26:01 -0800 | [diff] [blame] | 105 | const static int kMaxAlertCountPerConfig = 100; |
Yao Chen | 52b478b | 2018-03-27 10:59:45 -0700 | [diff] [blame] | 106 | const static int kMaxConditionCountPerConfig = 300; |
| 107 | const static int kMaxMetricCountPerConfig = 1000; |
| 108 | const static int kMaxMatcherCountPerConfig = 800; |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 109 | |
Yao Chen | f6723df | 2018-01-08 15:11:58 -0800 | [diff] [blame] | 110 | // The max number of old config stats we keep. |
| 111 | const static int kMaxIceBoxSize = 20; |
| 112 | |
Yao Chen | 163d260 | 2018-04-10 10:39:53 -0700 | [diff] [blame] | 113 | const static int kMaxLoggerErrors = 20; |
Yangster-mac | b8382a1 | 2018-04-04 10:39:12 -0700 | [diff] [blame] | 114 | |
Yangster-mac | 892f3d3 | 2018-05-02 14:16:48 -0700 | [diff] [blame] | 115 | const static int kMaxSystemServerRestarts = 20; |
| 116 | |
Yao Chen | 0fac5b1 | 2017-11-28 16:07:02 -0800 | [diff] [blame] | 117 | const static int kMaxTimestampCount = 20; |
| 118 | |
Yao Chen | d10f7b1 | 2017-12-18 12:53:50 -0800 | [diff] [blame] | 119 | const static int kMaxLogSourceCount = 50; |
| 120 | |
David Chen | 4c6d97a | 2018-03-22 16:31:40 -0700 | [diff] [blame] | 121 | // Max memory allowed for storing metrics per configuration. If this limit is exceeded, statsd |
| 122 | // drops the metrics data in memory. |
Yao Chen | bd88bde | 2018-11-26 15:31:53 -0800 | [diff] [blame] | 123 | static const size_t kMaxMetricsBytesPerConfig = 2 * 1024 * 1024; |
David Chen | 4c6d97a | 2018-03-22 16:31:40 -0700 | [diff] [blame] | 124 | |
| 125 | // Soft memory limit per configuration. Once this limit is exceeded, we begin notifying the |
| 126 | // data subscriber that it's time to call getData. |
Yangster-mac | db185c9 | 2018-04-19 10:54:30 -0700 | [diff] [blame] | 127 | static const size_t kBytesPerConfigTriggerGetData = 192 * 1024; |
David Chen | d9269e2 | 2017-12-05 13:43:51 -0800 | [diff] [blame] | 128 | |
David Chen | c136f45 | 2017-11-27 11:52:26 -0800 | [diff] [blame] | 129 | // Cap the UID map's memory usage to this. This should be fairly high since the UID information |
| 130 | // is critical for understanding the metrics. |
| 131 | const static size_t kMaxBytesUsedUidMap = 50 * 1024; |
| 132 | |
David Chen | bd12527 | 2018-04-04 19:02:50 -0700 | [diff] [blame] | 133 | // The number of deleted apps that are stored in the uid map. |
| 134 | const static int kMaxDeletedAppsInUidMap = 100; |
| 135 | |
David Chen | d9269e2 | 2017-12-05 13:43:51 -0800 | [diff] [blame] | 136 | /* Minimum period between two broadcasts in nanoseconds. */ |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 137 | static const int64_t kMinBroadcastPeriodNs = 60 * NS_PER_SEC; |
David Chen | d9269e2 | 2017-12-05 13:43:51 -0800 | [diff] [blame] | 138 | |
| 139 | /* Min period between two checks of byte size per config key in nanoseconds. */ |
Yao Chen | bd88bde | 2018-11-26 15:31:53 -0800 | [diff] [blame] | 140 | static const int64_t kMinByteSizeCheckPeriodNs = 60 * NS_PER_SEC; |
David Chen | d9269e2 | 2017-12-05 13:43:51 -0800 | [diff] [blame] | 141 | |
Tej Singh | 6ede28b | 2019-01-29 17:06:54 -0800 | [diff] [blame] | 142 | /* Minimum period between two activation broadcasts in nanoseconds. */ |
| 143 | static const int64_t kMinActivationBroadcastPeriodNs = 10 * NS_PER_SEC; |
| 144 | |
yro | 98a2850 | 2018-01-18 17:00:14 -0800 | [diff] [blame] | 145 | // Maximum age (30 days) that files on disk can exist in seconds. |
| 146 | static const int kMaxAgeSecond = 60 * 60 * 24 * 30; |
| 147 | |
| 148 | // Maximum number of files (1000) that can be in stats directory on disk. |
| 149 | static const int kMaxFileNumber = 1000; |
| 150 | |
| 151 | // Maximum size of all files that can be written to stats directory on disk. |
| 152 | static const int kMaxFileSize = 50 * 1024 * 1024; |
| 153 | |
Chenjie Yu | fa22d65 | 2018-02-05 14:37:48 -0800 | [diff] [blame] | 154 | // How long to try to clear puller cache from last time |
| 155 | static const long kPullerCacheClearIntervalSec = 1; |
| 156 | |
Chenjie Yu | 0bd73db | 2018-12-16 07:37:04 -0800 | [diff] [blame] | 157 | // Max time to do a pull. |
| 158 | static const int64_t kPullMaxDelayNs = 10 * NS_PER_SEC; |
Tej Singh | a0c89dd | 2019-01-25 16:39:18 -0800 | [diff] [blame] | 159 | |
| 160 | // Max platform atom tag number. |
| 161 | static const int32_t kMaxPlatformAtomTag = 100000; |
| 162 | |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 163 | /** |
| 164 | * Report a new config has been received and report the static stats about the config. |
| 165 | * |
| 166 | * The static stats include: the count of metrics, conditions, matchers, and alerts. |
| 167 | * If the config is not valid, this config stats will be put into icebox immediately. |
| 168 | */ |
| 169 | void noteConfigReceived(const ConfigKey& key, int metricsCount, int conditionsCount, |
David Chen | faa1af5 | 2018-03-30 15:14:04 -0700 | [diff] [blame] | 170 | int matchersCount, int alertCount, |
| 171 | const std::list<std::pair<const int64_t, const int32_t>>& annotations, |
| 172 | bool isValid); |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 173 | /** |
| 174 | * Report a config has been removed. |
| 175 | */ |
| 176 | void noteConfigRemoved(const ConfigKey& key); |
Yao Chen | a80e5c0 | 2018-09-04 13:55:29 -0700 | [diff] [blame] | 177 | /** |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 178 | * Report a config has been reset when ttl expires. |
| 179 | */ |
| 180 | void noteConfigReset(const ConfigKey& key); |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 181 | |
| 182 | /** |
| 183 | * Report a broadcast has been sent to a config owner to collect the data. |
| 184 | */ |
| 185 | void noteBroadcastSent(const ConfigKey& key); |
| 186 | |
| 187 | /** |
Tej Singh | 6ede28b | 2019-01-29 17:06:54 -0800 | [diff] [blame] | 188 | * Report that a config has become activated or deactivated. |
| 189 | * This can be different from whether or not a broadcast is sent if the |
| 190 | * guardrail prevented the broadcast from being sent. |
| 191 | */ |
| 192 | void noteActiveStatusChanged(const ConfigKey& key, bool activate); |
| 193 | |
| 194 | /** |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 195 | * Report a config's metrics data has been dropped. |
| 196 | */ |
Chenjie Yu | c3c30c0 | 2018-10-26 09:48:07 -0700 | [diff] [blame] | 197 | void noteDataDropped(const ConfigKey& key, const size_t totalBytes); |
Yao Chen | 69f1baf | 2017-11-27 17:25:36 -0800 | [diff] [blame] | 198 | |
| 199 | /** |
| 200 | * Report metrics data report has been sent. |
| 201 | * |
| 202 | * The report may be requested via StatsManager API, or through adb cmd. |
| 203 | */ |
Yangster-mac | e68f3a5 | 2018-04-04 00:01:43 -0700 | [diff] [blame] | 204 | void noteMetricsReportSent(const ConfigKey& key, const size_t num_bytes); |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 205 | |
| 206 | /** |
| 207 | * Report the size of output tuple of a condition. |
| 208 | * |
| 209 | * Note: only report when the condition has an output dimension, and the tuple |
| 210 | * count > kDimensionKeySizeSoftLimit. |
| 211 | * |
| 212 | * [key]: The config key that this condition belongs to. |
Yangster-mac | 94e197c | 2018-01-02 16:03:03 -0800 | [diff] [blame] | 213 | * [id]: The id of the condition. |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 214 | * [size]: The output tuple size. |
| 215 | */ |
Yangster-mac | 94e197c | 2018-01-02 16:03:03 -0800 | [diff] [blame] | 216 | void noteConditionDimensionSize(const ConfigKey& key, const int64_t& id, int size); |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 217 | |
| 218 | /** |
| 219 | * Report the size of output tuple of a metric. |
| 220 | * |
| 221 | * Note: only report when the metric has an output dimension, and the tuple |
| 222 | * count > kDimensionKeySizeSoftLimit. |
| 223 | * |
| 224 | * [key]: The config key that this metric belongs to. |
Yangster-mac | 94e197c | 2018-01-02 16:03:03 -0800 | [diff] [blame] | 225 | * [id]: The id of the metric. |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 226 | * [size]: The output tuple size. |
| 227 | */ |
Yangster-mac | 94e197c | 2018-01-02 16:03:03 -0800 | [diff] [blame] | 228 | void noteMetricDimensionSize(const ConfigKey& key, const int64_t& id, int size); |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 229 | |
Yangster-mac | 306ccc2 | 2018-03-24 15:03:40 -0700 | [diff] [blame] | 230 | /** |
| 231 | * Report the max size of output tuple of dimension in condition across dimensions in what. |
| 232 | * |
| 233 | * Note: only report when the metric has an output dimension in condition, and the max tuple |
| 234 | * count > kDimensionKeySizeSoftLimit. |
| 235 | * |
| 236 | * [key]: The config key that this metric belongs to. |
| 237 | * [id]: The id of the metric. |
| 238 | * [size]: The output tuple size. |
| 239 | */ |
| 240 | void noteMetricDimensionInConditionSize(const ConfigKey& key, const int64_t& id, int size); |
| 241 | |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 242 | /** |
| 243 | * Report a matcher has been matched. |
| 244 | * |
| 245 | * [key]: The config key that this matcher belongs to. |
Yangster-mac | 94e197c | 2018-01-02 16:03:03 -0800 | [diff] [blame] | 246 | * [id]: The id of the matcher. |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 247 | */ |
Yangster-mac | 94e197c | 2018-01-02 16:03:03 -0800 | [diff] [blame] | 248 | void noteMatcherMatched(const ConfigKey& key, const int64_t& id); |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 249 | |
| 250 | /** |
Bookatz | 8f2f3d8 | 2017-12-07 13:53:21 -0800 | [diff] [blame] | 251 | * Report that an anomaly detection alert has been declared. |
| 252 | * |
| 253 | * [key]: The config key that this alert belongs to. |
Yangster-mac | 94e197c | 2018-01-02 16:03:03 -0800 | [diff] [blame] | 254 | * [id]: The id of the alert. |
Bookatz | 8f2f3d8 | 2017-12-07 13:53:21 -0800 | [diff] [blame] | 255 | */ |
Yangster-mac | 94e197c | 2018-01-02 16:03:03 -0800 | [diff] [blame] | 256 | void noteAnomalyDeclared(const ConfigKey& key, const int64_t& id); |
Bookatz | 8f2f3d8 | 2017-12-07 13:53:21 -0800 | [diff] [blame] | 257 | |
| 258 | /** |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 259 | * Report an atom event has been logged. |
| 260 | */ |
| 261 | void noteAtomLogged(int atomId, int32_t timeSec); |
| 262 | |
| 263 | /** |
Bookatz | 1d0136d | 2017-12-01 11:13:32 -0800 | [diff] [blame] | 264 | * Report that statsd modified the anomaly alarm registered with StatsCompanionService. |
| 265 | */ |
| 266 | void noteRegisteredAnomalyAlarmChanged(); |
| 267 | |
| 268 | /** |
Yangster-mac | 932ecec | 2018-02-01 10:23:52 -0800 | [diff] [blame] | 269 | * Report that statsd modified the periodic alarm registered with StatsCompanionService. |
| 270 | */ |
| 271 | void noteRegisteredPeriodicAlarmChanged(); |
| 272 | |
| 273 | /** |
David Chen | bd12527 | 2018-04-04 19:02:50 -0700 | [diff] [blame] | 274 | * Records the number of delta entries that are being dropped from the uid map. |
David Chen | c136f45 | 2017-11-27 11:52:26 -0800 | [diff] [blame] | 275 | */ |
David Chen | bd12527 | 2018-04-04 19:02:50 -0700 | [diff] [blame] | 276 | void noteUidMapDropped(int deltas); |
David Chen | c136f45 | 2017-11-27 11:52:26 -0800 | [diff] [blame] | 277 | |
| 278 | /** |
David Chen | bd12527 | 2018-04-04 19:02:50 -0700 | [diff] [blame] | 279 | * Records that an app was deleted (from statsd's map). |
David Chen | c136f45 | 2017-11-27 11:52:26 -0800 | [diff] [blame] | 280 | */ |
David Chen | bd12527 | 2018-04-04 19:02:50 -0700 | [diff] [blame] | 281 | void noteUidMapAppDeletionDropped(); |
| 282 | |
| 283 | /** |
| 284 | * Updates the number of changes currently stored in the uid map. |
| 285 | */ |
David Chen | c136f45 | 2017-11-27 11:52:26 -0800 | [diff] [blame] | 286 | void setUidMapChanges(int changes); |
| 287 | void setCurrentUidMapMemory(int bytes); |
| 288 | |
Chenjie Yu | 48ed1cc | 2018-10-31 17:36:38 -0700 | [diff] [blame] | 289 | /* |
| 290 | * Updates minimum interval between pulls for an pulled atom. |
| 291 | */ |
Chenjie Yu | b038b70 | 2017-12-18 15:15:34 -0800 | [diff] [blame] | 292 | void updateMinPullIntervalSec(int pullAtomId, long intervalSec); |
| 293 | |
Chenjie Yu | 48ed1cc | 2018-10-31 17:36:38 -0700 | [diff] [blame] | 294 | /* |
| 295 | * Notes an atom is pulled. |
| 296 | */ |
Chenjie Yu | b038b70 | 2017-12-18 15:15:34 -0800 | [diff] [blame] | 297 | void notePull(int pullAtomId); |
| 298 | |
Chenjie Yu | 48ed1cc | 2018-10-31 17:36:38 -0700 | [diff] [blame] | 299 | /* |
| 300 | * Notes an atom is served from puller cache. |
| 301 | */ |
Chenjie Yu | b038b70 | 2017-12-18 15:15:34 -0800 | [diff] [blame] | 302 | void notePullFromCache(int pullAtomId); |
| 303 | |
Yangster-mac | 892f3d3 | 2018-05-02 14:16:48 -0700 | [diff] [blame] | 304 | /* |
Chenjie Yu | c715b9e | 2018-10-19 07:52:12 -0700 | [diff] [blame] | 305 | * Notify data error for pulled atom. |
| 306 | */ |
| 307 | void notePullDataError(int pullAtomId); |
| 308 | |
| 309 | /* |
Chenjie Yu | 48ed1cc | 2018-10-31 17:36:38 -0700 | [diff] [blame] | 310 | * Records time for actual pulling, not including those served from cache and not including |
| 311 | * statsd processing delays. |
| 312 | */ |
| 313 | void notePullTime(int pullAtomId, int64_t pullTimeNs); |
| 314 | |
| 315 | /* |
| 316 | * Records pull delay for a pulled atom, including those served from cache and including statsd |
| 317 | * processing delays. |
| 318 | */ |
| 319 | void notePullDelay(int pullAtomId, int64_t pullDelayNs); |
| 320 | |
| 321 | /* |
Chenjie Yu | 0bd73db | 2018-12-16 07:37:04 -0800 | [diff] [blame] | 322 | * Records pull exceeds timeout for the puller. |
| 323 | */ |
| 324 | void notePullTimeout(int pullAtomId); |
| 325 | |
| 326 | /* |
| 327 | * Records pull exceeds max delay for a metric. |
| 328 | */ |
| 329 | void notePullExceedMaxDelay(int pullAtomId); |
| 330 | |
| 331 | /* |
Yao Chen | a80e5c0 | 2018-09-04 13:55:29 -0700 | [diff] [blame] | 332 | * Records when system server restarts. |
| 333 | */ |
Yangster-mac | 892f3d3 | 2018-05-02 14:16:48 -0700 | [diff] [blame] | 334 | void noteSystemServerRestart(int32_t timeSec); |
| 335 | |
Yao Chen | 884c8c1 | 2018-01-26 10:36:25 -0800 | [diff] [blame] | 336 | /** |
Yangster-mac | b8382a1 | 2018-04-04 10:39:12 -0700 | [diff] [blame] | 337 | * Records statsd skipped an event. |
| 338 | */ |
Yao Chen | 49d7dd7 | 2019-03-26 14:02:11 -0700 | [diff] [blame] | 339 | void noteLogLost(int32_t wallClockTimeSec, int32_t count, int32_t lastError, |
| 340 | int32_t lastAtomTag, int32_t uid, int32_t pid); |
Yangster-mac | b8382a1 | 2018-04-04 10:39:12 -0700 | [diff] [blame] | 341 | |
| 342 | /** |
Misha Wagner | 1eee221 | 2019-01-22 11:47:11 +0000 | [diff] [blame] | 343 | * Records that the pull of an atom has failed |
| 344 | */ |
| 345 | void notePullFailed(int atomId); |
| 346 | |
| 347 | /** |
| 348 | * Records that the pull of StatsCompanionService atom has failed |
| 349 | */ |
| 350 | void noteStatsCompanionPullFailed(int atomId); |
| 351 | |
| 352 | /** |
| 353 | * Records that the pull of a StatsCompanionService atom has failed due to a failed binder |
| 354 | * transaction. This can happen when StatsCompanionService returns too |
| 355 | * much data (the max Binder parcel size is 1MB) |
| 356 | */ |
| 357 | void noteStatsCompanionPullBinderTransactionFailed(int atomId); |
| 358 | |
| 359 | /** |
| 360 | * A pull with no data occurred |
| 361 | */ |
| 362 | void noteEmptyData(int atomId); |
| 363 | |
| 364 | /** |
Tej Singh | a0c89dd | 2019-01-25 16:39:18 -0800 | [diff] [blame] | 365 | * Records that a puller callback for the given atomId was registered or unregistered. |
| 366 | * |
| 367 | * @param registered True if the callback was registered, false if was unregistered. |
| 368 | */ |
| 369 | void notePullerCallbackRegistrationChanged(int atomId, bool registered); |
| 370 | |
| 371 | /** |
Misha Wagner | 1eee221 | 2019-01-22 11:47:11 +0000 | [diff] [blame] | 372 | * Hard limit was reached in the cardinality of an atom |
| 373 | */ |
Olivier Gaillard | 2310b19 | 2019-02-08 10:23:05 +0000 | [diff] [blame] | 374 | void noteHardDimensionLimitReached(int64_t metricId); |
Misha Wagner | 1eee221 | 2019-01-22 11:47:11 +0000 | [diff] [blame] | 375 | |
| 376 | /** |
| 377 | * A log event was too late, arrived in the wrong bucket and was skipped |
| 378 | */ |
Olivier Gaillard | 2310b19 | 2019-02-08 10:23:05 +0000 | [diff] [blame] | 379 | void noteLateLogEventSkipped(int64_t metricId); |
Misha Wagner | 1eee221 | 2019-01-22 11:47:11 +0000 | [diff] [blame] | 380 | |
| 381 | /** |
| 382 | * Buckets were skipped as time elapsed without any data for them |
| 383 | */ |
Olivier Gaillard | 2310b19 | 2019-02-08 10:23:05 +0000 | [diff] [blame] | 384 | void noteSkippedForwardBuckets(int64_t metricId); |
Misha Wagner | 1eee221 | 2019-01-22 11:47:11 +0000 | [diff] [blame] | 385 | |
| 386 | /** |
| 387 | * An unsupported value type was received |
| 388 | */ |
Olivier Gaillard | 2310b19 | 2019-02-08 10:23:05 +0000 | [diff] [blame] | 389 | void noteBadValueType(int64_t metricId); |
Misha Wagner | 1eee221 | 2019-01-22 11:47:11 +0000 | [diff] [blame] | 390 | |
| 391 | /** |
Olivier Gaillard | 320952b | 2019-02-06 13:57:24 +0000 | [diff] [blame] | 392 | * Buckets were dropped due to reclaim memory. |
| 393 | */ |
Olivier Gaillard | 2310b19 | 2019-02-08 10:23:05 +0000 | [diff] [blame] | 394 | void noteBucketDropped(int64_t metricId); |
Olivier Gaillard | 320952b | 2019-02-06 13:57:24 +0000 | [diff] [blame] | 395 | |
| 396 | /** |
Misha Wagner | 1eee221 | 2019-01-22 11:47:11 +0000 | [diff] [blame] | 397 | * A condition change was too late, arrived in the wrong bucket and was skipped |
| 398 | */ |
Olivier Gaillard | 2310b19 | 2019-02-08 10:23:05 +0000 | [diff] [blame] | 399 | void noteConditionChangeInNextBucket(int64_t metricId); |
Misha Wagner | 1eee221 | 2019-01-22 11:47:11 +0000 | [diff] [blame] | 400 | |
| 401 | /** |
Olivier Gaillard | 9a5d359 | 2019-02-05 15:12:39 +0000 | [diff] [blame] | 402 | * A bucket has been tagged as invalid. |
| 403 | */ |
Olivier Gaillard | 2310b19 | 2019-02-08 10:23:05 +0000 | [diff] [blame] | 404 | void noteInvalidatedBucket(int64_t metricId); |
Olivier Gaillard | 9a5d359 | 2019-02-05 15:12:39 +0000 | [diff] [blame] | 405 | |
| 406 | /** |
Olivier Gaillard | f248c0d | 2019-02-21 15:56:58 +0000 | [diff] [blame] | 407 | * Tracks the total number of buckets (include skipped/invalid buckets). |
| 408 | */ |
| 409 | void noteBucketCount(int64_t metricId); |
| 410 | |
| 411 | /** |
Olivier Gaillard | 3ba2e93 | 2019-02-08 14:25:35 +0000 | [diff] [blame] | 412 | * For pulls at bucket boundaries, it represents the misalignment between the real timestamp and |
| 413 | * the end of the bucket. |
| 414 | */ |
| 415 | void noteBucketBoundaryDelayNs(int64_t metricId, int64_t timeDelayNs); |
| 416 | |
| 417 | /** |
Olivier Gaillard | e63d9e0 | 2019-02-12 14:43:59 +0000 | [diff] [blame] | 418 | * Number of buckets with unknown condition. |
| 419 | */ |
| 420 | void noteBucketUnknownCondition(int64_t metricId); |
| 421 | |
| 422 | /** |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 423 | * Reset the historical stats. Including all stats in icebox, and the tracked stats about |
| 424 | * metrics, matchers, and atoms. The active configs will be kept and StatsdStats will continue |
| 425 | * to collect stats after reset() has been called. |
| 426 | */ |
| 427 | void reset(); |
| 428 | |
| 429 | /** |
| 430 | * Output the stats in protobuf binary format to [buffer]. |
| 431 | * |
| 432 | * [reset]: whether to clear the historical stats after the call. |
| 433 | */ |
Yao Chen | 69f1baf | 2017-11-27 17:25:36 -0800 | [diff] [blame] | 434 | void dumpStats(std::vector<uint8_t>* buffer, bool reset); |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 435 | |
Yao Chen | f5acabe | 2018-01-17 14:10:34 -0800 | [diff] [blame] | 436 | /** |
Yao Chen | a80e5c0 | 2018-09-04 13:55:29 -0700 | [diff] [blame] | 437 | * Output statsd stats in human readable format to [out] file descriptor. |
Yao Chen | f5acabe | 2018-01-17 14:10:34 -0800 | [diff] [blame] | 438 | */ |
Yao Chen | a80e5c0 | 2018-09-04 13:55:29 -0700 | [diff] [blame] | 439 | void dumpStats(int outFd) const; |
Yao Chen | f5acabe | 2018-01-17 14:10:34 -0800 | [diff] [blame] | 440 | |
Chenjie Yu | b038b70 | 2017-12-18 15:15:34 -0800 | [diff] [blame] | 441 | typedef struct { |
Chenjie Yu | 48ed1cc | 2018-10-31 17:36:38 -0700 | [diff] [blame] | 442 | long totalPull = 0; |
| 443 | long totalPullFromCache = 0; |
| 444 | long minPullIntervalSec = LONG_MAX; |
| 445 | int64_t avgPullTimeNs = 0; |
| 446 | int64_t maxPullTimeNs = 0; |
| 447 | long numPullTime = 0; |
| 448 | int64_t avgPullDelayNs = 0; |
| 449 | int64_t maxPullDelayNs = 0; |
| 450 | long numPullDelay = 0; |
Chenjie Yu | c715b9e | 2018-10-19 07:52:12 -0700 | [diff] [blame] | 451 | long dataError = 0; |
Chenjie Yu | 0bd73db | 2018-12-16 07:37:04 -0800 | [diff] [blame] | 452 | long pullTimeout = 0; |
| 453 | long pullExceedMaxDelay = 0; |
Misha Wagner | 1eee221 | 2019-01-22 11:47:11 +0000 | [diff] [blame] | 454 | long pullFailed = 0; |
| 455 | long statsCompanionPullFailed = 0; |
| 456 | long statsCompanionPullBinderTransactionFailed = 0; |
| 457 | long emptyData = 0; |
Tej Singh | a0c89dd | 2019-01-25 16:39:18 -0800 | [diff] [blame] | 458 | long registeredCount = 0; |
| 459 | long unregisteredCount = 0; |
Chenjie Yu | b038b70 | 2017-12-18 15:15:34 -0800 | [diff] [blame] | 460 | } PulledAtomStats; |
| 461 | |
Misha Wagner | 1eee221 | 2019-01-22 11:47:11 +0000 | [diff] [blame] | 462 | typedef struct { |
| 463 | long hardDimensionLimitReached = 0; |
| 464 | long lateLogEventSkipped = 0; |
| 465 | long skippedForwardBuckets = 0; |
| 466 | long badValueType = 0; |
| 467 | long conditionChangeInNextBucket = 0; |
Olivier Gaillard | 9a5d359 | 2019-02-05 15:12:39 +0000 | [diff] [blame] | 468 | long invalidatedBucket = 0; |
Olivier Gaillard | 320952b | 2019-02-06 13:57:24 +0000 | [diff] [blame] | 469 | long bucketDropped = 0; |
Olivier Gaillard | 3ba2e93 | 2019-02-08 14:25:35 +0000 | [diff] [blame] | 470 | int64_t minBucketBoundaryDelayNs = 0; |
| 471 | int64_t maxBucketBoundaryDelayNs = 0; |
Olivier Gaillard | e63d9e0 | 2019-02-12 14:43:59 +0000 | [diff] [blame] | 472 | long bucketUnknownCondition = 0; |
Olivier Gaillard | f248c0d | 2019-02-21 15:56:58 +0000 | [diff] [blame] | 473 | long bucketCount = 0; |
Misha Wagner | 1eee221 | 2019-01-22 11:47:11 +0000 | [diff] [blame] | 474 | } AtomMetricStats; |
| 475 | |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 476 | private: |
| 477 | StatsdStats(); |
| 478 | |
| 479 | mutable std::mutex mLock; |
| 480 | |
Yao Chen | 69f1baf | 2017-11-27 17:25:36 -0800 | [diff] [blame] | 481 | int32_t mStartTimeSec; |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 482 | |
David Chen | c136f45 | 2017-11-27 11:52:26 -0800 | [diff] [blame] | 483 | // Track the number of dropped entries used by the uid map. |
Yao Chen | 20e9e62 | 2018-02-28 11:18:51 -0800 | [diff] [blame] | 484 | UidMapStats mUidMapStats; |
David Chen | c136f45 | 2017-11-27 11:52:26 -0800 | [diff] [blame] | 485 | |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 486 | // The stats about the configs that are still in use. |
Yao Chen | f6723df | 2018-01-08 15:11:58 -0800 | [diff] [blame] | 487 | // The map size is capped by kMaxConfigCount. |
Yao Chen | 20e9e62 | 2018-02-28 11:18:51 -0800 | [diff] [blame] | 488 | std::map<const ConfigKey, std::shared_ptr<ConfigStats>> mConfigStats; |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 489 | |
| 490 | // Stores the stats for the configs that are no longer in use. |
Yao Chen | f6723df | 2018-01-08 15:11:58 -0800 | [diff] [blame] | 491 | // The size of the vector is capped by kMaxIceBoxSize. |
Yao Chen | 20e9e62 | 2018-02-28 11:18:51 -0800 | [diff] [blame] | 492 | std::list<const std::shared_ptr<ConfigStats>> mIceBox; |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 493 | |
| 494 | // Stores the number of times a pushed atom is logged. |
| 495 | // The size of the vector is the largest pushed atom id in atoms.proto + 1. Atoms |
| 496 | // out of that range will be dropped (it's either pulled atoms or test atoms). |
| 497 | // This is a vector, not a map because it will be accessed A LOT -- for each stats log. |
| 498 | std::vector<int> mPushedAtomStats; |
| 499 | |
Yao Chen | f6723df | 2018-01-08 15:11:58 -0800 | [diff] [blame] | 500 | // Maps PullAtomId to its stats. The size is capped by the puller atom counts. |
Chenjie Yu | b038b70 | 2017-12-18 15:15:34 -0800 | [diff] [blame] | 501 | std::map<int, PulledAtomStats> mPulledAtomStats; |
| 502 | |
Misha Wagner | 1eee221 | 2019-01-22 11:47:11 +0000 | [diff] [blame] | 503 | // Maps metric ID to its stats. The size is capped by the number of metrics. |
Olivier Gaillard | f48fb04 | 2019-02-13 11:03:20 +0000 | [diff] [blame] | 504 | std::map<int64_t, AtomMetricStats> mAtomMetricStats; |
Misha Wagner | 1eee221 | 2019-01-22 11:47:11 +0000 | [diff] [blame] | 505 | |
Yao Chen | 39b6799 | 2018-11-08 15:32:17 -0800 | [diff] [blame] | 506 | struct LogLossStats { |
Yao Chen | 49d7dd7 | 2019-03-26 14:02:11 -0700 | [diff] [blame] | 507 | LogLossStats(int32_t sec, int32_t count, int32_t error, int32_t tag, int32_t uid, |
| 508 | int32_t pid) |
| 509 | : mWallClockSec(sec), |
| 510 | mCount(count), |
| 511 | mLastError(error), |
| 512 | mLastTag(tag), |
| 513 | mUid(uid), |
| 514 | mPid(pid) { |
Yao Chen | 39b6799 | 2018-11-08 15:32:17 -0800 | [diff] [blame] | 515 | } |
| 516 | int32_t mWallClockSec; |
| 517 | int32_t mCount; |
| 518 | // error code defined in linux/errno.h |
| 519 | int32_t mLastError; |
Yao Chen | 49d7dd7 | 2019-03-26 14:02:11 -0700 | [diff] [blame] | 520 | int32_t mLastTag; |
| 521 | int32_t mUid; |
| 522 | int32_t mPid; |
Yao Chen | 39b6799 | 2018-11-08 15:32:17 -0800 | [diff] [blame] | 523 | }; |
| 524 | |
Yao Chen | 3ff3a49 | 2018-08-06 16:17:37 -0700 | [diff] [blame] | 525 | // Timestamps when we detect log loss, and the number of logs lost. |
Yao Chen | 39b6799 | 2018-11-08 15:32:17 -0800 | [diff] [blame] | 526 | std::list<LogLossStats> mLogLossStats; |
Yangster-mac | b8382a1 | 2018-04-04 10:39:12 -0700 | [diff] [blame] | 527 | |
Yangster-mac | 892f3d3 | 2018-05-02 14:16:48 -0700 | [diff] [blame] | 528 | std::list<int32_t> mSystemServerRestartSec; |
| 529 | |
Bookatz | 1d0136d | 2017-12-01 11:13:32 -0800 | [diff] [blame] | 530 | // Stores the number of times statsd modified the anomaly alarm registered with |
| 531 | // StatsCompanionService. |
| 532 | int mAnomalyAlarmRegisteredStats = 0; |
| 533 | |
Yangster-mac | 932ecec | 2018-02-01 10:23:52 -0800 | [diff] [blame] | 534 | // Stores the number of times statsd registers the periodic alarm changes |
| 535 | int mPeriodicAlarmRegisteredStats = 0; |
| 536 | |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 537 | void noteConfigResetInternalLocked(const ConfigKey& key); |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 538 | |
| 539 | void noteConfigRemovedInternalLocked(const ConfigKey& key); |
| 540 | |
| 541 | void resetInternalLocked(); |
| 542 | |
Chenjie Yu | c3c30c0 | 2018-10-26 09:48:07 -0700 | [diff] [blame] | 543 | void noteDataDropped(const ConfigKey& key, const size_t totalBytes, int32_t timeSec); |
Yao Chen | 0fac5b1 | 2017-11-28 16:07:02 -0800 | [diff] [blame] | 544 | |
Yangster-mac | e68f3a5 | 2018-04-04 00:01:43 -0700 | [diff] [blame] | 545 | void noteMetricsReportSent(const ConfigKey& key, const size_t num_bytes, int32_t timeSec); |
Yao Chen | 0fac5b1 | 2017-11-28 16:07:02 -0800 | [diff] [blame] | 546 | |
| 547 | void noteBroadcastSent(const ConfigKey& key, int32_t timeSec); |
| 548 | |
Tej Singh | 6ede28b | 2019-01-29 17:06:54 -0800 | [diff] [blame] | 549 | void noteActiveStatusChanged(const ConfigKey& key, bool activate, int32_t timeSec); |
| 550 | |
Yao Chen | 20e9e62 | 2018-02-28 11:18:51 -0800 | [diff] [blame] | 551 | void addToIceBoxLocked(std::shared_ptr<ConfigStats>& stats); |
Yao Chen | f6723df | 2018-01-08 15:11:58 -0800 | [diff] [blame] | 552 | |
Misha Wagner | 1eee221 | 2019-01-22 11:47:11 +0000 | [diff] [blame] | 553 | /** |
| 554 | * Get a reference to AtomMetricStats for a metric. If none exists, create it. The reference |
| 555 | * will live as long as `this`. |
| 556 | */ |
Olivier Gaillard | f48fb04 | 2019-02-13 11:03:20 +0000 | [diff] [blame] | 557 | StatsdStats::AtomMetricStats& getAtomMetricStats(int64_t metricId); |
Misha Wagner | 1eee221 | 2019-01-22 11:47:11 +0000 | [diff] [blame] | 558 | |
Yao Chen | 69f1baf | 2017-11-27 17:25:36 -0800 | [diff] [blame] | 559 | FRIEND_TEST(StatsdStatsTest, TestValidConfigAdd); |
| 560 | FRIEND_TEST(StatsdStatsTest, TestInvalidConfigAdd); |
| 561 | FRIEND_TEST(StatsdStatsTest, TestConfigRemove); |
| 562 | FRIEND_TEST(StatsdStatsTest, TestSubStats); |
| 563 | FRIEND_TEST(StatsdStatsTest, TestAtomLog); |
Yao Chen | 0fac5b1 | 2017-11-28 16:07:02 -0800 | [diff] [blame] | 564 | FRIEND_TEST(StatsdStatsTest, TestTimestampThreshold); |
Bookatz | 1d0136d | 2017-12-01 11:13:32 -0800 | [diff] [blame] | 565 | FRIEND_TEST(StatsdStatsTest, TestAnomalyMonitor); |
Yangster-mac | 892f3d3 | 2018-05-02 14:16:48 -0700 | [diff] [blame] | 566 | FRIEND_TEST(StatsdStatsTest, TestSystemServerCrash); |
Chenjie Yu | 48ed1cc | 2018-10-31 17:36:38 -0700 | [diff] [blame] | 567 | FRIEND_TEST(StatsdStatsTest, TestPullAtomStats); |
Olivier Gaillard | e3e969c | 2019-02-11 11:22:51 +0000 | [diff] [blame] | 568 | FRIEND_TEST(StatsdStatsTest, TestAtomMetricsStats); |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 569 | }; |
| 570 | |
| 571 | } // namespace statsd |
| 572 | } // namespace os |
Stefan Lafon | c6f2fa2 | 2018-01-04 22:03:29 -0800 | [diff] [blame] | 573 | } // namespace android |