Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2017 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #ifndef METRIC_PRODUCER_H |
| 18 | #define METRIC_PRODUCER_H |
| 19 | |
Muhammad Qureshi | 844694b | 2019-04-05 10:10:40 -0700 | [diff] [blame] | 20 | #include <frameworks/base/cmds/statsd/src/active_config_list.pb.h> |
tsaichristine | d21aacf | 2019-10-07 14:47:38 -0700 | [diff] [blame] | 21 | #include <utils/RefBase.h> |
| 22 | |
| 23 | #include <unordered_map> |
| 24 | |
Yao Chen | 8a8d16c | 2018-02-08 14:50:40 -0800 | [diff] [blame] | 25 | #include "HashableDimensionKey.h" |
Yangster-mac | e2cd6d5 | 2017-11-09 20:38:30 -0800 | [diff] [blame] | 26 | #include "anomaly/AnomalyTracker.h" |
Yao Chen | 729093d | 2017-10-16 10:33:26 -0700 | [diff] [blame] | 27 | #include "condition/ConditionWizard.h" |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 28 | #include "config/ConfigKey.h" |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 29 | #include "matchers/matcher_util.h" |
| 30 | #include "packages/PackageInfoListener.h" |
tsaichristine | d21aacf | 2019-10-07 14:47:38 -0700 | [diff] [blame] | 31 | #include "state/StateListener.h" |
tsaichristine | 69000e6 | 2019-10-18 17:34:52 -0700 | [diff] [blame] | 32 | #include "state/StateManager.h" |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 33 | |
| 34 | namespace android { |
| 35 | namespace os { |
| 36 | namespace statsd { |
| 37 | |
Tej Singh | f53d445 | 2019-05-09 18:17:59 -0700 | [diff] [blame] | 38 | // Keep this in sync with DumpReportReason enum in stats_log.proto |
| 39 | enum DumpReportReason { |
| 40 | DEVICE_SHUTDOWN = 1, |
| 41 | CONFIG_UPDATED = 2, |
| 42 | CONFIG_REMOVED = 3, |
| 43 | GET_DATA_CALLED = 4, |
| 44 | ADB_DUMP = 5, |
| 45 | CONFIG_RESET = 6, |
| 46 | STATSCOMPANION_DIED = 7, |
| 47 | TERMINATION_SIGNAL_RECEIVED = 8 |
| 48 | }; |
| 49 | |
Yangster-mac | 849dfdc2 | 2018-10-12 15:41:45 -0700 | [diff] [blame] | 50 | // If the metric has no activation requirement, it will be active once the metric producer is |
| 51 | // created. |
| 52 | // If the metric needs to be activated by atoms, the metric producer will start |
Chenjie Yu | a9a310e | 2019-02-06 13:40:10 -0800 | [diff] [blame] | 53 | // with kNotActive state, turn to kActive or kActiveOnBoot when the activation event arrives, become |
| 54 | // kNotActive when it reaches the duration limit (timebomb). If the activation event arrives again |
| 55 | // before or after it expires, the event producer will be re-activated and ttl will be reset. |
Yangster-mac | 849dfdc2 | 2018-10-12 15:41:45 -0700 | [diff] [blame] | 56 | enum ActivationState { |
| 57 | kNotActive = 0, |
| 58 | kActive = 1, |
Chenjie Yu | a9a310e | 2019-02-06 13:40:10 -0800 | [diff] [blame] | 59 | kActiveOnBoot = 2, |
Yangster-mac | 849dfdc2 | 2018-10-12 15:41:45 -0700 | [diff] [blame] | 60 | }; |
| 61 | |
Olivier Gaillard | 6c75ecd | 2019-02-20 09:57:33 +0000 | [diff] [blame] | 62 | enum DumpLatency { |
| 63 | // In some cases, we only have a short time range to do the dump, e.g. statsd is being killed. |
| 64 | // We might be able to return all the data in this mode. For instance, pull metrics might need |
| 65 | // to be pulled when the current bucket is requested. |
| 66 | FAST = 1, |
| 67 | // In other cases, it is fine for a dump to take more than a few milliseconds, e.g. config |
| 68 | // updates. |
| 69 | NO_TIME_CONSTRAINTS = 2 |
| 70 | }; |
| 71 | |
tsaichristine | b87ca15 | 2019-12-09 15:19:41 -0800 | [diff] [blame] | 72 | // Keep this in sync with BucketDropReason enum in stats_log.proto |
| 73 | enum BucketDropReason { |
| 74 | // For ValueMetric, a bucket is dropped during a dump report request iff |
| 75 | // current bucket should be included, a pull is needed (pulled metric and |
| 76 | // condition is true), and we are under fast time constraints. |
| 77 | DUMP_REPORT_REQUESTED = 1, |
| 78 | EVENT_IN_WRONG_BUCKET = 2, |
| 79 | CONDITION_UNKNOWN = 3, |
| 80 | PULL_FAILED = 4, |
| 81 | PULL_DELAYED = 5, |
| 82 | DIMENSION_GUARDRAIL_REACHED = 6, |
| 83 | MULTIPLE_BUCKETS_SKIPPED = 7, |
| 84 | // Not an invalid bucket case, but the bucket is dropped. |
| 85 | BUCKET_TOO_SMALL = 8 |
| 86 | }; |
| 87 | |
Ruchir Rastogi | 21a287b | 2019-10-02 12:04:33 -0700 | [diff] [blame] | 88 | struct Activation { |
| 89 | Activation(const ActivationType& activationType, const int64_t ttlNs) |
| 90 | : ttl_ns(ttlNs), |
| 91 | start_ns(0), |
| 92 | state(ActivationState::kNotActive), |
| 93 | activationType(activationType) {} |
| 94 | |
| 95 | const int64_t ttl_ns; |
| 96 | int64_t start_ns; |
| 97 | ActivationState state; |
| 98 | const ActivationType activationType; |
| 99 | }; |
| 100 | |
tsaichristine | b87ca15 | 2019-12-09 15:19:41 -0800 | [diff] [blame] | 101 | struct DropEvent { |
| 102 | // Reason for dropping the bucket and/or marking the bucket invalid. |
| 103 | BucketDropReason reason; |
| 104 | // The timestamp of the drop event. |
| 105 | int64_t dropTimeNs; |
| 106 | }; |
| 107 | |
| 108 | struct SkippedBucket { |
| 109 | // Start time of the dropped bucket. |
| 110 | int64_t bucketStartTimeNs; |
| 111 | // End time of the dropped bucket. |
| 112 | int64_t bucketEndTimeNs; |
| 113 | // List of events that invalidated this bucket. |
| 114 | std::vector<DropEvent> dropEvents; |
| 115 | |
| 116 | void reset() { |
| 117 | bucketStartTimeNs = 0; |
| 118 | bucketEndTimeNs = 0; |
| 119 | dropEvents.clear(); |
| 120 | } |
| 121 | }; |
| 122 | |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 123 | // A MetricProducer is responsible for compute one single metrics, creating stats log report, and |
David Chen | de70169 | 2017-10-05 13:16:02 -0700 | [diff] [blame] | 124 | // writing the report to dropbox. MetricProducers should respond to package changes as required in |
| 125 | // PackageInfoListener, but if none of the metrics are slicing by package name, then the update can |
| 126 | // be a no-op. |
Tej Singh | 9ec159a | 2019-11-14 11:59:48 -0800 | [diff] [blame] | 127 | class MetricProducer : public virtual android::RefBase, public virtual StateListener { |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 128 | public: |
Yangster-mac | 15f6bbc | 2018-04-08 11:52:26 -0700 | [diff] [blame] | 129 | MetricProducer(const int64_t& metricId, const ConfigKey& key, const int64_t timeBaseNs, |
Ruchir Rastogi | 21a287b | 2019-10-02 12:04:33 -0700 | [diff] [blame] | 130 | const int conditionIndex, const sp<ConditionWizard>& wizard, |
| 131 | const std::unordered_map<int, std::shared_ptr<Activation>>& eventActivationMap, |
| 132 | const std::unordered_map<int, std::vector<std::shared_ptr<Activation>>>& |
tsaichristine | d21aacf | 2019-10-07 14:47:38 -0700 | [diff] [blame] | 133 | eventDeactivationMap, |
| 134 | const vector<int>& slicedStateAtoms, |
| 135 | const unordered_map<int, unordered_map<int, int64_t>>& stateGroupMap); |
Yangster-mac | 2087716 | 2017-12-22 17:19:39 -0800 | [diff] [blame] | 136 | |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 137 | virtual ~MetricProducer(){}; |
| 138 | |
Olivier Gaillard | e35b282 | 2019-02-27 17:09:40 +0000 | [diff] [blame] | 139 | ConditionState initialCondition(const int conditionIndex) const { |
| 140 | return conditionIndex >= 0 ? ConditionState::kUnknown : ConditionState::kTrue; |
| 141 | } |
| 142 | |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 143 | /** |
| 144 | * Forces this metric to split into a partial bucket right now. If we're past a full bucket, we |
| 145 | * first call the standard flushing code to flush up to the latest full bucket. Then we call |
| 146 | * the flush again when the end timestamp is forced to be now, and then after flushing, update |
| 147 | * the start timestamp to be now. |
| 148 | */ |
Tej Singh | 9ec159a | 2019-11-14 11:59:48 -0800 | [diff] [blame] | 149 | virtual void notifyAppUpgrade(const int64_t& eventTimeNs, const string& apk, const int uid, |
| 150 | const int64_t version) { |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 151 | std::lock_guard<std::mutex> lock(mMutex); |
| 152 | |
| 153 | if (eventTimeNs > getCurrentBucketEndTimeNs()) { |
| 154 | // Flush full buckets on the normal path up to the latest bucket boundary. |
| 155 | flushIfNeededLocked(eventTimeNs); |
| 156 | } |
| 157 | // Now flush a partial bucket. |
Olivier Gaillard | 6c75ecd | 2019-02-20 09:57:33 +0000 | [diff] [blame] | 158 | flushCurrentBucketLocked(eventTimeNs, eventTimeNs); |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 159 | // Don't update the current bucket number so that the anomaly tracker knows this bucket |
| 160 | // is a partial bucket and can merge it with the previous bucket. |
| 161 | }; |
| 162 | |
Tej Singh | 9ec159a | 2019-11-14 11:59:48 -0800 | [diff] [blame] | 163 | void notifyAppRemoved(const int64_t& eventTimeNs, const string& apk, const int uid) { |
David Chen | bd12527 | 2018-04-04 19:02:50 -0700 | [diff] [blame] | 164 | // Force buckets to split on removal also. |
| 165 | notifyAppUpgrade(eventTimeNs, apk, uid, 0); |
Yao Chen | d10f7b1 | 2017-12-18 12:53:50 -0800 | [diff] [blame] | 166 | }; |
| 167 | |
Yao Chen | caf339d | 2017-10-06 16:01:10 -0700 | [diff] [blame] | 168 | // Consume the parsed stats log entry that already matched the "what" of the metric. |
Chenjie Yu | a7259ab | 2017-12-10 08:31:05 -0800 | [diff] [blame] | 169 | void onMatchedLogEvent(const size_t matcherIndex, const LogEvent& event) { |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 170 | std::lock_guard<std::mutex> lock(mMutex); |
Muhammad Qureshi | 18e4692 | 2019-05-24 16:38:49 -0700 | [diff] [blame] | 171 | onMatchedLogEventLocked(matcherIndex, event); |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 172 | } |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 173 | |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 174 | void onConditionChanged(const bool condition, const int64_t eventTime) { |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 175 | std::lock_guard<std::mutex> lock(mMutex); |
Muhammad Qureshi | 18e4692 | 2019-05-24 16:38:49 -0700 | [diff] [blame] | 176 | onConditionChangedLocked(condition, eventTime); |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 177 | } |
Yao Chen | caf339d | 2017-10-06 16:01:10 -0700 | [diff] [blame] | 178 | |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 179 | void onSlicedConditionMayChange(bool overallCondition, const int64_t eventTime) { |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 180 | std::lock_guard<std::mutex> lock(mMutex); |
Muhammad Qureshi | 18e4692 | 2019-05-24 16:38:49 -0700 | [diff] [blame] | 181 | onSlicedConditionMayChangeLocked(overallCondition, eventTime); |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 182 | } |
| 183 | |
| 184 | bool isConditionSliced() const { |
| 185 | std::lock_guard<std::mutex> lock(mMutex); |
| 186 | return mConditionSliced; |
| 187 | }; |
Yao Chen | 729093d | 2017-10-16 10:33:26 -0700 | [diff] [blame] | 188 | |
tsaichristine | 8d73dc9 | 2019-12-06 02:11:02 -0800 | [diff] [blame] | 189 | void onStateChanged(const int64_t eventTimeNs, const int32_t atomId, |
tsaichristine | 1449fa4 | 2020-01-02 12:12:05 -0800 | [diff] [blame] | 190 | const HashableDimensionKey& primaryKey, const int32_t oldState, |
| 191 | const int32_t newState){}; |
tsaichristine | d21aacf | 2019-10-07 14:47:38 -0700 | [diff] [blame] | 192 | |
Yao Chen | 288c600 | 2017-12-12 13:43:18 -0800 | [diff] [blame] | 193 | // Output the metrics data to [protoOutput]. All metrics reports end with the same timestamp. |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 194 | // This method clears all the past buckets. |
Yangster-mac | e68f3a5 | 2018-04-04 00:01:43 -0700 | [diff] [blame] | 195 | void onDumpReport(const int64_t dumpTimeNs, |
| 196 | const bool include_current_partial_bucket, |
Bookatz | ff71cad | 2018-09-20 17:17:49 -0700 | [diff] [blame] | 197 | const bool erase_data, |
Olivier Gaillard | 6c75ecd | 2019-02-20 09:57:33 +0000 | [diff] [blame] | 198 | const DumpLatency dumpLatency, |
Yangster-mac | 9def8e3 | 2018-04-17 13:55:51 -0700 | [diff] [blame] | 199 | std::set<string> *str_set, |
Yangster-mac | e68f3a5 | 2018-04-04 00:01:43 -0700 | [diff] [blame] | 200 | android::util::ProtoOutputStream* protoOutput) { |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 201 | std::lock_guard<std::mutex> lock(mMutex); |
Bookatz | ff71cad | 2018-09-20 17:17:49 -0700 | [diff] [blame] | 202 | return onDumpReportLocked(dumpTimeNs, include_current_partial_bucket, erase_data, |
Olivier Gaillard | 6c75ecd | 2019-02-20 09:57:33 +0000 | [diff] [blame] | 203 | dumpLatency, str_set, protoOutput); |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 204 | } |
Yao Chen | 729093d | 2017-10-16 10:33:26 -0700 | [diff] [blame] | 205 | |
Yangster-mac | a802d73 | 2018-04-24 07:50:38 -0700 | [diff] [blame] | 206 | void clearPastBuckets(const int64_t dumpTimeNs) { |
| 207 | std::lock_guard<std::mutex> lock(mMutex); |
| 208 | return clearPastBucketsLocked(dumpTimeNs); |
| 209 | } |
| 210 | |
Tej Singh | 3be093b | 2020-03-04 20:08:38 -0800 | [diff] [blame] | 211 | void prepareFirstBucket() { |
| 212 | std::lock_guard<std::mutex> lock(mMutex); |
| 213 | prepareFirstBucketLocked(); |
| 214 | } |
| 215 | |
David Chen | 1d7b0cd | 2017-11-15 14:20:04 -0800 | [diff] [blame] | 216 | // Returns the memory in bytes currently used to store this metric's data. Does not change |
| 217 | // state. |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 218 | size_t byteSize() const { |
| 219 | std::lock_guard<std::mutex> lock(mMutex); |
| 220 | return byteSizeLocked(); |
| 221 | } |
yro | 69007c8 | 2017-10-26 20:42:57 -0700 | [diff] [blame] | 222 | |
tsaichristine | a05cfe0 | 2019-09-18 16:28:48 -0700 | [diff] [blame] | 223 | void dumpStates(FILE* out, bool verbose) const { |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 224 | std::lock_guard<std::mutex> lock(mMutex); |
tsaichristine | a05cfe0 | 2019-09-18 16:28:48 -0700 | [diff] [blame] | 225 | dumpStatesLocked(out, verbose); |
Chenjie Yu | c7939cb | 2019-02-04 17:25:45 -0800 | [diff] [blame] | 226 | } |
| 227 | |
Yao Chen | 06dba5d | 2018-01-26 13:38:16 -0800 | [diff] [blame] | 228 | // Let MetricProducer drop in-memory data to save memory. |
| 229 | // We still need to keep future data valid and anomaly tracking work, which means we will |
| 230 | // have to flush old data, informing anomaly trackers then safely drop old data. |
| 231 | // We still keep current bucket data for future metrics' validity. |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 232 | void dropData(const int64_t dropTimeNs) { |
Yao Chen | 06dba5d | 2018-01-26 13:38:16 -0800 | [diff] [blame] | 233 | std::lock_guard<std::mutex> lock(mMutex); |
| 234 | dropDataLocked(dropTimeNs); |
| 235 | } |
| 236 | |
tsaichristine | a05cfe0 | 2019-09-18 16:28:48 -0700 | [diff] [blame] | 237 | void loadActiveMetric(const ActiveMetric& activeMetric, int64_t currentTimeNs) { |
| 238 | std::lock_guard<std::mutex> lock(mMutex); |
| 239 | loadActiveMetricLocked(activeMetric, currentTimeNs); |
Yangster-mac | 15f6bbc | 2018-04-08 11:52:26 -0700 | [diff] [blame] | 240 | } |
| 241 | |
Yangster-mac | 849dfdc2 | 2018-10-12 15:41:45 -0700 | [diff] [blame] | 242 | void activate(int activationTrackerIndex, int64_t elapsedTimestampNs) { |
| 243 | std::lock_guard<std::mutex> lock(mMutex); |
| 244 | activateLocked(activationTrackerIndex, elapsedTimestampNs); |
| 245 | } |
| 246 | |
Muhammad Qureshi | 3a5ebf5 | 2019-03-28 12:38:21 -0700 | [diff] [blame] | 247 | void cancelEventActivation(int deactivationTrackerIndex) { |
| 248 | std::lock_guard<std::mutex> lock(mMutex); |
| 249 | cancelEventActivationLocked(deactivationTrackerIndex); |
| 250 | } |
| 251 | |
Chenjie Yu | c7939cb | 2019-02-04 17:25:45 -0800 | [diff] [blame] | 252 | bool isActive() const { |
| 253 | std::lock_guard<std::mutex> lock(mMutex); |
| 254 | return isActiveLocked(); |
| 255 | } |
| 256 | |
tsaichristine | a05cfe0 | 2019-09-18 16:28:48 -0700 | [diff] [blame] | 257 | void flushIfExpire(int64_t elapsedTimestampNs); |
| 258 | |
Muhammad Qureshi | 844694b | 2019-04-05 10:10:40 -0700 | [diff] [blame] | 259 | void writeActiveMetricToProtoOutputStream( |
Tej Singh | f53d445 | 2019-05-09 18:17:59 -0700 | [diff] [blame] | 260 | int64_t currentTimeNs, const DumpReportReason reason, ProtoOutputStream* proto); |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 261 | |
tsaichristine | a05cfe0 | 2019-09-18 16:28:48 -0700 | [diff] [blame] | 262 | // Start: getters/setters |
| 263 | inline const int64_t& getMetricId() const { |
| 264 | return mMetricId; |
Yang Lu | b472291 | 2018-11-15 11:02:03 -0800 | [diff] [blame] | 265 | } |
| 266 | |
tsaichristine | a05cfe0 | 2019-09-18 16:28:48 -0700 | [diff] [blame] | 267 | // For test only. |
| 268 | inline int64_t getCurrentBucketNum() const { |
| 269 | return mCurrentBucketNum; |
| 270 | } |
Chenjie Yu | c7939cb | 2019-02-04 17:25:45 -0800 | [diff] [blame] | 271 | |
tsaichristine | a05cfe0 | 2019-09-18 16:28:48 -0700 | [diff] [blame] | 272 | int64_t getBucketSizeInNs() const { |
| 273 | std::lock_guard<std::mutex> lock(mMutex); |
| 274 | return mBucketSizeNs; |
| 275 | } |
| 276 | |
tsaichristine | d21aacf | 2019-10-07 14:47:38 -0700 | [diff] [blame] | 277 | inline const std::vector<int> getSlicedStateAtoms() { |
| 278 | std::lock_guard<std::mutex> lock(mMutex); |
| 279 | return mSlicedStateAtoms; |
| 280 | } |
| 281 | |
tsaichristine | a05cfe0 | 2019-09-18 16:28:48 -0700 | [diff] [blame] | 282 | /* If alert is valid, adds an AnomalyTracker and returns it. If invalid, returns nullptr. */ |
| 283 | virtual sp<AnomalyTracker> addAnomalyTracker(const Alert &alert, |
| 284 | const sp<AlarmMonitor>& anomalyAlarmMonitor) { |
| 285 | std::lock_guard<std::mutex> lock(mMutex); |
| 286 | sp<AnomalyTracker> anomalyTracker = new AnomalyTracker(alert, mConfigKey); |
| 287 | if (anomalyTracker != nullptr) { |
| 288 | mAnomalyTrackers.push_back(anomalyTracker); |
| 289 | } |
| 290 | return anomalyTracker; |
| 291 | } |
| 292 | // End: getters/setters |
| 293 | protected: |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 294 | /** |
Yangster-mac | e68f3a5 | 2018-04-04 00:01:43 -0700 | [diff] [blame] | 295 | * Flushes the current bucket if the eventTime is after the current bucket's end time. This will |
| 296 | also flush the current partial bucket in memory. |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 297 | */ |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 298 | virtual void flushIfNeededLocked(const int64_t& eventTime){}; |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 299 | |
| 300 | /** |
| 301 | * For metrics that aggregate (ie, every metric producer except for EventMetricProducer), |
| 302 | * we need to be able to flush the current buckets on demand (ie, end the current bucket and |
| 303 | * start new bucket). If this function is called when eventTimeNs is greater than the current |
| 304 | * bucket's end timestamp, than we flush up to the end of the latest full bucket; otherwise, |
| 305 | * we assume that we want to flush a partial bucket. The bucket start timestamp and bucket |
| 306 | * number are not changed by this function. This method should only be called by |
Muhammad Qureshi | 18e4692 | 2019-05-24 16:38:49 -0700 | [diff] [blame] | 307 | * flushIfNeededLocked or flushLocked or the app upgrade handler; the caller MUST update the |
| 308 | * bucket timestamp and bucket number as needed. |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 309 | */ |
Olivier Gaillard | 6c75ecd | 2019-02-20 09:57:33 +0000 | [diff] [blame] | 310 | virtual void flushCurrentBucketLocked(const int64_t& eventTimeNs, |
| 311 | const int64_t& nextBucketStartTimeNs) {}; |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 312 | |
tsaichristine | a05cfe0 | 2019-09-18 16:28:48 -0700 | [diff] [blame] | 313 | /** |
| 314 | * Flushes all the data including the current partial bucket. |
| 315 | */ |
| 316 | virtual void flushLocked(const int64_t& eventTimeNs) { |
| 317 | flushIfNeededLocked(eventTimeNs); |
| 318 | flushCurrentBucketLocked(eventTimeNs, eventTimeNs); |
| 319 | }; |
Yangster-mac | e2cd6d5 | 2017-11-09 20:38:30 -0800 | [diff] [blame] | 320 | |
Yao Chen | b704177 | 2017-10-20 16:59:25 -0700 | [diff] [blame] | 321 | /* |
| 322 | * Individual metrics can implement their own business logic here. All pre-processing is done. |
| 323 | * |
| 324 | * [matcherIndex]: the index of the matcher which matched this event. This is interesting to |
| 325 | * DurationMetric, because it has start/stop/stop_all 3 matchers. |
| 326 | * [eventKey]: the extracted dimension key for the final output. if the metric doesn't have |
| 327 | * dimensions, it will be DEFAULT_DIMENSION_KEY |
| 328 | * [conditionKey]: the keys of conditions which should be used to query the condition for this |
Stefan Lafon | a5b5191 | 2017-12-05 21:43:52 -0800 | [diff] [blame] | 329 | * target event (from MetricConditionLink). This is passed to individual metrics |
Yao Chen | b704177 | 2017-10-20 16:59:25 -0700 | [diff] [blame] | 330 | * because DurationMetric needs it to be cached. |
| 331 | * [condition]: whether condition is met. If condition is sliced, this is the result coming from |
| 332 | * query with ConditionWizard; If condition is not sliced, this is the |
| 333 | * nonSlicedCondition. |
| 334 | * [event]: the log event, just in case the metric needs its data, e.g., EventMetric. |
| 335 | */ |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 336 | virtual void onMatchedLogEventInternalLocked( |
Yangster-mac | 9369446 | 2018-01-22 20:49:31 -0800 | [diff] [blame] | 337 | const size_t matcherIndex, const MetricDimensionKey& eventKey, |
tsaichristine | c876b49 | 2019-12-10 13:47:05 -0800 | [diff] [blame] | 338 | const ConditionKey& conditionKey, bool condition, const LogEvent& event, |
| 339 | const map<int, HashableDimensionKey>& statePrimaryKeys) = 0; |
yro | 2b0f886 | 2017-11-06 14:27:31 -0800 | [diff] [blame] | 340 | |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 341 | // Consume the parsed stats log entry that already matched the "what" of the metric. |
Yangster-mac | 5392888 | 2018-02-25 23:02:56 -0800 | [diff] [blame] | 342 | virtual void onMatchedLogEventLocked(const size_t matcherIndex, const LogEvent& event); |
tsaichristine | a05cfe0 | 2019-09-18 16:28:48 -0700 | [diff] [blame] | 343 | virtual void onConditionChangedLocked(const bool condition, const int64_t eventTime) = 0; |
| 344 | virtual void onSlicedConditionMayChangeLocked(bool overallCondition, |
| 345 | const int64_t eventTime) = 0; |
| 346 | virtual void onDumpReportLocked(const int64_t dumpTimeNs, |
| 347 | const bool include_current_partial_bucket, |
| 348 | const bool erase_data, |
| 349 | const DumpLatency dumpLatency, |
| 350 | std::set<string> *str_set, |
| 351 | android::util::ProtoOutputStream* protoOutput) = 0; |
| 352 | virtual void clearPastBucketsLocked(const int64_t dumpTimeNs) = 0; |
Tej Singh | 3be093b | 2020-03-04 20:08:38 -0800 | [diff] [blame] | 353 | virtual void prepareFirstBucketLocked(){}; |
tsaichristine | a05cfe0 | 2019-09-18 16:28:48 -0700 | [diff] [blame] | 354 | virtual size_t byteSizeLocked() const = 0; |
| 355 | virtual void dumpStatesLocked(FILE* out, bool verbose) const = 0; |
| 356 | virtual void dropDataLocked(const int64_t dropTimeNs) = 0; |
tsaichristine | a05cfe0 | 2019-09-18 16:28:48 -0700 | [diff] [blame] | 357 | void loadActiveMetricLocked(const ActiveMetric& activeMetric, int64_t currentTimeNs); |
| 358 | void activateLocked(int activationTrackerIndex, int64_t elapsedTimestampNs); |
| 359 | void cancelEventActivationLocked(int deactivationTrackerIndex); |
| 360 | |
| 361 | bool evaluateActiveStateLocked(int64_t elapsedTimestampNs); |
| 362 | |
| 363 | virtual void onActiveStateChangedLocked(const int64_t& eventTimeNs) { |
| 364 | if (!mIsActive) { |
| 365 | flushLocked(eventTimeNs); |
| 366 | } |
| 367 | } |
| 368 | |
| 369 | inline bool isActiveLocked() const { |
| 370 | return mIsActive; |
| 371 | } |
| 372 | |
| 373 | // Convenience to compute the current bucket's end time, which is always aligned with the |
| 374 | // start time of the metric. |
| 375 | int64_t getCurrentBucketEndTimeNs() const { |
| 376 | return mTimeBaseNs + (mCurrentBucketNum + 1) * mBucketSizeNs; |
| 377 | } |
| 378 | |
| 379 | int64_t getBucketNumFromEndTimeNs(const int64_t endNs) { |
| 380 | return (endNs - mTimeBaseNs) / mBucketSizeNs - 1; |
| 381 | } |
| 382 | |
tsaichristine | 1449fa4 | 2020-01-02 12:12:05 -0800 | [diff] [blame] | 383 | // Query StateManager for original state value using the queryKey. |
| 384 | // The field and value are output. |
| 385 | void queryStateValue(const int32_t atomId, const HashableDimensionKey& queryKey, |
| 386 | FieldValue* value); |
| 387 | |
| 388 | // If a state map exists for the given atom, replace the original state |
| 389 | // value with the group id mapped to the value. |
| 390 | // If no state map exists, keep the original state value. |
| 391 | void mapStateValue(const int32_t atomId, FieldValue* value); |
tsaichristine | 69000e6 | 2019-10-18 17:34:52 -0700 | [diff] [blame] | 392 | |
tsaichristine | b87ca15 | 2019-12-09 15:19:41 -0800 | [diff] [blame] | 393 | DropEvent buildDropEvent(const int64_t dropTimeNs, const BucketDropReason reason); |
| 394 | |
| 395 | // Returns true if the number of drop events in the current bucket has |
| 396 | // exceeded the maximum number allowed, which is currently capped at 10. |
| 397 | bool maxDropEventsReached(); |
| 398 | |
tsaichristine | a05cfe0 | 2019-09-18 16:28:48 -0700 | [diff] [blame] | 399 | const int64_t mMetricId; |
| 400 | |
| 401 | const ConfigKey mConfigKey; |
| 402 | |
| 403 | // The time when this metric producer was first created. The end time for the current bucket |
| 404 | // can be computed from this based on mCurrentBucketNum. |
| 405 | int64_t mTimeBaseNs; |
| 406 | |
| 407 | // Start time may not be aligned with the start of statsd if there is an app upgrade in the |
| 408 | // middle of a bucket. |
| 409 | int64_t mCurrentBucketStartTimeNs; |
| 410 | |
| 411 | // Used by anomaly detector to track which bucket we are in. This is not sent with the produced |
| 412 | // report. |
| 413 | int64_t mCurrentBucketNum; |
| 414 | |
| 415 | int64_t mBucketSizeNs; |
| 416 | |
| 417 | ConditionState mCondition; |
| 418 | |
| 419 | int mConditionTrackerIndex; |
| 420 | |
| 421 | bool mConditionSliced; |
| 422 | |
| 423 | sp<ConditionWizard> mWizard; |
| 424 | |
| 425 | bool mContainANYPositionInDimensionsInWhat; |
| 426 | |
| 427 | bool mSliceByPositionALL; |
| 428 | |
| 429 | vector<Matcher> mDimensionsInWhat; // The dimensions_in_what defined in statsd_config |
| 430 | |
| 431 | // True iff the metric to condition links cover all dimension fields in the condition tracker. |
| 432 | // This field is always false for combinational condition trackers. |
| 433 | bool mHasLinksToAllConditionDimensionsInTracker; |
| 434 | |
| 435 | std::vector<Metric2Condition> mMetric2ConditionLinks; |
| 436 | |
| 437 | std::vector<sp<AnomalyTracker>> mAnomalyTrackers; |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 438 | |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 439 | mutable std::mutex mMutex; |
Yangster-mac | 849dfdc2 | 2018-10-12 15:41:45 -0700 | [diff] [blame] | 440 | |
Yangster-mac | 849dfdc2 | 2018-10-12 15:41:45 -0700 | [diff] [blame] | 441 | // When the metric producer has multiple activations, these activations are ORed to determine |
| 442 | // whether the metric producer is ready to generate metrics. |
Muhammad Qureshi | 3a5ebf5 | 2019-03-28 12:38:21 -0700 | [diff] [blame] | 443 | std::unordered_map<int, std::shared_ptr<Activation>> mEventActivationMap; |
| 444 | |
Tej Singh | ee4495e | 2019-06-03 18:37:35 -0700 | [diff] [blame] | 445 | // Maps index of atom matcher for deactivation to a list of Activation structs. |
| 446 | std::unordered_map<int, std::vector<std::shared_ptr<Activation>>> mEventDeactivationMap; |
Yangster-mac | 849dfdc2 | 2018-10-12 15:41:45 -0700 | [diff] [blame] | 447 | |
| 448 | bool mIsActive; |
| 449 | |
tsaichristine | d21aacf | 2019-10-07 14:47:38 -0700 | [diff] [blame] | 450 | // The slice_by_state atom ids defined in statsd_config. |
tsaichristine | 69000e6 | 2019-10-18 17:34:52 -0700 | [diff] [blame] | 451 | std::vector<int32_t> mSlicedStateAtoms; |
tsaichristine | d21aacf | 2019-10-07 14:47:38 -0700 | [diff] [blame] | 452 | |
| 453 | // Maps atom ids and state values to group_ids (<atom_id, <value, group_id>>). |
Muhammad Qureshi | bfc4bdb | 2020-04-08 06:26:49 -0700 | [diff] [blame] | 454 | const std::unordered_map<int32_t, std::unordered_map<int, int64_t>> mStateGroupMap; |
tsaichristine | d21aacf | 2019-10-07 14:47:38 -0700 | [diff] [blame] | 455 | |
tsaichristine | 69000e6 | 2019-10-18 17:34:52 -0700 | [diff] [blame] | 456 | // MetricStateLinks defined in statsd_config that link fields in the state |
| 457 | // atom to fields in the "what" atom. |
| 458 | std::vector<Metric2State> mMetric2StateLinks; |
| 459 | |
tsaichristine | b87ca15 | 2019-12-09 15:19:41 -0800 | [diff] [blame] | 460 | SkippedBucket mCurrentSkippedBucket; |
| 461 | // Buckets that were invalidated and had their data dropped. |
| 462 | std::vector<SkippedBucket> mSkippedBuckets; |
| 463 | |
tsaichristine | 69000e6 | 2019-10-18 17:34:52 -0700 | [diff] [blame] | 464 | FRIEND_TEST(CountMetricE2eTest, TestSlicedState); |
| 465 | FRIEND_TEST(CountMetricE2eTest, TestSlicedStateWithMap); |
| 466 | FRIEND_TEST(CountMetricE2eTest, TestMultipleSlicedStates); |
| 467 | FRIEND_TEST(CountMetricE2eTest, TestSlicedStateWithPrimaryFields); |
tsaichristine | d21aacf | 2019-10-07 14:47:38 -0700 | [diff] [blame] | 468 | |
Muhammad Qureshi | 18e4692 | 2019-05-24 16:38:49 -0700 | [diff] [blame] | 469 | FRIEND_TEST(DurationMetricE2eTest, TestOneBucket); |
| 470 | FRIEND_TEST(DurationMetricE2eTest, TestTwoBuckets); |
| 471 | FRIEND_TEST(DurationMetricE2eTest, TestWithActivation); |
| 472 | FRIEND_TEST(DurationMetricE2eTest, TestWithCondition); |
| 473 | FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedCondition); |
| 474 | FRIEND_TEST(DurationMetricE2eTest, TestWithActivationAndSlicedCondition); |
tsaichristine | 1449fa4 | 2020-01-02 12:12:05 -0800 | [diff] [blame] | 475 | FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedState); |
| 476 | FRIEND_TEST(DurationMetricE2eTest, TestWithConditionAndSlicedState); |
| 477 | FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedStateMapped); |
| 478 | FRIEND_TEST(DurationMetricE2eTest, TestSlicedStatePrimaryFieldsNotSubsetDimInWhat); |
| 479 | FRIEND_TEST(DurationMetricE2eTest, TestWithSlicedStatePrimaryFieldsSubset); |
Muhammad Qureshi | 18e4692 | 2019-05-24 16:38:49 -0700 | [diff] [blame] | 480 | |
Yangster-mac | 849dfdc2 | 2018-10-12 15:41:45 -0700 | [diff] [blame] | 481 | FRIEND_TEST(MetricActivationE2eTest, TestCountMetric); |
Muhammad Qureshi | 3a5ebf5 | 2019-03-28 12:38:21 -0700 | [diff] [blame] | 482 | FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithOneDeactivation); |
| 483 | FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithTwoDeactivations); |
Tej Singh | ee4495e | 2019-06-03 18:37:35 -0700 | [diff] [blame] | 484 | FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithSameDeactivation); |
Muhammad Qureshi | 3a5ebf5 | 2019-03-28 12:38:21 -0700 | [diff] [blame] | 485 | FRIEND_TEST(MetricActivationE2eTest, TestCountMetricWithTwoMetricsTwoDeactivations); |
Chenjie Yu | c7939cb | 2019-02-04 17:25:45 -0800 | [diff] [blame] | 486 | |
| 487 | FRIEND_TEST(StatsLogProcessorTest, TestActiveConfigMetricDiskWriteRead); |
Chenjie Yu | a9a310e | 2019-02-06 13:40:10 -0800 | [diff] [blame] | 488 | FRIEND_TEST(StatsLogProcessorTest, TestActivationOnBoot); |
Muhammad Qureshi | 844694b | 2019-04-05 10:10:40 -0700 | [diff] [blame] | 489 | FRIEND_TEST(StatsLogProcessorTest, TestActivationOnBootMultipleActivations); |
Muhammad Qureshi | 15f8da9 | 2019-04-05 10:10:40 -0700 | [diff] [blame] | 490 | FRIEND_TEST(StatsLogProcessorTest, |
| 491 | TestActivationOnBootMultipleActivationsDifferentActivationTypes); |
Tej Singh | f53d445 | 2019-05-09 18:17:59 -0700 | [diff] [blame] | 492 | FRIEND_TEST(StatsLogProcessorTest, TestActivationsPersistAcrossSystemServerRestart); |
tsaichristine | c876b49 | 2019-12-10 13:47:05 -0800 | [diff] [blame] | 493 | |
| 494 | FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState); |
| 495 | FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithDimensions); |
| 496 | FRIEND_TEST(ValueMetricE2eTest, TestInitWithSlicedState_WithIncorrectDimensions); |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 497 | }; |
| 498 | |
| 499 | } // namespace statsd |
| 500 | } // namespace os |
| 501 | } // namespace android |
| 502 | #endif // METRIC_PRODUCER_H |