Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2017 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 17 | #pragma once |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 18 | |
Yangster-mac | 932ecec | 2018-02-01 10:23:52 -0800 | [diff] [blame] | 19 | #include "anomaly/AlarmMonitor.h" |
| 20 | #include "anomaly/AlarmTracker.h" |
Yangster-mac | e2cd6d5 | 2017-11-09 20:38:30 -0800 | [diff] [blame] | 21 | #include "anomaly/AnomalyTracker.h" |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 22 | #include "condition/ConditionTracker.h" |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 23 | #include "config/ConfigKey.h" |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 24 | #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" |
Joe Onorato | c4dfae5 | 2017-10-17 23:38:21 -0700 | [diff] [blame] | 25 | #include "logd/LogEvent.h" |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 26 | #include "matchers/LogMatchingTracker.h" |
| 27 | #include "metrics/MetricProducer.h" |
Yao Chen | d10f7b1 | 2017-12-18 12:53:50 -0800 | [diff] [blame] | 28 | #include "packages/UidMap.h" |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 29 | |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 30 | #include <unordered_map> |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 31 | |
| 32 | namespace android { |
| 33 | namespace os { |
| 34 | namespace statsd { |
| 35 | |
| 36 | // A MetricsManager is responsible for managing metrics from one single config source. |
Yao Chen | d10f7b1 | 2017-12-18 12:53:50 -0800 | [diff] [blame] | 37 | class MetricsManager : public PackageInfoListener { |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 38 | public: |
Yangster-mac | c04feba | 2018-04-02 14:37:33 -0700 | [diff] [blame] | 39 | MetricsManager(const ConfigKey& configKey, const StatsdConfig& config, |
Yangster-mac | 15f6bbc | 2018-04-08 11:52:26 -0700 | [diff] [blame] | 40 | const int64_t timeBaseNs, const int64_t currentTimeNs, |
Yangster-mac | 932ecec | 2018-02-01 10:23:52 -0800 | [diff] [blame] | 41 | const sp<UidMap>& uidMap, const sp<AlarmMonitor>& anomalyAlarmMonitor, |
| 42 | const sp<AlarmMonitor>& periodicAlarmMonitor); |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 43 | |
David Chen | d9269e2 | 2017-12-05 13:43:51 -0800 | [diff] [blame] | 44 | virtual ~MetricsManager(); |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 45 | |
Yao Chen | caf339d | 2017-10-06 16:01:10 -0700 | [diff] [blame] | 46 | // Return whether the configuration is valid. |
| 47 | bool isConfigValid() const; |
| 48 | |
Joe Onorato | c4dfae5 | 2017-10-17 23:38:21 -0700 | [diff] [blame] | 49 | void onLogEvent(const LogEvent& event); |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 50 | |
Yangster-mac | 2087716 | 2017-12-22 17:19:39 -0800 | [diff] [blame] | 51 | void onAnomalyAlarmFired( |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 52 | const int64_t& timestampNs, |
Yangster-mac | 932ecec | 2018-02-01 10:23:52 -0800 | [diff] [blame] | 53 | unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>>& alarmSet); |
Yangster-mac | e2cd6d5 | 2017-11-09 20:38:30 -0800 | [diff] [blame] | 54 | |
Yangster-mac | 932ecec | 2018-02-01 10:23:52 -0800 | [diff] [blame] | 55 | void onPeriodicAlarmFired( |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 56 | const int64_t& timestampNs, |
Yangster-mac | 932ecec | 2018-02-01 10:23:52 -0800 | [diff] [blame] | 57 | unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>>& alarmSet); |
Yangster-mac | e2cd6d5 | 2017-11-09 20:38:30 -0800 | [diff] [blame] | 58 | |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 59 | void notifyAppUpgrade(const int64_t& eventTimeNs, const string& apk, const int uid, |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 60 | const int64_t version) override; |
Yao Chen | d10f7b1 | 2017-12-18 12:53:50 -0800 | [diff] [blame] | 61 | |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 62 | void notifyAppRemoved(const int64_t& eventTimeNs, const string& apk, const int uid) override; |
Yao Chen | d10f7b1 | 2017-12-18 12:53:50 -0800 | [diff] [blame] | 63 | |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 64 | void onUidMapReceived(const int64_t& eventTimeNs) override; |
Yao Chen | d10f7b1 | 2017-12-18 12:53:50 -0800 | [diff] [blame] | 65 | |
Yao Chen | 147ce60 | 2017-12-22 14:35:34 -0800 | [diff] [blame] | 66 | bool shouldAddUidMapListener() const { |
| 67 | return !mAllowedPkg.empty(); |
| 68 | } |
| 69 | |
Yao Chen | 884c8c1 | 2018-01-26 10:36:25 -0800 | [diff] [blame] | 70 | void dumpStates(FILE* out, bool verbose); |
| 71 | |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 72 | inline bool isInTtl(const int64_t timestampNs) const { |
| 73 | return mTtlNs <= 0 || timestampNs < mTtlEndNs; |
| 74 | }; |
| 75 | |
| 76 | void refreshTtl(const int64_t currentTimestampNs) { |
| 77 | if (mTtlNs > 0) { |
| 78 | mTtlEndNs = currentTimestampNs + mTtlNs; |
| 79 | } |
| 80 | }; |
| 81 | |
David Chen | 1604957 | 2018-02-01 18:27:51 -0800 | [diff] [blame] | 82 | // Returns the elapsed realtime when this metric manager last reported metrics. |
Yangster-mac | 3fa5d7f | 2018-03-10 21:50:27 -0800 | [diff] [blame] | 83 | inline int64_t getLastReportTimeNs() const { |
David Chen | 1604957 | 2018-02-01 18:27:51 -0800 | [diff] [blame] | 84 | return mLastReportTimeNs; |
| 85 | }; |
| 86 | |
Yangster-mac | 3fa5d7f | 2018-03-10 21:50:27 -0800 | [diff] [blame] | 87 | inline int64_t getLastReportWallClockNs() const { |
| 88 | return mLastReportWallClockNs; |
| 89 | }; |
| 90 | |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 91 | virtual void dropData(const int64_t dropTimeNs); |
Yao Chen | 06dba5d | 2018-01-26 13:38:16 -0800 | [diff] [blame] | 92 | |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 93 | virtual void onDumpReport(const int64_t dumpTimeNs, |
Yangster-mac | e68f3a5 | 2018-04-04 00:01:43 -0700 | [diff] [blame] | 94 | const bool include_current_partial_bucket, |
Yao Chen | 8a8d16c | 2018-02-08 14:50:40 -0800 | [diff] [blame] | 95 | android::util::ProtoOutputStream* protoOutput); |
Yao Chen | 729093d | 2017-10-16 10:33:26 -0700 | [diff] [blame] | 96 | |
David Chen | 1d7b0cd | 2017-11-15 14:20:04 -0800 | [diff] [blame] | 97 | // Computes the total byte size of all metrics managed by a single config source. |
| 98 | // Does not change the state. |
David Chen | d9269e2 | 2017-12-05 13:43:51 -0800 | [diff] [blame] | 99 | virtual size_t byteSize(); |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 100 | |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 101 | private: |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 102 | // For test only. |
| 103 | inline int64_t getTtlEndNs() const { return mTtlEndNs; } |
| 104 | |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 105 | const ConfigKey mConfigKey; |
| 106 | |
Yao Chen | d10f7b1 | 2017-12-18 12:53:50 -0800 | [diff] [blame] | 107 | sp<UidMap> mUidMap; |
| 108 | |
| 109 | bool mConfigValid = false; |
| 110 | |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 111 | const int64_t mTtlNs; |
| 112 | int64_t mTtlEndNs; |
| 113 | |
Yangster-mac | 3fa5d7f | 2018-03-10 21:50:27 -0800 | [diff] [blame] | 114 | int64_t mLastReportTimeNs; |
| 115 | int64_t mLastReportWallClockNs; |
David Chen | 1604957 | 2018-02-01 18:27:51 -0800 | [diff] [blame] | 116 | |
Yao Chen | d10f7b1 | 2017-12-18 12:53:50 -0800 | [diff] [blame] | 117 | // The uid log sources from StatsdConfig. |
| 118 | std::vector<int32_t> mAllowedUid; |
| 119 | |
| 120 | // The pkg log sources from StatsdConfig. |
| 121 | std::vector<std::string> mAllowedPkg; |
| 122 | |
| 123 | // The combined uid sources (after translating pkg name to uid). |
| 124 | // Logs from uids that are not in the list will be ignored to avoid spamming. |
| 125 | std::set<int32_t> mAllowedLogSources; |
| 126 | |
David Chen | faa1af5 | 2018-03-30 15:14:04 -0700 | [diff] [blame] | 127 | // Contains the annotations passed in with StatsdConfig. |
| 128 | std::list<std::pair<const int64_t, const int32_t>> mAnnotations; |
| 129 | |
Yao Chen | d10f7b1 | 2017-12-18 12:53:50 -0800 | [diff] [blame] | 130 | // To guard access to mAllowedLogSources |
| 131 | mutable std::mutex mAllowedLogSourcesMutex; |
| 132 | |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 133 | // All event tags that are interesting to my metrics. |
| 134 | std::set<int> mTagIds; |
| 135 | |
Yao Chen | caf339d | 2017-10-06 16:01:10 -0700 | [diff] [blame] | 136 | // We only store the sp of LogMatchingTracker, MetricProducer, and ConditionTracker in |
Bookatz | d3606c7 | 2017-10-19 10:13:49 -0700 | [diff] [blame] | 137 | // MetricsManager. There are relationships between them, and the relationships are denoted by |
| 138 | // index instead of pointers. The reasons for this are: (1) the relationship between them are |
| 139 | // complicated, so storing index instead of pointers reduces the risk that A holds B's sp, and B |
| 140 | // holds A's sp. (2) When we evaluate matcher results, or condition results, we can quickly get |
| 141 | // the related results from a cache using the index. |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 142 | |
Stefan Lafon | b8c9aa8 | 2017-12-03 14:27:25 -0800 | [diff] [blame] | 143 | // Hold all the atom matchers from the config. |
| 144 | std::vector<sp<LogMatchingTracker>> mAllAtomMatchers; |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 145 | |
Yao Chen | caf339d | 2017-10-06 16:01:10 -0700 | [diff] [blame] | 146 | // Hold all the conditions from the config. |
| 147 | std::vector<sp<ConditionTracker>> mAllConditionTrackers; |
| 148 | |
| 149 | // Hold all metrics from the config. |
| 150 | std::vector<sp<MetricProducer>> mAllMetricProducers; |
| 151 | |
Yangster-mac | e2cd6d5 | 2017-11-09 20:38:30 -0800 | [diff] [blame] | 152 | // Hold all alert trackers. |
| 153 | std::vector<sp<AnomalyTracker>> mAllAnomalyTrackers; |
| 154 | |
Yangster-mac | 932ecec | 2018-02-01 10:23:52 -0800 | [diff] [blame] | 155 | // Hold all periodic alarm trackers. |
| 156 | std::vector<sp<AlarmTracker>> mAllPeriodicAlarmTrackers; |
| 157 | |
Yao Chen | caf339d | 2017-10-06 16:01:10 -0700 | [diff] [blame] | 158 | // To make the log processing more efficient, we want to do as much filtering as possible |
| 159 | // before we go into individual trackers and conditions to match. |
| 160 | |
| 161 | // 1st filter: check if the event tag id is in mTagIds. |
| 162 | // 2nd filter: if it is, we parse the event because there is at least one member is interested. |
| 163 | // then pass to all LogMatchingTrackers (itself also filter events by ids). |
| 164 | // 3nd filter: for LogMatchingTrackers that matched this event, we pass this event to the |
| 165 | // ConditionTrackers and MetricProducers that use this matcher. |
| 166 | // 4th filter: for ConditionTrackers that changed value due to this event, we pass |
| 167 | // new conditions to metrics that use this condition. |
| 168 | |
| 169 | // The following map is initialized from the statsd_config. |
| 170 | |
| 171 | // maps from the index of the LogMatchingTracker to index of MetricProducer. |
| 172 | std::unordered_map<int, std::vector<int>> mTrackerToMetricMap; |
| 173 | |
| 174 | // maps from LogMatchingTracker to ConditionTracker |
| 175 | std::unordered_map<int, std::vector<int>> mTrackerToConditionMap; |
| 176 | |
| 177 | // maps from ConditionTracker to MetricProducer |
| 178 | std::unordered_map<int, std::vector<int>> mConditionToMetricMap; |
| 179 | |
Yao Chen | d10f7b1 | 2017-12-18 12:53:50 -0800 | [diff] [blame] | 180 | void initLogSourceWhiteList(); |
Yangster-mac | 2087716 | 2017-12-22 17:19:39 -0800 | [diff] [blame] | 181 | |
Yangster-mac | 94e197c | 2018-01-02 16:03:03 -0800 | [diff] [blame] | 182 | // The metrics that don't need to be uploaded or even reported. |
| 183 | std::set<int64_t> mNoReportMetricIds; |
| 184 | |
Yangster-mac | 2087716 | 2017-12-22 17:19:39 -0800 | [diff] [blame] | 185 | FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensions); |
| 186 | FRIEND_TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks); |
Yangster-mac | e06cfd7 | 2018-03-10 23:22:59 -0800 | [diff] [blame] | 187 | FRIEND_TEST(AttributionE2eTest, TestAttributionMatchAndSliceByFirstUid); |
| 188 | FRIEND_TEST(AttributionE2eTest, TestAttributionMatchAndSliceByChain); |
Yangster-mac | 87718e2 | 2018-01-11 16:16:26 -0800 | [diff] [blame] | 189 | FRIEND_TEST(GaugeMetricE2eTest, TestMultipleFieldsForPushedEvent); |
Yangster-mac | 15f6bbc | 2018-04-08 11:52:26 -0700 | [diff] [blame] | 190 | FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvents); |
| 191 | FRIEND_TEST(GaugeMetricE2eTest, TestRandomSamplePulledEvent_LateAlarm); |
| 192 | FRIEND_TEST(GaugeMetricE2eTest, TestAllConditionChangesSamplePulledEvents); |
| 193 | FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents); |
| 194 | FRIEND_TEST(ValueMetricE2eTest, TestPulledEvents_LateAlarm); |
Yangster | 13fb7e4 | 2018-03-07 17:30:49 -0800 | [diff] [blame] | 195 | FRIEND_TEST(DimensionInConditionE2eTest, TestCreateCountMetric_NoLink_OR_CombinationCondition); |
| 196 | FRIEND_TEST(DimensionInConditionE2eTest, TestCreateCountMetric_Link_OR_CombinationCondition); |
| 197 | FRIEND_TEST(DimensionInConditionE2eTest, TestDurationMetric_NoLink_OR_CombinationCondition); |
| 198 | FRIEND_TEST(DimensionInConditionE2eTest, TestDurationMetric_Link_OR_CombinationCondition); |
| 199 | |
| 200 | FRIEND_TEST(DimensionInConditionE2eTest, TestDurationMetric_NoLink_SimpleCondition); |
| 201 | FRIEND_TEST(DimensionInConditionE2eTest, TestDurationMetric_Link_SimpleCondition); |
| 202 | FRIEND_TEST(DimensionInConditionE2eTest, TestDurationMetric_PartialLink_SimpleCondition); |
| 203 | FRIEND_TEST(DimensionInConditionE2eTest, TestDurationMetric_NoLink_AND_CombinationCondition); |
| 204 | FRIEND_TEST(DimensionInConditionE2eTest, TestDurationMetric_Link_AND_CombinationCondition); |
| 205 | FRIEND_TEST(DimensionInConditionE2eTest, TestDurationMetric_PartialLink_AND_CombinationCondition); |
| 206 | |
Yangster-mac | be10ddf | 2018-03-13 15:39:51 -0700 | [diff] [blame] | 207 | FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_single_bucket); |
| 208 | FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_multiple_buckets); |
| 209 | FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_single_bucket); |
| 210 | FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_multiple_buckets); |
| 211 | FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_long_refractory_period); |
| 212 | |
Yangster-mac | 684d195 | 2018-03-24 16:47:16 -0700 | [diff] [blame] | 213 | FRIEND_TEST(AlarmE2eTest, TestMultipleAlarms); |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 214 | FRIEND_TEST(ConfigTtlE2eTest, TestCountMetric); |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 215 | }; |
| 216 | |
| 217 | } // namespace statsd |
| 218 | } // namespace os |
| 219 | } // namespace android |