Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2017 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #pragma once |
| 18 | |
| 19 | #include <unordered_map> |
| 20 | |
yro | 2b0f886 | 2017-11-06 14:27:31 -0800 | [diff] [blame] | 21 | #include <android/util/ProtoOutputStream.h> |
Yangster-mac | e2cd6d5 | 2017-11-09 20:38:30 -0800 | [diff] [blame] | 22 | #include <gtest/gtest_prod.h> |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 23 | #include "../condition/ConditionTracker.h" |
| 24 | #include "../external/PullDataReceiver.h" |
| 25 | #include "../external/StatsPullerManager.h" |
| 26 | #include "../matchers/matcher_util.h" |
Yangster-mac | 32f07af | 2018-10-13 17:08:11 -0700 | [diff] [blame] | 27 | #include "../matchers/EventMatcherWizard.h" |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 28 | #include "MetricProducer.h" |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 29 | #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" |
Chenjie Yu | d9dfda7 | 2017-12-11 17:41:20 -0800 | [diff] [blame] | 30 | #include "../stats_util.h" |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 31 | |
| 32 | namespace android { |
| 33 | namespace os { |
| 34 | namespace statsd { |
| 35 | |
Yangster-mac | 34ea110 | 2018-01-29 12:40:55 -0800 | [diff] [blame] | 36 | struct GaugeAtom { |
Bookatz | fe2dde8 | 2018-08-28 13:24:40 -0700 | [diff] [blame] | 37 | GaugeAtom(std::shared_ptr<vector<FieldValue>> fields, int64_t elapsedTimeNs) |
Muhammad Qureshi | a7de000 | 2020-04-15 11:34:35 -0700 | [diff] [blame] | 38 | : mFields(fields), mElapsedTimestampNs(elapsedTimeNs) { |
Yao Chen | 8a8d16c | 2018-02-08 14:50:40 -0800 | [diff] [blame] | 39 | } |
| 40 | std::shared_ptr<vector<FieldValue>> mFields; |
Muhammad Qureshi | a7de000 | 2020-04-15 11:34:35 -0700 | [diff] [blame] | 41 | int64_t mElapsedTimestampNs; |
Yangster-mac | 34ea110 | 2018-01-29 12:40:55 -0800 | [diff] [blame] | 42 | }; |
| 43 | |
yro | 2b0f886 | 2017-11-06 14:27:31 -0800 | [diff] [blame] | 44 | struct GaugeBucket { |
| 45 | int64_t mBucketStartNs; |
| 46 | int64_t mBucketEndNs; |
Yangster-mac | 34ea110 | 2018-01-29 12:40:55 -0800 | [diff] [blame] | 47 | std::vector<GaugeAtom> mGaugeAtoms; |
yro | 2b0f886 | 2017-11-06 14:27:31 -0800 | [diff] [blame] | 48 | }; |
| 49 | |
Yangster-mac | 9369446 | 2018-01-22 20:49:31 -0800 | [diff] [blame] | 50 | typedef std::unordered_map<MetricDimensionKey, std::vector<GaugeAtom>> |
Yangster-mac | 34ea110 | 2018-01-29 12:40:55 -0800 | [diff] [blame] | 51 | DimToGaugeAtomsMap; |
Yangster-mac | 2087716 | 2017-12-22 17:19:39 -0800 | [diff] [blame] | 52 | |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 53 | // This gauge metric producer first register the puller to automatically pull the gauge at the |
| 54 | // beginning of each bucket. If the condition is met, insert it to the bucket info. Otherwise |
| 55 | // proactively pull the gauge when the condition is changed to be true. Therefore, the gauge metric |
| 56 | // producer always reports the guage at the earliest time of the bucket when the condition is met. |
| 57 | class GaugeMetricProducer : public virtual MetricProducer, public virtual PullDataReceiver { |
| 58 | public: |
tsaichristine | d21aacf | 2019-10-07 14:47:38 -0700 | [diff] [blame] | 59 | GaugeMetricProducer( |
| 60 | const ConfigKey& key, const GaugeMetric& gaugeMetric, const int conditionIndex, |
| 61 | const sp<ConditionWizard>& conditionWizard, const int whatMatcherIndex, |
| 62 | const sp<EventMatcherWizard>& matcherWizard, const int pullTagId, |
| 63 | const int triggerAtomId, const int atomId, const int64_t timeBaseNs, |
| 64 | const int64_t startTimeNs, const sp<StatsPullerManager>& pullerManager, |
| 65 | const std::unordered_map<int, std::shared_ptr<Activation>>& eventActivationMap = {}, |
| 66 | const std::unordered_map<int, std::vector<std::shared_ptr<Activation>>>& |
Tej Singh | 3be093b | 2020-03-04 20:08:38 -0800 | [diff] [blame] | 67 | eventDeactivationMap = {}); |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 68 | |
| 69 | virtual ~GaugeMetricProducer(); |
| 70 | |
| 71 | // Handles when the pulled data arrives. |
Olivier Gaillard | c5f11c4 | 2019-02-05 12:44:58 +0000 | [diff] [blame] | 72 | void onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& data, |
Olivier Gaillard | 11203df | 2019-02-06 13:18:09 +0000 | [diff] [blame] | 73 | bool pullSuccess, int64_t originalPullTimeNs) override; |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 74 | |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 75 | // GaugeMetric needs to immediately trigger another pull when we create the partial bucket. |
Tej Singh | e678cb7 | 2020-04-14 16:23:30 -0700 | [diff] [blame] | 76 | void notifyAppUpgrade(const int64_t& eventTimeNs) override { |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 77 | std::lock_guard<std::mutex> lock(mMutex); |
| 78 | |
Chenjie Yu | cd1b797 | 2019-01-16 20:38:15 -0800 | [diff] [blame] | 79 | if (!mSplitBucketForAppUpgrade) { |
| 80 | return; |
| 81 | } |
Tej Singh | e678cb7 | 2020-04-14 16:23:30 -0700 | [diff] [blame] | 82 | flushLocked(eventTimeNs); |
| 83 | if (mIsPulled && mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE) { |
| 84 | pullAndMatchEventsLocked(eventTimeNs); |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 85 | } |
Tej Singh | e678cb7 | 2020-04-14 16:23:30 -0700 | [diff] [blame] | 86 | }; |
| 87 | |
| 88 | // GaugeMetric needs to immediately trigger another pull when we create the partial bucket. |
| 89 | void onStatsdInitCompleted(const int64_t& eventTimeNs) override { |
| 90 | std::lock_guard<std::mutex> lock(mMutex); |
| 91 | |
| 92 | flushLocked(eventTimeNs); |
Chenjie Yu | e077f57 | 2018-11-02 11:51:37 -0700 | [diff] [blame] | 93 | if (mIsPulled && mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE) { |
Yangster-mac | 32f07af | 2018-10-13 17:08:11 -0700 | [diff] [blame] | 94 | pullAndMatchEventsLocked(eventTimeNs); |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 95 | } |
| 96 | }; |
| 97 | |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 98 | protected: |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 99 | void onMatchedLogEventInternalLocked( |
Yangster-mac | 9369446 | 2018-01-22 20:49:31 -0800 | [diff] [blame] | 100 | const size_t matcherIndex, const MetricDimensionKey& eventKey, |
tsaichristine | c876b49 | 2019-12-10 13:47:05 -0800 | [diff] [blame] | 101 | const ConditionKey& conditionKey, bool condition, const LogEvent& event, |
| 102 | const std::map<int, HashableDimensionKey>& statePrimaryKeys) override; |
yro | 2b0f886 | 2017-11-06 14:27:31 -0800 | [diff] [blame] | 103 | |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 104 | private: |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 105 | void onDumpReportLocked(const int64_t dumpTimeNs, |
Yangster-mac | e68f3a5 | 2018-04-04 00:01:43 -0700 | [diff] [blame] | 106 | const bool include_current_partial_bucket, |
Bookatz | ff71cad | 2018-09-20 17:17:49 -0700 | [diff] [blame] | 107 | const bool erase_data, |
Olivier Gaillard | 6c75ecd | 2019-02-20 09:57:33 +0000 | [diff] [blame] | 108 | const DumpLatency dumpLatency, |
Yangster-mac | 9def8e3 | 2018-04-17 13:55:51 -0700 | [diff] [blame] | 109 | std::set<string> *str_set, |
Yao Chen | 288c600 | 2017-12-12 13:43:18 -0800 | [diff] [blame] | 110 | android::util::ProtoOutputStream* protoOutput) override; |
Yangster-mac | a802d73 | 2018-04-24 07:50:38 -0700 | [diff] [blame] | 111 | void clearPastBucketsLocked(const int64_t dumpTimeNs) override; |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 112 | |
| 113 | // Internal interface to handle condition change. |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 114 | void onConditionChangedLocked(const bool conditionMet, const int64_t eventTime) override; |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 115 | |
Muhammad Qureshi | 18e4692 | 2019-05-24 16:38:49 -0700 | [diff] [blame] | 116 | // Internal interface to handle active state change. |
| 117 | void onActiveStateChangedLocked(const int64_t& eventTimeNs) override; |
| 118 | |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 119 | // Internal interface to handle sliced condition change. |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 120 | void onSlicedConditionMayChangeLocked(bool overallCondition, const int64_t eventTime) override; |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 121 | |
| 122 | // Internal function to calculate the current used bytes. |
| 123 | size_t byteSizeLocked() const override; |
| 124 | |
Yangster-mac | a78d008 | 2018-03-12 12:02:56 -0700 | [diff] [blame] | 125 | void dumpStatesLocked(FILE* out, bool verbose) const override; |
Yao Chen | 884c8c1 | 2018-01-26 10:36:25 -0800 | [diff] [blame] | 126 | |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 127 | void dropDataLocked(const int64_t dropTimeNs) override; |
Yao Chen | 06dba5d | 2018-01-26 13:38:16 -0800 | [diff] [blame] | 128 | |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 129 | // Util function to flush the old packet. |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 130 | void flushIfNeededLocked(const int64_t& eventTime) override; |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 131 | |
Olivier Gaillard | 6c75ecd | 2019-02-20 09:57:33 +0000 | [diff] [blame] | 132 | void flushCurrentBucketLocked(const int64_t& eventTimeNs, |
| 133 | const int64_t& nextBucketStartTimeNs) override; |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 134 | |
Tej Singh | 3be093b | 2020-03-04 20:08:38 -0800 | [diff] [blame] | 135 | void prepareFirstBucketLocked() override; |
| 136 | |
Yangster-mac | 32f07af | 2018-10-13 17:08:11 -0700 | [diff] [blame] | 137 | void pullAndMatchEventsLocked(const int64_t timestampNs); |
| 138 | |
| 139 | const int mWhatMatcherIndex; |
| 140 | |
| 141 | sp<EventMatcherWizard> mEventMatcherWizard; |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 142 | |
Chenjie Yu | e221920 | 2018-06-08 10:07:51 -0700 | [diff] [blame] | 143 | sp<StatsPullerManager> mPullerManager; |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 144 | // tagId for pulled data. -1 if this is not pulled |
| 145 | const int mPullTagId; |
| 146 | |
Chenjie Yu | 8858897 | 2018-08-03 09:49:22 -0700 | [diff] [blame] | 147 | // tagId for atoms that trigger the pulling, if any |
| 148 | const int mTriggerAtomId; |
| 149 | |
| 150 | // tagId for output atom |
| 151 | const int mAtomId; |
| 152 | |
Chenjie Yu | e1361ed | 2018-07-23 17:33:09 -0700 | [diff] [blame] | 153 | // if this is pulled metric |
| 154 | const bool mIsPulled; |
| 155 | |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 156 | // Save the past buckets and we can clear when the StatsLogReport is dumped. |
Yangster-mac | 9369446 | 2018-01-22 20:49:31 -0800 | [diff] [blame] | 157 | std::unordered_map<MetricDimensionKey, std::vector<GaugeBucket>> mPastBuckets; |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 158 | |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 159 | // The current partial bucket. |
Yangster-mac | 34ea110 | 2018-01-29 12:40:55 -0800 | [diff] [blame] | 160 | std::shared_ptr<DimToGaugeAtomsMap> mCurrentSlicedBucket; |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 161 | |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 162 | // The current full bucket for anomaly detection. This is updated to the latest value seen for |
| 163 | // this slice (ie, for partial buckets, we use the last partial bucket in this full bucket). |
Yangster-mac | 2087716 | 2017-12-22 17:19:39 -0800 | [diff] [blame] | 164 | std::shared_ptr<DimToValMap> mCurrentSlicedBucketForAnomaly; |
Chenjie Yu | d9dfda7 | 2017-12-11 17:41:20 -0800 | [diff] [blame] | 165 | |
David Chen | 81245fd | 2018-04-12 14:33:37 -0700 | [diff] [blame] | 166 | const int64_t mMinBucketSizeNs; |
| 167 | |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 168 | // Translate Atom based bucket to single numeric value bucket for anomaly and updates the map |
| 169 | // for each slice with the latest value. |
Chenjie Yu | d9dfda7 | 2017-12-11 17:41:20 -0800 | [diff] [blame] | 170 | void updateCurrentSlicedBucketForAnomaly(); |
| 171 | |
Chenjie Yu | d9dfda7 | 2017-12-11 17:41:20 -0800 | [diff] [blame] | 172 | // Whitelist of fields to report. Empty means all are reported. |
Yao Chen | 8a8d16c | 2018-02-08 14:50:40 -0800 | [diff] [blame] | 173 | std::vector<Matcher> mFieldMatchers; |
Chenjie Yu | d9dfda7 | 2017-12-11 17:41:20 -0800 | [diff] [blame] | 174 | |
Yangster-mac | 34ea110 | 2018-01-29 12:40:55 -0800 | [diff] [blame] | 175 | GaugeMetric::SamplingType mSamplingType; |
| 176 | |
Chenjie Yu | 0bd73db | 2018-12-16 07:37:04 -0800 | [diff] [blame] | 177 | const int64_t mMaxPullDelayNs; |
| 178 | |
Chenjie Yu | d9dfda7 | 2017-12-11 17:41:20 -0800 | [diff] [blame] | 179 | // apply a whitelist on the original input |
Yao Chen | 8a8d16c | 2018-02-08 14:50:40 -0800 | [diff] [blame] | 180 | std::shared_ptr<vector<FieldValue>> getGaugeFields(const LogEvent& event); |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 181 | |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 182 | // Util function to check whether the specified dimension hits the guardrail. |
Yangster-mac | 9369446 | 2018-01-22 20:49:31 -0800 | [diff] [blame] | 183 | bool hitGuardRailLocked(const MetricDimensionKey& newKey); |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 184 | |
Yangster-mac | e2cd6d5 | 2017-11-09 20:38:30 -0800 | [diff] [blame] | 185 | static const size_t kBucketSize = sizeof(GaugeBucket{}); |
yro | 2b0f886 | 2017-11-06 14:27:31 -0800 | [diff] [blame] | 186 | |
Chenjie Yu | c587505 | 2018-03-09 10:13:11 -0800 | [diff] [blame] | 187 | const size_t mDimensionSoftLimit; |
| 188 | |
| 189 | const size_t mDimensionHardLimit; |
| 190 | |
Yangster-mac | 50b0c9a | 2018-05-10 17:13:12 -0700 | [diff] [blame] | 191 | const size_t mGaugeAtomsPerDimensionLimit; |
| 192 | |
Chenjie Yu | cd1b797 | 2019-01-16 20:38:15 -0800 | [diff] [blame] | 193 | const bool mSplitBucketForAppUpgrade; |
| 194 | |
Chenjie Yu | e1361ed | 2018-07-23 17:33:09 -0700 | [diff] [blame] | 195 | FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsWithCondition); |
| 196 | FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsWithSlicedCondition); |
| 197 | FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsNoCondition); |
Chenjie Yu | cd1b797 | 2019-01-16 20:38:15 -0800 | [diff] [blame] | 198 | FRIEND_TEST(GaugeMetricProducerTest, TestPulledWithAppUpgradeDisabled); |
Chenjie Yu | e1361ed | 2018-07-23 17:33:09 -0700 | [diff] [blame] | 199 | FRIEND_TEST(GaugeMetricProducerTest, TestPulledEventsAnomalyDetection); |
| 200 | FRIEND_TEST(GaugeMetricProducerTest, TestFirstBucket); |
Chenjie Yu | 8858897 | 2018-08-03 09:49:22 -0700 | [diff] [blame] | 201 | FRIEND_TEST(GaugeMetricProducerTest, TestPullOnTrigger); |
Chenjie Yu | 4c31f67 | 2018-08-21 15:42:40 -0700 | [diff] [blame] | 202 | FRIEND_TEST(GaugeMetricProducerTest, TestRemoveDimensionInOutput); |
Tej Singh | e678cb7 | 2020-04-14 16:23:30 -0700 | [diff] [blame] | 203 | |
| 204 | FRIEND_TEST(GaugeMetricProducerTest_PartialBucket, TestPushedEvents); |
| 205 | FRIEND_TEST(GaugeMetricProducerTest_PartialBucket, TestPulled); |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 206 | }; |
| 207 | |
| 208 | } // namespace statsd |
| 209 | } // namespace os |
| 210 | } // namespace android |