Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2017 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #pragma once |
| 18 | |
| 19 | #include <unordered_map> |
| 20 | |
yro | 2b0f886 | 2017-11-06 14:27:31 -0800 | [diff] [blame] | 21 | #include <android/util/ProtoOutputStream.h> |
Yangster-mac | e2cd6d5 | 2017-11-09 20:38:30 -0800 | [diff] [blame] | 22 | #include <gtest/gtest_prod.h> |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 23 | #include "../condition/ConditionTracker.h" |
| 24 | #include "../external/PullDataReceiver.h" |
| 25 | #include "../external/StatsPullerManager.h" |
| 26 | #include "../matchers/matcher_util.h" |
| 27 | #include "MetricProducer.h" |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 28 | #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" |
Chenjie Yu | d9dfda7 | 2017-12-11 17:41:20 -0800 | [diff] [blame] | 29 | #include "../stats_util.h" |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 30 | |
| 31 | namespace android { |
| 32 | namespace os { |
| 33 | namespace statsd { |
| 34 | |
Yangster-mac | 34ea110 | 2018-01-29 12:40:55 -0800 | [diff] [blame] | 35 | struct GaugeAtom { |
Yangster-mac | 15f6bbc | 2018-04-08 11:52:26 -0700 | [diff] [blame] | 36 | GaugeAtom(std::shared_ptr<vector<FieldValue>> fields, int64_t elapsedTimeNs, int wallClockNs) |
| 37 | : mFields(fields), mElapsedTimestamps(elapsedTimeNs), mWallClockTimestampNs(wallClockNs) { |
Yao Chen | 8a8d16c | 2018-02-08 14:50:40 -0800 | [diff] [blame] | 38 | } |
| 39 | std::shared_ptr<vector<FieldValue>> mFields; |
Yangster-mac | 15f6bbc | 2018-04-08 11:52:26 -0700 | [diff] [blame] | 40 | int64_t mElapsedTimestamps; |
| 41 | int64_t mWallClockTimestampNs; |
Yangster-mac | 34ea110 | 2018-01-29 12:40:55 -0800 | [diff] [blame] | 42 | }; |
| 43 | |
yro | 2b0f886 | 2017-11-06 14:27:31 -0800 | [diff] [blame] | 44 | struct GaugeBucket { |
| 45 | int64_t mBucketStartNs; |
| 46 | int64_t mBucketEndNs; |
Yangster-mac | 34ea110 | 2018-01-29 12:40:55 -0800 | [diff] [blame] | 47 | std::vector<GaugeAtom> mGaugeAtoms; |
yro | 2b0f886 | 2017-11-06 14:27:31 -0800 | [diff] [blame] | 48 | }; |
| 49 | |
Yangster-mac | 9369446 | 2018-01-22 20:49:31 -0800 | [diff] [blame] | 50 | typedef std::unordered_map<MetricDimensionKey, std::vector<GaugeAtom>> |
Yangster-mac | 34ea110 | 2018-01-29 12:40:55 -0800 | [diff] [blame] | 51 | DimToGaugeAtomsMap; |
Yangster-mac | 2087716 | 2017-12-22 17:19:39 -0800 | [diff] [blame] | 52 | |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 53 | // This gauge metric producer first register the puller to automatically pull the gauge at the |
| 54 | // beginning of each bucket. If the condition is met, insert it to the bucket info. Otherwise |
| 55 | // proactively pull the gauge when the condition is changed to be true. Therefore, the gauge metric |
| 56 | // producer always reports the guage at the earliest time of the bucket when the condition is met. |
| 57 | class GaugeMetricProducer : public virtual MetricProducer, public virtual PullDataReceiver { |
| 58 | public: |
Yangster-mac | 34ea110 | 2018-01-29 12:40:55 -0800 | [diff] [blame] | 59 | GaugeMetricProducer(const ConfigKey& key, const GaugeMetric& gaugeMetric, |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 60 | const int conditionIndex, const sp<ConditionWizard>& wizard, |
Yangster-mac | 15f6bbc | 2018-04-08 11:52:26 -0700 | [diff] [blame] | 61 | const int pullTagId, const int64_t timeBaseNs, const int64_t startTimeNs); |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 62 | |
| 63 | virtual ~GaugeMetricProducer(); |
| 64 | |
| 65 | // Handles when the pulled data arrives. |
| 66 | void onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& data) override; |
| 67 | |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 68 | // GaugeMetric needs to immediately trigger another pull when we create the partial bucket. |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 69 | void notifyAppUpgrade(const int64_t& eventTimeNs, const string& apk, const int uid, |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 70 | const int64_t version) override { |
| 71 | std::lock_guard<std::mutex> lock(mMutex); |
| 72 | |
| 73 | if (eventTimeNs > getCurrentBucketEndTimeNs()) { |
| 74 | // Flush full buckets on the normal path up to the latest bucket boundary. |
| 75 | flushIfNeededLocked(eventTimeNs); |
| 76 | } |
| 77 | flushCurrentBucketLocked(eventTimeNs); |
| 78 | mCurrentBucketStartTimeNs = eventTimeNs; |
| 79 | if (mPullTagId != -1) { |
Yangster-mac | 15f6bbc | 2018-04-08 11:52:26 -0700 | [diff] [blame] | 80 | pullLocked(eventTimeNs); |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 81 | } |
| 82 | }; |
| 83 | |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 84 | protected: |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 85 | void onMatchedLogEventInternalLocked( |
Yangster-mac | 9369446 | 2018-01-22 20:49:31 -0800 | [diff] [blame] | 86 | const size_t matcherIndex, const MetricDimensionKey& eventKey, |
Yangster-mac | 2087716 | 2017-12-22 17:19:39 -0800 | [diff] [blame] | 87 | const ConditionKey& conditionKey, bool condition, |
Chenjie Yu | a7259ab | 2017-12-10 08:31:05 -0800 | [diff] [blame] | 88 | const LogEvent& event) override; |
yro | 2b0f886 | 2017-11-06 14:27:31 -0800 | [diff] [blame] | 89 | |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 90 | private: |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 91 | void onDumpReportLocked(const int64_t dumpTimeNs, |
Yangster-mac | e68f3a5 | 2018-04-04 00:01:43 -0700 | [diff] [blame] | 92 | const bool include_current_partial_bucket, |
Yao Chen | 288c600 | 2017-12-12 13:43:18 -0800 | [diff] [blame] | 93 | android::util::ProtoOutputStream* protoOutput) override; |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 94 | |
Chenjie Yu | d9dfda7 | 2017-12-11 17:41:20 -0800 | [diff] [blame] | 95 | // for testing |
| 96 | GaugeMetricProducer(const ConfigKey& key, const GaugeMetric& gaugeMetric, |
| 97 | const int conditionIndex, const sp<ConditionWizard>& wizard, |
Yangster-mac | 15f6bbc | 2018-04-08 11:52:26 -0700 | [diff] [blame] | 98 | const int pullTagId, |
| 99 | const int64_t timeBaseNs, const int64_t startTimeNs, |
Chenjie Yu | d9dfda7 | 2017-12-11 17:41:20 -0800 | [diff] [blame] | 100 | std::shared_ptr<StatsPullerManager> statsPullerManager); |
| 101 | |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 102 | // Internal interface to handle condition change. |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 103 | void onConditionChangedLocked(const bool conditionMet, const int64_t eventTime) override; |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 104 | |
| 105 | // Internal interface to handle sliced condition change. |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 106 | void onSlicedConditionMayChangeLocked(bool overallCondition, const int64_t eventTime) override; |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 107 | |
| 108 | // Internal function to calculate the current used bytes. |
| 109 | size_t byteSizeLocked() const override; |
| 110 | |
Yangster-mac | a78d008 | 2018-03-12 12:02:56 -0700 | [diff] [blame] | 111 | void dumpStatesLocked(FILE* out, bool verbose) const override; |
Yao Chen | 884c8c1 | 2018-01-26 10:36:25 -0800 | [diff] [blame] | 112 | |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 113 | void dropDataLocked(const int64_t dropTimeNs) override; |
Yao Chen | 06dba5d | 2018-01-26 13:38:16 -0800 | [diff] [blame] | 114 | |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 115 | // Util function to flush the old packet. |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 116 | void flushIfNeededLocked(const int64_t& eventTime) override; |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 117 | |
Yangster-mac | b142cc8 | 2018-03-30 15:22:08 -0700 | [diff] [blame] | 118 | void flushCurrentBucketLocked(const int64_t& eventTimeNs) override; |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 119 | |
Yangster-mac | 15f6bbc | 2018-04-08 11:52:26 -0700 | [diff] [blame] | 120 | void pullLocked(const int64_t timestampNs); |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 121 | |
Yao Chen | 8a8d16c | 2018-02-08 14:50:40 -0800 | [diff] [blame] | 122 | int mTagId; |
| 123 | |
Chenjie Yu | d9dfda7 | 2017-12-11 17:41:20 -0800 | [diff] [blame] | 124 | std::shared_ptr<StatsPullerManager> mStatsPullerManager; |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 125 | // tagId for pulled data. -1 if this is not pulled |
| 126 | const int mPullTagId; |
| 127 | |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 128 | // Save the past buckets and we can clear when the StatsLogReport is dumped. |
yro | 2b0f886 | 2017-11-06 14:27:31 -0800 | [diff] [blame] | 129 | // TODO: Add a lock to mPastBuckets. |
Yangster-mac | 9369446 | 2018-01-22 20:49:31 -0800 | [diff] [blame] | 130 | std::unordered_map<MetricDimensionKey, std::vector<GaugeBucket>> mPastBuckets; |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 131 | |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 132 | // The current partial bucket. |
Yangster-mac | 34ea110 | 2018-01-29 12:40:55 -0800 | [diff] [blame] | 133 | std::shared_ptr<DimToGaugeAtomsMap> mCurrentSlicedBucket; |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 134 | |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 135 | // The current full bucket for anomaly detection. This is updated to the latest value seen for |
| 136 | // this slice (ie, for partial buckets, we use the last partial bucket in this full bucket). |
Yangster-mac | 2087716 | 2017-12-22 17:19:39 -0800 | [diff] [blame] | 137 | std::shared_ptr<DimToValMap> mCurrentSlicedBucketForAnomaly; |
Chenjie Yu | d9dfda7 | 2017-12-11 17:41:20 -0800 | [diff] [blame] | 138 | |
David Chen | 81245fd | 2018-04-12 14:33:37 -0700 | [diff] [blame^] | 139 | // Pairs of (elapsed start, elapsed end) denoting buckets that were skipped. |
| 140 | std::list<std::pair<int64_t, int64_t>> mSkippedBuckets; |
| 141 | |
| 142 | const int64_t mMinBucketSizeNs; |
| 143 | |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 144 | // Translate Atom based bucket to single numeric value bucket for anomaly and updates the map |
| 145 | // for each slice with the latest value. |
Chenjie Yu | d9dfda7 | 2017-12-11 17:41:20 -0800 | [diff] [blame] | 146 | void updateCurrentSlicedBucketForAnomaly(); |
| 147 | |
Chenjie Yu | d9dfda7 | 2017-12-11 17:41:20 -0800 | [diff] [blame] | 148 | // Whitelist of fields to report. Empty means all are reported. |
Yao Chen | 8a8d16c | 2018-02-08 14:50:40 -0800 | [diff] [blame] | 149 | std::vector<Matcher> mFieldMatchers; |
Chenjie Yu | d9dfda7 | 2017-12-11 17:41:20 -0800 | [diff] [blame] | 150 | |
Yangster-mac | 34ea110 | 2018-01-29 12:40:55 -0800 | [diff] [blame] | 151 | GaugeMetric::SamplingType mSamplingType; |
| 152 | |
Chenjie Yu | d9dfda7 | 2017-12-11 17:41:20 -0800 | [diff] [blame] | 153 | // apply a whitelist on the original input |
Yao Chen | 8a8d16c | 2018-02-08 14:50:40 -0800 | [diff] [blame] | 154 | std::shared_ptr<vector<FieldValue>> getGaugeFields(const LogEvent& event); |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 155 | |
Yangster | f2bee6f | 2017-11-29 12:01:05 -0800 | [diff] [blame] | 156 | // Util function to check whether the specified dimension hits the guardrail. |
Yangster-mac | 9369446 | 2018-01-22 20:49:31 -0800 | [diff] [blame] | 157 | bool hitGuardRailLocked(const MetricDimensionKey& newKey); |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 158 | |
Yangster-mac | e2cd6d5 | 2017-11-09 20:38:30 -0800 | [diff] [blame] | 159 | static const size_t kBucketSize = sizeof(GaugeBucket{}); |
yro | 2b0f886 | 2017-11-06 14:27:31 -0800 | [diff] [blame] | 160 | |
Chenjie Yu | c587505 | 2018-03-09 10:13:11 -0800 | [diff] [blame] | 161 | const size_t mDimensionSoftLimit; |
| 162 | |
| 163 | const size_t mDimensionHardLimit; |
| 164 | |
Yangster-mac | e2cd6d5 | 2017-11-09 20:38:30 -0800 | [diff] [blame] | 165 | FRIEND_TEST(GaugeMetricProducerTest, TestWithCondition); |
Yao Chen | 427d372 | 2018-03-22 15:21:52 -0700 | [diff] [blame] | 166 | FRIEND_TEST(GaugeMetricProducerTest, TestWithSlicedCondition); |
Yangster-mac | e2cd6d5 | 2017-11-09 20:38:30 -0800 | [diff] [blame] | 167 | FRIEND_TEST(GaugeMetricProducerTest, TestNoCondition); |
David Chen | 27785a8 | 2018-01-19 17:06:45 -0800 | [diff] [blame] | 168 | FRIEND_TEST(GaugeMetricProducerTest, TestPushedEventsWithUpgrade); |
| 169 | FRIEND_TEST(GaugeMetricProducerTest, TestPulledWithUpgrade); |
Yangster-mac | e2cd6d5 | 2017-11-09 20:38:30 -0800 | [diff] [blame] | 170 | FRIEND_TEST(GaugeMetricProducerTest, TestAnomalyDetection); |
Yangster | 1d4d686 | 2017-10-31 12:58:51 -0700 | [diff] [blame] | 171 | }; |
| 172 | |
| 173 | } // namespace statsd |
| 174 | } // namespace os |
| 175 | } // namespace android |