| /* |
| * Copyright (C) 2017 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| #pragma once |
| |
| #include <gtest/gtest_prod.h> |
| #include <utils/threads.h> |
| #include <list> |
| #include "anomaly/AnomalyTracker.h" |
| #include "condition/ConditionTimer.h" |
| #include "condition/ConditionTracker.h" |
| #include "external/PullDataReceiver.h" |
| #include "external/StatsPullerManager.h" |
| #include "matchers/EventMatcherWizard.h" |
| #include "stats_log_util.h" |
| #include "MetricProducer.h" |
| #include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" |
| |
| namespace android { |
| namespace os { |
| namespace statsd { |
| |
| struct ValueBucket { |
| int64_t mBucketStartNs; |
| int64_t mBucketEndNs; |
| std::vector<int> valueIndex; |
| std::vector<Value> values; |
| // If the metric has no condition, then this field is just wasted. |
| // When we tune statsd memory usage in the future, this is a candidate to optimize. |
| int64_t mConditionTrueNs; |
| }; |
| |
| |
| // Aggregates values within buckets. |
| // |
| // There are different events that might complete a bucket |
| // - a condition change |
| // - an app upgrade |
| // - an alarm set to the end of the bucket |
| class ValueMetricProducer : public virtual MetricProducer, public virtual PullDataReceiver { |
| public: |
| ValueMetricProducer( |
| const ConfigKey& key, const ValueMetric& valueMetric, const int conditionIndex, |
| const sp<ConditionWizard>& conditionWizard, const int whatMatcherIndex, |
| const sp<EventMatcherWizard>& matcherWizard, const int pullTagId, |
| const int64_t timeBaseNs, const int64_t startTimeNs, |
| const sp<StatsPullerManager>& pullerManager, |
| const std::unordered_map<int, std::shared_ptr<Activation>>& eventActivationMap = {}, |
| const std::unordered_map<int, std::vector<std::shared_ptr<Activation>>>& |
| eventDeactivationMap = {}, |
| const vector<int>& slicedStateAtoms = {}, |
| const unordered_map<int, unordered_map<int, int64_t>>& stateGroupMap = {}); |
| |
| virtual ~ValueMetricProducer(); |
| |
| // Process data pulled on bucket boundary. |
| void onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& data, |
| bool pullSuccess, int64_t originalPullTimeNs) override; |
| |
| // ValueMetric needs special logic if it's a pulled atom. |
| void notifyAppUpgrade(const int64_t& eventTimeNs, const string& apk, const int uid, |
| const int64_t version) override { |
| std::lock_guard<std::mutex> lock(mMutex); |
| if (!mSplitBucketForAppUpgrade) { |
| return; |
| } |
| if (mIsPulled && mCondition) { |
| pullAndMatchEventsLocked(eventTimeNs); |
| } |
| flushCurrentBucketLocked(eventTimeNs, eventTimeNs); |
| }; |
| |
| protected: |
| void onMatchedLogEventInternalLocked( |
| const size_t matcherIndex, const MetricDimensionKey& eventKey, |
| const ConditionKey& conditionKey, bool condition, |
| const LogEvent& event) override; |
| |
| private: |
| void onDumpReportLocked(const int64_t dumpTimeNs, |
| const bool include_current_partial_bucket, |
| const bool erase_data, |
| const DumpLatency dumpLatency, |
| std::set<string> *str_set, |
| android::util::ProtoOutputStream* protoOutput) override; |
| void clearPastBucketsLocked(const int64_t dumpTimeNs) override; |
| |
| // Internal interface to handle active state change. |
| void onActiveStateChangedLocked(const int64_t& eventTimeNs) override; |
| |
| // Internal interface to handle condition change. |
| void onConditionChangedLocked(const bool conditionMet, const int64_t eventTime) override; |
| |
| // Internal interface to handle sliced condition change. |
| void onSlicedConditionMayChangeLocked(bool overallCondition, const int64_t eventTime) override; |
| |
| // Internal function to calculate the current used bytes. |
| size_t byteSizeLocked() const override; |
| |
| void dumpStatesLocked(FILE* out, bool verbose) const override; |
| |
| // For pulled metrics, this method should only be called if a pull has be done. Else we will |
| // not have complete data for the bucket. |
| void flushIfNeededLocked(const int64_t& eventTime) override; |
| |
| // For pulled metrics, this method should only be called if a pulled have be done. Else we will |
| // not have complete data for the bucket. |
| void flushCurrentBucketLocked(const int64_t& eventTimeNs, |
| const int64_t& nextBucketStartTimeNs) override; |
| |
| void dropDataLocked(const int64_t dropTimeNs) override; |
| |
| // Calculate previous bucket end time based on current time. |
| int64_t calcPreviousBucketEndTime(const int64_t currentTimeNs); |
| |
| // Calculate how many buckets are present between the current bucket and eventTimeNs. |
| int64_t calcBucketsForwardCount(const int64_t& eventTimeNs) const; |
| |
| // Mark the data as invalid. |
| void invalidateCurrentBucket(); |
| void invalidateCurrentBucketWithoutResetBase(); |
| |
| const int mWhatMatcherIndex; |
| |
| sp<EventMatcherWizard> mEventMatcherWizard; |
| |
| sp<StatsPullerManager> mPullerManager; |
| |
| // Value fields for matching. |
| std::vector<Matcher> mFieldMatchers; |
| |
| // Value fields for matching. |
| std::set<MetricDimensionKey> mMatchedMetricDimensionKeys; |
| |
| // tagId for pulled data. -1 if this is not pulled |
| const int mPullTagId; |
| |
| // if this is pulled metric |
| const bool mIsPulled; |
| |
| // internal state of an ongoing aggregation bucket. |
| typedef struct { |
| // Index in multi value aggregation. |
| int valueIndex; |
| // Holds current base value of the dimension. Take diff and update if necessary. |
| Value base; |
| // Whether there is a base to diff to. |
| bool hasBase; |
| // Current value, depending on the aggregation type. |
| Value value; |
| // Number of samples collected. |
| int sampleSize; |
| // If this dimension has any non-tainted value. If not, don't report the |
| // dimension. |
| bool hasValue = false; |
| // Whether new data is seen in the bucket. |
| bool seenNewData = false; |
| } Interval; |
| |
| std::unordered_map<MetricDimensionKey, std::vector<Interval>> mCurrentSlicedBucket; |
| |
| std::unordered_map<MetricDimensionKey, int64_t> mCurrentFullBucket; |
| |
| // Save the past buckets and we can clear when the StatsLogReport is dumped. |
| std::unordered_map<MetricDimensionKey, std::vector<ValueBucket>> mPastBuckets; |
| |
| // Pairs of (elapsed start, elapsed end) denoting buckets that were skipped. |
| std::list<std::pair<int64_t, int64_t>> mSkippedBuckets; |
| |
| const int64_t mMinBucketSizeNs; |
| |
| // Util function to check whether the specified dimension hits the guardrail. |
| bool hitGuardRailLocked(const MetricDimensionKey& newKey); |
| bool hasReachedGuardRailLimit() const; |
| |
| bool hitFullBucketGuardRailLocked(const MetricDimensionKey& newKey); |
| |
| void pullAndMatchEventsLocked(const int64_t timestampNs); |
| |
| void accumulateEvents(const std::vector<std::shared_ptr<LogEvent>>& allData, |
| int64_t originalPullTimeNs, int64_t eventElapsedTimeNs); |
| |
| ValueBucket buildPartialBucket(int64_t bucketEndTime, |
| const std::vector<Interval>& intervals); |
| void initCurrentSlicedBucket(int64_t nextBucketStartTimeNs); |
| void appendToFullBucket(int64_t eventTimeNs, int64_t fullBucketEndTimeNs); |
| |
| // Reset diff base and mHasGlobalBase |
| void resetBase(); |
| |
| static const size_t kBucketSize = sizeof(ValueBucket{}); |
| |
| const size_t mDimensionSoftLimit; |
| |
| const size_t mDimensionHardLimit; |
| |
| const bool mUseAbsoluteValueOnReset; |
| |
| const ValueMetric::AggregationType mAggregationType; |
| |
| const bool mUseDiff; |
| |
| const ValueMetric::ValueDirection mValueDirection; |
| |
| const bool mSkipZeroDiffOutput; |
| |
| // If true, use a zero value as base to compute the diff. |
| // This is used for new keys which are present in the new data but was not |
| // present in the base data. |
| // The default base will only be used if we have a global base. |
| const bool mUseZeroDefaultBase; |
| |
| // For pulled metrics, this is always set to true whenever a pull succeeds. |
| // It is set to false when a pull fails, or upon condition change to false. |
| // This is used to decide if we have the right base data to compute the |
| // diff against. |
| bool mHasGlobalBase; |
| |
| // Invalid bucket. There was a problem in collecting data in the current bucket so we cannot |
| // trust any of the data in this bucket. |
| // |
| // For instance, one pull failed. |
| bool mCurrentBucketIsInvalid; |
| |
| const int64_t mMaxPullDelayNs; |
| |
| const bool mSplitBucketForAppUpgrade; |
| |
| ConditionTimer mConditionTimer; |
| |
| FRIEND_TEST(ValueMetricProducerTest, TestAnomalyDetection); |
| FRIEND_TEST(ValueMetricProducerTest, TestBaseSetOnConditionChange); |
| FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundariesOnAppUpgrade); |
| FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundariesOnConditionChange); |
| FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundaryNoCondition); |
| FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition); |
| FRIEND_TEST(ValueMetricProducerTest, TestBucketBoundaryWithCondition2); |
| FRIEND_TEST(ValueMetricProducerTest, TestBucketIncludingUnknownConditionIsInvalid); |
| FRIEND_TEST(ValueMetricProducerTest, TestBucketInvalidIfGlobalBaseIsNotSet); |
| FRIEND_TEST(ValueMetricProducerTest, TestCalcPreviousBucketEndTime); |
| FRIEND_TEST(ValueMetricProducerTest, TestDataIsNotUpdatedWhenNoConditionChanged); |
| FRIEND_TEST(ValueMetricProducerTest, TestEmptyDataResetsBase_onBucketBoundary); |
| FRIEND_TEST(ValueMetricProducerTest, TestEmptyDataResetsBase_onConditionChanged); |
| FRIEND_TEST(ValueMetricProducerTest, TestEmptyDataResetsBase_onDataPulled); |
| FRIEND_TEST(ValueMetricProducerTest, TestEventsWithNonSlicedCondition); |
| FRIEND_TEST(ValueMetricProducerTest, TestFirstBucket); |
| FRIEND_TEST(ValueMetricProducerTest, TestFullBucketResetWhenLastBucketInvalid); |
| FRIEND_TEST(ValueMetricProducerTest, TestInvalidBucketWhenGuardRailHit); |
| FRIEND_TEST(ValueMetricProducerTest, TestInvalidBucketWhenInitialPullFailed); |
| FRIEND_TEST(ValueMetricProducerTest, TestInvalidBucketWhenLastPullFailed); |
| FRIEND_TEST(ValueMetricProducerTest, TestInvalidBucketWhenOneConditionFailed); |
| FRIEND_TEST(ValueMetricProducerTest, TestLateOnDataPulledWithDiff); |
| FRIEND_TEST(ValueMetricProducerTest, TestLateOnDataPulledWithoutDiff); |
| FRIEND_TEST(ValueMetricProducerTest, TestPartialBucketCreated); |
| FRIEND_TEST(ValueMetricProducerTest, TestPartialResetOnBucketBoundaries); |
| FRIEND_TEST(ValueMetricProducerTest, TestPulledData_noDiff_bucketBoundaryFalse); |
| FRIEND_TEST(ValueMetricProducerTest, TestPulledData_noDiff_bucketBoundaryTrue); |
| FRIEND_TEST(ValueMetricProducerTest, TestPulledData_noDiff_withFailure); |
| FRIEND_TEST(ValueMetricProducerTest, TestPulledData_noDiff_withMultipleConditionChanges); |
| FRIEND_TEST(ValueMetricProducerTest, TestPulledData_noDiff_withoutCondition); |
| FRIEND_TEST(ValueMetricProducerTest, TestPulledEventsNoCondition); |
| FRIEND_TEST(ValueMetricProducerTest, TestPulledEventsTakeAbsoluteValueOnReset); |
| FRIEND_TEST(ValueMetricProducerTest, TestPulledEventsTakeZeroOnReset); |
| FRIEND_TEST(ValueMetricProducerTest, TestPulledEventsWithFiltering); |
| FRIEND_TEST(ValueMetricProducerTest, TestPulledValueWithUpgrade); |
| FRIEND_TEST(ValueMetricProducerTest, TestPulledValueWithUpgradeWhileConditionFalse); |
| FRIEND_TEST(ValueMetricProducerTest, TestPulledWithAppUpgradeDisabled); |
| FRIEND_TEST(ValueMetricProducerTest, TestPushedAggregateAvg); |
| FRIEND_TEST(ValueMetricProducerTest, TestPushedAggregateMax); |
| FRIEND_TEST(ValueMetricProducerTest, TestPushedAggregateMin); |
| FRIEND_TEST(ValueMetricProducerTest, TestPushedAggregateSum); |
| FRIEND_TEST(ValueMetricProducerTest, TestPushedEventsWithCondition); |
| FRIEND_TEST(ValueMetricProducerTest, TestPushedEventsWithUpgrade); |
| FRIEND_TEST(ValueMetricProducerTest, TestPushedEventsWithoutCondition); |
| FRIEND_TEST(ValueMetricProducerTest, TestResetBaseOnPullDelayExceeded); |
| FRIEND_TEST(ValueMetricProducerTest, TestResetBaseOnPullFailAfterConditionChange); |
| FRIEND_TEST(ValueMetricProducerTest, TestResetBaseOnPullFailAfterConditionChange_EndOfBucket); |
| FRIEND_TEST(ValueMetricProducerTest, TestResetBaseOnPullFailBeforeConditionChange); |
| FRIEND_TEST(ValueMetricProducerTest, TestResetBaseOnPullTooLate); |
| FRIEND_TEST(ValueMetricProducerTest, TestSkipZeroDiffOutput); |
| FRIEND_TEST(ValueMetricProducerTest, TestSkipZeroDiffOutputMultiValue); |
| FRIEND_TEST(ValueMetricProducerTest, TestTrimUnusedDimensionKey); |
| FRIEND_TEST(ValueMetricProducerTest, TestUseZeroDefaultBase); |
| FRIEND_TEST(ValueMetricProducerTest, TestUseZeroDefaultBaseWithPullFailures); |
| friend class ValueMetricProducerTestHelper; |
| }; |
| |
| } // namespace statsd |
| } // namespace os |
| } // namespace android |