blob: 64344e837a51ae028751cc60f341c391889ee709 [file] [log] [blame]
Yangster1d4d6862017-10-31 12:58:51 -07001/*
2* Copyright (C) 2017 The Android Open Source Project
3*
4* Licensed under the Apache License, Version 2.0 (the "License");
5* you may not use this file except in compliance with the License.
6* You may obtain a copy of the License at
7*
8* http://www.apache.org/licenses/LICENSE-2.0
9*
10* Unless required by applicable law or agreed to in writing, software
11* distributed under the License is distributed on an "AS IS" BASIS,
12* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13* See the License for the specific language governing permissions and
14* limitations under the License.
15*/
16
Yao Chen3c0b95c2017-12-16 14:34:20 -080017#define DEBUG false // STOPSHIP if true
Yangster1d4d6862017-10-31 12:58:51 -070018#include "Log.h"
19
Chenjie Yuc5875052018-03-09 10:13:11 -080020#include "../guardrail/StatsdStats.h"
Yangster1d4d6862017-10-31 12:58:51 -070021#include "GaugeMetricProducer.h"
Chenjie Yuc5875052018-03-09 10:13:11 -080022#include "../stats_log_util.h"
Yangster1d4d6862017-10-31 12:58:51 -070023
24#include <cutils/log.h>
Yangster1d4d6862017-10-31 12:58:51 -070025
yrob0378b02017-11-09 20:36:25 -080026using android::util::FIELD_COUNT_REPEATED;
yro2b0f8862017-11-06 14:27:31 -080027using android::util::FIELD_TYPE_BOOL;
28using android::util::FIELD_TYPE_FLOAT;
29using android::util::FIELD_TYPE_INT32;
30using android::util::FIELD_TYPE_INT64;
31using android::util::FIELD_TYPE_MESSAGE;
Yangster-macd1815dc2017-11-13 21:43:15 -080032using android::util::FIELD_TYPE_STRING;
yro2b0f8862017-11-06 14:27:31 -080033using android::util::ProtoOutputStream;
Yangster1d4d6862017-10-31 12:58:51 -070034using std::map;
35using std::string;
36using std::unordered_map;
37using std::vector;
Chenjie Yud9dfda72017-12-11 17:41:20 -080038using std::make_shared;
39using std::shared_ptr;
Yangster1d4d6862017-10-31 12:58:51 -070040
41namespace android {
42namespace os {
43namespace statsd {
44
yro2b0f8862017-11-06 14:27:31 -080045// for StatsLogReport
Yangster-mac94e197c2018-01-02 16:03:03 -080046const int FIELD_ID_ID = 1;
yro2b0f8862017-11-06 14:27:31 -080047const int FIELD_ID_GAUGE_METRICS = 8;
Yangster-mac9def8e32018-04-17 13:55:51 -070048const int FIELD_ID_TIME_BASE = 9;
49const int FIELD_ID_BUCKET_SIZE = 10;
50const int FIELD_ID_DIMENSION_PATH_IN_WHAT = 11;
Howard Ro9440e092018-12-16 19:15:21 -080051const int FIELD_ID_IS_ACTIVE = 14;
yro2b0f8862017-11-06 14:27:31 -080052// for GaugeMetricDataWrapper
53const int FIELD_ID_DATA = 1;
David Chen81245fd2018-04-12 14:33:37 -070054const int FIELD_ID_SKIPPED = 2;
Yangster-mac9def8e32018-04-17 13:55:51 -070055const int FIELD_ID_SKIPPED_START_MILLIS = 3;
56const int FIELD_ID_SKIPPED_END_MILLIS = 4;
yro2b0f8862017-11-06 14:27:31 -080057// for GaugeMetricData
Yangster-mac468ff042018-01-17 12:26:34 -080058const int FIELD_ID_DIMENSION_IN_WHAT = 1;
Yangster-mac468ff042018-01-17 12:26:34 -080059const int FIELD_ID_BUCKET_INFO = 3;
Yangster-mac9def8e32018-04-17 13:55:51 -070060const int FIELD_ID_DIMENSION_LEAF_IN_WHAT = 4;
yro2b0f8862017-11-06 14:27:31 -080061// for GaugeBucketInfo
Chenjie Yud9dfda72017-12-11 17:41:20 -080062const int FIELD_ID_ATOM = 3;
Yangster-mac330af582018-02-08 15:24:38 -080063const int FIELD_ID_ELAPSED_ATOM_TIMESTAMP = 4;
Yangster-mac9def8e32018-04-17 13:55:51 -070064const int FIELD_ID_BUCKET_NUM = 6;
65const int FIELD_ID_START_BUCKET_ELAPSED_MILLIS = 7;
66const int FIELD_ID_END_BUCKET_ELAPSED_MILLIS = 8;
yro2b0f8862017-11-06 14:27:31 -080067
Chenjie Yu0bd73db2018-12-16 07:37:04 -080068GaugeMetricProducer::GaugeMetricProducer(
69 const ConfigKey& key, const GaugeMetric& metric, const int conditionIndex,
70 const sp<ConditionWizard>& wizard, const int whatMatcherIndex,
71 const sp<EventMatcherWizard>& matcherWizard, const int pullTagId, const int triggerAtomId,
72 const int atomId, const int64_t timeBaseNs, const int64_t startTimeNs,
Ruchir Rastogi21a287b2019-10-02 12:04:33 -070073 const sp<StatsPullerManager>& pullerManager,
74 const unordered_map<int, shared_ptr<Activation>>& eventActivationMap,
tsaichristined21aacf2019-10-07 14:47:38 -070075 const unordered_map<int, vector<shared_ptr<Activation>>>& eventDeactivationMap,
76 const vector<int>& slicedStateAtoms,
77 const unordered_map<int, unordered_map<int, int64_t>>& stateGroupMap)
Ruchir Rastogi21a287b2019-10-02 12:04:33 -070078 : MetricProducer(metric.id(), key, timeBaseNs, conditionIndex, wizard, eventActivationMap,
tsaichristined21aacf2019-10-07 14:47:38 -070079 eventDeactivationMap, slicedStateAtoms, stateGroupMap),
Yangster-mac32f07af2018-10-13 17:08:11 -070080 mWhatMatcherIndex(whatMatcherIndex),
81 mEventMatcherWizard(matcherWizard),
Chenjie Yue2219202018-06-08 10:07:51 -070082 mPullerManager(pullerManager),
Chenjie Yuc5875052018-03-09 10:13:11 -080083 mPullTagId(pullTagId),
Chenjie Yu88588972018-08-03 09:49:22 -070084 mTriggerAtomId(triggerAtomId),
85 mAtomId(atomId),
Chenjie Yue1361ed2018-07-23 17:33:09 -070086 mIsPulled(pullTagId != -1),
David Chen81245fd2018-04-12 14:33:37 -070087 mMinBucketSizeNs(metric.min_bucket_size_nanos()),
Chenjie Yu0bd73db2018-12-16 07:37:04 -080088 mMaxPullDelayNs(metric.max_pull_delay_sec() > 0 ? metric.max_pull_delay_sec() * NS_PER_SEC
89 : StatsdStats::kPullMaxDelayNs),
Chenjie Yuc5875052018-03-09 10:13:11 -080090 mDimensionSoftLimit(StatsdStats::kAtomDimensionKeySizeLimitMap.find(pullTagId) !=
91 StatsdStats::kAtomDimensionKeySizeLimitMap.end()
92 ? StatsdStats::kAtomDimensionKeySizeLimitMap.at(pullTagId).first
93 : StatsdStats::kDimensionKeySizeSoftLimit),
94 mDimensionHardLimit(StatsdStats::kAtomDimensionKeySizeLimitMap.find(pullTagId) !=
95 StatsdStats::kAtomDimensionKeySizeLimitMap.end()
96 ? StatsdStats::kAtomDimensionKeySizeLimitMap.at(pullTagId).second
Yangster-mac50b0c9a2018-05-10 17:13:12 -070097 : StatsdStats::kDimensionKeySizeHardLimit),
Chenjie Yucd1b7972019-01-16 20:38:15 -080098 mGaugeAtomsPerDimensionLimit(metric.max_num_gauge_atoms_per_bucket()),
99 mSplitBucketForAppUpgrade(metric.split_bucket_for_app_upgrade()) {
Yangster-mac34ea1102018-01-29 12:40:55 -0800100 mCurrentSlicedBucket = std::make_shared<DimToGaugeAtomsMap>();
Yangster-mac20877162017-12-22 17:19:39 -0800101 mCurrentSlicedBucketForAnomaly = std::make_shared<DimToValMap>();
Yangster-macb8144812018-01-04 10:56:23 -0800102 int64_t bucketSizeMills = 0;
103 if (metric.has_bucket()) {
yro59cc24d2018-02-13 20:17:32 -0800104 bucketSizeMills = TimeUnitToBucketSizeInMillisGuardrailed(key.GetUid(), metric.bucket());
Yangster1d4d6862017-10-31 12:58:51 -0700105 } else {
Yangster-macb8144812018-01-04 10:56:23 -0800106 bucketSizeMills = TimeUnitToBucketSizeInMillis(ONE_HOUR);
Yangster1d4d6862017-10-31 12:58:51 -0700107 }
Yangster-macb8144812018-01-04 10:56:23 -0800108 mBucketSizeNs = bucketSizeMills * 1000000;
Yangster1d4d6862017-10-31 12:58:51 -0700109
Yangster-mac34ea1102018-01-29 12:40:55 -0800110 mSamplingType = metric.sampling_type();
Yao Chen8a8d16c2018-02-08 14:50:40 -0800111 if (!metric.gauge_fields_filter().include_all()) {
112 translateFieldMatcher(metric.gauge_fields_filter().fields(), &mFieldMatchers);
113 }
Chenjie Yud9dfda72017-12-11 17:41:20 -0800114
Yao Chen8a8d16c2018-02-08 14:50:40 -0800115 if (metric.has_dimensions_in_what()) {
116 translateFieldMatcher(metric.dimensions_in_what(), &mDimensionsInWhat);
Yangster13fb7e42018-03-07 17:30:49 -0800117 mContainANYPositionInDimensionsInWhat = HasPositionANY(metric.dimensions_in_what());
Yao Chen8a8d16c2018-02-08 14:50:40 -0800118 }
119
Yangster1d4d6862017-10-31 12:58:51 -0700120 if (metric.links().size() > 0) {
Yao Chen8a8d16c2018-02-08 14:50:40 -0800121 for (const auto& link : metric.links()) {
122 Metric2Condition mc;
123 mc.conditionId = link.condition();
124 translateFieldMatcher(link.fields_in_what(), &mc.metricFields);
125 translateFieldMatcher(link.fields_in_condition(), &mc.conditionFields);
126 mMetric2ConditionLinks.push_back(mc);
127 }
tsaichristine76853372019-08-06 17:17:03 -0700128 mConditionSliced = true;
Yangster1d4d6862017-10-31 12:58:51 -0700129 }
tsaichristine76853372019-08-06 17:17:03 -0700130 mSliceByPositionALL = HasPositionALL(metric.dimensions_in_what());
Yangster1d4d6862017-10-31 12:58:51 -0700131
Yangster-mac15f6bbc2018-04-08 11:52:26 -0700132 flushIfNeededLocked(startTimeNs);
Yangster1d4d6862017-10-31 12:58:51 -0700133 // Kicks off the puller immediately.
Chenjie Yue1361ed2018-07-23 17:33:09 -0700134 if (mIsPulled && mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE) {
Chenjie Yue2219202018-06-08 10:07:51 -0700135 mPullerManager->RegisterReceiver(mPullTagId, this, getCurrentBucketEndTimeNs(),
136 mBucketSizeNs);
Yangster1d4d6862017-10-31 12:58:51 -0700137 }
138
Ruchir Rastogi21a287b2019-10-02 12:04:33 -0700139 // Adjust start for partial first bucket and then pull if needed
Chenjie Yue1361ed2018-07-23 17:33:09 -0700140 mCurrentBucketStartTimeNs = startTimeNs;
Ruchir Rastogi21a287b2019-10-02 12:04:33 -0700141 if (mIsActive && mIsPulled && mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE) {
142 pullAndMatchEventsLocked(mCurrentBucketStartTimeNs);
143 }
Chenjie Yue1361ed2018-07-23 17:33:09 -0700144
Yao Chen427d3722018-03-22 15:21:52 -0700145 VLOG("Gauge metric %lld created. bucket size %lld start_time: %lld sliced %d",
Yangster-mac15f6bbc2018-04-08 11:52:26 -0700146 (long long)metric.id(), (long long)mBucketSizeNs, (long long)mTimeBaseNs,
Yao Chen427d3722018-03-22 15:21:52 -0700147 mConditionSliced);
Yangster1d4d6862017-10-31 12:58:51 -0700148}
149
150GaugeMetricProducer::~GaugeMetricProducer() {
151 VLOG("~GaugeMetricProducer() called");
Chenjie Yue1361ed2018-07-23 17:33:09 -0700152 if (mIsPulled && mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE) {
Chenjie Yue2219202018-06-08 10:07:51 -0700153 mPullerManager->UnRegisterReceiver(mPullTagId, this);
Chenjie Yu032fefc2017-12-01 23:30:59 -0800154 }
Yangster1d4d6862017-10-31 12:58:51 -0700155}
156
Yangster-maca78d0082018-03-12 12:02:56 -0700157void GaugeMetricProducer::dumpStatesLocked(FILE* out, bool verbose) const {
158 if (mCurrentSlicedBucket == nullptr ||
159 mCurrentSlicedBucket->size() == 0) {
160 return;
161 }
162
163 fprintf(out, "GaugeMetric %lld dimension size %lu\n", (long long)mMetricId,
164 (unsigned long)mCurrentSlicedBucket->size());
165 if (verbose) {
166 for (const auto& it : *mCurrentSlicedBucket) {
tsaichristine69000e62019-10-18 17:34:52 -0700167 fprintf(out, "\t(what)%s\t(states)%s %d atoms\n",
168 it.first.getDimensionKeyInWhat().toString().c_str(),
169 it.first.getStateValuesKey().toString().c_str(), (int)it.second.size());
Yangster-maca78d0082018-03-12 12:02:56 -0700170 }
171 }
172}
173
Yangster-maca802d732018-04-24 07:50:38 -0700174void GaugeMetricProducer::clearPastBucketsLocked(const int64_t dumpTimeNs) {
175 flushIfNeededLocked(dumpTimeNs);
176 mPastBuckets.clear();
177 mSkippedBuckets.clear();
178}
179
Yangster-macb142cc82018-03-30 15:22:08 -0700180void GaugeMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs,
Yangster-mace68f3a52018-04-04 00:01:43 -0700181 const bool include_current_partial_bucket,
Bookatzff71cad2018-09-20 17:17:49 -0700182 const bool erase_data,
Olivier Gaillard6c75ecd2019-02-20 09:57:33 +0000183 const DumpLatency dumpLatency,
Yangster-mac9def8e32018-04-17 13:55:51 -0700184 std::set<string> *str_set,
Yao Chen288c6002017-12-12 13:43:18 -0800185 ProtoOutputStream* protoOutput) {
Yao Chen427d3722018-03-22 15:21:52 -0700186 VLOG("Gauge metric %lld report now...", (long long)mMetricId);
Yangster-mace68f3a52018-04-04 00:01:43 -0700187 if (include_current_partial_bucket) {
188 flushLocked(dumpTimeNs);
189 } else {
190 flushIfNeededLocked(dumpTimeNs);
191 }
Yao Chen6a8c7992017-11-29 20:02:07 +0000192
Yang Lub4722912018-11-15 11:02:03 -0800193 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)mMetricId);
Howard Ro07e23ff2018-12-17 17:28:07 -0800194 protoOutput->write(FIELD_TYPE_BOOL | FIELD_ID_IS_ACTIVE, isActiveLocked());
Yang Lub4722912018-11-15 11:02:03 -0800195
Yangster-mac635b4b32018-01-23 20:17:35 -0800196 if (mPastBuckets.empty()) {
197 return;
198 }
Yao Chen288c6002017-12-12 13:43:18 -0800199
Yangster-mac9def8e32018-04-17 13:55:51 -0700200 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_TIME_BASE, (long long)mTimeBaseNs);
201 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_SIZE, (long long)mBucketSizeNs);
202
203 // Fills the dimension path if not slicing by ALL.
204 if (!mSliceByPositionALL) {
205 if (!mDimensionsInWhat.empty()) {
206 uint64_t dimenPathToken = protoOutput->start(
207 FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_WHAT);
208 writeDimensionPathToProto(mDimensionsInWhat, protoOutput);
209 protoOutput->end(dimenPathToken);
210 }
Yangster-mac9def8e32018-04-17 13:55:51 -0700211 }
212
Yi Jin5ee07872018-03-05 18:18:27 -0800213 uint64_t protoToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_GAUGE_METRICS);
Yao Chen6a8c7992017-11-29 20:02:07 +0000214
David Chen81245fd2018-04-12 14:33:37 -0700215 for (const auto& pair : mSkippedBuckets) {
216 uint64_t wrapperToken =
217 protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_SKIPPED);
Yangster-mac9def8e32018-04-17 13:55:51 -0700218 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_SKIPPED_START_MILLIS,
219 (long long)(NanoToMillis(pair.first)));
220 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_SKIPPED_END_MILLIS,
221 (long long)(NanoToMillis(pair.second)));
David Chen81245fd2018-04-12 14:33:37 -0700222 protoOutput->end(wrapperToken);
223 }
David Chen81245fd2018-04-12 14:33:37 -0700224
Yangster1d4d6862017-10-31 12:58:51 -0700225 for (const auto& pair : mPastBuckets) {
Yangster-mac93694462018-01-22 20:49:31 -0800226 const MetricDimensionKey& dimensionKey = pair.first;
Yangster1d4d6862017-10-31 12:58:51 -0700227
Yao Chen427d3722018-03-22 15:21:52 -0700228 VLOG("Gauge dimension key %s", dimensionKey.toString().c_str());
Yi Jin5ee07872018-03-05 18:18:27 -0800229 uint64_t wrapperToken =
Yao Chen288c6002017-12-12 13:43:18 -0800230 protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_DATA);
yro2b0f8862017-11-06 14:27:31 -0800231
Yangster-mac20877162017-12-22 17:19:39 -0800232 // First fill dimension.
Yangster-mac9def8e32018-04-17 13:55:51 -0700233 if (mSliceByPositionALL) {
234 uint64_t dimensionToken = protoOutput->start(
235 FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT);
236 writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), str_set, protoOutput);
237 protoOutput->end(dimensionToken);
Yangster-mac9def8e32018-04-17 13:55:51 -0700238 } else {
239 writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInWhat(),
240 FIELD_ID_DIMENSION_LEAF_IN_WHAT, str_set, protoOutput);
Yangster-mac93694462018-01-22 20:49:31 -0800241 }
242
yro2b0f8862017-11-06 14:27:31 -0800243 // Then fill bucket_info (GaugeBucketInfo).
244 for (const auto& bucket : pair.second) {
Yi Jin5ee07872018-03-05 18:18:27 -0800245 uint64_t bucketInfoToken = protoOutput->start(
Yao Chen288c6002017-12-12 13:43:18 -0800246 FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_BUCKET_INFO);
Yangster-mac9def8e32018-04-17 13:55:51 -0700247
248 if (bucket.mBucketEndNs - bucket.mBucketStartNs != mBucketSizeNs) {
249 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_BUCKET_ELAPSED_MILLIS,
250 (long long)NanoToMillis(bucket.mBucketStartNs));
251 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_END_BUCKET_ELAPSED_MILLIS,
252 (long long)NanoToMillis(bucket.mBucketEndNs));
253 } else {
254 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_NUM,
255 (long long)(getBucketNumFromEndTimeNs(bucket.mBucketEndNs)));
256 }
Yangster-mac34ea1102018-01-29 12:40:55 -0800257
258 if (!bucket.mGaugeAtoms.empty()) {
Yangster-mac34ea1102018-01-29 12:40:55 -0800259 for (const auto& atom : bucket.mGaugeAtoms) {
Yangster-mac3fa5d7f2018-03-10 21:50:27 -0800260 uint64_t atomsToken =
261 protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |
262 FIELD_ID_ATOM);
Chenjie Yu88588972018-08-03 09:49:22 -0700263 writeFieldValueTreeToStream(mAtomId, *(atom.mFields), protoOutput);
Yangster-mac3fa5d7f2018-03-10 21:50:27 -0800264 protoOutput->end(atomsToken);
Yangster-mac34ea1102018-01-29 12:40:55 -0800265 }
Yangster-mac34ea1102018-01-29 12:40:55 -0800266 for (const auto& atom : bucket.mGaugeAtoms) {
Tej Singh29ac6012019-06-05 18:10:12 -0700267 const int64_t elapsedTimestampNs =
268 truncateTimestampIfNecessary(mAtomId, atom.mElapsedTimestamps);
Yangster-mac330af582018-02-08 15:24:38 -0800269 protoOutput->write(
270 FIELD_TYPE_INT64 | FIELD_COUNT_REPEATED | FIELD_ID_ELAPSED_ATOM_TIMESTAMP,
Yangster-mac15f6bbc2018-04-08 11:52:26 -0700271 (long long)elapsedTimestampNs);
Yangster-mac34ea1102018-01-29 12:40:55 -0800272 }
273 }
Yao Chen288c6002017-12-12 13:43:18 -0800274 protoOutput->end(bucketInfoToken);
Yao Chen427d3722018-03-22 15:21:52 -0700275 VLOG("Gauge \t bucket [%lld - %lld] includes %d atoms.",
276 (long long)bucket.mBucketStartNs, (long long)bucket.mBucketEndNs,
277 (int)bucket.mGaugeAtoms.size());
yro2b0f8862017-11-06 14:27:31 -0800278 }
Yao Chen288c6002017-12-12 13:43:18 -0800279 protoOutput->end(wrapperToken);
Yangster1d4d6862017-10-31 12:58:51 -0700280 }
Yao Chen288c6002017-12-12 13:43:18 -0800281 protoOutput->end(protoToken);
yro2b0f8862017-11-06 14:27:31 -0800282
Bookatzff71cad2018-09-20 17:17:49 -0700283
284 if (erase_data) {
285 mPastBuckets.clear();
286 mSkippedBuckets.clear();
287 }
Yangster1d4d6862017-10-31 12:58:51 -0700288}
289
Tej Singh597c7162019-04-17 16:41:45 -0700290
Yangster-mac32f07af2018-10-13 17:08:11 -0700291void GaugeMetricProducer::pullAndMatchEventsLocked(const int64_t timestampNs) {
Yangster-mac34ea1102018-01-29 12:40:55 -0800292 bool triggerPuller = false;
293 switch(mSamplingType) {
294 // When the metric wants to do random sampling and there is already one gauge atom for the
295 // current bucket, do not do it again.
296 case GaugeMetric::RANDOM_ONE_SAMPLE: {
Olivier Gaillarde63d9e02019-02-12 14:43:59 +0000297 triggerPuller = mCondition == ConditionState::kTrue && mCurrentSlicedBucket->empty();
Yangster-mac34ea1102018-01-29 12:40:55 -0800298 break;
299 }
Chenjie Yue077fd22018-11-19 13:29:40 -0800300 case GaugeMetric::CONDITION_CHANGE_TO_TRUE: {
Olivier Gaillarde63d9e02019-02-12 14:43:59 +0000301 triggerPuller = mCondition == ConditionState::kTrue;
Yangster-mac34ea1102018-01-29 12:40:55 -0800302 break;
303 }
Chenjie Yue077fd22018-11-19 13:29:40 -0800304 case GaugeMetric::FIRST_N_SAMPLES: {
Olivier Gaillarde63d9e02019-02-12 14:43:59 +0000305 triggerPuller = mCondition == ConditionState::kTrue;
Yangsterec3c7a32018-05-09 15:51:07 -0700306 break;
307 }
Yangster-mac34ea1102018-01-29 12:40:55 -0800308 default:
309 break;
310 }
311 if (!triggerPuller) {
Yao Chen6a8c7992017-11-29 20:02:07 +0000312 return;
313 }
Yao Chen6a8c7992017-11-29 20:02:07 +0000314 vector<std::shared_ptr<LogEvent>> allData;
Chenjie Yu0bd73db2018-12-16 07:37:04 -0800315 if (!mPullerManager->Pull(mPullTagId, &allData)) {
Chenjie Yue1361ed2018-07-23 17:33:09 -0700316 ALOGE("Gauge Stats puller failed for tag: %d at %lld", mPullTagId, (long long)timestampNs);
Yao Chen6a8c7992017-11-29 20:02:07 +0000317 return;
318 }
Chenjie Yu0bd73db2018-12-16 07:37:04 -0800319 const int64_t pullDelayNs = getElapsedRealtimeNs() - timestampNs;
320 if (pullDelayNs > mMaxPullDelayNs) {
321 ALOGE("Pull finish too late for atom %d", mPullTagId);
322 StatsdStats::getInstance().notePullExceedMaxDelay(mPullTagId);
323 StatsdStats::getInstance().notePullDelay(mPullTagId, pullDelayNs);
324 return;
325 }
326 StatsdStats::getInstance().notePullDelay(mPullTagId, pullDelayNs);
Yangster1d4d6862017-10-31 12:58:51 -0700327 for (const auto& data : allData) {
Chenjie Yu0bd73db2018-12-16 07:37:04 -0800328 LogEvent localCopy = data->makeCopy();
329 localCopy.setElapsedTimestampNs(timestampNs);
330 if (mEventMatcherWizard->matchLogEvent(localCopy, mWhatMatcherIndex) ==
331 MatchingState::kMatched) {
332 onMatchedLogEventLocked(mWhatMatcherIndex, localCopy);
Yangster-mac32f07af2018-10-13 17:08:11 -0700333 }
Yangster1d4d6862017-10-31 12:58:51 -0700334 }
Yangster1d4d6862017-10-31 12:58:51 -0700335}
336
Muhammad Qureshi18e46922019-05-24 16:38:49 -0700337void GaugeMetricProducer::onActiveStateChangedLocked(const int64_t& eventTimeNs) {
338 MetricProducer::onActiveStateChangedLocked(eventTimeNs);
339 if (ConditionState::kTrue != mCondition || !mIsPulled) {
340 return;
341 }
342 if (mTriggerAtomId == -1 || (mIsActive && mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE)) {
343 pullAndMatchEventsLocked(eventTimeNs);
344 }
345
346}
347
Yao Chen427d3722018-03-22 15:21:52 -0700348void GaugeMetricProducer::onConditionChangedLocked(const bool conditionMet,
Yangster-mac15f6bbc2018-04-08 11:52:26 -0700349 const int64_t eventTimeNs) {
Yao Chen427d3722018-03-22 15:21:52 -0700350 VLOG("GaugeMetric %lld onConditionChanged", (long long)mMetricId);
Muhammad Qureshi18e46922019-05-24 16:38:49 -0700351
Olivier Gaillarde63d9e02019-02-12 14:43:59 +0000352 mCondition = conditionMet ? ConditionState::kTrue : ConditionState::kFalse;
Muhammad Qureshi18e46922019-05-24 16:38:49 -0700353 if (!mIsActive) {
354 return;
355 }
356
357 flushIfNeededLocked(eventTimeNs);
Chenjie Yue077fd22018-11-19 13:29:40 -0800358 if (mIsPulled && mTriggerAtomId == -1) {
Yangster-mac32f07af2018-10-13 17:08:11 -0700359 pullAndMatchEventsLocked(eventTimeNs);
Yao Chen427d3722018-03-22 15:21:52 -0700360 } // else: Push mode. No need to proactively pull the gauge data.
361}
362
363void GaugeMetricProducer::onSlicedConditionMayChangeLocked(bool overallCondition,
Yangster-mac15f6bbc2018-04-08 11:52:26 -0700364 const int64_t eventTimeNs) {
Yao Chen427d3722018-03-22 15:21:52 -0700365 VLOG("GaugeMetric %lld onSlicedConditionMayChange overall condition %d", (long long)mMetricId,
366 overallCondition);
Muhammad Qureshi18e46922019-05-24 16:38:49 -0700367 mCondition = overallCondition ? ConditionState::kTrue : ConditionState::kFalse;
368 if (!mIsActive) {
369 return;
370 }
371
Yangster-mac15f6bbc2018-04-08 11:52:26 -0700372 flushIfNeededLocked(eventTimeNs);
Yao Chen427d3722018-03-22 15:21:52 -0700373 // If the condition is sliced, mCondition is true if any of the dimensions is true. And we will
374 // pull for every dimension.
Chenjie Yue077fd22018-11-19 13:29:40 -0800375 if (mIsPulled && mTriggerAtomId == -1) {
Yangster-mac32f07af2018-10-13 17:08:11 -0700376 pullAndMatchEventsLocked(eventTimeNs);
Yao Chen427d3722018-03-22 15:21:52 -0700377 } // else: Push mode. No need to proactively pull the gauge data.
Yangster1d4d6862017-10-31 12:58:51 -0700378}
379
Yao Chen8a8d16c2018-02-08 14:50:40 -0800380std::shared_ptr<vector<FieldValue>> GaugeMetricProducer::getGaugeFields(const LogEvent& event) {
Chenjie Yu4c31f672018-08-21 15:42:40 -0700381 std::shared_ptr<vector<FieldValue>> gaugeFields;
Yao Chen8a8d16c2018-02-08 14:50:40 -0800382 if (mFieldMatchers.size() > 0) {
Chenjie Yu4c31f672018-08-21 15:42:40 -0700383 gaugeFields = std::make_shared<vector<FieldValue>>();
Yao Chen8a8d16c2018-02-08 14:50:40 -0800384 filterGaugeValues(mFieldMatchers, event.getValues(), gaugeFields.get());
Yao Chen8a8d16c2018-02-08 14:50:40 -0800385 } else {
Chenjie Yu4c31f672018-08-21 15:42:40 -0700386 gaugeFields = std::make_shared<vector<FieldValue>>(event.getValues());
Yangster1d4d6862017-10-31 12:58:51 -0700387 }
Chenjie Yu4c31f672018-08-21 15:42:40 -0700388 // Trim all dimension fields from output. Dimensions will appear in output report and will
389 // benefit from dictionary encoding. For large pulled atoms, this can give the benefit of
390 // optional repeated field.
391 for (const auto& field : mDimensionsInWhat) {
392 for (auto it = gaugeFields->begin(); it != gaugeFields->end();) {
393 if (it->mField.matches(field)) {
394 it = gaugeFields->erase(it);
395 } else {
396 it++;
397 }
398 }
399 }
400 return gaugeFields;
Yangster1d4d6862017-10-31 12:58:51 -0700401}
402
Olivier Gaillardc5f11c42019-02-05 12:44:58 +0000403void GaugeMetricProducer::onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& allData,
Olivier Gaillard11203df2019-02-06 13:18:09 +0000404 bool pullSuccess, int64_t originalPullTimeNs) {
Yangsterf2bee6f2017-11-29 12:01:05 -0800405 std::lock_guard<std::mutex> lock(mMutex);
Olivier Gaillardc5f11c42019-02-05 12:44:58 +0000406 if (!pullSuccess || allData.size() == 0) {
Chenjie Yud9dfda72017-12-11 17:41:20 -0800407 return;
408 }
Yangster1d4d6862017-10-31 12:58:51 -0700409 for (const auto& data : allData) {
Yangster-mac32f07af2018-10-13 17:08:11 -0700410 if (mEventMatcherWizard->matchLogEvent(
411 *data, mWhatMatcherIndex) == MatchingState::kMatched) {
412 onMatchedLogEventLocked(mWhatMatcherIndex, *data);
413 }
Yangster1d4d6862017-10-31 12:58:51 -0700414 }
Yangster1d4d6862017-10-31 12:58:51 -0700415}
416
Yangster-mac93694462018-01-22 20:49:31 -0800417bool GaugeMetricProducer::hitGuardRailLocked(const MetricDimensionKey& newKey) {
Yao Chenb3561512017-11-21 18:07:17 -0800418 if (mCurrentSlicedBucket->find(newKey) != mCurrentSlicedBucket->end()) {
419 return false;
420 }
421 // 1. Report the tuple count if the tuple count > soft limit
Chenjie Yuc5875052018-03-09 10:13:11 -0800422 if (mCurrentSlicedBucket->size() > mDimensionSoftLimit - 1) {
Yao Chenb3561512017-11-21 18:07:17 -0800423 size_t newTupleCount = mCurrentSlicedBucket->size() + 1;
Yangster-mac94e197c2018-01-02 16:03:03 -0800424 StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetricId, newTupleCount);
Yao Chenb3561512017-11-21 18:07:17 -0800425 // 2. Don't add more tuples, we are above the allowed threshold. Drop the data.
Chenjie Yuc5875052018-03-09 10:13:11 -0800426 if (newTupleCount > mDimensionHardLimit) {
Yangster-mac94e197c2018-01-02 16:03:03 -0800427 ALOGE("GaugeMetric %lld dropping data for dimension key %s",
Yangster13fb7e42018-03-07 17:30:49 -0800428 (long long)mMetricId, newKey.toString().c_str());
Muhammad Qureshi87348a62019-02-14 16:17:52 -0800429 StatsdStats::getInstance().noteHardDimensionLimitReached(mMetricId);
Yao Chenb3561512017-11-21 18:07:17 -0800430 return true;
431 }
432 }
433
434 return false;
435}
436
Yangsterf2bee6f2017-11-29 12:01:05 -0800437void GaugeMetricProducer::onMatchedLogEventInternalLocked(
Yangster-mac93694462018-01-22 20:49:31 -0800438 const size_t matcherIndex, const MetricDimensionKey& eventKey,
Yangster-mac20877162017-12-22 17:19:39 -0800439 const ConditionKey& conditionKey, bool condition,
Chenjie Yua7259ab2017-12-10 08:31:05 -0800440 const LogEvent& event) {
Yangster1d4d6862017-10-31 12:58:51 -0700441 if (condition == false) {
442 return;
443 }
Yangster-macb142cc82018-03-30 15:22:08 -0700444 int64_t eventTimeNs = event.GetElapsedTimestampNs();
Yangster1d4d6862017-10-31 12:58:51 -0700445 if (eventTimeNs < mCurrentBucketStartTimeNs) {
Yao Chen427d3722018-03-22 15:21:52 -0700446 VLOG("Gauge Skip event due to late arrival: %lld vs %lld", (long long)eventTimeNs,
Yangster1d4d6862017-10-31 12:58:51 -0700447 (long long)mCurrentBucketStartTimeNs);
448 return;
449 }
Chenjie Yud9dfda72017-12-11 17:41:20 -0800450 flushIfNeededLocked(eventTimeNs);
Yao Chen6a8c7992017-11-29 20:02:07 +0000451
Chenjie Yu88588972018-08-03 09:49:22 -0700452 if (mTriggerAtomId == event.GetTagId()) {
Yangster-mac32f07af2018-10-13 17:08:11 -0700453 pullAndMatchEventsLocked(eventTimeNs);
Chenjie Yu88588972018-08-03 09:49:22 -0700454 return;
455 }
456
Yangster-mac34ea1102018-01-29 12:40:55 -0800457 // When gauge metric wants to randomly sample the output atom, we just simply use the first
458 // gauge in the given bucket.
459 if (mCurrentSlicedBucket->find(eventKey) != mCurrentSlicedBucket->end() &&
460 mSamplingType == GaugeMetric::RANDOM_ONE_SAMPLE) {
Yangster1d4d6862017-10-31 12:58:51 -0700461 return;
462 }
Chenjie Yud9dfda72017-12-11 17:41:20 -0800463 if (hitGuardRailLocked(eventKey)) {
464 return;
Yao Chen6a8c7992017-11-29 20:02:07 +0000465 }
Yangster-mac50b0c9a2018-05-10 17:13:12 -0700466 if ((*mCurrentSlicedBucket)[eventKey].size() >= mGaugeAtomsPerDimensionLimit) {
467 return;
468 }
Bookatzfe2dde82018-08-28 13:24:40 -0700469 GaugeAtom gaugeAtom(getGaugeFields(event), eventTimeNs);
Yangster-mac34ea1102018-01-29 12:40:55 -0800470 (*mCurrentSlicedBucket)[eventKey].push_back(gaugeAtom);
Chenjie Yud9dfda72017-12-11 17:41:20 -0800471 // Anomaly detection on gauge metric only works when there is one numeric
472 // field specified.
473 if (mAnomalyTrackers.size() > 0) {
Yangster-mac34ea1102018-01-29 12:40:55 -0800474 if (gaugeAtom.mFields->size() == 1) {
Yao Chen8a8d16c2018-02-08 14:50:40 -0800475 const Value& value = gaugeAtom.mFields->begin()->mValue;
Chenjie Yud9dfda72017-12-11 17:41:20 -0800476 long gaugeVal = 0;
Yao Chen8a8d16c2018-02-08 14:50:40 -0800477 if (value.getType() == INT) {
478 gaugeVal = (long)value.int_value;
479 } else if (value.getType() == LONG) {
480 gaugeVal = value.long_value;
Chenjie Yud9dfda72017-12-11 17:41:20 -0800481 }
482 for (auto& tracker : mAnomalyTrackers) {
Yao Chen4ce07292019-02-13 13:06:36 -0800483 tracker->detectAndDeclareAnomaly(eventTimeNs, mCurrentBucketNum, mMetricId,
484 eventKey, gaugeVal);
Chenjie Yud9dfda72017-12-11 17:41:20 -0800485 }
486 }
487 }
488}
489
490void GaugeMetricProducer::updateCurrentSlicedBucketForAnomaly() {
Chenjie Yud9dfda72017-12-11 17:41:20 -0800491 for (const auto& slice : *mCurrentSlicedBucket) {
Yao Chen8a8d16c2018-02-08 14:50:40 -0800492 if (slice.second.empty()) {
Yangster-mac34ea1102018-01-29 12:40:55 -0800493 continue;
494 }
Yao Chen8a8d16c2018-02-08 14:50:40 -0800495 const Value& value = slice.second.front().mFields->front().mValue;
Chenjie Yud9dfda72017-12-11 17:41:20 -0800496 long gaugeVal = 0;
Yao Chen8a8d16c2018-02-08 14:50:40 -0800497 if (value.getType() == INT) {
498 gaugeVal = (long)value.int_value;
499 } else if (value.getType() == LONG) {
500 gaugeVal = value.long_value;
Chenjie Yud9dfda72017-12-11 17:41:20 -0800501 }
502 (*mCurrentSlicedBucketForAnomaly)[slice.first] = gaugeVal;
Yangster1d4d6862017-10-31 12:58:51 -0700503 }
504}
505
Yangster-macb142cc82018-03-30 15:22:08 -0700506void GaugeMetricProducer::dropDataLocked(const int64_t dropTimeNs) {
Yao Chen06dba5d2018-01-26 13:38:16 -0800507 flushIfNeededLocked(dropTimeNs);
Olivier Gaillard320952b2019-02-06 13:57:24 +0000508 StatsdStats::getInstance().noteBucketDropped(mMetricId);
Yao Chen06dba5d2018-01-26 13:38:16 -0800509 mPastBuckets.clear();
510}
511
Yangster1d4d6862017-10-31 12:58:51 -0700512// When a new matched event comes in, we check if event falls into the current
513// bucket. If not, flush the old counter to past buckets and initialize the new
514// bucket.
515// if data is pushed, onMatchedLogEvent will only be called through onConditionChanged() inside
516// the GaugeMetricProducer while holding the lock.
Yangster-macb142cc82018-03-30 15:22:08 -0700517void GaugeMetricProducer::flushIfNeededLocked(const int64_t& eventTimeNs) {
518 int64_t currentBucketEndTimeNs = getCurrentBucketEndTimeNs();
David Chen27785a82018-01-19 17:06:45 -0800519
520 if (eventTimeNs < currentBucketEndTimeNs) {
Yao Chen427d3722018-03-22 15:21:52 -0700521 VLOG("Gauge eventTime is %lld, less than next bucket start time %lld",
522 (long long)eventTimeNs, (long long)(mCurrentBucketStartTimeNs + mBucketSizeNs));
Yangster1d4d6862017-10-31 12:58:51 -0700523 return;
524 }
525
David Chen27785a82018-01-19 17:06:45 -0800526 // Adjusts the bucket start and end times.
527 int64_t numBucketsForward = 1 + (eventTimeNs - currentBucketEndTimeNs) / mBucketSizeNs;
Muhammad Qureshi902529a2019-03-14 16:03:21 -0700528 int64_t nextBucketNs = currentBucketEndTimeNs + (numBucketsForward - 1) * mBucketSizeNs;
529 flushCurrentBucketLocked(eventTimeNs, nextBucketNs);
530
David Chen27785a82018-01-19 17:06:45 -0800531 mCurrentBucketNum += numBucketsForward;
Yao Chen427d3722018-03-22 15:21:52 -0700532 VLOG("Gauge metric %lld: new bucket start time: %lld", (long long)mMetricId,
David Chen27785a82018-01-19 17:06:45 -0800533 (long long)mCurrentBucketStartTimeNs);
534}
535
Olivier Gaillard6c75ecd2019-02-20 09:57:33 +0000536void GaugeMetricProducer::flushCurrentBucketLocked(const int64_t& eventTimeNs,
537 const int64_t& nextBucketStartTimeNs) {
Yangster-macb142cc82018-03-30 15:22:08 -0700538 int64_t fullBucketEndTimeNs = getCurrentBucketEndTimeNs();
David Chen27785a82018-01-19 17:06:45 -0800539
yro2b0f8862017-11-06 14:27:31 -0800540 GaugeBucket info;
541 info.mBucketStartNs = mCurrentBucketStartTimeNs;
David Chen27785a82018-01-19 17:06:45 -0800542 if (eventTimeNs < fullBucketEndTimeNs) {
543 info.mBucketEndNs = eventTimeNs;
544 } else {
545 info.mBucketEndNs = fullBucketEndTimeNs;
546 }
Yangster1d4d6862017-10-31 12:58:51 -0700547
David Chen81245fd2018-04-12 14:33:37 -0700548 if (info.mBucketEndNs - mCurrentBucketStartTimeNs >= mMinBucketSizeNs) {
549 for (const auto& slice : *mCurrentSlicedBucket) {
550 info.mGaugeAtoms = slice.second;
551 auto& bucketList = mPastBuckets[slice.first];
552 bucketList.push_back(info);
553 VLOG("Gauge gauge metric %lld, dump key value: %s", (long long)mMetricId,
554 slice.first.toString().c_str());
555 }
556 } else {
557 mSkippedBuckets.emplace_back(info.mBucketStartNs, info.mBucketEndNs);
Yangster1d4d6862017-10-31 12:58:51 -0700558 }
Yangster1d4d6862017-10-31 12:58:51 -0700559
David Chen27785a82018-01-19 17:06:45 -0800560 // If we have anomaly trackers, we need to update the partial bucket values.
Chenjie Yud9dfda72017-12-11 17:41:20 -0800561 if (mAnomalyTrackers.size() > 0) {
562 updateCurrentSlicedBucketForAnomaly();
David Chen27785a82018-01-19 17:06:45 -0800563
564 if (eventTimeNs > fullBucketEndTimeNs) {
565 // This is known to be a full bucket, so send this data to the anomaly tracker.
566 for (auto& tracker : mAnomalyTrackers) {
567 tracker->addPastBucket(mCurrentSlicedBucketForAnomaly, mCurrentBucketNum);
568 }
569 mCurrentSlicedBucketForAnomaly = std::make_shared<DimToValMap>();
Chenjie Yud9dfda72017-12-11 17:41:20 -0800570 }
Yangster-mace2cd6d52017-11-09 20:38:30 -0800571 }
572
Olivier Gaillardf248c0d2019-02-21 15:56:58 +0000573 StatsdStats::getInstance().noteBucketCount(mMetricId);
Yangster-mac34ea1102018-01-29 12:40:55 -0800574 mCurrentSlicedBucket = std::make_shared<DimToGaugeAtomsMap>();
Muhammad Qureshi902529a2019-03-14 16:03:21 -0700575 mCurrentBucketStartTimeNs = nextBucketStartTimeNs;
Yangster1d4d6862017-10-31 12:58:51 -0700576}
577
Yangsterf2bee6f2017-11-29 12:01:05 -0800578size_t GaugeMetricProducer::byteSizeLocked() const {
Yangster-mace2cd6d52017-11-09 20:38:30 -0800579 size_t totalSize = 0;
580 for (const auto& pair : mPastBuckets) {
Yangster-macb2532da2018-04-11 13:55:04 -0700581 for (const auto& bucket : pair.second) {
582 totalSize += bucket.mGaugeAtoms.size() * sizeof(GaugeAtom);
583 for (const auto& atom : bucket.mGaugeAtoms) {
584 if (atom.mFields != nullptr) {
585 totalSize += atom.mFields->size() * sizeof(FieldValue);
586 }
587 }
588 }
Yangster-mace2cd6d52017-11-09 20:38:30 -0800589 }
590 return totalSize;
yro2b0f8862017-11-06 14:27:31 -0800591}
592
Yangster1d4d6862017-10-31 12:58:51 -0700593} // namespace statsd
594} // namespace os
595} // namespace android