blob: 192a54b7e0a38d020aa85cde739139f79f48a996 [file] [log] [blame]
Chenjie Yub3dda412017-10-24 13:41:59 -07001/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
Chenjie Yu88588972018-08-03 09:49:22 -070017#define DEBUG false // STOPSHIP if true
Chenjie Yub3dda412017-10-24 13:41:59 -070018#include "Log.h"
19
20#include "ValueMetricProducer.h"
Chenjie Yuc5875052018-03-09 10:13:11 -080021#include "../guardrail/StatsdStats.h"
22#include "../stats_log_util.h"
Chenjie Yub3dda412017-10-24 13:41:59 -070023
24#include <cutils/log.h>
25#include <limits.h>
26#include <stdlib.h>
27
yrob0378b02017-11-09 20:36:25 -080028using android::util::FIELD_COUNT_REPEATED;
yro2b0f8862017-11-06 14:27:31 -080029using android::util::FIELD_TYPE_BOOL;
Chenjie Yua0f02242018-07-06 16:14:34 -070030using android::util::FIELD_TYPE_DOUBLE;
yro2b0f8862017-11-06 14:27:31 -080031using android::util::FIELD_TYPE_INT32;
32using android::util::FIELD_TYPE_INT64;
33using android::util::FIELD_TYPE_MESSAGE;
Yangster-macd1815dc2017-11-13 21:43:15 -080034using android::util::FIELD_TYPE_STRING;
yro2b0f8862017-11-06 14:27:31 -080035using android::util::ProtoOutputStream;
Chenjie Yub3dda412017-10-24 13:41:59 -070036using std::list;
Chenjie Yu6736c892017-11-09 10:50:09 -080037using std::make_pair;
Chenjie Yub3dda412017-10-24 13:41:59 -070038using std::make_shared;
Yao Chen93fe3a32017-11-02 13:52:59 -070039using std::map;
Chenjie Yub3dda412017-10-24 13:41:59 -070040using std::shared_ptr;
41using std::unique_ptr;
Yao Chen93fe3a32017-11-02 13:52:59 -070042using std::unordered_map;
Chenjie Yub3dda412017-10-24 13:41:59 -070043
44namespace android {
45namespace os {
46namespace statsd {
47
yro2b0f8862017-11-06 14:27:31 -080048// for StatsLogReport
Yangster-mac94e197c2018-01-02 16:03:03 -080049const int FIELD_ID_ID = 1;
yro2b0f8862017-11-06 14:27:31 -080050const int FIELD_ID_VALUE_METRICS = 7;
Yangster-mac9def8e32018-04-17 13:55:51 -070051const int FIELD_ID_TIME_BASE = 9;
52const int FIELD_ID_BUCKET_SIZE = 10;
53const int FIELD_ID_DIMENSION_PATH_IN_WHAT = 11;
54const int FIELD_ID_DIMENSION_PATH_IN_CONDITION = 12;
yro2b0f8862017-11-06 14:27:31 -080055// for ValueMetricDataWrapper
56const int FIELD_ID_DATA = 1;
David Chen81245fd2018-04-12 14:33:37 -070057const int FIELD_ID_SKIPPED = 2;
Yangster-mac9def8e32018-04-17 13:55:51 -070058const int FIELD_ID_SKIPPED_START_MILLIS = 3;
59const int FIELD_ID_SKIPPED_END_MILLIS = 4;
yro2b0f8862017-11-06 14:27:31 -080060// for ValueMetricData
Yangster-mac468ff042018-01-17 12:26:34 -080061const int FIELD_ID_DIMENSION_IN_WHAT = 1;
62const int FIELD_ID_DIMENSION_IN_CONDITION = 2;
63const int FIELD_ID_BUCKET_INFO = 3;
Yangster-mac9def8e32018-04-17 13:55:51 -070064const int FIELD_ID_DIMENSION_LEAF_IN_WHAT = 4;
65const int FIELD_ID_DIMENSION_LEAF_IN_CONDITION = 5;
yro2b0f8862017-11-06 14:27:31 -080066// for ValueBucketInfo
Chenjie Yuc3f00772018-07-26 13:50:58 -070067const int FIELD_ID_VALUE_LONG = 7;
68const int FIELD_ID_VALUE_DOUBLE = 8;
Yangster-mac9def8e32018-04-17 13:55:51 -070069const int FIELD_ID_BUCKET_NUM = 4;
70const int FIELD_ID_START_BUCKET_ELAPSED_MILLIS = 5;
71const int FIELD_ID_END_BUCKET_ELAPSED_MILLIS = 6;
yro2b0f8862017-11-06 14:27:31 -080072
Chenjie Yub3dda412017-10-24 13:41:59 -070073// ValueMetric has a minimum bucket size of 10min so that we don't pull too frequently
Yao Chenb3561512017-11-21 18:07:17 -080074ValueMetricProducer::ValueMetricProducer(const ConfigKey& key, const ValueMetric& metric,
75 const int conditionIndex,
Yao Chen93fe3a32017-11-02 13:52:59 -070076 const sp<ConditionWizard>& wizard, const int pullTagId,
Chenjie Yue1361ed2018-07-23 17:33:09 -070077 const int64_t timeBaseNs, const int64_t startTimeNs,
Chenjie Yue2219202018-06-08 10:07:51 -070078 const sp<StatsPullerManager>& pullerManager)
Yangster-mac15f6bbc2018-04-08 11:52:26 -070079 : MetricProducer(metric.id(), key, timeBaseNs, conditionIndex, wizard),
Chenjie Yue2219202018-06-08 10:07:51 -070080 mPullerManager(pullerManager),
Yao Chenf09569f2017-12-13 17:00:51 -080081 mValueField(metric.value_field()),
Chenjie Yuc5875052018-03-09 10:13:11 -080082 mPullTagId(pullTagId),
Chenjie Yua0f02242018-07-06 16:14:34 -070083 mIsPulled(pullTagId != -1),
David Chen81245fd2018-04-12 14:33:37 -070084 mMinBucketSizeNs(metric.min_bucket_size_nanos()),
Chenjie Yuc5875052018-03-09 10:13:11 -080085 mDimensionSoftLimit(StatsdStats::kAtomDimensionKeySizeLimitMap.find(pullTagId) !=
86 StatsdStats::kAtomDimensionKeySizeLimitMap.end()
87 ? StatsdStats::kAtomDimensionKeySizeLimitMap.at(pullTagId).first
88 : StatsdStats::kDimensionKeySizeSoftLimit),
89 mDimensionHardLimit(StatsdStats::kAtomDimensionKeySizeLimitMap.find(pullTagId) !=
90 StatsdStats::kAtomDimensionKeySizeLimitMap.end()
91 ? StatsdStats::kAtomDimensionKeySizeLimitMap.at(pullTagId).second
Chenjie Yu47234642018-05-14 10:14:16 -070092 : StatsdStats::kDimensionKeySizeHardLimit),
Chenjie Yua0f02242018-07-06 16:14:34 -070093 mUseAbsoluteValueOnReset(metric.use_absolute_value_on_reset()),
94 mAggregationType(metric.aggregation_type()),
95 mValueType(metric.aggregation_type() == ValueMetric::AVG ? DOUBLE : LONG) {
Yangster-macb8144812018-01-04 10:56:23 -080096 int64_t bucketSizeMills = 0;
97 if (metric.has_bucket()) {
yro59cc24d2018-02-13 20:17:32 -080098 bucketSizeMills = TimeUnitToBucketSizeInMillisGuardrailed(key.GetUid(), metric.bucket());
Chenjie Yu6736c892017-11-09 10:50:09 -080099 } else {
Yangster-macb8144812018-01-04 10:56:23 -0800100 bucketSizeMills = TimeUnitToBucketSizeInMillis(ONE_HOUR);
Chenjie Yu6736c892017-11-09 10:50:09 -0800101 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700102
Yangster-macb8144812018-01-04 10:56:23 -0800103 mBucketSizeNs = bucketSizeMills * 1000000;
Yao Chen8a8d16c2018-02-08 14:50:40 -0800104 if (metric.has_dimensions_in_what()) {
105 translateFieldMatcher(metric.dimensions_in_what(), &mDimensionsInWhat);
Yangster13fb7e42018-03-07 17:30:49 -0800106 mContainANYPositionInDimensionsInWhat = HasPositionANY(metric.dimensions_in_what());
Yao Chen8a8d16c2018-02-08 14:50:40 -0800107 }
108
109 if (metric.has_dimensions_in_condition()) {
110 translateFieldMatcher(metric.dimensions_in_condition(), &mDimensionsInCondition);
111 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700112
Yao Chen93fe3a32017-11-02 13:52:59 -0700113 if (metric.links().size() > 0) {
Yao Chen8a8d16c2018-02-08 14:50:40 -0800114 for (const auto& link : metric.links()) {
115 Metric2Condition mc;
116 mc.conditionId = link.condition();
117 translateFieldMatcher(link.fields_in_what(), &mc.metricFields);
118 translateFieldMatcher(link.fields_in_condition(), &mc.conditionFields);
119 mMetric2ConditionLinks.push_back(mc);
120 }
Yao Chen93fe3a32017-11-02 13:52:59 -0700121 }
Yao Chen8a8d16c2018-02-08 14:50:40 -0800122
Yangster-mac15f6bbc2018-04-08 11:52:26 -0700123 if (mValueField.child_size() > 0) {
Yao Chen8a8d16c2018-02-08 14:50:40 -0800124 mField = mValueField.child(0).field();
125 }
126 mConditionSliced = (metric.links().size() > 0) || (mDimensionsInCondition.size() > 0);
Yangster-mac9def8e32018-04-17 13:55:51 -0700127 mSliceByPositionALL = HasPositionALL(metric.dimensions_in_what()) ||
128 HasPositionALL(metric.dimensions_in_condition());
Chenjie Yub3dda412017-10-24 13:41:59 -0700129
Chenjie Yue1361ed2018-07-23 17:33:09 -0700130 flushIfNeededLocked(startTimeNs);
Chenjie Yua0f02242018-07-06 16:14:34 -0700131 // Kicks off the puller immediately.
132 if (mIsPulled) {
Chenjie Yue1361ed2018-07-23 17:33:09 -0700133 mPullerManager->RegisterReceiver(mPullTagId, this, getCurrentBucketEndTimeNs(),
134 mBucketSizeNs);
Yao Chen93fe3a32017-11-02 13:52:59 -0700135 }
Chenjie Yu1a0a9412018-03-28 10:07:22 -0700136
Chenjie Yue1361ed2018-07-23 17:33:09 -0700137 // TODO: Only do this for partial buckets like first bucket. All other buckets should use
138 // flushIfNeeded to adjust start and end to bucket boundaries.
139 // Adjust start for partial bucket
140 mCurrentBucketStartTimeNs = startTimeNs;
141 if (mIsPulled) {
142 pullLocked(startTimeNs);
143 }
Yangster-mac94e197c2018-01-02 16:03:03 -0800144 VLOG("value metric %lld created. bucket size %lld start_time: %lld",
Yangster-mac15f6bbc2018-04-08 11:52:26 -0700145 (long long)metric.id(), (long long)mBucketSizeNs, (long long)mTimeBaseNs);
Chenjie Yub3dda412017-10-24 13:41:59 -0700146}
147
148ValueMetricProducer::~ValueMetricProducer() {
Yao Chen93fe3a32017-11-02 13:52:59 -0700149 VLOG("~ValueMetricProducer() called");
Chenjie Yua0f02242018-07-06 16:14:34 -0700150 if (mIsPulled) {
Chenjie Yue2219202018-06-08 10:07:51 -0700151 mPullerManager->UnRegisterReceiver(mPullTagId, this);
Chenjie Yu6736c892017-11-09 10:50:09 -0800152 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700153}
154
Yao Chen427d3722018-03-22 15:21:52 -0700155void ValueMetricProducer::onSlicedConditionMayChangeLocked(bool overallCondition,
Yangster-macb142cc82018-03-30 15:22:08 -0700156 const int64_t eventTime) {
Yangster-mac94e197c2018-01-02 16:03:03 -0800157 VLOG("Metric %lld onSlicedConditionMayChange", (long long)mMetricId);
Chenjie Yub3dda412017-10-24 13:41:59 -0700158}
159
Yangster-macb142cc82018-03-30 15:22:08 -0700160void ValueMetricProducer::dropDataLocked(const int64_t dropTimeNs) {
Yao Chen06dba5d2018-01-26 13:38:16 -0800161 flushIfNeededLocked(dropTimeNs);
162 mPastBuckets.clear();
163}
164
Yangster-maca802d732018-04-24 07:50:38 -0700165void ValueMetricProducer::clearPastBucketsLocked(const int64_t dumpTimeNs) {
166 flushIfNeededLocked(dumpTimeNs);
167 mPastBuckets.clear();
168 mSkippedBuckets.clear();
169}
170
Yangster-macb142cc82018-03-30 15:22:08 -0700171void ValueMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs,
Yangster-mace68f3a52018-04-04 00:01:43 -0700172 const bool include_current_partial_bucket,
Yangster-mac9def8e32018-04-17 13:55:51 -0700173 std::set<string> *str_set,
Yao Chen288c6002017-12-12 13:43:18 -0800174 ProtoOutputStream* protoOutput) {
Yangster-mac94e197c2018-01-02 16:03:03 -0800175 VLOG("metric %lld dump report now...", (long long)mMetricId);
Yangster-mace68f3a52018-04-04 00:01:43 -0700176 if (include_current_partial_bucket) {
177 flushLocked(dumpTimeNs);
178 } else {
179 flushIfNeededLocked(dumpTimeNs);
180 }
David Chen81245fd2018-04-12 14:33:37 -0700181 if (mPastBuckets.empty() && mSkippedBuckets.empty()) {
Yangster-mac635b4b32018-01-23 20:17:35 -0800182 return;
183 }
Yangster-mac94e197c2018-01-02 16:03:03 -0800184 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)mMetricId);
Yangster-mac9def8e32018-04-17 13:55:51 -0700185 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_TIME_BASE, (long long)mTimeBaseNs);
186 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_SIZE, (long long)mBucketSizeNs);
187 // Fills the dimension path if not slicing by ALL.
188 if (!mSliceByPositionALL) {
189 if (!mDimensionsInWhat.empty()) {
190 uint64_t dimenPathToken = protoOutput->start(
191 FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_WHAT);
192 writeDimensionPathToProto(mDimensionsInWhat, protoOutput);
193 protoOutput->end(dimenPathToken);
194 }
195 if (!mDimensionsInCondition.empty()) {
196 uint64_t dimenPathToken = protoOutput->start(
197 FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_CONDITION);
198 writeDimensionPathToProto(mDimensionsInCondition, protoOutput);
199 protoOutput->end(dimenPathToken);
200 }
201 }
202
Yi Jin5ee07872018-03-05 18:18:27 -0800203 uint64_t protoToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_VALUE_METRICS);
Yao Chen6a8c7992017-11-29 20:02:07 +0000204
David Chen81245fd2018-04-12 14:33:37 -0700205 for (const auto& pair : mSkippedBuckets) {
206 uint64_t wrapperToken =
207 protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_SKIPPED);
Yangster-mac9def8e32018-04-17 13:55:51 -0700208 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_SKIPPED_START_MILLIS,
209 (long long)(NanoToMillis(pair.first)));
210 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_SKIPPED_END_MILLIS,
211 (long long)(NanoToMillis(pair.second)));
David Chen81245fd2018-04-12 14:33:37 -0700212 protoOutput->end(wrapperToken);
213 }
214 mSkippedBuckets.clear();
215
Yao Chen93fe3a32017-11-02 13:52:59 -0700216 for (const auto& pair : mPastBuckets) {
Yangster-mac93694462018-01-22 20:49:31 -0800217 const MetricDimensionKey& dimensionKey = pair.first;
Yangster13fb7e42018-03-07 17:30:49 -0800218 VLOG(" dimension key %s", dimensionKey.toString().c_str());
Yi Jin5ee07872018-03-05 18:18:27 -0800219 uint64_t wrapperToken =
Yao Chen288c6002017-12-12 13:43:18 -0800220 protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_DATA);
Chenjie Yub3dda412017-10-24 13:41:59 -0700221
Yangster-mac20877162017-12-22 17:19:39 -0800222 // First fill dimension.
Yangster-mac9def8e32018-04-17 13:55:51 -0700223 if (mSliceByPositionALL) {
224 uint64_t dimensionToken = protoOutput->start(
225 FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT);
226 writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), str_set, protoOutput);
227 protoOutput->end(dimensionToken);
228 if (dimensionKey.hasDimensionKeyInCondition()) {
229 uint64_t dimensionInConditionToken = protoOutput->start(
230 FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_CONDITION);
231 writeDimensionToProto(dimensionKey.getDimensionKeyInCondition(),
232 str_set, protoOutput);
233 protoOutput->end(dimensionInConditionToken);
234 }
235 } else {
236 writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInWhat(),
237 FIELD_ID_DIMENSION_LEAF_IN_WHAT, str_set, protoOutput);
238 if (dimensionKey.hasDimensionKeyInCondition()) {
239 writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInCondition(),
240 FIELD_ID_DIMENSION_LEAF_IN_CONDITION,
241 str_set, protoOutput);
242 }
Yangster-mac93694462018-01-22 20:49:31 -0800243 }
yro2b0f8862017-11-06 14:27:31 -0800244
245 // Then fill bucket_info (ValueBucketInfo).
246 for (const auto& bucket : pair.second) {
Yi Jin5ee07872018-03-05 18:18:27 -0800247 uint64_t bucketInfoToken = protoOutput->start(
Yao Chen288c6002017-12-12 13:43:18 -0800248 FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_BUCKET_INFO);
Yangster-mac9def8e32018-04-17 13:55:51 -0700249
250 if (bucket.mBucketEndNs - bucket.mBucketStartNs != mBucketSizeNs) {
251 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_BUCKET_ELAPSED_MILLIS,
252 (long long)NanoToMillis(bucket.mBucketStartNs));
253 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_END_BUCKET_ELAPSED_MILLIS,
254 (long long)NanoToMillis(bucket.mBucketEndNs));
255 } else {
256 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_NUM,
257 (long long)(getBucketNumFromEndTimeNs(bucket.mBucketEndNs)));
258 }
Chenjie Yua0f02242018-07-06 16:14:34 -0700259 if (mValueType == LONG) {
260 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_VALUE_LONG,
261 (long long)bucket.mValueLong);
262 } else {
263 protoOutput->write(FIELD_TYPE_DOUBLE | FIELD_ID_VALUE_DOUBLE, bucket.mValueDouble);
264 }
Yao Chen288c6002017-12-12 13:43:18 -0800265 protoOutput->end(bucketInfoToken);
Chenjie Yua0f02242018-07-06 16:14:34 -0700266 VLOG("\t bucket [%lld - %lld] count: %lld, %.2f", (long long)bucket.mBucketStartNs,
267 (long long)bucket.mBucketEndNs, (long long)bucket.mValueLong, bucket.mValueDouble);
yro2b0f8862017-11-06 14:27:31 -0800268 }
Yao Chen288c6002017-12-12 13:43:18 -0800269 protoOutput->end(wrapperToken);
Chenjie Yub3dda412017-10-24 13:41:59 -0700270 }
Yao Chen288c6002017-12-12 13:43:18 -0800271 protoOutput->end(protoToken);
yro2b0f8862017-11-06 14:27:31 -0800272
Yangster-mac94e197c2018-01-02 16:03:03 -0800273 VLOG("metric %lld dump report now...", (long long)mMetricId);
Yao Chen6a8c7992017-11-29 20:02:07 +0000274 mPastBuckets.clear();
Chenjie Yub3dda412017-10-24 13:41:59 -0700275}
276
Chenjie Yu1a0a9412018-03-28 10:07:22 -0700277void ValueMetricProducer::onConditionChangedLocked(const bool condition,
Yangster-macb142cc82018-03-30 15:22:08 -0700278 const int64_t eventTimeNs) {
Yao Chen6a8c7992017-11-29 20:02:07 +0000279 mCondition = condition;
Chenjie Yub3dda412017-10-24 13:41:59 -0700280
Chenjie Yu1a0a9412018-03-28 10:07:22 -0700281 if (eventTimeNs < mCurrentBucketStartTimeNs) {
282 VLOG("Skip event due to late arrival: %lld vs %lld", (long long)eventTimeNs,
Chenjie Yua7259ab2017-12-10 08:31:05 -0800283 (long long)mCurrentBucketStartTimeNs);
Yao Chen2794da22017-12-13 16:01:55 -0800284 return;
285 }
286
Chenjie Yu1a0a9412018-03-28 10:07:22 -0700287 flushIfNeededLocked(eventTimeNs);
Chenjie Yua7259ab2017-12-10 08:31:05 -0800288
Chenjie Yua0f02242018-07-06 16:14:34 -0700289 if (mIsPulled) {
Chenjie Yue1361ed2018-07-23 17:33:09 -0700290 pullLocked(eventTimeNs);
291 }
292}
293
294void ValueMetricProducer::pullLocked(const int64_t timestampNs) {
295 vector<std::shared_ptr<LogEvent>> allData;
296 if (mPullerManager->Pull(mPullTagId, timestampNs, &allData)) {
297 if (allData.size() == 0) {
298 return;
Yao Chen6a8c7992017-11-29 20:02:07 +0000299 }
Chenjie Yue1361ed2018-07-23 17:33:09 -0700300 for (const auto& data : allData) {
301 onMatchedLogEventLocked(0, *data);
302 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700303 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700304}
305
306void ValueMetricProducer::onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& allData) {
Yangsterf2bee6f2017-11-29 12:01:05 -0800307 std::lock_guard<std::mutex> lock(mMutex);
308
Yao Chenf09569f2017-12-13 17:00:51 -0800309 if (mCondition == true || mConditionTrackerIndex < 0) {
Chenjie Yub3dda412017-10-24 13:41:59 -0700310 if (allData.size() == 0) {
311 return;
312 }
Chenjie Yua7259ab2017-12-10 08:31:05 -0800313 // For scheduled pulled data, the effective event time is snap to the nearest
314 // bucket boundary to make bucket finalize.
Yangster-macb142cc82018-03-30 15:22:08 -0700315 int64_t realEventTime = allData.at(0)->GetElapsedTimestampNs();
Yangster-mac15f6bbc2018-04-08 11:52:26 -0700316 int64_t eventTime = mTimeBaseNs +
317 ((realEventTime - mTimeBaseNs) / mBucketSizeNs) * mBucketSizeNs;
Chenjie Yua7259ab2017-12-10 08:31:05 -0800318
Chenjie Yue1361ed2018-07-23 17:33:09 -0700319 // close the end of the bucket
Chenjie Yua7259ab2017-12-10 08:31:05 -0800320 mCondition = false;
Chenjie Yub3dda412017-10-24 13:41:59 -0700321 for (const auto& data : allData) {
Yangster-mac330af582018-02-08 15:24:38 -0800322 data->setElapsedTimestampNs(eventTime - 1);
Chenjie Yua7259ab2017-12-10 08:31:05 -0800323 onMatchedLogEventLocked(0, *data);
Chenjie Yub3dda412017-10-24 13:41:59 -0700324 }
Chenjie Yua7259ab2017-12-10 08:31:05 -0800325
Chenjie Yue1361ed2018-07-23 17:33:09 -0700326 // start a new bucket
Chenjie Yua7259ab2017-12-10 08:31:05 -0800327 mCondition = true;
328 for (const auto& data : allData) {
Yangster-mac330af582018-02-08 15:24:38 -0800329 data->setElapsedTimestampNs(eventTime);
Chenjie Yua7259ab2017-12-10 08:31:05 -0800330 onMatchedLogEventLocked(0, *data);
331 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700332 }
333}
334
Yangster-maca78d0082018-03-12 12:02:56 -0700335void ValueMetricProducer::dumpStatesLocked(FILE* out, bool verbose) const {
336 if (mCurrentSlicedBucket.size() == 0) {
337 return;
338 }
339
340 fprintf(out, "ValueMetric %lld dimension size %lu\n", (long long)mMetricId,
341 (unsigned long)mCurrentSlicedBucket.size());
342 if (verbose) {
343 for (const auto& it : mCurrentSlicedBucket) {
Chenjie Yua0f02242018-07-06 16:14:34 -0700344 fprintf(out, "\t(what)%s\t(condition)%s (value)%s\n",
345 it.first.getDimensionKeyInWhat().toString().c_str(),
346 it.first.getDimensionKeyInCondition().toString().c_str(),
347 it.second.value.toString().c_str());
Yangster-maca78d0082018-03-12 12:02:56 -0700348 }
349 }
350}
351
Yangster-mac93694462018-01-22 20:49:31 -0800352bool ValueMetricProducer::hitGuardRailLocked(const MetricDimensionKey& newKey) {
Yao Chenb3561512017-11-21 18:07:17 -0800353 // ===========GuardRail==============
354 // 1. Report the tuple count if the tuple count > soft limit
355 if (mCurrentSlicedBucket.find(newKey) != mCurrentSlicedBucket.end()) {
356 return false;
357 }
Chenjie Yuc5875052018-03-09 10:13:11 -0800358 if (mCurrentSlicedBucket.size() > mDimensionSoftLimit - 1) {
Yao Chenb3561512017-11-21 18:07:17 -0800359 size_t newTupleCount = mCurrentSlicedBucket.size() + 1;
Yangster-mac94e197c2018-01-02 16:03:03 -0800360 StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetricId, newTupleCount);
Yao Chenb3561512017-11-21 18:07:17 -0800361 // 2. Don't add more tuples, we are above the allowed threshold. Drop the data.
Chenjie Yuc5875052018-03-09 10:13:11 -0800362 if (newTupleCount > mDimensionHardLimit) {
Yangster-mac94e197c2018-01-02 16:03:03 -0800363 ALOGE("ValueMetric %lld dropping data for dimension key %s",
Yangster13fb7e42018-03-07 17:30:49 -0800364 (long long)mMetricId, newKey.toString().c_str());
Yao Chenb3561512017-11-21 18:07:17 -0800365 return true;
366 }
367 }
368
369 return false;
370}
371
Chenjie Yua0f02242018-07-06 16:14:34 -0700372const Value getDoubleOrLong(const Value& value) {
373 Value v;
374 switch (value.type) {
375 case INT:
376 v.setLong(value.int_value);
377 break;
378 case LONG:
379 v.setLong(value.long_value);
380 break;
381 case FLOAT:
382 v.setDouble(value.float_value);
383 break;
384 case DOUBLE:
385 v.setDouble(value.double_value);
386 break;
387 default:
388 break;
389 }
390 return v;
391}
392
Yangsterf2bee6f2017-11-29 12:01:05 -0800393void ValueMetricProducer::onMatchedLogEventInternalLocked(
Yangster-mac93694462018-01-22 20:49:31 -0800394 const size_t matcherIndex, const MetricDimensionKey& eventKey,
Yangster-mac20877162017-12-22 17:19:39 -0800395 const ConditionKey& conditionKey, bool condition,
Chenjie Yua7259ab2017-12-10 08:31:05 -0800396 const LogEvent& event) {
Yangster-macb142cc82018-03-30 15:22:08 -0700397 int64_t eventTimeNs = event.GetElapsedTimestampNs();
Yao Chen6a8c7992017-11-29 20:02:07 +0000398 if (eventTimeNs < mCurrentBucketStartTimeNs) {
399 VLOG("Skip event due to late arrival: %lld vs %lld", (long long)eventTimeNs,
400 (long long)mCurrentBucketStartTimeNs);
401 return;
402 }
403
Chenjie Yua7259ab2017-12-10 08:31:05 -0800404 flushIfNeededLocked(eventTimeNs);
405
Yangsterf2bee6f2017-11-29 12:01:05 -0800406 if (hitGuardRailLocked(eventKey)) {
Yangster8de69392017-11-27 13:48:29 -0800407 return;
408 }
Yao Chen6a8c7992017-11-29 20:02:07 +0000409 Interval& interval = mCurrentSlicedBucket[eventKey];
410
Chenjie Yua0f02242018-07-06 16:14:34 -0700411 if (mField > event.size()) {
412 VLOG("Failed to extract value field %d from atom %s. %d", mField, event.ToString().c_str(),
413 (int)event.size());
Yangster-maca7fb12d2018-01-03 17:17:20 -0800414 return;
415 }
Chenjie Yua0f02242018-07-06 16:14:34 -0700416 Value value = getDoubleOrLong(event.getValues()[mField - 1].mValue);
Yao Chen6a8c7992017-11-29 20:02:07 +0000417
Chenjie Yua0f02242018-07-06 16:14:34 -0700418 Value diff;
419 bool hasDiff = false;
420 if (mIsPulled) {
421 // Always require condition for pulled events. In the case of no condition, only pull
422 // on bucket boundaries, in which we fake condition changes.
Chenjie Yua7259ab2017-12-10 08:31:05 -0800423 if (mCondition == true) {
Chenjie Yu6d370f42018-03-25 14:57:30 -0700424 if (!interval.startUpdated) {
425 interval.start = value;
426 interval.startUpdated = true;
427 } else {
Chenjie Yua0f02242018-07-06 16:14:34 -0700428 // Skip it if there is already value recorded for the start. Happens when puller
429 // takes too long to finish. In this case we take the previous value.
Chenjie Yu6d370f42018-03-25 14:57:30 -0700430 VLOG("Already recorded value for this dimension %s", eventKey.toString().c_str());
431 }
Yao Chen6a8c7992017-11-29 20:02:07 +0000432 } else {
Chenjie Yu0ed268b2018-01-10 09:37:10 -0800433 // Generally we expect value to be monotonically increasing.
Chenjie Yu47234642018-05-14 10:14:16 -0700434 // If not, take absolute value or drop it, based on config.
Chenjie Yua7259ab2017-12-10 08:31:05 -0800435 if (interval.startUpdated) {
Yangster613a7e22018-05-08 15:12:30 -0700436 if (value >= interval.start) {
Chenjie Yua0f02242018-07-06 16:14:34 -0700437 diff = (value - interval.start);
438 hasDiff = true;
Chenjie Yu0ed268b2018-01-10 09:37:10 -0800439 } else {
Chenjie Yu47234642018-05-14 10:14:16 -0700440 if (mUseAbsoluteValueOnReset) {
Chenjie Yua0f02242018-07-06 16:14:34 -0700441 diff = value;
442 hasDiff = true;
Chenjie Yu47234642018-05-14 10:14:16 -0700443 } else {
Chenjie Yua0f02242018-07-06 16:14:34 -0700444 VLOG("Dropping data for atom %d, prev: %s, now: %s", mPullTagId,
445 interval.start.toString().c_str(), value.toString().c_str());
Chenjie Yu47234642018-05-14 10:14:16 -0700446 }
Chenjie Yu0ed268b2018-01-10 09:37:10 -0800447 }
Chenjie Yua7259ab2017-12-10 08:31:05 -0800448 interval.startUpdated = false;
Yao Chen6a8c7992017-11-29 20:02:07 +0000449 } else {
Chenjie Yua0f02242018-07-06 16:14:34 -0700450 VLOG("No start for matching end %s", value.toString().c_str());
Yao Chen6a8c7992017-11-29 20:02:07 +0000451 }
452 }
Chenjie Yua0f02242018-07-06 16:14:34 -0700453 } else {
454 // for pushed events, only aggregate when sliced condition is true
455 if (condition == true || mConditionTrackerIndex < 0) {
456 diff = value;
457 hasDiff = true;
Chenjie Yu021e2532018-05-16 12:23:07 -0700458 }
Yangster8de69392017-11-27 13:48:29 -0800459 }
Chenjie Yua0f02242018-07-06 16:14:34 -0700460 if (hasDiff) {
461 if (interval.hasValue) {
462 switch (mAggregationType) {
463 case ValueMetric::SUM:
464 // for AVG, we add up and take average when flushing the bucket
465 case ValueMetric::AVG:
466 interval.value += diff;
467 break;
468 case ValueMetric::MIN:
469 interval.value = diff < interval.value ? diff : interval.value;
470 break;
471 case ValueMetric::MAX:
472 interval.value = diff > interval.value ? diff : interval.value;
473 break;
474 default:
475 break;
476 }
477 } else {
478 interval.value = diff;
479 interval.hasValue = true;
480 }
481 interval.sampleSize += 1;
482 }
Bookatzde1b55622017-12-14 18:38:27 -0800483
Chenjie Yua0f02242018-07-06 16:14:34 -0700484 // TODO: propgate proper values down stream when anomaly support doubles
485 long wholeBucketVal = interval.value.long_value;
David Chen27785a82018-01-19 17:06:45 -0800486 auto prev = mCurrentFullBucket.find(eventKey);
487 if (prev != mCurrentFullBucket.end()) {
488 wholeBucketVal += prev->second;
489 }
Bookatzde1b55622017-12-14 18:38:27 -0800490 for (auto& tracker : mAnomalyTrackers) {
David Chen27785a82018-01-19 17:06:45 -0800491 tracker->detectAndDeclareAnomaly(eventTimeNs, mCurrentBucketNum, eventKey, wholeBucketVal);
Bookatzde1b55622017-12-14 18:38:27 -0800492 }
Yangster8de69392017-11-27 13:48:29 -0800493}
494
Yangster-macb142cc82018-03-30 15:22:08 -0700495void ValueMetricProducer::flushIfNeededLocked(const int64_t& eventTimeNs) {
496 int64_t currentBucketEndTimeNs = getCurrentBucketEndTimeNs();
David Chen27785a82018-01-19 17:06:45 -0800497
Yangster-mac15f6bbc2018-04-08 11:52:26 -0700498 if (eventTimeNs < currentBucketEndTimeNs) {
Chenjie Yub3dda412017-10-24 13:41:59 -0700499 VLOG("eventTime is %lld, less than next bucket start time %lld", (long long)eventTimeNs,
David Chen27785a82018-01-19 17:06:45 -0800500 (long long)(currentBucketEndTimeNs));
Chenjie Yub3dda412017-10-24 13:41:59 -0700501 return;
502 }
David Chen27785a82018-01-19 17:06:45 -0800503
504 flushCurrentBucketLocked(eventTimeNs);
505
506 int64_t numBucketsForward = 1 + (eventTimeNs - currentBucketEndTimeNs) / mBucketSizeNs;
507 mCurrentBucketStartTimeNs = currentBucketEndTimeNs + (numBucketsForward - 1) * mBucketSizeNs;
508 mCurrentBucketNum += numBucketsForward;
509
510 if (numBucketsForward > 1) {
511 VLOG("Skipping forward %lld buckets", (long long)numBucketsForward);
512 }
513 VLOG("metric %lld: new bucket start time: %lld", (long long)mMetricId,
514 (long long)mCurrentBucketStartTimeNs);
515}
516
Yangster-macb142cc82018-03-30 15:22:08 -0700517void ValueMetricProducer::flushCurrentBucketLocked(const int64_t& eventTimeNs) {
Chenjie Yub3dda412017-10-24 13:41:59 -0700518 VLOG("finalizing bucket for %ld, dumping %d slices", (long)mCurrentBucketStartTimeNs,
519 (int)mCurrentSlicedBucket.size());
Yangster-macb142cc82018-03-30 15:22:08 -0700520 int64_t fullBucketEndTimeNs = getCurrentBucketEndTimeNs();
David Chen27785a82018-01-19 17:06:45 -0800521
yro2b0f8862017-11-06 14:27:31 -0800522 ValueBucket info;
523 info.mBucketStartNs = mCurrentBucketStartTimeNs;
David Chen27785a82018-01-19 17:06:45 -0800524 if (eventTimeNs < fullBucketEndTimeNs) {
525 info.mBucketEndNs = eventTimeNs;
526 } else {
527 info.mBucketEndNs = fullBucketEndTimeNs;
528 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700529
David Chen81245fd2018-04-12 14:33:37 -0700530 if (info.mBucketEndNs - mCurrentBucketStartTimeNs >= mMinBucketSizeNs) {
531 // The current bucket is large enough to keep.
David Chen81245fd2018-04-12 14:33:37 -0700532 for (const auto& slice : mCurrentSlicedBucket) {
David Chen81245fd2018-04-12 14:33:37 -0700533 if (slice.second.hasValue) {
Chenjie Yua0f02242018-07-06 16:14:34 -0700534 info.mValueLong = slice.second.value.long_value;
535 info.mValueDouble = (double)slice.second.value.long_value / slice.second.sampleSize;
David Chen81245fd2018-04-12 14:33:37 -0700536 // it will auto create new vector of ValuebucketInfo if the key is not found.
537 auto& bucketList = mPastBuckets[slice.first];
538 bucketList.push_back(info);
539 }
Chenjie Yuae63b0a2018-04-10 14:59:31 -0700540 }
David Chen81245fd2018-04-12 14:33:37 -0700541 } else {
542 mSkippedBuckets.emplace_back(info.mBucketStartNs, info.mBucketEndNs);
Chenjie Yub3dda412017-10-24 13:41:59 -0700543 }
544
David Chen27785a82018-01-19 17:06:45 -0800545 if (eventTimeNs > fullBucketEndTimeNs) { // If full bucket, send to anomaly tracker.
546 // Accumulate partial buckets with current value and then send to anomaly tracker.
547 if (mCurrentFullBucket.size() > 0) {
548 for (const auto& slice : mCurrentSlicedBucket) {
Chenjie Yua0f02242018-07-06 16:14:34 -0700549 // TODO: fix this when anomaly can accept double values
550 mCurrentFullBucket[slice.first] += slice.second.value.long_value;
David Chen27785a82018-01-19 17:06:45 -0800551 }
552 for (const auto& slice : mCurrentFullBucket) {
553 for (auto& tracker : mAnomalyTrackers) {
554 if (tracker != nullptr) {
555 tracker->addPastBucket(slice.first, slice.second, mCurrentBucketNum);
556 }
557 }
558 }
559 mCurrentFullBucket.clear();
560 } else {
561 // Skip aggregating the partial buckets since there's no previous partial bucket.
562 for (const auto& slice : mCurrentSlicedBucket) {
563 for (auto& tracker : mAnomalyTrackers) {
564 if (tracker != nullptr) {
Chenjie Yua0f02242018-07-06 16:14:34 -0700565 // TODO: fix this when anomaly can accept double values
566 tracker->addPastBucket(slice.first, slice.second.value.long_value,
567 mCurrentBucketNum);
David Chen27785a82018-01-19 17:06:45 -0800568 }
569 }
570 }
571 }
572 } else {
573 // Accumulate partial bucket.
574 for (const auto& slice : mCurrentSlicedBucket) {
Chenjie Yua0f02242018-07-06 16:14:34 -0700575 // TODO: fix this when anomaly can accept double values
576 mCurrentFullBucket[slice.first] += slice.second.value.long_value;
David Chen27785a82018-01-19 17:06:45 -0800577 }
578 }
579
Chenjie Yub3dda412017-10-24 13:41:59 -0700580 // Reset counters
Chenjie Yua7259ab2017-12-10 08:31:05 -0800581 mCurrentSlicedBucket.clear();
Chenjie Yub3dda412017-10-24 13:41:59 -0700582}
583
Yangsterf2bee6f2017-11-29 12:01:05 -0800584size_t ValueMetricProducer::byteSizeLocked() const {
Yangster-mace2cd6d52017-11-09 20:38:30 -0800585 size_t totalSize = 0;
586 for (const auto& pair : mPastBuckets) {
587 totalSize += pair.second.size() * kBucketSize;
588 }
589 return totalSize;
yro2b0f8862017-11-06 14:27:31 -0800590}
591
Chenjie Yub3dda412017-10-24 13:41:59 -0700592} // namespace statsd
593} // namespace os
Yao Chen93fe3a32017-11-02 13:52:59 -0700594} // namespace android