blob: 63674798552607fd8b376d012f2b4adc63805c74 [file] [log] [blame]
Chenjie Yub3dda412017-10-24 13:41:59 -07001/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
Chenjie Yu88588972018-08-03 09:49:22 -070017#define DEBUG false // STOPSHIP if true
Chenjie Yub3dda412017-10-24 13:41:59 -070018#include "Log.h"
19
20#include "ValueMetricProducer.h"
Chenjie Yuc5875052018-03-09 10:13:11 -080021#include "../guardrail/StatsdStats.h"
22#include "../stats_log_util.h"
Chenjie Yub3dda412017-10-24 13:41:59 -070023
24#include <cutils/log.h>
25#include <limits.h>
26#include <stdlib.h>
27
yrob0378b02017-11-09 20:36:25 -080028using android::util::FIELD_COUNT_REPEATED;
yro2b0f8862017-11-06 14:27:31 -080029using android::util::FIELD_TYPE_BOOL;
Chenjie Yua0f02242018-07-06 16:14:34 -070030using android::util::FIELD_TYPE_DOUBLE;
yro2b0f8862017-11-06 14:27:31 -080031using android::util::FIELD_TYPE_INT32;
32using android::util::FIELD_TYPE_INT64;
33using android::util::FIELD_TYPE_MESSAGE;
Yangster-macd1815dc2017-11-13 21:43:15 -080034using android::util::FIELD_TYPE_STRING;
yro2b0f8862017-11-06 14:27:31 -080035using android::util::ProtoOutputStream;
Chenjie Yub3dda412017-10-24 13:41:59 -070036using std::list;
Chenjie Yu6736c892017-11-09 10:50:09 -080037using std::make_pair;
Chenjie Yub3dda412017-10-24 13:41:59 -070038using std::make_shared;
Yao Chen93fe3a32017-11-02 13:52:59 -070039using std::map;
Chenjie Yub3dda412017-10-24 13:41:59 -070040using std::shared_ptr;
41using std::unique_ptr;
Yao Chen93fe3a32017-11-02 13:52:59 -070042using std::unordered_map;
Chenjie Yub3dda412017-10-24 13:41:59 -070043
44namespace android {
45namespace os {
46namespace statsd {
47
yro2b0f8862017-11-06 14:27:31 -080048// for StatsLogReport
Yangster-mac94e197c2018-01-02 16:03:03 -080049const int FIELD_ID_ID = 1;
yro2b0f8862017-11-06 14:27:31 -080050const int FIELD_ID_VALUE_METRICS = 7;
Yangster-mac9def8e32018-04-17 13:55:51 -070051const int FIELD_ID_TIME_BASE = 9;
52const int FIELD_ID_BUCKET_SIZE = 10;
53const int FIELD_ID_DIMENSION_PATH_IN_WHAT = 11;
54const int FIELD_ID_DIMENSION_PATH_IN_CONDITION = 12;
yro2b0f8862017-11-06 14:27:31 -080055// for ValueMetricDataWrapper
56const int FIELD_ID_DATA = 1;
David Chen81245fd2018-04-12 14:33:37 -070057const int FIELD_ID_SKIPPED = 2;
Yangster-mac9def8e32018-04-17 13:55:51 -070058const int FIELD_ID_SKIPPED_START_MILLIS = 3;
59const int FIELD_ID_SKIPPED_END_MILLIS = 4;
yro2b0f8862017-11-06 14:27:31 -080060// for ValueMetricData
Yangster-mac468ff042018-01-17 12:26:34 -080061const int FIELD_ID_DIMENSION_IN_WHAT = 1;
62const int FIELD_ID_DIMENSION_IN_CONDITION = 2;
63const int FIELD_ID_BUCKET_INFO = 3;
Yangster-mac9def8e32018-04-17 13:55:51 -070064const int FIELD_ID_DIMENSION_LEAF_IN_WHAT = 4;
65const int FIELD_ID_DIMENSION_LEAF_IN_CONDITION = 5;
yro2b0f8862017-11-06 14:27:31 -080066// for ValueBucketInfo
Chenjie Yuc3f00772018-07-26 13:50:58 -070067const int FIELD_ID_VALUE_LONG = 7;
68const int FIELD_ID_VALUE_DOUBLE = 8;
Yangster-mac9def8e32018-04-17 13:55:51 -070069const int FIELD_ID_BUCKET_NUM = 4;
70const int FIELD_ID_START_BUCKET_ELAPSED_MILLIS = 5;
71const int FIELD_ID_END_BUCKET_ELAPSED_MILLIS = 6;
yro2b0f8862017-11-06 14:27:31 -080072
Chenjie Yub3dda412017-10-24 13:41:59 -070073// ValueMetric has a minimum bucket size of 10min so that we don't pull too frequently
Yao Chenb3561512017-11-21 18:07:17 -080074ValueMetricProducer::ValueMetricProducer(const ConfigKey& key, const ValueMetric& metric,
75 const int conditionIndex,
Yao Chen93fe3a32017-11-02 13:52:59 -070076 const sp<ConditionWizard>& wizard, const int pullTagId,
Chenjie Yue1361ed2018-07-23 17:33:09 -070077 const int64_t timeBaseNs, const int64_t startTimeNs,
Chenjie Yue2219202018-06-08 10:07:51 -070078 const sp<StatsPullerManager>& pullerManager)
Yangster-mac15f6bbc2018-04-08 11:52:26 -070079 : MetricProducer(metric.id(), key, timeBaseNs, conditionIndex, wizard),
Chenjie Yue2219202018-06-08 10:07:51 -070080 mPullerManager(pullerManager),
Yao Chenf09569f2017-12-13 17:00:51 -080081 mValueField(metric.value_field()),
Chenjie Yuc5875052018-03-09 10:13:11 -080082 mPullTagId(pullTagId),
Chenjie Yua0f02242018-07-06 16:14:34 -070083 mIsPulled(pullTagId != -1),
David Chen81245fd2018-04-12 14:33:37 -070084 mMinBucketSizeNs(metric.min_bucket_size_nanos()),
Chenjie Yuc5875052018-03-09 10:13:11 -080085 mDimensionSoftLimit(StatsdStats::kAtomDimensionKeySizeLimitMap.find(pullTagId) !=
86 StatsdStats::kAtomDimensionKeySizeLimitMap.end()
87 ? StatsdStats::kAtomDimensionKeySizeLimitMap.at(pullTagId).first
88 : StatsdStats::kDimensionKeySizeSoftLimit),
89 mDimensionHardLimit(StatsdStats::kAtomDimensionKeySizeLimitMap.find(pullTagId) !=
90 StatsdStats::kAtomDimensionKeySizeLimitMap.end()
91 ? StatsdStats::kAtomDimensionKeySizeLimitMap.at(pullTagId).second
Chenjie Yu47234642018-05-14 10:14:16 -070092 : StatsdStats::kDimensionKeySizeHardLimit),
Chenjie Yua0f02242018-07-06 16:14:34 -070093 mUseAbsoluteValueOnReset(metric.use_absolute_value_on_reset()),
94 mAggregationType(metric.aggregation_type()),
95 mValueType(metric.aggregation_type() == ValueMetric::AVG ? DOUBLE : LONG) {
Yangster-macb8144812018-01-04 10:56:23 -080096 int64_t bucketSizeMills = 0;
97 if (metric.has_bucket()) {
yro59cc24d2018-02-13 20:17:32 -080098 bucketSizeMills = TimeUnitToBucketSizeInMillisGuardrailed(key.GetUid(), metric.bucket());
Chenjie Yu6736c892017-11-09 10:50:09 -080099 } else {
Yangster-macb8144812018-01-04 10:56:23 -0800100 bucketSizeMills = TimeUnitToBucketSizeInMillis(ONE_HOUR);
Chenjie Yu6736c892017-11-09 10:50:09 -0800101 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700102
Yangster-macb8144812018-01-04 10:56:23 -0800103 mBucketSizeNs = bucketSizeMills * 1000000;
Yao Chen8a8d16c2018-02-08 14:50:40 -0800104 if (metric.has_dimensions_in_what()) {
105 translateFieldMatcher(metric.dimensions_in_what(), &mDimensionsInWhat);
Yangster13fb7e42018-03-07 17:30:49 -0800106 mContainANYPositionInDimensionsInWhat = HasPositionANY(metric.dimensions_in_what());
Yao Chen8a8d16c2018-02-08 14:50:40 -0800107 }
108
109 if (metric.has_dimensions_in_condition()) {
110 translateFieldMatcher(metric.dimensions_in_condition(), &mDimensionsInCondition);
111 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700112
Yao Chen93fe3a32017-11-02 13:52:59 -0700113 if (metric.links().size() > 0) {
Yao Chen8a8d16c2018-02-08 14:50:40 -0800114 for (const auto& link : metric.links()) {
115 Metric2Condition mc;
116 mc.conditionId = link.condition();
117 translateFieldMatcher(link.fields_in_what(), &mc.metricFields);
118 translateFieldMatcher(link.fields_in_condition(), &mc.conditionFields);
119 mMetric2ConditionLinks.push_back(mc);
120 }
Yao Chen93fe3a32017-11-02 13:52:59 -0700121 }
Yao Chen8a8d16c2018-02-08 14:50:40 -0800122
Yangster-mac15f6bbc2018-04-08 11:52:26 -0700123 if (mValueField.child_size() > 0) {
Yao Chen8a8d16c2018-02-08 14:50:40 -0800124 mField = mValueField.child(0).field();
125 }
126 mConditionSliced = (metric.links().size() > 0) || (mDimensionsInCondition.size() > 0);
Yangster-mac9def8e32018-04-17 13:55:51 -0700127 mSliceByPositionALL = HasPositionALL(metric.dimensions_in_what()) ||
128 HasPositionALL(metric.dimensions_in_condition());
Chenjie Yub3dda412017-10-24 13:41:59 -0700129
Chenjie Yue1361ed2018-07-23 17:33:09 -0700130 flushIfNeededLocked(startTimeNs);
Chenjie Yua0f02242018-07-06 16:14:34 -0700131 // Kicks off the puller immediately.
132 if (mIsPulled) {
Chenjie Yue1361ed2018-07-23 17:33:09 -0700133 mPullerManager->RegisterReceiver(mPullTagId, this, getCurrentBucketEndTimeNs(),
134 mBucketSizeNs);
Yao Chen93fe3a32017-11-02 13:52:59 -0700135 }
Chenjie Yu1a0a9412018-03-28 10:07:22 -0700136
Chenjie Yue1361ed2018-07-23 17:33:09 -0700137 // TODO: Only do this for partial buckets like first bucket. All other buckets should use
138 // flushIfNeeded to adjust start and end to bucket boundaries.
139 // Adjust start for partial bucket
140 mCurrentBucketStartTimeNs = startTimeNs;
141 if (mIsPulled) {
142 pullLocked(startTimeNs);
143 }
Yangster-mac94e197c2018-01-02 16:03:03 -0800144 VLOG("value metric %lld created. bucket size %lld start_time: %lld",
Yangster-mac15f6bbc2018-04-08 11:52:26 -0700145 (long long)metric.id(), (long long)mBucketSizeNs, (long long)mTimeBaseNs);
Chenjie Yub3dda412017-10-24 13:41:59 -0700146}
147
148ValueMetricProducer::~ValueMetricProducer() {
Yao Chen93fe3a32017-11-02 13:52:59 -0700149 VLOG("~ValueMetricProducer() called");
Chenjie Yua0f02242018-07-06 16:14:34 -0700150 if (mIsPulled) {
Chenjie Yue2219202018-06-08 10:07:51 -0700151 mPullerManager->UnRegisterReceiver(mPullTagId, this);
Chenjie Yu6736c892017-11-09 10:50:09 -0800152 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700153}
154
Yao Chen427d3722018-03-22 15:21:52 -0700155void ValueMetricProducer::onSlicedConditionMayChangeLocked(bool overallCondition,
Yangster-macb142cc82018-03-30 15:22:08 -0700156 const int64_t eventTime) {
Yangster-mac94e197c2018-01-02 16:03:03 -0800157 VLOG("Metric %lld onSlicedConditionMayChange", (long long)mMetricId);
Chenjie Yub3dda412017-10-24 13:41:59 -0700158}
159
Yangster-macb142cc82018-03-30 15:22:08 -0700160void ValueMetricProducer::dropDataLocked(const int64_t dropTimeNs) {
Yao Chen06dba5d2018-01-26 13:38:16 -0800161 flushIfNeededLocked(dropTimeNs);
162 mPastBuckets.clear();
163}
164
Yangster-maca802d732018-04-24 07:50:38 -0700165void ValueMetricProducer::clearPastBucketsLocked(const int64_t dumpTimeNs) {
166 flushIfNeededLocked(dumpTimeNs);
167 mPastBuckets.clear();
168 mSkippedBuckets.clear();
169}
170
Yangster-macb142cc82018-03-30 15:22:08 -0700171void ValueMetricProducer::onDumpReportLocked(const int64_t dumpTimeNs,
Yangster-mace68f3a52018-04-04 00:01:43 -0700172 const bool include_current_partial_bucket,
Bookatzff71cad2018-09-20 17:17:49 -0700173 const bool erase_data,
Yangster-mac9def8e32018-04-17 13:55:51 -0700174 std::set<string> *str_set,
Yao Chen288c6002017-12-12 13:43:18 -0800175 ProtoOutputStream* protoOutput) {
Yangster-mac94e197c2018-01-02 16:03:03 -0800176 VLOG("metric %lld dump report now...", (long long)mMetricId);
Yangster-mace68f3a52018-04-04 00:01:43 -0700177 if (include_current_partial_bucket) {
178 flushLocked(dumpTimeNs);
179 } else {
180 flushIfNeededLocked(dumpTimeNs);
181 }
David Chen81245fd2018-04-12 14:33:37 -0700182 if (mPastBuckets.empty() && mSkippedBuckets.empty()) {
Yangster-mac635b4b32018-01-23 20:17:35 -0800183 return;
184 }
Yangster-mac94e197c2018-01-02 16:03:03 -0800185 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)mMetricId);
Yangster-mac9def8e32018-04-17 13:55:51 -0700186 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_TIME_BASE, (long long)mTimeBaseNs);
187 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_SIZE, (long long)mBucketSizeNs);
188 // Fills the dimension path if not slicing by ALL.
189 if (!mSliceByPositionALL) {
190 if (!mDimensionsInWhat.empty()) {
191 uint64_t dimenPathToken = protoOutput->start(
192 FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_WHAT);
193 writeDimensionPathToProto(mDimensionsInWhat, protoOutput);
194 protoOutput->end(dimenPathToken);
195 }
196 if (!mDimensionsInCondition.empty()) {
197 uint64_t dimenPathToken = protoOutput->start(
198 FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_PATH_IN_CONDITION);
199 writeDimensionPathToProto(mDimensionsInCondition, protoOutput);
200 protoOutput->end(dimenPathToken);
201 }
202 }
203
Yi Jin5ee07872018-03-05 18:18:27 -0800204 uint64_t protoToken = protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_ID_VALUE_METRICS);
Yao Chen6a8c7992017-11-29 20:02:07 +0000205
David Chen81245fd2018-04-12 14:33:37 -0700206 for (const auto& pair : mSkippedBuckets) {
207 uint64_t wrapperToken =
208 protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_SKIPPED);
Yangster-mac9def8e32018-04-17 13:55:51 -0700209 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_SKIPPED_START_MILLIS,
210 (long long)(NanoToMillis(pair.first)));
211 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_SKIPPED_END_MILLIS,
212 (long long)(NanoToMillis(pair.second)));
David Chen81245fd2018-04-12 14:33:37 -0700213 protoOutput->end(wrapperToken);
214 }
David Chen81245fd2018-04-12 14:33:37 -0700215
Yao Chen93fe3a32017-11-02 13:52:59 -0700216 for (const auto& pair : mPastBuckets) {
Yangster-mac93694462018-01-22 20:49:31 -0800217 const MetricDimensionKey& dimensionKey = pair.first;
Yangster13fb7e42018-03-07 17:30:49 -0800218 VLOG(" dimension key %s", dimensionKey.toString().c_str());
Yi Jin5ee07872018-03-05 18:18:27 -0800219 uint64_t wrapperToken =
Yao Chen288c6002017-12-12 13:43:18 -0800220 protoOutput->start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_DATA);
Chenjie Yub3dda412017-10-24 13:41:59 -0700221
Yangster-mac20877162017-12-22 17:19:39 -0800222 // First fill dimension.
Yangster-mac9def8e32018-04-17 13:55:51 -0700223 if (mSliceByPositionALL) {
224 uint64_t dimensionToken = protoOutput->start(
225 FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_WHAT);
226 writeDimensionToProto(dimensionKey.getDimensionKeyInWhat(), str_set, protoOutput);
227 protoOutput->end(dimensionToken);
228 if (dimensionKey.hasDimensionKeyInCondition()) {
229 uint64_t dimensionInConditionToken = protoOutput->start(
230 FIELD_TYPE_MESSAGE | FIELD_ID_DIMENSION_IN_CONDITION);
231 writeDimensionToProto(dimensionKey.getDimensionKeyInCondition(),
232 str_set, protoOutput);
233 protoOutput->end(dimensionInConditionToken);
234 }
235 } else {
236 writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInWhat(),
237 FIELD_ID_DIMENSION_LEAF_IN_WHAT, str_set, protoOutput);
238 if (dimensionKey.hasDimensionKeyInCondition()) {
239 writeDimensionLeafNodesToProto(dimensionKey.getDimensionKeyInCondition(),
240 FIELD_ID_DIMENSION_LEAF_IN_CONDITION,
241 str_set, protoOutput);
242 }
Yangster-mac93694462018-01-22 20:49:31 -0800243 }
yro2b0f8862017-11-06 14:27:31 -0800244
245 // Then fill bucket_info (ValueBucketInfo).
246 for (const auto& bucket : pair.second) {
Yi Jin5ee07872018-03-05 18:18:27 -0800247 uint64_t bucketInfoToken = protoOutput->start(
Yao Chen288c6002017-12-12 13:43:18 -0800248 FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_BUCKET_INFO);
Yangster-mac9def8e32018-04-17 13:55:51 -0700249
250 if (bucket.mBucketEndNs - bucket.mBucketStartNs != mBucketSizeNs) {
251 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_START_BUCKET_ELAPSED_MILLIS,
252 (long long)NanoToMillis(bucket.mBucketStartNs));
253 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_END_BUCKET_ELAPSED_MILLIS,
254 (long long)NanoToMillis(bucket.mBucketEndNs));
255 } else {
256 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_BUCKET_NUM,
257 (long long)(getBucketNumFromEndTimeNs(bucket.mBucketEndNs)));
258 }
Chenjie Yua0f02242018-07-06 16:14:34 -0700259 if (mValueType == LONG) {
260 protoOutput->write(FIELD_TYPE_INT64 | FIELD_ID_VALUE_LONG,
261 (long long)bucket.mValueLong);
262 } else {
263 protoOutput->write(FIELD_TYPE_DOUBLE | FIELD_ID_VALUE_DOUBLE, bucket.mValueDouble);
264 }
Yao Chen288c6002017-12-12 13:43:18 -0800265 protoOutput->end(bucketInfoToken);
Chenjie Yua0f02242018-07-06 16:14:34 -0700266 VLOG("\t bucket [%lld - %lld] count: %lld, %.2f", (long long)bucket.mBucketStartNs,
267 (long long)bucket.mBucketEndNs, (long long)bucket.mValueLong, bucket.mValueDouble);
yro2b0f8862017-11-06 14:27:31 -0800268 }
Yao Chen288c6002017-12-12 13:43:18 -0800269 protoOutput->end(wrapperToken);
Chenjie Yub3dda412017-10-24 13:41:59 -0700270 }
Yao Chen288c6002017-12-12 13:43:18 -0800271 protoOutput->end(protoToken);
yro2b0f8862017-11-06 14:27:31 -0800272
Yangster-mac94e197c2018-01-02 16:03:03 -0800273 VLOG("metric %lld dump report now...", (long long)mMetricId);
Bookatzff71cad2018-09-20 17:17:49 -0700274 if (erase_data) {
275 mPastBuckets.clear();
276 mSkippedBuckets.clear();
277 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700278}
279
Chenjie Yu1a0a9412018-03-28 10:07:22 -0700280void ValueMetricProducer::onConditionChangedLocked(const bool condition,
Yangster-macb142cc82018-03-30 15:22:08 -0700281 const int64_t eventTimeNs) {
Yao Chen6a8c7992017-11-29 20:02:07 +0000282 mCondition = condition;
Chenjie Yub3dda412017-10-24 13:41:59 -0700283
Chenjie Yu1a0a9412018-03-28 10:07:22 -0700284 if (eventTimeNs < mCurrentBucketStartTimeNs) {
285 VLOG("Skip event due to late arrival: %lld vs %lld", (long long)eventTimeNs,
Chenjie Yua7259ab2017-12-10 08:31:05 -0800286 (long long)mCurrentBucketStartTimeNs);
Yao Chen2794da22017-12-13 16:01:55 -0800287 return;
288 }
289
Chenjie Yu1a0a9412018-03-28 10:07:22 -0700290 flushIfNeededLocked(eventTimeNs);
Chenjie Yua7259ab2017-12-10 08:31:05 -0800291
Chenjie Yua0f02242018-07-06 16:14:34 -0700292 if (mIsPulled) {
Chenjie Yue1361ed2018-07-23 17:33:09 -0700293 pullLocked(eventTimeNs);
294 }
295}
296
297void ValueMetricProducer::pullLocked(const int64_t timestampNs) {
298 vector<std::shared_ptr<LogEvent>> allData;
299 if (mPullerManager->Pull(mPullTagId, timestampNs, &allData)) {
300 if (allData.size() == 0) {
301 return;
Yao Chen6a8c7992017-11-29 20:02:07 +0000302 }
Chenjie Yue1361ed2018-07-23 17:33:09 -0700303 for (const auto& data : allData) {
304 onMatchedLogEventLocked(0, *data);
305 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700306 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700307}
308
309void ValueMetricProducer::onDataPulled(const std::vector<std::shared_ptr<LogEvent>>& allData) {
Yangsterf2bee6f2017-11-29 12:01:05 -0800310 std::lock_guard<std::mutex> lock(mMutex);
311
Yao Chenf09569f2017-12-13 17:00:51 -0800312 if (mCondition == true || mConditionTrackerIndex < 0) {
Chenjie Yub3dda412017-10-24 13:41:59 -0700313 if (allData.size() == 0) {
314 return;
315 }
Chenjie Yua7259ab2017-12-10 08:31:05 -0800316 // For scheduled pulled data, the effective event time is snap to the nearest
317 // bucket boundary to make bucket finalize.
Yangster-macb142cc82018-03-30 15:22:08 -0700318 int64_t realEventTime = allData.at(0)->GetElapsedTimestampNs();
Yangster-mac15f6bbc2018-04-08 11:52:26 -0700319 int64_t eventTime = mTimeBaseNs +
320 ((realEventTime - mTimeBaseNs) / mBucketSizeNs) * mBucketSizeNs;
Chenjie Yua7259ab2017-12-10 08:31:05 -0800321
Chenjie Yue1361ed2018-07-23 17:33:09 -0700322 // close the end of the bucket
Chenjie Yua7259ab2017-12-10 08:31:05 -0800323 mCondition = false;
Chenjie Yub3dda412017-10-24 13:41:59 -0700324 for (const auto& data : allData) {
Yangster-mac330af582018-02-08 15:24:38 -0800325 data->setElapsedTimestampNs(eventTime - 1);
Chenjie Yua7259ab2017-12-10 08:31:05 -0800326 onMatchedLogEventLocked(0, *data);
Chenjie Yub3dda412017-10-24 13:41:59 -0700327 }
Chenjie Yua7259ab2017-12-10 08:31:05 -0800328
Chenjie Yue1361ed2018-07-23 17:33:09 -0700329 // start a new bucket
Chenjie Yua7259ab2017-12-10 08:31:05 -0800330 mCondition = true;
331 for (const auto& data : allData) {
Yangster-mac330af582018-02-08 15:24:38 -0800332 data->setElapsedTimestampNs(eventTime);
Chenjie Yua7259ab2017-12-10 08:31:05 -0800333 onMatchedLogEventLocked(0, *data);
334 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700335 }
336}
337
Yangster-maca78d0082018-03-12 12:02:56 -0700338void ValueMetricProducer::dumpStatesLocked(FILE* out, bool verbose) const {
339 if (mCurrentSlicedBucket.size() == 0) {
340 return;
341 }
342
343 fprintf(out, "ValueMetric %lld dimension size %lu\n", (long long)mMetricId,
344 (unsigned long)mCurrentSlicedBucket.size());
345 if (verbose) {
346 for (const auto& it : mCurrentSlicedBucket) {
Chenjie Yua0f02242018-07-06 16:14:34 -0700347 fprintf(out, "\t(what)%s\t(condition)%s (value)%s\n",
348 it.first.getDimensionKeyInWhat().toString().c_str(),
349 it.first.getDimensionKeyInCondition().toString().c_str(),
350 it.second.value.toString().c_str());
Yangster-maca78d0082018-03-12 12:02:56 -0700351 }
352 }
353}
354
Yangster-mac93694462018-01-22 20:49:31 -0800355bool ValueMetricProducer::hitGuardRailLocked(const MetricDimensionKey& newKey) {
Yao Chenb3561512017-11-21 18:07:17 -0800356 // ===========GuardRail==============
357 // 1. Report the tuple count if the tuple count > soft limit
358 if (mCurrentSlicedBucket.find(newKey) != mCurrentSlicedBucket.end()) {
359 return false;
360 }
Chenjie Yuc5875052018-03-09 10:13:11 -0800361 if (mCurrentSlicedBucket.size() > mDimensionSoftLimit - 1) {
Yao Chenb3561512017-11-21 18:07:17 -0800362 size_t newTupleCount = mCurrentSlicedBucket.size() + 1;
Yangster-mac94e197c2018-01-02 16:03:03 -0800363 StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetricId, newTupleCount);
Yao Chenb3561512017-11-21 18:07:17 -0800364 // 2. Don't add more tuples, we are above the allowed threshold. Drop the data.
Chenjie Yuc5875052018-03-09 10:13:11 -0800365 if (newTupleCount > mDimensionHardLimit) {
Yangster-mac94e197c2018-01-02 16:03:03 -0800366 ALOGE("ValueMetric %lld dropping data for dimension key %s",
Yangster13fb7e42018-03-07 17:30:49 -0800367 (long long)mMetricId, newKey.toString().c_str());
Yao Chenb3561512017-11-21 18:07:17 -0800368 return true;
369 }
370 }
371
372 return false;
373}
374
Chenjie Yua0f02242018-07-06 16:14:34 -0700375const Value getDoubleOrLong(const Value& value) {
376 Value v;
377 switch (value.type) {
378 case INT:
379 v.setLong(value.int_value);
380 break;
381 case LONG:
382 v.setLong(value.long_value);
383 break;
384 case FLOAT:
385 v.setDouble(value.float_value);
386 break;
387 case DOUBLE:
388 v.setDouble(value.double_value);
389 break;
390 default:
391 break;
392 }
393 return v;
394}
395
Yangsterf2bee6f2017-11-29 12:01:05 -0800396void ValueMetricProducer::onMatchedLogEventInternalLocked(
Yangster-mac93694462018-01-22 20:49:31 -0800397 const size_t matcherIndex, const MetricDimensionKey& eventKey,
Yangster-mac20877162017-12-22 17:19:39 -0800398 const ConditionKey& conditionKey, bool condition,
Chenjie Yua7259ab2017-12-10 08:31:05 -0800399 const LogEvent& event) {
Yangster-macb142cc82018-03-30 15:22:08 -0700400 int64_t eventTimeNs = event.GetElapsedTimestampNs();
Yao Chen6a8c7992017-11-29 20:02:07 +0000401 if (eventTimeNs < mCurrentBucketStartTimeNs) {
402 VLOG("Skip event due to late arrival: %lld vs %lld", (long long)eventTimeNs,
403 (long long)mCurrentBucketStartTimeNs);
404 return;
405 }
406
Chenjie Yua7259ab2017-12-10 08:31:05 -0800407 flushIfNeededLocked(eventTimeNs);
408
Yangsterf2bee6f2017-11-29 12:01:05 -0800409 if (hitGuardRailLocked(eventKey)) {
Yangster8de69392017-11-27 13:48:29 -0800410 return;
411 }
Yao Chen6a8c7992017-11-29 20:02:07 +0000412 Interval& interval = mCurrentSlicedBucket[eventKey];
413
Chenjie Yua0f02242018-07-06 16:14:34 -0700414 if (mField > event.size()) {
415 VLOG("Failed to extract value field %d from atom %s. %d", mField, event.ToString().c_str(),
416 (int)event.size());
Yangster-maca7fb12d2018-01-03 17:17:20 -0800417 return;
418 }
Chenjie Yua0f02242018-07-06 16:14:34 -0700419 Value value = getDoubleOrLong(event.getValues()[mField - 1].mValue);
Yao Chen6a8c7992017-11-29 20:02:07 +0000420
Chenjie Yua0f02242018-07-06 16:14:34 -0700421 Value diff;
422 bool hasDiff = false;
423 if (mIsPulled) {
424 // Always require condition for pulled events. In the case of no condition, only pull
425 // on bucket boundaries, in which we fake condition changes.
Chenjie Yua7259ab2017-12-10 08:31:05 -0800426 if (mCondition == true) {
Chenjie Yu6d370f42018-03-25 14:57:30 -0700427 if (!interval.startUpdated) {
428 interval.start = value;
429 interval.startUpdated = true;
430 } else {
Chenjie Yua0f02242018-07-06 16:14:34 -0700431 // Skip it if there is already value recorded for the start. Happens when puller
432 // takes too long to finish. In this case we take the previous value.
Chenjie Yu6d370f42018-03-25 14:57:30 -0700433 VLOG("Already recorded value for this dimension %s", eventKey.toString().c_str());
434 }
Yao Chen6a8c7992017-11-29 20:02:07 +0000435 } else {
Chenjie Yu0ed268b2018-01-10 09:37:10 -0800436 // Generally we expect value to be monotonically increasing.
Chenjie Yu47234642018-05-14 10:14:16 -0700437 // If not, take absolute value or drop it, based on config.
Chenjie Yua7259ab2017-12-10 08:31:05 -0800438 if (interval.startUpdated) {
Yangster613a7e22018-05-08 15:12:30 -0700439 if (value >= interval.start) {
Chenjie Yua0f02242018-07-06 16:14:34 -0700440 diff = (value - interval.start);
441 hasDiff = true;
Chenjie Yu0ed268b2018-01-10 09:37:10 -0800442 } else {
Chenjie Yu47234642018-05-14 10:14:16 -0700443 if (mUseAbsoluteValueOnReset) {
Chenjie Yua0f02242018-07-06 16:14:34 -0700444 diff = value;
445 hasDiff = true;
Chenjie Yu47234642018-05-14 10:14:16 -0700446 } else {
Chenjie Yua0f02242018-07-06 16:14:34 -0700447 VLOG("Dropping data for atom %d, prev: %s, now: %s", mPullTagId,
448 interval.start.toString().c_str(), value.toString().c_str());
Chenjie Yu47234642018-05-14 10:14:16 -0700449 }
Chenjie Yu0ed268b2018-01-10 09:37:10 -0800450 }
Chenjie Yua7259ab2017-12-10 08:31:05 -0800451 interval.startUpdated = false;
Yao Chen6a8c7992017-11-29 20:02:07 +0000452 } else {
Chenjie Yua0f02242018-07-06 16:14:34 -0700453 VLOG("No start for matching end %s", value.toString().c_str());
Yao Chen6a8c7992017-11-29 20:02:07 +0000454 }
455 }
Chenjie Yua0f02242018-07-06 16:14:34 -0700456 } else {
457 // for pushed events, only aggregate when sliced condition is true
458 if (condition == true || mConditionTrackerIndex < 0) {
459 diff = value;
460 hasDiff = true;
Chenjie Yu021e2532018-05-16 12:23:07 -0700461 }
Yangster8de69392017-11-27 13:48:29 -0800462 }
Chenjie Yua0f02242018-07-06 16:14:34 -0700463 if (hasDiff) {
464 if (interval.hasValue) {
465 switch (mAggregationType) {
466 case ValueMetric::SUM:
467 // for AVG, we add up and take average when flushing the bucket
468 case ValueMetric::AVG:
469 interval.value += diff;
470 break;
471 case ValueMetric::MIN:
472 interval.value = diff < interval.value ? diff : interval.value;
473 break;
474 case ValueMetric::MAX:
475 interval.value = diff > interval.value ? diff : interval.value;
476 break;
477 default:
478 break;
479 }
480 } else {
481 interval.value = diff;
482 interval.hasValue = true;
483 }
484 interval.sampleSize += 1;
485 }
Bookatzde1b55622017-12-14 18:38:27 -0800486
Chenjie Yua0f02242018-07-06 16:14:34 -0700487 // TODO: propgate proper values down stream when anomaly support doubles
488 long wholeBucketVal = interval.value.long_value;
David Chen27785a82018-01-19 17:06:45 -0800489 auto prev = mCurrentFullBucket.find(eventKey);
490 if (prev != mCurrentFullBucket.end()) {
491 wholeBucketVal += prev->second;
492 }
Bookatzde1b55622017-12-14 18:38:27 -0800493 for (auto& tracker : mAnomalyTrackers) {
David Chen27785a82018-01-19 17:06:45 -0800494 tracker->detectAndDeclareAnomaly(eventTimeNs, mCurrentBucketNum, eventKey, wholeBucketVal);
Bookatzde1b55622017-12-14 18:38:27 -0800495 }
Yangster8de69392017-11-27 13:48:29 -0800496}
497
Yangster-macb142cc82018-03-30 15:22:08 -0700498void ValueMetricProducer::flushIfNeededLocked(const int64_t& eventTimeNs) {
499 int64_t currentBucketEndTimeNs = getCurrentBucketEndTimeNs();
David Chen27785a82018-01-19 17:06:45 -0800500
Yangster-mac15f6bbc2018-04-08 11:52:26 -0700501 if (eventTimeNs < currentBucketEndTimeNs) {
Chenjie Yub3dda412017-10-24 13:41:59 -0700502 VLOG("eventTime is %lld, less than next bucket start time %lld", (long long)eventTimeNs,
David Chen27785a82018-01-19 17:06:45 -0800503 (long long)(currentBucketEndTimeNs));
Chenjie Yub3dda412017-10-24 13:41:59 -0700504 return;
505 }
David Chen27785a82018-01-19 17:06:45 -0800506
507 flushCurrentBucketLocked(eventTimeNs);
508
509 int64_t numBucketsForward = 1 + (eventTimeNs - currentBucketEndTimeNs) / mBucketSizeNs;
510 mCurrentBucketStartTimeNs = currentBucketEndTimeNs + (numBucketsForward - 1) * mBucketSizeNs;
511 mCurrentBucketNum += numBucketsForward;
512
513 if (numBucketsForward > 1) {
514 VLOG("Skipping forward %lld buckets", (long long)numBucketsForward);
515 }
516 VLOG("metric %lld: new bucket start time: %lld", (long long)mMetricId,
517 (long long)mCurrentBucketStartTimeNs);
518}
519
Yangster-macb142cc82018-03-30 15:22:08 -0700520void ValueMetricProducer::flushCurrentBucketLocked(const int64_t& eventTimeNs) {
Chenjie Yub3dda412017-10-24 13:41:59 -0700521 VLOG("finalizing bucket for %ld, dumping %d slices", (long)mCurrentBucketStartTimeNs,
522 (int)mCurrentSlicedBucket.size());
Yangster-macb142cc82018-03-30 15:22:08 -0700523 int64_t fullBucketEndTimeNs = getCurrentBucketEndTimeNs();
David Chen27785a82018-01-19 17:06:45 -0800524
yro2b0f8862017-11-06 14:27:31 -0800525 ValueBucket info;
526 info.mBucketStartNs = mCurrentBucketStartTimeNs;
David Chen27785a82018-01-19 17:06:45 -0800527 if (eventTimeNs < fullBucketEndTimeNs) {
528 info.mBucketEndNs = eventTimeNs;
529 } else {
530 info.mBucketEndNs = fullBucketEndTimeNs;
531 }
Chenjie Yub3dda412017-10-24 13:41:59 -0700532
David Chen81245fd2018-04-12 14:33:37 -0700533 if (info.mBucketEndNs - mCurrentBucketStartTimeNs >= mMinBucketSizeNs) {
534 // The current bucket is large enough to keep.
David Chen81245fd2018-04-12 14:33:37 -0700535 for (const auto& slice : mCurrentSlicedBucket) {
David Chen81245fd2018-04-12 14:33:37 -0700536 if (slice.second.hasValue) {
Chenjie Yua0f02242018-07-06 16:14:34 -0700537 info.mValueLong = slice.second.value.long_value;
538 info.mValueDouble = (double)slice.second.value.long_value / slice.second.sampleSize;
David Chen81245fd2018-04-12 14:33:37 -0700539 // it will auto create new vector of ValuebucketInfo if the key is not found.
540 auto& bucketList = mPastBuckets[slice.first];
541 bucketList.push_back(info);
542 }
Chenjie Yuae63b0a2018-04-10 14:59:31 -0700543 }
David Chen81245fd2018-04-12 14:33:37 -0700544 } else {
545 mSkippedBuckets.emplace_back(info.mBucketStartNs, info.mBucketEndNs);
Chenjie Yub3dda412017-10-24 13:41:59 -0700546 }
547
David Chen27785a82018-01-19 17:06:45 -0800548 if (eventTimeNs > fullBucketEndTimeNs) { // If full bucket, send to anomaly tracker.
549 // Accumulate partial buckets with current value and then send to anomaly tracker.
550 if (mCurrentFullBucket.size() > 0) {
551 for (const auto& slice : mCurrentSlicedBucket) {
Chenjie Yua0f02242018-07-06 16:14:34 -0700552 // TODO: fix this when anomaly can accept double values
553 mCurrentFullBucket[slice.first] += slice.second.value.long_value;
David Chen27785a82018-01-19 17:06:45 -0800554 }
555 for (const auto& slice : mCurrentFullBucket) {
556 for (auto& tracker : mAnomalyTrackers) {
557 if (tracker != nullptr) {
558 tracker->addPastBucket(slice.first, slice.second, mCurrentBucketNum);
559 }
560 }
561 }
562 mCurrentFullBucket.clear();
563 } else {
564 // Skip aggregating the partial buckets since there's no previous partial bucket.
565 for (const auto& slice : mCurrentSlicedBucket) {
566 for (auto& tracker : mAnomalyTrackers) {
567 if (tracker != nullptr) {
Chenjie Yua0f02242018-07-06 16:14:34 -0700568 // TODO: fix this when anomaly can accept double values
569 tracker->addPastBucket(slice.first, slice.second.value.long_value,
570 mCurrentBucketNum);
David Chen27785a82018-01-19 17:06:45 -0800571 }
572 }
573 }
574 }
575 } else {
576 // Accumulate partial bucket.
577 for (const auto& slice : mCurrentSlicedBucket) {
Chenjie Yua0f02242018-07-06 16:14:34 -0700578 // TODO: fix this when anomaly can accept double values
579 mCurrentFullBucket[slice.first] += slice.second.value.long_value;
David Chen27785a82018-01-19 17:06:45 -0800580 }
581 }
582
Chenjie Yub3dda412017-10-24 13:41:59 -0700583 // Reset counters
Chenjie Yua7259ab2017-12-10 08:31:05 -0800584 mCurrentSlicedBucket.clear();
Chenjie Yub3dda412017-10-24 13:41:59 -0700585}
586
Yangsterf2bee6f2017-11-29 12:01:05 -0800587size_t ValueMetricProducer::byteSizeLocked() const {
Yangster-mace2cd6d52017-11-09 20:38:30 -0800588 size_t totalSize = 0;
589 for (const auto& pair : mPastBuckets) {
590 totalSize += pair.second.size() * kBucketSize;
591 }
592 return totalSize;
yro2b0f8862017-11-06 14:27:31 -0800593}
594
Chenjie Yub3dda412017-10-24 13:41:59 -0700595} // namespace statsd
596} // namespace os
Yao Chen93fe3a32017-11-02 13:52:59 -0700597} // namespace android