Persist active metrics to disk and read back
For metric activation that spans across boots, we need to persist active
metrics onto disk upon shutdown and load them on boot.
Bug: 123904359
Test: unit test
Change-Id: I5a4142a42595c8c132175fb574c3aa2ad30dcac0
diff --git a/cmds/statsd/src/StatsLogProcessor.cpp b/cmds/statsd/src/StatsLogProcessor.cpp
index 69cb264..250f5bf 100644
--- a/cmds/statsd/src/StatsLogProcessor.cpp
+++ b/cmds/statsd/src/StatsLogProcessor.cpp
@@ -14,18 +14,19 @@
* limitations under the License.
*/
-#define DEBUG false // STOPSHIP if true
+#define DEBUG false // STOPSHIP if true
#include "Log.h"
#include "statslog.h"
#include <android-base/file.h>
#include <dirent.h>
+#include <frameworks/base/cmds/statsd/src/active_config_list.pb.h>
#include "StatsLogProcessor.h"
-#include "stats_log_util.h"
#include "android-base/stringprintf.h"
+#include "external/StatsPullerManager.h"
#include "guardrail/StatsdStats.h"
#include "metrics/CountMetricProducer.h"
-#include "external/StatsPullerManager.h"
+#include "stats_log_util.h"
#include "stats_util.h"
#include "storage/StorageManager.h"
@@ -67,9 +68,17 @@
const int FIELD_ID_DUMP_REPORT_REASON = 8;
const int FIELD_ID_STRINGS = 9;
+const int FIELD_ID_ACTIVE_CONFIG_LIST = 1;
+const int FIELD_ID_CONFIG_ID = 1;
+const int FIELD_ID_CONFIG_UID = 2;
+const int FIELD_ID_ACTIVE_METRIC = 3;
+const int FIELD_ID_METRIC_ID = 1;
+const int FIELD_ID_TIME_TO_LIVE_NANOS = 2;
+
#define NS_PER_HOUR 3600 * NS_PER_SEC
#define STATS_DATA_DIR "/data/misc/stats-data"
+#define STATS_ACTIVE_METRIC_DIR "/data/misc/stats-active-metric"
// Cool down period for writing data to disk to avoid overwriting files.
#define WRITE_DATA_COOL_DOWN_SEC 5
@@ -507,6 +516,70 @@
mOnDiskDataConfigs.insert(key);
}
+void StatsLogProcessor::WriteMetricsActivationToDisk(int64_t currentTimeNs) {
+ std::lock_guard<std::mutex> lock(mMetricsMutex);
+ ProtoOutputStream proto;
+
+ for (const auto& pair : mMetricsManagers) {
+ uint64_t activeConfigListToken = proto.start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |
+ FIELD_ID_ACTIVE_CONFIG_LIST);
+ proto.write(FIELD_TYPE_INT64 | FIELD_ID_CONFIG_ID, (long long)pair.first.GetId());
+ proto.write(FIELD_TYPE_INT32 | FIELD_ID_CONFIG_UID, pair.first.GetUid());
+
+ vector<const MetricProducer*> acrtiveMetrics;
+ pair.second->getActiveMetrics(acrtiveMetrics);
+ for (const MetricProducer* metric : acrtiveMetrics) {
+ if (metric->isActive()) {
+ uint64_t metricToken = proto.start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED |
+ FIELD_ID_ACTIVE_METRIC);
+ proto.write(FIELD_TYPE_INT64 | FIELD_ID_METRIC_ID,
+ (long long)metric->getMetricId());
+ proto.write(FIELD_TYPE_INT64 | FIELD_ID_TIME_TO_LIVE_NANOS,
+ (long long)metric->getRemainingTtlNs(currentTimeNs));
+ proto.end(metricToken);
+ }
+ }
+ proto.end(activeConfigListToken);
+ }
+
+ string file_name = StringPrintf("%s/active_metrics", STATS_ACTIVE_METRIC_DIR);
+ StorageManager::deleteFile(file_name.c_str());
+ android::base::unique_fd fd(
+ open(file_name.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, S_IRUSR | S_IWUSR));
+ if (fd == -1) {
+ ALOGE("Attempt to write %s but failed", file_name.c_str());
+ return;
+ }
+ proto.flush(fd.get());
+}
+
+void StatsLogProcessor::LoadMetricsActivationFromDisk() {
+ string file_name = StringPrintf("%s/active_metrics", STATS_ACTIVE_METRIC_DIR);
+ int fd = open(file_name.c_str(), O_RDONLY | O_CLOEXEC);
+ if (fd != -1) {
+ string content;
+ if (android::base::ReadFdToString(fd, &content)) {
+ ActiveConfigList activeConfigList;
+ if (activeConfigList.ParseFromString(content)) {
+ for (int i = 0; i < activeConfigList.active_config_size(); i++) {
+ const auto& config = activeConfigList.active_config(i);
+ ConfigKey key(config.uid(), config.config_id());
+ auto it = mMetricsManagers.find(key);
+ if (it == mMetricsManagers.end()) {
+ ALOGE("No config found for config %s", key.ToString().c_str());
+ continue;
+ }
+ VLOG("Setting active config %s", key.ToString().c_str());
+ it->second->setActiveMetrics(config, mTimeBaseNs);
+ }
+ }
+ VLOG("Successfully loaded %d active configs.", activeConfigList.active_config_size());
+ }
+ close(fd);
+ }
+ StorageManager::deleteFile(file_name.c_str());
+}
+
void StatsLogProcessor::WriteDataToDiskLocked(const DumpReportReason dumpReportReason) {
const int64_t timeNs = getElapsedRealtimeNs();
// Do not write to disk if we already have in the last few seconds.
diff --git a/cmds/statsd/src/StatsLogProcessor.h b/cmds/statsd/src/StatsLogProcessor.h
index a5ce9b6..77d9a2f 100644
--- a/cmds/statsd/src/StatsLogProcessor.h
+++ b/cmds/statsd/src/StatsLogProcessor.h
@@ -80,6 +80,12 @@
/* Flushes data to disk. Data on memory will be gone after written to disk. */
void WriteDataToDisk(const DumpReportReason dumpReportReason);
+ /* Persist metric activation status onto disk. */
+ void WriteMetricsActivationToDisk(int64_t currentTimeNs);
+
+ /* Load metric activation status from disk. */
+ void LoadMetricsActivationFromDisk();
+
// Reset all configs.
void resetConfigs();
@@ -188,6 +194,8 @@
FRIEND_TEST(StatsLogProcessorTest, TestRateLimitByteSize);
FRIEND_TEST(StatsLogProcessorTest, TestRateLimitBroadcast);
FRIEND_TEST(StatsLogProcessorTest, TestDropWhenByteSizeTooLarge);
+ FRIEND_TEST(StatsLogProcessorTest, TestActiveConfigMetricDiskWriteRead);
+
FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration1);
FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration2);
FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensionsForSumDuration3);
diff --git a/cmds/statsd/src/StatsService.cpp b/cmds/statsd/src/StatsService.cpp
index b26c713..86bf3ec 100644
--- a/cmds/statsd/src/StatsService.cpp
+++ b/cmds/statsd/src/StatsService.cpp
@@ -161,7 +161,8 @@
mConfigManager = new ConfigManager();
mProcessor = new StatsLogProcessor(
mUidMap, mPullerManager, mAnomalyAlarmMonitor, mPeriodicAlarmMonitor,
- getElapsedRealtimeNs(), [this](const ConfigKey& key) {
+ getElapsedRealtimeNs(),
+ [this](const ConfigKey& key) {
sp<IStatsCompanionService> sc = getStatsCompanionService();
auto receiver = mConfigManager->GetConfigReceiver(key);
if (sc == nullptr) {
@@ -867,6 +868,7 @@
ENFORCE_UID(AID_SYSTEM);
VLOG("StatsService::informDeviceShutdown");
mProcessor->WriteDataToDisk(DEVICE_SHUTDOWN);
+ mProcessor->WriteMetricsActivationToDisk(getElapsedRealtimeNs());
return Status::ok();
}
@@ -901,6 +903,7 @@
void StatsService::Startup() {
mConfigManager->Startup();
+ mProcessor->LoadMetricsActivationFromDisk();
}
void StatsService::Terminate() {
diff --git a/cmds/statsd/src/active_config_list.proto b/cmds/statsd/src/active_config_list.proto
new file mode 100644
index 0000000..0e9ee03
--- /dev/null
+++ b/cmds/statsd/src/active_config_list.proto
@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto2";
+
+package android.os.statsd;
+option java_package = "com.android.os";
+option java_multiple_files = true;
+option java_outer_classname = "ActiveConfigProto";
+
+message ActiveMetric {
+ // metric id
+ optional int64 metric_id = 1;
+ // Remaining time to live in nano seconds. -1 for infinity.
+ optional int64 time_to_live_nanos = 2;
+}
+
+message ActiveConfig {
+ // config id
+ optional int64 config_id = 1;
+ // config uid
+ optional int32 uid = 2;
+ // metrics
+ repeated ActiveMetric active_metric = 3;
+}
+
+// all configs and their metrics on device.
+message ActiveConfigList {
+ repeated ActiveConfig active_config = 1;
+}
\ No newline at end of file
diff --git a/cmds/statsd/src/metrics/MetricProducer.cpp b/cmds/statsd/src/metrics/MetricProducer.cpp
index f87849e..b362e37 100644
--- a/cmds/statsd/src/metrics/MetricProducer.cpp
+++ b/cmds/statsd/src/metrics/MetricProducer.cpp
@@ -112,6 +112,28 @@
mIsActive = true;
}
+void MetricProducer::setActiveLocked(int64_t currentTimeNs, int64_t remainingTtlNs) {
+ if (mEventActivationMap.size() == 0) {
+ return;
+ }
+ auto& activation = mEventActivationMap.begin()->second;
+ activation.activation_ns = currentTimeNs + remainingTtlNs - activation.ttl_ns;
+ activation.state = kActive;
+ mIsActive = true;
+ VLOG("setting new activation time to %lld, %lld, %lld", (long long)activation.activation_ns,
+ (long long)currentTimeNs, (long long)remainingTtlNs);
+}
+
+int64_t MetricProducer::getRemainingTtlNsLocked(int64_t currentTimeNs) const {
+ int64_t maxTtl = 0;
+ for (const auto& activation : mEventActivationMap) {
+ if (activation.second.state == kActive) {
+ maxTtl = std::max(maxTtl, activation.second.ttl_ns + activation.second.activation_ns -
+ currentTimeNs);
+ }
+ }
+ return maxTtl;
+}
} // namespace statsd
} // namespace os
diff --git a/cmds/statsd/src/metrics/MetricProducer.h b/cmds/statsd/src/metrics/MetricProducer.h
index 09e2409..ca37bbb 100644
--- a/cmds/statsd/src/metrics/MetricProducer.h
+++ b/cmds/statsd/src/metrics/MetricProducer.h
@@ -179,10 +179,21 @@
mBucketSizeNs = bucketSize;
}
- inline const int64_t& getMetricId() {
+ inline const int64_t& getMetricId() const {
return mMetricId;
}
+ int64_t getRemainingTtlNs(int64_t currentTimeNs) const {
+ std::lock_guard<std::mutex> lock(mMutex);
+ return getRemainingTtlNsLocked(currentTimeNs);
+ }
+
+ // Set metric to active for ttlNs.
+ void setActive(int64_t currentTimeNs, int64_t remainingTtlNs) {
+ std::lock_guard<std::mutex> lock(mMutex);
+ setActiveLocked(currentTimeNs, remainingTtlNs);
+ }
+
// Let MetricProducer drop in-memory data to save memory.
// We still need to keep future data valid and anomaly tracking work, which means we will
// have to flush old data, informing anomaly trackers then safely drop old data.
@@ -202,6 +213,11 @@
activateLocked(activationTrackerIndex, elapsedTimestampNs);
}
+ bool isActive() const {
+ std::lock_guard<std::mutex> lock(mMutex);
+ return isActiveLocked();
+ }
+
void addActivation(int activationTrackerIndex, int64_t ttl_seconds);
void flushIfExpire(int64_t elapsedTimestampNs);
@@ -227,6 +243,10 @@
return mIsActive;
}
+ int64_t getRemainingTtlNsLocked(int64_t currentTimeNs) const;
+
+ void setActiveLocked(int64_t currentTimeNs, int64_t remainingTtlNs);
+
/**
* Flushes the current bucket if the eventTime is after the current bucket's end time. This will
also flush the current partial bucket in memory.
@@ -348,6 +368,8 @@
bool mIsActive;
FRIEND_TEST(MetricActivationE2eTest, TestCountMetric);
+
+ FRIEND_TEST(StatsLogProcessorTest, TestActiveConfigMetricDiskWriteRead);
};
} // namespace statsd
diff --git a/cmds/statsd/src/metrics/MetricsManager.cpp b/cmds/statsd/src/metrics/MetricsManager.cpp
index dd969c0..ca68117 100644
--- a/cmds/statsd/src/metrics/MetricsManager.cpp
+++ b/cmds/statsd/src/metrics/MetricsManager.cpp
@@ -122,6 +122,13 @@
StatsdStats::getInstance().noteConfigReceived(
key, mAllMetricProducers.size(), mAllConditionTrackers.size(), mAllAtomMatchers.size(),
mAllAnomalyTrackers.size(), mAnnotations, mConfigValid);
+ // Check active
+ for (const auto& metric : mAllMetricProducers) {
+ if (metric->isActive()) {
+ mIsActive = true;
+ break;
+ }
+ }
}
MetricsManager::~MetricsManager() {
@@ -304,8 +311,10 @@
int tagId = event.GetTagId();
int64_t eventTimeNs = event.GetElapsedTimestampNs();
+ bool isActive = false;
for (int metric : mMetricIndexesWithActivation) {
mAllMetricProducers[metric]->flushIfExpire(eventTimeNs);
+ isActive |= mAllMetricProducers[metric]->isActive();
}
if (mTagIds.find(tagId) == mTagIds.end()) {
@@ -323,10 +332,13 @@
if (matcherCache[it.first] == MatchingState::kMatched) {
for (int metricIndex : it.second) {
mAllMetricProducers[metricIndex]->activate(it.first, eventTimeNs);
+ isActive |= mAllMetricProducers[metricIndex]->isActive();
}
}
}
+ mIsActive = isActive;
+
// A bitmap to see which ConditionTracker needs to be re-evaluated.
vector<bool> conditionToBeEvaluated(mAllConditionTrackers.size(), false);
@@ -418,6 +430,25 @@
return totalSize;
}
+void MetricsManager::setActiveMetrics(ActiveConfig config, int64_t currentTimeNs) {
+ if (config.active_metric_size() == 0) {
+ ALOGW("No active metric for config %s", mConfigKey.ToString().c_str());
+ return;
+ }
+
+ for (int i = 0; i < config.active_metric_size(); i++) {
+ for (int metric : mMetricIndexesWithActivation) {
+ if (mAllMetricProducers[metric]->getMetricId() == config.active_metric(i).metric_id()) {
+ VLOG("Setting active metric: %lld",
+ (long long)mAllMetricProducers[metric]->getMetricId());
+ mAllMetricProducers[metric]->setActive(
+ currentTimeNs, config.active_metric(i).time_to_live_nanos());
+ mIsActive = true;
+ }
+ }
+ }
+}
+
} // namespace statsd
} // namespace os
} // namespace android
diff --git a/cmds/statsd/src/metrics/MetricsManager.h b/cmds/statsd/src/metrics/MetricsManager.h
index a31efbd..80982c3 100644
--- a/cmds/statsd/src/metrics/MetricsManager.h
+++ b/cmds/statsd/src/metrics/MetricsManager.h
@@ -16,12 +16,13 @@
#pragma once
-#include "external/StatsPullerManager.h"
+#include <frameworks/base/cmds/statsd/src/active_config_list.pb.h>
#include "anomaly/AlarmMonitor.h"
#include "anomaly/AlarmTracker.h"
#include "anomaly/AnomalyTracker.h"
#include "condition/ConditionTracker.h"
#include "config/ConfigKey.h"
+#include "external/StatsPullerManager.h"
#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h"
#include "logd/LogEvent.h"
#include "matchers/LogMatchingTracker.h"
@@ -123,6 +124,20 @@
// Does not change the state.
virtual size_t byteSize();
+ inline bool isActive() const {
+ return mIsActive;
+ }
+
+ inline void getActiveMetrics(std::vector<const MetricProducer*>& metrics) const {
+ for (const auto& metric : mAllMetricProducers) {
+ if (metric->isActive()) {
+ metrics.push_back(metric.get());
+ }
+ }
+ }
+
+ void setActiveMetrics(ActiveConfig config, int64_t currentTimeNs);
+
private:
// For test only.
inline int64_t getTtlEndNs() const { return mTtlEndNs; }
@@ -216,6 +231,9 @@
// The metrics that don't need to be uploaded or even reported.
std::set<int64_t> mNoReportMetricIds;
+ // Any metric active means the config is active.
+ bool mIsActive;
+
FRIEND_TEST(WakelockDurationE2eTest, TestAggregatedPredicateDimensions);
FRIEND_TEST(MetricConditionLinkE2eTest, TestMultiplePredicatesAndLinks);
FRIEND_TEST(AttributionE2eTest, TestAttributionMatchAndSliceByFirstUid);
@@ -247,6 +265,8 @@
FRIEND_TEST(AlarmE2eTest, TestMultipleAlarms);
FRIEND_TEST(ConfigTtlE2eTest, TestCountMetric);
FRIEND_TEST(MetricActivationE2eTest, TestCountMetric);
+
+ FRIEND_TEST(StatsLogProcessorTest, TestActiveConfigMetricDiskWriteRead);
};
} // namespace statsd