Load Metadata from Disk
Bug: 148280505
Test: bit statsd_test:*
Test: Manual - Added Config, Tested System Server Crash
Test: Manual - Added Config, Tested Device restart
Change-Id: Icd1b1e57bbf24e4a5676820a2b955f2034d4b37d
diff --git a/cmds/statsd/Android.bp b/cmds/statsd/Android.bp
index 750b942..ac537cc 100644
--- a/cmds/statsd/Android.bp
+++ b/cmds/statsd/Android.bp
@@ -340,6 +340,7 @@
"tests/log_event/LogEventQueue_test.cpp",
"tests/LogEntryMatcher_test.cpp",
"tests/LogEvent_test.cpp",
+ "tests/metadata_util_test.cpp",
"tests/metrics/CountMetricProducer_test.cpp",
"tests/metrics/DurationMetricProducer_test.cpp",
"tests/metrics/EventMetricProducer_test.cpp",
diff --git a/cmds/statsd/src/StatsLogProcessor.cpp b/cmds/statsd/src/StatsLogProcessor.cpp
index 45cb07b..0a61572 100644
--- a/cmds/statsd/src/StatsLogProcessor.cpp
+++ b/cmds/statsd/src/StatsLogProcessor.cpp
@@ -902,6 +902,60 @@
}
}
+void StatsLogProcessor::LoadMetadataFromDisk(int64_t currentWallClockTimeNs,
+ int64_t systemElapsedTimeNs) {
+ std::lock_guard<std::mutex> lock(mMetricsMutex);
+ string file_name = StringPrintf("%s/metadata", STATS_METADATA_DIR);
+ int fd = open(file_name.c_str(), O_RDONLY | O_CLOEXEC);
+ if (-1 == fd) {
+ VLOG("Attempt to read %s but failed", file_name.c_str());
+ StorageManager::deleteFile(file_name.c_str());
+ return;
+ }
+ string content;
+ if (!android::base::ReadFdToString(fd, &content)) {
+ ALOGE("Attempt to read %s but failed", file_name.c_str());
+ close(fd);
+ StorageManager::deleteFile(file_name.c_str());
+ return;
+ }
+
+ close(fd);
+
+ metadata::StatsMetadataList statsMetadataList;
+ if (!statsMetadataList.ParseFromString(content)) {
+ ALOGE("Attempt to read %s but failed; failed to metadata", file_name.c_str());
+ StorageManager::deleteFile(file_name.c_str());
+ return;
+ }
+ SetMetadataStateLocked(statsMetadataList, currentWallClockTimeNs, systemElapsedTimeNs);
+ StorageManager::deleteFile(file_name.c_str());
+}
+
+void StatsLogProcessor::SetMetadataState(const metadata::StatsMetadataList& statsMetadataList,
+ int64_t currentWallClockTimeNs,
+ int64_t systemElapsedTimeNs) {
+ std::lock_guard<std::mutex> lock(mMetricsMutex);
+ SetMetadataStateLocked(statsMetadataList, currentWallClockTimeNs, systemElapsedTimeNs);
+}
+
+void StatsLogProcessor::SetMetadataStateLocked(
+ const metadata::StatsMetadataList& statsMetadataList,
+ int64_t currentWallClockTimeNs,
+ int64_t systemElapsedTimeNs) {
+ for (const metadata::StatsMetadata& metadata : statsMetadataList.stats_metadata()) {
+ ConfigKey key(metadata.config_key().uid(), metadata.config_key().config_id());
+ auto it = mMetricsManagers.find(key);
+ if (it == mMetricsManagers.end()) {
+ ALOGE("No config found for configKey %s", key.ToString().c_str());
+ continue;
+ }
+ VLOG("Setting metadata %s", key.ToString().c_str());
+ it->second->loadMetadata(metadata, currentWallClockTimeNs, systemElapsedTimeNs);
+ }
+ VLOG("Successfully loaded %d metadata.", statsMetadataList.stats_metadata_size());
+}
+
void StatsLogProcessor::WriteActiveConfigsToProtoOutputStream(
int64_t currentTimeNs, const DumpReportReason reason, ProtoOutputStream* proto) {
std::lock_guard<std::mutex> lock(mMetricsMutex);
diff --git a/cmds/statsd/src/StatsLogProcessor.h b/cmds/statsd/src/StatsLogProcessor.h
index 760785a..14585c3 100644
--- a/cmds/statsd/src/StatsLogProcessor.h
+++ b/cmds/statsd/src/StatsLogProcessor.h
@@ -98,6 +98,15 @@
int64_t systemElapsedTimeNs,
metadata::StatsMetadataList* metadataList);
+ /* Load stats metadata for configs and metrics from disk. */
+ void LoadMetadataFromDisk(int64_t currentWallClockTimeNs,
+ int64_t systemElapsedTimeNs);
+
+ /* Sets the metadata for all configs and metrics */
+ void SetMetadataState(const metadata::StatsMetadataList& statsMetadataList,
+ int64_t currentWallClockTimeNs,
+ int64_t systemElapsedTimeNs);
+
/* Sets the active status/ttl for all configs and metrics to the status in ActiveConfigList. */
void SetConfigsActiveState(const ActiveConfigList& activeConfigList, int64_t currentTimeNs);
@@ -182,6 +191,10 @@
void SetConfigsActiveStateLocked(const ActiveConfigList& activeConfigList,
int64_t currentTimeNs);
+ void SetMetadataStateLocked(const metadata::StatsMetadataList& statsMetadataList,
+ int64_t currentWallClockTimeNs,
+ int64_t systemElapsedTimeNs);
+
void WriteMetadataToProtoLocked(int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs,
metadata::StatsMetadataList* metadataList);
@@ -297,6 +310,7 @@
FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_multiple_buckets);
FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk_no_data_written);
FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk);
+ FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_load_refractory_from_disk);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_single_bucket);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_multiple_buckets);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_long_refractory_period);
diff --git a/cmds/statsd/src/StatsService.cpp b/cmds/statsd/src/StatsService.cpp
index 03955bc..f7e135a 100644
--- a/cmds/statsd/src/StatsService.cpp
+++ b/cmds/statsd/src/StatsService.cpp
@@ -1042,6 +1042,7 @@
void StatsService::Startup() {
mConfigManager->Startup();
mProcessor->LoadActiveConfigsFromDisk();
+ mProcessor->LoadMetadataFromDisk(getWallClockNs(), getElapsedRealtimeNs());
}
void StatsService::Terminate() {
@@ -1298,6 +1299,7 @@
mProcessor->SetConfigsActiveState(activeConfigs, systemServerRestartNs);
}
}
+ mProcessor->SetMetadataState(metadataList, getWallClockNs(), systemServerRestartNs);
}
mAnomalyAlarmMonitor->setStatsCompanionService(nullptr);
mPeriodicAlarmMonitor->setStatsCompanionService(nullptr);
diff --git a/cmds/statsd/src/anomaly/AnomalyTracker.cpp b/cmds/statsd/src/anomaly/AnomalyTracker.cpp
index 2ff2589..619752c 100644
--- a/cmds/statsd/src/anomaly/AnomalyTracker.cpp
+++ b/cmds/statsd/src/anomaly/AnomalyTracker.cpp
@@ -298,6 +298,24 @@
return metadataWritten;
}
+void AnomalyTracker::loadAlertMetadata(
+ const metadata::AlertMetadata& alertMetadata,
+ int64_t currentWallClockTimeNs,
+ int64_t systemElapsedTimeNs) {
+ for (const metadata::AlertDimensionKeyedData& keyedData :
+ alertMetadata.alert_dim_keyed_data()) {
+ if ((uint64_t) keyedData.last_refractory_ends_sec() < currentWallClockTimeNs / NS_PER_SEC) {
+ // Do not update the timestamp if it has already expired.
+ continue;
+ }
+ MetricDimensionKey metricKey = loadMetricDimensionKeyFromProto(
+ keyedData.dimension_key());
+ int32_t refractoryPeriodEndsSec = (int32_t) keyedData.last_refractory_ends_sec() -
+ currentWallClockTimeNs / NS_PER_SEC + systemElapsedTimeNs / NS_PER_SEC;
+ mRefractoryPeriodEndsSec[metricKey] = refractoryPeriodEndsSec;
+ }
+}
+
} // namespace statsd
} // namespace os
} // namespace android
diff --git a/cmds/statsd/src/anomaly/AnomalyTracker.h b/cmds/statsd/src/anomaly/AnomalyTracker.h
index 0daa535..bf36a3b 100644
--- a/cmds/statsd/src/anomaly/AnomalyTracker.h
+++ b/cmds/statsd/src/anomaly/AnomalyTracker.h
@@ -24,7 +24,7 @@
#include "AlarmMonitor.h"
#include "config/ConfigKey.h"
#include "frameworks/base/cmds/statsd/src/statsd_config.pb.h" // Alert
-#include "frameworks/base/cmds/statsd/src/statsd_metadata.pb.h" // AlertMetadata
+#include "frameworks/base/cmds/statsd/src/statsd_metadata.pb.h" // AlertMetadata
#include "stats_util.h" // HashableDimensionKey and DimToValMap
namespace android {
@@ -119,6 +119,11 @@
int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs, metadata::AlertMetadata* alertMetadata);
+ void loadAlertMetadata(
+ const metadata::AlertMetadata& alertMetadata,
+ int64_t currentWallClockTimeNs,
+ int64_t systemElapsedTimeNs);
+
protected:
// For testing only.
// Returns the alarm timestamp in seconds for the query dimension if it exists. Otherwise
diff --git a/cmds/statsd/src/metadata_util.cpp b/cmds/statsd/src/metadata_util.cpp
index bfe850f..27ee59b 100644
--- a/cmds/statsd/src/metadata_util.cpp
+++ b/cmds/statsd/src/metadata_util.cpp
@@ -21,6 +21,8 @@
namespace os {
namespace statsd {
+using google::protobuf::RepeatedPtrField;
+
void writeValueToProto(metadata::FieldValue* metadataFieldValue, const Value& value) {
std::string storage_value;
switch (value.getType()) {
@@ -68,6 +70,53 @@
}
}
+void writeFieldValuesFromMetadata(
+ const RepeatedPtrField<metadata::FieldValue>& repeatedFieldValueList,
+ std::vector<FieldValue>* fieldValues) {
+ for (const metadata::FieldValue& metadataFieldValue : repeatedFieldValueList) {
+ Field field(metadataFieldValue.field().tag(), metadataFieldValue.field().field());
+ Value value;
+ switch (metadataFieldValue.value_case()) {
+ case metadata::FieldValue::ValueCase::kValueInt:
+ value = Value(metadataFieldValue.value_int());
+ break;
+ case metadata::FieldValue::ValueCase::kValueLong:
+ value = Value(metadataFieldValue.value_long());
+ break;
+ case metadata::FieldValue::ValueCase::kValueFloat:
+ value = Value(metadataFieldValue.value_float());
+ break;
+ case metadata::FieldValue::ValueCase::kValueDouble:
+ value = Value(metadataFieldValue.value_double());
+ break;
+ case metadata::FieldValue::ValueCase::kValueStr:
+ value = Value(metadataFieldValue.value_str());
+ break;
+ case metadata::FieldValue::ValueCase::kValueStorage:
+ value = Value(metadataFieldValue.value_storage());
+ break;
+ default:
+ break;
+ }
+ FieldValue fieldValue(field, value);
+ fieldValues->emplace_back(field, value);
+ }
+}
+
+MetricDimensionKey loadMetricDimensionKeyFromProto(
+ const metadata::MetricDimensionKey& metricDimensionKey) {
+ std::vector<FieldValue> dimKeyInWhatFieldValues;
+ writeFieldValuesFromMetadata(metricDimensionKey.dimension_key_in_what(),
+ &dimKeyInWhatFieldValues);
+ std::vector<FieldValue> stateValuesFieldValues;
+ writeFieldValuesFromMetadata(metricDimensionKey.state_values_key(), &stateValuesFieldValues);
+
+ HashableDimensionKey dimKeyInWhat(dimKeyInWhatFieldValues);
+ HashableDimensionKey stateValues(stateValuesFieldValues);
+ MetricDimensionKey metricKey(dimKeyInWhat, stateValues);
+ return metricKey;
+}
+
} // namespace statsd
} // namespace os
} // namespace android
\ No newline at end of file
diff --git a/cmds/statsd/src/metadata_util.h b/cmds/statsd/src/metadata_util.h
index 23e94b0..84a39ff 100644
--- a/cmds/statsd/src/metadata_util.h
+++ b/cmds/statsd/src/metadata_util.h
@@ -24,6 +24,9 @@
void writeMetricDimensionKeyToMetadataDimensionKey(const MetricDimensionKey& metricKey,
metadata::MetricDimensionKey* metadataMetricKey);
+MetricDimensionKey loadMetricDimensionKeyFromProto(
+ const metadata::MetricDimensionKey& metricDimensionKey);
+
} // namespace statsd
} // namespace os
} // namespace android
\ No newline at end of file
diff --git a/cmds/statsd/src/metrics/MetricsManager.cpp b/cmds/statsd/src/metrics/MetricsManager.cpp
index 9e9b6ff..829ba21 100644
--- a/cmds/statsd/src/metrics/MetricsManager.cpp
+++ b/cmds/statsd/src/metrics/MetricsManager.cpp
@@ -579,6 +579,22 @@
return metadataWritten;
}
+void MetricsManager::loadMetadata(const metadata::StatsMetadata& metadata,
+ int64_t currentWallClockTimeNs,
+ int64_t systemElapsedTimeNs) {
+ for (const metadata::AlertMetadata& alertMetadata : metadata.alert_metadata()) {
+ int64_t alertId = alertMetadata.alert_id();
+ auto it = mAlertTrackerMap.find(alertId);
+ if (it == mAlertTrackerMap.end()) {
+ ALOGE("No anomalyTracker found for alertId %lld", (long long) alertId);
+ continue;
+ }
+ mAllAnomalyTrackers[it->second]->loadAlertMetadata(alertMetadata,
+ currentWallClockTimeNs,
+ systemElapsedTimeNs);
+ }
+}
+
} // namespace statsd
} // namespace os
} // namespace android
diff --git a/cmds/statsd/src/metrics/MetricsManager.h b/cmds/statsd/src/metrics/MetricsManager.h
index 1813957..3b63b0a 100644
--- a/cmds/statsd/src/metrics/MetricsManager.h
+++ b/cmds/statsd/src/metrics/MetricsManager.h
@@ -144,6 +144,10 @@
bool writeMetadataToProto(int64_t currentWallClockTimeNs,
int64_t systemElapsedTimeNs,
metadata::StatsMetadata* statsMetadata);
+
+ void loadMetadata(const metadata::StatsMetadata& metadata,
+ int64_t currentWallClockTimeNs,
+ int64_t systemElapsedTimeNs);
private:
// For test only.
inline int64_t getTtlEndNs() const { return mTtlEndNs; }
@@ -270,6 +274,7 @@
FRIEND_TEST(AnomalyDetectionE2eTest, TestSlicedCountMetric_multiple_buckets);
FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk_no_data_written);
FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_save_refractory_to_disk);
+ FRIEND_TEST(AnomalyDetectionE2eTest, TestCountMetric_load_refractory_from_disk);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_single_bucket);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_multiple_buckets);
FRIEND_TEST(AnomalyDetectionE2eTest, TestDurationMetric_SUM_long_refractory_period);
diff --git a/cmds/statsd/tests/e2e/Anomaly_count_e2e_test.cpp b/cmds/statsd/tests/e2e/Anomaly_count_e2e_test.cpp
index 8e0687a..c2d7043 100644
--- a/cmds/statsd/tests/e2e/Anomaly_count_e2e_test.cpp
+++ b/cmds/statsd/tests/e2e/Anomaly_count_e2e_test.cpp
@@ -328,6 +328,59 @@
EXPECT_EQ(dimKeyInWhat.field().field(), fieldValue1.mField.getField());
EXPECT_EQ(dimKeyInWhat.value_int(), fieldValue1.mValue.int_value);
}
+
+TEST(AnomalyDetectionE2eTest, TestCountMetric_load_refractory_from_disk) {
+ const int num_buckets = 1;
+ const int threshold = 0;
+ const int refractory_period_sec = 86400 * 365; // 1 year
+ auto config = CreateStatsdConfig(num_buckets, threshold, refractory_period_sec);
+ const int64_t alert_id = config.alert(0).id();
+
+ int64_t bucketStartTimeNs = 10000000000;
+
+ int configUid = 2000;
+ int64_t configId = 1000;
+ ConfigKey cfgKey(configUid, configId);
+ auto processor = CreateStatsLogProcessor(bucketStartTimeNs, bucketStartTimeNs, config, cfgKey);
+ EXPECT_EQ(processor->mMetricsManagers.size(), 1u);
+ EXPECT_TRUE(processor->mMetricsManagers.begin()->second->isConfigValid());
+ EXPECT_EQ(1u, processor->mMetricsManagers.begin()->second->mAllAnomalyTrackers.size());
+
+ sp<AnomalyTracker> anomalyTracker =
+ processor->mMetricsManagers.begin()->second->mAllAnomalyTrackers[0];
+
+ std::vector<int> attributionUids1 = {111};
+ std::vector<string> attributionTags1 = {"App1"};
+ std::vector<int> attributionUids2 = {111, 222};
+ std::vector<string> attributionTags2 = {"App1", "GMSCoreModule1"};
+
+ FieldValue fieldValue1(Field(util::WAKELOCK_STATE_CHANGED, (int32_t)0x02010101),
+ Value((int32_t)111));
+ HashableDimensionKey whatKey1({fieldValue1});
+ MetricDimensionKey dimensionKey1(whatKey1, DEFAULT_DIMENSION_KEY);
+
+ auto event = CreateAcquireWakelockEvent(bucketStartTimeNs + 2, attributionUids1,
+ attributionTags1, "wl1");
+ processor->OnLogEvent(event.get());
+ EXPECT_EQ(refractory_period_sec + (bucketStartTimeNs + 2) / NS_PER_SEC + 1,
+ anomalyTracker->getRefractoryPeriodEndsSec(dimensionKey1));
+
+ int64_t mockWallClockNs = 1584991200 * NS_PER_SEC;
+ int64_t mockElapsedTimeNs = bucketStartTimeNs + 5000 * NS_PER_SEC;
+ processor->SaveMetadataToDisk(mockWallClockNs, mockElapsedTimeNs);
+
+ auto processor2 = CreateStatsLogProcessor(bucketStartTimeNs, bucketStartTimeNs, config, cfgKey);
+ int64_t mockElapsedTimeSinceBoot = 10 * NS_PER_SEC;
+ processor2->LoadMetadataFromDisk(mockWallClockNs, mockElapsedTimeSinceBoot);
+
+ sp<AnomalyTracker> anomalyTracker2 =
+ processor2->mMetricsManagers.begin()->second->mAllAnomalyTrackers[0];
+ EXPECT_EQ(anomalyTracker2->getRefractoryPeriodEndsSec(dimensionKey1) -
+ mockElapsedTimeSinceBoot / NS_PER_SEC,
+ anomalyTracker->getRefractoryPeriodEndsSec(dimensionKey1) -
+ mockElapsedTimeNs / NS_PER_SEC);
+}
+
#else
GTEST_LOG_(INFO) << "This test does nothing.\n";
#endif
diff --git a/cmds/statsd/tests/metadata_util_test.cpp b/cmds/statsd/tests/metadata_util_test.cpp
new file mode 100644
index 0000000..7707890
--- /dev/null
+++ b/cmds/statsd/tests/metadata_util_test.cpp
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <gtest/gtest.h>
+
+#include "metadata_util.h"
+#include "tests/statsd_test_util.h"
+
+#ifdef __ANDROID__
+
+namespace android {
+namespace os {
+namespace statsd {
+
+TEST(MetadataUtilTest, TestWriteAndReadMetricDimensionKey) {
+ HashableDimensionKey dim;
+ HashableDimensionKey dim2;
+ int pos1[] = {1, 1, 1};
+ int pos2[] = {1, 1, 2};
+ int pos3[] = {1, 1, 3};
+ int pos4[] = {2, 0, 0};
+ Field field1(10, pos1, 2);
+ Field field2(10, pos2, 2);
+ Field field3(10, pos3, 2);
+ Field field4(10, pos4, 0);
+
+ Value value1((int32_t)10025);
+ Value value2("tag");
+ Value value3((int32_t)987654);
+ Value value4((int32_t)99999);
+
+ dim.addValue(FieldValue(field1, value1));
+ dim.addValue(FieldValue(field2, value2));
+ dim.addValue(FieldValue(field3, value3));
+ dim.addValue(FieldValue(field4, value4));
+
+ dim2.addValue(FieldValue(field1, value1));
+ dim2.addValue(FieldValue(field2, value2));
+
+ MetricDimensionKey dimKey(dim, dim2);
+
+ metadata::MetricDimensionKey metadataDimKey;
+ writeMetricDimensionKeyToMetadataDimensionKey(dimKey, &metadataDimKey);
+
+ MetricDimensionKey loadedDimKey = loadMetricDimensionKeyFromProto(metadataDimKey);
+
+ ASSERT_EQ(loadedDimKey, dimKey);
+ ASSERT_EQ(std::hash<MetricDimensionKey>{}(loadedDimKey),
+ std::hash<MetricDimensionKey>{}(dimKey));
+}
+
+} // namespace statsd
+} // namespace os
+} // namespace android
+#else
+GTEST_LOG_(INFO) << "This test does nothing.\n";
+#endif