commit b356151e63140085cb96fa16804ee18b3862a4fc
author Yao Chen <yaochen@google.com> Tue Nov 21 18:07:17 2017 -0800
committer Yao Chen <yaochen@google.com> Mon Nov 27 10:52:54 2017 -0800
tree ecb892f399742cc0d7190b7700379b1a120bcb1e
parent 16fd2e9a9a0a064491196d02957c7e94eaa870a7

Add StatsdStats and guardrail.

+ StatsdStats is the global class that tracks the stats about statsd.

+ Added guardrail for classes that have a map which could potentially grow
  unboundedly with the number of logs.

TODO: add unit tests & CTS for StatsdStats, and guardrail
      add stats for pulled atoms.

Test: statsd_test

Change-Id: I0ea562de4dd3f6162f7923a9c193420b482c1d51

cmds/statsd/src/metrics/ValueMetricProducer.cpp

diff --git a/cmds/statsd/src/metrics/ValueMetricProducer.cpp b/cmds/statsd/src/metrics/ValueMetricProducer.cpp
index cc02c69..66c8419 100644
--- a/cmds/statsd/src/metrics/ValueMetricProducer.cpp
+++ b/cmds/statsd/src/metrics/ValueMetricProducer.cpp
@@ -18,6 +18,7 @@
 #include "Log.h"
 
 #include "ValueMetricProducer.h"
+#include "guardrail/StatsdStats.h"
 
 #include <cutils/log.h>
 #include <limits.h>
@@ -67,11 +68,12 @@
 static const uint64_t kDefaultBucketSizeMillis = 60 * 60 * 1000L;
 
 // ValueMetric has a minimum bucket size of 10min so that we don't pull too frequently
-ValueMetricProducer::ValueMetricProducer(const ValueMetric& metric, const int conditionIndex,
+ValueMetricProducer::ValueMetricProducer(const ConfigKey& key, const ValueMetric& metric,
+                                         const int conditionIndex,
                                          const sp<ConditionWizard>& wizard, const int pullTagId,
                                          const uint64_t startTimeNs,
                                          shared_ptr<StatsPullerManager> statsPullerManager)
-    : MetricProducer(startTimeNs, conditionIndex, wizard),
+    : MetricProducer(key, startTimeNs, conditionIndex, wizard),
       mMetric(metric),
       mStatsPullerManager(statsPullerManager),
       mPullTagId(pullTagId) {
@@ -103,10 +105,11 @@
 }
 
 // for testing
-ValueMetricProducer::ValueMetricProducer(const ValueMetric& metric, const int conditionIndex,
+ValueMetricProducer::ValueMetricProducer(const ConfigKey& key, const ValueMetric& metric,
+                                         const int conditionIndex,
                                          const sp<ConditionWizard>& wizard, const int pullTagId,
                                          const uint64_t startTimeNs)
-    : ValueMetricProducer(metric, conditionIndex, wizard, pullTagId, startTimeNs,
+    : ValueMetricProducer(key, metric, conditionIndex, wizard, pullTagId, startTimeNs,
                           make_shared<StatsPullerManager>()) {
 }
 
@@ -238,6 +241,27 @@
     }
 }
 
+bool ValueMetricProducer::hitGuardRail(const HashableDimensionKey& newKey) {
+    // ===========GuardRail==============
+    // 1. Report the tuple count if the tuple count > soft limit
+    if (mCurrentSlicedBucket.find(newKey) != mCurrentSlicedBucket.end()) {
+        return false;
+    }
+    if (mCurrentSlicedBucket.size() > StatsdStats::kDimensionKeySizeSoftLimit - 1) {
+        size_t newTupleCount = mCurrentSlicedBucket.size() + 1;
+        StatsdStats::getInstance().noteMetricDimensionSize(mConfigKey, mMetric.name(),
+                                                           newTupleCount);
+        // 2. Don't add more tuples, we are above the allowed threshold. Drop the data.
+        if (newTupleCount > StatsdStats::kDimensionKeySizeHardLimit) {
+            ALOGE("ValueMetric %s dropping data for dimension key %s", mMetric.name().c_str(),
+                  newKey.c_str());
+            return true;
+        }
+    }
+
+    return false;
+}
+
 void ValueMetricProducer::onMatchedLogEventInternal(
         const size_t matcherIndex, const HashableDimensionKey& eventKey,
         const map<string, HashableDimensionKey>& conditionKey, bool condition,
@@ -249,6 +273,9 @@
         return;
     }
 
+    if (hitGuardRail(eventKey)) {
+        return;
+    }
     Interval& interval = mCurrentSlicedBucket[eventKey];
 
     long value = get_value(event);