Yao Chen | ab273e2 | 2017-09-06 12:53:50 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2017 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 17 | #define DEBUG true // STOPSHIP if true |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 18 | #include "Log.h" |
David Chen | 2158296 | 2017-11-01 17:32:46 -0700 | [diff] [blame] | 19 | #include "statslog.h" |
Yao Chen | ab273e2 | 2017-09-06 12:53:50 -0700 | [diff] [blame] | 20 | |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 21 | #include <android-base/file.h> |
| 22 | #include <dirent.h> |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 23 | #include "StatsLogProcessor.h" |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 24 | #include "android-base/stringprintf.h" |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 25 | #include "guardrail/StatsdStats.h" |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 26 | #include "metrics/CountMetricProducer.h" |
Chenjie Yu | 85ed838 | 2017-12-14 16:48:54 -0800 | [diff] [blame] | 27 | #include "external/StatsPullerManager.h" |
Yangster-mac | d40053e | 2018-01-09 16:29:22 -0800 | [diff] [blame] | 28 | #include "dimension.h" |
| 29 | #include "field_util.h" |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 30 | #include "stats_util.h" |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 31 | #include "storage/StorageManager.h" |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 32 | |
yro | 00698da | 2017-09-15 10:06:40 -0700 | [diff] [blame] | 33 | #include <log/log_event_list.h> |
Yao Chen | ef99c4f | 2017-09-22 16:26:54 -0700 | [diff] [blame] | 34 | #include <utils/Errors.h> |
David Chen | 1604957 | 2018-02-01 18:27:51 -0800 | [diff] [blame] | 35 | #include <utils/SystemClock.h> |
Yao Chen | ab273e2 | 2017-09-06 12:53:50 -0700 | [diff] [blame] | 36 | |
| 37 | using namespace android; |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 38 | using android::base::StringPrintf; |
yro | b0378b0 | 2017-11-09 20:36:25 -0800 | [diff] [blame] | 39 | using android::util::FIELD_COUNT_REPEATED; |
yro | 17adac9 | 2017-11-08 23:16:29 -0800 | [diff] [blame] | 40 | using android::util::FIELD_TYPE_BOOL; |
| 41 | using android::util::FIELD_TYPE_FLOAT; |
| 42 | using android::util::FIELD_TYPE_INT32; |
| 43 | using android::util::FIELD_TYPE_INT64; |
| 44 | using android::util::FIELD_TYPE_MESSAGE; |
| 45 | using android::util::FIELD_TYPE_STRING; |
| 46 | using android::util::ProtoOutputStream; |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 47 | using std::make_unique; |
| 48 | using std::unique_ptr; |
| 49 | using std::vector; |
Bookatz | 906a35c | 2017-09-20 15:26:44 -0700 | [diff] [blame] | 50 | |
| 51 | namespace android { |
| 52 | namespace os { |
| 53 | namespace statsd { |
Yao Chen | ab273e2 | 2017-09-06 12:53:50 -0700 | [diff] [blame] | 54 | |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 55 | // for ConfigMetricsReportList |
yro | 17adac9 | 2017-11-08 23:16:29 -0800 | [diff] [blame] | 56 | const int FIELD_ID_CONFIG_KEY = 1; |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 57 | const int FIELD_ID_REPORTS = 2; |
yro | 17adac9 | 2017-11-08 23:16:29 -0800 | [diff] [blame] | 58 | // for ConfigKey |
| 59 | const int FIELD_ID_UID = 1; |
Yangster-mac | 94e197c | 2018-01-02 16:03:03 -0800 | [diff] [blame] | 60 | const int FIELD_ID_ID = 2; |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 61 | // for ConfigMetricsReport |
| 62 | const int FIELD_ID_METRICS = 1; |
| 63 | const int FIELD_ID_UID_MAP = 2; |
David Chen | 1604957 | 2018-02-01 18:27:51 -0800 | [diff] [blame] | 64 | const int FIELD_ID_LAST_REPORT_NANOS = 3; |
| 65 | const int FIELD_ID_CURRENT_REPORT_NANOS = 4; |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 66 | |
yro | 03faf09 | 2017-12-12 00:17:50 -0800 | [diff] [blame] | 67 | #define STATS_DATA_DIR "/data/misc/stats-data" |
yro | 17adac9 | 2017-11-08 23:16:29 -0800 | [diff] [blame] | 68 | |
yro | 31eb67b | 2017-10-24 13:33:21 -0700 | [diff] [blame] | 69 | StatsLogProcessor::StatsLogProcessor(const sp<UidMap>& uidMap, |
Yangster-mac | e2cd6d5 | 2017-11-09 20:38:30 -0800 | [diff] [blame] | 70 | const sp<AnomalyMonitor>& anomalyMonitor, |
Yangster-mac | 2087716 | 2017-12-22 17:19:39 -0800 | [diff] [blame] | 71 | const long timeBaseSec, |
David Chen | 1d7b0cd | 2017-11-15 14:20:04 -0800 | [diff] [blame] | 72 | const std::function<void(const ConfigKey&)>& sendBroadcast) |
Chenjie Yu | 85ed838 | 2017-12-14 16:48:54 -0800 | [diff] [blame] | 73 | : mUidMap(uidMap), |
| 74 | mAnomalyMonitor(anomalyMonitor), |
| 75 | mSendBroadcast(sendBroadcast), |
Yangster-mac | 2087716 | 2017-12-22 17:19:39 -0800 | [diff] [blame] | 76 | mTimeBaseSec(timeBaseSec) { |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 77 | // On each initialization of StatsLogProcessor, check stats-data directory to see if there is |
| 78 | // any left over data to be read. |
| 79 | StorageManager::sendBroadcast(STATS_DATA_DIR, mSendBroadcast); |
Chenjie Yu | 85ed838 | 2017-12-14 16:48:54 -0800 | [diff] [blame] | 80 | StatsPullerManager statsPullerManager; |
| 81 | statsPullerManager.SetTimeBaseSec(mTimeBaseSec); |
Yao Chen | ab273e2 | 2017-09-06 12:53:50 -0700 | [diff] [blame] | 82 | } |
| 83 | |
Yao Chen | ef99c4f | 2017-09-22 16:26:54 -0700 | [diff] [blame] | 84 | StatsLogProcessor::~StatsLogProcessor() { |
Yao Chen | ab273e2 | 2017-09-06 12:53:50 -0700 | [diff] [blame] | 85 | } |
| 86 | |
Yangster-mac | e2cd6d5 | 2017-11-09 20:38:30 -0800 | [diff] [blame] | 87 | void StatsLogProcessor::onAnomalyAlarmFired( |
| 88 | const uint64_t timestampNs, |
| 89 | unordered_set<sp<const AnomalyAlarm>, SpHash<AnomalyAlarm>> anomalySet) { |
Yangster-mac | b0d0628 | 2018-01-05 15:44:07 -0800 | [diff] [blame] | 90 | std::lock_guard<std::mutex> lock(mMetricsMutex); |
Bookatz | cc5adef | 2017-11-21 14:36:23 -0800 | [diff] [blame] | 91 | for (const auto& itr : mMetricsManagers) { |
| 92 | itr.second->onAnomalyAlarmFired(timestampNs, anomalySet); |
Yangster-mac | e2cd6d5 | 2017-11-09 20:38:30 -0800 | [diff] [blame] | 93 | } |
| 94 | } |
| 95 | |
Yangster-mac | d40053e | 2018-01-09 16:29:22 -0800 | [diff] [blame] | 96 | void StatsLogProcessor::mapIsolatedUidToHostUidIfNecessaryLocked(LogEvent* event) const { |
Yangster-mac | 7ba8fc3 | 2018-01-24 16:16:46 -0800 | [diff] [blame] | 97 | std::set<Field, FieldCmp> uidFields; |
Yangster-mac | 6898580 | 2018-01-21 10:05:09 -0800 | [diff] [blame] | 98 | if (android::util::kAtomsWithAttributionChain.find(event->GetTagId()) != |
| 99 | android::util::kAtomsWithAttributionChain.end()) { |
Yangster-mac | 7ba8fc3 | 2018-01-24 16:16:46 -0800 | [diff] [blame] | 100 | FieldMatcher matcher; |
| 101 | buildAttributionUidFieldMatcher(event->GetTagId(), Position::ANY, &matcher); |
| 102 | findFields(event->getFieldValueMap(), matcher, &uidFields); |
Yangster-mac | 6898580 | 2018-01-21 10:05:09 -0800 | [diff] [blame] | 103 | } else if (android::util::kAtomsWithUidField.find(event->GetTagId()) != |
| 104 | android::util::kAtomsWithUidField.end()) { |
Yangster-mac | 7ba8fc3 | 2018-01-24 16:16:46 -0800 | [diff] [blame] | 105 | FieldMatcher matcher; |
| 106 | buildSimpleAtomFieldMatcher( |
| 107 | event->GetTagId(), 1 /* uid is always the 1st field. */, &matcher); |
| 108 | findFields(event->getFieldValueMap(), matcher, &uidFields); |
Yangster-mac | 6898580 | 2018-01-21 10:05:09 -0800 | [diff] [blame] | 109 | } |
| 110 | |
Yangster-mac | 7ba8fc3 | 2018-01-24 16:16:46 -0800 | [diff] [blame] | 111 | for (const auto& uidField : uidFields) { |
| 112 | DimensionsValue* value = event->findFieldValueOrNull(uidField); |
Yangster-mac | d40053e | 2018-01-09 16:29:22 -0800 | [diff] [blame] | 113 | if (value != nullptr && value->value_case() == DimensionsValue::ValueCase::kValueInt) { |
| 114 | const int uid = mUidMap->getHostUidOrSelf(value->value_int()); |
| 115 | value->set_value_int(uid); |
| 116 | } |
Yao Chen | 312e898 | 2017-12-05 15:29:03 -0800 | [diff] [blame] | 117 | } |
Yangster-mac | d40053e | 2018-01-09 16:29:22 -0800 | [diff] [blame] | 118 | } |
| 119 | |
| 120 | void StatsLogProcessor::onIsolatedUidChangedEventLocked(const LogEvent& event) { |
| 121 | status_t err = NO_ERROR, err2 = NO_ERROR, err3 = NO_ERROR; |
| 122 | bool is_create = event.GetBool(3, &err); |
| 123 | auto parent_uid = int(event.GetLong(1, &err2)); |
| 124 | auto isolated_uid = int(event.GetLong(2, &err3)); |
| 125 | if (err == NO_ERROR && err2 == NO_ERROR && err3 == NO_ERROR) { |
| 126 | if (is_create) { |
| 127 | mUidMap->assignIsolatedUid(isolated_uid, parent_uid); |
| 128 | } else { |
| 129 | mUidMap->removeIsolatedUid(isolated_uid, parent_uid); |
| 130 | } |
| 131 | } else { |
| 132 | ALOGE("Failed to parse uid in the isolated uid change event."); |
| 133 | } |
| 134 | } |
| 135 | |
| 136 | // TODO: what if statsd service restarts? How do we know what logs are already processed before? |
| 137 | void StatsLogProcessor::OnLogEvent(LogEvent* event) { |
| 138 | std::lock_guard<std::mutex> lock(mMetricsMutex); |
| 139 | StatsdStats::getInstance().noteAtomLogged( |
| 140 | event->GetTagId(), event->GetTimestampNs() / NS_PER_SEC); |
| 141 | |
David Chen | 2158296 | 2017-11-01 17:32:46 -0700 | [diff] [blame] | 142 | // Hard-coded logic to update the isolated uid's in the uid-map. |
Stefan Lafon | ae2df01 | 2017-11-14 09:17:21 -0800 | [diff] [blame] | 143 | // The field numbers need to be currently updated by hand with atoms.proto |
Yangster-mac | d40053e | 2018-01-09 16:29:22 -0800 | [diff] [blame] | 144 | if (event->GetTagId() == android::util::ISOLATED_UID_CHANGED) { |
| 145 | onIsolatedUidChangedEventLocked(*event); |
David Chen | cfc311d | 2018-01-23 17:55:54 -0800 | [diff] [blame] | 146 | } |
| 147 | |
| 148 | if (mMetricsManagers.empty()) { |
| 149 | return; |
| 150 | } |
| 151 | |
Chenjie Yu | fa22d65 | 2018-02-05 14:37:48 -0800 | [diff] [blame] | 152 | long curTime = time(nullptr); |
| 153 | if (curTime - mLastPullerCacheClearTimeSec > StatsdStats::kPullerCacheClearIntervalSec) { |
| 154 | mStatsPullerManager.ClearPullerCacheIfNecessary(curTime); |
| 155 | mLastPullerCacheClearTimeSec = curTime; |
| 156 | } |
| 157 | |
David Chen | cfc311d | 2018-01-23 17:55:54 -0800 | [diff] [blame] | 158 | if (event->GetTagId() != android::util::ISOLATED_UID_CHANGED) { |
Yangster-mac | d40053e | 2018-01-09 16:29:22 -0800 | [diff] [blame] | 159 | // Map the isolated uid to host uid if necessary. |
| 160 | mapIsolatedUidToHostUidIfNecessaryLocked(event); |
| 161 | } |
| 162 | |
| 163 | // pass the event to metrics managers. |
| 164 | for (auto& pair : mMetricsManagers) { |
| 165 | pair.second->onLogEvent(*event); |
| 166 | flushIfNecessaryLocked(event->GetTimestampNs(), pair.first, *(pair.second)); |
David Chen | 2158296 | 2017-11-01 17:32:46 -0700 | [diff] [blame] | 167 | } |
Yao Chen | ab273e2 | 2017-09-06 12:53:50 -0700 | [diff] [blame] | 168 | } |
| 169 | |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 170 | void StatsLogProcessor::OnConfigUpdated(const ConfigKey& key, const StatsdConfig& config) { |
Yangster-mac | b0d0628 | 2018-01-05 15:44:07 -0800 | [diff] [blame] | 171 | std::lock_guard<std::mutex> lock(mMetricsMutex); |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 172 | ALOGD("Updated configuration for key %s", key.ToString().c_str()); |
Yao Chen | d10f7b1 | 2017-12-18 12:53:50 -0800 | [diff] [blame] | 173 | sp<MetricsManager> newMetricsManager = new MetricsManager(key, config, mTimeBaseSec, mUidMap); |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 174 | auto it = mMetricsManagers.find(key); |
Yao Chen | 288c600 | 2017-12-12 13:43:18 -0800 | [diff] [blame] | 175 | if (it == mMetricsManagers.end() && mMetricsManagers.size() > StatsdStats::kMaxConfigCount) { |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 176 | ALOGE("Can't accept more configs!"); |
| 177 | return; |
Yao Chen | 44cf27c | 2017-09-14 22:32:50 -0700 | [diff] [blame] | 178 | } |
| 179 | |
Yao Chen | caf339d | 2017-10-06 16:01:10 -0700 | [diff] [blame] | 180 | if (newMetricsManager->isConfigValid()) { |
David Chen | d689689 | 2017-10-25 11:49:03 -0700 | [diff] [blame] | 181 | mUidMap->OnConfigUpdated(key); |
Yangster-mac | e2cd6d5 | 2017-11-09 20:38:30 -0800 | [diff] [blame] | 182 | newMetricsManager->setAnomalyMonitor(mAnomalyMonitor); |
Yao Chen | 147ce60 | 2017-12-22 14:35:34 -0800 | [diff] [blame] | 183 | if (newMetricsManager->shouldAddUidMapListener()) { |
Yao Chen | d10f7b1 | 2017-12-18 12:53:50 -0800 | [diff] [blame] | 184 | // We have to add listener after the MetricsManager is constructed because it's |
| 185 | // not safe to create wp or sp from this pointer inside its constructor. |
| 186 | mUidMap->addListener(newMetricsManager.get()); |
| 187 | } |
| 188 | mMetricsManagers[key] = newMetricsManager; |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 189 | // Why doesn't this work? mMetricsManagers.insert({key, std::move(newMetricsManager)}); |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 190 | VLOG("StatsdConfig valid"); |
Yao Chen | caf339d | 2017-10-06 16:01:10 -0700 | [diff] [blame] | 191 | } else { |
| 192 | // If there is any error in the config, don't use it. |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 193 | ALOGE("StatsdConfig NOT valid"); |
Yao Chen | caf339d | 2017-10-06 16:01:10 -0700 | [diff] [blame] | 194 | } |
yro | 00698da | 2017-09-15 10:06:40 -0700 | [diff] [blame] | 195 | } |
Bookatz | 906a35c | 2017-09-20 15:26:44 -0700 | [diff] [blame] | 196 | |
Yangster | 7c334a1 | 2017-11-22 14:24:24 -0800 | [diff] [blame] | 197 | size_t StatsLogProcessor::GetMetricsSize(const ConfigKey& key) const { |
Yangster-mac | b0d0628 | 2018-01-05 15:44:07 -0800 | [diff] [blame] | 198 | std::lock_guard<std::mutex> lock(mMetricsMutex); |
Yao Chen | 729093d | 2017-10-16 10:33:26 -0700 | [diff] [blame] | 199 | auto it = mMetricsManagers.find(key); |
| 200 | if (it == mMetricsManagers.end()) { |
| 201 | ALOGW("Config source %s does not exist", key.ToString().c_str()); |
David Chen | 1d7b0cd | 2017-11-15 14:20:04 -0800 | [diff] [blame] | 202 | return 0; |
Yao Chen | 729093d | 2017-10-16 10:33:26 -0700 | [diff] [blame] | 203 | } |
David Chen | 1d7b0cd | 2017-11-15 14:20:04 -0800 | [diff] [blame] | 204 | return it->second->byteSize(); |
| 205 | } |
| 206 | |
Yao Chen | 884c8c1 | 2018-01-26 10:36:25 -0800 | [diff] [blame] | 207 | void StatsLogProcessor::dumpStates(FILE* out, bool verbose) { |
| 208 | std::lock_guard<std::mutex> lock(mMetricsMutex); |
| 209 | fprintf(out, "MetricsManager count: %lu\n", (unsigned long)mMetricsManagers.size()); |
| 210 | for (auto metricsManager : mMetricsManagers) { |
| 211 | metricsManager.second->dumpStates(out, verbose); |
| 212 | } |
| 213 | } |
| 214 | |
Yao Chen | 147ce60 | 2017-12-22 14:35:34 -0800 | [diff] [blame] | 215 | void StatsLogProcessor::onDumpReport(const ConfigKey& key, const uint64_t& dumpTimeStampNs, |
| 216 | ConfigMetricsReportList* report) { |
Yangster-mac | b0d0628 | 2018-01-05 15:44:07 -0800 | [diff] [blame] | 217 | std::lock_guard<std::mutex> lock(mMetricsMutex); |
Yangster-mac | 2087716 | 2017-12-22 17:19:39 -0800 | [diff] [blame] | 218 | auto it = mMetricsManagers.find(key); |
| 219 | if (it == mMetricsManagers.end()) { |
| 220 | ALOGW("Config source %s does not exist", key.ToString().c_str()); |
| 221 | return; |
| 222 | } |
| 223 | report->mutable_config_key()->set_uid(key.GetUid()); |
Yangster-mac | 94e197c | 2018-01-02 16:03:03 -0800 | [diff] [blame] | 224 | report->mutable_config_key()->set_id(key.GetId()); |
Yangster-mac | 2087716 | 2017-12-22 17:19:39 -0800 | [diff] [blame] | 225 | ConfigMetricsReport* configMetricsReport = report->add_reports(); |
| 226 | it->second->onDumpReport(dumpTimeStampNs, configMetricsReport); |
| 227 | // TODO: dump uid mapping. |
| 228 | } |
| 229 | |
David Chen | 1d7b0cd | 2017-11-15 14:20:04 -0800 | [diff] [blame] | 230 | void StatsLogProcessor::onDumpReport(const ConfigKey& key, vector<uint8_t>* outData) { |
Yangster-mac | b0d0628 | 2018-01-05 15:44:07 -0800 | [diff] [blame] | 231 | std::lock_guard<std::mutex> lock(mMetricsMutex); |
Yangster-mac | 8617950 | 2018-01-23 15:47:15 -0800 | [diff] [blame] | 232 | onDumpReportLocked(key, outData); |
| 233 | } |
| 234 | |
| 235 | void StatsLogProcessor::onDumpReportLocked(const ConfigKey& key, vector<uint8_t>* outData) { |
David Chen | 1d7b0cd | 2017-11-15 14:20:04 -0800 | [diff] [blame] | 236 | auto it = mMetricsManagers.find(key); |
| 237 | if (it == mMetricsManagers.end()) { |
| 238 | ALOGW("Config source %s does not exist", key.ToString().c_str()); |
| 239 | return; |
| 240 | } |
| 241 | |
| 242 | // This allows another broadcast to be sent within the rate-limit period if we get close to |
| 243 | // filling the buffer again soon. |
David Chen | 1d7b0cd | 2017-11-15 14:20:04 -0800 | [diff] [blame] | 244 | mLastBroadcastTimes.erase(key); |
Yao Chen | 729093d | 2017-10-16 10:33:26 -0700 | [diff] [blame] | 245 | |
yro | 17adac9 | 2017-11-08 23:16:29 -0800 | [diff] [blame] | 246 | ProtoOutputStream proto; |
| 247 | |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 248 | // Start of ConfigKey. |
yro | 17adac9 | 2017-11-08 23:16:29 -0800 | [diff] [blame] | 249 | long long configKeyToken = proto.start(FIELD_TYPE_MESSAGE | FIELD_ID_CONFIG_KEY); |
| 250 | proto.write(FIELD_TYPE_INT32 | FIELD_ID_UID, key.GetUid()); |
Yangster-mac | 94e197c | 2018-01-02 16:03:03 -0800 | [diff] [blame] | 251 | proto.write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)key.GetId()); |
yro | 17adac9 | 2017-11-08 23:16:29 -0800 | [diff] [blame] | 252 | proto.end(configKeyToken); |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 253 | // End of ConfigKey. |
yro | 17adac9 | 2017-11-08 23:16:29 -0800 | [diff] [blame] | 254 | |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 255 | // Start of ConfigMetricsReport (reports). |
| 256 | long long reportsToken = |
| 257 | proto.start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS); |
| 258 | |
| 259 | // First, fill in ConfigMetricsReport using current data on memory, which |
| 260 | // starts from filling in StatsLogReport's. |
Yao Chen | 288c600 | 2017-12-12 13:43:18 -0800 | [diff] [blame] | 261 | it->second->onDumpReport(&proto); |
yro | 17adac9 | 2017-11-08 23:16:29 -0800 | [diff] [blame] | 262 | |
| 263 | // Fill in UidMap. |
| 264 | auto uidMap = mUidMap->getOutput(key); |
| 265 | const int uidMapSize = uidMap.ByteSize(); |
| 266 | char uidMapBuffer[uidMapSize]; |
| 267 | uidMap.SerializeToArray(&uidMapBuffer[0], uidMapSize); |
| 268 | proto.write(FIELD_TYPE_MESSAGE | FIELD_ID_UID_MAP, uidMapBuffer, uidMapSize); |
| 269 | |
David Chen | 1604957 | 2018-02-01 18:27:51 -0800 | [diff] [blame] | 270 | // Fill in the timestamps. |
| 271 | proto.write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_NANOS, |
| 272 | (long long)it->second->getLastReportTimeNs()); |
| 273 | proto.write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_NANOS, |
| 274 | (long long)::android::elapsedRealtimeNano()); |
| 275 | |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 276 | // End of ConfigMetricsReport (reports). |
| 277 | proto.end(reportsToken); |
| 278 | |
| 279 | // Then, check stats-data directory to see there's any file containing |
| 280 | // ConfigMetricsReport from previous shutdowns to concatenate to reports. |
yro | 98a2850 | 2018-01-18 17:00:14 -0800 | [diff] [blame] | 281 | StorageManager::appendConfigMetricsReport(proto); |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 282 | |
David Chen | 1d7b0cd | 2017-11-15 14:20:04 -0800 | [diff] [blame] | 283 | if (outData != nullptr) { |
| 284 | outData->clear(); |
| 285 | outData->resize(proto.size()); |
| 286 | size_t pos = 0; |
| 287 | auto iter = proto.data(); |
| 288 | while (iter.readBuffer() != NULL) { |
| 289 | size_t toRead = iter.currentToRead(); |
| 290 | std::memcpy(&((*outData)[pos]), iter.readBuffer(), toRead); |
| 291 | pos += toRead; |
| 292 | iter.rp()->move(toRead); |
| 293 | } |
yro | 17adac9 | 2017-11-08 23:16:29 -0800 | [diff] [blame] | 294 | } |
Yao Chen | 69f1baf | 2017-11-27 17:25:36 -0800 | [diff] [blame] | 295 | StatsdStats::getInstance().noteMetricsReportSent(key); |
Yao Chen | 729093d | 2017-10-16 10:33:26 -0700 | [diff] [blame] | 296 | } |
| 297 | |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 298 | void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) { |
Yangster-mac | b0d0628 | 2018-01-05 15:44:07 -0800 | [diff] [blame] | 299 | std::lock_guard<std::mutex> lock(mMetricsMutex); |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 300 | auto it = mMetricsManagers.find(key); |
| 301 | if (it != mMetricsManagers.end()) { |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 302 | mMetricsManagers.erase(it); |
David Chen | d689689 | 2017-10-25 11:49:03 -0700 | [diff] [blame] | 303 | mUidMap->OnConfigRemoved(key); |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 304 | } |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 305 | StatsdStats::getInstance().noteConfigRemoved(key); |
David Chen | 1d7b0cd | 2017-11-15 14:20:04 -0800 | [diff] [blame] | 306 | |
David Chen | 1d7b0cd | 2017-11-15 14:20:04 -0800 | [diff] [blame] | 307 | mLastBroadcastTimes.erase(key); |
Chenjie Yu | fa22d65 | 2018-02-05 14:37:48 -0800 | [diff] [blame] | 308 | |
| 309 | if (mMetricsManagers.empty()) { |
| 310 | mStatsPullerManager.ForceClearPullerCache(); |
| 311 | } |
Joe Onorato | 9fc9edf | 2017-10-15 20:08:52 -0700 | [diff] [blame] | 312 | } |
| 313 | |
Yangster-mac | b0d0628 | 2018-01-05 15:44:07 -0800 | [diff] [blame] | 314 | void StatsLogProcessor::flushIfNecessaryLocked( |
| 315 | uint64_t timestampNs, const ConfigKey& key, MetricsManager& metricsManager) { |
David Chen | d9269e2 | 2017-12-05 13:43:51 -0800 | [diff] [blame] | 316 | auto lastCheckTime = mLastByteSizeTimes.find(key); |
| 317 | if (lastCheckTime != mLastByteSizeTimes.end()) { |
| 318 | if (timestampNs - lastCheckTime->second < StatsdStats::kMinByteSizeCheckPeriodNs) { |
| 319 | return; |
| 320 | } |
| 321 | } |
| 322 | |
| 323 | // We suspect that the byteSize() computation is expensive, so we set a rate limit. |
| 324 | size_t totalBytes = metricsManager.byteSize(); |
| 325 | mLastByteSizeTimes[key] = timestampNs; |
| 326 | if (totalBytes > |
| 327 | StatsdStats::kMaxMetricsBytesPerConfig) { // Too late. We need to start clearing data. |
Yao Chen | 288c600 | 2017-12-12 13:43:18 -0800 | [diff] [blame] | 328 | // TODO(b/70571383): By 12/15/2017 add API to drop data directly |
| 329 | ProtoOutputStream proto; |
| 330 | metricsManager.onDumpReport(&proto); |
David Chen | 1294295 | 2017-12-04 14:28:43 -0800 | [diff] [blame] | 331 | StatsdStats::getInstance().noteDataDropped(key); |
| 332 | VLOG("StatsD had to toss out metrics for %s", key.ToString().c_str()); |
David Chen | d9269e2 | 2017-12-05 13:43:51 -0800 | [diff] [blame] | 333 | } else if (totalBytes > .9 * StatsdStats::kMaxMetricsBytesPerConfig) { |
| 334 | // Send broadcast so that receivers can pull data. |
| 335 | auto lastBroadcastTime = mLastBroadcastTimes.find(key); |
| 336 | if (lastBroadcastTime != mLastBroadcastTimes.end()) { |
| 337 | if (timestampNs - lastBroadcastTime->second < StatsdStats::kMinBroadcastPeriodNs) { |
| 338 | VLOG("StatsD would've sent a broadcast but the rate limit stopped us."); |
David Chen | 1d7b0cd | 2017-11-15 14:20:04 -0800 | [diff] [blame] | 339 | return; |
| 340 | } |
| 341 | } |
| 342 | mLastBroadcastTimes[key] = timestampNs; |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 343 | VLOG("StatsD requesting broadcast for %s", key.ToString().c_str()); |
David Chen | 1d7b0cd | 2017-11-15 14:20:04 -0800 | [diff] [blame] | 344 | mSendBroadcast(key); |
Yao Chen | b356151 | 2017-11-21 18:07:17 -0800 | [diff] [blame] | 345 | StatsdStats::getInstance().noteBroadcastSent(key); |
yro | 31eb67b | 2017-10-24 13:33:21 -0700 | [diff] [blame] | 346 | } |
| 347 | } |
| 348 | |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 349 | void StatsLogProcessor::WriteDataToDisk() { |
Yangster-mac | b0d0628 | 2018-01-05 15:44:07 -0800 | [diff] [blame] | 350 | std::lock_guard<std::mutex> lock(mMetricsMutex); |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 351 | for (auto& pair : mMetricsManagers) { |
| 352 | const ConfigKey& key = pair.first; |
| 353 | vector<uint8_t> data; |
Yangster-mac | 8617950 | 2018-01-23 15:47:15 -0800 | [diff] [blame] | 354 | onDumpReportLocked(key, &data); |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 355 | // TODO: Add a guardrail to prevent accumulation of file on disk. |
yro | 98a2850 | 2018-01-18 17:00:14 -0800 | [diff] [blame] | 356 | string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR, time(nullptr), |
| 357 | key.GetUid(), (long long)key.GetId()); |
yro | 947fbce | 2017-11-15 22:50:23 -0800 | [diff] [blame] | 358 | StorageManager::writeFile(file_name.c_str(), &data[0], data.size()); |
| 359 | } |
| 360 | } |
| 361 | |
Yao Chen | ef99c4f | 2017-09-22 16:26:54 -0700 | [diff] [blame] | 362 | } // namespace statsd |
| 363 | } // namespace os |
| 364 | } // namespace android |