Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 1 | // Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style license that can be |
| 3 | // found in the LICENSE file. |
| 4 | |
| 5 | #include "update_engine/payload_state.h" |
| 6 | |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 7 | #include <algorithm> |
| 8 | |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 9 | #include <base/logging.h> |
Jay Srinivasan | 19409b7 | 2013-04-12 19:23:36 -0700 | [diff] [blame] | 10 | #include "base/string_util.h" |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 11 | #include <base/stringprintf.h> |
| 12 | |
Jay Srinivasan | d29695d | 2013-04-08 15:08:05 -0700 | [diff] [blame] | 13 | #include "update_engine/constants.h" |
Jay Srinivasan | 19409b7 | 2013-04-12 19:23:36 -0700 | [diff] [blame] | 14 | #include "update_engine/prefs.h" |
| 15 | #include "update_engine/system_state.h" |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 16 | #include "update_engine/utils.h" |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 17 | |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 18 | using base::Time; |
| 19 | using base::TimeDelta; |
| 20 | using std::min; |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 21 | using std::string; |
| 22 | |
| 23 | namespace chromeos_update_engine { |
| 24 | |
David Zeuthen | 9a017f2 | 2013-04-11 16:10:26 -0700 | [diff] [blame] | 25 | const TimeDelta PayloadState::kDurationSlack = TimeDelta::FromSeconds(600); |
| 26 | |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 27 | // We want to upperbound backoffs to 16 days |
| 28 | static const uint32_t kMaxBackoffDays = 16; |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 29 | |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 30 | // We want to randomize retry attempts after the backoff by +/- 6 hours. |
| 31 | static const uint32_t kMaxBackoffFuzzMinutes = 12 * 60; |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 32 | |
Jay Srinivasan | 19409b7 | 2013-04-12 19:23:36 -0700 | [diff] [blame] | 33 | PayloadState::PayloadState() |
| 34 | : prefs_(NULL), |
| 35 | payload_attempt_number_(0), |
| 36 | url_index_(0), |
| 37 | url_failure_count_(0) { |
| 38 | for (int i = 0; i <= kNumDownloadSources; i++) |
| 39 | total_bytes_downloaded_[i] = current_bytes_downloaded_[i] = 0; |
| 40 | } |
| 41 | |
| 42 | bool PayloadState::Initialize(SystemState* system_state) { |
| 43 | system_state_ = system_state; |
| 44 | prefs_ = system_state_->prefs(); |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 45 | LoadResponseSignature(); |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 46 | LoadPayloadAttemptNumber(); |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 47 | LoadUrlIndex(); |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 48 | LoadUrlFailureCount(); |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 49 | LoadBackoffExpiryTime(); |
David Zeuthen | 9a017f2 | 2013-04-11 16:10:26 -0700 | [diff] [blame] | 50 | LoadUpdateTimestampStart(); |
| 51 | // The LoadUpdateDurationUptime() method relies on LoadUpdateTimestampStart() |
| 52 | // being called before it. Don't reorder. |
| 53 | LoadUpdateDurationUptime(); |
Jay Srinivasan | 19409b7 | 2013-04-12 19:23:36 -0700 | [diff] [blame] | 54 | for (int i = 0; i < kNumDownloadSources; i++) { |
| 55 | DownloadSource source = static_cast<DownloadSource>(i); |
| 56 | LoadCurrentBytesDownloaded(source); |
| 57 | LoadTotalBytesDownloaded(source); |
| 58 | } |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 59 | return true; |
| 60 | } |
| 61 | |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 62 | void PayloadState::SetResponse(const OmahaResponse& omaha_response) { |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 63 | // Always store the latest response. |
| 64 | response_ = omaha_response; |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 65 | |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 66 | // Check if the "signature" of this response (i.e. the fields we care about) |
| 67 | // has changed. |
| 68 | string new_response_signature = CalculateResponseSignature(); |
| 69 | bool has_response_changed = (response_signature_ != new_response_signature); |
| 70 | |
| 71 | // If the response has changed, we should persist the new signature and |
| 72 | // clear away all the existing state. |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 73 | if (has_response_changed) { |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 74 | LOG(INFO) << "Resetting all persisted state as this is a new response"; |
| 75 | SetResponseSignature(new_response_signature); |
| 76 | ResetPersistedState(); |
| 77 | return; |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 78 | } |
| 79 | |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 80 | // This is the earliest point at which we can validate whether the URL index |
| 81 | // we loaded from the persisted state is a valid value. If the response |
| 82 | // hasn't changed but the URL index is invalid, it's indicative of some |
| 83 | // tampering of the persisted state. |
| 84 | if (url_index_ >= GetNumUrls()) { |
| 85 | LOG(INFO) << "Resetting all payload state as the url index seems to have " |
| 86 | "been tampered with"; |
| 87 | ResetPersistedState(); |
| 88 | return; |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 89 | } |
Jay Srinivasan | 19409b7 | 2013-04-12 19:23:36 -0700 | [diff] [blame] | 90 | |
| 91 | // Update the current download source which depends on the latest value of |
| 92 | // the response. |
| 93 | UpdateCurrentDownloadSource(); |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 94 | } |
| 95 | |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 96 | void PayloadState::DownloadComplete() { |
| 97 | LOG(INFO) << "Payload downloaded successfully"; |
| 98 | IncrementPayloadAttemptNumber(); |
| 99 | } |
| 100 | |
| 101 | void PayloadState::DownloadProgress(size_t count) { |
| 102 | if (count == 0) |
| 103 | return; |
| 104 | |
David Zeuthen | 9a017f2 | 2013-04-11 16:10:26 -0700 | [diff] [blame] | 105 | CalculateUpdateDurationUptime(); |
Jay Srinivasan | 19409b7 | 2013-04-12 19:23:36 -0700 | [diff] [blame] | 106 | UpdateBytesDownloaded(count); |
David Zeuthen | 9a017f2 | 2013-04-11 16:10:26 -0700 | [diff] [blame] | 107 | |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 108 | // We've received non-zero bytes from a recent download operation. Since our |
| 109 | // URL failure count is meant to penalize a URL only for consecutive |
| 110 | // failures, downloading bytes successfully means we should reset the failure |
| 111 | // count (as we know at least that the URL is working). In future, we can |
| 112 | // design this to be more sophisticated to check for more intelligent failure |
| 113 | // patterns, but right now, even 1 byte downloaded will mark the URL to be |
| 114 | // good unless it hits 10 (or configured number of) consecutive failures |
| 115 | // again. |
| 116 | |
| 117 | if (GetUrlFailureCount() == 0) |
| 118 | return; |
| 119 | |
| 120 | LOG(INFO) << "Resetting failure count of Url" << GetUrlIndex() |
| 121 | << " to 0 as we received " << count << " bytes successfully"; |
| 122 | SetUrlFailureCount(0); |
| 123 | } |
| 124 | |
Jay Srinivasan | 19409b7 | 2013-04-12 19:23:36 -0700 | [diff] [blame] | 125 | void PayloadState::UpdateRestarted() { |
| 126 | LOG(INFO) << "Starting a new update"; |
| 127 | ResetDownloadSourcesOnNewUpdate(); |
| 128 | } |
| 129 | |
David Zeuthen | 9a017f2 | 2013-04-11 16:10:26 -0700 | [diff] [blame] | 130 | void PayloadState::UpdateSucceeded() { |
Jay Srinivasan | 19409b7 | 2013-04-12 19:23:36 -0700 | [diff] [blame] | 131 | // Send the relevant metrics that are tracked in this class to UMA. |
David Zeuthen | 9a017f2 | 2013-04-11 16:10:26 -0700 | [diff] [blame] | 132 | CalculateUpdateDurationUptime(); |
| 133 | SetUpdateTimestampEnd(Time::Now()); |
Jay Srinivasan | 19409b7 | 2013-04-12 19:23:36 -0700 | [diff] [blame] | 134 | ReportBytesDownloadedMetrics(); |
David Zeuthen | 9a017f2 | 2013-04-11 16:10:26 -0700 | [diff] [blame] | 135 | } |
| 136 | |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 137 | void PayloadState::UpdateFailed(ActionExitCode error) { |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 138 | ActionExitCode base_error = utils::GetBaseErrorCode(error); |
Jay Srinivasan | 55f50c2 | 2013-01-10 19:24:35 -0800 | [diff] [blame] | 139 | LOG(INFO) << "Updating payload state for error code: " << base_error |
| 140 | << " (" << utils::CodeToString(base_error) << ")"; |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 141 | |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 142 | if (GetNumUrls() == 0) { |
| 143 | // This means we got this error even before we got a valid Omaha response. |
| 144 | // So we should not advance the url_index_ in such cases. |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 145 | LOG(INFO) << "Ignoring failures until we get a valid Omaha response."; |
| 146 | return; |
| 147 | } |
| 148 | |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 149 | switch (base_error) { |
| 150 | // Errors which are good indicators of a problem with a particular URL or |
| 151 | // the protocol used in the URL or entities in the communication channel |
| 152 | // (e.g. proxies). We should try the next available URL in the next update |
| 153 | // check to quickly recover from these errors. |
| 154 | case kActionCodePayloadHashMismatchError: |
| 155 | case kActionCodePayloadSizeMismatchError: |
| 156 | case kActionCodeDownloadPayloadVerificationError: |
| 157 | case kActionCodeDownloadPayloadPubKeyVerificationError: |
| 158 | case kActionCodeSignedDeltaPayloadExpectedError: |
| 159 | case kActionCodeDownloadInvalidMetadataMagicString: |
| 160 | case kActionCodeDownloadSignatureMissingInManifest: |
| 161 | case kActionCodeDownloadManifestParseError: |
| 162 | case kActionCodeDownloadMetadataSignatureError: |
| 163 | case kActionCodeDownloadMetadataSignatureVerificationError: |
| 164 | case kActionCodeDownloadMetadataSignatureMismatch: |
| 165 | case kActionCodeDownloadOperationHashVerificationError: |
| 166 | case kActionCodeDownloadOperationExecutionError: |
| 167 | case kActionCodeDownloadOperationHashMismatch: |
| 168 | case kActionCodeDownloadInvalidMetadataSize: |
| 169 | case kActionCodeDownloadInvalidMetadataSignature: |
| 170 | case kActionCodeDownloadOperationHashMissingError: |
| 171 | case kActionCodeDownloadMetadataSignatureMissingError: |
| 172 | IncrementUrlIndex(); |
| 173 | break; |
| 174 | |
| 175 | // Errors which seem to be just transient network/communication related |
| 176 | // failures and do not indicate any inherent problem with the URL itself. |
| 177 | // So, we should keep the current URL but just increment the |
| 178 | // failure count to give it more chances. This way, while we maximize our |
| 179 | // chances of downloading from the URLs that appear earlier in the response |
| 180 | // (because download from a local server URL that appears earlier in a |
| 181 | // response is preferable than downloading from the next URL which could be |
| 182 | // a internet URL and thus could be more expensive). |
| 183 | case kActionCodeError: |
| 184 | case kActionCodeDownloadTransferError: |
| 185 | case kActionCodeDownloadWriteError: |
| 186 | case kActionCodeDownloadStateInitializationError: |
| 187 | case kActionCodeOmahaErrorInHTTPResponse: // Aggregate code for HTTP errors. |
| 188 | IncrementFailureCount(); |
| 189 | break; |
| 190 | |
| 191 | // Errors which are not specific to a URL and hence shouldn't result in |
| 192 | // the URL being penalized. This can happen in two cases: |
| 193 | // 1. We haven't started downloading anything: These errors don't cost us |
| 194 | // anything in terms of actual payload bytes, so we should just do the |
| 195 | // regular retries at the next update check. |
| 196 | // 2. We have successfully downloaded the payload: In this case, the |
| 197 | // payload attempt number would have been incremented and would take care |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 198 | // of the backoff at the next update check. |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 199 | // In either case, there's no need to update URL index or failure count. |
| 200 | case kActionCodeOmahaRequestError: |
| 201 | case kActionCodeOmahaResponseHandlerError: |
| 202 | case kActionCodePostinstallRunnerError: |
| 203 | case kActionCodeFilesystemCopierError: |
| 204 | case kActionCodeInstallDeviceOpenError: |
| 205 | case kActionCodeKernelDeviceOpenError: |
| 206 | case kActionCodeDownloadNewPartitionInfoError: |
| 207 | case kActionCodeNewRootfsVerificationError: |
| 208 | case kActionCodeNewKernelVerificationError: |
| 209 | case kActionCodePostinstallBootedFromFirmwareB: |
| 210 | case kActionCodeOmahaRequestEmptyResponseError: |
| 211 | case kActionCodeOmahaRequestXMLParseError: |
| 212 | case kActionCodeOmahaResponseInvalid: |
| 213 | case kActionCodeOmahaUpdateIgnoredPerPolicy: |
| 214 | case kActionCodeOmahaUpdateDeferredPerPolicy: |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 215 | case kActionCodeOmahaUpdateDeferredForBackoff: |
Jay Srinivasan | ae4697c | 2013-03-18 17:08:08 -0700 | [diff] [blame] | 216 | case kActionCodePostinstallPowerwashError: |
Jay Srinivasan | 1c0fe79 | 2013-03-28 16:45:25 -0700 | [diff] [blame] | 217 | case kActionCodeUpdateCanceledByChannelChange: |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 218 | LOG(INFO) << "Not incrementing URL index or failure count for this error"; |
| 219 | break; |
| 220 | |
| 221 | case kActionCodeSuccess: // success code |
| 222 | case kActionCodeSetBootableFlagError: // unused |
| 223 | case kActionCodeUmaReportedMax: // not an error code |
| 224 | case kActionCodeOmahaRequestHTTPResponseBase: // aggregated already |
Jay Srinivasan | 55f50c2 | 2013-01-10 19:24:35 -0800 | [diff] [blame] | 225 | case kActionCodeDevModeFlag: // not an error code |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 226 | case kActionCodeResumedFlag: // not an error code |
Jay Srinivasan | 55f50c2 | 2013-01-10 19:24:35 -0800 | [diff] [blame] | 227 | case kActionCodeTestImageFlag: // not an error code |
| 228 | case kActionCodeTestOmahaUrlFlag: // not an error code |
| 229 | case kSpecialFlags: // not an error code |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 230 | // These shouldn't happen. Enumerating these explicitly here so that we |
| 231 | // can let the compiler warn about new error codes that are added to |
| 232 | // action_processor.h but not added here. |
| 233 | LOG(WARNING) << "Unexpected error code for UpdateFailed"; |
| 234 | break; |
| 235 | |
| 236 | // Note: Not adding a default here so as to let the compiler warn us of |
| 237 | // any new enums that were added in the .h but not listed in this switch. |
| 238 | } |
| 239 | } |
| 240 | |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 241 | bool PayloadState::ShouldBackoffDownload() { |
| 242 | if (response_.disable_payload_backoff) { |
| 243 | LOG(INFO) << "Payload backoff logic is disabled. " |
| 244 | "Can proceed with the download"; |
| 245 | return false; |
| 246 | } |
| 247 | |
| 248 | if (response_.is_delta_payload) { |
| 249 | // If delta payloads fail, we want to fallback quickly to full payloads as |
| 250 | // they are more likely to succeed. Exponential backoffs would greatly |
| 251 | // slow down the fallback to full payloads. So we don't backoff for delta |
| 252 | // payloads. |
| 253 | LOG(INFO) << "No backoffs for delta payloads. " |
| 254 | << "Can proceed with the download"; |
| 255 | return false; |
| 256 | } |
| 257 | |
| 258 | if (!utils::IsOfficialBuild()) { |
| 259 | // Backoffs are needed only for official builds. We do not want any delays |
| 260 | // or update failures due to backoffs during testing or development. |
| 261 | LOG(INFO) << "No backoffs for test/dev images. " |
| 262 | << "Can proceed with the download"; |
| 263 | return false; |
| 264 | } |
| 265 | |
| 266 | if (backoff_expiry_time_.is_null()) { |
| 267 | LOG(INFO) << "No backoff expiry time has been set. " |
| 268 | << "Can proceed with the download"; |
| 269 | return false; |
| 270 | } |
| 271 | |
| 272 | if (backoff_expiry_time_ < Time::Now()) { |
| 273 | LOG(INFO) << "The backoff expiry time (" |
| 274 | << utils::ToString(backoff_expiry_time_) |
| 275 | << ") has elapsed. Can proceed with the download"; |
| 276 | return false; |
| 277 | } |
| 278 | |
| 279 | LOG(INFO) << "Cannot proceed with downloads as we need to backoff until " |
| 280 | << utils::ToString(backoff_expiry_time_); |
| 281 | return true; |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 282 | } |
| 283 | |
| 284 | void PayloadState::IncrementPayloadAttemptNumber() { |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 285 | if (response_.is_delta_payload) { |
| 286 | LOG(INFO) << "Not incrementing payload attempt number for delta payloads"; |
| 287 | return; |
| 288 | } |
| 289 | |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 290 | LOG(INFO) << "Incrementing the payload attempt number"; |
| 291 | SetPayloadAttemptNumber(GetPayloadAttemptNumber() + 1); |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 292 | UpdateBackoffExpiryTime(); |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 293 | } |
| 294 | |
| 295 | void PayloadState::IncrementUrlIndex() { |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 296 | uint32_t next_url_index = GetUrlIndex() + 1; |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 297 | if (next_url_index < GetNumUrls()) { |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 298 | LOG(INFO) << "Incrementing the URL index for next attempt"; |
| 299 | SetUrlIndex(next_url_index); |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 300 | } else { |
| 301 | LOG(INFO) << "Resetting the current URL index (" << GetUrlIndex() << ") to " |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 302 | << "0 as we only have " << GetNumUrls() << " URL(s)"; |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 303 | SetUrlIndex(0); |
| 304 | IncrementPayloadAttemptNumber(); |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 305 | } |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 306 | |
| 307 | // Whenever we update the URL index, we should also clear the URL failure |
| 308 | // count so we can start over fresh for the new URL. |
| 309 | SetUrlFailureCount(0); |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 310 | } |
| 311 | |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 312 | void PayloadState::IncrementFailureCount() { |
| 313 | uint32_t next_url_failure_count = GetUrlFailureCount() + 1; |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 314 | if (next_url_failure_count < response_.max_failure_count_per_url) { |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 315 | LOG(INFO) << "Incrementing the URL failure count"; |
| 316 | SetUrlFailureCount(next_url_failure_count); |
| 317 | } else { |
| 318 | LOG(INFO) << "Reached max number of failures for Url" << GetUrlIndex() |
| 319 | << ". Trying next available URL"; |
| 320 | IncrementUrlIndex(); |
| 321 | } |
| 322 | } |
| 323 | |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 324 | void PayloadState::UpdateBackoffExpiryTime() { |
| 325 | if (response_.disable_payload_backoff) { |
| 326 | LOG(INFO) << "Resetting backoff expiry time as payload backoff is disabled"; |
| 327 | SetBackoffExpiryTime(Time()); |
| 328 | return; |
| 329 | } |
| 330 | |
| 331 | if (GetPayloadAttemptNumber() == 0) { |
| 332 | SetBackoffExpiryTime(Time()); |
| 333 | return; |
| 334 | } |
| 335 | |
| 336 | // Since we're doing left-shift below, make sure we don't shift more |
| 337 | // than this. E.g. if uint32_t is 4-bytes, don't left-shift more than 30 bits, |
| 338 | // since we don't expect value of kMaxBackoffDays to be more than 100 anyway. |
| 339 | uint32_t num_days = 1; // the value to be shifted. |
| 340 | const uint32_t kMaxShifts = (sizeof(num_days) * 8) - 2; |
| 341 | |
| 342 | // Normal backoff days is 2 raised to (payload_attempt_number - 1). |
| 343 | // E.g. if payload_attempt_number is over 30, limit power to 30. |
| 344 | uint32_t power = min(GetPayloadAttemptNumber() - 1, kMaxShifts); |
| 345 | |
| 346 | // The number of days is the minimum of 2 raised to (payload_attempt_number |
| 347 | // - 1) or kMaxBackoffDays. |
| 348 | num_days = min(num_days << power, kMaxBackoffDays); |
| 349 | |
| 350 | // We don't want all retries to happen exactly at the same time when |
| 351 | // retrying after backoff. So add some random minutes to fuzz. |
| 352 | int fuzz_minutes = utils::FuzzInt(0, kMaxBackoffFuzzMinutes); |
| 353 | TimeDelta next_backoff_interval = TimeDelta::FromDays(num_days) + |
| 354 | TimeDelta::FromMinutes(fuzz_minutes); |
| 355 | LOG(INFO) << "Incrementing the backoff expiry time by " |
| 356 | << utils::FormatTimeDelta(next_backoff_interval); |
| 357 | SetBackoffExpiryTime(Time::Now() + next_backoff_interval); |
| 358 | } |
| 359 | |
Jay Srinivasan | 19409b7 | 2013-04-12 19:23:36 -0700 | [diff] [blame] | 360 | void PayloadState::UpdateCurrentDownloadSource() { |
| 361 | current_download_source_ = kNumDownloadSources; |
| 362 | |
| 363 | if (GetUrlIndex() < response_.payload_urls.size()) { |
| 364 | string current_url = response_.payload_urls[GetUrlIndex()]; |
| 365 | if (StartsWithASCII(current_url, "https://", false)) |
| 366 | current_download_source_ = kDownloadSourceHttpsServer; |
| 367 | else if (StartsWithASCII(current_url, "http://", false)) |
| 368 | current_download_source_ = kDownloadSourceHttpServer; |
| 369 | } |
| 370 | |
| 371 | LOG(INFO) << "Current download source: " |
| 372 | << utils::ToString(current_download_source_); |
| 373 | } |
| 374 | |
| 375 | void PayloadState::UpdateBytesDownloaded(size_t count) { |
| 376 | SetCurrentBytesDownloaded( |
| 377 | current_download_source_, |
| 378 | GetCurrentBytesDownloaded(current_download_source_) + count, |
| 379 | false); |
| 380 | SetTotalBytesDownloaded( |
| 381 | current_download_source_, |
| 382 | GetTotalBytesDownloaded(current_download_source_) + count, |
| 383 | false); |
| 384 | } |
| 385 | |
| 386 | void PayloadState::ReportBytesDownloadedMetrics() { |
| 387 | // Report metrics collected from all known download sources to UMA. |
| 388 | // The reported data is in Megabytes in order to represent a larger |
| 389 | // sample range. |
| 390 | for (int i = 0; i < kNumDownloadSources; i++) { |
| 391 | DownloadSource source = static_cast<DownloadSource>(i); |
| 392 | const int kMaxMiBs = 10240; // Anything above 10GB goes in the last bucket. |
| 393 | |
| 394 | string metric = "Installer.SuccessfulMBsDownloadedFrom" + |
| 395 | utils::ToString(source); |
| 396 | uint64_t mbs = GetCurrentBytesDownloaded(source) / kNumBytesInOneMiB; |
| 397 | LOG(INFO) << "Uploading " << mbs << " (MBs) for metric " << metric; |
| 398 | system_state_->metrics_lib()->SendToUMA(metric, |
| 399 | mbs, |
| 400 | 0, // min |
| 401 | kMaxMiBs, |
| 402 | kNumDefaultUmaBuckets); |
| 403 | SetCurrentBytesDownloaded(source, 0, true); |
| 404 | |
| 405 | metric = "Installer.TotalMBsDownloadedFrom" + utils::ToString(source); |
| 406 | mbs = GetTotalBytesDownloaded(source) / kNumBytesInOneMiB; |
| 407 | LOG(INFO) << "Uploading " << mbs << " (MBs) for metric " << metric; |
| 408 | system_state_->metrics_lib()->SendToUMA(metric, |
| 409 | mbs, |
| 410 | 0, // min |
| 411 | kMaxMiBs, |
| 412 | kNumDefaultUmaBuckets); |
| 413 | |
| 414 | SetTotalBytesDownloaded(source, 0, true); |
| 415 | } |
| 416 | } |
| 417 | |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 418 | void PayloadState::ResetPersistedState() { |
| 419 | SetPayloadAttemptNumber(0); |
| 420 | SetUrlIndex(0); |
| 421 | SetUrlFailureCount(0); |
| 422 | UpdateBackoffExpiryTime(); // This will reset the backoff expiry time. |
David Zeuthen | 9a017f2 | 2013-04-11 16:10:26 -0700 | [diff] [blame] | 423 | SetUpdateTimestampStart(Time::Now()); |
| 424 | SetUpdateTimestampEnd(Time()); // Set to null time |
| 425 | SetUpdateDurationUptime(TimeDelta::FromSeconds(0)); |
Jay Srinivasan | 19409b7 | 2013-04-12 19:23:36 -0700 | [diff] [blame] | 426 | ResetDownloadSourcesOnNewUpdate(); |
| 427 | } |
| 428 | |
| 429 | void PayloadState::ResetDownloadSourcesOnNewUpdate() { |
| 430 | for (int i = 0; i < kNumDownloadSources; i++) { |
| 431 | DownloadSource source = static_cast<DownloadSource>(i); |
| 432 | SetCurrentBytesDownloaded(source, 0, true); |
| 433 | // Note: Not resetting the TotalBytesDownloaded as we want that metric |
| 434 | // to count the bytes downloaded across various update attempts until |
| 435 | // we have successfully applied the update. |
| 436 | } |
| 437 | } |
| 438 | |
| 439 | int64_t PayloadState::GetPersistedValue(const string& key) { |
| 440 | CHECK(prefs_); |
| 441 | if (!prefs_->Exists(key)) |
| 442 | return 0; |
| 443 | |
| 444 | int64_t stored_value; |
| 445 | if (!prefs_->GetInt64(key, &stored_value)) |
| 446 | return 0; |
| 447 | |
| 448 | if (stored_value < 0) { |
| 449 | LOG(ERROR) << key << ": Invalid value (" << stored_value |
| 450 | << ") in persisted state. Defaulting to 0"; |
| 451 | return 0; |
| 452 | } |
| 453 | |
| 454 | return stored_value; |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 455 | } |
| 456 | |
| 457 | string PayloadState::CalculateResponseSignature() { |
| 458 | string response_sign = StringPrintf("NumURLs = %d\n", |
| 459 | response_.payload_urls.size()); |
| 460 | |
| 461 | for (size_t i = 0; i < response_.payload_urls.size(); i++) |
| 462 | response_sign += StringPrintf("Url%d = %s\n", |
| 463 | i, response_.payload_urls[i].c_str()); |
| 464 | |
| 465 | response_sign += StringPrintf("Payload Size = %llu\n" |
| 466 | "Payload Sha256 Hash = %s\n" |
| 467 | "Metadata Size = %llu\n" |
| 468 | "Metadata Signature = %s\n" |
| 469 | "Is Delta Payload = %d\n" |
| 470 | "Max Failure Count Per Url = %d\n" |
| 471 | "Disable Payload Backoff = %d\n", |
| 472 | response_.size, |
| 473 | response_.hash.c_str(), |
| 474 | response_.metadata_size, |
| 475 | response_.metadata_signature.c_str(), |
| 476 | response_.is_delta_payload, |
| 477 | response_.max_failure_count_per_url, |
| 478 | response_.disable_payload_backoff); |
| 479 | return response_sign; |
| 480 | } |
| 481 | |
| 482 | void PayloadState::LoadResponseSignature() { |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 483 | CHECK(prefs_); |
| 484 | string stored_value; |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 485 | if (prefs_->Exists(kPrefsCurrentResponseSignature) && |
| 486 | prefs_->GetString(kPrefsCurrentResponseSignature, &stored_value)) { |
| 487 | SetResponseSignature(stored_value); |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 488 | } |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 489 | } |
| 490 | |
Jay Srinivasan | 19409b7 | 2013-04-12 19:23:36 -0700 | [diff] [blame] | 491 | void PayloadState::SetResponseSignature(const string& response_signature) { |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 492 | CHECK(prefs_); |
| 493 | response_signature_ = response_signature; |
| 494 | LOG(INFO) << "Current Response Signature = \n" << response_signature_; |
| 495 | prefs_->SetString(kPrefsCurrentResponseSignature, response_signature_); |
| 496 | } |
| 497 | |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 498 | void PayloadState::LoadPayloadAttemptNumber() { |
Jay Srinivasan | 19409b7 | 2013-04-12 19:23:36 -0700 | [diff] [blame] | 499 | SetPayloadAttemptNumber(GetPersistedValue(kPrefsPayloadAttemptNumber)); |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 500 | } |
| 501 | |
| 502 | void PayloadState::SetPayloadAttemptNumber(uint32_t payload_attempt_number) { |
| 503 | CHECK(prefs_); |
| 504 | payload_attempt_number_ = payload_attempt_number; |
| 505 | LOG(INFO) << "Payload Attempt Number = " << payload_attempt_number_; |
| 506 | prefs_->SetInt64(kPrefsPayloadAttemptNumber, payload_attempt_number_); |
| 507 | } |
| 508 | |
| 509 | void PayloadState::LoadUrlIndex() { |
Jay Srinivasan | 19409b7 | 2013-04-12 19:23:36 -0700 | [diff] [blame] | 510 | SetUrlIndex(GetPersistedValue(kPrefsCurrentUrlIndex)); |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 511 | } |
| 512 | |
| 513 | void PayloadState::SetUrlIndex(uint32_t url_index) { |
| 514 | CHECK(prefs_); |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 515 | url_index_ = url_index; |
| 516 | LOG(INFO) << "Current URL Index = " << url_index_; |
| 517 | prefs_->SetInt64(kPrefsCurrentUrlIndex, url_index_); |
Jay Srinivasan | 19409b7 | 2013-04-12 19:23:36 -0700 | [diff] [blame] | 518 | |
| 519 | // Also update the download source, which is purely dependent on the |
| 520 | // current URL index alone. |
| 521 | UpdateCurrentDownloadSource(); |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 522 | } |
| 523 | |
| 524 | void PayloadState::LoadUrlFailureCount() { |
Jay Srinivasan | 19409b7 | 2013-04-12 19:23:36 -0700 | [diff] [blame] | 525 | SetUrlFailureCount(GetPersistedValue(kPrefsCurrentUrlFailureCount)); |
Jay Srinivasan | 2b5a0f0 | 2012-12-19 17:25:56 -0800 | [diff] [blame] | 526 | } |
| 527 | |
| 528 | void PayloadState::SetUrlFailureCount(uint32_t url_failure_count) { |
| 529 | CHECK(prefs_); |
| 530 | url_failure_count_ = url_failure_count; |
| 531 | LOG(INFO) << "Current URL (Url" << GetUrlIndex() |
| 532 | << ")'s Failure Count = " << url_failure_count_; |
| 533 | prefs_->SetInt64(kPrefsCurrentUrlFailureCount, url_failure_count_); |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 534 | } |
| 535 | |
Jay Srinivasan | 0826288 | 2012-12-28 19:29:43 -0800 | [diff] [blame] | 536 | void PayloadState::LoadBackoffExpiryTime() { |
| 537 | CHECK(prefs_); |
| 538 | int64_t stored_value; |
| 539 | if (!prefs_->Exists(kPrefsBackoffExpiryTime)) |
| 540 | return; |
| 541 | |
| 542 | if (!prefs_->GetInt64(kPrefsBackoffExpiryTime, &stored_value)) |
| 543 | return; |
| 544 | |
| 545 | Time stored_time = Time::FromInternalValue(stored_value); |
| 546 | if (stored_time > Time::Now() + TimeDelta::FromDays(kMaxBackoffDays)) { |
| 547 | LOG(ERROR) << "Invalid backoff expiry time (" |
| 548 | << utils::ToString(stored_time) |
| 549 | << ") in persisted state. Resetting."; |
| 550 | stored_time = Time(); |
| 551 | } |
| 552 | SetBackoffExpiryTime(stored_time); |
| 553 | } |
| 554 | |
| 555 | void PayloadState::SetBackoffExpiryTime(const Time& new_time) { |
| 556 | CHECK(prefs_); |
| 557 | backoff_expiry_time_ = new_time; |
| 558 | LOG(INFO) << "Backoff Expiry Time = " |
| 559 | << utils::ToString(backoff_expiry_time_); |
| 560 | prefs_->SetInt64(kPrefsBackoffExpiryTime, |
| 561 | backoff_expiry_time_.ToInternalValue()); |
| 562 | } |
| 563 | |
David Zeuthen | 9a017f2 | 2013-04-11 16:10:26 -0700 | [diff] [blame] | 564 | TimeDelta PayloadState::GetUpdateDuration() { |
| 565 | Time end_time = update_timestamp_end_.is_null() ? Time::Now() : |
| 566 | update_timestamp_end_; |
| 567 | return end_time - update_timestamp_start_; |
| 568 | } |
| 569 | |
| 570 | void PayloadState::LoadUpdateTimestampStart() { |
| 571 | int64_t stored_value; |
| 572 | Time stored_time; |
| 573 | |
| 574 | CHECK(prefs_); |
| 575 | |
| 576 | Time now = Time::Now(); |
| 577 | |
| 578 | if (!prefs_->Exists(kPrefsUpdateTimestampStart)) { |
| 579 | // The preference missing is not unexpected - in that case, just |
| 580 | // use the current time as start time |
| 581 | stored_time = now; |
| 582 | } else if (!prefs_->GetInt64(kPrefsUpdateTimestampStart, &stored_value)) { |
| 583 | LOG(ERROR) << "Invalid UpdateTimestampStart value. Resetting."; |
| 584 | stored_time = now; |
| 585 | } else { |
| 586 | stored_time = Time::FromInternalValue(stored_value); |
| 587 | } |
| 588 | |
| 589 | // Sanity check: If the time read from disk is in the future |
| 590 | // (modulo some slack to account for possible NTP drift |
| 591 | // adjustments), something is fishy and we should report and |
| 592 | // reset. |
| 593 | TimeDelta duration_according_to_stored_time = now - stored_time; |
| 594 | if (duration_according_to_stored_time < -kDurationSlack) { |
| 595 | LOG(ERROR) << "The UpdateTimestampStart value (" |
| 596 | << utils::ToString(stored_time) |
| 597 | << ") in persisted state is " |
| 598 | << duration_according_to_stored_time.InSeconds() |
| 599 | << " seconds in the future. Resetting."; |
| 600 | stored_time = now; |
| 601 | } |
| 602 | |
| 603 | SetUpdateTimestampStart(stored_time); |
| 604 | } |
| 605 | |
| 606 | void PayloadState::SetUpdateTimestampStart(const Time& value) { |
| 607 | CHECK(prefs_); |
| 608 | update_timestamp_start_ = value; |
| 609 | prefs_->SetInt64(kPrefsUpdateTimestampStart, |
| 610 | update_timestamp_start_.ToInternalValue()); |
| 611 | LOG(INFO) << "Update Timestamp Start = " |
| 612 | << utils::ToString(update_timestamp_start_); |
| 613 | } |
| 614 | |
| 615 | void PayloadState::SetUpdateTimestampEnd(const Time& value) { |
| 616 | update_timestamp_end_ = value; |
| 617 | LOG(INFO) << "Update Timestamp End = " |
| 618 | << utils::ToString(update_timestamp_end_); |
| 619 | } |
| 620 | |
| 621 | TimeDelta PayloadState::GetUpdateDurationUptime() { |
| 622 | return update_duration_uptime_; |
| 623 | } |
| 624 | |
| 625 | void PayloadState::LoadUpdateDurationUptime() { |
| 626 | int64_t stored_value; |
| 627 | TimeDelta stored_delta; |
| 628 | |
| 629 | CHECK(prefs_); |
| 630 | |
| 631 | if (!prefs_->Exists(kPrefsUpdateDurationUptime)) { |
| 632 | // The preference missing is not unexpected - in that case, just |
| 633 | // we'll use zero as the delta |
| 634 | } else if (!prefs_->GetInt64(kPrefsUpdateDurationUptime, &stored_value)) { |
| 635 | LOG(ERROR) << "Invalid UpdateDurationUptime value. Resetting."; |
| 636 | stored_delta = TimeDelta::FromSeconds(0); |
| 637 | } else { |
| 638 | stored_delta = TimeDelta::FromInternalValue(stored_value); |
| 639 | } |
| 640 | |
| 641 | // Sanity-check: Uptime can never be greater than the wall-clock |
| 642 | // difference (modulo some slack). If it is, report and reset |
| 643 | // to the wall-clock difference. |
| 644 | TimeDelta diff = GetUpdateDuration() - stored_delta; |
| 645 | if (diff < -kDurationSlack) { |
| 646 | LOG(ERROR) << "The UpdateDurationUptime value (" |
| 647 | << stored_delta.InSeconds() << " seconds" |
| 648 | << ") in persisted state is " |
| 649 | << diff.InSeconds() |
| 650 | << " seconds larger than the wall-clock delta. Resetting."; |
| 651 | stored_delta = update_duration_current_; |
| 652 | } |
| 653 | |
| 654 | SetUpdateDurationUptime(stored_delta); |
| 655 | } |
| 656 | |
| 657 | void PayloadState::SetUpdateDurationUptimeExtended(const TimeDelta& value, |
| 658 | const Time& timestamp, |
| 659 | bool use_logging) { |
| 660 | CHECK(prefs_); |
| 661 | update_duration_uptime_ = value; |
| 662 | update_duration_uptime_timestamp_ = timestamp; |
| 663 | prefs_->SetInt64(kPrefsUpdateDurationUptime, |
| 664 | update_duration_uptime_.ToInternalValue()); |
| 665 | if (use_logging) { |
| 666 | LOG(INFO) << "Update Duration Uptime = " |
| 667 | << update_duration_uptime_.InSeconds() |
| 668 | << " seconds"; |
| 669 | } |
| 670 | } |
| 671 | |
| 672 | void PayloadState::SetUpdateDurationUptime(const TimeDelta& value) { |
| 673 | SetUpdateDurationUptimeExtended(value, utils::GetMonotonicTime(), true); |
| 674 | } |
| 675 | |
| 676 | void PayloadState::CalculateUpdateDurationUptime() { |
| 677 | Time now = utils::GetMonotonicTime(); |
| 678 | TimeDelta uptime_since_last_update = now - update_duration_uptime_timestamp_; |
| 679 | TimeDelta new_uptime = update_duration_uptime_ + uptime_since_last_update; |
| 680 | // We're frequently called so avoid logging this write |
| 681 | SetUpdateDurationUptimeExtended(new_uptime, now, false); |
| 682 | } |
| 683 | |
Jay Srinivasan | 19409b7 | 2013-04-12 19:23:36 -0700 | [diff] [blame] | 684 | string PayloadState::GetPrefsKey(const string& prefix, DownloadSource source) { |
| 685 | return prefix + "-from-" + utils::ToString(source); |
| 686 | } |
| 687 | |
| 688 | void PayloadState::LoadCurrentBytesDownloaded(DownloadSource source) { |
| 689 | string key = GetPrefsKey(kPrefsCurrentBytesDownloaded, source); |
| 690 | SetCurrentBytesDownloaded(source, GetPersistedValue(key), true); |
| 691 | } |
| 692 | |
| 693 | void PayloadState::SetCurrentBytesDownloaded( |
| 694 | DownloadSource source, |
| 695 | uint64_t current_bytes_downloaded, |
| 696 | bool log) { |
| 697 | CHECK(prefs_); |
| 698 | |
| 699 | if (source >= kNumDownloadSources) |
| 700 | return; |
| 701 | |
| 702 | // Update the in-memory value. |
| 703 | current_bytes_downloaded_[source] = current_bytes_downloaded; |
| 704 | |
| 705 | string prefs_key = GetPrefsKey(kPrefsCurrentBytesDownloaded, source); |
| 706 | prefs_->SetInt64(prefs_key, current_bytes_downloaded); |
| 707 | LOG_IF(INFO, log) << "Current bytes downloaded for " |
| 708 | << utils::ToString(source) << " = " |
| 709 | << GetCurrentBytesDownloaded(source); |
| 710 | } |
| 711 | |
| 712 | void PayloadState::LoadTotalBytesDownloaded(DownloadSource source) { |
| 713 | string key = GetPrefsKey(kPrefsTotalBytesDownloaded, source); |
| 714 | SetTotalBytesDownloaded(source, GetPersistedValue(key), true); |
| 715 | } |
| 716 | |
| 717 | void PayloadState::SetTotalBytesDownloaded( |
| 718 | DownloadSource source, |
| 719 | uint64_t total_bytes_downloaded, |
| 720 | bool log) { |
| 721 | CHECK(prefs_); |
| 722 | |
| 723 | if (source >= kNumDownloadSources) |
| 724 | return; |
| 725 | |
| 726 | // Update the in-memory value. |
| 727 | total_bytes_downloaded_[source] = total_bytes_downloaded; |
| 728 | |
| 729 | // Persist. |
| 730 | string prefs_key = GetPrefsKey(kPrefsTotalBytesDownloaded, source); |
| 731 | prefs_->SetInt64(prefs_key, total_bytes_downloaded); |
| 732 | LOG_IF(INFO, log) << "Total bytes downloaded for " |
| 733 | << utils::ToString(source) << " = " |
| 734 | << GetTotalBytesDownloaded(source); |
| 735 | } |
| 736 | |
Jay Srinivasan | 6f6ea00 | 2012-12-14 11:26:28 -0800 | [diff] [blame] | 737 | } // namespace chromeos_update_engine |