blob: 429dbc4f320aa48231e610490d48ee7974c21987 [file] [log] [blame]
mflodman@webrtc.orge6168f52013-06-26 11:23:01 +00001/*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "video/overuse_frame_detector.h"
mflodman@webrtc.orge6168f52013-06-26 11:23:01 +000012
pbos@webrtc.orga9575702013-08-30 17:16:32 +000013#include <math.h>
Piotr Tworek5e4833c2017-12-12 12:09:31 +010014#include <stdio.h>
mflodman@webrtc.orgd4412fe2013-07-31 16:42:21 +000015
asapersson@webrtc.org9e5b0342013-12-04 13:47:44 +000016#include <algorithm>
17#include <list>
asapersson@webrtc.org734a5322014-06-10 06:35:22 +000018#include <map>
Mirko Bonadei317a1f02019-09-17 17:06:18 +020019#include <memory>
sprangc5d62e22017-04-02 23:53:04 -070020#include <string>
21#include <utility>
asapersson@webrtc.org9e5b0342013-12-04 13:47:44 +000022
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/video_frame.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020024#include "rtc_base/checks.h"
25#include "rtc_base/logging.h"
Niels Möller7dc26b72017-12-06 10:27:48 +010026#include "rtc_base/numerics/exp_filter.h"
Steve Anton10542f22019-01-11 09:11:00 -080027#include "rtc_base/time_utils.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020028#include "system_wrappers/include/field_trial.h"
mflodman@webrtc.orge6168f52013-06-26 11:23:01 +000029
pbosa1025072016-05-14 03:04:19 -070030#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
torbjorng448468d2016-02-10 08:11:57 -080031#include <mach/mach.h>
pbosa1025072016-05-14 03:04:19 -070032#endif // defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
torbjorng448468d2016-02-10 08:11:57 -080033
mflodman@webrtc.orge6168f52013-06-26 11:23:01 +000034namespace webrtc {
35
mflodman@webrtc.orgd4412fe2013-07-31 16:42:21 +000036namespace {
perkjd52063f2016-09-07 06:32:18 -070037const int64_t kCheckForOveruseIntervalMs = 5000;
38const int64_t kTimeToFirstCheckForOveruseMs = 100;
mflodman@webrtc.orgd4412fe2013-07-31 16:42:21 +000039
pbos@webrtc.orga9575702013-08-30 17:16:32 +000040// Delay between consecutive rampups. (Used for quick recovery.)
41const int kQuickRampUpDelayMs = 10 * 1000;
42// Delay between rampup attempts. Initially uses standard, scales up to max.
asapersson@webrtc.org23a4d852014-08-13 14:33:49 +000043const int kStandardRampUpDelayMs = 40 * 1000;
asapersson@webrtc.org2881ab12014-06-12 08:46:46 +000044const int kMaxRampUpDelayMs = 240 * 1000;
pbos@webrtc.orga9575702013-08-30 17:16:32 +000045// Expontential back-off factor, to prevent annoying up-down behaviour.
46const double kRampUpBackoffFactor = 2.0;
mflodman@webrtc.orgd4412fe2013-07-31 16:42:21 +000047
asapersson@webrtc.orgd9803072014-06-16 14:27:19 +000048// Max number of overuses detected before always applying the rampup delay.
asapersson@webrtc.org23a4d852014-08-13 14:33:49 +000049const int kMaxOverusesBeforeApplyRampupDelay = 4;
asapersson@webrtc.org9e5b0342013-12-04 13:47:44 +000050
Niels Möller7dc26b72017-12-06 10:27:48 +010051// The maximum exponent to use in VCMExpFilter.
52const float kMaxExp = 7.0f;
53// Default value used before first reconfiguration.
54const int kDefaultFrameRate = 30;
55// Default sample diff, default frame rate.
56const float kDefaultSampleDiffMs = 1000.0f / kDefaultFrameRate;
57// A factor applied to the sample diff on OnTargetFramerateUpdated to determine
58// a max limit for the sample diff. For instance, with a framerate of 30fps,
59// the sample diff is capped to (1000 / 30) * 1.35 = 45ms. This prevents
60// triggering too soon if there are individual very large outliers.
61const float kMaxSampleDiffMarginFactor = 1.35f;
62// Minimum framerate allowed for usage calculation. This prevents crazy long
63// encode times from being accepted if the frame rate happens to be low.
64const int kMinFramerate = 7;
65const int kMaxFramerate = 30;
66
sprangb1ca0732017-02-01 08:38:12 -080067const auto kScaleReasonCpu = AdaptationObserverInterface::AdaptReason::kCpu;
torbjorng448468d2016-02-10 08:11:57 -080068
asapersson@webrtc.org9aed0022014-10-16 06:57:12 +000069// Class for calculating the processing usage on the send-side (the average
70// processing time of a frame divided by the average time difference between
71// captured frames).
Niels Möller83dbeac2017-12-14 16:39:44 +010072class SendProcessingUsage1 : public OveruseFrameDetector::ProcessingUsage {
asapersson@webrtc.org9e5b0342013-12-04 13:47:44 +000073 public:
Niels Möller83dbeac2017-12-14 16:39:44 +010074 explicit SendProcessingUsage1(const CpuOveruseOptions& options)
Niels Möller7dc26b72017-12-06 10:27:48 +010075 : kWeightFactorFrameDiff(0.998f),
76 kWeightFactorProcessing(0.995f),
77 kInitialSampleDiffMs(40.0f),
Niels Möller7dc26b72017-12-06 10:27:48 +010078 options_(options),
Niels Möllere08cf3a2017-12-07 15:23:58 +010079 count_(0),
80 last_processed_capture_time_us_(-1),
Niels Möller7dc26b72017-12-06 10:27:48 +010081 max_sample_diff_ms_(kDefaultSampleDiffMs * kMaxSampleDiffMarginFactor),
82 filtered_processing_ms_(new rtc::ExpFilter(kWeightFactorProcessing)),
83 filtered_frame_diff_ms_(new rtc::ExpFilter(kWeightFactorFrameDiff)) {
asapersson@webrtc.orgce12f1f2014-03-24 21:59:16 +000084 Reset();
asapersson@webrtc.org9e5b0342013-12-04 13:47:44 +000085 }
Mirko Bonadeife055c12019-01-29 22:53:28 +010086 ~SendProcessingUsage1() override {}
asapersson@webrtc.org9e5b0342013-12-04 13:47:44 +000087
Niels Möller904f8692017-12-07 11:22:39 +010088 void Reset() override {
Niels Möllere08cf3a2017-12-07 15:23:58 +010089 frame_timing_.clear();
Niels Möller7dc26b72017-12-06 10:27:48 +010090 count_ = 0;
Niels Möllere08cf3a2017-12-07 15:23:58 +010091 last_processed_capture_time_us_ = -1;
Niels Möller7dc26b72017-12-06 10:27:48 +010092 max_sample_diff_ms_ = kDefaultSampleDiffMs * kMaxSampleDiffMarginFactor;
93 filtered_frame_diff_ms_->Reset(kWeightFactorFrameDiff);
94 filtered_frame_diff_ms_->Apply(1.0f, kInitialSampleDiffMs);
95 filtered_processing_ms_->Reset(kWeightFactorProcessing);
96 filtered_processing_ms_->Apply(1.0f, InitialProcessingMs());
asapersson@webrtc.orgce12f1f2014-03-24 21:59:16 +000097 }
98
Niels Möller904f8692017-12-07 11:22:39 +010099 void SetMaxSampleDiffMs(float diff_ms) override {
100 max_sample_diff_ms_ = diff_ms;
101 }
sprangfda496a2017-06-15 04:21:07 -0700102
Niels Möllere08cf3a2017-12-07 15:23:58 +0100103 void FrameCaptured(const VideoFrame& frame,
104 int64_t time_when_first_seen_us,
105 int64_t last_capture_time_us) override {
106 if (last_capture_time_us != -1)
107 AddCaptureSample(1e-3 * (time_when_first_seen_us - last_capture_time_us));
108
109 frame_timing_.push_back(FrameTiming(frame.timestamp_us(), frame.timestamp(),
110 time_when_first_seen_us));
Niels Möller7dc26b72017-12-06 10:27:48 +0100111 }
112
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200113 absl::optional<int> FrameSent(
Niels Möller83dbeac2017-12-14 16:39:44 +0100114 uint32_t timestamp,
115 int64_t time_sent_in_us,
116 int64_t /* capture_time_us */,
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200117 absl::optional<int> /* encode_duration_us */) override {
118 absl::optional<int> encode_duration_us;
Niels Möllere08cf3a2017-12-07 15:23:58 +0100119 // Delay before reporting actual encoding time, used to have the ability to
120 // detect total encoding time when encoding more than one layer. Encoding is
121 // here assumed to finish within a second (or that we get enough long-time
122 // samples before one second to trigger an overuse even when this is not the
123 // case).
124 static const int64_t kEncodingTimeMeasureWindowMs = 1000;
125 for (auto& it : frame_timing_) {
126 if (it.timestamp == timestamp) {
127 it.last_send_us = time_sent_in_us;
128 break;
129 }
130 }
131 // TODO(pbos): Handle the case/log errors when not finding the corresponding
132 // frame (either very slow encoding or incorrect wrong timestamps returned
133 // from the encoder).
134 // This is currently the case for all frames on ChromeOS, so logging them
135 // would be spammy, and triggering overuse would be wrong.
136 // https://crbug.com/350106
137 while (!frame_timing_.empty()) {
138 FrameTiming timing = frame_timing_.front();
139 if (time_sent_in_us - timing.capture_us <
140 kEncodingTimeMeasureWindowMs * rtc::kNumMicrosecsPerMillisec) {
141 break;
142 }
143 if (timing.last_send_us != -1) {
144 encode_duration_us.emplace(
145 static_cast<int>(timing.last_send_us - timing.capture_us));
Niels Möller6b642f72017-12-08 14:11:14 +0100146
Niels Möllere08cf3a2017-12-07 15:23:58 +0100147 if (last_processed_capture_time_us_ != -1) {
148 int64_t diff_us = timing.capture_us - last_processed_capture_time_us_;
149 AddSample(1e-3 * (*encode_duration_us), 1e-3 * diff_us);
150 }
151 last_processed_capture_time_us_ = timing.capture_us;
152 }
153 frame_timing_.pop_front();
154 }
155 return encode_duration_us;
Niels Möller7dc26b72017-12-06 10:27:48 +0100156 }
157
Niels Möller904f8692017-12-07 11:22:39 +0100158 int Value() override {
Niels Möller7dc26b72017-12-06 10:27:48 +0100159 if (count_ < static_cast<uint32_t>(options_.min_frame_samples)) {
160 return static_cast<int>(InitialUsageInPercent() + 0.5f);
asapersson@webrtc.orgce12f1f2014-03-24 21:59:16 +0000161 }
Niels Möller7dc26b72017-12-06 10:27:48 +0100162 float frame_diff_ms = std::max(filtered_frame_diff_ms_->filtered(), 1.0f);
163 frame_diff_ms = std::min(frame_diff_ms, max_sample_diff_ms_);
164 float encode_usage_percent =
165 100.0f * filtered_processing_ms_->filtered() / frame_diff_ms;
166 return static_cast<int>(encode_usage_percent + 0.5);
asapersson@webrtc.org9e5b0342013-12-04 13:47:44 +0000167 }
168
asapersson@webrtc.org2881ab12014-06-12 08:46:46 +0000169 private:
Niels Möllere08cf3a2017-12-07 15:23:58 +0100170 struct FrameTiming {
171 FrameTiming(int64_t capture_time_us, uint32_t timestamp, int64_t now)
172 : capture_time_us(capture_time_us),
173 timestamp(timestamp),
174 capture_us(now),
175 last_send_us(-1) {}
176 int64_t capture_time_us;
177 uint32_t timestamp;
178 int64_t capture_us;
179 int64_t last_send_us;
180 };
181
182 void AddCaptureSample(float sample_ms) {
183 float exp = sample_ms / kDefaultSampleDiffMs;
184 exp = std::min(exp, kMaxExp);
185 filtered_frame_diff_ms_->Apply(exp, sample_ms);
186 }
187
188 void AddSample(float processing_ms, int64_t diff_last_sample_ms) {
189 ++count_;
190 float exp = diff_last_sample_ms / kDefaultSampleDiffMs;
191 exp = std::min(exp, kMaxExp);
192 filtered_processing_ms_->Apply(exp, processing_ms);
193 }
194
Niels Möller7dc26b72017-12-06 10:27:48 +0100195 float InitialUsageInPercent() const {
196 // Start in between the underuse and overuse threshold.
197 return (options_.low_encode_usage_threshold_percent +
Yves Gerey665174f2018-06-19 15:03:05 +0200198 options_.high_encode_usage_threshold_percent) /
199 2.0f;
Niels Möller7dc26b72017-12-06 10:27:48 +0100200 }
201
202 float InitialProcessingMs() const {
203 return InitialUsageInPercent() * kInitialSampleDiffMs / 100;
204 }
205
206 const float kWeightFactorFrameDiff;
207 const float kWeightFactorProcessing;
208 const float kInitialSampleDiffMs;
Niels Möllere08cf3a2017-12-07 15:23:58 +0100209
Peter Boström4b91bd02015-06-26 06:58:16 +0200210 const CpuOveruseOptions options_;
Niels Möllere08cf3a2017-12-07 15:23:58 +0100211 std::list<FrameTiming> frame_timing_;
212 uint64_t count_;
213 int64_t last_processed_capture_time_us_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100214 float max_sample_diff_ms_;
215 std::unique_ptr<rtc::ExpFilter> filtered_processing_ms_;
216 std::unique_ptr<rtc::ExpFilter> filtered_frame_diff_ms_;
asapersson@webrtc.org9e5b0342013-12-04 13:47:44 +0000217};
218
Niels Möller83dbeac2017-12-14 16:39:44 +0100219// New cpu load estimator.
220// TODO(bugs.webrtc.org/8504): For some period of time, we need to
221// switch between the two versions of the estimator for experiments.
222// When problems are sorted out, the old estimator should be deleted.
223class SendProcessingUsage2 : public OveruseFrameDetector::ProcessingUsage {
sprangc5d62e22017-04-02 23:53:04 -0700224 public:
Niels Möller83dbeac2017-12-14 16:39:44 +0100225 explicit SendProcessingUsage2(const CpuOveruseOptions& options)
226 : options_(options) {
227 Reset();
228 }
Mirko Bonadeife055c12019-01-29 22:53:28 +0100229 ~SendProcessingUsage2() override = default;
Niels Möller83dbeac2017-12-14 16:39:44 +0100230
231 void Reset() override {
232 prev_time_us_ = -1;
233 // Start in between the underuse and overuse threshold.
234 load_estimate_ = (options_.low_encode_usage_threshold_percent +
235 options_.high_encode_usage_threshold_percent) /
236 200.0;
237 }
238
239 void SetMaxSampleDiffMs(float /* diff_ms */) override {}
240
241 void FrameCaptured(const VideoFrame& frame,
242 int64_t time_when_first_seen_us,
243 int64_t last_capture_time_us) override {}
244
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200245 absl::optional<int> FrameSent(
Niels Möllerf5033ad2018-08-14 17:00:46 +0200246 uint32_t /* timestamp */,
247 int64_t /* time_sent_in_us */,
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200248 int64_t capture_time_us,
249 absl::optional<int> encode_duration_us) override {
Niels Möller83dbeac2017-12-14 16:39:44 +0100250 if (encode_duration_us) {
Niels Möllerf5033ad2018-08-14 17:00:46 +0200251 int duration_per_frame_us =
252 DurationPerInputFrame(capture_time_us, *encode_duration_us);
Niels Möller83dbeac2017-12-14 16:39:44 +0100253 if (prev_time_us_ != -1) {
Niels Möller58d2a5e2018-08-07 16:37:18 +0200254 if (capture_time_us < prev_time_us_) {
255 // The weighting in AddSample assumes that samples are processed with
256 // non-decreasing measurement timestamps. We could implement
257 // appropriate weights for samples arriving late, but since it is a
258 // rare case, keep things simple, by just pushing those measurements a
259 // bit forward in time.
260 capture_time_us = prev_time_us_;
261 }
Niels Möllerf5033ad2018-08-14 17:00:46 +0200262 AddSample(1e-6 * duration_per_frame_us,
Niels Möller83dbeac2017-12-14 16:39:44 +0100263 1e-6 * (capture_time_us - prev_time_us_));
264 }
265 }
266 prev_time_us_ = capture_time_us;
267
268 return encode_duration_us;
269 }
270
271 private:
272 void AddSample(double encode_time, double diff_time) {
273 RTC_CHECK_GE(diff_time, 0.0);
274
275 // Use the filter update
276 //
277 // load <-- x/d (1-exp (-d/T)) + exp (-d/T) load
278 //
279 // where we must take care for small d, using the proper limit
280 // (1 - exp(-d/tau)) / d = 1/tau - d/2tau^2 + O(d^2)
281 double tau = (1e-3 * options_.filter_time_ms);
282 double e = diff_time / tau;
283 double c;
284 if (e < 0.0001) {
285 c = (1 - e / 2) / tau;
286 } else {
287 c = -expm1(-e) / diff_time;
288 }
289 load_estimate_ = c * encode_time + exp(-e) * load_estimate_;
290 }
291
Niels Möllerf5033ad2018-08-14 17:00:46 +0200292 int64_t DurationPerInputFrame(int64_t capture_time_us,
293 int64_t encode_time_us) {
294 // Discard data on old frames; limit 2 seconds.
295 static constexpr int64_t kMaxAge = 2 * rtc::kNumMicrosecsPerSec;
296 for (auto it = max_encode_time_per_input_frame_.begin();
297 it != max_encode_time_per_input_frame_.end() &&
298 it->first < capture_time_us - kMaxAge;) {
299 it = max_encode_time_per_input_frame_.erase(it);
300 }
301
302 std::map<int64_t, int>::iterator it;
303 bool inserted;
304 std::tie(it, inserted) = max_encode_time_per_input_frame_.emplace(
305 capture_time_us, encode_time_us);
306 if (inserted) {
307 // First encoded frame for this input frame.
308 return encode_time_us;
309 }
310 if (encode_time_us <= it->second) {
311 // Shorter encode time than previous frame (unlikely). Count it as being
312 // done in parallel.
313 return 0;
314 }
315 // Record new maximum encode time, and return increase from previous max.
316 int increase = encode_time_us - it->second;
317 it->second = encode_time_us;
318 return increase;
319 }
320
Niels Möller83dbeac2017-12-14 16:39:44 +0100321 int Value() override {
322 return static_cast<int>(100.0 * load_estimate_ + 0.5);
323 }
324
Niels Möller83dbeac2017-12-14 16:39:44 +0100325 const CpuOveruseOptions options_;
Niels Möllerf5033ad2018-08-14 17:00:46 +0200326 // Indexed by the capture timestamp, used as frame id.
327 std::map<int64_t, int> max_encode_time_per_input_frame_;
328
Niels Möller83dbeac2017-12-14 16:39:44 +0100329 int64_t prev_time_us_ = -1;
330 double load_estimate_;
331};
332
333// Class used for manual testing of overuse, enabled via field trial flag.
334class OverdoseInjector : public OveruseFrameDetector::ProcessingUsage {
335 public:
336 OverdoseInjector(std::unique_ptr<OveruseFrameDetector::ProcessingUsage> usage,
sprangc5d62e22017-04-02 23:53:04 -0700337 int64_t normal_period_ms,
338 int64_t overuse_period_ms,
339 int64_t underuse_period_ms)
Niels Möller83dbeac2017-12-14 16:39:44 +0100340 : usage_(std::move(usage)),
sprangc5d62e22017-04-02 23:53:04 -0700341 normal_period_ms_(normal_period_ms),
342 overuse_period_ms_(overuse_period_ms),
343 underuse_period_ms_(underuse_period_ms),
344 state_(State::kNormal),
345 last_toggling_ms_(-1) {
346 RTC_DCHECK_GT(overuse_period_ms, 0);
347 RTC_DCHECK_GT(normal_period_ms, 0);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100348 RTC_LOG(LS_INFO) << "Simulating overuse with intervals " << normal_period_ms
349 << "ms normal mode, " << overuse_period_ms
350 << "ms overuse mode.";
sprangc5d62e22017-04-02 23:53:04 -0700351 }
352
353 ~OverdoseInjector() override {}
354
Niels Möller83dbeac2017-12-14 16:39:44 +0100355 void Reset() override { usage_->Reset(); }
356
357 void SetMaxSampleDiffMs(float diff_ms) override {
358 usage_->SetMaxSampleDiffMs(diff_ms);
359 }
360
361 void FrameCaptured(const VideoFrame& frame,
362 int64_t time_when_first_seen_us,
363 int64_t last_capture_time_us) override {
364 usage_->FrameCaptured(frame, time_when_first_seen_us, last_capture_time_us);
365 }
366
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200367 absl::optional<int> FrameSent(
Niels Möller83dbeac2017-12-14 16:39:44 +0100368 // These two argument used by old estimator.
369 uint32_t timestamp,
370 int64_t time_sent_in_us,
371 // And these two by the new estimator.
372 int64_t capture_time_us,
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200373 absl::optional<int> encode_duration_us) override {
Niels Möller83dbeac2017-12-14 16:39:44 +0100374 return usage_->FrameSent(timestamp, time_sent_in_us, capture_time_us,
375 encode_duration_us);
376 }
377
sprangc5d62e22017-04-02 23:53:04 -0700378 int Value() override {
379 int64_t now_ms = rtc::TimeMillis();
380 if (last_toggling_ms_ == -1) {
381 last_toggling_ms_ = now_ms;
382 } else {
383 switch (state_) {
384 case State::kNormal:
385 if (now_ms > last_toggling_ms_ + normal_period_ms_) {
386 state_ = State::kOveruse;
387 last_toggling_ms_ = now_ms;
Mirko Bonadei675513b2017-11-09 11:09:25 +0100388 RTC_LOG(LS_INFO) << "Simulating CPU overuse.";
sprangc5d62e22017-04-02 23:53:04 -0700389 }
390 break;
391 case State::kOveruse:
392 if (now_ms > last_toggling_ms_ + overuse_period_ms_) {
393 state_ = State::kUnderuse;
394 last_toggling_ms_ = now_ms;
Mirko Bonadei675513b2017-11-09 11:09:25 +0100395 RTC_LOG(LS_INFO) << "Simulating CPU underuse.";
sprangc5d62e22017-04-02 23:53:04 -0700396 }
397 break;
398 case State::kUnderuse:
399 if (now_ms > last_toggling_ms_ + underuse_period_ms_) {
400 state_ = State::kNormal;
401 last_toggling_ms_ = now_ms;
Mirko Bonadei675513b2017-11-09 11:09:25 +0100402 RTC_LOG(LS_INFO) << "Actual CPU overuse measurements in effect.";
sprangc5d62e22017-04-02 23:53:04 -0700403 }
404 break;
405 }
406 }
407
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200408 absl::optional<int> overried_usage_value;
sprangc5d62e22017-04-02 23:53:04 -0700409 switch (state_) {
410 case State::kNormal:
411 break;
412 case State::kOveruse:
413 overried_usage_value.emplace(250);
414 break;
415 case State::kUnderuse:
416 overried_usage_value.emplace(5);
417 break;
418 }
Niels Möller7dc26b72017-12-06 10:27:48 +0100419
Niels Möller83dbeac2017-12-14 16:39:44 +0100420 return overried_usage_value.value_or(usage_->Value());
sprangc5d62e22017-04-02 23:53:04 -0700421 }
422
423 private:
Niels Möller83dbeac2017-12-14 16:39:44 +0100424 const std::unique_ptr<OveruseFrameDetector::ProcessingUsage> usage_;
sprangc5d62e22017-04-02 23:53:04 -0700425 const int64_t normal_period_ms_;
426 const int64_t overuse_period_ms_;
427 const int64_t underuse_period_ms_;
428 enum class State { kNormal, kOveruse, kUnderuse } state_;
429 int64_t last_toggling_ms_;
430};
431
Niels Möller904f8692017-12-07 11:22:39 +0100432} // namespace
433
434CpuOveruseOptions::CpuOveruseOptions()
435 : high_encode_usage_threshold_percent(85),
436 frame_timeout_interval_ms(1500),
437 min_frame_samples(120),
438 min_process_count(3),
Niels Möller83dbeac2017-12-14 16:39:44 +0100439 high_threshold_consecutive_count(2),
440 // Disabled by default.
441 filter_time_ms(0) {
Niels Möller904f8692017-12-07 11:22:39 +0100442#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
443 // This is proof-of-concept code for letting the physical core count affect
444 // the interval into which we attempt to scale. For now, the code is Mac OS
445 // specific, since that's the platform were we saw most problems.
446 // TODO(torbjorng): Enhance SystemInfo to return this metric.
447
448 mach_port_t mach_host = mach_host_self();
449 host_basic_info hbi = {};
450 mach_msg_type_number_t info_count = HOST_BASIC_INFO_COUNT;
451 kern_return_t kr =
452 host_info(mach_host, HOST_BASIC_INFO, reinterpret_cast<host_info_t>(&hbi),
453 &info_count);
454 mach_port_deallocate(mach_task_self(), mach_host);
455
456 int n_physical_cores;
457 if (kr != KERN_SUCCESS) {
458 // If we couldn't get # of physical CPUs, don't panic. Assume we have 1.
459 n_physical_cores = 1;
460 RTC_LOG(LS_ERROR)
461 << "Failed to determine number of physical cores, assuming 1";
462 } else {
463 n_physical_cores = hbi.physical_cpu;
464 RTC_LOG(LS_INFO) << "Number of physical cores:" << n_physical_cores;
465 }
466
467 // Change init list default for few core systems. The assumption here is that
468 // encoding, which we measure here, takes about 1/4 of the processing of a
469 // two-way call. This is roughly true for x86 using both vp8 and vp9 without
470 // hardware encoding. Since we don't affect the incoming stream here, we only
471 // control about 1/2 of the total processing needs, but this is not taken into
472 // account.
473 if (n_physical_cores == 1)
474 high_encode_usage_threshold_percent = 20; // Roughly 1/4 of 100%.
475 else if (n_physical_cores == 2)
476 high_encode_usage_threshold_percent = 40; // Roughly 1/4 of 200%.
477#endif // defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
478
479 // Note that we make the interval 2x+epsilon wide, since libyuv scaling steps
480 // are close to that (when squared). This wide interval makes sure that
481 // scaling up or down does not jump all the way across the interval.
482 low_encode_usage_threshold_percent =
483 (high_encode_usage_threshold_percent - 1) / 2;
484}
485
486std::unique_ptr<OveruseFrameDetector::ProcessingUsage>
Yves Gerey665174f2018-06-19 15:03:05 +0200487OveruseFrameDetector::CreateProcessingUsage(const CpuOveruseOptions& options) {
Niels Möller904f8692017-12-07 11:22:39 +0100488 std::unique_ptr<ProcessingUsage> instance;
Niels Möller83dbeac2017-12-14 16:39:44 +0100489 if (options.filter_time_ms > 0) {
Mirko Bonadei317a1f02019-09-17 17:06:18 +0200490 instance = std::make_unique<SendProcessingUsage2>(options);
Niels Möller83dbeac2017-12-14 16:39:44 +0100491 } else {
Mirko Bonadei317a1f02019-09-17 17:06:18 +0200492 instance = std::make_unique<SendProcessingUsage1>(options);
Niels Möller83dbeac2017-12-14 16:39:44 +0100493 }
sprangc5d62e22017-04-02 23:53:04 -0700494 std::string toggling_interval =
495 field_trial::FindFullName("WebRTC-ForceSimulatedOveruseIntervalMs");
496 if (!toggling_interval.empty()) {
497 int normal_period_ms = 0;
498 int overuse_period_ms = 0;
499 int underuse_period_ms = 0;
500 if (sscanf(toggling_interval.c_str(), "%d-%d-%d", &normal_period_ms,
501 &overuse_period_ms, &underuse_period_ms) == 3) {
502 if (normal_period_ms > 0 && overuse_period_ms > 0 &&
503 underuse_period_ms > 0) {
Mirko Bonadei317a1f02019-09-17 17:06:18 +0200504 instance = std::make_unique<OverdoseInjector>(
Yves Gerey665174f2018-06-19 15:03:05 +0200505 std::move(instance), normal_period_ms, overuse_period_ms,
506 underuse_period_ms);
sprangc5d62e22017-04-02 23:53:04 -0700507 } else {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100508 RTC_LOG(LS_WARNING)
sprangc5d62e22017-04-02 23:53:04 -0700509 << "Invalid (non-positive) normal/overuse/underuse periods: "
510 << normal_period_ms << " / " << overuse_period_ms << " / "
511 << underuse_period_ms;
512 }
513 } else {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100514 RTC_LOG(LS_WARNING) << "Malformed toggling interval: "
515 << toggling_interval;
sprangc5d62e22017-04-02 23:53:04 -0700516 }
517 }
sprangc5d62e22017-04-02 23:53:04 -0700518 return instance;
519}
520
pbos@webrtc.org3e6e2712015-02-26 12:19:31 +0000521OveruseFrameDetector::OveruseFrameDetector(
pbos@webrtc.org3e6e2712015-02-26 12:19:31 +0000522 CpuOveruseMetricsObserver* metrics_observer)
Sebastian Janssonecb68972019-01-18 10:30:54 +0100523 : metrics_observer_(metrics_observer),
asapersson@webrtc.orgb60346e2014-02-17 19:02:15 +0000524 num_process_times_(0),
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200525 // TODO(nisse): Use absl::optional
nissee0e3bdf2017-01-18 02:16:20 -0800526 last_capture_time_us_(-1),
asapersson74d85e12015-09-24 00:53:32 -0700527 num_pixels_(0),
Niels Möller7dc26b72017-12-06 10:27:48 +0100528 max_framerate_(kDefaultFrameRate),
Peter Boströme4499152016-02-05 11:13:28 +0100529 last_overuse_time_ms_(-1),
pbos@webrtc.orga9575702013-08-30 17:16:32 +0000530 checks_above_threshold_(0),
asapersson@webrtc.orgd9803072014-06-16 14:27:19 +0000531 num_overuse_detections_(0),
Peter Boströme4499152016-02-05 11:13:28 +0100532 last_rampup_time_ms_(-1),
pbos@webrtc.orga9575702013-08-30 17:16:32 +0000533 in_quick_rampup_(false),
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200534 current_rampup_delay_ms_(kStandardRampUpDelayMs) {
perkjd52063f2016-09-07 06:32:18 -0700535 task_checker_.Detach();
Niels Möllereea92882019-04-25 08:44:04 +0200536 ParseFieldTrial({&filter_time_constant_},
537 field_trial::FindFullName("WebRTC-CpuLoadEstimator"));
asapersson@webrtc.org9e5b0342013-12-04 13:47:44 +0000538}
mflodman@webrtc.orge6168f52013-06-26 11:23:01 +0000539
Sebastian Janssonecb68972019-01-18 10:30:54 +0100540OveruseFrameDetector::~OveruseFrameDetector() {}
perkjd52063f2016-09-07 06:32:18 -0700541
Niels Möller73f29cb2018-01-31 16:09:31 +0100542void OveruseFrameDetector::StartCheckForOveruse(
Sebastian Janssoncda86dd2019-03-11 17:26:36 +0100543 rtc::TaskQueue* task_queue,
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200544 const CpuOveruseOptions& options,
Niels Möller73f29cb2018-01-31 16:09:31 +0100545 AdaptationObserverInterface* overuse_observer) {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200546 RTC_DCHECK_RUN_ON(&task_checker_);
Sebastian Janssonecb68972019-01-18 10:30:54 +0100547 RTC_DCHECK(!check_overuse_task_.Running());
Niels Möller73f29cb2018-01-31 16:09:31 +0100548 RTC_DCHECK(overuse_observer != nullptr);
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200549
550 SetOptions(options);
Sebastian Janssonecb68972019-01-18 10:30:54 +0100551 check_overuse_task_ = RepeatingTaskHandle::DelayedStart(
Sebastian Janssoncda86dd2019-03-11 17:26:36 +0100552 task_queue->Get(), TimeDelta::ms(kTimeToFirstCheckForOveruseMs),
553 [this, overuse_observer] {
Sebastian Janssonecb68972019-01-18 10:30:54 +0100554 CheckForOveruse(overuse_observer);
555 return TimeDelta::ms(kCheckForOveruseIntervalMs);
556 });
perkjd52063f2016-09-07 06:32:18 -0700557}
558void OveruseFrameDetector::StopCheckForOveruse() {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200559 RTC_DCHECK_RUN_ON(&task_checker_);
Sebastian Janssonecb68972019-01-18 10:30:54 +0100560 check_overuse_task_.Stop();
mflodman@webrtc.orge6168f52013-06-26 11:23:01 +0000561}
562
Peter Boströme4499152016-02-05 11:13:28 +0100563void OveruseFrameDetector::EncodedFrameTimeMeasured(int encode_duration_ms) {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200564 RTC_DCHECK_RUN_ON(&task_checker_);
Niels Möller213618e2018-07-24 09:29:58 +0200565 encode_usage_percent_ = usage_->Value();
asapersson@webrtc.org9aed0022014-10-16 06:57:12 +0000566
Niels Möller213618e2018-07-24 09:29:58 +0200567 metrics_observer_->OnEncodedFrameTimeMeasured(encode_duration_ms,
568 *encode_usage_percent_);
asapersson@webrtc.orgab6bf4f2014-05-27 07:43:15 +0000569}
570
asapersson@webrtc.org8a8c3ef2014-03-20 13:15:01 +0000571bool OveruseFrameDetector::FrameSizeChanged(int num_pixels) const {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200572 RTC_DCHECK_RUN_ON(&task_checker_);
asapersson@webrtc.org8a8c3ef2014-03-20 13:15:01 +0000573 if (num_pixels != num_pixels_) {
574 return true;
575 }
576 return false;
577}
578
nissee0e3bdf2017-01-18 02:16:20 -0800579bool OveruseFrameDetector::FrameTimeoutDetected(int64_t now_us) const {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200580 RTC_DCHECK_RUN_ON(&task_checker_);
nissee0e3bdf2017-01-18 02:16:20 -0800581 if (last_capture_time_us_ == -1)
asapersson@webrtc.orgb60346e2014-02-17 19:02:15 +0000582 return false;
nissee0e3bdf2017-01-18 02:16:20 -0800583 return (now_us - last_capture_time_us_) >
Yves Gerey665174f2018-06-19 15:03:05 +0200584 options_.frame_timeout_interval_ms * rtc::kNumMicrosecsPerMillisec;
asapersson@webrtc.org8a8c3ef2014-03-20 13:15:01 +0000585}
586
Niels Möller7dc26b72017-12-06 10:27:48 +0100587void OveruseFrameDetector::ResetAll(int num_pixels) {
588 // Reset state, as a result resolution being changed. Do not however change
589 // the current frame rate back to the default.
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200590 RTC_DCHECK_RUN_ON(&task_checker_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100591 num_pixels_ = num_pixels;
asapersson@webrtc.org9aed0022014-10-16 06:57:12 +0000592 usage_->Reset();
nissee0e3bdf2017-01-18 02:16:20 -0800593 last_capture_time_us_ = -1;
asapersson@webrtc.org8a8c3ef2014-03-20 13:15:01 +0000594 num_process_times_ = 0;
Niels Möller213618e2018-07-24 09:29:58 +0200595 encode_usage_percent_ = absl::nullopt;
Niels Möller7dc26b72017-12-06 10:27:48 +0100596 OnTargetFramerateUpdated(max_framerate_);
sprangfda496a2017-06-15 04:21:07 -0700597}
598
Niels Möller7dc26b72017-12-06 10:27:48 +0100599void OveruseFrameDetector::OnTargetFramerateUpdated(int framerate_fps) {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200600 RTC_DCHECK_RUN_ON(&task_checker_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100601 RTC_DCHECK_GE(framerate_fps, 0);
602 max_framerate_ = std::min(kMaxFramerate, framerate_fps);
603 usage_->SetMaxSampleDiffMs((1000 / std::max(kMinFramerate, max_framerate_)) *
604 kMaxSampleDiffMarginFactor);
asapersson@webrtc.org9aed0022014-10-16 06:57:12 +0000605}
606
Niels Möller7dc26b72017-12-06 10:27:48 +0100607void OveruseFrameDetector::FrameCaptured(const VideoFrame& frame,
608 int64_t time_when_first_seen_us) {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200609 RTC_DCHECK_RUN_ON(&task_checker_);
Niels Möllereee7ced2017-12-01 11:25:01 +0100610
Niels Möller7dc26b72017-12-06 10:27:48 +0100611 if (FrameSizeChanged(frame.width() * frame.height()) ||
612 FrameTimeoutDetected(time_when_first_seen_us)) {
613 ResetAll(frame.width() * frame.height());
614 }
615
Niels Möllere08cf3a2017-12-07 15:23:58 +0100616 usage_->FrameCaptured(frame, time_when_first_seen_us, last_capture_time_us_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100617 last_capture_time_us_ = time_when_first_seen_us;
Niels Möller7dc26b72017-12-06 10:27:48 +0100618}
619
620void OveruseFrameDetector::FrameSent(uint32_t timestamp,
Niels Möller83dbeac2017-12-14 16:39:44 +0100621 int64_t time_sent_in_us,
622 int64_t capture_time_us,
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200623 absl::optional<int> encode_duration_us) {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200624 RTC_DCHECK_RUN_ON(&task_checker_);
Niels Möller83dbeac2017-12-14 16:39:44 +0100625 encode_duration_us = usage_->FrameSent(timestamp, time_sent_in_us,
626 capture_time_us, encode_duration_us);
Niels Möllere08cf3a2017-12-07 15:23:58 +0100627
628 if (encode_duration_us) {
629 EncodedFrameTimeMeasured(*encode_duration_us /
630 rtc::kNumMicrosecsPerMillisec);
Peter Boströme4499152016-02-05 11:13:28 +0100631 }
asapersson@webrtc.orgc7ff8f92013-11-26 11:12:33 +0000632}
633
Niels Möller73f29cb2018-01-31 16:09:31 +0100634void OveruseFrameDetector::CheckForOveruse(
635 AdaptationObserverInterface* observer) {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200636 RTC_DCHECK_RUN_ON(&task_checker_);
Niels Möller73f29cb2018-01-31 16:09:31 +0100637 RTC_DCHECK(observer);
perkjd52063f2016-09-07 06:32:18 -0700638 ++num_process_times_;
Niels Möller213618e2018-07-24 09:29:58 +0200639 if (num_process_times_ <= options_.min_process_count ||
640 !encode_usage_percent_)
perkjd52063f2016-09-07 06:32:18 -0700641 return;
pbos@webrtc.orga9575702013-08-30 17:16:32 +0000642
nissee0e3bdf2017-01-18 02:16:20 -0800643 int64_t now_ms = rtc::TimeMillis();
pbos@webrtc.orga9575702013-08-30 17:16:32 +0000644
Niels Möller213618e2018-07-24 09:29:58 +0200645 if (IsOverusing(*encode_usage_percent_)) {
pbos@webrtc.orga9575702013-08-30 17:16:32 +0000646 // If the last thing we did was going up, and now have to back down, we need
647 // to check if this peak was short. If so we should back off to avoid going
648 // back and forth between this load, the system doesn't seem to handle it.
Peter Boströme4499152016-02-05 11:13:28 +0100649 bool check_for_backoff = last_rampup_time_ms_ > last_overuse_time_ms_;
pbos@webrtc.orga9575702013-08-30 17:16:32 +0000650 if (check_for_backoff) {
nissee0e3bdf2017-01-18 02:16:20 -0800651 if (now_ms - last_rampup_time_ms_ < kStandardRampUpDelayMs ||
asapersson@webrtc.orgd9803072014-06-16 14:27:19 +0000652 num_overuse_detections_ > kMaxOverusesBeforeApplyRampupDelay) {
pbos@webrtc.orga9575702013-08-30 17:16:32 +0000653 // Going up was not ok for very long, back off.
654 current_rampup_delay_ms_ *= kRampUpBackoffFactor;
655 if (current_rampup_delay_ms_ > kMaxRampUpDelayMs)
656 current_rampup_delay_ms_ = kMaxRampUpDelayMs;
657 } else {
658 // Not currently backing off, reset rampup delay.
659 current_rampup_delay_ms_ = kStandardRampUpDelayMs;
660 }
661 }
662
nissee0e3bdf2017-01-18 02:16:20 -0800663 last_overuse_time_ms_ = now_ms;
pbos@webrtc.orga9575702013-08-30 17:16:32 +0000664 in_quick_rampup_ = false;
665 checks_above_threshold_ = 0;
asapersson@webrtc.orgd9803072014-06-16 14:27:19 +0000666 ++num_overuse_detections_;
pbos@webrtc.orga9575702013-08-30 17:16:32 +0000667
Niels Möller73f29cb2018-01-31 16:09:31 +0100668 observer->AdaptDown(kScaleReasonCpu);
Niels Möller213618e2018-07-24 09:29:58 +0200669 } else if (IsUnderusing(*encode_usage_percent_, now_ms)) {
nissee0e3bdf2017-01-18 02:16:20 -0800670 last_rampup_time_ms_ = now_ms;
pbos@webrtc.orga9575702013-08-30 17:16:32 +0000671 in_quick_rampup_ = true;
672
Niels Möller73f29cb2018-01-31 16:09:31 +0100673 observer->AdaptUp(kScaleReasonCpu);
mflodman@webrtc.orge6168f52013-06-26 11:23:01 +0000674 }
pbos@webrtc.orga9575702013-08-30 17:16:32 +0000675
mflodman@webrtc.org5574dac2014-04-07 10:56:31 +0000676 int rampup_delay =
677 in_quick_rampup_ ? kQuickRampUpDelayMs : current_rampup_delay_ms_;
asapersson74d85e12015-09-24 00:53:32 -0700678
Mirko Bonadei675513b2017-11-09 11:09:25 +0100679 RTC_LOG(LS_VERBOSE) << " Frame stats: "
Niels Möller213618e2018-07-24 09:29:58 +0200680 << " encode usage " << *encode_usage_percent_
Mirko Bonadei675513b2017-11-09 11:09:25 +0100681 << " overuse detections " << num_overuse_detections_
682 << " rampup delay " << rampup_delay;
asapersson@webrtc.orge2af6222013-09-23 20:05:39 +0000683}
684
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200685void OveruseFrameDetector::SetOptions(const CpuOveruseOptions& options) {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200686 RTC_DCHECK_RUN_ON(&task_checker_);
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200687 options_ = options;
Niels Möllereea92882019-04-25 08:44:04 +0200688
689 // Time constant config overridable by field trial.
690 if (filter_time_constant_) {
691 options_.filter_time_ms = filter_time_constant_->ms();
692 }
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200693 // Force reset with next frame.
694 num_pixels_ = 0;
695 usage_ = CreateProcessingUsage(options);
696}
697
Niels Möller213618e2018-07-24 09:29:58 +0200698bool OveruseFrameDetector::IsOverusing(int usage_percent) {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200699 RTC_DCHECK_RUN_ON(&task_checker_);
sprangc5d62e22017-04-02 23:53:04 -0700700
Niels Möller213618e2018-07-24 09:29:58 +0200701 if (usage_percent >= options_.high_encode_usage_threshold_percent) {
pbos@webrtc.orga9575702013-08-30 17:16:32 +0000702 ++checks_above_threshold_;
703 } else {
704 checks_above_threshold_ = 0;
705 }
asapersson@webrtc.org8a8c3ef2014-03-20 13:15:01 +0000706 return checks_above_threshold_ >= options_.high_threshold_consecutive_count;
mflodman@webrtc.orgd4412fe2013-07-31 16:42:21 +0000707}
708
Niels Möller213618e2018-07-24 09:29:58 +0200709bool OveruseFrameDetector::IsUnderusing(int usage_percent, int64_t time_now) {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200710 RTC_DCHECK_RUN_ON(&task_checker_);
pbos@webrtc.orga9575702013-08-30 17:16:32 +0000711 int delay = in_quick_rampup_ ? kQuickRampUpDelayMs : current_rampup_delay_ms_;
Peter Boströme4499152016-02-05 11:13:28 +0100712 if (time_now < last_rampup_time_ms_ + delay)
mflodman@webrtc.orgd4412fe2013-07-31 16:42:21 +0000713 return false;
mflodman@webrtc.orgd4412fe2013-07-31 16:42:21 +0000714
Niels Möller213618e2018-07-24 09:29:58 +0200715 return usage_percent < options_.low_encode_usage_threshold_percent;
mflodman@webrtc.orgd4412fe2013-07-31 16:42:21 +0000716}
mflodman@webrtc.orge6168f52013-06-26 11:23:01 +0000717} // namespace webrtc