Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2019 The WebRTC Project Authors. All rights reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
| 11 | #include "video/overuse_frame_detector_resource_adaptation_module.h" |
| 12 | |
| 13 | #include <algorithm> |
| 14 | #include <limits> |
| 15 | #include <memory> |
| 16 | #include <string> |
| 17 | #include <utility> |
| 18 | |
| 19 | #include "absl/algorithm/container.h" |
Danil Chapovalov | 64f1f3f | 2020-01-16 14:41:10 +0100 | [diff] [blame] | 20 | #include "absl/base/macros.h" |
Henrik Boström | 07b17df | 2020-01-15 11:42:12 +0100 | [diff] [blame] | 21 | #include "api/task_queue/task_queue_base.h" |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 22 | #include "api/video/video_source_interface.h" |
Henrik Boström | 48258ac | 2020-02-06 12:49:57 +0100 | [diff] [blame] | 23 | #include "call/adaptation/resource.h" |
Henrik Boström | ce0ea49 | 2020-01-13 11:27:18 +0100 | [diff] [blame] | 24 | #include "call/adaptation/video_source_restrictions.h" |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 25 | #include "rtc_base/logging.h" |
Henrik Boström | d238200 | 2020-01-10 15:44:01 +0100 | [diff] [blame] | 26 | #include "rtc_base/numerics/safe_conversions.h" |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 27 | #include "rtc_base/strings/string_builder.h" |
Henrik Boström | ad515a2 | 2020-01-27 13:38:05 +0100 | [diff] [blame] | 28 | #include "rtc_base/time_utils.h" |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 29 | #include "video/video_stream_encoder.h" |
| 30 | |
| 31 | namespace webrtc { |
| 32 | |
| 33 | namespace { |
| 34 | |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 35 | bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) { |
| 36 | return degradation_preference == DegradationPreference::MAINTAIN_FRAMERATE || |
| 37 | degradation_preference == DegradationPreference::BALANCED; |
| 38 | } |
| 39 | |
| 40 | bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) { |
| 41 | return degradation_preference == DegradationPreference::MAINTAIN_RESOLUTION || |
| 42 | degradation_preference == DegradationPreference::BALANCED; |
| 43 | } |
| 44 | |
Henrik Boström | 8234b92 | 2020-01-13 17:26:50 +0100 | [diff] [blame] | 45 | // Returns modified restrictions where any constraints that don't apply to the |
| 46 | // degradation preference are cleared. |
| 47 | VideoSourceRestrictions ApplyDegradationPreference( |
| 48 | VideoSourceRestrictions source_restrictions, |
| 49 | DegradationPreference degradation_preference) { |
| 50 | switch (degradation_preference) { |
| 51 | case DegradationPreference::BALANCED: |
| 52 | break; |
| 53 | case DegradationPreference::MAINTAIN_FRAMERATE: |
| 54 | source_restrictions.set_max_frame_rate(absl::nullopt); |
| 55 | break; |
| 56 | case DegradationPreference::MAINTAIN_RESOLUTION: |
| 57 | source_restrictions.set_max_pixels_per_frame(absl::nullopt); |
| 58 | source_restrictions.set_target_pixels_per_frame(absl::nullopt); |
| 59 | break; |
| 60 | case DegradationPreference::DISABLED: |
| 61 | source_restrictions.set_max_pixels_per_frame(absl::nullopt); |
| 62 | source_restrictions.set_target_pixels_per_frame(absl::nullopt); |
| 63 | source_restrictions.set_max_frame_rate(absl::nullopt); |
| 64 | } |
| 65 | return source_restrictions; |
| 66 | } |
| 67 | |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 68 | // Returns AdaptationCounters where constraints that don't apply to the |
Henrik Boström | efbec9a | 2020-03-06 10:41:25 +0100 | [diff] [blame] | 69 | // degredation preference are cleared. This behaviour must reflect that of |
| 70 | // ApplyDegredationPreference for SourceRestrictions. Any to that method must |
| 71 | // also change this one. |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 72 | AdaptationCounters ApplyDegradationPreference( |
| 73 | AdaptationCounters counters, |
| 74 | DegradationPreference degradation_preference) { |
| 75 | switch (degradation_preference) { |
| 76 | case DegradationPreference::BALANCED: |
| 77 | break; |
| 78 | case DegradationPreference::MAINTAIN_FRAMERATE: |
| 79 | counters.fps_adaptations = 0; |
| 80 | break; |
| 81 | case DegradationPreference::MAINTAIN_RESOLUTION: |
| 82 | counters.resolution_adaptations = 0; |
| 83 | break; |
| 84 | case DegradationPreference::DISABLED: |
| 85 | counters.resolution_adaptations = 0; |
| 86 | counters.fps_adaptations = 0; |
| 87 | break; |
| 88 | default: |
| 89 | RTC_NOTREACHED(); |
| 90 | } |
| 91 | return counters; |
| 92 | } |
| 93 | |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 94 | } // namespace |
| 95 | |
Henrik Boström | 8cfecac | 2020-02-07 11:29:14 +0100 | [diff] [blame] | 96 | class OveruseFrameDetectorResourceAdaptationModule::InitialFrameDropper { |
| 97 | public: |
| 98 | explicit InitialFrameDropper(QualityScalerResource* quality_scaler_resource) |
| 99 | : quality_scaler_resource_(quality_scaler_resource), |
| 100 | quality_scaler_settings_(QualityScalerSettings::ParseFromFieldTrials()), |
| 101 | has_seen_first_bwe_drop_(false), |
| 102 | set_start_bitrate_(DataRate::Zero()), |
| 103 | set_start_bitrate_time_ms_(0), |
| 104 | initial_framedrop_(0) { |
| 105 | RTC_DCHECK(quality_scaler_resource_); |
| 106 | } |
| 107 | |
| 108 | // Output signal. |
| 109 | bool DropInitialFrames() const { |
| 110 | return initial_framedrop_ < kMaxInitialFramedrop; |
| 111 | } |
| 112 | |
| 113 | // Input signals. |
| 114 | void SetStartBitrate(DataRate start_bitrate, int64_t now_ms) { |
| 115 | set_start_bitrate_ = start_bitrate; |
| 116 | set_start_bitrate_time_ms_ = now_ms; |
| 117 | } |
| 118 | |
| 119 | void SetTargetBitrate(DataRate target_bitrate, int64_t now_ms) { |
| 120 | if (set_start_bitrate_ > DataRate::Zero() && !has_seen_first_bwe_drop_ && |
| 121 | quality_scaler_resource_->is_started() && |
| 122 | quality_scaler_settings_.InitialBitrateIntervalMs() && |
| 123 | quality_scaler_settings_.InitialBitrateFactor()) { |
| 124 | int64_t diff_ms = now_ms - set_start_bitrate_time_ms_; |
| 125 | if (diff_ms < |
| 126 | quality_scaler_settings_.InitialBitrateIntervalMs().value() && |
| 127 | (target_bitrate < |
| 128 | (set_start_bitrate_ * |
| 129 | quality_scaler_settings_.InitialBitrateFactor().value()))) { |
| 130 | RTC_LOG(LS_INFO) << "Reset initial_framedrop_. Start bitrate: " |
| 131 | << set_start_bitrate_.bps() |
| 132 | << ", target bitrate: " << target_bitrate.bps(); |
| 133 | initial_framedrop_ = 0; |
| 134 | has_seen_first_bwe_drop_ = true; |
| 135 | } |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | void OnFrameDroppedDueToSize() { ++initial_framedrop_; } |
| 140 | |
Mirko Bonadei | 2e161c4 | 2020-02-20 08:45:01 +0000 | [diff] [blame] | 141 | void OnMaybeEncodeFrame() { initial_framedrop_ = kMaxInitialFramedrop; } |
Henrik Boström | 8cfecac | 2020-02-07 11:29:14 +0100 | [diff] [blame] | 142 | |
| 143 | void OnQualityScalerSettingsUpdated() { |
| 144 | if (quality_scaler_resource_->is_started()) { |
| 145 | // Restart frame drops due to size. |
| 146 | initial_framedrop_ = 0; |
| 147 | } else { |
| 148 | // Quality scaling disabled so we shouldn't drop initial frames. |
| 149 | initial_framedrop_ = kMaxInitialFramedrop; |
| 150 | } |
| 151 | } |
| 152 | |
| 153 | private: |
| 154 | // The maximum number of frames to drop at beginning of stream to try and |
| 155 | // achieve desired bitrate. |
| 156 | static const int kMaxInitialFramedrop = 4; |
| 157 | |
| 158 | const QualityScalerResource* quality_scaler_resource_; |
| 159 | const QualityScalerSettings quality_scaler_settings_; |
| 160 | bool has_seen_first_bwe_drop_; |
| 161 | DataRate set_start_bitrate_; |
| 162 | int64_t set_start_bitrate_time_ms_; |
| 163 | // Counts how many frames we've dropped in the initial framedrop phase. |
| 164 | int initial_framedrop_; |
| 165 | }; |
| 166 | |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 167 | OveruseFrameDetectorResourceAdaptationModule:: |
| 168 | OveruseFrameDetectorResourceAdaptationModule( |
Evan Shrubsole | 7c3a1fc | 2020-02-04 16:26:38 +0100 | [diff] [blame] | 169 | Clock* clock, |
Henrik Boström | ad515a2 | 2020-01-27 13:38:05 +0100 | [diff] [blame] | 170 | bool experiment_cpu_load_estimator, |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 171 | std::unique_ptr<OveruseFrameDetector> overuse_detector, |
Henrik Boström | d238200 | 2020-01-10 15:44:01 +0100 | [diff] [blame] | 172 | VideoStreamEncoderObserver* encoder_stats_observer, |
| 173 | ResourceAdaptationModuleListener* adaptation_listener) |
Henrik Boström | 07b17df | 2020-01-15 11:42:12 +0100 | [diff] [blame] | 174 | : adaptation_listener_(adaptation_listener), |
Evan Shrubsole | 7c3a1fc | 2020-02-04 16:26:38 +0100 | [diff] [blame] | 175 | clock_(clock), |
Evan Shrubsole | aa6fbc1 | 2020-02-25 16:26:01 +0100 | [diff] [blame] | 176 | state_(State::kStopped), |
Henrik Boström | ad515a2 | 2020-01-27 13:38:05 +0100 | [diff] [blame] | 177 | experiment_cpu_load_estimator_(experiment_cpu_load_estimator), |
Henrik Boström | a3d4252 | 2020-01-16 13:55:29 +0100 | [diff] [blame] | 178 | has_input_video_(false), |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 179 | degradation_preference_(DegradationPreference::DISABLED), |
Henrik Boström | efbec9a | 2020-03-06 10:41:25 +0100 | [diff] [blame] | 180 | stream_adapter_(std::make_unique<VideoStreamAdapter>()), |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 181 | encode_usage_resource_( |
Henrik Boström | 48258ac | 2020-02-06 12:49:57 +0100 | [diff] [blame] | 182 | std::make_unique<EncodeUsageResource>(std::move(overuse_detector))), |
| 183 | quality_scaler_resource_(std::make_unique<QualityScalerResource>()), |
Henrik Boström | 8cfecac | 2020-02-07 11:29:14 +0100 | [diff] [blame] | 184 | initial_frame_dropper_(std::make_unique<InitialFrameDropper>( |
| 185 | quality_scaler_resource_.get())), |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 186 | quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), |
Henrik Boström | d4578ae | 2020-01-22 16:16:04 +0100 | [diff] [blame] | 187 | last_input_frame_size_(absl::nullopt), |
Henrik Boström | fae6f0e | 2020-01-20 11:16:50 +0100 | [diff] [blame] | 188 | target_frame_rate_(absl::nullopt), |
Evan Shrubsole | 7c3a1fc | 2020-02-04 16:26:38 +0100 | [diff] [blame] | 189 | encoder_target_bitrate_bps_(absl::nullopt), |
Evan Shrubsole | e331a12 | 2020-02-05 13:30:33 +0100 | [diff] [blame] | 190 | quality_rampup_done_(false), |
| 191 | quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()), |
Henrik Boström | 4bab2fc | 2020-01-21 11:18:06 +0100 | [diff] [blame] | 192 | encoder_settings_(absl::nullopt), |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 193 | encoder_stats_observer_(encoder_stats_observer), |
| 194 | active_counts_() { |
Henrik Boström | d238200 | 2020-01-10 15:44:01 +0100 | [diff] [blame] | 195 | RTC_DCHECK(adaptation_listener_); |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 196 | RTC_DCHECK(encoder_stats_observer_); |
Evan Shrubsole | aa6fbc1 | 2020-02-25 16:26:01 +0100 | [diff] [blame] | 197 | AddResource(encode_usage_resource_.get(), |
| 198 | AdaptationObserverInterface::AdaptReason::kCpu); |
| 199 | AddResource(quality_scaler_resource_.get(), |
| 200 | AdaptationObserverInterface::AdaptReason::kQuality); |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 201 | } |
| 202 | |
| 203 | OveruseFrameDetectorResourceAdaptationModule:: |
Evan Shrubsole | aa6fbc1 | 2020-02-25 16:26:01 +0100 | [diff] [blame] | 204 | ~OveruseFrameDetectorResourceAdaptationModule() { |
| 205 | RTC_DCHECK_EQ(state_, State::kStopped); |
| 206 | } |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 207 | |
Henrik Boström | a3d4252 | 2020-01-16 13:55:29 +0100 | [diff] [blame] | 208 | void OveruseFrameDetectorResourceAdaptationModule::StartResourceAdaptation( |
Henrik Boström | d238200 | 2020-01-10 15:44:01 +0100 | [diff] [blame] | 209 | ResourceAdaptationModuleListener* adaptation_listener) { |
Evan Shrubsole | aa6fbc1 | 2020-02-25 16:26:01 +0100 | [diff] [blame] | 210 | RTC_DCHECK_EQ(state_, State::kStopped); |
Henrik Boström | 4bab2fc | 2020-01-21 11:18:06 +0100 | [diff] [blame] | 211 | RTC_DCHECK(encoder_settings_.has_value()); |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 212 | // TODO(https://crbug.com/webrtc/11222): Rethink when the adaptation listener |
| 213 | // should be passed in and why. If resources are separated from modules then |
| 214 | // those resources may be started or stopped separately from the module. |
Henrik Boström | d238200 | 2020-01-10 15:44:01 +0100 | [diff] [blame] | 215 | RTC_DCHECK_EQ(adaptation_listener, adaptation_listener_); |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 216 | encode_usage_resource_->StartCheckForOveruse(GetCpuOveruseOptions()); |
Evan Shrubsole | aa6fbc1 | 2020-02-25 16:26:01 +0100 | [diff] [blame] | 217 | for (auto& resource_and_reason : resources_) { |
| 218 | resource_and_reason.resource->RegisterListener(this); |
| 219 | } |
| 220 | state_ = State::kStarted; |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 221 | } |
| 222 | |
Henrik Boström | a3d4252 | 2020-01-16 13:55:29 +0100 | [diff] [blame] | 223 | void OveruseFrameDetectorResourceAdaptationModule::StopResourceAdaptation() { |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 224 | encode_usage_resource_->StopCheckForOveruse(); |
| 225 | quality_scaler_resource_->StopCheckForOveruse(); |
Evan Shrubsole | aa6fbc1 | 2020-02-25 16:26:01 +0100 | [diff] [blame] | 226 | for (auto& resource_and_reason : resources_) { |
| 227 | resource_and_reason.resource->UnregisterListener(this); |
| 228 | } |
| 229 | state_ = State::kStopped; |
| 230 | } |
| 231 | |
| 232 | void OveruseFrameDetectorResourceAdaptationModule::AddResource( |
| 233 | Resource* resource) { |
| 234 | return AddResource(resource, AdaptationObserverInterface::AdaptReason::kCpu); |
| 235 | } |
| 236 | |
| 237 | void OveruseFrameDetectorResourceAdaptationModule::AddResource( |
| 238 | Resource* resource, |
| 239 | AdaptationObserverInterface::AdaptReason reason) { |
| 240 | RTC_DCHECK(resource); |
| 241 | RTC_DCHECK(absl::c_find_if(resources_, |
| 242 | [resource](const ResourceAndReason& r) { |
| 243 | return r.resource == resource; |
| 244 | }) == resources_.end()) |
| 245 | << "Resource " << resource->name() << " already was inserted"; |
| 246 | resources_.emplace_back(resource, reason); |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 247 | } |
| 248 | |
Henrik Boström | a3d4252 | 2020-01-16 13:55:29 +0100 | [diff] [blame] | 249 | void OveruseFrameDetectorResourceAdaptationModule::SetHasInputVideo( |
| 250 | bool has_input_video) { |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 251 | // While false, OnResourceUnderuse() and OnResourceOveruse() are NO-OPS. |
Henrik Boström | a3d4252 | 2020-01-16 13:55:29 +0100 | [diff] [blame] | 252 | has_input_video_ = has_input_video; |
| 253 | } |
| 254 | |
| 255 | void OveruseFrameDetectorResourceAdaptationModule::SetDegradationPreference( |
| 256 | DegradationPreference degradation_preference) { |
Henrik Boström | a3d4252 | 2020-01-16 13:55:29 +0100 | [diff] [blame] | 257 | degradation_preference_ = degradation_preference; |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 258 | if (stream_adapter_->SetDegradationPreference(degradation_preference) == |
| 259 | VideoStreamAdapter::SetDegradationPreferenceResult:: |
| 260 | kRestrictionsCleared) { |
| 261 | active_counts_.fill(AdaptationCounters()); |
| 262 | } |
Henrik Boström | a3d4252 | 2020-01-16 13:55:29 +0100 | [diff] [blame] | 263 | MaybeUpdateVideoSourceRestrictions(); |
| 264 | } |
| 265 | |
Henrik Boström | 4bab2fc | 2020-01-21 11:18:06 +0100 | [diff] [blame] | 266 | void OveruseFrameDetectorResourceAdaptationModule::SetEncoderSettings( |
| 267 | EncoderSettings encoder_settings) { |
| 268 | encoder_settings_ = std::move(encoder_settings); |
Evan Shrubsole | e331a12 | 2020-02-05 13:30:33 +0100 | [diff] [blame] | 269 | |
| 270 | quality_rampup_experiment_.SetMaxBitrate( |
| 271 | LastInputFrameSizeOrDefault(), |
| 272 | encoder_settings_->video_codec().maxBitrate); |
Henrik Boström | 4bab2fc | 2020-01-21 11:18:06 +0100 | [diff] [blame] | 273 | MaybeUpdateTargetFrameRate(); |
| 274 | } |
| 275 | |
Evan Shrubsole | 7c3a1fc | 2020-02-04 16:26:38 +0100 | [diff] [blame] | 276 | void OveruseFrameDetectorResourceAdaptationModule::SetStartBitrate( |
| 277 | DataRate start_bitrate) { |
| 278 | if (!start_bitrate.IsZero()) |
| 279 | encoder_target_bitrate_bps_ = start_bitrate.bps(); |
Henrik Boström | 8cfecac | 2020-02-07 11:29:14 +0100 | [diff] [blame] | 280 | initial_frame_dropper_->SetStartBitrate(start_bitrate, |
| 281 | clock_->TimeInMicroseconds()); |
Evan Shrubsole | 7c3a1fc | 2020-02-04 16:26:38 +0100 | [diff] [blame] | 282 | } |
| 283 | |
| 284 | void OveruseFrameDetectorResourceAdaptationModule::SetTargetBitrate( |
| 285 | DataRate target_bitrate) { |
| 286 | if (!target_bitrate.IsZero()) |
| 287 | encoder_target_bitrate_bps_ = target_bitrate.bps(); |
Henrik Boström | 8cfecac | 2020-02-07 11:29:14 +0100 | [diff] [blame] | 288 | initial_frame_dropper_->SetTargetBitrate(target_bitrate, |
| 289 | clock_->TimeInMilliseconds()); |
Henrik Boström | ede69c0 | 2020-01-21 17:45:35 +0100 | [diff] [blame] | 290 | } |
| 291 | |
Evan Shrubsole | e331a12 | 2020-02-05 13:30:33 +0100 | [diff] [blame] | 292 | void OveruseFrameDetectorResourceAdaptationModule::SetEncoderRates( |
| 293 | const VideoEncoder::RateControlParameters& encoder_rates) { |
| 294 | encoder_rates_ = encoder_rates; |
| 295 | } |
| 296 | |
Henrik Boström | fae6f0e | 2020-01-20 11:16:50 +0100 | [diff] [blame] | 297 | void OveruseFrameDetectorResourceAdaptationModule:: |
| 298 | ResetVideoSourceRestrictions() { |
Henrik Boström | efbec9a | 2020-03-06 10:41:25 +0100 | [diff] [blame] | 299 | stream_adapter_->ClearRestrictions(); |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 300 | active_counts_.fill(AdaptationCounters()); |
Henrik Boström | fae6f0e | 2020-01-20 11:16:50 +0100 | [diff] [blame] | 301 | MaybeUpdateVideoSourceRestrictions(); |
| 302 | } |
| 303 | |
Henrik Boström | d4578ae | 2020-01-22 16:16:04 +0100 | [diff] [blame] | 304 | void OveruseFrameDetectorResourceAdaptationModule::OnFrame( |
| 305 | const VideoFrame& frame) { |
| 306 | last_input_frame_size_ = frame.size(); |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 307 | } |
| 308 | |
Henrik Boström | d4578ae | 2020-01-22 16:16:04 +0100 | [diff] [blame] | 309 | void OveruseFrameDetectorResourceAdaptationModule::OnFrameDroppedDueToSize() { |
Henrik Boström | efbec9a | 2020-03-06 10:41:25 +0100 | [diff] [blame] | 310 | AdaptationCounters counters_before = stream_adapter_->adaptation_counters(); |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 311 | OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality); |
Evan Shrubsole | 6c13fd9 | 2020-01-21 09:57:54 +0100 | [diff] [blame] | 312 | if (degradation_preference() == DegradationPreference::BALANCED && |
Henrik Boström | efbec9a | 2020-03-06 10:41:25 +0100 | [diff] [blame] | 313 | stream_adapter_->adaptation_counters().fps_adaptations > |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 314 | counters_before.fps_adaptations) { |
Evan Shrubsole | 6c13fd9 | 2020-01-21 09:57:54 +0100 | [diff] [blame] | 315 | // Adapt framerate in same step as resolution. |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 316 | OnResourceOveruse(AdaptationObserverInterface::AdaptReason::kQuality); |
Evan Shrubsole | 6c13fd9 | 2020-01-21 09:57:54 +0100 | [diff] [blame] | 317 | } |
Henrik Boström | efbec9a | 2020-03-06 10:41:25 +0100 | [diff] [blame] | 318 | if (stream_adapter_->adaptation_counters().resolution_adaptations > |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 319 | counters_before.resolution_adaptations) { |
Evan Shrubsole | 6c13fd9 | 2020-01-21 09:57:54 +0100 | [diff] [blame] | 320 | encoder_stats_observer_->OnInitialQualityResolutionAdaptDown(); |
| 321 | } |
Henrik Boström | 8cfecac | 2020-02-07 11:29:14 +0100 | [diff] [blame] | 322 | initial_frame_dropper_->OnFrameDroppedDueToSize(); |
Evan Shrubsole | 6c13fd9 | 2020-01-21 09:57:54 +0100 | [diff] [blame] | 323 | } |
| 324 | |
Henrik Boström | d4578ae | 2020-01-22 16:16:04 +0100 | [diff] [blame] | 325 | void OveruseFrameDetectorResourceAdaptationModule::OnEncodeStarted( |
| 326 | const VideoFrame& cropped_frame, |
| 327 | int64_t time_when_first_seen_us) { |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 328 | encode_usage_resource_->OnEncodeStarted(cropped_frame, |
| 329 | time_when_first_seen_us); |
Henrik Boström | d4578ae | 2020-01-22 16:16:04 +0100 | [diff] [blame] | 330 | } |
| 331 | |
| 332 | void OveruseFrameDetectorResourceAdaptationModule::OnEncodeCompleted( |
Evan Shrubsole | bfe3ef8 | 2020-01-30 14:29:35 +0100 | [diff] [blame] | 333 | const EncodedImage& encoded_image, |
Henrik Boström | d4578ae | 2020-01-22 16:16:04 +0100 | [diff] [blame] | 334 | int64_t time_sent_in_us, |
Henrik Boström | d4578ae | 2020-01-22 16:16:04 +0100 | [diff] [blame] | 335 | absl::optional<int> encode_duration_us) { |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 336 | // Inform |encode_usage_resource_| of the encode completed event. |
Evan Shrubsole | bfe3ef8 | 2020-01-30 14:29:35 +0100 | [diff] [blame] | 337 | uint32_t timestamp = encoded_image.Timestamp(); |
| 338 | int64_t capture_time_us = |
| 339 | encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec; |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 340 | encode_usage_resource_->OnEncodeCompleted( |
| 341 | timestamp, time_sent_in_us, capture_time_us, encode_duration_us); |
| 342 | // Inform |quality_scaler_resource_| of the encode completed event. |
| 343 | quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us); |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 344 | } |
| 345 | |
Evan Shrubsole | c809e8b | 2020-01-31 15:36:35 +0100 | [diff] [blame] | 346 | void OveruseFrameDetectorResourceAdaptationModule::OnFrameDropped( |
| 347 | EncodedImageCallback::DropReason reason) { |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 348 | quality_scaler_resource_->OnFrameDropped(reason); |
Evan Shrubsole | c809e8b | 2020-01-31 15:36:35 +0100 | [diff] [blame] | 349 | } |
| 350 | |
Henrik Boström | 8cfecac | 2020-02-07 11:29:14 +0100 | [diff] [blame] | 351 | bool OveruseFrameDetectorResourceAdaptationModule::DropInitialFrames() const { |
| 352 | return initial_frame_dropper_->DropInitialFrames(); |
Evan Shrubsole | f2be3ef | 2020-02-03 10:43:31 +0100 | [diff] [blame] | 353 | } |
| 354 | |
Mirko Bonadei | 2e161c4 | 2020-02-20 08:45:01 +0000 | [diff] [blame] | 355 | void OveruseFrameDetectorResourceAdaptationModule::OnMaybeEncodeFrame() { |
| 356 | initial_frame_dropper_->OnMaybeEncodeFrame(); |
| 357 | MaybePerformQualityRampupExperiment(); |
| 358 | } |
| 359 | |
Evan Shrubsole | cf05952 | 2020-01-29 17:04:44 +0100 | [diff] [blame] | 360 | void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings( |
| 361 | absl::optional<VideoEncoder::QpThresholds> qp_thresholds) { |
| 362 | if (qp_thresholds.has_value()) { |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 363 | quality_scaler_resource_->StopCheckForOveruse(); |
| 364 | quality_scaler_resource_->StartCheckForOveruse(qp_thresholds.value()); |
Evan Shrubsole | cf05952 | 2020-01-29 17:04:44 +0100 | [diff] [blame] | 365 | } else { |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 366 | quality_scaler_resource_->StopCheckForOveruse(); |
Evan Shrubsole | cf05952 | 2020-01-29 17:04:44 +0100 | [diff] [blame] | 367 | } |
Henrik Boström | 8cfecac | 2020-02-07 11:29:14 +0100 | [diff] [blame] | 368 | initial_frame_dropper_->OnQualityScalerSettingsUpdated(); |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 369 | } |
| 370 | |
Evan Shrubsole | c81798b | 2020-02-03 15:46:08 +0100 | [diff] [blame] | 371 | void OveruseFrameDetectorResourceAdaptationModule::ConfigureQualityScaler( |
| 372 | const VideoEncoder::EncoderInfo& encoder_info) { |
| 373 | const auto scaling_settings = encoder_info.scaling_settings; |
| 374 | const bool quality_scaling_allowed = |
| 375 | IsResolutionScalingEnabled(degradation_preference_) && |
| 376 | scaling_settings.thresholds; |
| 377 | |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 378 | // TODO(https://crbug.com/webrtc/11222): Should this move to |
| 379 | // QualityScalerResource? |
Evan Shrubsole | c81798b | 2020-02-03 15:46:08 +0100 | [diff] [blame] | 380 | if (quality_scaling_allowed) { |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 381 | if (!quality_scaler_resource_->is_started()) { |
Evan Shrubsole | c81798b | 2020-02-03 15:46:08 +0100 | [diff] [blame] | 382 | // Quality scaler has not already been configured. |
| 383 | |
| 384 | // Use experimental thresholds if available. |
| 385 | absl::optional<VideoEncoder::QpThresholds> experimental_thresholds; |
| 386 | if (quality_scaling_experiment_enabled_) { |
| 387 | experimental_thresholds = QualityScalingExperiment::GetQpThresholds( |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 388 | GetVideoCodecTypeOrGeneric(encoder_settings_)); |
Evan Shrubsole | c81798b | 2020-02-03 15:46:08 +0100 | [diff] [blame] | 389 | } |
| 390 | UpdateQualityScalerSettings(experimental_thresholds |
| 391 | ? *experimental_thresholds |
| 392 | : *(scaling_settings.thresholds)); |
| 393 | } |
| 394 | } else { |
| 395 | UpdateQualityScalerSettings(absl::nullopt); |
| 396 | } |
| 397 | |
| 398 | // Set the qp-thresholds to the balanced settings if balanced mode. |
| 399 | if (degradation_preference_ == DegradationPreference::BALANCED && |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 400 | quality_scaler_resource_->is_started()) { |
Evan Shrubsole | c81798b | 2020-02-03 15:46:08 +0100 | [diff] [blame] | 401 | absl::optional<VideoEncoder::QpThresholds> thresholds = |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 402 | stream_adapter_->balanced_settings().GetQpThresholds( |
| 403 | GetVideoCodecTypeOrGeneric(encoder_settings_), |
| 404 | LastInputFrameSizeOrDefault()); |
Evan Shrubsole | c81798b | 2020-02-03 15:46:08 +0100 | [diff] [blame] | 405 | if (thresholds) { |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 406 | quality_scaler_resource_->SetQpThresholds(*thresholds); |
Evan Shrubsole | c81798b | 2020-02-03 15:46:08 +0100 | [diff] [blame] | 407 | } |
| 408 | } |
| 409 | |
| 410 | encoder_stats_observer_->OnAdaptationChanged( |
| 411 | VideoStreamEncoderObserver::AdaptationReason::kNone, |
| 412 | GetActiveCounts(AdaptationObserverInterface::AdaptReason::kCpu), |
| 413 | GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality)); |
| 414 | } |
| 415 | |
Henrik Boström | 48258ac | 2020-02-06 12:49:57 +0100 | [diff] [blame] | 416 | ResourceListenerResponse |
| 417 | OveruseFrameDetectorResourceAdaptationModule::OnResourceUsageStateMeasured( |
| 418 | const Resource& resource) { |
Evan Shrubsole | aa6fbc1 | 2020-02-25 16:26:01 +0100 | [diff] [blame] | 419 | const auto& registered_resource = |
| 420 | absl::c_find_if(resources_, [&resource](const ResourceAndReason& r) { |
| 421 | return r.resource == &resource; |
| 422 | }); |
| 423 | RTC_DCHECK(registered_resource != resources_.end()) |
| 424 | << resource.name() << " not found."; |
| 425 | |
| 426 | const AdaptationObserverInterface::AdaptReason reason = |
| 427 | registered_resource->reason; |
Henrik Boström | 48258ac | 2020-02-06 12:49:57 +0100 | [diff] [blame] | 428 | switch (resource.usage_state()) { |
| 429 | case ResourceUsageState::kOveruse: |
| 430 | return OnResourceOveruse(reason); |
| 431 | case ResourceUsageState::kStable: |
| 432 | // Do nothing. |
| 433 | // |
| 434 | // This module has two resources: |encoude_usage_resource_| and |
| 435 | // |quality_scaler_resource_|. A smarter adaptation module might not |
| 436 | // attempt to adapt up unless ALL resources were underused, but this |
| 437 | // module acts on each resource's measurement in isolation - without |
| 438 | // taking the current usage of any other resource into account. |
| 439 | return ResourceListenerResponse::kNothing; |
| 440 | case ResourceUsageState::kUnderuse: |
| 441 | OnResourceUnderuse(reason); |
| 442 | return ResourceListenerResponse::kNothing; |
| 443 | } |
| 444 | } |
| 445 | |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 446 | absl::optional<VideoStreamAdapter::AdaptationTarget> |
Henrik Boström | 6038383 | 2020-02-28 09:03:53 +0100 | [diff] [blame] | 447 | OveruseFrameDetectorResourceAdaptationModule::GetAdaptUpTarget( |
| 448 | int input_pixels, |
| 449 | int input_fps, |
| 450 | AdaptationObserverInterface::AdaptReason reason) const { |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 451 | // We can't adapt up if we're already at the highest setting. |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 452 | // Note that this only includes counts relevant to the current degradation |
| 453 | // preference. e.g. we previously adapted resolution, now prefer adpating fps, |
| 454 | // only count the fps adaptations and not the previous resolution adaptations. |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 455 | // |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 456 | // TODO(https://crbug.com/webrtc/11394): Checking the counts for reason should |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 457 | // be replaced with checking the overuse state of all resources. This is |
| 458 | // effectively trying to infer if the the Resource specified by |reason| is OK |
| 459 | // with adapting up by looking at active counters. If the relevant Resources |
| 460 | // simply told us this directly we wouldn't have to depend on stats counters |
| 461 | // to abort GetAdaptUpTarget(). |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 462 | int num_downgrades = ApplyDegradationPreference(active_counts_[reason], |
| 463 | degradation_preference_) |
| 464 | .Total(); |
Henrik Boström | d2a1f09 | 2020-02-25 09:46:36 +0100 | [diff] [blame] | 465 | RTC_DCHECK_GE(num_downgrades, 0); |
| 466 | if (num_downgrades == 0) |
Henrik Boström | 6038383 | 2020-02-28 09:03:53 +0100 | [diff] [blame] | 467 | return absl::nullopt; |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 468 | return stream_adapter_->GetAdaptUpTarget( |
| 469 | encoder_settings_, encoder_target_bitrate_bps_, GetVideoInputMode(), |
| 470 | input_pixels, input_fps, reason); |
Henrik Boström | d2a1f09 | 2020-02-25 09:46:36 +0100 | [diff] [blame] | 471 | } |
| 472 | |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 473 | absl::optional<VideoStreamAdapter::AdaptationTarget> |
Henrik Boström | 6038383 | 2020-02-28 09:03:53 +0100 | [diff] [blame] | 474 | OveruseFrameDetectorResourceAdaptationModule::GetAdaptDownTarget( |
| 475 | int input_pixels, |
| 476 | int input_fps, |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 477 | int min_pixels_per_frame) const { |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 478 | return stream_adapter_->GetAdaptDownTarget( |
| 479 | encoder_settings_, GetVideoInputMode(), input_pixels, input_fps, |
| 480 | min_pixels_per_frame, encoder_stats_observer_); |
Henrik Boström | 6038383 | 2020-02-28 09:03:53 +0100 | [diff] [blame] | 481 | } |
| 482 | |
| 483 | void OveruseFrameDetectorResourceAdaptationModule::ApplyAdaptationTarget( |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 484 | const VideoStreamAdapter::AdaptationTarget& target, |
| 485 | int input_pixels, |
| 486 | int input_fps, |
| 487 | int min_pixels_per_frame) { |
| 488 | stream_adapter_->ApplyAdaptationTarget(target, GetVideoInputMode(), |
| 489 | input_pixels, input_fps, |
| 490 | min_pixels_per_frame); |
Henrik Boström | 6038383 | 2020-02-28 09:03:53 +0100 | [diff] [blame] | 491 | } |
| 492 | |
| 493 | void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse( |
| 494 | AdaptationObserverInterface::AdaptReason reason) { |
| 495 | int input_pixels = LastInputFrameSizeOrDefault(); |
| 496 | int input_fps = encoder_stats_observer_->GetInputFrameRate(); |
| 497 | int min_pixels_per_frame = MinPixelsPerFrame(); |
| 498 | // Should we adapt, if so to what target? |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 499 | absl::optional<VideoStreamAdapter::AdaptationTarget> target = |
Henrik Boström | 6038383 | 2020-02-28 09:03:53 +0100 | [diff] [blame] | 500 | GetAdaptUpTarget(input_pixels, input_fps, reason); |
| 501 | if (!target.has_value()) |
| 502 | return; |
| 503 | // Apply target. |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 504 | ApplyAdaptationTarget(target.value(), input_pixels, input_fps, |
| 505 | min_pixels_per_frame); |
Henrik Boström | 6038383 | 2020-02-28 09:03:53 +0100 | [diff] [blame] | 506 | // Update VideoSourceRestrictions based on adaptation. This also informs the |
| 507 | // |adaptation_listener_|. |
| 508 | MaybeUpdateVideoSourceRestrictions(); |
| 509 | // Stats and logging. |
| 510 | UpdateAdaptationStats(reason); |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 511 | RTC_LOG(LS_INFO) << ActiveCountsToString(); |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 512 | } |
| 513 | |
Henrik Boström | 48258ac | 2020-02-06 12:49:57 +0100 | [diff] [blame] | 514 | ResourceListenerResponse |
| 515 | OveruseFrameDetectorResourceAdaptationModule::OnResourceOveruse( |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 516 | AdaptationObserverInterface::AdaptReason reason) { |
Henrik Boström | a3d4252 | 2020-01-16 13:55:29 +0100 | [diff] [blame] | 517 | if (!has_input_video_) |
Henrik Boström | 48258ac | 2020-02-06 12:49:57 +0100 | [diff] [blame] | 518 | return ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency; |
Henrik Boström | 6038383 | 2020-02-28 09:03:53 +0100 | [diff] [blame] | 519 | int input_pixels = LastInputFrameSizeOrDefault(); |
| 520 | int input_fps = encoder_stats_observer_->GetInputFrameRate(); |
| 521 | int min_pixels_per_frame = MinPixelsPerFrame(); |
| 522 | // Should we adapt, if so to what target? |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 523 | absl::optional<VideoStreamAdapter::AdaptationTarget> target = |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 524 | GetAdaptDownTarget(input_pixels, input_fps, min_pixels_per_frame); |
Henrik Boström | 6038383 | 2020-02-28 09:03:53 +0100 | [diff] [blame] | 525 | if (!target.has_value()) |
Henrik Boström | d2a1f09 | 2020-02-25 09:46:36 +0100 | [diff] [blame] | 526 | return ResourceListenerResponse::kNothing; |
Henrik Boström | 6038383 | 2020-02-28 09:03:53 +0100 | [diff] [blame] | 527 | // Apply target. |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 528 | ApplyAdaptationTarget(target.value(), input_pixels, input_fps, |
| 529 | min_pixels_per_frame); |
Henrik Boström | 6038383 | 2020-02-28 09:03:53 +0100 | [diff] [blame] | 530 | // Update VideoSourceRestrictions based on adaptation. This also informs the |
| 531 | // |adaptation_listener_|. |
Henrik Boström | 07b17df | 2020-01-15 11:42:12 +0100 | [diff] [blame] | 532 | MaybeUpdateVideoSourceRestrictions(); |
Henrik Boström | 6038383 | 2020-02-28 09:03:53 +0100 | [diff] [blame] | 533 | // Stats and logging. |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 534 | UpdateAdaptationStats(reason); |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 535 | RTC_LOG(INFO) << ActiveCountsToString(); |
Henrik Boström | 6038383 | 2020-02-28 09:03:53 +0100 | [diff] [blame] | 536 | // In BALANCED, if requested FPS is higher or close to input FPS to the target |
| 537 | // we tell the QualityScaler to increase its frequency. |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 538 | if (stream_adapter_->EffectiveDegradationPreference(GetVideoInputMode()) == |
| 539 | DegradationPreference::BALANCED && |
| 540 | target->action == |
| 541 | VideoStreamAdapter::AdaptationAction::kDecreaseFrameRate) { |
| 542 | absl::optional<int> min_diff = |
| 543 | stream_adapter_->balanced_settings().MinFpsDiff(input_pixels); |
Henrik Boström | 6038383 | 2020-02-28 09:03:53 +0100 | [diff] [blame] | 544 | if (min_diff && input_fps > 0) { |
| 545 | int fps_diff = input_fps - target->value; |
| 546 | if (fps_diff < min_diff.value()) { |
| 547 | return ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency; |
| 548 | } |
| 549 | } |
| 550 | } |
| 551 | return ResourceListenerResponse::kNothing; |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 552 | } |
| 553 | |
Henrik Boström | ad515a2 | 2020-01-27 13:38:05 +0100 | [diff] [blame] | 554 | // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle |
| 555 | // pipelining encoders better (multiple input frames before something comes |
| 556 | // out). This should effectively turn off CPU adaptations for systems that |
| 557 | // remotely cope with the load right now. |
| 558 | CpuOveruseOptions |
| 559 | OveruseFrameDetectorResourceAdaptationModule::GetCpuOveruseOptions() const { |
| 560 | // This is already ensured by the only caller of this method: |
| 561 | // StartResourceAdaptation(). |
| 562 | RTC_DCHECK(encoder_settings_.has_value()); |
| 563 | CpuOveruseOptions options; |
| 564 | // Hardware accelerated encoders are assumed to be pipelined; give them |
| 565 | // additional overuse time. |
| 566 | if (encoder_settings_->encoder_info().is_hardware_accelerated) { |
| 567 | options.low_encode_usage_threshold_percent = 150; |
| 568 | options.high_encode_usage_threshold_percent = 200; |
| 569 | } |
| 570 | if (experiment_cpu_load_estimator_) { |
| 571 | options.filter_time_ms = 5 * rtc::kNumMillisecsPerSec; |
| 572 | } |
| 573 | return options; |
| 574 | } |
| 575 | |
Henrik Boström | d4578ae | 2020-01-22 16:16:04 +0100 | [diff] [blame] | 576 | int OveruseFrameDetectorResourceAdaptationModule::LastInputFrameSizeOrDefault() |
| 577 | const { |
| 578 | // The dependency on this hardcoded resolution is inherited from old code, |
| 579 | // which used this resolution as a stand-in for not knowing the resolution |
| 580 | // yet. |
| 581 | // TODO(hbos): Can we simply DCHECK has_value() before usage instead? Having a |
| 582 | // DCHECK passed all the tests but adding it does change the requirements of |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 583 | // this class (= not being allowed to call OnResourceUnderuse() or |
| 584 | // OnResourceOveruse() before OnFrame()) and deserves a standalone CL. |
Henrik Boström | d4578ae | 2020-01-22 16:16:04 +0100 | [diff] [blame] | 585 | return last_input_frame_size_.value_or( |
| 586 | VideoStreamEncoder::kDefaultLastFrameInfoWidth * |
| 587 | VideoStreamEncoder::kDefaultLastFrameInfoHeight); |
| 588 | } |
| 589 | |
Henrik Boström | 6038383 | 2020-02-28 09:03:53 +0100 | [diff] [blame] | 590 | int OveruseFrameDetectorResourceAdaptationModule::MinPixelsPerFrame() const { |
| 591 | return encoder_settings_.has_value() |
| 592 | ? encoder_settings_->encoder_info() |
| 593 | .scaling_settings.min_pixels_per_frame |
| 594 | : kDefaultMinPixelsPerFrame; |
| 595 | } |
| 596 | |
Henrik Boström | 8234b92 | 2020-01-13 17:26:50 +0100 | [diff] [blame] | 597 | void OveruseFrameDetectorResourceAdaptationModule:: |
Henrik Boström | 07b17df | 2020-01-15 11:42:12 +0100 | [diff] [blame] | 598 | MaybeUpdateVideoSourceRestrictions() { |
| 599 | VideoSourceRestrictions new_restrictions = ApplyDegradationPreference( |
Henrik Boström | efbec9a | 2020-03-06 10:41:25 +0100 | [diff] [blame] | 600 | stream_adapter_->source_restrictions(), degradation_preference_); |
Henrik Boström | 07b17df | 2020-01-15 11:42:12 +0100 | [diff] [blame] | 601 | if (video_source_restrictions_ != new_restrictions) { |
| 602 | video_source_restrictions_ = std::move(new_restrictions); |
Henrik Boström | 8234b92 | 2020-01-13 17:26:50 +0100 | [diff] [blame] | 603 | adaptation_listener_->OnVideoSourceRestrictionsUpdated( |
Henrik Boström | 07b17df | 2020-01-15 11:42:12 +0100 | [diff] [blame] | 604 | video_source_restrictions_); |
Henrik Boström | fae6f0e | 2020-01-20 11:16:50 +0100 | [diff] [blame] | 605 | MaybeUpdateTargetFrameRate(); |
| 606 | } |
| 607 | } |
| 608 | |
| 609 | void OveruseFrameDetectorResourceAdaptationModule:: |
| 610 | MaybeUpdateTargetFrameRate() { |
Henrik Boström | 4bab2fc | 2020-01-21 11:18:06 +0100 | [diff] [blame] | 611 | absl::optional<double> codec_max_frame_rate = |
| 612 | encoder_settings_.has_value() |
| 613 | ? absl::optional<double>( |
| 614 | encoder_settings_->video_codec().maxFramerate) |
| 615 | : absl::nullopt; |
Henrik Boström | fae6f0e | 2020-01-20 11:16:50 +0100 | [diff] [blame] | 616 | // The current target framerate is the maximum frame rate as specified by |
| 617 | // the current codec configuration or any limit imposed by the adaptation |
| 618 | // module. This is used to make sure overuse detection doesn't needlessly |
| 619 | // trigger in low and/or variable framerate scenarios. |
| 620 | absl::optional<double> target_frame_rate = |
Henrik Boström | efbec9a | 2020-03-06 10:41:25 +0100 | [diff] [blame] | 621 | ApplyDegradationPreference(stream_adapter_->source_restrictions(), |
Henrik Boström | fae6f0e | 2020-01-20 11:16:50 +0100 | [diff] [blame] | 622 | degradation_preference_) |
| 623 | .max_frame_rate(); |
| 624 | if (!target_frame_rate.has_value() || |
Henrik Boström | 4bab2fc | 2020-01-21 11:18:06 +0100 | [diff] [blame] | 625 | (codec_max_frame_rate.has_value() && |
| 626 | codec_max_frame_rate.value() < target_frame_rate.value())) { |
| 627 | target_frame_rate = codec_max_frame_rate; |
Henrik Boström | fae6f0e | 2020-01-20 11:16:50 +0100 | [diff] [blame] | 628 | } |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 629 | encode_usage_resource_->SetTargetFrameRate(target_frame_rate); |
Henrik Boström | 8234b92 | 2020-01-13 17:26:50 +0100 | [diff] [blame] | 630 | } |
| 631 | |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 632 | void OveruseFrameDetectorResourceAdaptationModule::OnAdaptationCountChanged( |
| 633 | const AdaptationCounters& adaptation_count, |
| 634 | AdaptationCounters* active_count, |
| 635 | AdaptationCounters* other_active) { |
| 636 | RTC_DCHECK(active_count); |
| 637 | RTC_DCHECK(other_active); |
| 638 | const int active_total = active_count->Total(); |
| 639 | const int other_total = other_active->Total(); |
| 640 | const AdaptationCounters prev_total = *active_count + *other_active; |
| 641 | const AdaptationCounters delta = adaptation_count - prev_total; |
| 642 | |
| 643 | RTC_DCHECK_EQ( |
| 644 | std::abs(delta.resolution_adaptations) + std::abs(delta.fps_adaptations), |
| 645 | 1) |
| 646 | << "Adaptation took more than one step!"; |
| 647 | |
| 648 | if (delta.resolution_adaptations > 0) { |
| 649 | ++active_count->resolution_adaptations; |
| 650 | } else if (delta.resolution_adaptations < 0) { |
| 651 | if (active_count->resolution_adaptations == 0) { |
| 652 | RTC_DCHECK_GT(active_count->fps_adaptations, 0) << "No downgrades left"; |
| 653 | RTC_DCHECK_GT(other_active->resolution_adaptations, 0) |
| 654 | << "No resolution adaptation to borrow from"; |
| 655 | // Lend an fps adaptation to other and take one resolution adaptation. |
| 656 | --active_count->fps_adaptations; |
| 657 | ++other_active->fps_adaptations; |
| 658 | --other_active->resolution_adaptations; |
| 659 | } else { |
| 660 | --active_count->resolution_adaptations; |
| 661 | } |
| 662 | } |
| 663 | if (delta.fps_adaptations > 0) { |
| 664 | ++active_count->fps_adaptations; |
| 665 | } else if (delta.fps_adaptations < 0) { |
| 666 | if (active_count->fps_adaptations == 0) { |
| 667 | RTC_DCHECK_GT(active_count->resolution_adaptations, 0) |
| 668 | << "No downgrades left"; |
| 669 | RTC_DCHECK_GT(other_active->fps_adaptations, 0) |
| 670 | << "No fps adaptation to borrow from"; |
| 671 | // Lend a resolution adaptation to other and take one fps adaptation. |
| 672 | --active_count->resolution_adaptations; |
| 673 | ++other_active->resolution_adaptations; |
| 674 | --other_active->fps_adaptations; |
| 675 | } else { |
| 676 | --active_count->fps_adaptations; |
| 677 | } |
| 678 | } |
| 679 | |
| 680 | RTC_DCHECK(*active_count + *other_active == adaptation_count); |
| 681 | RTC_DCHECK_EQ(other_active->Total(), other_total); |
| 682 | RTC_DCHECK_EQ(active_count->Total(), active_total + delta.Total()); |
| 683 | RTC_DCHECK_GE(active_count->resolution_adaptations, 0); |
| 684 | RTC_DCHECK_GE(active_count->fps_adaptations, 0); |
| 685 | RTC_DCHECK_GE(other_active->resolution_adaptations, 0); |
| 686 | RTC_DCHECK_GE(other_active->fps_adaptations, 0); |
| 687 | } |
| 688 | |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 689 | // TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged. |
| 690 | void OveruseFrameDetectorResourceAdaptationModule::UpdateAdaptationStats( |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 691 | AdaptationObserverInterface::AdaptReason reason) { |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 692 | // Update active counts |
| 693 | AdaptationCounters& active_count = active_counts_[reason]; |
| 694 | AdaptationCounters& other_active = active_counts_[(reason + 1) % 2]; |
| 695 | const AdaptationCounters total_counts = |
Henrik Boström | efbec9a | 2020-03-06 10:41:25 +0100 | [diff] [blame] | 696 | stream_adapter_->adaptation_counters(); |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 697 | |
| 698 | OnAdaptationCountChanged(total_counts, &active_count, &other_active); |
| 699 | |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 700 | switch (reason) { |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 701 | case AdaptationObserverInterface::AdaptReason::kCpu: |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 702 | encoder_stats_observer_->OnAdaptationChanged( |
| 703 | VideoStreamEncoderObserver::AdaptationReason::kCpu, |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 704 | GetActiveCounts(AdaptationObserverInterface::AdaptReason::kCpu), |
| 705 | GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality)); |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 706 | break; |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 707 | case AdaptationObserverInterface::AdaptReason::kQuality: |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 708 | encoder_stats_observer_->OnAdaptationChanged( |
| 709 | VideoStreamEncoderObserver::AdaptationReason::kQuality, |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 710 | GetActiveCounts(AdaptationObserverInterface::AdaptReason::kCpu), |
| 711 | GetActiveCounts(AdaptationObserverInterface::AdaptReason::kQuality)); |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 712 | break; |
| 713 | } |
| 714 | } |
| 715 | |
| 716 | VideoStreamEncoderObserver::AdaptationSteps |
| 717 | OveruseFrameDetectorResourceAdaptationModule::GetActiveCounts( |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 718 | AdaptationObserverInterface::AdaptReason reason) { |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 719 | // TODO(https://crbug.com/webrtc/11392) Ideally this shuold be moved out of |
| 720 | // this class and into the encoder_stats_observer_. |
| 721 | const AdaptationCounters counters = active_counts_[reason]; |
| 722 | |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 723 | VideoStreamEncoderObserver::AdaptationSteps counts = |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 724 | VideoStreamEncoderObserver::AdaptationSteps(); |
| 725 | counts.num_resolution_reductions = counters.resolution_adaptations; |
| 726 | counts.num_framerate_reductions = counters.fps_adaptations; |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 727 | switch (reason) { |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 728 | case AdaptationObserverInterface::AdaptReason::kCpu: |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 729 | if (!IsFramerateScalingEnabled(degradation_preference_)) |
| 730 | counts.num_framerate_reductions = absl::nullopt; |
| 731 | if (!IsResolutionScalingEnabled(degradation_preference_)) |
| 732 | counts.num_resolution_reductions = absl::nullopt; |
| 733 | break; |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 734 | case AdaptationObserverInterface::AdaptReason::kQuality: |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 735 | if (!IsFramerateScalingEnabled(degradation_preference_) || |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 736 | !quality_scaler_resource_->is_started()) { |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 737 | counts.num_framerate_reductions = absl::nullopt; |
| 738 | } |
| 739 | if (!IsResolutionScalingEnabled(degradation_preference_) || |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 740 | !quality_scaler_resource_->is_started()) { |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 741 | counts.num_resolution_reductions = absl::nullopt; |
| 742 | } |
| 743 | break; |
| 744 | } |
| 745 | return counts; |
| 746 | } |
| 747 | |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 748 | VideoStreamAdapter::VideoInputMode |
| 749 | OveruseFrameDetectorResourceAdaptationModule::GetVideoInputMode() const { |
| 750 | if (!has_input_video_) |
| 751 | return VideoStreamAdapter::VideoInputMode::kNoVideo; |
Henrik Boström | 4bab2fc | 2020-01-21 11:18:06 +0100 | [diff] [blame] | 752 | return (encoder_settings_.has_value() && |
| 753 | encoder_settings_->encoder_config().content_type == |
Henrik Boström | b0f2e0c | 2020-03-06 13:32:03 +0100 | [diff] [blame^] | 754 | VideoEncoderConfig::ContentType::kScreen) |
| 755 | ? VideoStreamAdapter::VideoInputMode::kScreenshareVideo |
| 756 | : VideoStreamAdapter::VideoInputMode::kNormalVideo; |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 757 | } |
| 758 | |
Evan Shrubsole | e331a12 | 2020-02-05 13:30:33 +0100 | [diff] [blame] | 759 | void OveruseFrameDetectorResourceAdaptationModule:: |
| 760 | MaybePerformQualityRampupExperiment() { |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 761 | if (!quality_scaler_resource_->is_started()) |
Evan Shrubsole | e331a12 | 2020-02-05 13:30:33 +0100 | [diff] [blame] | 762 | return; |
| 763 | |
| 764 | if (quality_rampup_done_) |
| 765 | return; |
| 766 | |
| 767 | int64_t now_ms = clock_->TimeInMilliseconds(); |
| 768 | uint32_t bw_kbps = encoder_rates_.has_value() |
| 769 | ? encoder_rates_.value().bandwidth_allocation.kbps() |
| 770 | : 0; |
| 771 | |
| 772 | bool try_quality_rampup = false; |
| 773 | if (quality_rampup_experiment_.BwHigh(now_ms, bw_kbps)) { |
| 774 | // Verify that encoder is at max bitrate and the QP is low. |
| 775 | if (encoder_settings_ && |
| 776 | encoder_target_bitrate_bps_.value_or(0) == |
| 777 | encoder_settings_->video_codec().maxBitrate * 1000 && |
Henrik Boström | 7875c99 | 2020-02-06 10:35:00 +0100 | [diff] [blame] | 778 | quality_scaler_resource_->QpFastFilterLow()) { |
Evan Shrubsole | e331a12 | 2020-02-05 13:30:33 +0100 | [diff] [blame] | 779 | try_quality_rampup = true; |
| 780 | } |
| 781 | } |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 782 | // TODO(https://crbug.com/webrtc/11392): See if we can rely on the total |
| 783 | // counts or the stats, and not the active counts. |
| 784 | const AdaptationCounters& qp_counts = |
| 785 | std::get<AdaptationObserverInterface::kQuality>(active_counts_); |
| 786 | const AdaptationCounters& cpu_counts = |
| 787 | std::get<AdaptationObserverInterface::kCpu>(active_counts_); |
| 788 | if (try_quality_rampup && qp_counts.resolution_adaptations > 0 && |
| 789 | cpu_counts.Total() == 0) { |
Evan Shrubsole | e331a12 | 2020-02-05 13:30:33 +0100 | [diff] [blame] | 790 | RTC_LOG(LS_INFO) << "Reset quality limitations."; |
| 791 | ResetVideoSourceRestrictions(); |
| 792 | quality_rampup_done_ = true; |
| 793 | } |
| 794 | } |
| 795 | |
Evan Shrubsole | 33be9df | 2020-03-05 18:39:32 +0100 | [diff] [blame] | 796 | std::string OveruseFrameDetectorResourceAdaptationModule::ActiveCountsToString() |
| 797 | const { |
| 798 | rtc::StringBuilder ss; |
| 799 | |
| 800 | ss << "Downgrade counts: fps: {"; |
| 801 | for (size_t reason = 0; reason < active_counts_.size(); ++reason) { |
| 802 | ss << (reason ? " cpu" : "quality") << ":"; |
| 803 | ss << active_counts_[reason].fps_adaptations; |
| 804 | } |
| 805 | ss << "}, resolution {"; |
| 806 | for (size_t reason = 0; reason < active_counts_.size(); ++reason) { |
| 807 | ss << (reason ? " cpu" : "quality") << ":"; |
| 808 | ss << active_counts_[reason].resolution_adaptations; |
| 809 | } |
| 810 | ss << "}"; |
| 811 | |
| 812 | return ss.Release(); |
| 813 | } |
Henrik Boström | b08882b | 2020-01-07 10:11:17 +0100 | [diff] [blame] | 814 | } // namespace webrtc |