[Overuse] Make VideoStreamAdapter responsible for executing adaptation.
This CL moves GetAdaptUpTarget(), GetAdaptDownTarget() and
ApplyAdaptationTarget() - and related code - to the VideoStreamAdapter.
This includes pieces related to calculating how to adapt, including:
- DegradationPreference
- BalancedDegradationPreference
- AdaptationRequest and last_adaptation_request_
- CanAdaptUpResolution()
The VideoStreamAdapter's interface has changed: VideoSourceRestrictor
methods are now hidden in favor of methods exposing AdaptationTarget.
This CL also does some misc moves:
- GetEncoderBitrateLimits is moved and renamed to
VideoEncoder::EncoderInfo::GetEncoderBitrateLimitsForResolution.
- EncoderSettings moved to a separate file.
// For api/video_codecs/video_encoder.[cc/h] changes, which is the
// moving of a function.
TBR=sprang@webrtc.org
Bug: webrtc:11393
Change-Id: Ie6bd8ef644ce927d7eca6ab90a0a7bcace682f3c
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169842
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30708}
diff --git a/api/video_codecs/video_encoder.cc b/api/video_codecs/video_encoder.cc
index cf25987..4427d6c 100644
--- a/api/video_codecs/video_encoder.cc
+++ b/api/video_codecs/video_encoder.cc
@@ -11,6 +11,7 @@
#include "api/video_codecs/video_encoder.h"
#include <string.h>
+#include <algorithm>
#include "rtc_base/checks.h"
#include "rtc_base/strings/string_builder.h"
@@ -208,6 +209,42 @@
return true;
}
+absl::optional<VideoEncoder::ResolutionBitrateLimits>
+VideoEncoder::EncoderInfo::GetEncoderBitrateLimitsForResolution(
+ int frame_size_pixels) const {
+ std::vector<ResolutionBitrateLimits> bitrate_limits =
+ resolution_bitrate_limits;
+
+ // Sort the list of bitrate limits by resolution.
+ sort(bitrate_limits.begin(), bitrate_limits.end(),
+ [](const ResolutionBitrateLimits& lhs,
+ const ResolutionBitrateLimits& rhs) {
+ return lhs.frame_size_pixels < rhs.frame_size_pixels;
+ });
+
+ for (size_t i = 0; i < bitrate_limits.size(); ++i) {
+ RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps, 0);
+ RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps, 0);
+ RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps,
+ bitrate_limits[i].min_bitrate_bps);
+ if (i > 0) {
+ // The bitrate limits aren't expected to decrease with resolution.
+ RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps,
+ bitrate_limits[i - 1].min_bitrate_bps);
+ RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps,
+ bitrate_limits[i - 1].min_start_bitrate_bps);
+ RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps,
+ bitrate_limits[i - 1].max_bitrate_bps);
+ }
+
+ if (bitrate_limits[i].frame_size_pixels >= frame_size_pixels) {
+ return absl::optional<ResolutionBitrateLimits>(bitrate_limits[i]);
+ }
+ }
+
+ return absl::nullopt;
+}
+
VideoEncoder::RateControlParameters::RateControlParameters()
: bitrate(VideoBitrateAllocation()),
framerate_fps(0.0),
diff --git a/api/video_codecs/video_encoder.h b/api/video_codecs/video_encoder.h
index 34502c8..064dc8f 100644
--- a/api/video_codecs/video_encoder.h
+++ b/api/video_codecs/video_encoder.h
@@ -236,6 +236,11 @@
// Recommended bitrate limits for different resolutions.
std::vector<ResolutionBitrateLimits> resolution_bitrate_limits;
+ // Obtains the limits from |resolution_bitrate_limits| that best matches the
+ // |frame_size_pixels|.
+ absl::optional<ResolutionBitrateLimits>
+ GetEncoderBitrateLimitsForResolution(int frame_size_pixels) const;
+
// If true, this encoder has internal support for generating simulcast
// streams. Otherwise, an adapter class will be needed.
// Even if true, the config provided to InitEncode() might not be supported,
diff --git a/call/adaptation/BUILD.gn b/call/adaptation/BUILD.gn
index 5eb5af5..0288e24 100644
--- a/call/adaptation/BUILD.gn
+++ b/call/adaptation/BUILD.gn
@@ -10,6 +10,8 @@
rtc_library("resource_adaptation") {
sources = [
+ "encoder_settings.cc",
+ "encoder_settings.h",
"resource.cc",
"resource.h",
"resource_adaptation_module_interface.cc",
diff --git a/call/adaptation/encoder_settings.cc b/call/adaptation/encoder_settings.cc
new file mode 100644
index 0000000..84b4b17
--- /dev/null
+++ b/call/adaptation/encoder_settings.cc
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2020 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "call/adaptation/encoder_settings.h"
+
+#include <utility>
+
+namespace webrtc {
+
+EncoderSettings::EncoderSettings(VideoEncoder::EncoderInfo encoder_info,
+ VideoEncoderConfig encoder_config,
+ VideoCodec video_codec)
+ : encoder_info_(std::move(encoder_info)),
+ encoder_config_(std::move(encoder_config)),
+ video_codec_(std::move(video_codec)) {}
+
+const VideoEncoder::EncoderInfo& EncoderSettings::encoder_info() const {
+ return encoder_info_;
+}
+
+const VideoEncoderConfig& EncoderSettings::encoder_config() const {
+ return encoder_config_;
+}
+
+const VideoCodec& EncoderSettings::video_codec() const {
+ return video_codec_;
+}
+
+VideoCodecType GetVideoCodecTypeOrGeneric(
+ const absl::optional<EncoderSettings>& settings) {
+ return settings.has_value() ? settings->encoder_config().codec_type
+ : kVideoCodecGeneric;
+}
+
+} // namespace webrtc
diff --git a/call/adaptation/encoder_settings.h b/call/adaptation/encoder_settings.h
new file mode 100644
index 0000000..9cfd056
--- /dev/null
+++ b/call/adaptation/encoder_settings.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2020 The WebRTC Project Authors. All rights reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef CALL_ADAPTATION_ENCODER_SETTINGS_H_
+#define CALL_ADAPTATION_ENCODER_SETTINGS_H_
+
+#include "absl/types/optional.h"
+#include "api/video_codecs/video_codec.h"
+#include "api/video_codecs/video_encoder.h"
+#include "api/video_codecs/video_encoder_config.h"
+
+namespace webrtc {
+
+// Information about an encoder available when reconfiguring the encoder.
+class EncoderSettings {
+ public:
+ EncoderSettings(VideoEncoder::EncoderInfo encoder_info,
+ VideoEncoderConfig encoder_config,
+ VideoCodec video_codec);
+
+ // Encoder capabilities, implementation info, etc.
+ const VideoEncoder::EncoderInfo& encoder_info() const;
+ // Configuration parameters, ultimately coming from the API and negotiation.
+ const VideoEncoderConfig& encoder_config() const;
+ // Lower level config, heavily based on the VideoEncoderConfig.
+ const VideoCodec& video_codec() const;
+
+ private:
+ VideoEncoder::EncoderInfo encoder_info_;
+ VideoEncoderConfig encoder_config_;
+ VideoCodec video_codec_;
+};
+
+VideoCodecType GetVideoCodecTypeOrGeneric(
+ const absl::optional<EncoderSettings>& settings);
+
+} // namespace webrtc
+
+#endif // CALL_ADAPTATION_ENCODER_SETTINGS_H_
diff --git a/call/adaptation/resource_adaptation_module_interface.cc b/call/adaptation/resource_adaptation_module_interface.cc
index 63cfb72..e89d1ef 100644
--- a/call/adaptation/resource_adaptation_module_interface.cc
+++ b/call/adaptation/resource_adaptation_module_interface.cc
@@ -10,29 +10,8 @@
#include "call/adaptation/resource_adaptation_module_interface.h"
-#include <utility>
-
namespace webrtc {
-EncoderSettings::EncoderSettings(VideoEncoder::EncoderInfo encoder_info,
- VideoEncoderConfig encoder_config,
- VideoCodec video_codec)
- : encoder_info_(std::move(encoder_info)),
- encoder_config_(std::move(encoder_config)),
- video_codec_(std::move(video_codec)) {}
-
-const VideoEncoder::EncoderInfo& EncoderSettings::encoder_info() const {
- return encoder_info_;
-}
-
-const VideoEncoderConfig& EncoderSettings::encoder_config() const {
- return encoder_config_;
-}
-
-const VideoCodec& EncoderSettings::video_codec() const {
- return video_codec_;
-}
-
ResourceAdaptationModuleListener::~ResourceAdaptationModuleListener() {}
ResourceAdaptationModuleInterface::~ResourceAdaptationModuleInterface() {}
diff --git a/call/adaptation/resource_adaptation_module_interface.h b/call/adaptation/resource_adaptation_module_interface.h
index e961897..1248e17 100644
--- a/call/adaptation/resource_adaptation_module_interface.h
+++ b/call/adaptation/resource_adaptation_module_interface.h
@@ -14,33 +14,12 @@
#include "absl/types/optional.h"
#include "api/rtp_parameters.h"
#include "api/video/video_frame.h"
-#include "api/video_codecs/video_encoder.h"
-#include "api/video_codecs/video_encoder_config.h"
+#include "call/adaptation/encoder_settings.h"
#include "call/adaptation/resource.h"
#include "call/adaptation/video_source_restrictions.h"
namespace webrtc {
-// Information about an encoder available when reconfiguring the encoder.
-class EncoderSettings {
- public:
- EncoderSettings(VideoEncoder::EncoderInfo encoder_info,
- VideoEncoderConfig encoder_config,
- VideoCodec video_codec);
-
- // Encoder capabilities, implementation info, etc.
- const VideoEncoder::EncoderInfo& encoder_info() const;
- // Configuration parameters, ultimately coming from the API and negotiation.
- const VideoEncoderConfig& encoder_config() const;
- // Lower level config, heavily based on the VideoEncoderConfig.
- const VideoCodec& video_codec() const;
-
- private:
- VideoEncoder::EncoderInfo encoder_info_;
- VideoEncoderConfig encoder_config_;
- VideoCodec video_codec_;
-};
-
// The listener is responsible for carrying out the reconfiguration of the video
// source such that the VideoSourceRestrictions are fulfilled.
class ResourceAdaptationModuleListener {
diff --git a/video/adaptation/BUILD.gn b/video/adaptation/BUILD.gn
index e9e5955..95705cb 100644
--- a/video/adaptation/BUILD.gn
+++ b/video/adaptation/BUILD.gn
@@ -17,12 +17,17 @@
]
deps = [
+ "../../api:rtp_parameters",
+ "../../api/video:video_stream_encoder",
+ "../../api/video_codecs:video_codecs_api",
"../../call/adaptation:resource_adaptation",
+ "../../modules/video_coding:video_coding_utility",
"../../rtc_base:checks",
"../../rtc_base:logging",
"../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_event",
"../../rtc_base:rtc_numerics",
+ "../../rtc_base/experiments:balanced_degradation_settings",
"//third_party/abseil-cpp/absl/types:optional",
]
}
diff --git a/video/adaptation/video_stream_adapter.cc b/video/adaptation/video_stream_adapter.cc
index 87f7e90..8b589c3 100644
--- a/video/adaptation/video_stream_adapter.cc
+++ b/video/adaptation/video_stream_adapter.cc
@@ -14,50 +14,87 @@
#include <limits>
#include "absl/types/optional.h"
+#include "api/video_codecs/video_encoder.h"
#include "rtc_base/constructor_magic.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/safe_conversions.h"
namespace webrtc {
+namespace {
+
+const int kMinFramerateFps = 2;
+
+// Generate suggested higher and lower frame rates and resolutions, to be
+// applied to the VideoSourceRestrictor. These are used in "maintain-resolution"
+// and "maintain-framerate". The "balanced" degradation preference also makes
+// use of BalancedDegradationPreference when generating suggestions. The
+// VideoSourceRestrictor decidedes whether or not a proposed adaptation is
+// valid.
+
+// For frame rate, the steps we take are 2/3 (down) and 3/2 (up).
+int GetLowerFrameRateThan(int fps) {
+ RTC_DCHECK(fps != std::numeric_limits<int>::max());
+ return (fps * 2) / 3;
+}
+// TODO(hbos): Use absl::optional<> instead?
+int GetHigherFrameRateThan(int fps) {
+ return fps != std::numeric_limits<int>::max()
+ ? (fps * 3) / 2
+ : std::numeric_limits<int>::max();
+}
+
+// For resolution, the steps we take are 3/5 (down) and 5/3 (up).
+// Notice the asymmetry of which restriction property is set depending on if
+// we are adapting up or down:
+// - VideoSourceRestrictor::DecreaseResolution() sets the max_pixels_per_frame()
+// to the desired target and target_pixels_per_frame() to null.
+// - VideoSourceRestrictor::IncreaseResolutionTo() sets the
+// target_pixels_per_frame() to the desired target, and max_pixels_per_frame()
+// is set according to VideoSourceRestrictor::GetIncreasedMaxPixelsWanted().
+int GetLowerResolutionThan(int pixel_count) {
+ RTC_DCHECK(pixel_count != std::numeric_limits<int>::max());
+ return (pixel_count * 3) / 5;
+}
+// TODO(hbos): Use absl::optional<> instead?
+int GetHigherResolutionThan(int pixel_count) {
+ return pixel_count != std::numeric_limits<int>::max()
+ ? (pixel_count * 5) / 3
+ : std::numeric_limits<int>::max();
+}
+
+// One of the conditions used in VideoStreamAdapter::GetAdaptUpTarget().
+// TODO(hbos): Whether or not we can adapt up due to encoder settings and
+// bitrate should be expressed as a bandwidth-related Resource.
+bool CanAdaptUpResolution(
+ const absl::optional<EncoderSettings>& encoder_settings,
+ absl::optional<uint32_t> encoder_target_bitrate_bps,
+ int input_pixels) {
+ uint32_t bitrate_bps = encoder_target_bitrate_bps.value_or(0);
+ absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
+ encoder_settings.has_value()
+ ? encoder_settings->encoder_info()
+ .GetEncoderBitrateLimitsForResolution(
+ GetHigherResolutionThan(input_pixels))
+ : absl::nullopt;
+ if (!bitrate_limits.has_value() || bitrate_bps == 0) {
+ return true; // No limit configured or bitrate provided.
+ }
+ RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, input_pixels);
+ return bitrate_bps >=
+ static_cast<uint32_t>(bitrate_limits->min_start_bitrate_bps);
+}
+
+} // namespace
+
+VideoStreamAdapter::AdaptationTarget::AdaptationTarget(AdaptationAction action,
+ int value)
+ : action(action), value(value) {}
+
// VideoSourceRestrictor is responsible for keeping track of current
-// VideoSourceRestrictions. It suggests higher and lower frame rates and
-// resolutions (used by "maintain-resolution" and "maintain-framerate"), but is
-// ultimately not reponsible for determining when or how we should adapt up or
-// down (e.g. "balanced" mode also uses BalancedDegradationPreference).
+// VideoSourceRestrictions.
class VideoStreamAdapter::VideoSourceRestrictor {
public:
- // For frame rate, the steps we take are 2/3 (down) and 3/2 (up).
- static int GetLowerFrameRateThan(int fps) {
- RTC_DCHECK(fps != std::numeric_limits<int>::max());
- return (fps * 2) / 3;
- }
- // TODO(hbos): Use absl::optional<> instead?
- static int GetHigherFrameRateThan(int fps) {
- return fps != std::numeric_limits<int>::max()
- ? (fps * 3) / 2
- : std::numeric_limits<int>::max();
- }
-
- // For resolution, the steps we take are 3/5 (down) and 5/3 (up).
- // Notice the asymmetry of which restriction property is set depending on if
- // we are adapting up or down:
- // - DecreaseResolution() sets the max_pixels_per_frame() to the desired
- // target and target_pixels_per_frame() to null.
- // - IncreaseResolutionTo() sets the target_pixels_per_frame() to the desired
- // target, and max_pixels_per_frame() is set according to
- // GetIncreasedMaxPixelsWanted().
- static int GetLowerResolutionThan(int pixel_count) {
- RTC_DCHECK(pixel_count != std::numeric_limits<int>::max());
- return (pixel_count * 3) / 5;
- }
- // TODO(hbos): Use absl::optional<> instead?
- static int GetHigherResolutionThan(int pixel_count) {
- return pixel_count != std::numeric_limits<int>::max()
- ? (pixel_count * 5) / 3
- : std::numeric_limits<int>::max();
- }
-
VideoSourceRestrictor() {}
VideoSourceRestrictions source_restrictions() const {
@@ -168,30 +205,27 @@
RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceRestrictor);
};
-const int VideoStreamAdapter::kMinFramerateFps = 2;
-
// static
-int VideoStreamAdapter::GetLowerFrameRateThan(int fps) {
- return VideoSourceRestrictor::GetLowerFrameRateThan(fps);
-}
-
-// static
-int VideoStreamAdapter::GetHigherFrameRateThan(int fps) {
- return VideoSourceRestrictor::GetHigherFrameRateThan(fps);
-}
-
-// static
-int VideoStreamAdapter::GetLowerResolutionThan(int pixel_count) {
- return VideoSourceRestrictor::GetLowerResolutionThan(pixel_count);
-}
-
-// static
-int VideoStreamAdapter::GetHigherResolutionThan(int pixel_count) {
- return VideoSourceRestrictor::GetHigherResolutionThan(pixel_count);
+VideoStreamAdapter::AdaptationRequest::Mode
+VideoStreamAdapter::AdaptationRequest::GetModeFromAdaptationAction(
+ VideoStreamAdapter::AdaptationAction action) {
+ switch (action) {
+ case AdaptationAction::kIncreaseResolution:
+ return AdaptationRequest::Mode::kAdaptUp;
+ case AdaptationAction::kDecreaseResolution:
+ return AdaptationRequest::Mode::kAdaptDown;
+ case AdaptationAction::kIncreaseFrameRate:
+ return AdaptationRequest::Mode::kAdaptUp;
+ case AdaptationAction::kDecreaseFrameRate:
+ return AdaptationRequest::Mode::kAdaptDown;
+ }
}
VideoStreamAdapter::VideoStreamAdapter()
- : source_restrictor_(std::make_unique<VideoSourceRestrictor>()) {}
+ : source_restrictor_(std::make_unique<VideoSourceRestrictor>()),
+ balanced_settings_(),
+ degradation_preference_(DegradationPreference::DISABLED),
+ last_adaptation_request_(absl::nullopt) {}
VideoStreamAdapter::~VideoStreamAdapter() {}
@@ -203,43 +237,257 @@
return source_restrictor_->adaptation_counters();
}
+const BalancedDegradationSettings& VideoStreamAdapter::balanced_settings()
+ const {
+ return balanced_settings_;
+}
+
void VideoStreamAdapter::ClearRestrictions() {
source_restrictor_->ClearRestrictions();
+ last_adaptation_request_.reset();
}
-bool VideoStreamAdapter::CanDecreaseResolutionTo(int target_pixels,
- int min_pixels_per_frame) {
- return source_restrictor_->CanDecreaseResolutionTo(target_pixels,
- min_pixels_per_frame);
+VideoStreamAdapter::SetDegradationPreferenceResult
+VideoStreamAdapter::SetDegradationPreference(
+ DegradationPreference degradation_preference) {
+ bool did_clear = false;
+ if (degradation_preference_ != degradation_preference) {
+ if (degradation_preference == DegradationPreference::BALANCED ||
+ degradation_preference_ == DegradationPreference::BALANCED) {
+ ClearRestrictions();
+ did_clear = true;
+ }
+ }
+ degradation_preference_ = degradation_preference;
+ return did_clear ? SetDegradationPreferenceResult::kRestrictionsCleared
+ : SetDegradationPreferenceResult::kRestrictionsNotCleared;
}
-void VideoStreamAdapter::DecreaseResolutionTo(int target_pixels,
- int min_pixels_per_frame) {
- source_restrictor_->DecreaseResolutionTo(target_pixels, min_pixels_per_frame);
+DegradationPreference VideoStreamAdapter::EffectiveDegradationPreference(
+ VideoInputMode input_mode) const {
+ // Balanced mode for screenshare works via automatic animation detection:
+ // Resolution is capped for fullscreen animated content.
+ // Adapatation is done only via framerate downgrade.
+ // Thus effective degradation preference is MAINTAIN_RESOLUTION.
+ // TODO(hbos): Don't do this. This is not what "balanced" means. If the
+ // application wants to maintain resolution it should set that degradation
+ // preference rather than depend on non-standard behaviors.
+ return (input_mode == VideoInputMode::kScreenshareVideo &&
+ degradation_preference_ == DegradationPreference::BALANCED)
+ ? DegradationPreference::MAINTAIN_RESOLUTION
+ : degradation_preference_;
}
-bool VideoStreamAdapter::CanIncreaseResolutionTo(int target_pixels) {
- return source_restrictor_->CanIncreaseResolutionTo(target_pixels);
+absl::optional<VideoStreamAdapter::AdaptationTarget>
+VideoStreamAdapter::GetAdaptUpTarget(
+ const absl::optional<EncoderSettings>& encoder_settings,
+ absl::optional<uint32_t> encoder_target_bitrate_bps,
+ VideoInputMode input_mode,
+ int input_pixels,
+ int input_fps,
+ AdaptationObserverInterface::AdaptReason reason) const {
+ // Preconditions for being able to adapt up:
+ if (input_mode == VideoInputMode::kNoVideo)
+ return absl::nullopt;
+ // 1. We shouldn't adapt up if we're currently waiting for a previous upgrade
+ // to have an effect.
+ // TODO(hbos): What about in the case of other degradation preferences?
+ bool last_adaptation_was_up =
+ last_adaptation_request_ &&
+ last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
+ if (last_adaptation_was_up &&
+ degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
+ input_pixels <= last_adaptation_request_->input_pixel_count_) {
+ return absl::nullopt;
+ }
+ // 2. We shouldn't adapt up if BalancedSettings doesn't allow it, which is
+ // only applicable if reason is kQuality and preference is BALANCED.
+ if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
+ EffectiveDegradationPreference(input_mode) ==
+ DegradationPreference::BALANCED &&
+ !balanced_settings_.CanAdaptUp(
+ GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels,
+ encoder_target_bitrate_bps.value_or(0))) {
+ return absl::nullopt;
+ }
+
+ // Attempt to find an allowed adaptation target.
+ switch (EffectiveDegradationPreference(input_mode)) {
+ case DegradationPreference::BALANCED: {
+ // Attempt to increase target frame rate.
+ int target_fps = balanced_settings_.MaxFps(
+ GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels);
+ if (source_restrictor_->CanIncreaseFrameRateTo(target_fps)) {
+ return AdaptationTarget(AdaptationAction::kIncreaseFrameRate,
+ target_fps);
+ }
+ // Fall-through to maybe-adapting resolution, unless |balanced_settings_|
+ // forbids it based on bitrate.
+ if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
+ !balanced_settings_.CanAdaptUpResolution(
+ GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels,
+ encoder_target_bitrate_bps.value_or(0))) {
+ return absl::nullopt;
+ }
+ // Scale up resolution.
+ ABSL_FALLTHROUGH_INTENDED;
+ }
+ case DegradationPreference::MAINTAIN_FRAMERATE: {
+ // Don't adapt resolution if CanAdaptUpResolution() forbids it based on
+ // bitrate and limits specified by encoder capabilities.
+ if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
+ !CanAdaptUpResolution(encoder_settings, encoder_target_bitrate_bps,
+ input_pixels)) {
+ return absl::nullopt;
+ }
+ // Attempt to increase pixel count.
+ int target_pixels = input_pixels;
+ if (source_restrictor_->adaptation_counters().resolution_adaptations ==
+ 1) {
+ RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting.";
+ target_pixels = std::numeric_limits<int>::max();
+ }
+ target_pixels = GetHigherResolutionThan(target_pixels);
+ if (!source_restrictor_->CanIncreaseResolutionTo(target_pixels))
+ return absl::nullopt;
+ return AdaptationTarget(AdaptationAction::kIncreaseResolution,
+ target_pixels);
+ }
+ case DegradationPreference::MAINTAIN_RESOLUTION: {
+ // Scale up framerate.
+ int target_fps = input_fps;
+ if (source_restrictor_->adaptation_counters().fps_adaptations == 1) {
+ RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
+ target_fps = std::numeric_limits<int>::max();
+ }
+ target_fps = GetHigherFrameRateThan(target_fps);
+ if (!source_restrictor_->CanIncreaseFrameRateTo(target_fps))
+ return absl::nullopt;
+ return AdaptationTarget(AdaptationAction::kIncreaseFrameRate, target_fps);
+ }
+ case DegradationPreference::DISABLED:
+ return absl::nullopt;
+ }
}
-void VideoStreamAdapter::IncreaseResolutionTo(int target_pixels) {
- source_restrictor_->IncreaseResolutionTo(target_pixels);
+absl::optional<VideoStreamAdapter::AdaptationTarget>
+VideoStreamAdapter::GetAdaptDownTarget(
+ const absl::optional<EncoderSettings>& encoder_settings,
+ VideoInputMode input_mode,
+ int input_pixels,
+ int input_fps,
+ int min_pixels_per_frame,
+ VideoStreamEncoderObserver* encoder_stats_observer) const {
+ // Preconditions for being able to adapt down:
+ if (input_mode == VideoInputMode::kNoVideo)
+ return absl::nullopt;
+ // 1. We are not disabled.
+ // TODO(hbos): Don't support DISABLED, it doesn't exist in the spec and it
+ // causes scaling due to bandwidth constraints (QualityScalerResource) to be
+ // ignored, not just CPU signals. This is not a use case we want to support
+ // long-term; remove this enum value.
+ if (degradation_preference_ == DegradationPreference::DISABLED)
+ return absl::nullopt;
+ bool last_adaptation_was_down =
+ last_adaptation_request_ &&
+ last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown;
+ // 2. We shouldn't adapt down if our frame rate is below the minimum or if its
+ // currently unknown.
+ if (EffectiveDegradationPreference(input_mode) ==
+ DegradationPreference::MAINTAIN_RESOLUTION) {
+ // TODO(hbos): This usage of |last_adaptation_was_down| looks like a mistake
+ // - delete it.
+ if (input_fps <= 0 ||
+ (last_adaptation_was_down && input_fps < kMinFramerateFps)) {
+ return absl::nullopt;
+ }
+ }
+ // 3. We shouldn't adapt down if we're currently waiting for a previous
+ // downgrade to have an effect.
+ // TODO(hbos): What about in the case of other degradation preferences?
+ if (last_adaptation_was_down &&
+ degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
+ input_pixels >= last_adaptation_request_->input_pixel_count_) {
+ return absl::nullopt;
+ }
+
+ // Attempt to find an allowed adaptation target.
+ switch (EffectiveDegradationPreference(input_mode)) {
+ case DegradationPreference::BALANCED: {
+ // Try scale down framerate, if lower.
+ int target_fps = balanced_settings_.MinFps(
+ GetVideoCodecTypeOrGeneric(encoder_settings), input_pixels);
+ if (source_restrictor_->CanDecreaseFrameRateTo(target_fps)) {
+ return AdaptationTarget(AdaptationAction::kDecreaseFrameRate,
+ target_fps);
+ }
+ // Scale down resolution.
+ ABSL_FALLTHROUGH_INTENDED;
+ }
+ case DegradationPreference::MAINTAIN_FRAMERATE: {
+ // Scale down resolution.
+ int target_pixels = GetLowerResolutionThan(input_pixels);
+ // TODO(https://crbug.com/webrtc/11393): Move this logic to
+ // ApplyAdaptationTarget() or elsewhere - simply checking which adaptation
+ // target is available should not have side-effects.
+ if (target_pixels < min_pixels_per_frame)
+ encoder_stats_observer->OnMinPixelLimitReached();
+ if (!source_restrictor_->CanDecreaseResolutionTo(target_pixels,
+ min_pixels_per_frame)) {
+ return absl::nullopt;
+ }
+ return AdaptationTarget(AdaptationAction::kDecreaseResolution,
+ target_pixels);
+ }
+ case DegradationPreference::MAINTAIN_RESOLUTION: {
+ int target_fps = GetLowerFrameRateThan(input_fps);
+ if (!source_restrictor_->CanDecreaseFrameRateTo(target_fps))
+ return absl::nullopt;
+ return AdaptationTarget(AdaptationAction::kDecreaseFrameRate, target_fps);
+ }
+ case DegradationPreference::DISABLED:
+ RTC_NOTREACHED();
+ return absl::nullopt;
+ }
}
-bool VideoStreamAdapter::CanDecreaseFrameRateTo(int max_frame_rate) {
- return source_restrictor_->CanDecreaseFrameRateTo(max_frame_rate);
-}
-
-void VideoStreamAdapter::DecreaseFrameRateTo(int max_frame_rate) {
- source_restrictor_->DecreaseFrameRateTo(max_frame_rate);
-}
-
-bool VideoStreamAdapter::CanIncreaseFrameRateTo(int max_frame_rate) {
- return source_restrictor_->CanIncreaseFrameRateTo(max_frame_rate);
-}
-
-void VideoStreamAdapter::IncreaseFrameRateTo(int max_frame_rate) {
- source_restrictor_->IncreaseFrameRateTo(max_frame_rate);
+void VideoStreamAdapter::ApplyAdaptationTarget(const AdaptationTarget& target,
+ VideoInputMode input_mode,
+ int input_pixels,
+ int input_fps,
+ int min_pixels_per_frame) {
+ // Remember the input pixels and fps of this adaptation. Used to avoid
+ // adapting again before this adaptation has had an effect.
+ last_adaptation_request_.emplace(AdaptationRequest{
+ input_pixels, input_fps,
+ AdaptationRequest::GetModeFromAdaptationAction(target.action)});
+ switch (target.action) {
+ case AdaptationAction::kIncreaseResolution:
+ source_restrictor_->IncreaseResolutionTo(target.value);
+ return;
+ case AdaptationAction::kDecreaseResolution:
+ source_restrictor_->DecreaseResolutionTo(target.value,
+ min_pixels_per_frame);
+ return;
+ case AdaptationAction::kIncreaseFrameRate:
+ source_restrictor_->IncreaseFrameRateTo(target.value);
+ // TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps.
+ // GetAdaptUpTarget() should tell us the correct value, but BALANCED logic
+ // in DecrementFramerate() makes it hard to predict whether this will be
+ // the last step. Remove the dependency on GetConstAdaptCounter().
+ if (EffectiveDegradationPreference(input_mode) ==
+ DegradationPreference::BALANCED &&
+ source_restrictor_->adaptation_counters().fps_adaptations == 0 &&
+ target.value != std::numeric_limits<int>::max()) {
+ RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
+ source_restrictor_->IncreaseFrameRateTo(
+ std::numeric_limits<int>::max());
+ }
+ return;
+ case AdaptationAction::kDecreaseFrameRate:
+ source_restrictor_->DecreaseFrameRateTo(target.value);
+ return;
+ }
}
} // namespace webrtc
diff --git a/video/adaptation/video_stream_adapter.h b/video/adaptation/video_stream_adapter.h
index ddcd6f5..40e35ec 100644
--- a/video/adaptation/video_stream_adapter.h
+++ b/video/adaptation/video_stream_adapter.h
@@ -13,7 +13,13 @@
#include <memory>
+#include "absl/types/optional.h"
+#include "api/rtp_parameters.h"
+#include "api/video/video_stream_encoder_observer.h"
+#include "call/adaptation/encoder_settings.h"
#include "call/adaptation/video_source_restrictions.h"
+#include "modules/video_coding/utility/quality_scaler.h"
+#include "rtc_base/experiments/balanced_degradation_settings.h"
#include "video/adaptation/adaptation_counters.h"
namespace webrtc {
@@ -26,39 +32,128 @@
// 3. Modify the stream's restrictions in one of the valid ways.
class VideoStreamAdapter {
public:
- static const int kMinFramerateFps;
+ enum class SetDegradationPreferenceResult {
+ kRestrictionsNotCleared,
+ kRestrictionsCleared,
+ };
- static int GetLowerFrameRateThan(int fps);
- static int GetHigherFrameRateThan(int fps);
- static int GetLowerResolutionThan(int pixel_count);
- static int GetHigherResolutionThan(int pixel_count);
+ enum class VideoInputMode {
+ kNoVideo,
+ kNormalVideo,
+ kScreenshareVideo,
+ };
+
+ enum class AdaptationAction {
+ kIncreaseResolution,
+ kDecreaseResolution,
+ kIncreaseFrameRate,
+ kDecreaseFrameRate,
+ };
+
+ // Describes an adaptation step: increasing or decreasing resolution or frame
+ // rate to a given value.
+ // TODO(https://crbug.com/webrtc/11393): Make these private implementation
+ // details, and expose something that allows you to inspect the
+ // VideoSourceRestrictions instead. The adaptation steps could be expressed as
+ // a graph, for instance.
+ struct AdaptationTarget {
+ AdaptationTarget(AdaptationAction action, int value);
+ // Which action the VideoSourceRestrictor needs to take.
+ const AdaptationAction action;
+ // Target pixel count or frame rate depending on |action|.
+ const int value;
+
+ // Allow this struct to be instantiated as an optional, even though it's in
+ // a private namespace.
+ friend class absl::optional<AdaptationTarget>;
+ };
VideoStreamAdapter();
~VideoStreamAdapter();
- // TODO(hbos): Why isn't this const?
VideoSourceRestrictions source_restrictions() const;
const AdaptationCounters& adaptation_counters() const;
+ // TODO(hbos): Can we get rid of any external dependencies on
+ // BalancedDegradationPreference? How the adaptor generates possible next
+ // steps for adaptation should be an implementation detail. Can the relevant
+ // information be inferred from GetAdaptUpTarget()/GetAdaptDownTarget()?
+ const BalancedDegradationSettings& balanced_settings() const;
void ClearRestrictions();
- // "Can adapt?" and "do adapt!" methods.
- // TODO(https://crbug.com/webrtc/11393): Make the adapter responsible for
- // deciding what the next step are, i.e. taking on degradation preference
- // logic. Then, these can be expressed either as CanAdaptUp() and DoAdaptUp()
- // or as GetNextRestrictionsUp() and ApplyRestrictions().
- bool CanDecreaseResolutionTo(int target_pixels, int min_pixels_per_frame);
- void DecreaseResolutionTo(int target_pixels, int min_pixels_per_frame);
- bool CanIncreaseResolutionTo(int target_pixels);
- void IncreaseResolutionTo(int target_pixels);
- bool CanDecreaseFrameRateTo(int max_frame_rate);
- void DecreaseFrameRateTo(int max_frame_rate);
- bool CanIncreaseFrameRateTo(int max_frame_rate);
- void IncreaseFrameRateTo(int max_frame_rate);
+ // TODO(hbos): Setting the degradation preference should not clear
+ // restrictions! This is not defined in the spec and is unexpected, there is a
+ // tiny risk that people would discover and rely on this behavior.
+ SetDegradationPreferenceResult SetDegradationPreference(
+ DegradationPreference degradation_preference);
+ // TODO(hbos): This is only used in one place externally by
+ // OveruseFrameDetectorResourceAdaptationModule - can we get rid of that
+ // usage? This is exposing an implementation detail.
+ DegradationPreference EffectiveDegradationPreference(
+ VideoInputMode input_mode) const;
+
+ // Returns a target that we are guaranteed to be able to adapt to, or null if
+ // adaptation is not desired or not possible.
+ absl::optional<AdaptationTarget> GetAdaptUpTarget(
+ const absl::optional<EncoderSettings>& encoder_settings,
+ absl::optional<uint32_t> encoder_target_bitrate_bps,
+ VideoInputMode input_mode,
+ int input_pixels,
+ int input_fps,
+ AdaptationObserverInterface::AdaptReason reason) const;
+ // TODO(https://crbug.com/webrtc/11393): Remove the dependency on
+ // |encoder_stats_observer| - simply checking which adaptation target is
+ // available should not have side-effects.
+ absl::optional<AdaptationTarget> GetAdaptDownTarget(
+ const absl::optional<EncoderSettings>& encoder_settings,
+ VideoInputMode input_mode,
+ int input_pixels,
+ int input_fps,
+ int min_pixels_per_frame,
+ VideoStreamEncoderObserver* encoder_stats_observer) const;
+ // Applies the |target| to |source_restrictor_|.
+ void ApplyAdaptationTarget(const AdaptationTarget& target,
+ VideoInputMode input_mode,
+ int input_pixels,
+ int input_fps,
+ int min_pixels_per_frame);
private:
class VideoSourceRestrictor;
+ // The input frame rate and resolution at the time of an adaptation in the
+ // direction described by |mode_| (up or down).
+ // TODO(https://crbug.com/webrtc/11393): Can this be renamed? Can this be
+ // merged with AdaptationTarget?
+ struct AdaptationRequest {
+ // The pixel count produced by the source at the time of the adaptation.
+ int input_pixel_count_;
+ // Framerate received from the source at the time of the adaptation.
+ int framerate_fps_;
+ // Indicates if request was to adapt up or down.
+ enum class Mode { kAdaptUp, kAdaptDown } mode_;
+
+ // This is a static method rather than an anonymous namespace function due
+ // to namespace visiblity.
+ static Mode GetModeFromAdaptationAction(AdaptationAction action);
+ };
+
+ // Owner and modifier of the VideoSourceRestriction of this stream adaptor.
const std::unique_ptr<VideoSourceRestrictor> source_restrictor_;
+ // Decides the next adaptation target in DegradationPreference::BALANCED.
+ const BalancedDegradationSettings balanced_settings_;
+ // When deciding the next target up or down, different strategies are used
+ // depending on the DegradationPreference.
+ // https://w3c.github.io/mst-content-hint/#dom-rtcdegradationpreference
+ DegradationPreference degradation_preference_;
+
+ // The input frame rate, resolution and adaptation direction of the last
+ // ApplyAdaptationTarget(). Used to avoid adapting twice if a recent
+ // adaptation has not had an effect on the input frame rate or resolution yet.
+ // TODO(hbos): Can we implement a more general "cooldown" mechanism of
+ // resources intead? If we already have adapted it seems like we should wait
+ // a while before adapting again, so that we are not acting on usage
+ // measurements that are made obsolete/unreliable by an "ongoing" adaptation.
+ absl::optional<AdaptationRequest> last_adaptation_request_;
};
} // namespace webrtc
diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc
index c203b8f..d86b7d9 100644
--- a/video/overuse_frame_detector_resource_adaptation_module.cc
+++ b/video/overuse_frame_detector_resource_adaptation_module.cc
@@ -164,10 +164,6 @@
int initial_framedrop_;
};
-OveruseFrameDetectorResourceAdaptationModule::AdaptationTarget::
- AdaptationTarget(AdaptationAction action, int value)
- : action(action), value(value) {}
-
OveruseFrameDetectorResourceAdaptationModule::
OveruseFrameDetectorResourceAdaptationModule(
Clock* clock,
@@ -181,8 +177,6 @@
experiment_cpu_load_estimator_(experiment_cpu_load_estimator),
has_input_video_(false),
degradation_preference_(DegradationPreference::DISABLED),
- balanced_settings_(),
- last_adaptation_request_(absl::nullopt),
stream_adapter_(std::make_unique<VideoStreamAdapter>()),
encode_usage_resource_(
std::make_unique<EncodeUsageResource>(std::move(overuse_detector))),
@@ -260,17 +254,12 @@
void OveruseFrameDetectorResourceAdaptationModule::SetDegradationPreference(
DegradationPreference degradation_preference) {
- if (degradation_preference_ != degradation_preference) {
- // Reset adaptation state, so that we're not tricked into thinking there's
- // an already pending request of the same type.
- last_adaptation_request_.reset();
- if (degradation_preference == DegradationPreference::BALANCED ||
- degradation_preference_ == DegradationPreference::BALANCED) {
- stream_adapter_->ClearRestrictions();
- active_counts_.fill(AdaptationCounters());
- }
- }
degradation_preference_ = degradation_preference;
+ if (stream_adapter_->SetDegradationPreference(degradation_preference) ==
+ VideoStreamAdapter::SetDegradationPreferenceResult::
+ kRestrictionsCleared) {
+ active_counts_.fill(AdaptationCounters());
+ }
MaybeUpdateVideoSourceRestrictions();
}
@@ -307,7 +296,6 @@
void OveruseFrameDetectorResourceAdaptationModule::
ResetVideoSourceRestrictions() {
- last_adaptation_request_.reset();
stream_adapter_->ClearRestrictions();
active_counts_.fill(AdaptationCounters());
MaybeUpdateVideoSourceRestrictions();
@@ -397,7 +385,7 @@
absl::optional<VideoEncoder::QpThresholds> experimental_thresholds;
if (quality_scaling_experiment_enabled_) {
experimental_thresholds = QualityScalingExperiment::GetQpThresholds(
- GetVideoCodecTypeOrGeneric());
+ GetVideoCodecTypeOrGeneric(encoder_settings_));
}
UpdateQualityScalerSettings(experimental_thresholds
? *experimental_thresholds
@@ -411,8 +399,9 @@
if (degradation_preference_ == DegradationPreference::BALANCED &&
quality_scaler_resource_->is_started()) {
absl::optional<VideoEncoder::QpThresholds> thresholds =
- balanced_settings_.GetQpThresholds(GetVideoCodecTypeOrGeneric(),
- LastInputFrameSizeOrDefault());
+ stream_adapter_->balanced_settings().GetQpThresholds(
+ GetVideoCodecTypeOrGeneric(encoder_settings_),
+ LastInputFrameSizeOrDefault());
if (thresholds) {
quality_scaler_resource_->SetQpThresholds(*thresholds);
}
@@ -454,212 +443,51 @@
}
}
-absl::optional<OveruseFrameDetectorResourceAdaptationModule::AdaptationTarget>
+absl::optional<VideoStreamAdapter::AdaptationTarget>
OveruseFrameDetectorResourceAdaptationModule::GetAdaptUpTarget(
int input_pixels,
int input_fps,
AdaptationObserverInterface::AdaptReason reason) const {
- // Preconditions for being able to adapt up:
- if (!has_input_video_)
- return absl::nullopt;
- // 1. We can't adapt up if we're already at the highest setting.
+ // We can't adapt up if we're already at the highest setting.
// Note that this only includes counts relevant to the current degradation
// preference. e.g. we previously adapted resolution, now prefer adpating fps,
// only count the fps adaptations and not the previous resolution adaptations.
+ //
// TODO(https://crbug.com/webrtc/11394): Checking the counts for reason should
- // be replaced with checking the overuse state of all resources.
+ // be replaced with checking the overuse state of all resources. This is
+ // effectively trying to infer if the the Resource specified by |reason| is OK
+ // with adapting up by looking at active counters. If the relevant Resources
+ // simply told us this directly we wouldn't have to depend on stats counters
+ // to abort GetAdaptUpTarget().
int num_downgrades = ApplyDegradationPreference(active_counts_[reason],
degradation_preference_)
.Total();
RTC_DCHECK_GE(num_downgrades, 0);
if (num_downgrades == 0)
return absl::nullopt;
- // 2. We shouldn't adapt up if we're currently waiting for a previous upgrade
- // to have an effect.
- // TODO(hbos): What about in the case of other degradation preferences?
- bool last_adaptation_was_up =
- last_adaptation_request_ &&
- last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
- if (last_adaptation_was_up &&
- degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
- input_pixels <= last_adaptation_request_->input_pixel_count_) {
- return absl::nullopt;
- }
- // 3. We shouldn't adapt up if BalancedSettings doesn't allow it, which is
- // only applicable if reason is kQuality and preference is BALANCED.
- if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
- EffectiveDegradationPreference() == DegradationPreference::BALANCED &&
- !balanced_settings_.CanAdaptUp(GetVideoCodecTypeOrGeneric(), input_pixels,
- encoder_target_bitrate_bps_.value_or(0))) {
- return absl::nullopt;
- }
-
- // Attempt to find an allowed adaptation target.
- switch (EffectiveDegradationPreference()) {
- case DegradationPreference::BALANCED: {
- // Attempt to increase target frame rate.
- int target_fps =
- balanced_settings_.MaxFps(GetVideoCodecTypeOrGeneric(), input_pixels);
- if (stream_adapter_->CanIncreaseFrameRateTo(target_fps)) {
- return AdaptationTarget(AdaptationAction::kIncreaseFrameRate,
- target_fps);
- }
- // Fall-through to maybe-adapting resolution, unless |balanced_settings_|
- // forbids it based on bitrate.
- if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
- !balanced_settings_.CanAdaptUpResolution(
- GetVideoCodecTypeOrGeneric(), input_pixels,
- encoder_target_bitrate_bps_.value_or(0))) {
- return absl::nullopt;
- }
- // Scale up resolution.
- ABSL_FALLTHROUGH_INTENDED;
- }
- case DegradationPreference::MAINTAIN_FRAMERATE: {
- // Don't adapt resolution if CanAdaptUpResolution() forbids it based on
- // bitrate and limits specified by encoder capabilities.
- if (reason == AdaptationObserverInterface::AdaptReason::kQuality &&
- !CanAdaptUpResolution(input_pixels,
- encoder_target_bitrate_bps_.value_or(0))) {
- return absl::nullopt;
- }
- // Attempt to increase pixel count.
- int target_pixels = input_pixels;
- if (stream_adapter_->adaptation_counters().resolution_adaptations == 1) {
- RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting.";
- target_pixels = std::numeric_limits<int>::max();
- }
- target_pixels =
- VideoStreamAdapter::GetHigherResolutionThan(target_pixels);
- if (!stream_adapter_->CanIncreaseResolutionTo(target_pixels))
- return absl::nullopt;
- return AdaptationTarget(AdaptationAction::kIncreaseResolution,
- target_pixels);
- }
- case DegradationPreference::MAINTAIN_RESOLUTION: {
- // Scale up framerate.
- int target_fps = input_fps;
- if (stream_adapter_->adaptation_counters().fps_adaptations == 1) {
- RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
- target_fps = std::numeric_limits<int>::max();
- }
- target_fps = VideoStreamAdapter::GetHigherFrameRateThan(target_fps);
- if (!stream_adapter_->CanIncreaseFrameRateTo(target_fps))
- return absl::nullopt;
- return AdaptationTarget(AdaptationAction::kIncreaseFrameRate, target_fps);
- }
- case DegradationPreference::DISABLED:
- return absl::nullopt;
- }
+ return stream_adapter_->GetAdaptUpTarget(
+ encoder_settings_, encoder_target_bitrate_bps_, GetVideoInputMode(),
+ input_pixels, input_fps, reason);
}
-absl::optional<OveruseFrameDetectorResourceAdaptationModule::AdaptationTarget>
+absl::optional<VideoStreamAdapter::AdaptationTarget>
OveruseFrameDetectorResourceAdaptationModule::GetAdaptDownTarget(
int input_pixels,
int input_fps,
int min_pixels_per_frame) const {
- // Preconditions for being able to adapt down:
- if (!has_input_video_)
- return absl::nullopt;
- // 1. We are not disabled.
- // TODO(hbos): Don't support DISABLED, it doesn't exist in the spec and it
- // causes scaling due to bandwidth constraints (QualityScalerResource) to be
- // ignored, not just CPU signals. This is not a use case we want to support;
- // remove the enum value.
- if (degradation_preference_ == DegradationPreference::DISABLED)
- return absl::nullopt;
- bool last_adaptation_was_down =
- last_adaptation_request_ &&
- last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown;
- // 2. We shouldn't adapt down if our frame rate is below the minimum or if its
- // currently unknown.
- if (EffectiveDegradationPreference() ==
- DegradationPreference::MAINTAIN_RESOLUTION) {
- // TODO(hbos): This usage of |last_adaptation_was_down| looks like a mistake
- // - delete it.
- if (input_fps <= 0 || (last_adaptation_was_down &&
- input_fps < VideoStreamAdapter::kMinFramerateFps)) {
- return absl::nullopt;
- }
- }
- // 3. We shouldn't adapt down if we're currently waiting for a previous
- // downgrade to have an effect.
- // TODO(hbos): What about in the case of other degradation preferences?
- if (last_adaptation_was_down &&
- degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE &&
- input_pixels >= last_adaptation_request_->input_pixel_count_) {
- return absl::nullopt;
- }
-
- // Attempt to find an allowed adaptation target.
- switch (EffectiveDegradationPreference()) {
- case DegradationPreference::BALANCED: {
- // Try scale down framerate, if lower.
- int target_fps =
- balanced_settings_.MinFps(GetVideoCodecTypeOrGeneric(), input_pixels);
- if (stream_adapter_->CanDecreaseFrameRateTo(target_fps)) {
- return AdaptationTarget(AdaptationAction::kDecreaseFrameRate,
- target_fps);
- }
- // Scale down resolution.
- ABSL_FALLTHROUGH_INTENDED;
- }
- case DegradationPreference::MAINTAIN_FRAMERATE: {
- // Scale down resolution.
- int target_pixels =
- VideoStreamAdapter::GetLowerResolutionThan(input_pixels);
- // TODO(https://crbug.com/webrtc/11222): Move this logic to
- // ApplyAdaptationTarget() or elsewhere - simply checking which adaptation
- // target is available should not have side-effects.
- if (target_pixels < min_pixels_per_frame)
- encoder_stats_observer_->OnMinPixelLimitReached();
- if (!stream_adapter_->CanDecreaseResolutionTo(target_pixels,
- min_pixels_per_frame)) {
- return absl::nullopt;
- }
- return AdaptationTarget(AdaptationAction::kDecreaseResolution,
- target_pixels);
- }
- case DegradationPreference::MAINTAIN_RESOLUTION: {
- int target_fps = VideoStreamAdapter::GetLowerFrameRateThan(input_fps);
- if (!stream_adapter_->CanDecreaseFrameRateTo(target_fps))
- return absl::nullopt;
- return AdaptationTarget(AdaptationAction::kDecreaseFrameRate, target_fps);
- }
- case DegradationPreference::DISABLED:
- RTC_NOTREACHED();
- return absl::nullopt;
- }
+ return stream_adapter_->GetAdaptDownTarget(
+ encoder_settings_, GetVideoInputMode(), input_pixels, input_fps,
+ min_pixels_per_frame, encoder_stats_observer_);
}
void OveruseFrameDetectorResourceAdaptationModule::ApplyAdaptationTarget(
- const AdaptationTarget& target,
- int min_pixels_per_frame,
- AdaptationObserverInterface::AdaptReason reason) {
- switch (target.action) {
- case AdaptationAction::kIncreaseResolution:
- stream_adapter_->IncreaseResolutionTo(target.value);
- return;
- case AdaptationAction::kDecreaseResolution:
- stream_adapter_->DecreaseResolutionTo(target.value, min_pixels_per_frame);
- return;
- case AdaptationAction::kIncreaseFrameRate:
- stream_adapter_->IncreaseFrameRateTo(target.value);
- // TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps.
- // GetAdaptUpTarget() should tell us the correct value, but BALANCED logic
- // in DecrementFramerate() makes it hard to predict whether this will be
- // the last step. Remove the dependency on GetConstAdaptCounter().
- if (EffectiveDegradationPreference() == DegradationPreference::BALANCED &&
- stream_adapter_->adaptation_counters().fps_adaptations == 0 &&
- target.value != std::numeric_limits<int>::max()) {
- RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
- stream_adapter_->IncreaseFrameRateTo(std::numeric_limits<int>::max());
- }
- return;
- case AdaptationAction::kDecreaseFrameRate:
- stream_adapter_->DecreaseFrameRateTo(target.value);
- return;
- }
+ const VideoStreamAdapter::AdaptationTarget& target,
+ int input_pixels,
+ int input_fps,
+ int min_pixels_per_frame) {
+ stream_adapter_->ApplyAdaptationTarget(target, GetVideoInputMode(),
+ input_pixels, input_fps,
+ min_pixels_per_frame);
}
void OveruseFrameDetectorResourceAdaptationModule::OnResourceUnderuse(
@@ -668,14 +496,13 @@
int input_fps = encoder_stats_observer_->GetInputFrameRate();
int min_pixels_per_frame = MinPixelsPerFrame();
// Should we adapt, if so to what target?
- absl::optional<AdaptationTarget> target =
+ absl::optional<VideoStreamAdapter::AdaptationTarget> target =
GetAdaptUpTarget(input_pixels, input_fps, reason);
if (!target.has_value())
return;
// Apply target.
- ApplyAdaptationTarget(target.value(), min_pixels_per_frame, reason);
- last_adaptation_request_.emplace(AdaptationRequest{
- input_pixels, input_fps, AdaptationRequest::Mode::kAdaptUp});
+ ApplyAdaptationTarget(target.value(), input_pixels, input_fps,
+ min_pixels_per_frame);
// Update VideoSourceRestrictions based on adaptation. This also informs the
// |adaptation_listener_|.
MaybeUpdateVideoSourceRestrictions();
@@ -693,14 +520,13 @@
int input_fps = encoder_stats_observer_->GetInputFrameRate();
int min_pixels_per_frame = MinPixelsPerFrame();
// Should we adapt, if so to what target?
- absl::optional<AdaptationTarget> target =
+ absl::optional<VideoStreamAdapter::AdaptationTarget> target =
GetAdaptDownTarget(input_pixels, input_fps, min_pixels_per_frame);
if (!target.has_value())
return ResourceListenerResponse::kNothing;
// Apply target.
- ApplyAdaptationTarget(target.value(), min_pixels_per_frame, reason);
- last_adaptation_request_.emplace(AdaptationRequest{
- input_pixels, input_fps, AdaptationRequest::Mode::kAdaptDown});
+ ApplyAdaptationTarget(target.value(), input_pixels, input_fps,
+ min_pixels_per_frame);
// Update VideoSourceRestrictions based on adaptation. This also informs the
// |adaptation_listener_|.
MaybeUpdateVideoSourceRestrictions();
@@ -709,9 +535,12 @@
RTC_LOG(INFO) << ActiveCountsToString();
// In BALANCED, if requested FPS is higher or close to input FPS to the target
// we tell the QualityScaler to increase its frequency.
- if (EffectiveDegradationPreference() == DegradationPreference::BALANCED &&
- target->action == AdaptationAction::kDecreaseFrameRate) {
- absl::optional<int> min_diff = balanced_settings_.MinFpsDiff(input_pixels);
+ if (stream_adapter_->EffectiveDegradationPreference(GetVideoInputMode()) ==
+ DegradationPreference::BALANCED &&
+ target->action ==
+ VideoStreamAdapter::AdaptationAction::kDecreaseFrameRate) {
+ absl::optional<int> min_diff =
+ stream_adapter_->balanced_settings().MinFpsDiff(input_pixels);
if (min_diff && input_fps > 0) {
int fps_diff = input_fps - target->value;
if (fps_diff < min_diff.value()) {
@@ -744,14 +573,6 @@
return options;
}
-VideoCodecType
-OveruseFrameDetectorResourceAdaptationModule::GetVideoCodecTypeOrGeneric()
- const {
- return encoder_settings_.has_value()
- ? encoder_settings_->encoder_config().codec_type
- : kVideoCodecGeneric;
-}
-
int OveruseFrameDetectorResourceAdaptationModule::LastInputFrameSizeOrDefault()
const {
// The dependency on this hardcoded resolution is inherited from old code,
@@ -924,37 +745,17 @@
return counts;
}
-DegradationPreference
-OveruseFrameDetectorResourceAdaptationModule::EffectiveDegradationPreference()
- const {
- // Balanced mode for screenshare works via automatic animation detection:
- // Resolution is capped for fullscreen animated content.
- // Adapatation is done only via framerate downgrade.
- // Thus effective degradation preference is MAINTAIN_RESOLUTION.
+VideoStreamAdapter::VideoInputMode
+OveruseFrameDetectorResourceAdaptationModule::GetVideoInputMode() const {
+ if (!has_input_video_)
+ return VideoStreamAdapter::VideoInputMode::kNoVideo;
return (encoder_settings_.has_value() &&
encoder_settings_->encoder_config().content_type ==
- VideoEncoderConfig::ContentType::kScreen &&
- degradation_preference_ == DegradationPreference::BALANCED)
- ? DegradationPreference::MAINTAIN_RESOLUTION
- : degradation_preference_;
+ VideoEncoderConfig::ContentType::kScreen)
+ ? VideoStreamAdapter::VideoInputMode::kScreenshareVideo
+ : VideoStreamAdapter::VideoInputMode::kNormalVideo;
}
-bool OveruseFrameDetectorResourceAdaptationModule::CanAdaptUpResolution(
- int pixels,
- uint32_t bitrate_bps) const {
- absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
- encoder_settings_.has_value()
- ? GetEncoderBitrateLimits(
- encoder_settings_->encoder_info(),
- VideoStreamAdapter::GetHigherResolutionThan(pixels))
- : absl::nullopt;
- if (!bitrate_limits.has_value() || bitrate_bps == 0) {
- return true; // No limit configured or bitrate provided.
- }
- RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, pixels);
- return bitrate_bps >=
- static_cast<uint32_t>(bitrate_limits->min_start_bitrate_bps);
-}
void OveruseFrameDetectorResourceAdaptationModule::
MaybePerformQualityRampupExperiment() {
if (!quality_scaler_resource_->is_started())
diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h
index 14607fe..e80bd6f 100644
--- a/video/overuse_frame_detector_resource_adaptation_module.h
+++ b/video/overuse_frame_detector_resource_adaptation_module.h
@@ -128,55 +128,25 @@
AdaptationCounters* other_active);
private:
- class VideoSourceRestrictor;
class InitialFrameDropper;
enum class State { kStopped, kStarted };
- struct AdaptationRequest {
- // The pixel count produced by the source at the time of the adaptation.
- int input_pixel_count_;
- // Framerate received from the source at the time of the adaptation.
- int framerate_fps_;
- // Indicates if request was to adapt up or down.
- enum class Mode { kAdaptUp, kAdaptDown } mode_;
- };
-
- enum class AdaptationAction {
- kIncreaseResolution,
- kDecreaseResolution,
- kIncreaseFrameRate,
- kDecreaseFrameRate,
- };
-
- // Describes an adaptation step: increasing or decreasing resolution or frame
- // rate to a given value.
- struct AdaptationTarget {
- AdaptationTarget(AdaptationAction action, int value);
- // Which action the VideoSourceRestrictor needs to take.
- const AdaptationAction action;
- // Target pixel count or frame rate depending on |action|.
- const int value;
-
- // Allow this struct to be instantiated as an optional, even though it's in
- // a private namespace.
- friend class absl::optional<AdaptationTarget>;
- };
-
// Returns a target that we are guaranteed to be able to adapt to, or null if
// adaptation is not desired or not possible.
- absl::optional<AdaptationTarget> GetAdaptUpTarget(
+ absl::optional<VideoStreamAdapter::AdaptationTarget> GetAdaptUpTarget(
int input_pixels,
int input_fps,
AdaptationObserverInterface::AdaptReason reason) const;
- absl::optional<AdaptationTarget> GetAdaptDownTarget(
+ absl::optional<VideoStreamAdapter::AdaptationTarget> GetAdaptDownTarget(
int input_pixels,
int input_fps,
int min_pixels_per_frame) const;
// Applies the |target| to |source_restrictor_|.
- void ApplyAdaptationTarget(const AdaptationTarget& target,
- int min_pixels_per_frame,
- AdaptationObserverInterface::AdaptReason reason);
+ void ApplyAdaptationTarget(const VideoStreamAdapter::AdaptationTarget& target,
+ int input_pixels,
+ int input_fps,
+ int min_pixels_per_frame);
// Performs the adaptation by getting the next target, applying it and
// informing listeners of the new VideoSourceRestriction and adapt counters.
@@ -185,11 +155,11 @@
AdaptationObserverInterface::AdaptReason reason);
CpuOveruseOptions GetCpuOveruseOptions() const;
- VideoCodecType GetVideoCodecTypeOrGeneric() const;
int LastInputFrameSizeOrDefault() const;
int MinPixelsPerFrame() const;
VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts(
AdaptationObserverInterface::AdaptReason reason);
+ VideoStreamAdapter::VideoInputMode GetVideoInputMode() const;
// Makes |video_source_restrictions_| up-to-date and informs the
// |adaptation_listener_| if restrictions are changed, allowing the listener
@@ -204,8 +174,6 @@
absl::optional<VideoEncoder::QpThresholds> qp_thresholds);
void UpdateAdaptationStats(AdaptationObserverInterface::AdaptReason reason);
- DegradationPreference EffectiveDegradationPreference() const;
- bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const;
// Checks to see if we should execute the quality rampup experiment. The
// experiment resets all video restrictions at the start of the call in the
@@ -224,11 +192,12 @@
// The restrictions that |adaptation_listener_| is informed of.
VideoSourceRestrictions video_source_restrictions_;
bool has_input_video_;
+ // TODO(https://crbug.com/webrtc/11393): DegradationPreference has mostly
+ // moved to VideoStreamAdapter. Move it entirely and delete it from this
+ // class. If the responsibility of generating next steps for adaptations is
+ // owned by the adapter, this class has no buisness relying on implementation
+ // details of the adapter.
DegradationPreference degradation_preference_;
- const BalancedDegradationSettings balanced_settings_;
- // Stores a snapshot of the last adaptation request triggered by an AdaptUp
- // or AdaptDown signal.
- absl::optional<AdaptationRequest> last_adaptation_request_;
// Keeps track of source restrictions that this adaptation module outputs.
const std::unique_ptr<VideoStreamAdapter> stream_adapter_;
const std::unique_ptr<EncodeUsageResource> encode_usage_resource_;
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index b99e911..58fba37 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -171,43 +171,6 @@
}
} // namespace
-absl::optional<VideoEncoder::ResolutionBitrateLimits> GetEncoderBitrateLimits(
- const VideoEncoder::EncoderInfo& encoder_info,
- int frame_size_pixels) {
- std::vector<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
- encoder_info.resolution_bitrate_limits;
-
- // Sort the list of bitrate limits by resolution.
- sort(bitrate_limits.begin(), bitrate_limits.end(),
- [](const VideoEncoder::ResolutionBitrateLimits& lhs,
- const VideoEncoder::ResolutionBitrateLimits& rhs) {
- return lhs.frame_size_pixels < rhs.frame_size_pixels;
- });
-
- for (size_t i = 0; i < bitrate_limits.size(); ++i) {
- RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps, 0);
- RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps, 0);
- RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps,
- bitrate_limits[i].min_bitrate_bps);
- if (i > 0) {
- // The bitrate limits aren't expected to decrease with resolution.
- RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps,
- bitrate_limits[i - 1].min_bitrate_bps);
- RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps,
- bitrate_limits[i - 1].min_start_bitrate_bps);
- RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps,
- bitrate_limits[i - 1].max_bitrate_bps);
- }
-
- if (bitrate_limits[i].frame_size_pixels >= frame_size_pixels) {
- return absl::optional<VideoEncoder::ResolutionBitrateLimits>(
- bitrate_limits[i]);
- }
- }
-
- return absl::nullopt;
-}
-
const int VideoStreamEncoder::kDefaultLastFrameInfoWidth = 176;
const int VideoStreamEncoder::kDefaultLastFrameInfoHeight = 144;
@@ -503,9 +466,9 @@
encoder_reset_required = true;
}
- encoder_bitrate_limits_ = GetEncoderBitrateLimits(
- encoder_->GetEncoderInfo(),
- last_frame_info_->width * last_frame_info_->height);
+ encoder_bitrate_limits_ =
+ encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution(
+ last_frame_info_->width * last_frame_info_->height);
if (streams.size() == 1 && encoder_bitrate_limits_) {
// Bitrate limits can be set by app (in SDP or RtpEncodingParameters) or/and
@@ -1630,7 +1593,8 @@
}
absl::optional<VideoEncoder::ResolutionBitrateLimits> encoder_bitrate_limits =
- GetEncoderBitrateLimits(encoder_->GetEncoderInfo(), pixel_count);
+ encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution(
+ pixel_count);
if (encoder_bitrate_limits.has_value()) {
// Use bitrate limits provided by encoder.
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h
index 64f5e44..904a741 100644
--- a/video/video_stream_encoder.h
+++ b/video/video_stream_encoder.h
@@ -45,10 +45,6 @@
namespace webrtc {
-absl::optional<VideoEncoder::ResolutionBitrateLimits> GetEncoderBitrateLimits(
- const VideoEncoder::EncoderInfo& encoder_info,
- int frame_size_pixels);
-
// VideoStreamEncoder represent a video encoder that accepts raw video frames as
// input and produces an encoded bit stream.
// Usage: