Move initial frame drop to overuse module
It would be nice for this to stay in video stream encoder,
but this feature is mostly related to quality scaling. Perhaps
something easier to understand is possible in the future.
Bug: webrtc:11222
Change-Id: I71705f33ff94bbcf2fb9b5c94226c8e76dcba94c
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/168051
Commit-Queue: Evan Shrubsole <eshr@google.com>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30446}
diff --git a/call/adaptation/resource_adaptation_module_interface.h b/call/adaptation/resource_adaptation_module_interface.h
index 074857d..4f4a302 100644
--- a/call/adaptation/resource_adaptation_module_interface.h
+++ b/call/adaptation/resource_adaptation_module_interface.h
@@ -113,7 +113,13 @@
// TODO(hbos): If we take frame rate into account perhaps it would be valid to
// adapt down in frame rate as well.
virtual void OnFrameDroppedDueToSize() = 0;
- // 2.ii) An input frame is about to be encoded. It may have been cropped and
+ // 2.ii) If the frame will not be dropped due to size then signal that it may
+ // get encoded. However the frame is not guaranteed to be encoded right away
+ // or ever (for example if encoding is paused).
+ // TODO(eshr): Try replace OnMaybeEncodeFrame and merge behaviour into
+ // EncodeStarted.
+ virtual void OnMaybeEncodeFrame() = 0;
+ // 2.iii) An input frame is about to be encoded. It may have been cropped and
// have different dimensions than what was observed at OnFrame(). Next
// up: encoding completes or fails, see OnEncodeCompleted(). There is
// currently no signal for encode failure.
diff --git a/video/overuse_frame_detector_resource_adaptation_module.cc b/video/overuse_frame_detector_resource_adaptation_module.cc
index 4403857..c67e47e 100644
--- a/video/overuse_frame_detector_resource_adaptation_module.cc
+++ b/video/overuse_frame_detector_resource_adaptation_module.cc
@@ -66,6 +66,10 @@
return source_restrictions;
}
+// The maximum number of frames to drop at beginning of stream
+// to try and achieve desired bitrate.
+const int kMaxInitialFramedrop = 4;
+
} // namespace
// VideoSourceRestrictor is responsible for keeping track of current
@@ -361,7 +365,8 @@
target_bitrate_bps_(absl::nullopt),
quality_scaler_(nullptr),
encoder_settings_(absl::nullopt),
- encoder_stats_observer_(encoder_stats_observer) {
+ encoder_stats_observer_(encoder_stats_observer),
+ initial_framedrop_(0) {
RTC_DCHECK(adaptation_listener_);
RTC_DCHECK(overuse_detector_);
RTC_DCHECK(encoder_stats_observer_);
@@ -459,6 +464,7 @@
AdaptationObserverInterface::AdaptReason::kQuality) > res_count) {
encoder_stats_observer_->OnInitialQualityResolutionAdaptDown();
}
+ ++initial_framedrop_;
}
void OveruseFrameDetectorResourceAdaptationModule::OnEncodeStarted(
@@ -499,13 +505,29 @@
}
}
+void OveruseFrameDetectorResourceAdaptationModule::OnMaybeEncodeFrame() {
+ initial_framedrop_ = kMaxInitialFramedrop;
+}
+
+bool OveruseFrameDetectorResourceAdaptationModule::DropInitialFrames() const {
+ return initial_framedrop_ < kMaxInitialFramedrop;
+}
+
+void OveruseFrameDetectorResourceAdaptationModule::ResetInitialFrameDropping() {
+ initial_framedrop_ = 0;
+}
+
void OveruseFrameDetectorResourceAdaptationModule::UpdateQualityScalerSettings(
absl::optional<VideoEncoder::QpThresholds> qp_thresholds) {
if (qp_thresholds.has_value()) {
quality_scaler_ =
std::make_unique<QualityScaler>(this, qp_thresholds.value());
+ // Restart frame drops due to size.
+ initial_framedrop_ = 0;
} else {
quality_scaler_ = nullptr;
+ // Quality scaling disabled so we shouldn't drop initial frames.
+ initial_framedrop_ = kMaxInitialFramedrop;
}
}
diff --git a/video/overuse_frame_detector_resource_adaptation_module.h b/video/overuse_frame_detector_resource_adaptation_module.h
index 21c055b..379e1ad 100644
--- a/video/overuse_frame_detector_resource_adaptation_module.h
+++ b/video/overuse_frame_detector_resource_adaptation_module.h
@@ -79,12 +79,16 @@
void OnFrame(const VideoFrame& frame) override;
void OnFrameDroppedDueToSize() override;
+ void OnMaybeEncodeFrame() override;
void OnEncodeStarted(const VideoFrame& cropped_frame,
int64_t time_when_first_seen_us) override;
void OnEncodeCompleted(const EncodedImage& encoded_image,
int64_t time_sent_in_us,
absl::optional<int> encode_duration_us) override;
void OnFrameDropped(EncodedImageCallback::DropReason reason) override;
+ bool DropInitialFrames() const;
+ // TODO(eshr): Remove once all qp-scaling is in this class.
+ void ResetInitialFrameDropping();
// Use nullopt to disable quality scaling.
void UpdateQualityScalerSettings(
@@ -210,6 +214,8 @@
std::unique_ptr<QualityScaler> quality_scaler_;
absl::optional<EncoderSettings> encoder_settings_;
VideoStreamEncoderObserver* const encoder_stats_observer_;
+ // Counts how many frames we've dropped in the initial framedrop phase.
+ int initial_framedrop_;
};
} // namespace webrtc
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index cf71271..e1fdb8c 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -52,10 +52,6 @@
constexpr char kFrameDropperFieldTrial[] = "WebRTC-FrameDropper";
-// The maximum number of frames to drop at beginning of stream
-// to try and achieve desired bitrate.
-const int kMaxInitialFramedrop = 4;
-
// Averaging window spanning 90 frames at default 30fps, matching old media
// optimization module defaults.
const int64_t kFrameRateAvergingWindowSizeMs = (1000 / 30) * 90;
@@ -258,7 +254,6 @@
TaskQueueFactory* task_queue_factory)
: shutdown_event_(true /* manual_reset */, false),
number_of_cores_(number_of_cores),
- initial_framedrop_(0),
quality_rampup_done_(false),
quality_rampup_experiment_(QualityRampupExperiment::ParseSettings()),
quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()),
@@ -798,11 +793,9 @@
resource_adaptation_module_->UpdateQualityScalerSettings(
experimental_thresholds ? *experimental_thresholds
: *(scaling_settings.thresholds));
- initial_framedrop_ = 0;
}
} else {
resource_adaptation_module_->UpdateQualityScalerSettings(absl::nullopt);
- initial_framedrop_ = kMaxInitialFramedrop;
}
QualityScaler* quality_scaler = resource_adaptation_module_->quality_scaler();
@@ -1112,7 +1105,6 @@
if (DropDueToSize(video_frame.size())) {
RTC_LOG(LS_INFO) << "Dropping frame. Too large for target bitrate.";
resource_adaptation_module_->OnFrameDroppedDueToSize();
- ++initial_framedrop_;
// Storing references to a native buffer risks blocking frame capture.
if (video_frame.video_frame_buffer()->type() !=
VideoFrameBuffer::Type::kNative) {
@@ -1126,7 +1118,7 @@
}
return;
}
- initial_framedrop_ = kMaxInitialFramedrop;
+ resource_adaptation_module_->OnMaybeEncodeFrame();
if (!quality_rampup_done_ && TryQualityRampup(now_ms) &&
resource_adaptation_module_->GetConstAdaptCounter().ResolutionCount(
@@ -1608,7 +1600,7 @@
RTC_LOG(LS_INFO) << "Reset initial_framedrop_. Start bitrate: "
<< set_start_bitrate_bps_
<< ", target bitrate: " << target_bitrate.bps();
- initial_framedrop_ = 0;
+ resource_adaptation_module_->ResetInitialFrameDropping();
has_seen_first_bwe_drop_ = true;
}
}
@@ -1648,7 +1640,7 @@
}
bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const {
- if (initial_framedrop_ >= kMaxInitialFramedrop ||
+ if (!resource_adaptation_module_->DropInitialFrames() ||
!encoder_target_bitrate_bps_.has_value()) {
return false;
}
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h
index 97a4a66..d0b037d 100644
--- a/video/video_stream_encoder.h
+++ b/video/video_stream_encoder.h
@@ -213,8 +213,6 @@
rtc::Event shutdown_event_;
const uint32_t number_of_cores_;
- // Counts how many frames we've dropped in the initial framedrop phase.
- int initial_framedrop_;
bool quality_rampup_done_ RTC_GUARDED_BY(&encoder_queue_);
QualityRampupExperiment quality_rampup_experiment_
RTC_GUARDED_BY(&encoder_queue_);