Wire up internal libvpx VP9 scaler to statistics proxy
Bug: webrtc:11396
Change-Id: I5ac69208b00cc75d4e5dbb3ab86f234b3e1f29f8
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169922
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30725}
diff --git a/api/video/video_stream_encoder_observer.h b/api/video/video_stream_encoder_observer.h
index 0d639f3..9fd462c 100644
--- a/api/video/video_stream_encoder_observer.h
+++ b/api/video/video_stream_encoder_observer.h
@@ -95,6 +95,11 @@
const VideoCodec& codec,
const VideoBitrateAllocation& allocation) {}
+ // Informes observer if an internal encoder scaler has reduced video
+ // resolution or not. |is_scaled| is a flag indicating if the video is scaled
+ // down.
+ virtual void OnEncoderInternalScalerUpdate(bool is_scaled) {}
+
// TODO(nisse): VideoStreamEncoder wants to query the stats, which makes this
// not a pure observer. GetInputFrameRate is needed for the cpu adaptation, so
// can be deleted if that responsibility is moved out to a VideoStreamAdaptor
diff --git a/video/send_statistics_proxy.cc b/video/send_statistics_proxy.cc
index 094baa3..60d84f1 100644
--- a/video/send_statistics_proxy.cc
+++ b/video/send_statistics_proxy.cc
@@ -148,6 +148,7 @@
last_num_simulcast_streams_(0),
last_spatial_layer_use_{},
bw_limited_layers_(false),
+ internal_encoder_scaler_(false),
uma_container_(
new UmaSamplesContainer(GetUmaPrefix(content_type_), stats_, clock)) {
}
@@ -1083,7 +1084,7 @@
cpu_counts_.num_framerate_reductions > 0;
bool is_bandwidth_limited = quality_counts_.num_resolution_reductions > 0 ||
quality_counts_.num_framerate_reductions > 0 ||
- bw_limited_layers_;
+ bw_limited_layers_ || internal_encoder_scaler_;
if (is_bandwidth_limited) {
// We may be both CPU limited and bandwidth limited at the same time but
// there is no way to express this in standardized stats. Heuristically,
@@ -1117,6 +1118,10 @@
}
}
}
+ if (internal_encoder_scaler_) {
+ stats_.bw_limited_resolution = true;
+ }
+
stats_.quality_limitation_reason =
quality_limitation_reason_tracker_.current_reason();
@@ -1164,6 +1169,15 @@
last_num_simulcast_streams_ = num_simulcast_streams;
}
+// Informes observer if an internal encoder scaler has reduced video
+// resolution or not. |is_scaled| is a flag indicating if the video is scaled
+// down.
+void SendStatisticsProxy::OnEncoderInternalScalerUpdate(bool is_scaled) {
+ rtc::CritScope lock(&crit_);
+ internal_encoder_scaler_ = is_scaled;
+ UpdateAdaptationStats();
+}
+
// TODO(asapersson): Include fps changes.
void SendStatisticsProxy::OnInitialQualityResolutionAdaptDown() {
rtc::CritScope lock(&crit_);
diff --git a/video/send_statistics_proxy.h b/video/send_statistics_proxy.h
index a67725e..abe3999 100644
--- a/video/send_statistics_proxy.h
+++ b/video/send_statistics_proxy.h
@@ -78,6 +78,8 @@
const VideoCodec& codec,
const VideoBitrateAllocation& allocation) override;
+ void OnEncoderInternalScalerUpdate(bool is_scaled) override;
+
void OnMinPixelLimitReached() override;
void OnInitialQualityResolutionAdaptDown() override;
@@ -264,6 +266,8 @@
// Indicates if the latest bitrate allocation had layers disabled by low
// available bandwidth.
bool bw_limited_layers_ RTC_GUARDED_BY(crit_);
+ // Indicastes if the encoder internally downscales input image.
+ bool internal_encoder_scaler_ RTC_GUARDED_BY(crit_);
AdaptationSteps cpu_counts_ RTC_GUARDED_BY(crit_);
AdaptationSteps quality_counts_ RTC_GUARDED_BY(crit_);
diff --git a/video/send_statistics_proxy_unittest.cc b/video/send_statistics_proxy_unittest.cc
index db5c94b..3f5ebd5 100644
--- a/video/send_statistics_proxy_unittest.cc
+++ b/video/send_statistics_proxy_unittest.cc
@@ -2147,6 +2147,15 @@
allocation.set_bw_limited(true);
statistics_proxy_->OnBitrateAllocationUpdated(codec, allocation);
EXPECT_TRUE(statistics_proxy_->GetStats().bw_limited_resolution);
+
+ // Revert for the next test.
+ allocation.set_bw_limited(false);
+ statistics_proxy_->OnBitrateAllocationUpdated(codec, allocation);
+ EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution);
+
+ // Internal encoder scaler reduced resolution.
+ statistics_proxy_->OnEncoderInternalScalerUpdate(/*scaled=*/true);
+ EXPECT_TRUE(statistics_proxy_->GetStats().bw_limited_resolution);
}
TEST_F(SendStatisticsProxyTest, GetStatsReportsTargetMediaBitrate) {
diff --git a/video/video_quality_test.cc b/video/video_quality_test.cc
index b870f7c..826567c 100644
--- a/video/video_quality_test.cc
+++ b/video/video_quality_test.cc
@@ -870,6 +870,7 @@
VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
vp9_settings.denoisingOn = false;
vp9_settings.frameDroppingOn = false;
+ vp9_settings.automaticResizeOn = false;
vp9_settings.numberOfTemporalLayers = static_cast<unsigned char>(
params_.video[video_idx].num_temporal_layers);
vp9_settings.numberOfSpatialLayers = static_cast<unsigned char>(
@@ -892,6 +893,7 @@
vp9_settings.numberOfSpatialLayers =
static_cast<unsigned char>(params_.ss[video_idx].num_spatial_layers);
vp9_settings.interLayerPred = params_.ss[video_idx].inter_layer_pred;
+ vp9_settings.automaticResizeOn = false;
video_encoder_configs_[video_idx].encoder_specific_settings =
new rtc::RefCountedObject<
VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
@@ -904,7 +906,9 @@
VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings);
} else if (params_.video[video_idx].codec == "VP9") {
VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
- vp9_settings.automaticResizeOn = true;
+ // Only enable quality scaler for single spatial layer.
+ vp9_settings.automaticResizeOn =
+ params_.ss[video_idx].num_spatial_layers == 1;
video_encoder_configs_[video_idx].encoder_specific_settings =
new rtc::RefCountedObject<
VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
diff --git a/video/video_send_stream_tests.cc b/video/video_send_stream_tests.cc
index 7ceb9db..27bf0f0 100644
--- a/video/video_send_stream_tests.cc
+++ b/video/video_send_stream_tests.cc
@@ -3479,6 +3479,7 @@
vp9_settings_.flexibleMode = false;
vp9_settings_.frameDroppingOn = false;
+ vp9_settings_.automaticResizeOn = false;
vp9_settings_.keyFrameInterval = kKeyFrameInterval;
vp9_settings_.numberOfTemporalLayers = num_temporal_layers_;
vp9_settings_.numberOfSpatialLayers = num_spatial_layers_;
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index 58fba37..d9eda8e 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -169,6 +169,7 @@
new_allocation.set_bw_limited(allocation.is_bw_limited());
return new_allocation;
}
+
} // namespace
const int VideoStreamEncoder::kDefaultLastFrameInfoWidth = 176;
@@ -1360,6 +1361,37 @@
RTC_CHECK(videocontenttypehelpers::SetSimulcastId(
&image_copy.content_type_, static_cast<uint8_t>(spatial_idx + 1)));
+ // Currently internal quality scaler is used for VP9 instead of webrtc qp
+ // scaler (in no-svc case or if only a single spatial layer is encoded).
+ // It has to be explicitly detected and reported to adaptation metrics.
+ // Post a task because |send_codec_| requires |encoder_queue_| lock.
+ unsigned int image_width = image_copy._encodedWidth;
+ unsigned int image_height = image_copy._encodedHeight;
+ VideoCodecType codec = codec_specific_info
+ ? codec_specific_info->codecType
+ : VideoCodecType::kVideoCodecGeneric;
+ encoder_queue_.PostTask([this, codec, image_width, image_height] {
+ RTC_DCHECK_RUN_ON(&encoder_queue_);
+ if (codec == VideoCodecType::kVideoCodecVP9 &&
+ send_codec_.VP9()->automaticResizeOn) {
+ unsigned int expected_width = send_codec_.width;
+ unsigned int expected_height = send_codec_.height;
+ int num_active_layers = 0;
+ for (int i = 0; i < send_codec_.VP9()->numberOfSpatialLayers; ++i) {
+ if (send_codec_.spatialLayers[i].active) {
+ ++num_active_layers;
+ expected_width = send_codec_.spatialLayers[i].width;
+ expected_height = send_codec_.spatialLayers[i].height;
+ }
+ }
+ RTC_DCHECK_LE(num_active_layers, 1)
+ << "VP9 quality scaling is enabled for "
+ "SVC with several active layers.";
+ encoder_stats_observer_->OnEncoderInternalScalerUpdate(
+ image_width < expected_width || image_height < expected_height);
+ }
+ });
+
// Encoded is called on whatever thread the real encoder implementation run
// on. In the case of hardware encoders, there might be several encoders
// running in parallel on different threads.