Turn off error resilience for VP9 if no spatial or temporal layers are configured and NACK is enabled.
Error resilience is currently always enabled for VP9 which reduces quality.
Reland of https://codereview.webrtc.org/2532053002
BUG=webrtc:6783
Review-Url: https://codereview.webrtc.org/2925253002
Cr-Commit-Position: refs/heads/master@{#19385}
diff --git a/webrtc/modules/video_coding/video_codec_initializer.cc b/webrtc/modules/video_coding/video_codec_initializer.cc
index c759e84..df8f136 100644
--- a/webrtc/modules/video_coding/video_codec_initializer.cc
+++ b/webrtc/modules/video_coding/video_codec_initializer.cc
@@ -22,6 +22,15 @@
#include "webrtc/system_wrappers/include/clock.h"
namespace webrtc {
+namespace {
+bool TemporalLayersConfigured(const std::vector<VideoStream>& streams) {
+ for (const VideoStream& stream : streams) {
+ if (stream.temporal_layer_thresholds_bps.size() > 0)
+ return true;
+ }
+ return false;
+}
+} // namespace
bool VideoCodecInitializer::SetupCodec(
const VideoEncoderConfig& config,
@@ -121,12 +130,8 @@
*video_codec.VP8() = VideoEncoder::GetDefaultVp8Settings();
video_codec.VP8()->numberOfTemporalLayers = static_cast<unsigned char>(
streams.back().temporal_layer_thresholds_bps.size() + 1);
- bool temporal_layers_configured = false;
- for (const VideoStream& stream : streams) {
- if (stream.temporal_layer_thresholds_bps.size() > 0)
- temporal_layers_configured = true;
- }
- if (nack_enabled && !temporal_layers_configured) {
+
+ if (nack_enabled && !TemporalLayersConfigured(streams)) {
LOG(LS_INFO) << "No temporal layers and nack enabled -> resilience off";
video_codec.VP8()->resilience = kResilienceOff;
}
@@ -144,6 +149,13 @@
}
video_codec.VP9()->numberOfTemporalLayers = static_cast<unsigned char>(
streams.back().temporal_layer_thresholds_bps.size() + 1);
+
+ if (nack_enabled && !TemporalLayersConfigured(streams) &&
+ video_codec.VP9()->numberOfSpatialLayers == 1) {
+ LOG(LS_INFO) << "No temporal or spatial layers and nack enabled -> "
+ << "resilience off";
+ video_codec.VP9()->resilienceOn = false;
+ }
break;
}
case kVideoCodecH264: {
diff --git a/webrtc/video/video_stream_encoder_unittest.cc b/webrtc/video/video_stream_encoder_unittest.cc
index d7d09db..69b9119 100644
--- a/webrtc/video/video_stream_encoder_unittest.cc
+++ b/webrtc/video/video_stream_encoder_unittest.cc
@@ -34,6 +34,7 @@
const int kMinPixelsPerFrame = 320 * 180;
const int kMinFramerateFps = 2;
const int64_t kFrameTimeoutMs = 100;
+const unsigned char kNumSlDummy = 0;
} // namespace
namespace webrtc {
@@ -317,6 +318,7 @@
void ResetEncoder(const std::string& payload_name,
size_t num_streams,
size_t num_temporal_layers,
+ unsigned char num_spatial_layers,
bool nack_enabled,
bool screenshare) {
video_send_config_.encoder_settings.payload_name = payload_name;
@@ -330,6 +332,13 @@
video_encoder_config.content_type =
screenshare ? VideoEncoderConfig::ContentType::kScreen
: VideoEncoderConfig::ContentType::kRealtimeVideo;
+ if (payload_name == "VP9") {
+ VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
+ vp9_settings.numberOfSpatialLayers = num_spatial_layers;
+ video_encoder_config.encoder_specific_settings =
+ new rtc::RefCountedObject<
+ VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
+ }
ConfigureEncoder(std::move(video_encoder_config), nack_enabled);
}
@@ -799,7 +808,7 @@
const bool kNackEnabled = true;
const size_t kNumStreams = 1;
const size_t kNumTl = 1;
- ResetEncoder("VP8", kNumStreams, kNumTl, kNackEnabled, false);
+ ResetEncoder("VP8", kNumStreams, kNumTl, kNumSlDummy, kNackEnabled, false);
video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
// Capture a frame and wait for it to synchronize with the encoder thread.
@@ -819,7 +828,7 @@
const bool kNackEnabled = true;
const size_t kNumStreams = 2;
const size_t kNumTl = 1;
- ResetEncoder("VP8", kNumStreams, kNumTl, kNackEnabled, false);
+ ResetEncoder("VP8", kNumStreams, kNumTl, kNumSlDummy, kNackEnabled, false);
video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
// Capture a frame and wait for it to synchronize with the encoder thread.
@@ -839,7 +848,7 @@
const bool kNackEnabled = false;
const size_t kNumStreams = 1;
const size_t kNumTl = 1;
- ResetEncoder("VP8", kNumStreams, kNumTl, kNackEnabled, false);
+ ResetEncoder("VP8", kNumStreams, kNumTl, kNumSlDummy, kNackEnabled, false);
video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
// Capture a frame and wait for it to synchronize with the encoder thread.
@@ -859,7 +868,7 @@
const bool kNackEnabled = true;
const size_t kNumStreams = 1;
const size_t kNumTl = 2;
- ResetEncoder("VP8", kNumStreams, kNumTl, kNackEnabled, false);
+ ResetEncoder("VP8", kNumStreams, kNumTl, kNumSlDummy, kNackEnabled, false);
video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
// Capture a frame and wait for it to synchronize with the encoder thread.
@@ -875,6 +884,94 @@
video_stream_encoder_->Stop();
}
+TEST_F(VideoStreamEncoderTest, Vp9ResilienceIsOffFor1SL1TLWithNackEnabled) {
+ const bool kNackEnabled = true;
+ const size_t kNumStreams = 1;
+ const size_t kNumTl = 1;
+ const unsigned char kNumSl = 1;
+ ResetEncoder("VP9", kNumStreams, kNumTl, kNumSl, kNackEnabled, false);
+ video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+ // Capture a frame and wait for it to synchronize with the encoder thread.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ sink_.WaitForEncodedFrame(1);
+ // The encoder have been configured once when the first frame is received.
+ EXPECT_EQ(1, sink_.number_of_reconfigurations());
+ EXPECT_EQ(kVideoCodecVP9, fake_encoder_.codec_config().codecType);
+ EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
+ EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP9()->numberOfTemporalLayers);
+ EXPECT_EQ(kNumSl, fake_encoder_.codec_config().VP9()->numberOfSpatialLayers);
+ // Resilience is off for no spatial and temporal layers with nack on.
+ EXPECT_FALSE(fake_encoder_.codec_config().VP9()->resilienceOn);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, Vp9ResilienceIsOnFor1SL1TLWithNackDisabled) {
+ const bool kNackEnabled = false;
+ const size_t kNumStreams = 1;
+ const size_t kNumTl = 1;
+ const unsigned char kNumSl = 1;
+ ResetEncoder("VP9", kNumStreams, kNumTl, kNumSl, kNackEnabled, false);
+ video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+ // Capture a frame and wait for it to synchronize with the encoder thread.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ sink_.WaitForEncodedFrame(1);
+ // The encoder have been configured once when the first frame is received.
+ EXPECT_EQ(1, sink_.number_of_reconfigurations());
+ EXPECT_EQ(kVideoCodecVP9, fake_encoder_.codec_config().codecType);
+ EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
+ EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP9()->numberOfTemporalLayers);
+ EXPECT_EQ(kNumSl, fake_encoder_.codec_config().VP9()->numberOfSpatialLayers);
+ // Resilience is on if nack is off.
+ EXPECT_TRUE(fake_encoder_.codec_config().VP9()->resilienceOn);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, Vp9ResilienceIsOnFor2SL1TLWithNackEnabled) {
+ const bool kNackEnabled = true;
+ const size_t kNumStreams = 1;
+ const size_t kNumTl = 1;
+ const unsigned char kNumSl = 2;
+ ResetEncoder("VP9", kNumStreams, kNumTl, kNumSl, kNackEnabled, false);
+ video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+ // Capture a frame and wait for it to synchronize with the encoder thread.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ sink_.WaitForEncodedFrame(1);
+ // The encoder have been configured once when the first frame is received.
+ EXPECT_EQ(1, sink_.number_of_reconfigurations());
+ EXPECT_EQ(kVideoCodecVP9, fake_encoder_.codec_config().codecType);
+ EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
+ EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP9()->numberOfTemporalLayers);
+ EXPECT_EQ(kNumSl, fake_encoder_.codec_config().VP9()->numberOfSpatialLayers);
+ // Resilience is on for spatial layers.
+ EXPECT_TRUE(fake_encoder_.codec_config().VP9()->resilienceOn);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, Vp9ResilienceIsOnFor1SL2TLWithNackEnabled) {
+ const bool kNackEnabled = true;
+ const size_t kNumStreams = 1;
+ const size_t kNumTl = 2;
+ const unsigned char kNumSl = 1;
+ ResetEncoder("VP9", kNumStreams, kNumTl, kNumSl, kNackEnabled, false);
+ video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+ // Capture a frame and wait for it to synchronize with the encoder thread.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ sink_.WaitForEncodedFrame(1);
+ // The encoder have been configured once when the first frame is received.
+ EXPECT_EQ(1, sink_.number_of_reconfigurations());
+ EXPECT_EQ(kVideoCodecVP9, fake_encoder_.codec_config().codecType);
+ EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
+ EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP9()->numberOfTemporalLayers);
+ EXPECT_EQ(kNumSl, fake_encoder_.codec_config().VP9()->numberOfSpatialLayers);
+ // Resilience is on for temporal layers.
+ EXPECT_TRUE(fake_encoder_.codec_config().VP9()->resilienceOn);
+ video_stream_encoder_->Stop();
+}
+
TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
EXPECT_TRUE(video_source_.has_sinks());
test::FrameForwarder new_video_source;
@@ -2463,7 +2560,7 @@
TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
fake_encoder_.ForceInitEncodeFailure(true);
video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
- ResetEncoder("VP8", 2, 1, true, false);
+ ResetEncoder("VP8", 2, 1, 1, true, false);
const int kFrameWidth = 1280;
const int kFrameHeight = 720;
video_source_.IncomingCapturedFrame(
@@ -2610,7 +2707,7 @@
// Reconfigure encoder with two temporal layers and screensharing, which will
// disable frame dropping and make testing easier.
- ResetEncoder("VP8", 1, 2, true, true);
+ ResetEncoder("VP8", 1, 2, 1, true, true);
video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
video_stream_encoder_->SetSource(