Turn off error resilience for vp8 for no temporal layers if nack is enabled.
BUG=webrtc:6634
Review-Url: https://codereview.webrtc.org/2493893003
Cr-Commit-Position: refs/heads/master@{#15240}
diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
index eb5d2bd..c045100 100644
--- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
+++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
@@ -401,11 +401,7 @@
// Set the error resilience mode according to user settings.
switch (inst->VP8().resilience) {
case kResilienceOff:
- // TODO(marpan): We should set keep error resilience off for this mode,
- // independent of temporal layer settings, and make sure we set
- // |codecSpecific.VP8.resilience| = |kResilientStream| at higher level
- // code if we want to get error resilience on.
- configurations_[0].g_error_resilient = 1;
+ configurations_[0].g_error_resilient = 0;
break;
case kResilientStream:
configurations_[0].g_error_resilient = 1; // TODO(holmer): Replace with
diff --git a/webrtc/modules/video_coding/include/video_codec_initializer.h b/webrtc/modules/video_coding/include/video_codec_initializer.h
index 8514bbb..159218d 100644
--- a/webrtc/modules/video_coding/include/video_codec_initializer.h
+++ b/webrtc/modules/video_coding/include/video_codec_initializer.h
@@ -36,6 +36,7 @@
const VideoEncoderConfig& config,
const VideoSendStream::Config::EncoderSettings settings,
const std::vector<VideoStream>& streams,
+ bool nack_enabled,
VideoCodec* codec,
std::unique_ptr<VideoBitrateAllocator>* bitrate_allocator);
@@ -51,7 +52,8 @@
const VideoEncoderConfig& config,
const std::vector<VideoStream>& streams,
const std::string& payload_name,
- int payload_type);
+ int payload_type,
+ bool nack_enabled);
};
} // namespace webrtc
diff --git a/webrtc/modules/video_coding/video_codec_initializer.cc b/webrtc/modules/video_coding/video_codec_initializer.cc
index 5d32fd2..57f9b26 100644
--- a/webrtc/modules/video_coding/video_codec_initializer.cc
+++ b/webrtc/modules/video_coding/video_codec_initializer.cc
@@ -11,6 +11,7 @@
#include "webrtc/modules/video_coding/include/video_codec_initializer.h"
#include "webrtc/base/basictypes.h"
+#include "webrtc/base/logging.h"
#include "webrtc/common_video/include/video_bitrate_allocator.h"
#include "webrtc/common_types.h"
#include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h"
@@ -25,10 +26,12 @@
const VideoEncoderConfig& config,
const VideoSendStream::Config::EncoderSettings settings,
const std::vector<VideoStream>& streams,
+ bool nack_enabled,
VideoCodec* codec,
std::unique_ptr<VideoBitrateAllocator>* bitrate_allocator) {
- *codec = VideoEncoderConfigToVideoCodec(
- config, streams, settings.payload_name, settings.payload_type);
+ *codec =
+ VideoEncoderConfigToVideoCodec(config, streams, settings.payload_name,
+ settings.payload_type, nack_enabled);
std::unique_ptr<TemporalLayersFactory> tl_factory;
switch (codec->codecType) {
@@ -82,7 +85,8 @@
const VideoEncoderConfig& config,
const std::vector<VideoStream>& streams,
const std::string& payload_name,
- int payload_type) {
+ int payload_type,
+ bool nack_enabled) {
static const int kEncoderMinBitrateKbps = 30;
RTC_DCHECK(!streams.empty());
RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0);
@@ -115,6 +119,15 @@
*video_codec.VP8() = VideoEncoder::GetDefaultVp8Settings();
video_codec.VP8()->numberOfTemporalLayers = static_cast<unsigned char>(
streams.back().temporal_layer_thresholds_bps.size() + 1);
+ bool temporal_layers_configured = false;
+ for (const VideoStream& stream : streams) {
+ if (stream.temporal_layer_thresholds_bps.size() > 0)
+ temporal_layers_configured = true;
+ }
+ if (nack_enabled && !temporal_layers_configured) {
+ LOG(LS_INFO) << "No temporal layers and nack enabled -> resilience off";
+ video_codec.VP8()->resilience = kResilienceOff;
+ }
break;
}
case kVideoCodecVP9: {
diff --git a/webrtc/video/video_send_stream.cc b/webrtc/video/video_send_stream.cc
index 8eb8cba..964475f 100644
--- a/webrtc/video/video_send_stream.cc
+++ b/webrtc/video/video_send_stream.cc
@@ -636,8 +636,8 @@
// TODO(perkj): Some test cases in VideoSendStreamTest call
// ReconfigureVideoEncoder from the network thread.
// RTC_DCHECK_RUN_ON(&thread_checker_);
- vie_encoder_->ConfigureEncoder(std::move(config),
- config_.rtp.max_packet_size);
+ vie_encoder_->ConfigureEncoder(std::move(config), config_.rtp.max_packet_size,
+ config_.rtp.nack.rtp_history_ms > 0);
}
VideoSendStream::Stats VideoSendStream::GetStats() {
diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc
index 2c9c679..c2598f4 100644
--- a/webrtc/video/vie_encoder.cc
+++ b/webrtc/video/vie_encoder.cc
@@ -52,21 +52,24 @@
public:
ConfigureEncoderTask(ViEEncoder* vie_encoder,
VideoEncoderConfig config,
- size_t max_data_payload_length)
+ size_t max_data_payload_length,
+ bool nack_enabled)
: vie_encoder_(vie_encoder),
config_(std::move(config)),
- max_data_payload_length_(max_data_payload_length) {}
+ max_data_payload_length_(max_data_payload_length),
+ nack_enabled_(nack_enabled) {}
private:
bool Run() override {
- vie_encoder_->ConfigureEncoderOnTaskQueue(std::move(config_),
- max_data_payload_length_);
+ vie_encoder_->ConfigureEncoderOnTaskQueue(
+ std::move(config_), max_data_payload_length_, nack_enabled_);
return true;
}
ViEEncoder* const vie_encoder_;
VideoEncoderConfig config_;
size_t max_data_payload_length_;
+ bool nack_enabled_;
};
class ViEEncoder::EncodeTask : public rtc::QueuedTask {
@@ -246,6 +249,7 @@
pending_encoder_reconfiguration_(false),
encoder_start_bitrate_bps_(0),
max_data_payload_length_(0),
+ nack_enabled_(false),
last_observed_bitrate_bps_(0),
encoder_paused_and_dropped_frame_(false),
has_received_sli_(false),
@@ -344,19 +348,22 @@
}
void ViEEncoder::ConfigureEncoder(VideoEncoderConfig config,
- size_t max_data_payload_length) {
+ size_t max_data_payload_length,
+ bool nack_enabled) {
encoder_queue_.PostTask(
std::unique_ptr<rtc::QueuedTask>(new ConfigureEncoderTask(
- this, std::move(config), max_data_payload_length)));
+ this, std::move(config), max_data_payload_length, nack_enabled)));
}
void ViEEncoder::ConfigureEncoderOnTaskQueue(VideoEncoderConfig config,
- size_t max_data_payload_length) {
+ size_t max_data_payload_length,
+ bool nack_enabled) {
RTC_DCHECK_RUN_ON(&encoder_queue_);
RTC_DCHECK(sink_);
LOG(LS_INFO) << "ConfigureEncoder requested.";
max_data_payload_length_ = max_data_payload_length;
+ nack_enabled_ = nack_enabled;
encoder_config_ = std::move(config);
pending_encoder_reconfiguration_ = true;
@@ -382,7 +389,8 @@
VideoCodec codec;
if (!VideoCodecInitializer::SetupCodec(encoder_config_, settings_, streams,
- &codec, &rate_allocator_)) {
+ nack_enabled_, &codec,
+ &rate_allocator_)) {
LOG(LS_ERROR) << "Failed to create encoder configuration.";
}
diff --git a/webrtc/video/vie_encoder.h b/webrtc/video/vie_encoder.h
index 433e9ce..8373a3f 100644
--- a/webrtc/video/vie_encoder.h
+++ b/webrtc/video/vie_encoder.h
@@ -92,7 +92,8 @@
void SetStartBitrate(int start_bitrate_bps);
void ConfigureEncoder(VideoEncoderConfig config,
- size_t max_data_payload_length);
+ size_t max_data_payload_length,
+ bool nack_enabled);
// Permanently stop encoding. After this method has returned, it is
// guaranteed that no encoded frames will be delivered to the sink.
@@ -140,7 +141,8 @@
};
void ConfigureEncoderOnTaskQueue(VideoEncoderConfig config,
- size_t max_data_payload_length);
+ size_t max_data_payload_length,
+ bool nack_enabled);
void ReconfigureEncoder();
// Implements VideoSinkInterface.
@@ -193,6 +195,7 @@
rtc::Optional<VideoFrameInfo> last_frame_info_ ACCESS_ON(&encoder_queue_);
uint32_t encoder_start_bitrate_bps_ ACCESS_ON(&encoder_queue_);
size_t max_data_payload_length_ ACCESS_ON(&encoder_queue_);
+ bool nack_enabled_ ACCESS_ON(&encoder_queue_);
uint32_t last_observed_bitrate_bps_ ACCESS_ON(&encoder_queue_);
bool encoder_paused_and_dropped_frame_ ACCESS_ON(&encoder_queue_);
bool has_received_sli_ ACCESS_ON(&encoder_queue_);
diff --git a/webrtc/video/vie_encoder_unittest.cc b/webrtc/video/vie_encoder_unittest.cc
index f6046be..71e68d3 100644
--- a/webrtc/video/vie_encoder_unittest.cc
+++ b/webrtc/video/vie_encoder_unittest.cc
@@ -23,6 +23,9 @@
namespace webrtc {
namespace {
+const size_t kMaxPayloadLength = 1440;
+const int kTargetBitrateBps = 100000;
+
class TestBuffer : public webrtc::I420Buffer {
public:
TestBuffer(rtc::Event* event, int width, int height)
@@ -67,6 +70,29 @@
}
};
+class VideoStreamFactory
+ : public VideoEncoderConfig::VideoStreamFactoryInterface {
+ public:
+ explicit VideoStreamFactory(size_t num_temporal_layers)
+ : num_temporal_layers_(num_temporal_layers) {
+ EXPECT_GT(num_temporal_layers, 0u);
+ }
+
+ private:
+ std::vector<VideoStream> CreateEncoderStreams(
+ int width,
+ int height,
+ const VideoEncoderConfig& encoder_config) override {
+ std::vector<VideoStream> streams =
+ test::CreateVideoStreams(width, height, encoder_config);
+ for (VideoStream& stream : streams) {
+ stream.temporal_layer_thresholds_bps.resize(num_temporal_layers_ - 1);
+ }
+ return streams;
+ }
+ const size_t num_temporal_layers_;
+};
+
} // namespace
class ViEEncoderTest : public ::testing::Test {
@@ -94,13 +120,35 @@
VideoEncoderConfig video_encoder_config;
test::FillEncoderConfiguration(1, &video_encoder_config);
video_encoder_config_ = video_encoder_config.Copy();
+ ConfigureEncoder(std::move(video_encoder_config), true /* nack_enabled */);
+ }
+
+ void ConfigureEncoder(VideoEncoderConfig video_encoder_config,
+ bool nack_enabled) {
+ if (vie_encoder_)
+ vie_encoder_->Stop();
vie_encoder_.reset(new ViEEncoderUnderTest(
stats_proxy_.get(), video_send_config_.encoder_settings));
vie_encoder_->SetSink(&sink_, false /* rotation_applied */);
vie_encoder_->SetSource(&video_source_,
VideoSendStream::DegradationPreference::kBalanced);
vie_encoder_->SetStartBitrate(10000);
- vie_encoder_->ConfigureEncoder(std::move(video_encoder_config), 1440);
+ vie_encoder_->ConfigureEncoder(std::move(video_encoder_config),
+ kMaxPayloadLength, nack_enabled);
+ }
+
+ void ResetEncoder(const std::string& payload_name,
+ size_t num_streams,
+ size_t num_temporal_layers,
+ bool nack_enabled) {
+ video_send_config_.encoder_settings.payload_name = payload_name;
+
+ VideoEncoderConfig video_encoder_config;
+ video_encoder_config.number_of_streams = num_streams;
+ video_encoder_config.max_bitrate_bps = 1000000;
+ video_encoder_config.video_stream_factory =
+ new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers);
+ ConfigureEncoder(std::move(video_encoder_config), nack_enabled);
}
VideoFrame CreateFrame(int64_t ntp_ts, rtc::Event* destruction_event) const {
@@ -248,7 +296,6 @@
};
TEST_F(ViEEncoderTest, EncodeOneFrame) {
- const int kTargetBitrateBps = 100000;
vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
rtc::Event frame_destroyed_event(false, false);
video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
@@ -263,7 +310,6 @@
video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
- const int kTargetBitrateBps = 100000;
vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
@@ -272,7 +318,6 @@
}
TEST_F(ViEEncoderTest, DropsFramesWhenRateSetToZero) {
- const int kTargetBitrateBps = 100000;
vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sink_.WaitForEncodedFrame(1);
@@ -288,7 +333,6 @@
}
TEST_F(ViEEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
- const int kTargetBitrateBps = 100000;
vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sink_.WaitForEncodedFrame(1);
@@ -302,7 +346,6 @@
}
TEST_F(ViEEncoderTest, DropsFrameAfterStop) {
- const int kTargetBitrateBps = 100000;
vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
@@ -316,7 +359,6 @@
}
TEST_F(ViEEncoderTest, DropsPendingFramesOnSlowEncode) {
- const int kTargetBitrateBps = 100000;
vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
fake_encoder_.BlockNextEncode();
@@ -333,7 +375,6 @@
}
TEST_F(ViEEncoderTest, ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
- const int kTargetBitrateBps = 100000;
vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
EXPECT_EQ(0, sink_.number_of_reconfigurations());
@@ -347,7 +388,8 @@
VideoEncoderConfig video_encoder_config;
test::FillEncoderConfiguration(1, &video_encoder_config);
video_encoder_config.min_transmit_bitrate_bps = 9999;
- vie_encoder_->ConfigureEncoder(std::move(video_encoder_config), 1440);
+ vie_encoder_->ConfigureEncoder(std::move(video_encoder_config),
+ kMaxPayloadLength, true /* nack_enabled */);
// Capture a frame and wait for it to synchronize with the encoder thread.
video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
@@ -359,7 +401,6 @@
}
TEST_F(ViEEncoderTest, FrameResolutionChangeReconfigureEncoder) {
- const int kTargetBitrateBps = 100000;
vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
// Capture a frame and wait for it to synchronize with the encoder thread.
@@ -383,6 +424,86 @@
vie_encoder_->Stop();
}
+TEST_F(ViEEncoderTest, Vp8ResilienceIsOffFor1S1TLWithNackEnabled) {
+ const bool kNackEnabled = true;
+ const size_t kNumStreams = 1;
+ const size_t kNumTl = 1;
+ ResetEncoder("VP8", kNumStreams, kNumTl, kNackEnabled);
+ vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+ // Capture a frame and wait for it to synchronize with the encoder thread.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ sink_.WaitForEncodedFrame(1);
+ // The encoder have been configured once when the first frame is received.
+ EXPECT_EQ(1, sink_.number_of_reconfigurations());
+ EXPECT_EQ(kVideoCodecVP8, fake_encoder_.codec_config().codecType);
+ EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
+ EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP8()->numberOfTemporalLayers);
+ // Resilience is off for no temporal layers with nack on.
+ EXPECT_EQ(kResilienceOff, fake_encoder_.codec_config().VP8()->resilience);
+ vie_encoder_->Stop();
+}
+
+TEST_F(ViEEncoderTest, Vp8ResilienceIsOffFor2S1TlWithNackEnabled) {
+ const bool kNackEnabled = true;
+ const size_t kNumStreams = 2;
+ const size_t kNumTl = 1;
+ ResetEncoder("VP8", kNumStreams, kNumTl, kNackEnabled);
+ vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+ // Capture a frame and wait for it to synchronize with the encoder thread.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ sink_.WaitForEncodedFrame(1);
+ // The encoder have been configured once when the first frame is received.
+ EXPECT_EQ(1, sink_.number_of_reconfigurations());
+ EXPECT_EQ(kVideoCodecVP8, fake_encoder_.codec_config().codecType);
+ EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
+ EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP8()->numberOfTemporalLayers);
+ // Resilience is off for no temporal layers and >1 streams with nack on.
+ EXPECT_EQ(kResilienceOff, fake_encoder_.codec_config().VP8()->resilience);
+ vie_encoder_->Stop();
+}
+
+TEST_F(ViEEncoderTest, Vp8ResilienceIsOnFor1S1TLWithNackDisabled) {
+ const bool kNackEnabled = false;
+ const size_t kNumStreams = 1;
+ const size_t kNumTl = 1;
+ ResetEncoder("VP8", kNumStreams, kNumTl, kNackEnabled);
+ vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+ // Capture a frame and wait for it to synchronize with the encoder thread.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ sink_.WaitForEncodedFrame(1);
+ // The encoder have been configured once when the first frame is received.
+ EXPECT_EQ(1, sink_.number_of_reconfigurations());
+ EXPECT_EQ(kVideoCodecVP8, fake_encoder_.codec_config().codecType);
+ EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
+ EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP8()->numberOfTemporalLayers);
+ // Resilience is on for no temporal layers with nack off.
+ EXPECT_EQ(kResilientStream, fake_encoder_.codec_config().VP8()->resilience);
+ vie_encoder_->Stop();
+}
+
+TEST_F(ViEEncoderTest, Vp8ResilienceIsOnFor1S2TlWithNackEnabled) {
+ const bool kNackEnabled = true;
+ const size_t kNumStreams = 1;
+ const size_t kNumTl = 2;
+ ResetEncoder("VP8", kNumStreams, kNumTl, kNackEnabled);
+ vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
+
+ // Capture a frame and wait for it to synchronize with the encoder thread.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ sink_.WaitForEncodedFrame(1);
+ // The encoder have been configured once when the first frame is received.
+ EXPECT_EQ(1, sink_.number_of_reconfigurations());
+ EXPECT_EQ(kVideoCodecVP8, fake_encoder_.codec_config().codecType);
+ EXPECT_EQ(kNumStreams, fake_encoder_.codec_config().numberOfSimulcastStreams);
+ EXPECT_EQ(kNumTl, fake_encoder_.codec_config().VP8()->numberOfTemporalLayers);
+ // Resilience is on for temporal layers.
+ EXPECT_EQ(kResilientStream, fake_encoder_.codec_config().VP8()->resilience);
+ vie_encoder_->Stop();
+}
+
TEST_F(ViEEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
EXPECT_TRUE(video_source_.has_sinks());
test::FrameForwarder new_video_source;
@@ -402,7 +523,6 @@
}
TEST_F(ViEEncoderTest, SinkWantsFromOveruseDetector) {
- const int kTargetBitrateBps = 100000;
vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
EXPECT_FALSE(video_source_.sink_wants().max_pixel_count);
@@ -452,7 +572,6 @@
TEST_F(ViEEncoderTest,
ResolutionSinkWantsResetOnSetSourceWithDisabledResolutionScaling) {
- const int kTargetBitrateBps = 100000;
vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
EXPECT_FALSE(video_source_.sink_wants().max_pixel_count);
@@ -499,7 +618,6 @@
}
TEST_F(ViEEncoderTest, StatsTracksAdaptationStats) {
- const int kTargetBitrateBps = 100000;
vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
int frame_width = 1280;
@@ -536,7 +654,6 @@
}
TEST_F(ViEEncoderTest, StatsTracksAdaptationStatsWhenSwitchingSource) {
- const int kTargetBitrateBps = 100000;
vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
// Trigger CPU overuse.
@@ -598,7 +715,6 @@
}
TEST_F(ViEEncoderTest, StatsTracksPreferredBitrate) {
- const int kTargetBitrateBps = 100000;
vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
@@ -612,7 +728,6 @@
}
TEST_F(ViEEncoderTest, UMACpuLimitedResolutionInPercent) {
- const int kTargetBitrateBps = 100000;
vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
int frame_width = 640;