Remove simulcast constraints in SimulcastEncoderAdapter
The lowest and highest resolution layers are also identified instead
of assuming they are the first and last ones.
Bug: webrtc:10069
Change-Id: If9c76d647415c5065b79dc71850709db6bf16f61
Reviewed-on: https://webrtc-review.googlesource.com/c/114429
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Florent Castelli <orphis@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26343}diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc
index d73c59a..895489c 100644
--- a/media/engine/simulcast_encoder_adapter.cc
+++ b/media/engine/simulcast_encoder_adapter.cc
@@ -66,21 +66,6 @@
return streams;
}
-bool ValidSimulcastResolutions(const webrtc::VideoCodec& codec,
- int num_streams) {
- if (codec.width != codec.simulcastStream[num_streams - 1].width ||
- codec.height != codec.simulcastStream[num_streams - 1].height) {
- return false;
- }
- for (int i = 0; i < num_streams; ++i) {
- if (codec.width * codec.simulcastStream[i].height !=
- codec.height * codec.simulcastStream[i].width) {
- return false;
- }
- }
- return true;
-}
-
int VerifyCodec(const webrtc::VideoCodec* inst) {
if (inst == nullptr) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
@@ -102,6 +87,12 @@
return WEBRTC_VIDEO_CODEC_OK;
}
+bool StreamResolutionCompare(const webrtc::SimulcastStream& a,
+ const webrtc::SimulcastStream& b) {
+ return std::tie(a.height, a.width, a.maxBitrate, a.maxFramerate) <
+ std::tie(b.height, b.width, b.maxBitrate, b.maxFramerate);
+}
+
// An EncodedImageCallback implementation that forwards on calls to a
// SimulcastEncoderAdapter, but with the stream index it's registered with as
// the first parameter to Encoded.
@@ -195,10 +186,6 @@
RTC_DCHECK_LE(number_of_streams, kMaxSimulcastStreams);
const bool doing_simulcast = (number_of_streams > 1);
- if (doing_simulcast && !ValidSimulcastResolutions(*inst, number_of_streams)) {
- return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
- }
-
codec_ = *inst;
SimulcastRateAllocator rate_allocator(codec_);
VideoBitrateAllocation allocation = rate_allocator.GetAllocation(
@@ -212,6 +199,19 @@
encoder_info_.supports_native_handle = true;
encoder_info_.scaling_settings.thresholds = absl::nullopt;
// Create |number_of_streams| of encoder instances and init them.
+
+ const auto minmax = std::minmax_element(
+ std::begin(codec_.simulcastStream),
+ std::begin(codec_.simulcastStream) + number_of_streams,
+ StreamResolutionCompare);
+ const auto lowest_resolution_stream_index =
+ std::distance(std::begin(codec_.simulcastStream), minmax.first);
+ const auto highest_resolution_stream_index =
+ std::distance(std::begin(codec_.simulcastStream), minmax.second);
+
+ RTC_DCHECK_LT(lowest_resolution_stream_index, number_of_streams);
+ RTC_DCHECK_LT(highest_resolution_stream_index, number_of_streams);
+
for (int i = 0; i < number_of_streams; ++i) {
VideoCodec stream_codec;
uint32_t start_bitrate_kbps = start_bitrates[i];
@@ -223,11 +223,16 @@
} else {
// Cap start bitrate to the min bitrate in order to avoid strange codec
// behavior. Since sending will be false, this should not matter.
+ StreamResolution stream_resolution =
+ i == highest_resolution_stream_index
+ ? StreamResolution::HIGHEST
+ : i == lowest_resolution_stream_index ? StreamResolution::LOWEST
+ : StreamResolution::OTHER;
+
start_bitrate_kbps =
std::max(codec_.simulcastStream[i].minBitrate, start_bitrate_kbps);
- bool highest_resolution_stream = (i == (number_of_streams - 1));
- PopulateStreamCodec(codec_, i, start_bitrate_kbps,
- highest_resolution_stream, &stream_codec);
+ PopulateStreamCodec(codec_, i, start_bitrate_kbps, stream_resolution,
+ &stream_codec);
}
// TODO(ronghuawu): Remove once this is handled in LibvpxVp8Encoder.
@@ -511,7 +516,7 @@
const webrtc::VideoCodec& inst,
int stream_index,
uint32_t start_bitrate_kbps,
- bool highest_resolution_stream,
+ StreamResolution stream_resolution,
webrtc::VideoCodec* stream_codec) {
*stream_codec = inst;
@@ -523,8 +528,7 @@
stream_codec->minBitrate = inst.simulcastStream[stream_index].minBitrate;
stream_codec->qpMax = inst.simulcastStream[stream_index].qpMax;
// Settings that are based on stream/resolution.
- const bool lowest_resolution_stream = (stream_index == 0);
- if (lowest_resolution_stream) {
+ if (stream_resolution == StreamResolution::LOWEST) {
// Settings for lowest spatial resolutions.
if (inst.mode == VideoCodecMode::kScreensharing) {
if (experimental_boosted_screenshare_qp_) {
@@ -537,7 +541,7 @@
if (inst.codecType == webrtc::kVideoCodecVP8) {
stream_codec->VP8()->numberOfTemporalLayers =
inst.simulcastStream[stream_index].numberOfTemporalLayers;
- if (!highest_resolution_stream) {
+ if (stream_resolution != StreamResolution::HIGHEST) {
// For resolutions below CIF, set the codec |complexity| parameter to
// kComplexityHigher, which maps to cpu_used = -4.
int pixels_per_frame = stream_codec->width * stream_codec->height;
diff --git a/media/engine/simulcast_encoder_adapter.h b/media/engine/simulcast_encoder_adapter.h
index 696503d..b7b6c7f 100644
--- a/media/engine/simulcast_encoder_adapter.h
+++ b/media/engine/simulcast_encoder_adapter.h
@@ -83,11 +83,17 @@
bool send_stream;
};
+ enum class StreamResolution {
+ OTHER,
+ HIGHEST,
+ LOWEST,
+ };
+
// Populate the codec settings for each simulcast stream.
void PopulateStreamCodec(const webrtc::VideoCodec& inst,
int stream_index,
uint32_t start_bitrate_kbps,
- bool highest_resolution_stream,
+ StreamResolution stream_resolution,
webrtc::VideoCodec* stream_codec);
bool Initialized() const;
diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc
index 4cbaf56..c02a585 100644
--- a/media/engine/simulcast_encoder_adapter_unittest.cc
+++ b/media/engine/simulcast_encoder_adapter_unittest.cc
@@ -428,10 +428,13 @@
// always be 0.
}
- void InitRefCodec(int stream_index, VideoCodec* ref_codec) {
+ void InitRefCodec(int stream_index,
+ VideoCodec* ref_codec,
+ bool reverse_layer_order = false) {
*ref_codec = codec_;
ref_codec->VP8()->numberOfTemporalLayers =
- kTestTemporalLayerProfile[stream_index];
+ kTestTemporalLayerProfile[reverse_layer_order ? 2 - stream_index
+ : stream_index];
ref_codec->width = codec_.simulcastStream[stream_index].width;
ref_codec->height = codec_.simulcastStream[stream_index].height;
ref_codec->maxBitrate = codec_.simulcastStream[stream_index].maxBitrate;
@@ -963,6 +966,39 @@
VerifyCodec(ref_codec, 0);
}
+TEST_F(TestSimulcastEncoderAdapterFake,
+ DoesNotAlterMaxQpForScreenshareReversedLayer) {
+ const int kHighMaxQp = 56;
+ const int kLowMaxQp = 46;
+
+ SimulcastTestFixtureImpl::DefaultSettings(
+ &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
+ kVideoCodecVP8, true /* reverse_layer_order */);
+ codec_.numberOfSimulcastStreams = 3;
+ codec_.simulcastStream[2].qpMax = kHighMaxQp;
+ codec_.mode = VideoCodecMode::kScreensharing;
+
+ EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200));
+ EXPECT_EQ(3u, helper_->factory()->encoders().size());
+
+ // Just check the lowest stream, which is the one that where the adapter
+ // might alter the max qp setting.
+ VideoCodec ref_codec;
+ InitRefCodec(2, &ref_codec, true /* reverse_layer_order */);
+ ref_codec.qpMax = kHighMaxQp;
+ ref_codec.VP8()->complexity = webrtc::VideoCodecComplexity::kComplexityHigher;
+ ref_codec.VP8()->denoisingOn = false;
+ ref_codec.startBitrate = 100; // Should equal to the target bitrate.
+ VerifyCodec(ref_codec, 2);
+
+ // Change the max qp and try again.
+ codec_.simulcastStream[2].qpMax = kLowMaxQp;
+ EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200));
+ EXPECT_EQ(3u, helper_->factory()->encoders().size());
+ ref_codec.qpMax = kLowMaxQp;
+ VerifyCodec(ref_codec, 2);
+}
+
TEST_F(TestSimulcastEncoderAdapterFake, ActivatesCorrectStreamsInInitEncode) {
// Set up common settings for three streams.
SimulcastTestFixtureImpl::DefaultSettings(
diff --git a/modules/video_coding/utility/simulcast_rate_allocator.cc b/modules/video_coding/utility/simulcast_rate_allocator.cc
index 4e8b708..873f94f 100644
--- a/modules/video_coding/utility/simulcast_rate_allocator.cc
+++ b/modules/video_coding/utility/simulcast_rate_allocator.cc
@@ -13,7 +13,9 @@
#include <stdio.h>
#include <algorithm>
#include <cstdint>
+#include <numeric>
#include <string>
+#include <tuple>
#include <vector>
#include "common_types.h" // NOLINT(build/include)
@@ -110,10 +112,21 @@
}
return;
}
+
+ // Sort the layers by maxFramerate, they might not always be from smallest
+ // to biggest
+ std::vector<size_t> layer_index(codec_.numberOfSimulcastStreams);
+ std::iota(layer_index.begin(), layer_index.end(), 0);
+ std::stable_sort(layer_index.begin(), layer_index.end(),
+ [this](size_t a, size_t b) {
+ return std::tie(codec_.simulcastStream[a].maxBitrate) <
+ std::tie(codec_.simulcastStream[b].maxBitrate);
+ });
+
// Find the first active layer. We don't allocate to inactive layers.
size_t active_layer = 0;
for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) {
- if (codec_.simulcastStream[active_layer].active) {
+ if (codec_.simulcastStream[layer_index[active_layer]].active) {
// Found the first active layer.
break;
}
@@ -127,7 +140,8 @@
// active layer. Suspending below min bitrate is controlled outside the
// codec implementation and is not overridden by this.
left_to_allocate = std::max(
- codec_.simulcastStream[active_layer].minBitrate * 1000, left_to_allocate);
+ codec_.simulcastStream[layer_index[active_layer]].minBitrate * 1000,
+ left_to_allocate);
// Begin by allocating bitrate to simulcast streams, putting all bitrate in
// temporal layer 0. We'll then distribute this bitrate, across potential
@@ -144,15 +158,16 @@
size_t top_active_layer = active_layer;
// Allocate up to the target bitrate for each active simulcast layer.
for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) {
- const SimulcastStream& stream = codec_.simulcastStream[active_layer];
+ const SimulcastStream& stream =
+ codec_.simulcastStream[layer_index[active_layer]];
if (!stream.active) {
- stream_enabled_[active_layer] = false;
+ stream_enabled_[layer_index[active_layer]] = false;
continue;
}
// If we can't allocate to the current layer we can't allocate to higher
// layers because they require a higher minimum bitrate.
uint32_t min_bitrate = stream.minBitrate * 1000;
- if (!first_allocation && !stream_enabled_[active_layer]) {
+ if (!first_allocation && !stream_enabled_[layer_index[active_layer]]) {
min_bitrate = std::min(
static_cast<uint32_t>(hysteresis_factor_ * min_bitrate + 0.5),
stream.targetBitrate * 1000);
@@ -162,18 +177,19 @@
}
// We are allocating to this layer so it is the current active allocation.
- top_active_layer = active_layer;
- stream_enabled_[active_layer] = true;
+ top_active_layer = layer_index[active_layer];
+ stream_enabled_[layer_index[active_layer]] = true;
uint32_t allocation =
std::min(left_to_allocate, stream.targetBitrate * 1000);
- allocated_bitrates_bps->SetBitrate(active_layer, 0, allocation);
+ allocated_bitrates_bps->SetBitrate(layer_index[active_layer], 0,
+ allocation);
RTC_DCHECK_LE(allocation, left_to_allocate);
left_to_allocate -= allocation;
}
// All layers above this one are not active.
for (; active_layer < codec_.numberOfSimulcastStreams; ++active_layer) {
- stream_enabled_[active_layer] = false;
+ stream_enabled_[layer_index[active_layer]] = false;
}
// Next, try allocate remaining bitrate, up to max bitrate, in top active
diff --git a/modules/video_coding/utility/simulcast_test_fixture_impl.cc b/modules/video_coding/utility/simulcast_test_fixture_impl.cc
index be50286..781c7d0 100644
--- a/modules/video_coding/utility/simulcast_test_fixture_impl.cc
+++ b/modules/video_coding/utility/simulcast_test_fixture_impl.cc
@@ -212,7 +212,8 @@
void SimulcastTestFixtureImpl::DefaultSettings(
VideoCodec* settings,
const int* temporal_layer_profile,
- VideoCodecType codec_type) {
+ VideoCodecType codec_type,
+ bool reverse_layer_order) {
RTC_CHECK(settings);
memset(settings, 0, sizeof(VideoCodec));
settings->codecType = codec_type;
@@ -227,26 +228,34 @@
settings->numberOfSimulcastStreams = kNumberOfSimulcastStreams;
settings->active = true;
ASSERT_EQ(3, kNumberOfSimulcastStreams);
+ int layer_order[3] = {0, 1, 2};
+ if (reverse_layer_order) {
+ layer_order[0] = 2;
+ layer_order[2] = 0;
+ }
settings->timing_frame_thresholds = {kDefaultTimingFramesDelayMs,
kDefaultOutlierFrameSizePercent};
ConfigureStream(kDefaultWidth / 4, kDefaultHeight / 4, kMaxBitrates[0],
kMinBitrates[0], kTargetBitrates[0],
- &settings->simulcastStream[0], temporal_layer_profile[0]);
+ &settings->simulcastStream[layer_order[0]],
+ temporal_layer_profile[0]);
ConfigureStream(kDefaultWidth / 2, kDefaultHeight / 2, kMaxBitrates[1],
kMinBitrates[1], kTargetBitrates[1],
- &settings->simulcastStream[1], temporal_layer_profile[1]);
+ &settings->simulcastStream[layer_order[1]],
+ temporal_layer_profile[1]);
ConfigureStream(kDefaultWidth, kDefaultHeight, kMaxBitrates[2],
kMinBitrates[2], kTargetBitrates[2],
- &settings->simulcastStream[2], temporal_layer_profile[2]);
- if (codec_type == kVideoCodecVP8) {
- settings->VP8()->denoisingOn = true;
- settings->VP8()->automaticResizeOn = false;
- settings->VP8()->frameDroppingOn = true;
- settings->VP8()->keyFrameInterval = 3000;
- } else {
- settings->H264()->frameDroppingOn = true;
- settings->H264()->keyFrameInterval = 3000;
- }
+ &settings->simulcastStream[layer_order[2]],
+ temporal_layer_profile[2]);
+ if (codec_type == kVideoCodecVP8) {
+ settings->VP8()->denoisingOn = true;
+ settings->VP8()->automaticResizeOn = false;
+ settings->VP8()->frameDroppingOn = true;
+ settings->VP8()->keyFrameInterval = 3000;
+ } else {
+ settings->H264()->frameDroppingOn = true;
+ settings->H264()->keyFrameInterval = 3000;
+ }
}
SimulcastTestFixtureImpl::SimulcastTestFixtureImpl(
diff --git a/modules/video_coding/utility/simulcast_test_fixture_impl.h b/modules/video_coding/utility/simulcast_test_fixture_impl.h
index 2f834bd..8881e06 100644
--- a/modules/video_coding/utility/simulcast_test_fixture_impl.h
+++ b/modules/video_coding/utility/simulcast_test_fixture_impl.h
@@ -55,7 +55,8 @@
static void DefaultSettings(VideoCodec* settings,
const int* temporal_layer_profile,
- VideoCodecType codec_type);
+ VideoCodecType codec_type,
+ bool reverse_layer_order = false);
private:
class TestEncodedImageCallback;