Let Opus increase complexity for low bitrates
This change adds code that lets Opus increase the complexity setting
at low bitrates (only relevant for mobile where the default complexity
is not already maximum). The feature is default off.
Also adding a performance test to make sure the complexity adaptation
has desired effect.
BUG=webrtc:6708
Review-Url: https://codereview.webrtc.org/2503443002
Cr-Commit-Position: refs/heads/master@{#15182}
diff --git a/webrtc/BUILD.gn b/webrtc/BUILD.gn
index 17e3c2a..242d8ea 100644
--- a/webrtc/BUILD.gn
+++ b/webrtc/BUILD.gn
@@ -623,6 +623,7 @@
"call/call_perf_tests.cc",
"call/rampup_tests.cc",
"call/rampup_tests.h",
+ "modules/audio_coding/codecs/opus/opus_complexity_unittest.cc",
"modules/audio_coding/neteq/test/neteq_performance_unittest.cc",
"modules/audio_processing/audio_processing_performance_unittest.cc",
"modules/audio_processing/level_controller/level_controller_complexity_unittest.cc",
diff --git a/webrtc/build/webrtc.gni b/webrtc/build/webrtc.gni
index ced2c12..5ab5d35 100644
--- a/webrtc/build/webrtc.gni
+++ b/webrtc/build/webrtc.gni
@@ -17,6 +17,9 @@
# Disable this to avoid building the Opus audio codec.
rtc_include_opus = true
+ # Enable this to let the Opus audio codec change complexity on the fly.
+ rtc_opus_variable_complexity = false
+
# Disable to use absolute header paths for some libraries.
rtc_relative_path = true
diff --git a/webrtc/modules/audio_coding/BUILD.gn b/webrtc/modules/audio_coding/BUILD.gn
index f6188a0..0e78abf 100644
--- a/webrtc/modules/audio_coding/BUILD.gn
+++ b/webrtc/modules/audio_coding/BUILD.gn
@@ -711,10 +711,17 @@
"../../base:rtc_base_approved",
]
+ defines = []
+
if (rtc_build_opus) {
public_deps = [
rtc_opus_dir,
]
+ if (rtc_opus_variable_complexity) {
+ defines += [ "WEBRTC_OPUS_VARIABLE_COMPLEXITY=1" ]
+ } else {
+ defines += [ "WEBRTC_OPUS_VARIABLE_COMPLEXITY=0" ]
+ }
} else if (build_with_mozilla) {
include_dirs = [ getenv("DIST") + "/include/opus" ]
}
diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc
index f466926..0f0958c 100644
--- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc
+++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc
@@ -44,6 +44,9 @@
config.application = config.num_channels == 1 ? AudioEncoderOpus::kVoip
: AudioEncoderOpus::kAudio;
config.supported_frame_lengths_ms.push_back(config.frame_size_ms);
+#if WEBRTC_OPUS_VARIABLE_COMPLEXITY
+ config.low_rate_complexity = 9;
+#endif
return config;
}
@@ -118,7 +121,11 @@
rtc::ExpFilter smoother_;
};
-AudioEncoderOpus::Config::Config() = default;
+AudioEncoderOpus::Config::Config() {
+#if WEBRTC_OPUS_VARIABLE_COMPLEXITY
+ low_rate_complexity = 9;
+#endif
+}
AudioEncoderOpus::Config::Config(const Config&) = default;
AudioEncoderOpus::Config::~Config() = default;
auto AudioEncoderOpus::Config::operator=(const Config&) -> Config& = default;
@@ -133,6 +140,8 @@
return false;
if (complexity < 0 || complexity > 10)
return false;
+ if (low_rate_complexity < 0 || low_rate_complexity > 10)
+ return false;
return true;
}
@@ -144,6 +153,21 @@
return num_channels == 1 ? 32000 : 64000; // Default value.
}
+rtc::Optional<int> AudioEncoderOpus::Config::GetNewComplexity() const {
+ RTC_DCHECK(IsOk());
+ const int bitrate_bps = GetBitrateBps();
+ if (bitrate_bps >=
+ complexity_threshold_bps - complexity_threshold_window_bps &&
+ bitrate_bps <=
+ complexity_threshold_bps + complexity_threshold_window_bps) {
+ // Within the hysteresis window; make no change.
+ return rtc::Optional<int>();
+ }
+ return bitrate_bps <= complexity_threshold_bps
+ ? rtc::Optional<int>(low_rate_complexity)
+ : rtc::Optional<int>(complexity);
+}
+
AudioEncoderOpus::AudioEncoderOpus(
const Config& config,
AudioNetworkAdaptorCreator&& audio_network_adaptor_creator)
@@ -250,6 +274,11 @@
std::max(std::min(bits_per_second, kMaxBitrateBps), kMinBitrateBps));
RTC_DCHECK(config_.IsOk());
RTC_CHECK_EQ(0, WebRtcOpus_SetBitRate(inst_, config_.GetBitrateBps()));
+ const auto new_complexity = config_.GetNewComplexity();
+ if (new_complexity && complexity_ != *new_complexity) {
+ complexity_ = *new_complexity;
+ RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, complexity_));
+ }
}
bool AudioEncoderOpus::EnableAudioNetworkAdaptor(
@@ -399,7 +428,10 @@
}
RTC_CHECK_EQ(
0, WebRtcOpus_SetMaxPlaybackRate(inst_, config.max_playback_rate_hz));
- RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, config.complexity));
+ // Use the default complexity if the start bitrate is within the hysteresis
+ // window.
+ complexity_ = config.GetNewComplexity().value_or(config.complexity);
+ RTC_CHECK_EQ(0, WebRtcOpus_SetComplexity(inst_, complexity_));
if (config.dtx_enabled) {
RTC_CHECK_EQ(0, WebRtcOpus_EnableDtx(inst_));
} else {
diff --git a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h
index 81ca17f..f30d657 100644
--- a/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h
+++ b/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h
@@ -12,6 +12,8 @@
#define WEBRTC_MODULES_AUDIO_CODING_CODECS_OPUS_AUDIO_ENCODER_OPUS_H_
#include <functional>
+#include <memory>
+#include <string>
#include <vector>
#include "webrtc/base/constructormagic.h"
@@ -39,6 +41,11 @@
bool IsOk() const;
int GetBitrateBps() const;
+ // Returns empty if the current bitrate falls within the hysteresis window,
+ // defined by complexity_threshold_bps +/- complexity_threshold_window_bps.
+ // Otherwise, returns the current complexity depending on whether the
+ // current bitrate is above or below complexity_threshold_bps.
+ rtc::Optional<int> GetNewComplexity() const;
int frame_size_ms = 20;
size_t num_channels = 1;
@@ -48,6 +55,11 @@
bool fec_enabled = false;
int max_playback_rate_hz = 48000;
int complexity = kDefaultComplexity;
+ // This value may change in the struct's constructor.
+ int low_rate_complexity = kDefaultComplexity;
+ // low_rate_complexity is used when the bitrate is below this threshold.
+ int complexity_threshold_bps = 12500;
+ int complexity_threshold_window_bps = 1500;
bool dtx_enabled = false;
std::vector<int> supported_frame_lengths_ms;
const Clock* clock = nullptr;
@@ -140,6 +152,7 @@
uint32_t first_timestamp_in_buffer_;
size_t num_channels_to_encode_;
int next_frame_length_ms_;
+ int complexity_;
std::unique_ptr<PacketLossFractionSmoother> packet_loss_fraction_smoother_;
AudioNetworkAdaptorCreator audio_network_adaptor_creator_;
std::unique_ptr<AudioNetworkAdaptor> audio_network_adaptor_;
diff --git a/webrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc b/webrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc
new file mode 100644
index 0000000..83a4539
--- /dev/null
+++ b/webrtc/modules/audio_coding/codecs/opus/opus_complexity_unittest.cc
@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/base/format_macros.h"
+#include "webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h"
+#include "webrtc/modules/audio_coding/neteq/tools/audio_loop.h"
+#include "webrtc/test/gtest.h"
+#include "webrtc/test/testsupport/fileutils.h"
+#include "webrtc/test/testsupport/perf_test.h"
+#include "webrtc/system_wrappers/include/clock.h"
+
+namespace webrtc {
+
+namespace {
+int64_t RunComplexityTest(const AudioEncoderOpus::Config& config) {
+ // Create encoder.
+ AudioEncoderOpus encoder(config);
+ // Open speech file.
+ const std::string kInputFileName =
+ webrtc::test::ResourcePath("audio_coding/speech_mono_32_48kHz", "pcm");
+ test::AudioLoop audio_loop;
+ constexpr int kSampleRateHz = 48000;
+ EXPECT_EQ(kSampleRateHz, encoder.SampleRateHz());
+ constexpr size_t kMaxLoopLengthSamples =
+ kSampleRateHz * 10; // 10 second loop.
+ constexpr size_t kInputBlockSizeSamples =
+ 10 * kSampleRateHz / 1000; // 60 ms.
+ EXPECT_TRUE(audio_loop.Init(kInputFileName, kMaxLoopLengthSamples,
+ kInputBlockSizeSamples));
+ // Encode.
+ webrtc::Clock* clock = webrtc::Clock::GetRealTimeClock();
+ const int64_t start_time_ms = clock->TimeInMilliseconds();
+ AudioEncoder::EncodedInfo info;
+ rtc::Buffer encoded(500);
+ uint32_t rtp_timestamp = 0u;
+ for (size_t i = 0; i < 10000; ++i) {
+ encoded.Clear();
+ info = encoder.Encode(rtp_timestamp, audio_loop.GetNextBlock(), &encoded);
+ rtp_timestamp += kInputBlockSizeSamples;
+ }
+ return clock->TimeInMilliseconds() - start_time_ms;
+}
+} // namespace
+
+// This test encodes an audio file using Opus twice with different bitrates
+// (12.5 kbps and 15.5 kbps). The runtime for each is measured, and the ratio
+// between the two is calculated and tracked. This test explicitly sets the
+// low_rate_complexity to 9. When running on desktop platforms, this is the same
+// as the regular complexity, and the expectation is that the resulting ratio
+// should be less than 100% (since the encoder runs faster at lower bitrates,
+// given a fixed complexity setting). On the other hand, when running on
+// mobiles, the regular complexity is 5, and we expect the resulting ratio to
+// be higher, since we have explicitly asked for a higher complexity setting at
+// the lower rate.
+TEST(AudioEncoderOpusComplexityAdaptationTest, AdaptationOn) {
+ // Create config.
+ AudioEncoderOpus::Config config;
+ config.bitrate_bps = rtc::Optional<int>(12500);
+ config.low_rate_complexity = 9;
+ int64_t runtime_12500bps = RunComplexityTest(config);
+
+ config.bitrate_bps = rtc::Optional<int>(15500);
+ int64_t runtime_15500bps = RunComplexityTest(config);
+
+ test::PrintResult("opus_encoding_complexity_ratio", "", "adaptation_on",
+ 100.0 * runtime_12500bps / runtime_15500bps, "percent",
+ true);
+}
+
+// This test is identical to the one above, but without the complexity
+// adaptation enabled (neither on desktop, nor on mobile). The expectation is
+// that the resulting ratio is less than 100% at all times.
+TEST(AudioEncoderOpusComplexityAdaptationTest, AdaptationOff) {
+ // Create config.
+ AudioEncoderOpus::Config config;
+ config.bitrate_bps = rtc::Optional<int>(12500);
+ int64_t runtime_12500bps = RunComplexityTest(config);
+
+ config.bitrate_bps = rtc::Optional<int>(15500);
+ int64_t runtime_15500bps = RunComplexityTest(config);
+
+ test::PrintResult("opus_encoding_complexity_ratio", "", "adaptation_off",
+ 100.0 * runtime_12500bps / runtime_15500bps, "", true);
+}
+} // namespace webrtc