Encoder side of Multistream Opus.
Follows https://webrtc-review.googlesource.com/c/src/+/129768 closely.
Adds an ENCODER and sets it up to parse SDP config for multistream
opus.
E.g. this is the new SDP syntax for 6.1 surround sound:
"multiopus/48000/6 channel_mapping=0,4,1,2,3,5 num_streams=4 coupled_streams=2"
Bug: webrtc:8649
Change-Id: I3fc341e76f5c41dab0243cf65f6461e4c3d9d67d
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/132001
Commit-Queue: Alex Loiko <aleloi@webrtc.org>
Reviewed-by: Oskar Sundbom <ossu@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27775}
diff --git a/api/audio_codecs/opus/BUILD.gn b/api/audio_codecs/opus/BUILD.gn
index 498e45a..01a4884 100644
--- a/api/audio_codecs/opus/BUILD.gn
+++ b/api/audio_codecs/opus/BUILD.gn
@@ -15,7 +15,8 @@
rtc_static_library("audio_encoder_opus_config") {
visibility = [ "*" ]
sources = [
- "audio_decoder_multi_channel_opus_config.h",
+ "audio_encoder_multi_channel_opus_config.cc",
+ "audio_encoder_multi_channel_opus_config.h",
"audio_encoder_opus_config.cc",
"audio_encoder_opus_config.h",
]
@@ -32,6 +33,13 @@
}
}
+rtc_source_set("audio_decoder_opus_config") {
+ visibility = [ "*" ]
+ sources = [
+ "audio_decoder_multi_channel_opus_config.h",
+ ]
+}
+
rtc_source_set("audio_encoder_opus") {
visibility = [ "*" ]
poisonous = [ "audio_codecs" ]
@@ -70,6 +78,25 @@
]
}
+rtc_source_set("audio_encoder_multiopus") {
+ visibility = [ "*" ]
+ poisonous = [ "audio_codecs" ]
+ public = [
+ "audio_encoder_multi_channel_opus.h",
+ ]
+ sources = [
+ "audio_encoder_multi_channel_opus.cc",
+ ]
+ deps = [
+ "..:audio_codecs_api",
+ "../../../modules/audio_coding:webrtc_multiopus",
+ "../../../rtc_base:rtc_base_approved",
+ "../../../rtc_base/system:rtc_export",
+ "../opus:audio_encoder_opus_config",
+ "//third_party/abseil-cpp/absl/types:optional",
+ ]
+}
+
rtc_static_library("audio_decoder_multiopus") {
visibility = [ "*" ]
poisonous = [ "audio_codecs" ]
@@ -78,7 +105,7 @@
"audio_decoder_multi_channel_opus.h",
]
deps = [
- ":audio_encoder_opus_config",
+ ":audio_decoder_opus_config",
"..:audio_codecs_api",
"../../../modules/audio_coding:webrtc_multiopus",
"../../../rtc_base:rtc_base_approved",
diff --git a/api/audio_codecs/opus/audio_encoder_multi_channel_opus.cc b/api/audio_codecs/opus/audio_encoder_multi_channel_opus.cc
new file mode 100644
index 0000000..758eaae
--- /dev/null
+++ b/api/audio_codecs/opus/audio_encoder_multi_channel_opus.cc
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/audio_codecs/opus/audio_encoder_multi_channel_opus.h"
+
+#include <utility>
+
+#include "modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.h"
+
+namespace webrtc {
+
+absl::optional<AudioEncoderMultiChannelOpusConfig>
+AudioEncoderMultiChannelOpus::SdpToConfig(const SdpAudioFormat& format) {
+ return AudioEncoderMultiChannelOpusImpl::SdpToConfig(format);
+}
+
+void AudioEncoderMultiChannelOpus::AppendSupportedEncoders(
+ std::vector<AudioCodecSpec>* specs) {
+ // To get full utilization of the surround support of the Opus lib, we can
+ // mark which channel is the low frequency effects (LFE). But that is not done
+ // ATM.
+ {
+ AudioCodecInfo surround_5_1_opus_info{48000, 6,
+ /* default_bitrate_bps= */ 128000};
+ surround_5_1_opus_info.allow_comfort_noise = false;
+ surround_5_1_opus_info.supports_network_adaption = false;
+ SdpAudioFormat opus_format({"multiopus",
+ 48000,
+ 6,
+ {{"minptime", "10"},
+ {"useinbandfec", "1"},
+ {"channel_mapping", "0,4,1,2,3,5"},
+ {"num_streams", "4"},
+ {"coupled_streams", "2"}}});
+ specs->push_back({std::move(opus_format), surround_5_1_opus_info});
+ }
+ {
+ AudioCodecInfo surround_7_1_opus_info{48000, 8,
+ /* default_bitrate_bps= */ 200000};
+ surround_7_1_opus_info.allow_comfort_noise = false;
+ surround_7_1_opus_info.supports_network_adaption = false;
+ SdpAudioFormat opus_format({"multiopus",
+ 48000,
+ 8,
+ {{"minptime", "10"},
+ {"useinbandfec", "1"},
+ {"channel_mapping", "0,6,1,2,3,4,5,7"},
+ {"num_streams", "5"},
+ {"coupled_streams", "3"}}});
+ specs->push_back({std::move(opus_format), surround_7_1_opus_info});
+ }
+}
+
+AudioCodecInfo AudioEncoderMultiChannelOpus::QueryAudioEncoder(
+ const AudioEncoderMultiChannelOpusConfig& config) {
+ return AudioEncoderMultiChannelOpusImpl::QueryAudioEncoder(config);
+}
+
+std::unique_ptr<AudioEncoder> AudioEncoderMultiChannelOpus::MakeAudioEncoder(
+ const AudioEncoderMultiChannelOpusConfig& config,
+ int payload_type,
+ absl::optional<AudioCodecPairId> /*codec_pair_id*/) {
+ return AudioEncoderMultiChannelOpusImpl::MakeAudioEncoder(config,
+ payload_type);
+}
+
+} // namespace webrtc
diff --git a/api/audio_codecs/opus/audio_encoder_multi_channel_opus.h b/api/audio_codecs/opus/audio_encoder_multi_channel_opus.h
new file mode 100644
index 0000000..977a3a4
--- /dev/null
+++ b/api/audio_codecs/opus/audio_encoder_multi_channel_opus.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_AUDIO_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_H_
+#define API_AUDIO_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_H_
+
+#include <memory>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/audio_codecs/audio_codec_pair_id.h"
+#include "api/audio_codecs/audio_encoder.h"
+#include "api/audio_codecs/audio_format.h"
+#include "api/audio_codecs/opus/audio_encoder_multi_channel_opus_config.h"
+#include "rtc_base/system/rtc_export.h"
+
+namespace webrtc {
+
+// Opus encoder API for use as a template parameter to
+// CreateAudioEncoderFactory<...>().
+struct RTC_EXPORT AudioEncoderMultiChannelOpus {
+ using Config = AudioEncoderMultiChannelOpusConfig;
+ static absl::optional<Config> SdpToConfig(const SdpAudioFormat& audio_format);
+ static void AppendSupportedEncoders(std::vector<AudioCodecSpec>* specs);
+ static AudioCodecInfo QueryAudioEncoder(const Config& config);
+ static std::unique_ptr<AudioEncoder> MakeAudioEncoder(
+ const Config& config,
+ int payload_type,
+ absl::optional<AudioCodecPairId> codec_pair_id = absl::nullopt);
+};
+
+} // namespace webrtc
+
+#endif // API_AUDIO_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_H_
diff --git a/api/audio_codecs/opus/audio_encoder_multi_channel_opus_config.cc b/api/audio_codecs/opus/audio_encoder_multi_channel_opus_config.cc
new file mode 100644
index 0000000..f01caf1
--- /dev/null
+++ b/api/audio_codecs/opus/audio_encoder_multi_channel_opus_config.cc
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/audio_codecs/opus/audio_encoder_multi_channel_opus_config.h"
+
+namespace webrtc {
+
+namespace {
+constexpr int kDefaultComplexity = 9;
+} // namespace
+
+AudioEncoderMultiChannelOpusConfig::AudioEncoderMultiChannelOpusConfig()
+ : frame_size_ms(kDefaultFrameSizeMs),
+ num_channels(1),
+ application(ApplicationMode::kVoip),
+ bitrate_bps(32000),
+ fec_enabled(false),
+ cbr_enabled(false),
+ dtx_enabled(false),
+ max_playback_rate_hz(48000),
+ complexity(kDefaultComplexity),
+ num_streams(-1),
+ coupled_streams(-1) {}
+AudioEncoderMultiChannelOpusConfig::AudioEncoderMultiChannelOpusConfig(
+ const AudioEncoderMultiChannelOpusConfig&) = default;
+AudioEncoderMultiChannelOpusConfig::~AudioEncoderMultiChannelOpusConfig() =
+ default;
+AudioEncoderMultiChannelOpusConfig& AudioEncoderMultiChannelOpusConfig::
+operator=(const AudioEncoderMultiChannelOpusConfig&) = default;
+
+bool AudioEncoderMultiChannelOpusConfig::IsOk() const {
+ if (frame_size_ms <= 0 || frame_size_ms % 10 != 0)
+ return false;
+ if (num_channels < 0 || num_channels >= 255) {
+ return false;
+ }
+ if (bitrate_bps < kMinBitrateBps || bitrate_bps > kMaxBitrateBps)
+ return false;
+ if (complexity < 0 || complexity > 10)
+ return false;
+
+ // Check the lengths:
+ if (num_channels < 0 || num_streams < 0 || coupled_streams < 0) {
+ return false;
+ }
+ if (num_streams < coupled_streams) {
+ return false;
+ }
+ if (channel_mapping.size() != static_cast<size_t>(num_channels)) {
+ return false;
+ }
+
+ // Every mono stream codes one channel, every coupled stream codes two. This
+ // is the total coded channel count:
+ const int max_coded_channel = num_streams + coupled_streams;
+ for (const auto& x : channel_mapping) {
+ // Coded channels >= max_coded_channel don't exist. Except for 255, which
+ // tells Opus to ignore input channel x.
+ if (x >= max_coded_channel && x != 255) {
+ return false;
+ }
+ }
+
+ // Inverse mapping.
+ constexpr int kNotSet = -1;
+ std::vector<int> coded_channels_to_input_channels(max_coded_channel, kNotSet);
+ for (size_t i = 0; i < num_channels; ++i) {
+ if (channel_mapping[i] == 255) {
+ continue;
+ }
+
+ // If it's not ignored, put it in the inverted mapping. But first check if
+ // we've told Opus to use another input channel for this coded channel:
+ const int coded_channel = channel_mapping[i];
+ if (coded_channels_to_input_channels[coded_channel] != kNotSet) {
+ // Coded channel `coded_channel` comes from both input channels
+ // `coded_channels_to_input_channels[coded_channel]` and `i`.
+ return false;
+ }
+
+ coded_channels_to_input_channels[coded_channel] = i;
+ }
+
+ // Check that we specified what input the encoder should use to produce
+ // every coded channel.
+ for (int i = 0; i < max_coded_channel; ++i) {
+ if (coded_channels_to_input_channels[i] == kNotSet) {
+ // Coded channel `i` has unspecified input channel.
+ return false;
+ }
+ }
+
+ if (num_channels > 255 || max_coded_channel >= 255) {
+ return false;
+ }
+ return true;
+}
+
+} // namespace webrtc
diff --git a/api/audio_codecs/opus/audio_encoder_multi_channel_opus_config.h b/api/audio_codecs/opus/audio_encoder_multi_channel_opus_config.h
new file mode 100644
index 0000000..9b51246
--- /dev/null
+++ b/api/audio_codecs/opus/audio_encoder_multi_channel_opus_config.h
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_AUDIO_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_CONFIG_H_
+#define API_AUDIO_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_CONFIG_H_
+
+#include <stddef.h>
+
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/audio_codecs/opus/audio_encoder_opus_config.h"
+#include "rtc_base/system/rtc_export.h"
+
+namespace webrtc {
+
+struct RTC_EXPORT AudioEncoderMultiChannelOpusConfig {
+ static constexpr int kDefaultFrameSizeMs = 20;
+
+ // Opus API allows a min bitrate of 500bps, but Opus documentation suggests
+ // bitrate should be in the range of 6000 to 510000, inclusive.
+ static constexpr int kMinBitrateBps = 6000;
+ static constexpr int kMaxBitrateBps = 510000;
+
+ AudioEncoderMultiChannelOpusConfig();
+ AudioEncoderMultiChannelOpusConfig(const AudioEncoderMultiChannelOpusConfig&);
+ ~AudioEncoderMultiChannelOpusConfig();
+ AudioEncoderMultiChannelOpusConfig& operator=(
+ const AudioEncoderMultiChannelOpusConfig&);
+
+ int frame_size_ms;
+ size_t num_channels;
+ enum class ApplicationMode { kVoip, kAudio };
+ ApplicationMode application;
+ int bitrate_bps;
+ bool fec_enabled;
+ bool cbr_enabled;
+ bool dtx_enabled;
+ int max_playback_rate_hz;
+ std::vector<int> supported_frame_lengths_ms;
+
+ int complexity;
+
+ // Number of mono/stereo Opus streams.
+ int num_streams;
+
+ // Number of channel pairs coupled together, see RFC 7845 section
+ // 5.1.1. Has to be less than the number of streams
+ int coupled_streams;
+
+ // Channel mapping table, defines the mapping from encoded streams to input
+ // channels. See RFC 7845 section 5.1.1.
+ std::vector<unsigned char> channel_mapping;
+
+ bool IsOk() const;
+};
+
+} // namespace webrtc
+#endif // API_AUDIO_CODECS_OPUS_AUDIO_ENCODER_MULTI_CHANNEL_OPUS_CONFIG_H_
diff --git a/api/audio_codecs/opus/audio_encoder_opus_config.cc b/api/audio_codecs/opus/audio_encoder_opus_config.cc
index 2847cea..8d6e3a0 100644
--- a/api/audio_codecs/opus/audio_encoder_opus_config.cc
+++ b/api/audio_codecs/opus/audio_encoder_opus_config.cc
@@ -55,9 +55,9 @@
bool AudioEncoderOpusConfig::IsOk() const {
if (frame_size_ms <= 0 || frame_size_ms % 10 != 0)
return false;
- if (num_channels != 1 && num_channels != 2 && num_channels != 4 &&
- num_channels != 6 && num_channels != 8)
+ if (num_channels < 0 || num_channels >= 255) {
return false;
+ }
if (!bitrate_bps)
return false;
if (*bitrate_bps < kMinBitrateBps || *bitrate_bps > kMaxBitrateBps)