Render-side pre-processing in APM.
This CL adds a way to insert a custom render-side pre-processor to
APM. The pre-processor operates in full-band mode before anything
else. Currently the render processing chain is (if everything is
enabled):
Network --> [Pre processing] --> [Band split] -->
[IntelligibilityEnhancer] --> [Echo canceller (read-only)] -->
[Band merge] --> Playout
Since the render pre processor and capture post processor have the
same interface, I renamed webrtc::PostProcessing into
webrtc::CustomProcessing.
The old APM factory method PostProcessing will be deprecated and
dependencies updated as part of webrtc:8665
NOTRY=True
Bug: webrtc:8665
Change-Id: Ia381cbf12e336d6587406a14d77243d931f69a31
Reviewed-on: https://webrtc-review.googlesource.com/29201
Commit-Queue: Alex Loiko <aleloi@webrtc.org>
Reviewed-by: Per Åhgren <peah@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#21327}diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc
index d61ab97..2fbf369 100644
--- a/modules/audio_processing/audio_processing_impl.cc
+++ b/modules/audio_processing/audio_processing_impl.cc
@@ -172,8 +172,10 @@
static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero");
AudioProcessingImpl::ApmSubmoduleStates::ApmSubmoduleStates(
- bool capture_post_processor_enabled)
- : capture_post_processor_enabled_(capture_post_processor_enabled) {}
+ bool capture_post_processor_enabled,
+ bool render_pre_processor_enabled)
+ : capture_post_processor_enabled_(capture_post_processor_enabled),
+ render_pre_processor_enabled_(render_pre_processor_enabled) {}
bool AudioProcessingImpl::ApmSubmoduleStates::Update(
bool low_cut_filter_enabled,
@@ -264,6 +266,11 @@
echo_controller_enabled_;
}
+bool AudioProcessingImpl::ApmSubmoduleStates::RenderFullBandProcessingActive()
+ const {
+ return render_pre_processor_enabled_;
+}
+
bool AudioProcessingImpl::ApmSubmoduleStates::RenderMultiBandProcessingActive()
const {
#if WEBRTC_INTELLIGIBILITY_ENHANCER
@@ -294,9 +301,11 @@
struct AudioProcessingImpl::ApmPrivateSubmodules {
ApmPrivateSubmodules(NonlinearBeamformer* beamformer,
- std::unique_ptr<PostProcessing> capture_post_processor)
+ std::unique_ptr<CustomProcessing> capture_post_processor,
+ std::unique_ptr<CustomProcessing> render_pre_processor)
: beamformer(beamformer),
- capture_post_processor(std::move(capture_post_processor)) {}
+ capture_post_processor(std::move(capture_post_processor)),
+ render_pre_processor(std::move(render_pre_processor)) {}
// Accessed internally from capture or during initialization
std::unique_ptr<NonlinearBeamformer> beamformer;
std::unique_ptr<AgcManagerDirect> agc_manager;
@@ -305,31 +314,43 @@
std::unique_ptr<LevelController> level_controller;
std::unique_ptr<ResidualEchoDetector> residual_echo_detector;
std::unique_ptr<EchoControl> echo_controller;
- std::unique_ptr<PostProcessing> capture_post_processor;
+ std::unique_ptr<CustomProcessing> capture_post_processor;
+ std::unique_ptr<CustomProcessing> render_pre_processor;
};
AudioProcessing* AudioProcessing::Create() {
webrtc::Config config;
- return Create(config, nullptr, nullptr, nullptr);
+ return Create(config, nullptr, nullptr, nullptr, nullptr);
}
AudioProcessing* AudioProcessing::Create(const webrtc::Config& config) {
- return Create(config, nullptr, nullptr, nullptr);
+ return Create(config, nullptr, nullptr, nullptr, nullptr);
}
AudioProcessing* AudioProcessing::Create(const webrtc::Config& config,
NonlinearBeamformer* beamformer) {
- return Create(config, nullptr, nullptr, beamformer);
+ return Create(config, nullptr, nullptr, nullptr, beamformer);
}
AudioProcessing* AudioProcessing::Create(
const webrtc::Config& config,
- std::unique_ptr<PostProcessing> capture_post_processor,
+ std::unique_ptr<CustomProcessing> capture_post_processor,
+ std::unique_ptr<EchoControlFactory> echo_control_factory,
+ NonlinearBeamformer* beamformer) {
+ return Create(config, std::move(capture_post_processor), nullptr,
+ std::move(echo_control_factory), beamformer);
+}
+
+AudioProcessing* AudioProcessing::Create(
+ const webrtc::Config& config,
+ std::unique_ptr<CustomProcessing> capture_post_processor,
+ std::unique_ptr<CustomProcessing> render_pre_processor,
std::unique_ptr<EchoControlFactory> echo_control_factory,
NonlinearBeamformer* beamformer) {
AudioProcessingImpl* apm = new rtc::RefCountedObject<AudioProcessingImpl>(
config, std::move(capture_post_processor),
- std::move(echo_control_factory), beamformer);
+ std::move(render_pre_processor), std::move(echo_control_factory),
+ beamformer);
if (apm->Initialize() != kNoError) {
delete apm;
apm = nullptr;
@@ -339,20 +360,22 @@
}
AudioProcessingImpl::AudioProcessingImpl(const webrtc::Config& config)
- : AudioProcessingImpl(config, nullptr, nullptr, nullptr) {}
+ : AudioProcessingImpl(config, nullptr, nullptr, nullptr, nullptr) {}
AudioProcessingImpl::AudioProcessingImpl(
const webrtc::Config& config,
- std::unique_ptr<PostProcessing> capture_post_processor,
+ std::unique_ptr<CustomProcessing> capture_post_processor,
+ std::unique_ptr<CustomProcessing> render_pre_processor,
std::unique_ptr<EchoControlFactory> echo_control_factory,
NonlinearBeamformer* beamformer)
: high_pass_filter_impl_(new HighPassFilterImpl(this)),
echo_control_factory_(std::move(echo_control_factory)),
- submodule_states_(!!capture_post_processor),
+ submodule_states_(!!capture_post_processor, !!render_pre_processor),
public_submodules_(new ApmPublicSubmodules()),
private_submodules_(
new ApmPrivateSubmodules(beamformer,
- std::move(capture_post_processor))),
+ std::move(capture_post_processor),
+ std::move(render_pre_processor))),
constants_(config.Get<ExperimentalAgc>().startup_min_volume,
config.Get<ExperimentalAgc>().clipped_level_min,
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
@@ -405,6 +428,9 @@
RTC_LOG(LS_INFO) << "Capture post processor activated: "
<< !!private_submodules_->capture_post_processor;
+
+ RTC_LOG(LS_INFO) << "Render pre processor activated: "
+ << !!private_submodules_->render_pre_processor;
}
SetExtraOptions(config);
@@ -560,6 +586,7 @@
InitializeEchoController();
InitializeGainController2();
InitializePostProcessor();
+ InitializePreProcessor();
if (aec_dump_) {
aec_dump_->WriteInitMessage(ToStreamsConfig(formats_.api_format));
@@ -1345,7 +1372,8 @@
TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig");
rtc::CritScope cs(&crit_render_);
RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, input_config, output_config));
- if (submodule_states_.RenderMultiBandProcessingActive()) {
+ if (submodule_states_.RenderMultiBandProcessingActive() ||
+ submodule_states_.RenderFullBandProcessingActive()) {
render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(),
dest);
} else if (formats_.api_format.reverse_input_stream() !=
@@ -1434,7 +1462,8 @@
render_.render_audio->DeinterleaveFrom(frame);
RETURN_ON_ERR(ProcessRenderStreamLocked());
render_.render_audio->InterleaveTo(
- frame, submodule_states_.RenderMultiBandProcessingActive());
+ frame, submodule_states_.RenderMultiBandProcessingActive() ||
+ submodule_states_.RenderFullBandProcessingActive());
return kNoError;
}
@@ -1443,6 +1472,10 @@
QueueNonbandedRenderAudio(render_buffer);
+ if (private_submodules_->render_pre_processor) {
+ private_submodules_->render_pre_processor->Process(render_buffer);
+ }
+
if (submodule_states_.RenderMultiBandSubModulesActive() &&
SampleRateSupportsMultiBand(
formats_.render_processing_format.sample_rate_hz())) {
@@ -1792,6 +1825,14 @@
}
}
+void AudioProcessingImpl::InitializePreProcessor() {
+ if (private_submodules_->render_pre_processor) {
+ private_submodules_->render_pre_processor->Initialize(
+ formats_.render_processing_format.sample_rate_hz(),
+ formats_.render_processing_format.num_channels());
+ }
+}
+
void AudioProcessingImpl::MaybeUpdateHistograms() {
static const int kMinDiffDelayMs = 60;
diff --git a/modules/audio_processing/audio_processing_impl.h b/modules/audio_processing/audio_processing_impl.h
index 021a520..c05d238 100644
--- a/modules/audio_processing/audio_processing_impl.h
+++ b/modules/audio_processing/audio_processing_impl.h
@@ -42,7 +42,8 @@
// AudioProcessingImpl takes ownership of capture post processor and
// beamformer.
AudioProcessingImpl(const webrtc::Config& config,
- std::unique_ptr<PostProcessing> capture_post_processor,
+ std::unique_ptr<CustomProcessing> capture_post_processor,
+ std::unique_ptr<CustomProcessing> render_pre_processor,
std::unique_ptr<EchoControlFactory> echo_control_factory,
NonlinearBeamformer* beamformer);
~AudioProcessingImpl() override;
@@ -148,7 +149,8 @@
class ApmSubmoduleStates {
public:
- explicit ApmSubmoduleStates(bool capture_post_processor_enabled);
+ ApmSubmoduleStates(bool capture_post_processor_enabled,
+ bool render_pre_processor_enabled);
// Updates the submodule state and returns true if it has changed.
bool Update(bool low_cut_filter_enabled,
bool echo_canceller_enabled,
@@ -168,10 +170,12 @@
bool CaptureMultiBandProcessingActive() const;
bool CaptureFullBandProcessingActive() const;
bool RenderMultiBandSubModulesActive() const;
+ bool RenderFullBandProcessingActive() const;
bool RenderMultiBandProcessingActive() const;
private:
const bool capture_post_processor_enabled_ = false;
+ const bool render_pre_processor_enabled_ = false;
bool low_cut_filter_enabled_ = false;
bool echo_canceller_enabled_ = false;
bool mobile_echo_controller_enabled_ = false;
@@ -228,6 +232,7 @@
void InitializeEchoController() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
void InitializeGainController2() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
void InitializePostProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
+ void InitializePreProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
void EmptyQueuedRenderAudio();
void AllocateRenderQueue()
diff --git a/modules/audio_processing/audio_processing_unittest.cc b/modules/audio_processing/audio_processing_unittest.cc
index 5984ed7..b19a56b 100644
--- a/modules/audio_processing/audio_processing_unittest.cc
+++ b/modules/audio_processing/audio_processing_unittest.cc
@@ -1318,7 +1318,7 @@
testing::NiceMock<MockNonlinearBeamformer>* beamformer =
new testing::NiceMock<MockNonlinearBeamformer>(geometry, 1u);
std::unique_ptr<AudioProcessing> apm(
- AudioProcessing::Create(config, nullptr, nullptr, beamformer));
+ AudioProcessing::Create(config, nullptr, nullptr, nullptr, beamformer));
EXPECT_EQ(kNoErr, apm->gain_control()->Enable(true));
ChannelBuffer<float> src_buf(kSamplesPerChannel, kNumInputChannels);
ChannelBuffer<float> dest_buf(kSamplesPerChannel, kNumOutputChannels);
@@ -2912,11 +2912,11 @@
// Verify that apm uses a capture post processing module if one is provided.
webrtc::Config webrtc_config;
auto mock_post_processor_ptr =
- new testing::NiceMock<test::MockPostProcessing>();
+ new testing::NiceMock<test::MockCustomProcessing>();
auto mock_post_processor =
- std::unique_ptr<PostProcessing>(mock_post_processor_ptr);
+ std::unique_ptr<CustomProcessing>(mock_post_processor_ptr);
rtc::scoped_refptr<AudioProcessing> apm = AudioProcessing::Create(
- webrtc_config, std::move(mock_post_processor), nullptr, nullptr);
+ webrtc_config, std::move(mock_post_processor), nullptr, nullptr, nullptr);
AudioFrame audio;
audio.num_channels_ = 1;
@@ -2926,6 +2926,24 @@
apm->ProcessStream(&audio);
}
+TEST(ApmConfiguration, EnablePreProcessing) {
+ // Verify that apm uses a capture post processing module if one is provided.
+ webrtc::Config webrtc_config;
+ auto mock_pre_processor_ptr =
+ new testing::NiceMock<test::MockCustomProcessing>();
+ auto mock_pre_processor =
+ std::unique_ptr<CustomProcessing>(mock_pre_processor_ptr);
+ rtc::scoped_refptr<AudioProcessing> apm = AudioProcessing::Create(
+ webrtc_config, nullptr, std::move(mock_pre_processor), nullptr, nullptr);
+
+ AudioFrame audio;
+ audio.num_channels_ = 1;
+ SetFrameSampleRate(&audio, AudioProcessing::NativeRate::kSampleRate16kHz);
+
+ EXPECT_CALL(*mock_pre_processor_ptr, Process(testing::_)).Times(1);
+ apm->ProcessReverseStream(&audio);
+}
+
class MyEchoControlFactory : public EchoControlFactory {
public:
std::unique_ptr<EchoControl> Create(int sample_rate_hz) {
@@ -2943,8 +2961,9 @@
std::unique_ptr<EchoControlFactory> echo_control_factory(
new MyEchoControlFactory());
- rtc::scoped_refptr<AudioProcessing> apm = AudioProcessing::Create(
- webrtc_config, nullptr, std::move(echo_control_factory), nullptr);
+ rtc::scoped_refptr<AudioProcessing> apm =
+ AudioProcessing::Create(webrtc_config, nullptr, nullptr,
+ std::move(echo_control_factory), nullptr);
AudioFrame audio;
audio.num_channels_ = 1;
diff --git a/modules/audio_processing/include/audio_processing.h b/modules/audio_processing/include/audio_processing.h
index 3b8d2c4..4a3a30b 100644
--- a/modules/audio_processing/include/audio_processing.h
+++ b/modules/audio_processing/include/audio_processing.h
@@ -52,9 +52,12 @@
class HighPassFilter;
class LevelEstimator;
class NoiseSuppression;
-class PostProcessing;
+class CustomProcessing;
class VoiceDetection;
+// webrtc:8665, addedd temporarily to avoid breaking dependencies.
+typedef CustomProcessing PostProcessing;
+
// Use to enable the extended filter mode in the AEC, along with robustness
// measures around the reported system delays. It comes with a significant
// increase in AEC complexity, but is much more robust to unreliable reported
@@ -317,14 +320,24 @@
static AudioProcessing* Create();
// Allows passing in an optional configuration at create-time.
static AudioProcessing* Create(const webrtc::Config& config);
- // Deprecated. Use the Create below, with nullptr PostProcessing.
+ // Deprecated. Use the Create below, with nullptr CustomProcessing.
RTC_DEPRECATED
static AudioProcessing* Create(const webrtc::Config& config,
NonlinearBeamformer* beamformer);
+
+ // Will be deprecated and removed as part of webrtc:8665. Use the
+ // Create below, with nullptr CustomProcessing.
+ static AudioProcessing* Create(
+ const webrtc::Config& config,
+ std::unique_ptr<CustomProcessing> capture_post_processor,
+ std::unique_ptr<EchoControlFactory> echo_control_factory,
+ NonlinearBeamformer* beamformer);
+
// Allows passing in optional user-defined processing modules.
static AudioProcessing* Create(
const webrtc::Config& config,
- std::unique_ptr<PostProcessing> capture_post_processor,
+ std::unique_ptr<CustomProcessing> capture_post_processor,
+ std::unique_ptr<CustomProcessing> render_pre_processor,
std::unique_ptr<EchoControlFactory> echo_control_factory,
NonlinearBeamformer* beamformer);
~AudioProcessing() override {}
@@ -1087,8 +1100,8 @@
virtual ~NoiseSuppression() {}
};
-// Interface for a post processing submodule.
-class PostProcessing {
+// Interface for a custom processing submodule.
+class CustomProcessing {
public:
// (Re-)Initializes the submodule.
virtual void Initialize(int sample_rate_hz, int num_channels) = 0;
@@ -1097,7 +1110,7 @@
// Returns a string representation of the module state.
virtual std::string ToString() const = 0;
- virtual ~PostProcessing() {}
+ virtual ~CustomProcessing() {}
};
// The voice activity detection (VAD) component analyzes the stream to
diff --git a/modules/audio_processing/include/mock_audio_processing.h b/modules/audio_processing/include/mock_audio_processing.h
index f2bdc2f..8a1e463 100644
--- a/modules/audio_processing/include/mock_audio_processing.h
+++ b/modules/audio_processing/include/mock_audio_processing.h
@@ -105,9 +105,9 @@
MOCK_METHOD0(NoiseEstimate, std::vector<float>());
};
-class MockPostProcessing : public PostProcessing {
+class MockCustomProcessing : public CustomProcessing {
public:
- virtual ~MockPostProcessing() {}
+ virtual ~MockCustomProcessing() {}
MOCK_METHOD2(Initialize, void(int sample_rate_hz, int num_channels));
MOCK_METHOD1(Process, void(AudioBuffer* audio));
MOCK_CONST_METHOD0(ToString, std::string());
diff --git a/modules/audio_processing/test/audio_processing_simulator.cc b/modules/audio_processing/test/audio_processing_simulator.cc
index af10243..2477a1f 100644
--- a/modules/audio_processing/test/audio_processing_simulator.cc
+++ b/modules/audio_processing/test/audio_processing_simulator.cc
@@ -348,7 +348,7 @@
apm_config.residual_echo_detector.enabled = *settings_.use_ed;
}
- ap_.reset(AudioProcessing::Create(config, nullptr,
+ ap_.reset(AudioProcessing::Create(config, nullptr, nullptr,
std::move(echo_control_factory), nullptr));
RTC_CHECK(ap_);
diff --git a/modules/audio_processing/test/debug_dump_test.cc b/modules/audio_processing/test/debug_dump_test.cc
index 2c839d3..b901fd0 100644
--- a/modules/audio_processing/test/debug_dump_test.cc
+++ b/modules/audio_processing/test/debug_dump_test.cc
@@ -142,6 +142,7 @@
apm_(AudioProcessing::Create(
config,
nullptr,
+ nullptr,
(enable_aec3 ? std::unique_ptr<EchoControlFactory>(
new EchoCanceller3Factory())
: nullptr),