Only create AECm when needed
This CL ensures that the AECm is only created when needed.
The changes in the CL are bitexact when running AECm via
audioproc_f
The CL also corrects an issue where there is a risk for
AEC2 to not be correctly setup when the sample rate
changes inbetween activations.
Bug: webrtc:8671
Change-Id: Id3b33e20969b1543e28c885d47495246cfbe549d
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/134216
Commit-Queue: Per Åhgren <peah@webrtc.org>
Reviewed-by: Sam Zackrisson <saza@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27800}
diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc
index a796ed6..9ca81e6 100644
--- a/modules/audio_processing/audio_processing_impl.cc
+++ b/modules/audio_processing/audio_processing_impl.cc
@@ -428,7 +428,6 @@
new rtc::RefCountedObject<ResidualEchoDetector>();
}
- private_submodules_->echo_control_mobile.reset(new EchoControlMobileImpl());
// TODO(alessiob): Move the injected gain controller once injection is
// implemented.
private_submodules_->gain_controller2.reset(new GainController2());
@@ -548,10 +547,6 @@
AllocateRenderQueue();
- private_submodules_->echo_control_mobile->Initialize(
- proc_split_sample_rate_hz(), num_reverse_channels(),
- num_output_channels());
-
public_submodules_->gain_control->Initialize(num_proc_channels(),
proc_sample_rate_hz());
if (constants_.use_experimental_agc) {
@@ -1036,15 +1031,15 @@
}
void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) {
- EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(),
- num_reverse_channels(),
- &aec_render_queue_buffer_);
-
RTC_DCHECK_GE(160, audio->num_frames_per_band());
// Insert the samples into the queue.
if (private_submodules_->echo_cancellation) {
RTC_DCHECK(aec_render_signal_queue_);
+ EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(),
+ num_reverse_channels(),
+ &aec_render_queue_buffer_);
+
if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) {
// The data queue is full and needs to be emptied.
EmptyQueuedRenderAudio();
@@ -1055,18 +1050,21 @@
}
}
- EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(),
- num_reverse_channels(),
- &aecm_render_queue_buffer_);
+ if (private_submodules_->echo_control_mobile) {
+ EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(),
+ num_reverse_channels(),
+ &aecm_render_queue_buffer_);
+ RTC_DCHECK(aecm_render_signal_queue_);
+ // Insert the samples into the queue.
+ if (!aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_)) {
+ // The data queue is full and needs to be emptied.
+ EmptyQueuedRenderAudio();
- // Insert the samples into the queue.
- if (!aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_)) {
- // The data queue is full and needs to be emptied.
- EmptyQueuedRenderAudio();
-
- // Retry the insert (should always work).
- bool result = aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_);
- RTC_DCHECK(result);
+ // Retry the insert (should always work).
+ bool result =
+ aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_);
+ RTC_DCHECK(result);
+ }
}
if (!constants_.use_experimental_agc) {
@@ -1098,12 +1096,6 @@
}
void AudioProcessingImpl::AllocateRenderQueue() {
- const size_t new_aecm_render_queue_element_max_size =
- std::max(static_cast<size_t>(1),
- kMaxAllowedValuesOfSamplesPerBand *
- EchoControlMobileImpl::NumCancellersRequired(
- num_output_channels(), num_reverse_channels()));
-
const size_t new_agc_render_queue_element_max_size =
std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerBand);
@@ -1112,25 +1104,6 @@
// Reallocate the queues if the queue item sizes are too small to fit the
// data to put in the queues.
- if (aecm_render_queue_element_max_size_ <
- new_aecm_render_queue_element_max_size) {
- aecm_render_queue_element_max_size_ =
- new_aecm_render_queue_element_max_size;
-
- std::vector<int16_t> template_queue_element(
- aecm_render_queue_element_max_size_);
-
- aecm_render_signal_queue_.reset(
- new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
- kMaxNumFramesToBuffer, template_queue_element,
- RenderQueueItemVerifier<int16_t>(
- aecm_render_queue_element_max_size_)));
-
- aecm_render_queue_buffer_.resize(aecm_render_queue_element_max_size_);
- aecm_capture_queue_buffer_.resize(aecm_render_queue_element_max_size_);
- } else {
- aecm_render_signal_queue_->Clear();
- }
if (agc_render_queue_element_max_size_ <
new_agc_render_queue_element_max_size) {
@@ -1181,9 +1154,12 @@
}
}
- while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) {
- private_submodules_->echo_control_mobile->ProcessRenderAudio(
- aecm_capture_queue_buffer_);
+ if (private_submodules_->echo_control_mobile) {
+ RTC_DCHECK(aecm_render_signal_queue_);
+ while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) {
+ private_submodules_->echo_control_mobile->ProcessRenderAudio(
+ aecm_capture_queue_buffer_);
+ }
}
while (agc_render_signal_queue_->Remove(&agc_capture_queue_buffer_)) {
@@ -1272,9 +1248,10 @@
// Ensure that not both the AEC and AECM are active at the same time.
// TODO(peah): Simplify once the public API Enable functions for these
// are moved to APM.
- RTC_DCHECK(!((private_submodules_->echo_cancellation &&
- private_submodules_->echo_cancellation->is_enabled()) &&
- private_submodules_->echo_control_mobile->is_enabled()));
+ RTC_DCHECK_LE(!!private_submodules_->echo_controller +
+ !!private_submodules_->echo_cancellation +
+ !!private_submodules_->echo_control_mobile,
+ 1);
MaybeUpdateHistograms();
@@ -1356,47 +1333,44 @@
public_submodules_->gain_control->AnalyzeCaptureAudio(capture_buffer));
public_submodules_->noise_suppression->AnalyzeCaptureAudio(capture_buffer);
- // Ensure that the stream delay was set before the call to the
- // AEC ProcessCaptureAudio function.
- if (private_submodules_->echo_cancellation &&
- private_submodules_->echo_cancellation->is_enabled() &&
- !private_submodules_->echo_controller && !was_stream_delay_set()) {
- return AudioProcessing::kStreamParameterNotSetError;
- }
-
- if (private_submodules_->echo_controller) {
- data_dumper_->DumpRaw("stream_delay", stream_delay_ms());
-
- if (was_stream_delay_set()) {
- private_submodules_->echo_controller->SetAudioBufferDelay(
- stream_delay_ms());
+ if (private_submodules_->echo_control_mobile) {
+ // Ensure that the stream delay was set before the call to the
+ // AECM ProcessCaptureAudio function.
+ if (!was_stream_delay_set()) {
+ return AudioProcessing::kStreamParameterNotSetError;
}
- private_submodules_->echo_controller->ProcessCapture(
- capture_buffer, capture_.echo_path_gain_change);
- } else if (private_submodules_->echo_cancellation) {
- RETURN_ON_ERR(private_submodules_->echo_cancellation->ProcessCaptureAudio(
- capture_buffer, stream_delay_ms()));
- }
+ if (public_submodules_->noise_suppression->is_enabled()) {
+ capture_buffer->CopyLowPassToReference();
+ }
- if (private_submodules_->echo_control_mobile->is_enabled() &&
- public_submodules_->noise_suppression->is_enabled()) {
- capture_buffer->CopyLowPassToReference();
- }
- public_submodules_->noise_suppression->ProcessCaptureAudio(capture_buffer);
+ public_submodules_->noise_suppression->ProcessCaptureAudio(capture_buffer);
- // Ensure that the stream delay was set before the call to the
- // AECM ProcessCaptureAudio function.
- if (private_submodules_->echo_control_mobile->is_enabled() &&
- !was_stream_delay_set()) {
- return AudioProcessing::kStreamParameterNotSetError;
- }
-
- if (!(private_submodules_->echo_controller ||
- (private_submodules_->echo_cancellation &&
- private_submodules_->echo_cancellation->is_enabled()))) {
RETURN_ON_ERR(private_submodules_->echo_control_mobile->ProcessCaptureAudio(
capture_buffer, stream_delay_ms()));
+ } else {
+ if (private_submodules_->echo_controller) {
+ data_dumper_->DumpRaw("stream_delay", stream_delay_ms());
+
+ if (was_stream_delay_set()) {
+ private_submodules_->echo_controller->SetAudioBufferDelay(
+ stream_delay_ms());
+ }
+
+ private_submodules_->echo_controller->ProcessCapture(
+ capture_buffer, capture_.echo_path_gain_change);
+ } else if (private_submodules_->echo_cancellation) {
+ // Ensure that the stream delay was set before the call to the
+ // AEC ProcessCaptureAudio function.
+ if (!was_stream_delay_set()) {
+ return AudioProcessing::kStreamParameterNotSetError;
+ }
+
+ RETURN_ON_ERR(private_submodules_->echo_cancellation->ProcessCaptureAudio(
+ capture_buffer, stream_delay_ms()));
+ }
+
+ public_submodules_->noise_suppression->ProcessCaptureAudio(capture_buffer);
}
public_submodules_->voice_detection->ProcessCaptureAudio(capture_buffer);
@@ -1795,7 +1769,8 @@
config_.high_pass_filter.enabled,
private_submodules_->echo_cancellation &&
private_submodules_->echo_cancellation->is_enabled(),
- private_submodules_->echo_control_mobile->is_enabled(),
+ private_submodules_->echo_control_mobile &&
+ private_submodules_->echo_control_mobile->is_enabled(),
config_.residual_echo_detector.enabled,
public_submodules_->noise_suppression->is_enabled(),
public_submodules_->gain_control->is_enabled(),
@@ -1828,9 +1803,13 @@
}
void AudioProcessingImpl::InitializeEchoController() {
- if (echo_control_factory_ ||
+ bool use_echo_controller =
+ echo_control_factory_ ||
(config_.echo_canceller.enabled && !config_.echo_canceller.mobile_mode &&
- !config_.echo_canceller.use_legacy_aec)) {
+ !config_.echo_canceller.use_legacy_aec);
+
+ if (use_echo_controller) {
+ // Create and activate the echo controller.
if (echo_control_factory_) {
private_submodules_->echo_controller =
echo_control_factory_->Create(proc_sample_rate_hz());
@@ -1843,7 +1822,8 @@
private_submodules_->echo_cancellation.reset();
aec_render_signal_queue_.reset();
- private_submodules_->echo_control_mobile->Enable(false);
+ private_submodules_->echo_control_mobile.reset();
+ aecm_render_signal_queue_.reset();
return;
}
@@ -1853,13 +1833,35 @@
if (!config_.echo_canceller.enabled) {
private_submodules_->echo_cancellation.reset();
aec_render_signal_queue_.reset();
- private_submodules_->echo_control_mobile->Enable(false);
+ private_submodules_->echo_control_mobile.reset();
+ aecm_render_signal_queue_.reset();
return;
}
if (config_.echo_canceller.mobile_mode) {
// Create and activate AECM.
- // TODO(peah): Add an on-demand creation of AECmobile similar to AEC2.
+ size_t max_element_size =
+ std::max(static_cast<size_t>(1),
+ kMaxAllowedValuesOfSamplesPerBand *
+ EchoControlMobileImpl::NumCancellersRequired(
+ num_output_channels(), num_reverse_channels()));
+
+ std::vector<int16_t> template_queue_element(max_element_size);
+
+ aecm_render_signal_queue_.reset(
+ new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
+ kMaxNumFramesToBuffer, template_queue_element,
+ RenderQueueItemVerifier<int16_t>(max_element_size)));
+
+ aecm_render_queue_buffer_.resize(max_element_size);
+ aecm_capture_queue_buffer_.resize(max_element_size);
+
+ private_submodules_->echo_control_mobile.reset(new EchoControlMobileImpl());
+
+ private_submodules_->echo_control_mobile->Initialize(
+ proc_split_sample_rate_hz(), num_reverse_channels(),
+ num_output_channels());
+
private_submodules_->echo_control_mobile->Enable(true);
private_submodules_->echo_cancellation.reset();
@@ -1867,41 +1869,31 @@
return;
}
+ private_submodules_->echo_control_mobile.reset();
+ aecm_render_signal_queue_.reset();
+
// Create and activate AEC2.
- private_submodules_->echo_control_mobile->Enable(false);
private_submodules_->echo_cancellation.reset(new EchoCancellationImpl());
private_submodules_->echo_cancellation->SetExtraOptions(
capture_nonlocked_.use_aec2_extended_filter,
capture_nonlocked_.use_aec2_delay_agnostic,
capture_nonlocked_.use_aec2_refined_adaptive_filter);
- const size_t new_aec_render_queue_element_max_size =
+ size_t element_max_size =
std::max(static_cast<size_t>(1),
kMaxAllowedValuesOfSamplesPerBand *
EchoCancellationImpl::NumCancellersRequired(
num_output_channels(), num_reverse_channels()));
- // Reallocate the queues if the queue item sizes are too small to fit the
- // data to put in the queues.
- if (aec_render_queue_element_max_size_ <
- new_aec_render_queue_element_max_size ||
- !aec_render_signal_queue_) {
- aec_render_queue_element_max_size_ = new_aec_render_queue_element_max_size;
+ std::vector<float> template_queue_element(element_max_size);
- std::vector<float> template_queue_element(
- aec_render_queue_element_max_size_);
+ aec_render_signal_queue_.reset(
+ new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
+ kMaxNumFramesToBuffer, template_queue_element,
+ RenderQueueItemVerifier<float>(element_max_size)));
- aec_render_signal_queue_.reset(
- new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
- kMaxNumFramesToBuffer, template_queue_element,
- RenderQueueItemVerifier<float>(
- aec_render_queue_element_max_size_)));
-
- aec_render_queue_buffer_.resize(aec_render_queue_element_max_size_);
- aec_capture_queue_buffer_.resize(aec_render_queue_element_max_size_);
- } else {
- aec_render_signal_queue_->Clear();
- }
+ aec_render_queue_buffer_.resize(element_max_size);
+ aec_capture_queue_buffer_.resize(element_max_size);
private_submodules_->echo_cancellation->Initialize(
proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(),
@@ -1913,9 +1905,6 @@
config_.echo_canceller.legacy_moderate_suppression_level
? EchoCancellationImpl::SuppressionLevel::kModerateSuppression
: EchoCancellationImpl::SuppressionLevel::kHighSuppression);
-
- private_submodules_->echo_controller.reset();
- capture_nonlocked_.echo_controller_enabled = false;
}
void AudioProcessingImpl::InitializeGainController2() {
@@ -2077,11 +2066,16 @@
: 0;
apm_config.aecm_enabled =
+ private_submodules_->echo_control_mobile &&
private_submodules_->echo_control_mobile->is_enabled();
apm_config.aecm_comfort_noise_enabled =
+ private_submodules_->echo_control_mobile &&
private_submodules_->echo_control_mobile->is_comfort_noise_enabled();
- apm_config.aecm_routing_mode = static_cast<int>(
- private_submodules_->echo_control_mobile->routing_mode());
+ apm_config.aecm_routing_mode =
+ private_submodules_->echo_control_mobile
+ ? static_cast<int>(
+ private_submodules_->echo_control_mobile->routing_mode())
+ : 0;
apm_config.agc_enabled = public_submodules_->gain_control->is_enabled();
apm_config.agc_mode =