Adding CustomAudioAnalyzer interface in APM.
CustomAudioAnalyzer is an interface of a component into APM that
reads AudioBuffer without changing it.
The APM sub-module is optional. It operates in full band.
As described in the comments, it is an experimental interface which
may be changed in the nearest future.
Change-Id: I21edf729d97947529256407b10fa4b5219bb2bf5
Bug: webrtc:9678
Reviewed-on: https://webrtc-review.googlesource.com/96560
Reviewed-by: Alessio Bazzica <alessiob@webrtc.org>
Commit-Queue: Alessio Bazzica <alessiob@webrtc.org>
Commit-Queue: Valeriia Nemychnikova <valeriian@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24481}diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc
index 16053e6..4a1a86c 100644
--- a/modules/audio_processing/audio_processing_impl.cc
+++ b/modules/audio_processing/audio_processing_impl.cc
@@ -158,9 +158,11 @@
AudioProcessingImpl::ApmSubmoduleStates::ApmSubmoduleStates(
bool capture_post_processor_enabled,
- bool render_pre_processor_enabled)
+ bool render_pre_processor_enabled,
+ bool capture_analyzer_enabled)
: capture_post_processor_enabled_(capture_post_processor_enabled),
- render_pre_processor_enabled_(render_pre_processor_enabled) {}
+ render_pre_processor_enabled_(render_pre_processor_enabled),
+ capture_analyzer_enabled_(capture_analyzer_enabled) {}
bool AudioProcessingImpl::ApmSubmoduleStates::Update(
bool low_cut_filter_enabled,
@@ -240,6 +242,10 @@
pre_amplifier_enabled_;
}
+bool AudioProcessingImpl::ApmSubmoduleStates::CaptureAnalyzerActive() const {
+ return capture_analyzer_enabled_;
+}
+
bool AudioProcessingImpl::ApmSubmoduleStates::RenderMultiBandSubModulesActive()
const {
return RenderMultiBandProcessingActive() || echo_canceller_enabled_ ||
@@ -285,10 +291,12 @@
struct AudioProcessingImpl::ApmPrivateSubmodules {
ApmPrivateSubmodules(std::unique_ptr<CustomProcessing> capture_post_processor,
std::unique_ptr<CustomProcessing> render_pre_processor,
- rtc::scoped_refptr<EchoDetector> echo_detector)
+ rtc::scoped_refptr<EchoDetector> echo_detector,
+ std::unique_ptr<CustomAudioAnalyzer> capture_analyzer)
: echo_detector(std::move(echo_detector)),
capture_post_processor(std::move(capture_post_processor)),
- render_pre_processor(std::move(render_pre_processor)) {}
+ render_pre_processor(std::move(render_pre_processor)),
+ capture_analyzer(std::move(capture_analyzer)) {}
// Accessed internally from capture or during initialization
std::unique_ptr<AgcManagerDirect> agc_manager;
std::unique_ptr<GainController2> gain_controller2;
@@ -298,6 +306,7 @@
std::unique_ptr<CustomProcessing> capture_post_processor;
std::unique_ptr<CustomProcessing> render_pre_processor;
std::unique_ptr<GainApplier> pre_amplifier;
+ std::unique_ptr<CustomAudioAnalyzer> capture_analyzer;
};
AudioProcessingBuilder::AudioProcessingBuilder() = default;
@@ -315,6 +324,12 @@
return *this;
}
+AudioProcessingBuilder& AudioProcessingBuilder::SetCaptureAnalyzer(
+ std::unique_ptr<CustomAudioAnalyzer> capture_analyzer) {
+ capture_analyzer_ = std::move(capture_analyzer);
+ return *this;
+}
+
AudioProcessingBuilder& AudioProcessingBuilder::SetEchoControlFactory(
std::unique_ptr<EchoControlFactory> echo_control_factory) {
echo_control_factory_ = std::move(echo_control_factory);
@@ -336,7 +351,7 @@
AudioProcessingImpl* apm = new rtc::RefCountedObject<AudioProcessingImpl>(
config, std::move(capture_post_processing_),
std::move(render_pre_processing_), std::move(echo_control_factory_),
- std::move(echo_detector_));
+ std::move(echo_detector_), std::move(capture_analyzer_));
if (apm->Initialize() != AudioProcessing::kNoError) {
delete apm;
apm = nullptr;
@@ -345,7 +360,8 @@
}
AudioProcessingImpl::AudioProcessingImpl(const webrtc::Config& config)
- : AudioProcessingImpl(config, nullptr, nullptr, nullptr, nullptr) {}
+ : AudioProcessingImpl(config, nullptr, nullptr, nullptr, nullptr, nullptr) {
+}
int AudioProcessingImpl::instance_count_ = 0;
@@ -354,7 +370,8 @@
std::unique_ptr<CustomProcessing> capture_post_processor,
std::unique_ptr<CustomProcessing> render_pre_processor,
std::unique_ptr<EchoControlFactory> echo_control_factory,
- rtc::scoped_refptr<EchoDetector> echo_detector)
+ rtc::scoped_refptr<EchoDetector> echo_detector,
+ std::unique_ptr<CustomAudioAnalyzer> capture_analyzer)
: data_dumper_(
new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
capture_runtime_settings_(kRuntimeSettingQueueSize),
@@ -363,12 +380,15 @@
render_runtime_settings_enqueuer_(&render_runtime_settings_),
high_pass_filter_impl_(new HighPassFilterImpl(this)),
echo_control_factory_(std::move(echo_control_factory)),
- submodule_states_(!!capture_post_processor, !!render_pre_processor),
+ submodule_states_(!!capture_post_processor,
+ !!render_pre_processor,
+ !!capture_analyzer),
public_submodules_(new ApmPublicSubmodules()),
private_submodules_(
new ApmPrivateSubmodules(std::move(capture_post_processor),
std::move(render_pre_processor),
- std::move(echo_detector))),
+ std::move(echo_detector),
+ std::move(capture_analyzer))),
constants_(config.Get<ExperimentalAgc>().startup_min_volume,
config.Get<ExperimentalAgc>().clipped_level_min,
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
@@ -425,7 +445,9 @@
// implemented.
private_submodules_->gain_controller2.reset(new GainController2());
- RTC_LOG(LS_INFO) << "Capture post processor activated: "
+ RTC_LOG(LS_INFO) << "Capture analyzer activated: "
+ << !!private_submodules_->capture_analyzer
+ << "\nCapture post processor activated: "
<< !!private_submodules_->capture_post_processor
<< "\nRender pre processor activated: "
<< !!private_submodules_->render_pre_processor;
@@ -578,6 +600,7 @@
InitializeResidualEchoDetector();
InitializeEchoController();
InitializeGainController2();
+ InitializeAnalyzer();
InitializePostProcessor();
InitializePreProcessor();
@@ -1350,6 +1373,11 @@
capture_.key_pressed);
}
+ // Experimental APM sub-module that analyzes |capture_buffer|.
+ if (private_submodules_->capture_analyzer) {
+ private_submodules_->capture_analyzer->Analyze(capture_buffer);
+ }
+
if (config_.gain_controller2.enabled) {
private_submodules_->gain_controller2->NotifyAnalogLevel(
gain_control()->stream_analog_level());
@@ -1854,6 +1882,13 @@
formats_.render_processing_format.sample_rate_hz(), 1);
}
+void AudioProcessingImpl::InitializeAnalyzer() {
+ if (private_submodules_->capture_analyzer) {
+ private_submodules_->capture_analyzer->Initialize(proc_sample_rate_hz(),
+ num_proc_channels());
+ }
+}
+
void AudioProcessingImpl::InitializePostProcessor() {
if (private_submodules_->capture_post_processor) {
private_submodules_->capture_post_processor->Initialize(
diff --git a/modules/audio_processing/audio_processing_impl.h b/modules/audio_processing/audio_processing_impl.h
index 44d0d08..06fa70c 100644
--- a/modules/audio_processing/audio_processing_impl.h
+++ b/modules/audio_processing/audio_processing_impl.h
@@ -42,7 +42,8 @@
std::unique_ptr<CustomProcessing> capture_post_processor,
std::unique_ptr<CustomProcessing> render_pre_processor,
std::unique_ptr<EchoControlFactory> echo_control_factory,
- rtc::scoped_refptr<EchoDetector> echo_detector);
+ rtc::scoped_refptr<EchoDetector> echo_detector,
+ std::unique_ptr<CustomAudioAnalyzer> capture_analyzer);
~AudioProcessingImpl() override;
int Initialize() override;
int Initialize(int capture_input_sample_rate_hz,
@@ -174,7 +175,8 @@
class ApmSubmoduleStates {
public:
ApmSubmoduleStates(bool capture_post_processor_enabled,
- bool render_pre_processor_enabled);
+ bool render_pre_processor_enabled,
+ bool capture_analyzer_enabled);
// Updates the submodule state and returns true if it has changed.
bool Update(bool low_cut_filter_enabled,
bool echo_canceller_enabled,
@@ -192,6 +194,7 @@
bool CaptureMultiBandSubModulesActive() const;
bool CaptureMultiBandProcessingActive() const;
bool CaptureFullBandProcessingActive() const;
+ bool CaptureAnalyzerActive() const;
bool RenderMultiBandSubModulesActive() const;
bool RenderFullBandProcessingActive() const;
bool RenderMultiBandProcessingActive() const;
@@ -199,6 +202,7 @@
private:
const bool capture_post_processor_enabled_ = false;
const bool render_pre_processor_enabled_ = false;
+ const bool capture_analyzer_enabled_ = false;
bool low_cut_filter_enabled_ = false;
bool echo_canceller_enabled_ = false;
bool mobile_echo_controller_enabled_ = false;
@@ -252,6 +256,7 @@
void InitializeGainController2() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
void InitializePreAmplifier() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
void InitializePostProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
+ void InitializeAnalyzer() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
void InitializePreProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
// Empties and handles the respective RuntimeSetting queues.
diff --git a/modules/audio_processing/audio_processing_unittest.cc b/modules/audio_processing/audio_processing_unittest.cc
index 198c66d..b04f89a 100644
--- a/modules/audio_processing/audio_processing_unittest.cc
+++ b/modules/audio_processing/audio_processing_unittest.cc
@@ -2802,6 +2802,25 @@
apm->ProcessReverseStream(&audio);
}
+TEST(ApmConfiguration, EnableCaptureAnalyzer) {
+ // Verify that apm uses a capture analyzer if one is provided.
+ auto mock_capture_analyzer_ptr =
+ new testing::NiceMock<test::MockCustomAudioAnalyzer>();
+ auto mock_capture_analyzer =
+ std::unique_ptr<CustomAudioAnalyzer>(mock_capture_analyzer_ptr);
+ rtc::scoped_refptr<AudioProcessing> apm =
+ AudioProcessingBuilder()
+ .SetCaptureAnalyzer(std::move(mock_capture_analyzer))
+ .Create();
+
+ AudioFrame audio;
+ audio.num_channels_ = 1;
+ SetFrameSampleRate(&audio, AudioProcessing::NativeRate::kSampleRate16kHz);
+
+ EXPECT_CALL(*mock_capture_analyzer_ptr, Analyze(testing::_)).Times(1);
+ apm->ProcessStream(&audio);
+}
+
TEST(ApmConfiguration, PreProcessingReceivesRuntimeSettings) {
auto mock_pre_processor_ptr =
new testing::NiceMock<test::MockCustomProcessing>();
diff --git a/modules/audio_processing/include/audio_processing.h b/modules/audio_processing/include/audio_processing.h
index 22fa92d..e194be7 100644
--- a/modules/audio_processing/include/audio_processing.h
+++ b/modules/audio_processing/include/audio_processing.h
@@ -53,6 +53,7 @@
class HighPassFilter;
class LevelEstimator;
class NoiseSuppression;
+class CustomAudioAnalyzer;
class CustomProcessing;
class VoiceDetection;
@@ -675,6 +676,9 @@
// The AudioProcessingBuilder takes ownership of the echo_detector.
AudioProcessingBuilder& SetEchoDetector(
rtc::scoped_refptr<EchoDetector> echo_detector);
+ // The AudioProcessingBuilder takes ownership of the capture_analyzer.
+ AudioProcessingBuilder& SetCaptureAnalyzer(
+ std::unique_ptr<CustomAudioAnalyzer> capture_analyzer);
// This creates an APM instance using the previously set components. Calling
// the Create function resets the AudioProcessingBuilder to its initial state.
AudioProcessing* Create();
@@ -685,6 +689,7 @@
std::unique_ptr<CustomProcessing> capture_post_processing_;
std::unique_ptr<CustomProcessing> render_pre_processing_;
rtc::scoped_refptr<EchoDetector> echo_detector_;
+ std::unique_ptr<CustomAudioAnalyzer> capture_analyzer_;
RTC_DISALLOW_COPY_AND_ASSIGN(AudioProcessingBuilder);
};
@@ -1018,6 +1023,19 @@
virtual ~NoiseSuppression() {}
};
+// Experimental interface for a custom analysis submodule.
+class CustomAudioAnalyzer {
+ public:
+ // (Re-) Initializes the submodule.
+ virtual void Initialize(int sample_rate_hz, int num_channels) = 0;
+ // Analyzes the given capture or render signal.
+ virtual void Analyze(const AudioBuffer* audio) = 0;
+ // Returns a string representation of the module state.
+ virtual std::string ToString() const = 0;
+
+ virtual ~CustomAudioAnalyzer() {}
+};
+
// Interface for a custom processing submodule.
class CustomProcessing {
public:
diff --git a/modules/audio_processing/include/mock_audio_processing.h b/modules/audio_processing/include/mock_audio_processing.h
index 9ceee10..2864e48 100644
--- a/modules/audio_processing/include/mock_audio_processing.h
+++ b/modules/audio_processing/include/mock_audio_processing.h
@@ -115,6 +115,14 @@
MOCK_CONST_METHOD0(ToString, std::string());
};
+class MockCustomAudioAnalyzer : public CustomAudioAnalyzer {
+ public:
+ virtual ~MockCustomAudioAnalyzer() {}
+ MOCK_METHOD2(Initialize, void(int sample_rate_hz, int num_channels));
+ MOCK_METHOD1(Analyze, void(const AudioBuffer* audio));
+ MOCK_CONST_METHOD0(ToString, std::string());
+};
+
class MockEchoControl : public EchoControl {
public:
virtual ~MockEchoControl() {}