diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc index 16053e60ec..4a1a86ce86 100644 --- a/modules/audio_processing/audio_processing_impl.cc +++ b/modules/audio_processing/audio_processing_impl.cc @@ -158,9 +158,11 @@ static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); AudioProcessingImpl::ApmSubmoduleStates::ApmSubmoduleStates( bool capture_post_processor_enabled, - bool render_pre_processor_enabled) + bool render_pre_processor_enabled, + bool capture_analyzer_enabled) : capture_post_processor_enabled_(capture_post_processor_enabled), - render_pre_processor_enabled_(render_pre_processor_enabled) {} + render_pre_processor_enabled_(render_pre_processor_enabled), + capture_analyzer_enabled_(capture_analyzer_enabled) {} bool AudioProcessingImpl::ApmSubmoduleStates::Update( bool low_cut_filter_enabled, @@ -240,6 +242,10 @@ bool AudioProcessingImpl::ApmSubmoduleStates::CaptureFullBandProcessingActive() pre_amplifier_enabled_; } +bool AudioProcessingImpl::ApmSubmoduleStates::CaptureAnalyzerActive() const { + return capture_analyzer_enabled_; +} + bool AudioProcessingImpl::ApmSubmoduleStates::RenderMultiBandSubModulesActive() const { return RenderMultiBandProcessingActive() || echo_canceller_enabled_ || @@ -285,10 +291,12 @@ struct AudioProcessingImpl::ApmPublicSubmodules { struct AudioProcessingImpl::ApmPrivateSubmodules { ApmPrivateSubmodules(std::unique_ptr capture_post_processor, std::unique_ptr render_pre_processor, - rtc::scoped_refptr echo_detector) + rtc::scoped_refptr echo_detector, + std::unique_ptr capture_analyzer) : echo_detector(std::move(echo_detector)), capture_post_processor(std::move(capture_post_processor)), - render_pre_processor(std::move(render_pre_processor)) {} + render_pre_processor(std::move(render_pre_processor)), + capture_analyzer(std::move(capture_analyzer)) {} // Accessed internally from capture or during initialization std::unique_ptr agc_manager; std::unique_ptr gain_controller2; @@ -298,6 +306,7 @@ struct AudioProcessingImpl::ApmPrivateSubmodules { std::unique_ptr capture_post_processor; std::unique_ptr render_pre_processor; std::unique_ptr pre_amplifier; + std::unique_ptr capture_analyzer; }; AudioProcessingBuilder::AudioProcessingBuilder() = default; @@ -315,6 +324,12 @@ AudioProcessingBuilder& AudioProcessingBuilder::SetRenderPreProcessing( return *this; } +AudioProcessingBuilder& AudioProcessingBuilder::SetCaptureAnalyzer( + std::unique_ptr capture_analyzer) { + capture_analyzer_ = std::move(capture_analyzer); + return *this; +} + AudioProcessingBuilder& AudioProcessingBuilder::SetEchoControlFactory( std::unique_ptr echo_control_factory) { echo_control_factory_ = std::move(echo_control_factory); @@ -336,7 +351,7 @@ AudioProcessing* AudioProcessingBuilder::Create(const webrtc::Config& config) { AudioProcessingImpl* apm = new rtc::RefCountedObject( config, std::move(capture_post_processing_), std::move(render_pre_processing_), std::move(echo_control_factory_), - std::move(echo_detector_)); + std::move(echo_detector_), std::move(capture_analyzer_)); if (apm->Initialize() != AudioProcessing::kNoError) { delete apm; apm = nullptr; @@ -345,7 +360,8 @@ AudioProcessing* AudioProcessingBuilder::Create(const webrtc::Config& config) { } AudioProcessingImpl::AudioProcessingImpl(const webrtc::Config& config) - : AudioProcessingImpl(config, nullptr, nullptr, nullptr, nullptr) {} + : AudioProcessingImpl(config, nullptr, nullptr, nullptr, nullptr, nullptr) { +} int AudioProcessingImpl::instance_count_ = 0; @@ -354,7 +370,8 @@ AudioProcessingImpl::AudioProcessingImpl( std::unique_ptr capture_post_processor, std::unique_ptr render_pre_processor, std::unique_ptr echo_control_factory, - rtc::scoped_refptr echo_detector) + rtc::scoped_refptr echo_detector, + std::unique_ptr capture_analyzer) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), capture_runtime_settings_(kRuntimeSettingQueueSize), @@ -363,12 +380,15 @@ AudioProcessingImpl::AudioProcessingImpl( render_runtime_settings_enqueuer_(&render_runtime_settings_), high_pass_filter_impl_(new HighPassFilterImpl(this)), echo_control_factory_(std::move(echo_control_factory)), - submodule_states_(!!capture_post_processor, !!render_pre_processor), + submodule_states_(!!capture_post_processor, + !!render_pre_processor, + !!capture_analyzer), public_submodules_(new ApmPublicSubmodules()), private_submodules_( new ApmPrivateSubmodules(std::move(capture_post_processor), std::move(render_pre_processor), - std::move(echo_detector))), + std::move(echo_detector), + std::move(capture_analyzer))), constants_(config.Get().startup_min_volume, config.Get().clipped_level_min, #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) @@ -425,7 +445,9 @@ AudioProcessingImpl::AudioProcessingImpl( // implemented. private_submodules_->gain_controller2.reset(new GainController2()); - RTC_LOG(LS_INFO) << "Capture post processor activated: " + RTC_LOG(LS_INFO) << "Capture analyzer activated: " + << !!private_submodules_->capture_analyzer + << "\nCapture post processor activated: " << !!private_submodules_->capture_post_processor << "\nRender pre processor activated: " << !!private_submodules_->render_pre_processor; @@ -578,6 +600,7 @@ int AudioProcessingImpl::InitializeLocked() { InitializeResidualEchoDetector(); InitializeEchoController(); InitializeGainController2(); + InitializeAnalyzer(); InitializePostProcessor(); InitializePreProcessor(); @@ -1350,6 +1373,11 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { capture_.key_pressed); } + // Experimental APM sub-module that analyzes |capture_buffer|. + if (private_submodules_->capture_analyzer) { + private_submodules_->capture_analyzer->Analyze(capture_buffer); + } + if (config_.gain_controller2.enabled) { private_submodules_->gain_controller2->NotifyAnalogLevel( gain_control()->stream_analog_level()); @@ -1854,6 +1882,13 @@ void AudioProcessingImpl::InitializeResidualEchoDetector() { formats_.render_processing_format.sample_rate_hz(), 1); } +void AudioProcessingImpl::InitializeAnalyzer() { + if (private_submodules_->capture_analyzer) { + private_submodules_->capture_analyzer->Initialize(proc_sample_rate_hz(), + num_proc_channels()); + } +} + void AudioProcessingImpl::InitializePostProcessor() { if (private_submodules_->capture_post_processor) { private_submodules_->capture_post_processor->Initialize( diff --git a/modules/audio_processing/audio_processing_impl.h b/modules/audio_processing/audio_processing_impl.h index 44d0d087a5..06fa70c78a 100644 --- a/modules/audio_processing/audio_processing_impl.h +++ b/modules/audio_processing/audio_processing_impl.h @@ -42,7 +42,8 @@ class AudioProcessingImpl : public AudioProcessing { std::unique_ptr capture_post_processor, std::unique_ptr render_pre_processor, std::unique_ptr echo_control_factory, - rtc::scoped_refptr echo_detector); + rtc::scoped_refptr echo_detector, + std::unique_ptr capture_analyzer); ~AudioProcessingImpl() override; int Initialize() override; int Initialize(int capture_input_sample_rate_hz, @@ -174,7 +175,8 @@ class AudioProcessingImpl : public AudioProcessing { class ApmSubmoduleStates { public: ApmSubmoduleStates(bool capture_post_processor_enabled, - bool render_pre_processor_enabled); + bool render_pre_processor_enabled, + bool capture_analyzer_enabled); // Updates the submodule state and returns true if it has changed. bool Update(bool low_cut_filter_enabled, bool echo_canceller_enabled, @@ -192,6 +194,7 @@ class AudioProcessingImpl : public AudioProcessing { bool CaptureMultiBandSubModulesActive() const; bool CaptureMultiBandProcessingActive() const; bool CaptureFullBandProcessingActive() const; + bool CaptureAnalyzerActive() const; bool RenderMultiBandSubModulesActive() const; bool RenderFullBandProcessingActive() const; bool RenderMultiBandProcessingActive() const; @@ -199,6 +202,7 @@ class AudioProcessingImpl : public AudioProcessing { private: const bool capture_post_processor_enabled_ = false; const bool render_pre_processor_enabled_ = false; + const bool capture_analyzer_enabled_ = false; bool low_cut_filter_enabled_ = false; bool echo_canceller_enabled_ = false; bool mobile_echo_controller_enabled_ = false; @@ -252,6 +256,7 @@ class AudioProcessingImpl : public AudioProcessing { void InitializeGainController2() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); void InitializePreAmplifier() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); void InitializePostProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + void InitializeAnalyzer() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); void InitializePreProcessor() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_render_); // Empties and handles the respective RuntimeSetting queues. diff --git a/modules/audio_processing/audio_processing_unittest.cc b/modules/audio_processing/audio_processing_unittest.cc index 198c66d1d8..b04f89a82b 100644 --- a/modules/audio_processing/audio_processing_unittest.cc +++ b/modules/audio_processing/audio_processing_unittest.cc @@ -2802,6 +2802,25 @@ TEST(ApmConfiguration, EnablePreProcessing) { apm->ProcessReverseStream(&audio); } +TEST(ApmConfiguration, EnableCaptureAnalyzer) { + // Verify that apm uses a capture analyzer if one is provided. + auto mock_capture_analyzer_ptr = + new testing::NiceMock(); + auto mock_capture_analyzer = + std::unique_ptr(mock_capture_analyzer_ptr); + rtc::scoped_refptr apm = + AudioProcessingBuilder() + .SetCaptureAnalyzer(std::move(mock_capture_analyzer)) + .Create(); + + AudioFrame audio; + audio.num_channels_ = 1; + SetFrameSampleRate(&audio, AudioProcessing::NativeRate::kSampleRate16kHz); + + EXPECT_CALL(*mock_capture_analyzer_ptr, Analyze(testing::_)).Times(1); + apm->ProcessStream(&audio); +} + TEST(ApmConfiguration, PreProcessingReceivesRuntimeSettings) { auto mock_pre_processor_ptr = new testing::NiceMock(); diff --git a/modules/audio_processing/include/audio_processing.h b/modules/audio_processing/include/audio_processing.h index 22fa92d943..e194be72c5 100644 --- a/modules/audio_processing/include/audio_processing.h +++ b/modules/audio_processing/include/audio_processing.h @@ -53,6 +53,7 @@ class GainControl; class HighPassFilter; class LevelEstimator; class NoiseSuppression; +class CustomAudioAnalyzer; class CustomProcessing; class VoiceDetection; @@ -675,6 +676,9 @@ class AudioProcessingBuilder { // The AudioProcessingBuilder takes ownership of the echo_detector. AudioProcessingBuilder& SetEchoDetector( rtc::scoped_refptr echo_detector); + // The AudioProcessingBuilder takes ownership of the capture_analyzer. + AudioProcessingBuilder& SetCaptureAnalyzer( + std::unique_ptr capture_analyzer); // This creates an APM instance using the previously set components. Calling // the Create function resets the AudioProcessingBuilder to its initial state. AudioProcessing* Create(); @@ -685,6 +689,7 @@ class AudioProcessingBuilder { std::unique_ptr capture_post_processing_; std::unique_ptr render_pre_processing_; rtc::scoped_refptr echo_detector_; + std::unique_ptr capture_analyzer_; RTC_DISALLOW_COPY_AND_ASSIGN(AudioProcessingBuilder); }; @@ -1018,6 +1023,19 @@ class NoiseSuppression { virtual ~NoiseSuppression() {} }; +// Experimental interface for a custom analysis submodule. +class CustomAudioAnalyzer { + public: + // (Re-) Initializes the submodule. + virtual void Initialize(int sample_rate_hz, int num_channels) = 0; + // Analyzes the given capture or render signal. + virtual void Analyze(const AudioBuffer* audio) = 0; + // Returns a string representation of the module state. + virtual std::string ToString() const = 0; + + virtual ~CustomAudioAnalyzer() {} +}; + // Interface for a custom processing submodule. class CustomProcessing { public: diff --git a/modules/audio_processing/include/mock_audio_processing.h b/modules/audio_processing/include/mock_audio_processing.h index 9ceee106c0..2864e48cb8 100644 --- a/modules/audio_processing/include/mock_audio_processing.h +++ b/modules/audio_processing/include/mock_audio_processing.h @@ -115,6 +115,14 @@ class MockCustomProcessing : public CustomProcessing { MOCK_CONST_METHOD0(ToString, std::string()); }; +class MockCustomAudioAnalyzer : public CustomAudioAnalyzer { + public: + virtual ~MockCustomAudioAnalyzer() {} + MOCK_METHOD2(Initialize, void(int sample_rate_hz, int num_channels)); + MOCK_METHOD1(Analyze, void(const AudioBuffer* audio)); + MOCK_CONST_METHOD0(ToString, std::string()); +}; + class MockEchoControl : public EchoControl { public: virtual ~MockEchoControl() {}