From 8675eeec26686cc902f9789127f14322d5e787c7 Mon Sep 17 00:00:00 2001 From: Gustaf Ullberg Date: Wed, 9 Oct 2019 13:34:36 +0200 Subject: [PATCH] Bypass unnecessary resampling. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change keeps the original 48 kHz signal and uses it for the fullband processing given that the following requirements are fulfilled: - Input signal is 48 kHz - Output signal is 48 kHz - Multiband processing is performed at 32 kHz - The multiband processing does not modify the original signal This avoids unnecessary, lossy resampling and band merging. Bug: b/130016532 Change-Id: I690c26faba07eab0cbff6c0a95a81d89255dd1a1 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/155966 Commit-Queue: Gustaf Ullberg Reviewed-by: Per Åhgren Cr-Commit-Position: refs/heads/master@{#29425} --- api/audio/echo_control.h | 3 ++ .../audio_processing/aec3/echo_canceller3.cc | 4 +++ .../audio_processing/aec3/echo_canceller3.h | 2 ++ .../audio_processing/audio_processing_impl.cc | 31 ++++++++++++++----- .../audio_processing/audio_processing_impl.h | 3 +- .../include/mock_audio_processing.h | 1 + .../audio_processing/test/echo_control_mock.h | 1 + 7 files changed, 37 insertions(+), 8 deletions(-) diff --git a/api/audio/echo_control.h b/api/audio/echo_control.h index f549f40fbb..c8f73b8bac 100644 --- a/api/audio/echo_control.h +++ b/api/audio/echo_control.h @@ -41,6 +41,9 @@ class EchoControl { // Provides an optional external estimate of the audio buffer delay. virtual void SetAudioBufferDelay(size_t delay_ms) = 0; + // Returns wheter the signal is altered. + virtual bool ActiveProcessing() const = 0; + virtual ~EchoControl() {} }; diff --git a/modules/audio_processing/aec3/echo_canceller3.cc b/modules/audio_processing/aec3/echo_canceller3.cc index 85b2dc7630..ffff1b6ccf 100644 --- a/modules/audio_processing/aec3/echo_canceller3.cc +++ b/modules/audio_processing/aec3/echo_canceller3.cc @@ -371,6 +371,10 @@ void EchoCanceller3::SetAudioBufferDelay(size_t delay_ms) { block_processor_->SetAudioBufferDelay(delay_ms); } +bool EchoCanceller3::ActiveProcessing() const { + return true; +} + void EchoCanceller3::EmptyRenderQueue() { RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); bool frame_to_buffer = diff --git a/modules/audio_processing/aec3/echo_canceller3.h b/modules/audio_processing/aec3/echo_canceller3.h index 5b59674337..7a7df07171 100644 --- a/modules/audio_processing/aec3/echo_canceller3.h +++ b/modules/audio_processing/aec3/echo_canceller3.h @@ -111,6 +111,8 @@ class EchoCanceller3 : public EchoControl { // Provides an optional external estimate of the audio buffer delay. void SetAudioBufferDelay(size_t delay_ms) override; + bool ActiveProcessing() const override; + // Signals whether an external detector has detected echo leakage from the // echo canceller. // Note that in the case echo leakage has been flagged, it should be unflagged diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc index ceb100686e..3a70791bec 100644 --- a/modules/audio_processing/audio_processing_impl.cc +++ b/modules/audio_processing/audio_processing_impl.cc @@ -206,14 +206,21 @@ bool AudioProcessingImpl::ApmSubmoduleStates::Update( bool AudioProcessingImpl::ApmSubmoduleStates::CaptureMultiBandSubModulesActive() const { - return CaptureMultiBandProcessingActive() || voice_detector_enabled_; + return CaptureMultiBandProcessingPresent() || voice_detector_enabled_; } -bool AudioProcessingImpl::ApmSubmoduleStates::CaptureMultiBandProcessingActive() - const { +bool AudioProcessingImpl::ApmSubmoduleStates:: + CaptureMultiBandProcessingPresent() const { + // If echo controller is present, assume it performs active processing. + return CaptureMultiBandProcessingActive(/*ec_processing_active=*/true); +} + +bool AudioProcessingImpl::ApmSubmoduleStates::CaptureMultiBandProcessingActive( + bool ec_processing_active) const { return high_pass_filter_enabled_ || echo_canceller_enabled_ || mobile_echo_controller_enabled_ || noise_suppressor_enabled_ || - adaptive_gain_controller_enabled_ || echo_controller_enabled_; + adaptive_gain_controller_enabled_ || + (echo_controller_enabled_ && ec_processing_active); } bool AudioProcessingImpl::ApmSubmoduleStates::CaptureFullBandProcessingActive() @@ -987,6 +994,10 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, capture_.keyboard_info.Extract(src, formats_.api_format.input_stream()); capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream()); + if (capture_.capture_fullband_audio) { + capture_.capture_fullband_audio->CopyFrom( + src, formats_.api_format.input_stream()); + } RETURN_ON_ERR(ProcessCaptureStreamLocked()); if (capture_.capture_fullband_audio) { capture_.capture_fullband_audio->CopyTo(formats_.api_format.output_stream(), @@ -1287,7 +1298,7 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { capture_.capture_audio->CopyFrom(frame); RETURN_ON_ERR(ProcessCaptureStreamLocked()); - if (submodule_states_.CaptureMultiBandProcessingActive() || + if (submodule_states_.CaptureMultiBandProcessingPresent() || submodule_states_.CaptureFullBandProcessingActive()) { if (capture_.capture_fullband_audio) { capture_.capture_fullband_audio->CopyTo(frame); @@ -1469,14 +1480,20 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { private_submodules_->echo_cancellation && private_submodules_->echo_cancellation->stream_has_echo())); - if (submodule_states_.CaptureMultiBandProcessingActive() && + if (submodule_states_.CaptureMultiBandProcessingPresent() && SampleRateSupportsMultiBand( capture_nonlocked_.capture_processing_format.sample_rate_hz())) { capture_buffer->MergeFrequencyBands(); } if (capture_.capture_fullband_audio) { - capture_buffer->CopyTo(capture_.capture_fullband_audio.get()); + const auto& ec = private_submodules_->echo_controller; + bool ec_active = ec ? ec->ActiveProcessing() : false; + // Only update the fullband buffer if the multiband processing has changed + // the signal. Keep the original signal otherwise. + if (submodule_states_.CaptureMultiBandProcessingActive(ec_active)) { + capture_buffer->CopyTo(capture_.capture_fullband_audio.get()); + } capture_buffer = capture_.capture_fullband_audio.get(); } diff --git a/modules/audio_processing/audio_processing_impl.h b/modules/audio_processing/audio_processing_impl.h index c8e8c014e3..5d07fe3930 100644 --- a/modules/audio_processing/audio_processing_impl.h +++ b/modules/audio_processing/audio_processing_impl.h @@ -185,7 +185,8 @@ class AudioProcessingImpl : public AudioProcessing { bool level_estimator_enabled, bool transient_suppressor_enabled); bool CaptureMultiBandSubModulesActive() const; - bool CaptureMultiBandProcessingActive() const; + bool CaptureMultiBandProcessingPresent() const; + bool CaptureMultiBandProcessingActive(bool ec_processing_active) const; bool CaptureFullBandProcessingActive() const; bool CaptureAnalyzerActive() const; bool RenderMultiBandSubModulesActive() const; diff --git a/modules/audio_processing/include/mock_audio_processing.h b/modules/audio_processing/include/mock_audio_processing.h index e989fea89a..c5a1f0925a 100644 --- a/modules/audio_processing/include/mock_audio_processing.h +++ b/modules/audio_processing/include/mock_audio_processing.h @@ -89,6 +89,7 @@ class MockEchoControl : public EchoControl { void(AudioBuffer* capture, bool echo_path_change)); MOCK_CONST_METHOD0(GetMetrics, Metrics()); MOCK_METHOD1(SetAudioBufferDelay, void(size_t delay_ms)); + MOCK_CONST_METHOD0(ActiveProcessing, bool()); }; class MockAudioProcessing : public ::testing::NiceMock { diff --git a/modules/audio_processing/test/echo_control_mock.h b/modules/audio_processing/test/echo_control_mock.h index 0386762ed4..8479a7cd72 100644 --- a/modules/audio_processing/test/echo_control_mock.h +++ b/modules/audio_processing/test/echo_control_mock.h @@ -26,6 +26,7 @@ class MockEchoControl : public EchoControl { void(AudioBuffer* capture, bool echo_path_change)); MOCK_CONST_METHOD0(GetMetrics, EchoControl::Metrics()); MOCK_METHOD1(SetAudioBufferDelay, void(size_t delay_ms)); + MOCK_CONST_METHOD0(ActiveProcessing, bool()); }; } // namespace webrtc