diff --git a/modules/audio_processing/aec3/block_processor.cc b/modules/audio_processing/aec3/block_processor.cc index f2f3261489..2ee32b82dc 100644 --- a/modules/audio_processing/aec3/block_processor.cc +++ b/modules/audio_processing/aec3/block_processor.cc @@ -63,6 +63,7 @@ class BlockProcessorImpl final : public BlockProcessor { void GetMetrics(EchoControl::Metrics* metrics) const override; void SetAudioBufferDelay(int delay_ms) override; + void SetCaptureOutputUsage(bool capture_output_used) override; private: static int instance_count_; @@ -237,6 +238,10 @@ void BlockProcessorImpl::SetAudioBufferDelay(int delay_ms) { render_buffer_->SetAudioBufferDelay(delay_ms); } +void BlockProcessorImpl::SetCaptureOutputUsage(bool capture_output_used) { + echo_remover_->SetCaptureOutputUsage(capture_output_used); +} + } // namespace BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config, diff --git a/modules/audio_processing/aec3/block_processor.h b/modules/audio_processing/aec3/block_processor.h index 9bb0cf19f3..41ce016dc0 100644 --- a/modules/audio_processing/aec3/block_processor.h +++ b/modules/audio_processing/aec3/block_processor.h @@ -69,6 +69,12 @@ class BlockProcessor { // Reports whether echo leakage has been detected in the echo canceller // output. virtual void UpdateEchoLeakageStatus(bool leakage_detected) = 0; + + // Specifies whether the capture output will be used. The purpose of this is + // to allow the block processor to deactivate some of the processing when the + // resulting output is anyway not used, for instance when the endpoint is + // muted. + virtual void SetCaptureOutputUsage(bool capture_output_used) = 0; }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/echo_canceller3.cc b/modules/audio_processing/aec3/echo_canceller3.cc index 698ede7543..35a2cff7ea 100644 --- a/modules/audio_processing/aec3/echo_canceller3.cc +++ b/modules/audio_processing/aec3/echo_canceller3.cc @@ -834,8 +834,8 @@ void EchoCanceller3::SetAudioBufferDelay(int delay_ms) { } void EchoCanceller3::SetCaptureOutputUsage(bool capture_output_used) { - // TODO(b/177830919): Add functionality for reducing the complexity when the - // echo canceller output is not used. + RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); + block_processor_->SetCaptureOutputUsage(capture_output_used); } bool EchoCanceller3::ActiveProcessing() const { diff --git a/modules/audio_processing/aec3/echo_canceller3_unittest.cc b/modules/audio_processing/aec3/echo_canceller3_unittest.cc index acf84738af..4a3c466712 100644 --- a/modules/audio_processing/aec3/echo_canceller3_unittest.cc +++ b/modules/audio_processing/aec3/echo_canceller3_unittest.cc @@ -131,6 +131,8 @@ class CaptureTransportVerificationProcessor : public BlockProcessor { void GetMetrics(EchoControl::Metrics* metrics) const override {} void SetAudioBufferDelay(int delay_ms) override {} + + void SetCaptureOutputUsage(bool capture_output_used) {} }; // Class for testing that the render data is properly received by the block @@ -169,6 +171,8 @@ class RenderTransportVerificationProcessor : public BlockProcessor { void SetAudioBufferDelay(int delay_ms) override {} + void SetCaptureOutputUsage(bool capture_output_used) {} + private: std::deque>>> received_render_blocks_; diff --git a/modules/audio_processing/aec3/echo_remover.cc b/modules/audio_processing/aec3/echo_remover.cc index df539bfad0..1a83fefcf6 100644 --- a/modules/audio_processing/aec3/echo_remover.cc +++ b/modules/audio_processing/aec3/echo_remover.cc @@ -132,6 +132,10 @@ class EchoRemoverImpl final : public EchoRemover { echo_leakage_detected_ = leakage_detected; } + void SetCaptureOutputUsage(bool capture_output_used) override { + capture_output_used_ = capture_output_used; + } + private: // Selects which of the coarse and refined linear filter outputs that is most // appropriate to pass to the suppressor and forms the linear filter output by @@ -155,6 +159,7 @@ class EchoRemoverImpl final : public EchoRemover { RenderSignalAnalyzer render_signal_analyzer_; ResidualEchoEstimator residual_echo_estimator_; bool echo_leakage_detected_ = false; + bool capture_output_used_ = true; AecState aec_state_; EchoRemoverMetrics metrics_; std::vector> e_old_; @@ -391,42 +396,49 @@ void EchoRemoverImpl::ProcessCapture( 1); data_dumper_->DumpWav("aec3_output_linear2", kBlockSize, &e[0][0], 16000, 1); - // Estimate the residual echo power. - residual_echo_estimator_.Estimate(aec_state_, *render_buffer, S2_linear, Y2, - R2); - // Estimate the comfort noise. cng_.Compute(aec_state_.SaturatedCapture(), Y2, comfort_noise, high_band_comfort_noise); - // Suppressor nearend estimate. - if (aec_state_.UsableLinearEstimate()) { - // E2 is bound by Y2. - for (size_t ch = 0; ch < num_capture_channels_; ++ch) { - std::transform(E2[ch].begin(), E2[ch].end(), Y2[ch].begin(), - E2[ch].begin(), - [](float a, float b) { return std::min(a, b); }); - } - } - const auto& nearend_spectrum = aec_state_.UsableLinearEstimate() ? E2 : Y2; - - // Suppressor echo estimate. - const auto& echo_spectrum = - aec_state_.UsableLinearEstimate() ? S2_linear : R2; - - // Determine if the suppressor should assume clock drift. - const bool clock_drift = config_.echo_removal_control.has_clock_drift || - echo_path_variability.clock_drift; - - // Compute preferred gains. - float high_bands_gain; + // Only do the below processing if the output of the audio processing module + // is used. std::array G; - suppression_gain_.GetGain(nearend_spectrum, echo_spectrum, R2, - cng_.NoiseSpectrum(), render_signal_analyzer_, - aec_state_, x, clock_drift, &high_bands_gain, &G); + if (capture_output_used_) { + // Estimate the residual echo power. + residual_echo_estimator_.Estimate(aec_state_, *render_buffer, S2_linear, Y2, + R2); - suppression_filter_.ApplyGain(comfort_noise, high_band_comfort_noise, G, - high_bands_gain, Y_fft, y); + // Suppressor nearend estimate. + if (aec_state_.UsableLinearEstimate()) { + // E2 is bound by Y2. + for (size_t ch = 0; ch < num_capture_channels_; ++ch) { + std::transform(E2[ch].begin(), E2[ch].end(), Y2[ch].begin(), + E2[ch].begin(), + [](float a, float b) { return std::min(a, b); }); + } + } + const auto& nearend_spectrum = aec_state_.UsableLinearEstimate() ? E2 : Y2; + + // Suppressor echo estimate. + const auto& echo_spectrum = + aec_state_.UsableLinearEstimate() ? S2_linear : R2; + + // Determine if the suppressor should assume clock drift. + const bool clock_drift = config_.echo_removal_control.has_clock_drift || + echo_path_variability.clock_drift; + + // Compute preferred gains. + float high_bands_gain; + suppression_gain_.GetGain(nearend_spectrum, echo_spectrum, R2, + cng_.NoiseSpectrum(), render_signal_analyzer_, + aec_state_, x, clock_drift, &high_bands_gain, &G); + + suppression_filter_.ApplyGain(comfort_noise, high_band_comfort_noise, G, + high_bands_gain, Y_fft, y); + + } else { + G.fill(0.f); + } // Update the metrics. metrics_.Update(aec_state_, cng_.NoiseSpectrum()[0], G); diff --git a/modules/audio_processing/aec3/echo_remover.h b/modules/audio_processing/aec3/echo_remover.h index ef4164688b..486a9a72f4 100644 --- a/modules/audio_processing/aec3/echo_remover.h +++ b/modules/audio_processing/aec3/echo_remover.h @@ -48,6 +48,12 @@ class EchoRemover { // Updates the status on whether echo leakage is detected in the output of the // echo remover. virtual void UpdateEchoLeakageStatus(bool leakage_detected) = 0; + + // Specifies whether the capture output will be used. The purpose of this is + // to allow the echo remover to deactivate some of the processing when the + // resulting output is anyway not used, for instance when the endpoint is + // muted. + virtual void SetCaptureOutputUsage(bool capture_output_used) = 0; }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/mock/mock_block_processor.h b/modules/audio_processing/aec3/mock/mock_block_processor.h index e1eb26702f..aa612257ea 100644 --- a/modules/audio_processing/aec3/mock/mock_block_processor.h +++ b/modules/audio_processing/aec3/mock/mock_block_processor.h @@ -44,6 +44,10 @@ class MockBlockProcessor : public BlockProcessor { (EchoControl::Metrics * metrics), (const, override)); MOCK_METHOD(void, SetAudioBufferDelay, (int delay_ms), (override)); + MOCK_METHOD(void, + SetCaptureOutputUsage, + (bool capture_output_used), + (override)); }; } // namespace test diff --git a/modules/audio_processing/aec3/mock/mock_echo_remover.h b/modules/audio_processing/aec3/mock/mock_echo_remover.h index 8a3044bcf1..60c5bf433e 100644 --- a/modules/audio_processing/aec3/mock/mock_echo_remover.h +++ b/modules/audio_processing/aec3/mock/mock_echo_remover.h @@ -44,6 +44,10 @@ class MockEchoRemover : public EchoRemover { GetMetrics, (EchoControl::Metrics * metrics), (const, override)); + MOCK_METHOD(void, + SetCaptureOutputUsage, + (bool capture_output_used), + (override)); }; } // namespace test