diff --git a/modules/audio_processing/aec3/echo_remover.cc b/modules/audio_processing/aec3/echo_remover.cc index 8fef71c600..f75eb55d60 100644 --- a/modules/audio_processing/aec3/echo_remover.cc +++ b/modules/audio_processing/aec3/echo_remover.cc @@ -48,6 +48,10 @@ bool UseSmoothSignalTransitions() { "WebRTC-Aec3SmoothSignalTransitionsKillSwitch"); } +bool EnableBoundedNearend() { + return !field_trial::IsEnabled("WebRTC-Aec3BoundedNearendKillSwitch"); +} + void LinearEchoPower(const FftData& E, const FftData& Y, std::array* S2) { @@ -132,6 +136,7 @@ class EchoRemoverImpl final : public EchoRemover { const int sample_rate_hz_; const bool use_shadow_filter_output_; const bool use_smooth_signal_transitions_; + const bool enable_bounded_nearend_; Subtractor subtractor_; SuppressionGain suppression_gain_; ComfortNoiseGenerator cng_; @@ -166,6 +171,7 @@ EchoRemoverImpl::EchoRemoverImpl(const EchoCanceller3Config& config, UseShadowFilterOutput() && config_.filter.enable_shadow_filter_output_usage), use_smooth_signal_transitions_(UseSmoothSignalTransitions()), + enable_bounded_nearend_(EnableBoundedNearend()), subtractor_(config, data_dumper_.get(), optimization_), suppression_gain_(config_, optimization_, sample_rate_hz), cng_(optimization_), @@ -311,9 +317,18 @@ void EchoRemoverImpl::ProcessCapture( // Compute and apply the suppression gain. const auto& echo_spectrum = aec_state_.UsableLinearEstimate() ? S2_linear : R2; - suppression_gain_.GetGain(E2, echo_spectrum, R2, cng_.NoiseSpectrum(), E, Y, - render_signal_analyzer_, aec_state_, x, - &high_bands_gain, &G); + + std::array E2_bounded; + if (enable_bounded_nearend_) { + std::transform(E2.begin(), E2.end(), Y2.begin(), E2_bounded.begin(), + [](float a, float b) { return std::min(a, b); }); + } else { + std::copy(E2.begin(), E2.end(), E2_bounded.begin()); + } + + suppression_gain_.GetGain(E2, E2_bounded, echo_spectrum, R2, + cng_.NoiseSpectrum(), E, Y, render_signal_analyzer_, + aec_state_, x, &high_bands_gain, &G); suppression_filter_.ApplyGain(comfort_noise, high_band_comfort_noise, G, high_bands_gain, Y_fft, y); diff --git a/modules/audio_processing/aec3/suppression_gain.cc b/modules/audio_processing/aec3/suppression_gain.cc index c389a6a13e..6132566730 100644 --- a/modules/audio_processing/aec3/suppression_gain.cc +++ b/modules/audio_processing/aec3/suppression_gain.cc @@ -50,16 +50,13 @@ void AdjustForExternalFilters(std::array* gain) { // Scales the echo according to assessed audibility at the other end. void WeightEchoForAudibility(const EchoCanceller3Config& config, rtc::ArrayView echo, - rtc::ArrayView weighted_echo, - rtc::ArrayView one_by_weighted_echo) { + rtc::ArrayView weighted_echo) { RTC_DCHECK_EQ(kFftLengthBy2Plus1, echo.size()); RTC_DCHECK_EQ(kFftLengthBy2Plus1, weighted_echo.size()); - RTC_DCHECK_EQ(kFftLengthBy2Plus1, one_by_weighted_echo.size()); auto weigh = [](float threshold, float normalizer, size_t begin, size_t end, rtc::ArrayView echo, - rtc::ArrayView weighted_echo, - rtc::ArrayView one_by_weighted_echo) { + rtc::ArrayView weighted_echo) { for (size_t k = begin; k < end; ++k) { if (echo[k] < threshold) { float tmp = (threshold - echo[k]) * normalizer; @@ -67,26 +64,23 @@ void WeightEchoForAudibility(const EchoCanceller3Config& config, } else { weighted_echo[k] = echo[k]; } - one_by_weighted_echo[k] = - weighted_echo[k] > 0.f ? 1.f / weighted_echo[k] : 1.f; } }; float threshold = config.echo_audibility.floor_power * config.echo_audibility.audibility_threshold_lf; float normalizer = 1.f / (threshold - config.echo_audibility.floor_power); - weigh(threshold, normalizer, 0, 3, echo, weighted_echo, one_by_weighted_echo); + weigh(threshold, normalizer, 0, 3, echo, weighted_echo); threshold = config.echo_audibility.floor_power * config.echo_audibility.audibility_threshold_mf; normalizer = 1.f / (threshold - config.echo_audibility.floor_power); - weigh(threshold, normalizer, 3, 7, echo, weighted_echo, one_by_weighted_echo); + weigh(threshold, normalizer, 3, 7, echo, weighted_echo); threshold = config.echo_audibility.floor_power * config.echo_audibility.audibility_threshold_hf; normalizer = 1.f / (threshold - config.echo_audibility.floor_power); - weigh(threshold, normalizer, 7, kFftLengthBy2Plus1, echo, weighted_echo, - one_by_weighted_echo); + weigh(threshold, normalizer, 7, kFftLengthBy2Plus1, echo, weighted_echo); } // Computes the gain to reduce the echo to a non audible level. @@ -256,75 +250,98 @@ void SuppressionGain::GainToNoAudibleEcho( } } -// TODO(peah): Add further optimizations, in particular for the divisions. -void SuppressionGain::LowerBandGain( +// Compute the minimum gain as the attenuating gain to put the signal just +// above the zero sample values. +void SuppressionGain::GetMinGain( + rtc::ArrayView suppressor_input, + rtc::ArrayView weighted_residual_echo, bool low_noise_render, - const AecState& aec_state, - const std::array& nearend, - const std::array& echo, - const std::array& comfort_noise, - std::array* gain) { - const bool saturated_echo = aec_state.SaturatedEcho(); - const bool linear_echo_estimate = aec_state.UsableLinearEstimate(); - const auto& params = dominant_nearend_detector_.IsNearendState() - ? nearend_params_ - : normal_params_; - - // Weight echo power in terms of audibility. // Precompute 1/weighted echo - // (note that when the echo is zero, the precomputed value is never used). - std::array weighted_echo; - std::array one_by_weighted_echo; - WeightEchoForAudibility(config_, echo, weighted_echo, one_by_weighted_echo); - - // Compute the minimum gain as the attenuating gain to put the signal just - // above the zero sample values. - std::array min_gain; - const float min_echo_power = - low_noise_render ? config_.echo_audibility.low_render_limit - : config_.echo_audibility.normal_render_limit; + bool saturated_echo, + rtc::ArrayView min_gain) const { if (!saturated_echo) { - for (size_t k = 0; k < nearend.size(); ++k) { - const float denom = std::min(nearend[k], weighted_echo[k]); + const float min_echo_power = + low_noise_render ? config_.echo_audibility.low_render_limit + : config_.echo_audibility.normal_render_limit; + + for (size_t k = 0; k < suppressor_input.size(); ++k) { + const float denom = + std::min(suppressor_input[k], weighted_residual_echo[k]); min_gain[k] = denom > 0.f ? min_echo_power / denom : 1.f; min_gain[k] = std::min(min_gain[k], 1.f); } for (size_t k = 0; k < 6; ++k) { + const auto& dec = dominant_nearend_detector_.IsNearendState() + ? nearend_params_.max_dec_factor_lf + : normal_params_.max_dec_factor_lf; + // Make sure the gains of the low frequencies do not decrease too // quickly after strong nearend. if (last_nearend_[k] > last_echo_[k]) { - min_gain[k] = - std::max(min_gain[k], last_gain_[k] * params.max_dec_factor_lf); + min_gain[k] = std::max(min_gain[k], last_gain_[k] * dec); min_gain[k] = std::min(min_gain[k], 1.f); } } } else { - min_gain.fill(0.f); + std::fill(min_gain.begin(), min_gain.end(), 0.f); } +} - // Compute the maximum gain by limiting the gain increase from the previous - // gain. - std::array max_gain; - for (size_t k = 0; k < gain->size(); ++k) { - max_gain[k] = std::min(std::max(last_gain_[k] * params.max_inc_factor, - config_.suppressor.floor_first_increase), - 1.f); +// Compute the maximum gain by limiting the gain increase from the previous +// gain. +void SuppressionGain::GetMaxGain(rtc::ArrayView max_gain) const { + const auto& inc = dominant_nearend_detector_.IsNearendState() + ? nearend_params_.max_inc_factor + : normal_params_.max_inc_factor; + const auto& floor = config_.suppressor.floor_first_increase; + for (size_t k = 0; k < max_gain.size(); ++k) { + max_gain[k] = std::min(std::max(last_gain_[k] * inc, floor), 1.f); } +} + +// TODO(peah): Add further optimizations, in particular for the divisions. +void SuppressionGain::LowerBandGain( + bool low_noise_render, + const AecState& aec_state, + const std::array& suppressor_input, + const std::array& nearend, + const std::array& residual_echo, + const std::array& comfort_noise, + std::array* gain) { + const bool saturated_echo = aec_state.SaturatedEcho(); + + // Weight echo power in terms of audibility. // Precompute 1/weighted echo + // (note that when the echo is zero, the precomputed value is never used). + std::array weighted_residual_echo; + WeightEchoForAudibility(config_, residual_echo, weighted_residual_echo); + + std::array min_gain; + GetMinGain(suppressor_input, weighted_residual_echo, low_noise_render, + saturated_echo, min_gain); + + std::array max_gain; + GetMaxGain(max_gain); // Iteratively compute the gain required to attenuate the echo to a non // noticeable level. - std::array masker; + if (enable_new_suppression_) { - GainToNoAudibleEcho(nearend, weighted_echo, comfort_noise, min_gain, - max_gain, gain); + GainToNoAudibleEcho(nearend, weighted_residual_echo, comfort_noise, + min_gain, max_gain, gain); AdjustForExternalFilters(gain); } else { + const bool linear_echo_estimate = aec_state.UsableLinearEstimate(); + std::array masker; + std::array one_by_weighted_echo; + std::transform(weighted_residual_echo.begin(), weighted_residual_echo.end(), + one_by_weighted_echo.begin(), + [](float e) { return e > 0.f ? 1.f / e : 1.f; }); gain->fill(0.f); for (int k = 0; k < 2; ++k) { std::copy(comfort_noise.begin(), comfort_noise.end(), masker.begin()); GainToNoAudibleEchoFallback(config_, low_noise_render, saturated_echo, - linear_echo_estimate, nearend, weighted_echo, - masker, min_gain, max_gain, - one_by_weighted_echo, gain); + linear_echo_estimate, nearend, + weighted_residual_echo, masker, min_gain, + max_gain, one_by_weighted_echo, gain); AdjustForExternalFilters(gain); } } @@ -334,14 +351,16 @@ void SuppressionGain::LowerBandGain( // Store data required for the gain computation of the next block. std::copy(nearend.begin(), nearend.end(), last_nearend_.begin()); - std::copy(weighted_echo.begin(), weighted_echo.end(), last_echo_.begin()); + std::copy(weighted_residual_echo.begin(), weighted_residual_echo.end(), + last_echo_.begin()); std::copy(gain->begin(), gain->end(), last_gain_.begin()); aec3::VectorMath(optimization_).Sqrt(*gain); // Debug outputs for the purpose of development and analysis. data_dumper_->DumpRaw("aec3_suppressor_min_gain", min_gain); data_dumper_->DumpRaw("aec3_suppressor_max_gain", max_gain); - data_dumper_->DumpRaw("aec3_suppressor_masker", masker); + data_dumper_->DumpRaw("aec3_dominant_nearend", + dominant_nearend_detector_.IsNearendState()); } SuppressionGain::SuppressionGain(const EchoCanceller3Config& config, @@ -370,6 +389,7 @@ SuppressionGain::SuppressionGain(const EchoCanceller3Config& config, SuppressionGain::~SuppressionGain() = default; void SuppressionGain::GetGain( + const std::array& suppressor_input_spectrum, const std::array& nearend_spectrum, const std::array& echo_spectrum, const std::array& residual_echo_spectrum, @@ -400,10 +420,9 @@ void SuppressionGain::GetGain( // Compute gain for the lower band. bool low_noise_render = low_render_detector_.Detect(render); - const absl::optional narrow_peak_band = - render_signal_analyzer.NarrowPeakBand(); - LowerBandGain(low_noise_render, aec_state, nearend_average, - residual_echo_spectrum, comfort_noise_spectrum, low_band_gain); + LowerBandGain(low_noise_render, aec_state, suppressor_input_spectrum, + nearend_average, residual_echo_spectrum, comfort_noise_spectrum, + low_band_gain); // Limit the gain of the lower bands during start up and after resets. const float gain_upper_bound = aec_state.SuppressionGainLimit(); @@ -414,6 +433,9 @@ void SuppressionGain::GetGain( } // Compute the gain for the upper bands. + const absl::optional narrow_peak_band = + render_signal_analyzer.NarrowPeakBand(); + *high_bands_gain = UpperBandsGain(echo_spectrum, comfort_noise_spectrum, narrow_peak_band, aec_state.SaturatedEcho(), render, *low_band_gain); diff --git a/modules/audio_processing/aec3/suppression_gain.h b/modules/audio_processing/aec3/suppression_gain.h index b8519302bd..4eb8581a86 100644 --- a/modules/audio_processing/aec3/suppression_gain.h +++ b/modules/audio_processing/aec3/suppression_gain.h @@ -30,6 +30,7 @@ class SuppressionGain { int sample_rate_hz); ~SuppressionGain(); void GetGain( + const std::array& suppressor_input_spectrum, const std::array& nearend_spectrum, const std::array& echo_spectrum, const std::array& residual_echo_spectrum, @@ -63,12 +64,22 @@ class SuppressionGain { const std::array& max_gain, std::array* gain) const; - void LowerBandGain(bool stationary_with_low_power, - const AecState& aec_state, - const std::array& nearend, - const std::array& echo, - const std::array& comfort_noise, - std::array* gain); + void LowerBandGain( + bool stationary_with_low_power, + const AecState& aec_state, + const std::array& suppressor_input, + const std::array& nearend, + const std::array& residual_echo, + const std::array& comfort_noise, + std::array* gain); + + void GetMinGain(rtc::ArrayView suppressor_input, + rtc::ArrayView weighted_residual_echo, + bool low_noise_render, + bool saturated_echo, + rtc::ArrayView min_gain) const; + + void GetMaxGain(rtc::ArrayView max_gain) const; class LowNoiseRenderDetector { public: diff --git a/modules/audio_processing/aec3/suppression_gain_unittest.cc b/modules/audio_processing/aec3/suppression_gain_unittest.cc index ef31371fc8..1ff96ca6e0 100644 --- a/modules/audio_processing/aec3/suppression_gain_unittest.cc +++ b/modules/audio_processing/aec3/suppression_gain_unittest.cc @@ -45,7 +45,7 @@ TEST(SuppressionGain, NullOutputGains) { AecState aec_state(EchoCanceller3Config{}); EXPECT_DEATH( SuppressionGain(EchoCanceller3Config{}, DetectOptimization(), 16000) - .GetGain(E2, S2, R2, N2, E, Y, + .GetGain(E2, E2, S2, R2, N2, E, Y, RenderSignalAnalyzer((EchoCanceller3Config{})), aec_state, std::vector>( 3, std::vector(kBlockSize, 0.f)), @@ -106,7 +106,7 @@ TEST(SuppressionGain, BasicGainComputation) { subtractor.FilterImpulseResponse(), *render_delay_buffer->GetRenderBuffer(), E2, Y2, output, y); - suppression_gain.GetGain(E2, S2, R2, N2, E, Y, analyzer, aec_state, x, + suppression_gain.GetGain(E2, E2, S2, R2, N2, E, Y, analyzer, aec_state, x, &high_bands_gain, &g); } std::for_each(g.begin(), g.end(), @@ -126,7 +126,7 @@ TEST(SuppressionGain, BasicGainComputation) { subtractor.FilterImpulseResponse(), *render_delay_buffer->GetRenderBuffer(), E2, Y2, output, y); - suppression_gain.GetGain(E2, S2, R2, N2, E, Y, analyzer, aec_state, x, + suppression_gain.GetGain(E2, E2, S2, R2, N2, E, Y, analyzer, aec_state, x, &high_bands_gain, &g); } std::for_each(g.begin(), g.end(), @@ -138,7 +138,7 @@ TEST(SuppressionGain, BasicGainComputation) { E.re.fill(sqrtf(E2[0])); for (int k = 0; k < 10; ++k) { - suppression_gain.GetGain(E2, S2, R2, N2, E, Y, analyzer, aec_state, x, + suppression_gain.GetGain(E2, E2, S2, R2, N2, E, Y, analyzer, aec_state, x, &high_bands_gain, &g); } std::for_each(g.begin(), g.end(),