From 7682c6e2cb7778e157cb54a43baf2e785f02c275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20de=20Vicente=20Pe=C3=B1a?= Date: Thu, 22 Mar 2018 14:53:23 +0100 Subject: [PATCH] Improves in the ERLE estimation for AEC3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The estimation on how well the linear filter in the AEC3 is performing is done through an estimation of the ERLE. That estimation is then used for knowing how much the suppressor needs to react in order to cancel all the echoes. In the current code, the ERLE is quite conservative during farend inactivity and it is common that it goes to a minimum value during those periods. Under highly varying conditions, that is probably the right approach. However, in other scenarios where conditions does not change that fast there is a loss in transparency that could be avoided by means of a different ERLE estimation. In the current CL, the ERLE estimation has been changed in the following way: - During farend activity the ERLE is estimated through a 1st order AR smoother. This smoother goes faster toward lower ERLE values than to larger ones in order to avoid overestimation of this value. Furthermore, during the beginning of the farend burst, an estimation of the ERLE is done that aim to represent the performance of the linear filter during onsets. Under highly variant environments, those quantities, the ERLE during onsets and the one computed during the whole farend duration, would differ a lot. If the environment is more stationary, those quantities would be much more similar. - During nearend activity the ERLE estimation is decreased toward a value of the ERLE during onsets. Bug: webrtc:9040 Change-Id: Ieab86370a4333d2d0cd7041047d29651de4f6827 Reviewed-on: https://webrtc-review.googlesource.com/62342 Commit-Queue: Jesus de Vicente Pena Reviewed-by: Per Åhgren Cr-Commit-Position: refs/heads/master@{#22568} --- modules/audio_processing/aec3/aec_state.cc | 1 + modules/audio_processing/aec3/echo_remover.cc | 5 +- .../audio_processing/aec3/erle_estimator.cc | 54 ++++++++--- .../audio_processing/aec3/erle_estimator.h | 6 ++ .../aec3/erle_estimator_unittest.cc | 94 +++++++++++++++---- 5 files changed, 126 insertions(+), 34 deletions(-) diff --git a/modules/audio_processing/aec3/aec_state.cc b/modules/audio_processing/aec3/aec_state.cc index 3c3d095280..db61c5a118 100644 --- a/modules/audio_processing/aec3/aec_state.cc +++ b/modules/audio_processing/aec3/aec_state.cc @@ -221,6 +221,7 @@ void AecState::Update( use_linear_filter_output_ = usable_linear_estimate_ && !TransparentMode(); data_dumper_->DumpRaw("aec3_erle", Erle()); + data_dumper_->DumpRaw("aec3_erle_onset", erle_estimator_.ErleOnsets()); data_dumper_->DumpRaw("aec3_erl", Erl()); data_dumper_->DumpRaw("aec3_erle_time_domain", ErleTimeDomain()); data_dumper_->DumpRaw("aec3_erl_time_domain", ErlTimeDomain()); diff --git a/modules/audio_processing/aec3/echo_remover.cc b/modules/audio_processing/aec3/echo_remover.cc index 28c5c0bf7d..8e14032f2c 100644 --- a/modules/audio_processing/aec3/echo_remover.cc +++ b/modules/audio_processing/aec3/echo_remover.cc @@ -245,10 +245,9 @@ void EchoRemoverImpl::ProcessCapture( data_dumper_->DumpRaw("aec3_E2_shadow", E2_shadow); data_dumper_->DumpRaw("aec3_S2_linear", S2_linear); data_dumper_->DumpRaw("aec3_Y2", Y2); - data_dumper_->DumpRaw("aec3_X2", render_buffer->Spectrum(0)); + data_dumper_->DumpRaw( + "aec3_X2", render_buffer->Spectrum(aec_state_.FilterDelayBlocks())); data_dumper_->DumpRaw("aec3_R2", R2); - data_dumper_->DumpRaw("aec3_erle", aec_state_.Erle()); - data_dumper_->DumpRaw("aec3_erl", aec_state_.Erl()); data_dumper_->DumpRaw("aec3_filter_delay", aec_state_.FilterDelayBlocks()); data_dumper_->DumpRaw("aec3_capture_saturation", aec_state_.SaturatedCapture() ? 1 : 0); diff --git a/modules/audio_processing/aec3/erle_estimator.cc b/modules/audio_processing/aec3/erle_estimator.cc index 0e4cbe1469..18763cbda2 100644 --- a/modules/audio_processing/aec3/erle_estimator.cc +++ b/modules/audio_processing/aec3/erle_estimator.cc @@ -24,7 +24,9 @@ ErleEstimator::ErleEstimator(float min_erle, max_erle_lf_(max_erle_lf), max_erle_hf_(max_erle_hf) { erle_.fill(min_erle_); + erle_onsets_.fill(min_erle_); hold_counters_.fill(0); + coming_onset_.fill(true); erle_time_domain_ = min_erle_; hold_counter_time_domain_ = 0; } @@ -43,29 +45,55 @@ void ErleEstimator::Update(rtc::ArrayView render_spectrum, // Corresponds of WGN of power -46 dBFS. constexpr float kX2Min = 44015068.0f; + constexpr int kOnsetSizeBlocks = 4; + constexpr int kErleHold = 100; + constexpr int kErleOnsetHold = kErleHold + kOnsetSizeBlocks; + + auto erle_band_update = [](float erle_band, float new_erle, float alpha_inc, + float alpha_dec, float min_erle, float max_erle) { + float alpha = new_erle > erle_band ? alpha_inc : alpha_dec; + float erle_band_out = erle_band; + erle_band_out = erle_band + alpha * (new_erle - erle_band); + erle_band_out = rtc::SafeClamp(erle_band_out, min_erle, max_erle); + return erle_band_out; + }; // Update the estimates in a clamped minimum statistics manner. auto erle_update = [&](size_t start, size_t stop, float max_erle) { for (size_t k = start; k < stop; ++k) { if (X2[k] > kX2Min && E2[k] > 0.f) { const float new_erle = Y2[k] / E2[k]; - if (new_erle > erle_[k]) { - hold_counters_[k - 1] = 100; - erle_[k] += 0.1f * (new_erle - erle_[k]); - erle_[k] = rtc::SafeClamp(erle_[k], min_erle_, max_erle); + + if (coming_onset_[k - 1]) { + hold_counters_[k - 1] = kErleOnsetHold; + coming_onset_[k - 1] = false; } + if (hold_counters_[k - 1] > kErleHold) { + erle_onsets_[k] = erle_band_update(erle_onsets_[k], new_erle, 0.05f, + 0.1f, min_erle_, max_erle); + } else { + hold_counters_[k - 1] = kErleHold; + } + erle_[k] = erle_band_update(erle_[k], new_erle, 0.01f, 0.02f, min_erle_, + max_erle); } } }; - erle_update(1, kFftLengthBy2 / 2, max_erle_lf_); - erle_update(kFftLengthBy2 / 2, kFftLengthBy2, max_erle_hf_); - std::for_each(hold_counters_.begin(), hold_counters_.end(), - [](int& a) { --a; }); - std::transform(hold_counters_.begin(), hold_counters_.end(), - erle_.begin() + 1, erle_.begin() + 1, [&](int a, float b) { - return a > 0 ? b : std::max(min_erle_, 0.97f * b); - }); + constexpr size_t kFftLengthBy4 = kFftLengthBy2 / 2; + erle_update(1, kFftLengthBy4, max_erle_lf_); + erle_update(kFftLengthBy4, kFftLengthBy2, max_erle_hf_); + + for (size_t k = 0; k < hold_counters_.size(); ++k) { + hold_counters_[k]--; + if (hold_counters_[k] <= 0) { + coming_onset_[k] = true; + if (erle_[k + 1] > erle_onsets_[k + 1]) { + erle_[k + 1] = std::max(erle_onsets_[k + 1], 0.97f * erle_[k + 1]); + RTC_DCHECK_LE(min_erle_, erle_[k + 1]); + } + } + } erle_[0] = erle_[1]; erle_[kFftLengthBy2] = erle_[kFftLengthBy2 - 1]; @@ -77,7 +105,7 @@ void ErleEstimator::Update(rtc::ArrayView render_spectrum, const float Y2_sum = std::accumulate(Y2.begin(), Y2.end(), 0.0f); const float new_erle = Y2_sum / E2_sum; if (new_erle > erle_time_domain_) { - hold_counter_time_domain_ = 100; + hold_counter_time_domain_ = kErleHold; erle_time_domain_ += 0.1f * (new_erle - erle_time_domain_); erle_time_domain_ = rtc::SafeClamp(erle_time_domain_, min_erle_, max_erle_lf_); diff --git a/modules/audio_processing/aec3/erle_estimator.h b/modules/audio_processing/aec3/erle_estimator.h index cb9fce6e14..809466c76e 100644 --- a/modules/audio_processing/aec3/erle_estimator.h +++ b/modules/audio_processing/aec3/erle_estimator.h @@ -32,10 +32,16 @@ class ErleEstimator { // Returns the most recent ERLE estimate. const std::array& Erle() const { return erle_; } + // Returns the ERLE that is estimated during onsets. Use for logging/testing. + const std::array& ErleOnsets() const { + return erle_onsets_; + } float ErleTimeDomain() const { return erle_time_domain_; } private: std::array erle_; + std::array erle_onsets_; + std::array coming_onset_; std::array hold_counters_; float erle_time_domain_; int hold_counter_time_domain_; diff --git a/modules/audio_processing/aec3/erle_estimator_unittest.cc b/modules/audio_processing/aec3/erle_estimator_unittest.cc index f3dd7d9bbb..9ccdb20a4a 100644 --- a/modules/audio_processing/aec3/erle_estimator_unittest.cc +++ b/modules/audio_processing/aec3/erle_estimator_unittest.cc @@ -16,52 +16,109 @@ namespace webrtc { namespace { constexpr int kLowFrequencyLimit = kFftLengthBy2 / 2; +constexpr float kMaxErleLf = 8.f; +constexpr float kMaxErleHf = 1.5f; +constexpr float kMinErle = 1.0f; +constexpr float kTrueErle = 10.f; +constexpr float kTrueErleOnsets = 1.0f; -void VerifyErle(const std::array& erle, - float erle_time_domain, - float reference_lf, - float reference_hf) { +void VerifyErleBands(const std::array& erle, + float reference_lf, + float reference_hf) { std::for_each( erle.begin(), erle.begin() + kLowFrequencyLimit, [reference_lf](float a) { EXPECT_NEAR(reference_lf, a, 0.001); }); std::for_each( erle.begin() + kLowFrequencyLimit, erle.end(), [reference_hf](float a) { EXPECT_NEAR(reference_hf, a, 0.001); }); +} + +void VerifyErle(const std::array& erle, + float erle_time_domain, + float reference_lf, + float reference_hf) { + VerifyErleBands(erle, reference_lf, reference_hf); EXPECT_NEAR(reference_lf, erle_time_domain, 0.001); } +void FormFarendFrame(std::array* X2, + std::array* E2, + std::array* Y2, + float erle) { + X2->fill(500 * 1000.f * 1000.f); + E2->fill(1000.f * 1000.f); + Y2->fill(erle * (*E2)[0]); +} + +void FormNearendFrame(std::array* X2, + std::array* E2, + std::array* Y2) { + X2->fill(0.f); + Y2->fill(500.f * 1000.f * 1000.f); + E2->fill((*Y2)[0]); +} + } // namespace -// Verifies that the correct ERLE estimates are achieved. -TEST(ErleEstimator, Estimates) { +TEST(ErleEstimator, VerifyErleIncreaseAndHold) { std::array X2; std::array E2; std::array Y2; - ErleEstimator estimator(1.f, 8.f, 1.5f); + ErleEstimator estimator(kMinErle, kMaxErleLf, kMaxErleHf); + + // Verifies that the ERLE estimate is properly increased to higher values. + FormFarendFrame(&X2, &E2, &Y2, kTrueErle); - // Verifies that the ERLE estimate is properley increased to higher values. - X2.fill(500 * 1000.f * 1000.f); - E2.fill(1000.f * 1000.f); - Y2.fill(10 * E2[0]); for (size_t k = 0; k < 200; ++k) { estimator.Update(X2, Y2, E2); } VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), 8.f, 1.5f); - // Verifies that the ERLE is not immediately decreased when the ERLE in the - // data decreases. - Y2.fill(0.1f * E2[0]); + FormNearendFrame(&X2, &E2, &Y2); + // Verifies that the ERLE is not immediately decreased during nearend + // activity. for (size_t k = 0; k < 98; ++k) { estimator.Update(X2, Y2, E2); } VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), 8.f, 1.5f); +} - // Verifies that the minimum ERLE is eventually achieved. - for (size_t k = 0; k < 1000; ++k) { +TEST(ErleEstimator, VerifyErleTrackingOnOnsets) { + std::array X2; + std::array E2; + std::array Y2; + + ErleEstimator estimator(kMinErle, kMaxErleLf, kMaxErleHf); + + for (size_t burst = 0; burst < 20; ++burst) { + FormFarendFrame(&X2, &E2, &Y2, kTrueErleOnsets); + for (size_t k = 0; k < 10; ++k) { + estimator.Update(X2, Y2, E2); + } + FormFarendFrame(&X2, &E2, &Y2, kTrueErle); + for (size_t k = 0; k < 200; ++k) { + estimator.Update(X2, Y2, E2); + } + FormNearendFrame(&X2, &E2, &Y2); + for (size_t k = 0; k < 100; ++k) { + estimator.Update(X2, Y2, E2); + } + } + VerifyErleBands(estimator.ErleOnsets(), kMinErle, kMinErle); + FormNearendFrame(&X2, &E2, &Y2); + for (size_t k = 0; k < 1000; k++) { estimator.Update(X2, Y2, E2); } - VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), 1.f, 1.f); + // Verifies that during ne activity, Erle converges to the Erle for onsets. + VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), kMinErle, kMinErle); +} + +TEST(ErleEstimator, VerifyNoErleUpdateDuringLowActivity) { + std::array X2; + std::array E2; + std::array Y2; + ErleEstimator estimator(kMinErle, kMaxErleLf, kMaxErleHf); // Verifies that the ERLE estimate is is not updated for low-level render // signals. @@ -70,6 +127,7 @@ TEST(ErleEstimator, Estimates) { for (size_t k = 0; k < 200; ++k) { estimator.Update(X2, Y2, E2); } - VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), 1.f, 1.f); + VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), kMinErle, kMinErle); } + } // namespace webrtc