diff --git a/modules/audio_processing/aec3/aec_state.cc b/modules/audio_processing/aec3/aec_state.cc index 3c3d095280..db61c5a118 100644 --- a/modules/audio_processing/aec3/aec_state.cc +++ b/modules/audio_processing/aec3/aec_state.cc @@ -221,6 +221,7 @@ void AecState::Update( use_linear_filter_output_ = usable_linear_estimate_ && !TransparentMode(); data_dumper_->DumpRaw("aec3_erle", Erle()); + data_dumper_->DumpRaw("aec3_erle_onset", erle_estimator_.ErleOnsets()); data_dumper_->DumpRaw("aec3_erl", Erl()); data_dumper_->DumpRaw("aec3_erle_time_domain", ErleTimeDomain()); data_dumper_->DumpRaw("aec3_erl_time_domain", ErlTimeDomain()); diff --git a/modules/audio_processing/aec3/echo_remover.cc b/modules/audio_processing/aec3/echo_remover.cc index 28c5c0bf7d..8e14032f2c 100644 --- a/modules/audio_processing/aec3/echo_remover.cc +++ b/modules/audio_processing/aec3/echo_remover.cc @@ -245,10 +245,9 @@ void EchoRemoverImpl::ProcessCapture( data_dumper_->DumpRaw("aec3_E2_shadow", E2_shadow); data_dumper_->DumpRaw("aec3_S2_linear", S2_linear); data_dumper_->DumpRaw("aec3_Y2", Y2); - data_dumper_->DumpRaw("aec3_X2", render_buffer->Spectrum(0)); + data_dumper_->DumpRaw( + "aec3_X2", render_buffer->Spectrum(aec_state_.FilterDelayBlocks())); data_dumper_->DumpRaw("aec3_R2", R2); - data_dumper_->DumpRaw("aec3_erle", aec_state_.Erle()); - data_dumper_->DumpRaw("aec3_erl", aec_state_.Erl()); data_dumper_->DumpRaw("aec3_filter_delay", aec_state_.FilterDelayBlocks()); data_dumper_->DumpRaw("aec3_capture_saturation", aec_state_.SaturatedCapture() ? 1 : 0); diff --git a/modules/audio_processing/aec3/erle_estimator.cc b/modules/audio_processing/aec3/erle_estimator.cc index 0e4cbe1469..18763cbda2 100644 --- a/modules/audio_processing/aec3/erle_estimator.cc +++ b/modules/audio_processing/aec3/erle_estimator.cc @@ -24,7 +24,9 @@ ErleEstimator::ErleEstimator(float min_erle, max_erle_lf_(max_erle_lf), max_erle_hf_(max_erle_hf) { erle_.fill(min_erle_); + erle_onsets_.fill(min_erle_); hold_counters_.fill(0); + coming_onset_.fill(true); erle_time_domain_ = min_erle_; hold_counter_time_domain_ = 0; } @@ -43,29 +45,55 @@ void ErleEstimator::Update(rtc::ArrayView render_spectrum, // Corresponds of WGN of power -46 dBFS. constexpr float kX2Min = 44015068.0f; + constexpr int kOnsetSizeBlocks = 4; + constexpr int kErleHold = 100; + constexpr int kErleOnsetHold = kErleHold + kOnsetSizeBlocks; + + auto erle_band_update = [](float erle_band, float new_erle, float alpha_inc, + float alpha_dec, float min_erle, float max_erle) { + float alpha = new_erle > erle_band ? alpha_inc : alpha_dec; + float erle_band_out = erle_band; + erle_band_out = erle_band + alpha * (new_erle - erle_band); + erle_band_out = rtc::SafeClamp(erle_band_out, min_erle, max_erle); + return erle_band_out; + }; // Update the estimates in a clamped minimum statistics manner. auto erle_update = [&](size_t start, size_t stop, float max_erle) { for (size_t k = start; k < stop; ++k) { if (X2[k] > kX2Min && E2[k] > 0.f) { const float new_erle = Y2[k] / E2[k]; - if (new_erle > erle_[k]) { - hold_counters_[k - 1] = 100; - erle_[k] += 0.1f * (new_erle - erle_[k]); - erle_[k] = rtc::SafeClamp(erle_[k], min_erle_, max_erle); + + if (coming_onset_[k - 1]) { + hold_counters_[k - 1] = kErleOnsetHold; + coming_onset_[k - 1] = false; } + if (hold_counters_[k - 1] > kErleHold) { + erle_onsets_[k] = erle_band_update(erle_onsets_[k], new_erle, 0.05f, + 0.1f, min_erle_, max_erle); + } else { + hold_counters_[k - 1] = kErleHold; + } + erle_[k] = erle_band_update(erle_[k], new_erle, 0.01f, 0.02f, min_erle_, + max_erle); } } }; - erle_update(1, kFftLengthBy2 / 2, max_erle_lf_); - erle_update(kFftLengthBy2 / 2, kFftLengthBy2, max_erle_hf_); - std::for_each(hold_counters_.begin(), hold_counters_.end(), - [](int& a) { --a; }); - std::transform(hold_counters_.begin(), hold_counters_.end(), - erle_.begin() + 1, erle_.begin() + 1, [&](int a, float b) { - return a > 0 ? b : std::max(min_erle_, 0.97f * b); - }); + constexpr size_t kFftLengthBy4 = kFftLengthBy2 / 2; + erle_update(1, kFftLengthBy4, max_erle_lf_); + erle_update(kFftLengthBy4, kFftLengthBy2, max_erle_hf_); + + for (size_t k = 0; k < hold_counters_.size(); ++k) { + hold_counters_[k]--; + if (hold_counters_[k] <= 0) { + coming_onset_[k] = true; + if (erle_[k + 1] > erle_onsets_[k + 1]) { + erle_[k + 1] = std::max(erle_onsets_[k + 1], 0.97f * erle_[k + 1]); + RTC_DCHECK_LE(min_erle_, erle_[k + 1]); + } + } + } erle_[0] = erle_[1]; erle_[kFftLengthBy2] = erle_[kFftLengthBy2 - 1]; @@ -77,7 +105,7 @@ void ErleEstimator::Update(rtc::ArrayView render_spectrum, const float Y2_sum = std::accumulate(Y2.begin(), Y2.end(), 0.0f); const float new_erle = Y2_sum / E2_sum; if (new_erle > erle_time_domain_) { - hold_counter_time_domain_ = 100; + hold_counter_time_domain_ = kErleHold; erle_time_domain_ += 0.1f * (new_erle - erle_time_domain_); erle_time_domain_ = rtc::SafeClamp(erle_time_domain_, min_erle_, max_erle_lf_); diff --git a/modules/audio_processing/aec3/erle_estimator.h b/modules/audio_processing/aec3/erle_estimator.h index cb9fce6e14..809466c76e 100644 --- a/modules/audio_processing/aec3/erle_estimator.h +++ b/modules/audio_processing/aec3/erle_estimator.h @@ -32,10 +32,16 @@ class ErleEstimator { // Returns the most recent ERLE estimate. const std::array& Erle() const { return erle_; } + // Returns the ERLE that is estimated during onsets. Use for logging/testing. + const std::array& ErleOnsets() const { + return erle_onsets_; + } float ErleTimeDomain() const { return erle_time_domain_; } private: std::array erle_; + std::array erle_onsets_; + std::array coming_onset_; std::array hold_counters_; float erle_time_domain_; int hold_counter_time_domain_; diff --git a/modules/audio_processing/aec3/erle_estimator_unittest.cc b/modules/audio_processing/aec3/erle_estimator_unittest.cc index f3dd7d9bbb..9ccdb20a4a 100644 --- a/modules/audio_processing/aec3/erle_estimator_unittest.cc +++ b/modules/audio_processing/aec3/erle_estimator_unittest.cc @@ -16,52 +16,109 @@ namespace webrtc { namespace { constexpr int kLowFrequencyLimit = kFftLengthBy2 / 2; +constexpr float kMaxErleLf = 8.f; +constexpr float kMaxErleHf = 1.5f; +constexpr float kMinErle = 1.0f; +constexpr float kTrueErle = 10.f; +constexpr float kTrueErleOnsets = 1.0f; -void VerifyErle(const std::array& erle, - float erle_time_domain, - float reference_lf, - float reference_hf) { +void VerifyErleBands(const std::array& erle, + float reference_lf, + float reference_hf) { std::for_each( erle.begin(), erle.begin() + kLowFrequencyLimit, [reference_lf](float a) { EXPECT_NEAR(reference_lf, a, 0.001); }); std::for_each( erle.begin() + kLowFrequencyLimit, erle.end(), [reference_hf](float a) { EXPECT_NEAR(reference_hf, a, 0.001); }); +} + +void VerifyErle(const std::array& erle, + float erle_time_domain, + float reference_lf, + float reference_hf) { + VerifyErleBands(erle, reference_lf, reference_hf); EXPECT_NEAR(reference_lf, erle_time_domain, 0.001); } +void FormFarendFrame(std::array* X2, + std::array* E2, + std::array* Y2, + float erle) { + X2->fill(500 * 1000.f * 1000.f); + E2->fill(1000.f * 1000.f); + Y2->fill(erle * (*E2)[0]); +} + +void FormNearendFrame(std::array* X2, + std::array* E2, + std::array* Y2) { + X2->fill(0.f); + Y2->fill(500.f * 1000.f * 1000.f); + E2->fill((*Y2)[0]); +} + } // namespace -// Verifies that the correct ERLE estimates are achieved. -TEST(ErleEstimator, Estimates) { +TEST(ErleEstimator, VerifyErleIncreaseAndHold) { std::array X2; std::array E2; std::array Y2; - ErleEstimator estimator(1.f, 8.f, 1.5f); + ErleEstimator estimator(kMinErle, kMaxErleLf, kMaxErleHf); + + // Verifies that the ERLE estimate is properly increased to higher values. + FormFarendFrame(&X2, &E2, &Y2, kTrueErle); - // Verifies that the ERLE estimate is properley increased to higher values. - X2.fill(500 * 1000.f * 1000.f); - E2.fill(1000.f * 1000.f); - Y2.fill(10 * E2[0]); for (size_t k = 0; k < 200; ++k) { estimator.Update(X2, Y2, E2); } VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), 8.f, 1.5f); - // Verifies that the ERLE is not immediately decreased when the ERLE in the - // data decreases. - Y2.fill(0.1f * E2[0]); + FormNearendFrame(&X2, &E2, &Y2); + // Verifies that the ERLE is not immediately decreased during nearend + // activity. for (size_t k = 0; k < 98; ++k) { estimator.Update(X2, Y2, E2); } VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), 8.f, 1.5f); +} - // Verifies that the minimum ERLE is eventually achieved. - for (size_t k = 0; k < 1000; ++k) { +TEST(ErleEstimator, VerifyErleTrackingOnOnsets) { + std::array X2; + std::array E2; + std::array Y2; + + ErleEstimator estimator(kMinErle, kMaxErleLf, kMaxErleHf); + + for (size_t burst = 0; burst < 20; ++burst) { + FormFarendFrame(&X2, &E2, &Y2, kTrueErleOnsets); + for (size_t k = 0; k < 10; ++k) { + estimator.Update(X2, Y2, E2); + } + FormFarendFrame(&X2, &E2, &Y2, kTrueErle); + for (size_t k = 0; k < 200; ++k) { + estimator.Update(X2, Y2, E2); + } + FormNearendFrame(&X2, &E2, &Y2); + for (size_t k = 0; k < 100; ++k) { + estimator.Update(X2, Y2, E2); + } + } + VerifyErleBands(estimator.ErleOnsets(), kMinErle, kMinErle); + FormNearendFrame(&X2, &E2, &Y2); + for (size_t k = 0; k < 1000; k++) { estimator.Update(X2, Y2, E2); } - VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), 1.f, 1.f); + // Verifies that during ne activity, Erle converges to the Erle for onsets. + VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), kMinErle, kMinErle); +} + +TEST(ErleEstimator, VerifyNoErleUpdateDuringLowActivity) { + std::array X2; + std::array E2; + std::array Y2; + ErleEstimator estimator(kMinErle, kMaxErleLf, kMaxErleHf); // Verifies that the ERLE estimate is is not updated for low-level render // signals. @@ -70,6 +127,7 @@ TEST(ErleEstimator, Estimates) { for (size_t k = 0; k < 200; ++k) { estimator.Update(X2, Y2, E2); } - VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), 1.f, 1.f); + VerifyErle(estimator.Erle(), estimator.ErleTimeDomain(), kMinErle, kMinErle); } + } // namespace webrtc