From 8b7d206d37ba42a0b14e34440180ebb8566845b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20=C3=85hgren?= Date: Tue, 30 Oct 2018 23:44:40 +0100 Subject: [PATCH] AEC3: Decrease latency until the delay has been detected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL utilizes the existing, but unused, ability to set different histogram thresholds for early and late delay estimation. It does so by tuning the parameters for these. On top of that, some corrections are added to correctly handle resets and the use of the hysteresis thresholds. Bug: webrtc:19886,chromium:896334 Change-Id: I950ac107c124541af8f02b4403f477dda71cc1a1 Reviewed-on: https://webrtc-review.googlesource.com/c/106706 Reviewed-by: Sam Zackrisson Reviewed-by: Gustaf Ullberg Commit-Queue: Per Åhgren Cr-Commit-Position: refs/heads/master@{#25443} --- api/audio/echo_canceller3_config.h | 2 +- .../audio_processing/aec3/block_processor.cc | 10 +++--- .../audio_processing/aec3/block_processor2.cc | 6 ++-- .../audio_processing/aec3/echo_canceller3.cc | 8 +++++ .../aec3/echo_path_delay_estimator.cc | 21 ++++++----- .../aec3/echo_path_delay_estimator.h | 9 +++-- .../aec3/matched_filter_lag_aggregator.cc | 12 +++++-- .../aec3/matched_filter_lag_aggregator.h | 2 +- .../aec3/mock/mock_render_delay_controller.h | 2 +- .../aec3/render_delay_controller.cc | 35 +++++++++++++------ .../aec3/render_delay_controller.h | 5 +-- .../aec3/render_delay_controller2.cc | 32 ++++++++++++----- .../audio_processing_configs_fuzzer.cc | 1 + 13 files changed, 98 insertions(+), 47 deletions(-) diff --git a/api/audio/echo_canceller3_config.h b/api/audio/echo_canceller3_config.h index 8ff5aca90a..251f282539 100644 --- a/api/audio/echo_canceller3_config.h +++ b/api/audio/echo_canceller3_config.h @@ -50,7 +50,7 @@ struct RTC_EXPORT EchoCanceller3Config { struct DelaySelectionThresholds { int initial; int converged; - } delay_selection_thresholds = {25, 25}; + } delay_selection_thresholds = {5, 20}; } delay; struct Filter { diff --git a/modules/audio_processing/aec3/block_processor.cc b/modules/audio_processing/aec3/block_processor.cc index 4f57902e7d..590380f897 100644 --- a/modules/audio_processing/aec3/block_processor.cc +++ b/modules/audio_processing/aec3/block_processor.cc @@ -108,7 +108,7 @@ void BlockProcessorImpl::ProcessCapture( if (!capture_properly_started_) { capture_properly_started_ = true; render_buffer_->Reset(); - delay_controller_->Reset(); + delay_controller_->Reset(true); } } else { // If no render data has yet arrived, do not process the capture signal. @@ -123,7 +123,7 @@ void BlockProcessorImpl::ProcessCapture( render_properly_started_) { echo_path_variability.delay_change = EchoPathVariability::DelayAdjustment::kBufferFlush; - delay_controller_->Reset(); + delay_controller_->Reset(true); RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun at block " << capture_call_counter_; } @@ -139,7 +139,7 @@ void BlockProcessorImpl::ProcessCapture( estimated_delay_->quality == DelayEstimate::Quality::kRefined) { echo_path_variability.delay_change = EchoPathVariability::DelayAdjustment::kDelayReset; - delay_controller_->Reset(); + delay_controller_->Reset(true); capture_properly_started_ = false; render_properly_started_ = false; @@ -151,7 +151,7 @@ void BlockProcessorImpl::ProcessCapture( // echo. echo_path_variability.delay_change = EchoPathVariability::DelayAdjustment::kDelayReset; - delay_controller_->Reset(); + delay_controller_->Reset(true); capture_properly_started_ = false; render_properly_started_ = false; RTC_LOG(LS_WARNING) << "Reset due to render buffer api skew at block " @@ -184,7 +184,7 @@ void BlockProcessorImpl::ProcessCapture( if (estimated_delay_->quality == DelayEstimate::Quality::kRefined) { echo_path_variability.delay_change = EchoPathVariability::DelayAdjustment::kDelayReset; - delay_controller_->Reset(); + delay_controller_->Reset(true); render_buffer_->Reset(); capture_properly_started_ = false; render_properly_started_ = false; diff --git a/modules/audio_processing/aec3/block_processor2.cc b/modules/audio_processing/aec3/block_processor2.cc index 5fa5abe7e9..3616427ce2 100644 --- a/modules/audio_processing/aec3/block_processor2.cc +++ b/modules/audio_processing/aec3/block_processor2.cc @@ -115,7 +115,7 @@ void BlockProcessorImpl2::ProcessCapture( if (!capture_properly_started_) { capture_properly_started_ = true; render_buffer_->Reset(); - delay_controller_->Reset(); + delay_controller_->Reset(true); } } else { // If no render data has yet arrived, do not process the capture signal. @@ -130,7 +130,7 @@ void BlockProcessorImpl2::ProcessCapture( render_properly_started_) { echo_path_variability.delay_change = EchoPathVariability::DelayAdjustment::kBufferFlush; - delay_controller_->Reset(); + delay_controller_->Reset(true); RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun at block " << capture_call_counter_; } @@ -143,7 +143,7 @@ void BlockProcessorImpl2::ProcessCapture( render_buffer_->PrepareCaptureProcessing(); // Reset the delay controller at render buffer underrun. if (buffer_event == RenderDelayBuffer::BufferingEvent::kRenderUnderrun) { - delay_controller_->Reset(); + delay_controller_->Reset(false); } data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize, diff --git a/modules/audio_processing/aec3/echo_canceller3.cc b/modules/audio_processing/aec3/echo_canceller3.cc index 1b089f841d..5debcda349 100644 --- a/modules/audio_processing/aec3/echo_canceller3.cc +++ b/modules/audio_processing/aec3/echo_canceller3.cc @@ -82,6 +82,10 @@ bool EnableNewRenderBuffering() { return !field_trial::IsEnabled("WebRTC-Aec3NewRenderBufferingKillSwitch"); } +bool UseEarlyDelayDetection() { + return !field_trial::IsEnabled("WebRTC-Aec3EarlyDelayDetectionKillSwitch"); +} + // Method for adjusting config parameter dependencies.. EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) { EchoCanceller3Config adjusted_cfg = config; @@ -161,6 +165,10 @@ EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) { adjusted_cfg.echo_audibility.use_stationarity_properties_at_init = false; } + if (!UseEarlyDelayDetection()) { + adjusted_cfg.delay.delay_selection_thresholds = {25, 25}; + } + return adjusted_cfg; } diff --git a/modules/audio_processing/aec3/echo_path_delay_estimator.cc b/modules/audio_processing/aec3/echo_path_delay_estimator.cc index 3a25deb101..5c838aed8a 100644 --- a/modules/audio_processing/aec3/echo_path_delay_estimator.cc +++ b/modules/audio_processing/aec3/echo_path_delay_estimator.cc @@ -49,13 +49,8 @@ EchoPathDelayEstimator::EchoPathDelayEstimator( EchoPathDelayEstimator::~EchoPathDelayEstimator() = default; -void EchoPathDelayEstimator::Reset(bool soft_reset) { - if (!soft_reset) { - matched_filter_lag_aggregator_.Reset(); - } - matched_filter_.Reset(); - old_aggregated_lag_ = absl::nullopt; - consistent_estimate_counter_ = 0; +void EchoPathDelayEstimator::Reset(bool reset_delay_confidence) { + Reset(true, reset_delay_confidence); } absl::optional EchoPathDelayEstimator::EstimateDelay( @@ -102,10 +97,20 @@ absl::optional EchoPathDelayEstimator::EstimateDelay( old_aggregated_lag_ = aggregated_matched_filter_lag; constexpr size_t kNumBlocksPerSecondBy2 = kNumBlocksPerSecond / 2; if (consistent_estimate_counter_ > kNumBlocksPerSecondBy2) { - Reset(true); + Reset(false, false); } return aggregated_matched_filter_lag; } +void EchoPathDelayEstimator::Reset(bool reset_lag_aggregator, + bool reset_delay_confidence) { + if (reset_lag_aggregator) { + matched_filter_lag_aggregator_.Reset(reset_delay_confidence); + } + matched_filter_.Reset(); + old_aggregated_lag_ = absl::nullopt; + consistent_estimate_counter_ = 0; +} + } // namespace webrtc diff --git a/modules/audio_processing/aec3/echo_path_delay_estimator.h b/modules/audio_processing/aec3/echo_path_delay_estimator.h index b1c42473f5..060c8753bd 100644 --- a/modules/audio_processing/aec3/echo_path_delay_estimator.h +++ b/modules/audio_processing/aec3/echo_path_delay_estimator.h @@ -34,9 +34,9 @@ class EchoPathDelayEstimator { const EchoCanceller3Config& config); ~EchoPathDelayEstimator(); - // Resets the estimation. If the soft-reset is specified, only the matched - // filters are reset. - void Reset(bool soft_reset); + // Resets the estimation. If the delay confidence is reset, the reset behavior + // is as if the call is restarted. + void Reset(bool reset_delay_confidence); // Produce a delay estimate if such is avaliable. absl::optional EstimateDelay( @@ -59,6 +59,9 @@ class EchoPathDelayEstimator { absl::optional old_aggregated_lag_; size_t consistent_estimate_counter_ = 0; + // Internal reset method with more granularity. + void Reset(bool reset_lag_aggregator, bool reset_delay_confidence); + RTC_DISALLOW_COPY_AND_ASSIGN(EchoPathDelayEstimator); }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc b/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc index 5ca62683d1..f59525eee9 100644 --- a/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc +++ b/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc @@ -31,11 +31,13 @@ MatchedFilterLagAggregator::MatchedFilterLagAggregator( MatchedFilterLagAggregator::~MatchedFilterLagAggregator() = default; -void MatchedFilterLagAggregator::Reset() { +void MatchedFilterLagAggregator::Reset(bool hard_reset) { std::fill(histogram_.begin(), histogram_.end(), 0); histogram_data_.fill(0); histogram_data_index_ = 0; - significant_candidate_found_ = false; + if (hard_reset) { + significant_candidate_found_ = false; + } } absl::optional MatchedFilterLagAggregator::Aggregate( @@ -81,9 +83,13 @@ absl::optional MatchedFilterLagAggregator::Aggregate( if (histogram_[candidate] > thresholds_.converged || (histogram_[candidate] > thresholds_.initial && !significant_candidate_found_)) { - return DelayEstimate(DelayEstimate::Quality::kRefined, candidate); + DelayEstimate::Quality quality = significant_candidate_found_ + ? DelayEstimate::Quality::kRefined + : DelayEstimate::Quality::kCoarse; + return DelayEstimate(quality, candidate); } } + return absl::nullopt; } diff --git a/modules/audio_processing/aec3/matched_filter_lag_aggregator.h b/modules/audio_processing/aec3/matched_filter_lag_aggregator.h index c57051aa68..d7f34aed60 100644 --- a/modules/audio_processing/aec3/matched_filter_lag_aggregator.h +++ b/modules/audio_processing/aec3/matched_filter_lag_aggregator.h @@ -34,7 +34,7 @@ class MatchedFilterLagAggregator { ~MatchedFilterLagAggregator(); // Resets the aggregator. - void Reset(); + void Reset(bool hard_reset); // Aggregates the provided lag estimates. absl::optional Aggregate( diff --git a/modules/audio_processing/aec3/mock/mock_render_delay_controller.h b/modules/audio_processing/aec3/mock/mock_render_delay_controller.h index ed5971cda0..5520f764fb 100644 --- a/modules/audio_processing/aec3/mock/mock_render_delay_controller.h +++ b/modules/audio_processing/aec3/mock/mock_render_delay_controller.h @@ -25,7 +25,7 @@ class MockRenderDelayController : public RenderDelayController { MockRenderDelayController(); virtual ~MockRenderDelayController(); - MOCK_METHOD0(Reset, void()); + MOCK_METHOD1(Reset, void(bool reset_delay_statistics)); MOCK_METHOD0(LogRenderCall, void()); MOCK_METHOD4(GetDelay, absl::optional( diff --git a/modules/audio_processing/aec3/render_delay_controller.cc b/modules/audio_processing/aec3/render_delay_controller.cc index e81d9d3ecd..36e75d9fa9 100644 --- a/modules/audio_processing/aec3/render_delay_controller.cc +++ b/modules/audio_processing/aec3/render_delay_controller.cc @@ -45,6 +45,10 @@ bool UseOffsetBlocks() { return field_trial::IsEnabled("WebRTC-Aec3UseOffsetBlocks"); } +bool UseEarlyDelayDetection() { + return !field_trial::IsEnabled("WebRTC-Aec3EarlyDelayDetectionKillSwitch"); +} + constexpr int kSkewHistorySizeLog2 = 8; class RenderDelayControllerImpl final : public RenderDelayController { @@ -53,7 +57,7 @@ class RenderDelayControllerImpl final : public RenderDelayController { int non_causal_offset, int sample_rate_hz); ~RenderDelayControllerImpl() override; - void Reset() override; + void Reset(bool reset_delay_confidence) override; void LogRenderCall() override; absl::optional GetDelay( const DownsampledRenderBuffer& render_buffer, @@ -64,6 +68,7 @@ class RenderDelayControllerImpl final : public RenderDelayController { private: static int instance_count_; std::unique_ptr data_dumper_; + const bool use_early_delay_detection_; const int delay_headroom_blocks_; const int hysteresis_limit_1_blocks_; const int hysteresis_limit_2_blocks_; @@ -82,6 +87,7 @@ class RenderDelayControllerImpl final : public RenderDelayController { size_t capture_call_counter_ = 0; int delay_change_counter_ = 0; size_t soft_reset_counter_ = 0; + DelayEstimate::Quality last_delay_estimate_quality_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayControllerImpl); }; @@ -130,6 +136,7 @@ RenderDelayControllerImpl::RenderDelayControllerImpl( int sample_rate_hz) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + use_early_delay_detection_(UseEarlyDelayDetection()), delay_headroom_blocks_( static_cast(config.delay.delay_headroom_blocks)), hysteresis_limit_1_blocks_( @@ -140,7 +147,8 @@ RenderDelayControllerImpl::RenderDelayControllerImpl( use_offset_blocks_(UseOffsetBlocks()), delay_estimator_(data_dumper_.get(), config), delay_buf_(kBlockSize * non_causal_offset, 0.f), - skew_estimator_(kSkewHistorySizeLog2) { + skew_estimator_(kSkewHistorySizeLog2), + last_delay_estimate_quality_(DelayEstimate::Quality::kCoarse) { RTC_DCHECK(ValidFullBandRate(sample_rate_hz)); delay_estimator_.LogDelayEstimationProperties(sample_rate_hz, delay_buf_.size()); @@ -148,16 +156,19 @@ RenderDelayControllerImpl::RenderDelayControllerImpl( RenderDelayControllerImpl::~RenderDelayControllerImpl() = default; -void RenderDelayControllerImpl::Reset() { +void RenderDelayControllerImpl::Reset(bool reset_delay_confidence) { delay_ = absl::nullopt; delay_samples_ = absl::nullopt; skew_ = absl::nullopt; previous_offset_blocks_ = 0; std::fill(delay_buf_.begin(), delay_buf_.end(), 0.f); - delay_estimator_.Reset(false); + delay_estimator_.Reset(reset_delay_confidence); skew_estimator_.Reset(); delay_change_counter_ = 0; soft_reset_counter_ = 0; + if (reset_delay_confidence) { + last_delay_estimate_quality_ = DelayEstimate::Quality::kCoarse; + } } void RenderDelayControllerImpl::LogRenderCall() { @@ -195,9 +206,6 @@ absl::optional RenderDelayControllerImpl::GetDelay( absl::optional skew = skew_estimator_.GetSkewFromCapture(); if (delay_samples) { - // TODO(peah): Refactor the rest of the code to assume a kRefined estimate - // quality. - RTC_DCHECK(DelayEstimate::Quality::kRefined == delay_samples->quality); if (!delay_samples_ || delay_samples->delay != delay_samples_->delay) { delay_change_counter_ = 0; } @@ -240,7 +248,7 @@ absl::optional RenderDelayControllerImpl::GetDelay( } else if (soft_reset_counter_ > 10 * kNumBlocksPerSecond) { // Soft reset the delay estimator if there is a significant offset // detected. - delay_estimator_.Reset(true); + delay_estimator_.Reset(false); soft_reset_counter_ = 0; } } @@ -265,9 +273,14 @@ absl::optional RenderDelayControllerImpl::GetDelay( if (delay_samples_) { // Compute the render delay buffer delay. - delay_ = ComputeBufferDelay( - delay_, delay_headroom_blocks_, hysteresis_limit_1_blocks_, - hysteresis_limit_2_blocks_, offset_blocks, *delay_samples_); + const bool use_hysteresis = + last_delay_estimate_quality_ == DelayEstimate::Quality::kRefined && + delay_samples_->quality == DelayEstimate::Quality::kRefined; + delay_ = ComputeBufferDelay(delay_, delay_headroom_blocks_, + use_hysteresis ? hysteresis_limit_1_blocks_ : 0, + use_hysteresis ? hysteresis_limit_2_blocks_ : 0, + offset_blocks, *delay_samples_); + last_delay_estimate_quality_ = delay_samples_->quality; } metrics_.Update(delay_samples_ ? absl::optional(delay_samples_->delay) diff --git a/modules/audio_processing/aec3/render_delay_controller.h b/modules/audio_processing/aec3/render_delay_controller.h index a0cbe89f20..41ba4224c9 100644 --- a/modules/audio_processing/aec3/render_delay_controller.h +++ b/modules/audio_processing/aec3/render_delay_controller.h @@ -31,8 +31,9 @@ class RenderDelayController { int sample_rate_hz); virtual ~RenderDelayController() = default; - // Resets the delay controller. - virtual void Reset() = 0; + // Resets the delay controller. If the delay confidence is reset, the reset + // behavior is as if the call is restarted. + virtual void Reset(bool reset_delay_confidence) = 0; // Logs a render call. virtual void LogRenderCall() = 0; diff --git a/modules/audio_processing/aec3/render_delay_controller2.cc b/modules/audio_processing/aec3/render_delay_controller2.cc index 64f14cb64c..1b7c18d3e9 100644 --- a/modules/audio_processing/aec3/render_delay_controller2.cc +++ b/modules/audio_processing/aec3/render_delay_controller2.cc @@ -24,17 +24,22 @@ #include "rtc_base/atomicops.h" #include "rtc_base/checks.h" #include "rtc_base/constructormagic.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { namespace { +bool UseEarlyDelayDetection() { + return !field_trial::IsEnabled("WebRTC-Aec3EarlyDelayDetectionKillSwitch"); +} + class RenderDelayControllerImpl2 final : public RenderDelayController { public: RenderDelayControllerImpl2(const EchoCanceller3Config& config, int sample_rate_hz); ~RenderDelayControllerImpl2() override; - void Reset() override; + void Reset(bool reset_delay_confidence) override; void LogRenderCall() override; absl::optional GetDelay( const DownsampledRenderBuffer& render_buffer, @@ -45,6 +50,7 @@ class RenderDelayControllerImpl2 final : public RenderDelayController { private: static int instance_count_; std::unique_ptr data_dumper_; + const bool use_early_delay_detection_; const int delay_headroom_blocks_; const int hysteresis_limit_1_blocks_; const int hysteresis_limit_2_blocks_; @@ -54,6 +60,7 @@ class RenderDelayControllerImpl2 final : public RenderDelayController { absl::optional delay_samples_; size_t capture_call_counter_ = 0; int delay_change_counter_ = 0; + DelayEstimate::Quality last_delay_estimate_quality_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayControllerImpl2); }; @@ -100,24 +107,29 @@ RenderDelayControllerImpl2::RenderDelayControllerImpl2( int sample_rate_hz) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + use_early_delay_detection_(UseEarlyDelayDetection()), delay_headroom_blocks_( static_cast(config.delay.delay_headroom_blocks)), hysteresis_limit_1_blocks_( static_cast(config.delay.hysteresis_limit_1_blocks)), hysteresis_limit_2_blocks_( static_cast(config.delay.hysteresis_limit_2_blocks)), - delay_estimator_(data_dumper_.get(), config) { + delay_estimator_(data_dumper_.get(), config), + last_delay_estimate_quality_(DelayEstimate::Quality::kCoarse) { RTC_DCHECK(ValidFullBandRate(sample_rate_hz)); delay_estimator_.LogDelayEstimationProperties(sample_rate_hz, 0); } RenderDelayControllerImpl2::~RenderDelayControllerImpl2() = default; -void RenderDelayControllerImpl2::Reset() { +void RenderDelayControllerImpl2::Reset(bool reset_delay_confidence) { delay_ = absl::nullopt; delay_samples_ = absl::nullopt; - delay_estimator_.Reset(false); + delay_estimator_.Reset(reset_delay_confidence); delay_change_counter_ = 0; + if (reset_delay_confidence || true) { + last_delay_estimate_quality_ = DelayEstimate::Quality::kCoarse; + } } void RenderDelayControllerImpl2::LogRenderCall() {} @@ -141,9 +153,6 @@ absl::optional RenderDelayControllerImpl2::GetDelay( } if (delay_samples) { - // TODO(peah): Refactor the rest of the code to assume a kRefined estimate - // quality. - RTC_DCHECK(DelayEstimate::Quality::kRefined == delay_samples->quality); if (!delay_samples_ || delay_samples->delay != delay_samples_->delay) { delay_change_counter_ = 0; } @@ -171,9 +180,14 @@ absl::optional RenderDelayControllerImpl2::GetDelay( if (delay_samples_) { // Compute the render delay buffer delay. + const bool use_hysteresis = + last_delay_estimate_quality_ == DelayEstimate::Quality::kRefined && + delay_samples_->quality == DelayEstimate::Quality::kRefined; delay_ = ComputeBufferDelay(delay_, delay_headroom_blocks_, - hysteresis_limit_1_blocks_, - hysteresis_limit_2_blocks_, *delay_samples_); + use_hysteresis ? hysteresis_limit_1_blocks_ : 0, + use_hysteresis ? hysteresis_limit_2_blocks_ : 0, + *delay_samples_); + last_delay_estimate_quality_ = delay_samples_->quality; } metrics_.Update(delay_samples_ ? absl::optional(delay_samples_->delay) diff --git a/test/fuzzers/audio_processing_configs_fuzzer.cc b/test/fuzzers/audio_processing_configs_fuzzer.cc index a88df8ff07..6bbd0c0cb4 100644 --- a/test/fuzzers/audio_processing_configs_fuzzer.cc +++ b/test/fuzzers/audio_processing_configs_fuzzer.cc @@ -59,6 +59,7 @@ const std::string kFieldTrialNames[] = { "WebRTC-Aec3UseStationarityPropertiesKillSwitch", "WebRTC-Aec3UtilizeShadowFilterOutputKillSwitch", "WebRTC-Aec3ZeroExternalDelayHeadroomKillSwitch", + "WebRTC-Aec3EarlyDelayDetectionKillSwitch", }; std::unique_ptr CreateApm(test::FuzzDataHelper* fuzz_data,