diff --git a/api/audio/echo_canceller3_config.h b/api/audio/echo_canceller3_config.h index 8ff5aca90a..251f282539 100644 --- a/api/audio/echo_canceller3_config.h +++ b/api/audio/echo_canceller3_config.h @@ -50,7 +50,7 @@ struct RTC_EXPORT EchoCanceller3Config { struct DelaySelectionThresholds { int initial; int converged; - } delay_selection_thresholds = {25, 25}; + } delay_selection_thresholds = {5, 20}; } delay; struct Filter { diff --git a/modules/audio_processing/aec3/block_processor.cc b/modules/audio_processing/aec3/block_processor.cc index 4f57902e7d..590380f897 100644 --- a/modules/audio_processing/aec3/block_processor.cc +++ b/modules/audio_processing/aec3/block_processor.cc @@ -108,7 +108,7 @@ void BlockProcessorImpl::ProcessCapture( if (!capture_properly_started_) { capture_properly_started_ = true; render_buffer_->Reset(); - delay_controller_->Reset(); + delay_controller_->Reset(true); } } else { // If no render data has yet arrived, do not process the capture signal. @@ -123,7 +123,7 @@ void BlockProcessorImpl::ProcessCapture( render_properly_started_) { echo_path_variability.delay_change = EchoPathVariability::DelayAdjustment::kBufferFlush; - delay_controller_->Reset(); + delay_controller_->Reset(true); RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun at block " << capture_call_counter_; } @@ -139,7 +139,7 @@ void BlockProcessorImpl::ProcessCapture( estimated_delay_->quality == DelayEstimate::Quality::kRefined) { echo_path_variability.delay_change = EchoPathVariability::DelayAdjustment::kDelayReset; - delay_controller_->Reset(); + delay_controller_->Reset(true); capture_properly_started_ = false; render_properly_started_ = false; @@ -151,7 +151,7 @@ void BlockProcessorImpl::ProcessCapture( // echo. echo_path_variability.delay_change = EchoPathVariability::DelayAdjustment::kDelayReset; - delay_controller_->Reset(); + delay_controller_->Reset(true); capture_properly_started_ = false; render_properly_started_ = false; RTC_LOG(LS_WARNING) << "Reset due to render buffer api skew at block " @@ -184,7 +184,7 @@ void BlockProcessorImpl::ProcessCapture( if (estimated_delay_->quality == DelayEstimate::Quality::kRefined) { echo_path_variability.delay_change = EchoPathVariability::DelayAdjustment::kDelayReset; - delay_controller_->Reset(); + delay_controller_->Reset(true); render_buffer_->Reset(); capture_properly_started_ = false; render_properly_started_ = false; diff --git a/modules/audio_processing/aec3/block_processor2.cc b/modules/audio_processing/aec3/block_processor2.cc index 5fa5abe7e9..3616427ce2 100644 --- a/modules/audio_processing/aec3/block_processor2.cc +++ b/modules/audio_processing/aec3/block_processor2.cc @@ -115,7 +115,7 @@ void BlockProcessorImpl2::ProcessCapture( if (!capture_properly_started_) { capture_properly_started_ = true; render_buffer_->Reset(); - delay_controller_->Reset(); + delay_controller_->Reset(true); } } else { // If no render data has yet arrived, do not process the capture signal. @@ -130,7 +130,7 @@ void BlockProcessorImpl2::ProcessCapture( render_properly_started_) { echo_path_variability.delay_change = EchoPathVariability::DelayAdjustment::kBufferFlush; - delay_controller_->Reset(); + delay_controller_->Reset(true); RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun at block " << capture_call_counter_; } @@ -143,7 +143,7 @@ void BlockProcessorImpl2::ProcessCapture( render_buffer_->PrepareCaptureProcessing(); // Reset the delay controller at render buffer underrun. if (buffer_event == RenderDelayBuffer::BufferingEvent::kRenderUnderrun) { - delay_controller_->Reset(); + delay_controller_->Reset(false); } data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize, diff --git a/modules/audio_processing/aec3/echo_canceller3.cc b/modules/audio_processing/aec3/echo_canceller3.cc index 1b089f841d..5debcda349 100644 --- a/modules/audio_processing/aec3/echo_canceller3.cc +++ b/modules/audio_processing/aec3/echo_canceller3.cc @@ -82,6 +82,10 @@ bool EnableNewRenderBuffering() { return !field_trial::IsEnabled("WebRTC-Aec3NewRenderBufferingKillSwitch"); } +bool UseEarlyDelayDetection() { + return !field_trial::IsEnabled("WebRTC-Aec3EarlyDelayDetectionKillSwitch"); +} + // Method for adjusting config parameter dependencies.. EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) { EchoCanceller3Config adjusted_cfg = config; @@ -161,6 +165,10 @@ EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) { adjusted_cfg.echo_audibility.use_stationarity_properties_at_init = false; } + if (!UseEarlyDelayDetection()) { + adjusted_cfg.delay.delay_selection_thresholds = {25, 25}; + } + return adjusted_cfg; } diff --git a/modules/audio_processing/aec3/echo_path_delay_estimator.cc b/modules/audio_processing/aec3/echo_path_delay_estimator.cc index 3a25deb101..5c838aed8a 100644 --- a/modules/audio_processing/aec3/echo_path_delay_estimator.cc +++ b/modules/audio_processing/aec3/echo_path_delay_estimator.cc @@ -49,13 +49,8 @@ EchoPathDelayEstimator::EchoPathDelayEstimator( EchoPathDelayEstimator::~EchoPathDelayEstimator() = default; -void EchoPathDelayEstimator::Reset(bool soft_reset) { - if (!soft_reset) { - matched_filter_lag_aggregator_.Reset(); - } - matched_filter_.Reset(); - old_aggregated_lag_ = absl::nullopt; - consistent_estimate_counter_ = 0; +void EchoPathDelayEstimator::Reset(bool reset_delay_confidence) { + Reset(true, reset_delay_confidence); } absl::optional EchoPathDelayEstimator::EstimateDelay( @@ -102,10 +97,20 @@ absl::optional EchoPathDelayEstimator::EstimateDelay( old_aggregated_lag_ = aggregated_matched_filter_lag; constexpr size_t kNumBlocksPerSecondBy2 = kNumBlocksPerSecond / 2; if (consistent_estimate_counter_ > kNumBlocksPerSecondBy2) { - Reset(true); + Reset(false, false); } return aggregated_matched_filter_lag; } +void EchoPathDelayEstimator::Reset(bool reset_lag_aggregator, + bool reset_delay_confidence) { + if (reset_lag_aggregator) { + matched_filter_lag_aggregator_.Reset(reset_delay_confidence); + } + matched_filter_.Reset(); + old_aggregated_lag_ = absl::nullopt; + consistent_estimate_counter_ = 0; +} + } // namespace webrtc diff --git a/modules/audio_processing/aec3/echo_path_delay_estimator.h b/modules/audio_processing/aec3/echo_path_delay_estimator.h index b1c42473f5..060c8753bd 100644 --- a/modules/audio_processing/aec3/echo_path_delay_estimator.h +++ b/modules/audio_processing/aec3/echo_path_delay_estimator.h @@ -34,9 +34,9 @@ class EchoPathDelayEstimator { const EchoCanceller3Config& config); ~EchoPathDelayEstimator(); - // Resets the estimation. If the soft-reset is specified, only the matched - // filters are reset. - void Reset(bool soft_reset); + // Resets the estimation. If the delay confidence is reset, the reset behavior + // is as if the call is restarted. + void Reset(bool reset_delay_confidence); // Produce a delay estimate if such is avaliable. absl::optional EstimateDelay( @@ -59,6 +59,9 @@ class EchoPathDelayEstimator { absl::optional old_aggregated_lag_; size_t consistent_estimate_counter_ = 0; + // Internal reset method with more granularity. + void Reset(bool reset_lag_aggregator, bool reset_delay_confidence); + RTC_DISALLOW_COPY_AND_ASSIGN(EchoPathDelayEstimator); }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc b/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc index 5ca62683d1..f59525eee9 100644 --- a/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc +++ b/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc @@ -31,11 +31,13 @@ MatchedFilterLagAggregator::MatchedFilterLagAggregator( MatchedFilterLagAggregator::~MatchedFilterLagAggregator() = default; -void MatchedFilterLagAggregator::Reset() { +void MatchedFilterLagAggregator::Reset(bool hard_reset) { std::fill(histogram_.begin(), histogram_.end(), 0); histogram_data_.fill(0); histogram_data_index_ = 0; - significant_candidate_found_ = false; + if (hard_reset) { + significant_candidate_found_ = false; + } } absl::optional MatchedFilterLagAggregator::Aggregate( @@ -81,9 +83,13 @@ absl::optional MatchedFilterLagAggregator::Aggregate( if (histogram_[candidate] > thresholds_.converged || (histogram_[candidate] > thresholds_.initial && !significant_candidate_found_)) { - return DelayEstimate(DelayEstimate::Quality::kRefined, candidate); + DelayEstimate::Quality quality = significant_candidate_found_ + ? DelayEstimate::Quality::kRefined + : DelayEstimate::Quality::kCoarse; + return DelayEstimate(quality, candidate); } } + return absl::nullopt; } diff --git a/modules/audio_processing/aec3/matched_filter_lag_aggregator.h b/modules/audio_processing/aec3/matched_filter_lag_aggregator.h index c57051aa68..d7f34aed60 100644 --- a/modules/audio_processing/aec3/matched_filter_lag_aggregator.h +++ b/modules/audio_processing/aec3/matched_filter_lag_aggregator.h @@ -34,7 +34,7 @@ class MatchedFilterLagAggregator { ~MatchedFilterLagAggregator(); // Resets the aggregator. - void Reset(); + void Reset(bool hard_reset); // Aggregates the provided lag estimates. absl::optional Aggregate( diff --git a/modules/audio_processing/aec3/mock/mock_render_delay_controller.h b/modules/audio_processing/aec3/mock/mock_render_delay_controller.h index ed5971cda0..5520f764fb 100644 --- a/modules/audio_processing/aec3/mock/mock_render_delay_controller.h +++ b/modules/audio_processing/aec3/mock/mock_render_delay_controller.h @@ -25,7 +25,7 @@ class MockRenderDelayController : public RenderDelayController { MockRenderDelayController(); virtual ~MockRenderDelayController(); - MOCK_METHOD0(Reset, void()); + MOCK_METHOD1(Reset, void(bool reset_delay_statistics)); MOCK_METHOD0(LogRenderCall, void()); MOCK_METHOD4(GetDelay, absl::optional( diff --git a/modules/audio_processing/aec3/render_delay_controller.cc b/modules/audio_processing/aec3/render_delay_controller.cc index e81d9d3ecd..36e75d9fa9 100644 --- a/modules/audio_processing/aec3/render_delay_controller.cc +++ b/modules/audio_processing/aec3/render_delay_controller.cc @@ -45,6 +45,10 @@ bool UseOffsetBlocks() { return field_trial::IsEnabled("WebRTC-Aec3UseOffsetBlocks"); } +bool UseEarlyDelayDetection() { + return !field_trial::IsEnabled("WebRTC-Aec3EarlyDelayDetectionKillSwitch"); +} + constexpr int kSkewHistorySizeLog2 = 8; class RenderDelayControllerImpl final : public RenderDelayController { @@ -53,7 +57,7 @@ class RenderDelayControllerImpl final : public RenderDelayController { int non_causal_offset, int sample_rate_hz); ~RenderDelayControllerImpl() override; - void Reset() override; + void Reset(bool reset_delay_confidence) override; void LogRenderCall() override; absl::optional GetDelay( const DownsampledRenderBuffer& render_buffer, @@ -64,6 +68,7 @@ class RenderDelayControllerImpl final : public RenderDelayController { private: static int instance_count_; std::unique_ptr data_dumper_; + const bool use_early_delay_detection_; const int delay_headroom_blocks_; const int hysteresis_limit_1_blocks_; const int hysteresis_limit_2_blocks_; @@ -82,6 +87,7 @@ class RenderDelayControllerImpl final : public RenderDelayController { size_t capture_call_counter_ = 0; int delay_change_counter_ = 0; size_t soft_reset_counter_ = 0; + DelayEstimate::Quality last_delay_estimate_quality_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayControllerImpl); }; @@ -130,6 +136,7 @@ RenderDelayControllerImpl::RenderDelayControllerImpl( int sample_rate_hz) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + use_early_delay_detection_(UseEarlyDelayDetection()), delay_headroom_blocks_( static_cast(config.delay.delay_headroom_blocks)), hysteresis_limit_1_blocks_( @@ -140,7 +147,8 @@ RenderDelayControllerImpl::RenderDelayControllerImpl( use_offset_blocks_(UseOffsetBlocks()), delay_estimator_(data_dumper_.get(), config), delay_buf_(kBlockSize * non_causal_offset, 0.f), - skew_estimator_(kSkewHistorySizeLog2) { + skew_estimator_(kSkewHistorySizeLog2), + last_delay_estimate_quality_(DelayEstimate::Quality::kCoarse) { RTC_DCHECK(ValidFullBandRate(sample_rate_hz)); delay_estimator_.LogDelayEstimationProperties(sample_rate_hz, delay_buf_.size()); @@ -148,16 +156,19 @@ RenderDelayControllerImpl::RenderDelayControllerImpl( RenderDelayControllerImpl::~RenderDelayControllerImpl() = default; -void RenderDelayControllerImpl::Reset() { +void RenderDelayControllerImpl::Reset(bool reset_delay_confidence) { delay_ = absl::nullopt; delay_samples_ = absl::nullopt; skew_ = absl::nullopt; previous_offset_blocks_ = 0; std::fill(delay_buf_.begin(), delay_buf_.end(), 0.f); - delay_estimator_.Reset(false); + delay_estimator_.Reset(reset_delay_confidence); skew_estimator_.Reset(); delay_change_counter_ = 0; soft_reset_counter_ = 0; + if (reset_delay_confidence) { + last_delay_estimate_quality_ = DelayEstimate::Quality::kCoarse; + } } void RenderDelayControllerImpl::LogRenderCall() { @@ -195,9 +206,6 @@ absl::optional RenderDelayControllerImpl::GetDelay( absl::optional skew = skew_estimator_.GetSkewFromCapture(); if (delay_samples) { - // TODO(peah): Refactor the rest of the code to assume a kRefined estimate - // quality. - RTC_DCHECK(DelayEstimate::Quality::kRefined == delay_samples->quality); if (!delay_samples_ || delay_samples->delay != delay_samples_->delay) { delay_change_counter_ = 0; } @@ -240,7 +248,7 @@ absl::optional RenderDelayControllerImpl::GetDelay( } else if (soft_reset_counter_ > 10 * kNumBlocksPerSecond) { // Soft reset the delay estimator if there is a significant offset // detected. - delay_estimator_.Reset(true); + delay_estimator_.Reset(false); soft_reset_counter_ = 0; } } @@ -265,9 +273,14 @@ absl::optional RenderDelayControllerImpl::GetDelay( if (delay_samples_) { // Compute the render delay buffer delay. - delay_ = ComputeBufferDelay( - delay_, delay_headroom_blocks_, hysteresis_limit_1_blocks_, - hysteresis_limit_2_blocks_, offset_blocks, *delay_samples_); + const bool use_hysteresis = + last_delay_estimate_quality_ == DelayEstimate::Quality::kRefined && + delay_samples_->quality == DelayEstimate::Quality::kRefined; + delay_ = ComputeBufferDelay(delay_, delay_headroom_blocks_, + use_hysteresis ? hysteresis_limit_1_blocks_ : 0, + use_hysteresis ? hysteresis_limit_2_blocks_ : 0, + offset_blocks, *delay_samples_); + last_delay_estimate_quality_ = delay_samples_->quality; } metrics_.Update(delay_samples_ ? absl::optional(delay_samples_->delay) diff --git a/modules/audio_processing/aec3/render_delay_controller.h b/modules/audio_processing/aec3/render_delay_controller.h index a0cbe89f20..41ba4224c9 100644 --- a/modules/audio_processing/aec3/render_delay_controller.h +++ b/modules/audio_processing/aec3/render_delay_controller.h @@ -31,8 +31,9 @@ class RenderDelayController { int sample_rate_hz); virtual ~RenderDelayController() = default; - // Resets the delay controller. - virtual void Reset() = 0; + // Resets the delay controller. If the delay confidence is reset, the reset + // behavior is as if the call is restarted. + virtual void Reset(bool reset_delay_confidence) = 0; // Logs a render call. virtual void LogRenderCall() = 0; diff --git a/modules/audio_processing/aec3/render_delay_controller2.cc b/modules/audio_processing/aec3/render_delay_controller2.cc index 64f14cb64c..1b7c18d3e9 100644 --- a/modules/audio_processing/aec3/render_delay_controller2.cc +++ b/modules/audio_processing/aec3/render_delay_controller2.cc @@ -24,17 +24,22 @@ #include "rtc_base/atomicops.h" #include "rtc_base/checks.h" #include "rtc_base/constructormagic.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { namespace { +bool UseEarlyDelayDetection() { + return !field_trial::IsEnabled("WebRTC-Aec3EarlyDelayDetectionKillSwitch"); +} + class RenderDelayControllerImpl2 final : public RenderDelayController { public: RenderDelayControllerImpl2(const EchoCanceller3Config& config, int sample_rate_hz); ~RenderDelayControllerImpl2() override; - void Reset() override; + void Reset(bool reset_delay_confidence) override; void LogRenderCall() override; absl::optional GetDelay( const DownsampledRenderBuffer& render_buffer, @@ -45,6 +50,7 @@ class RenderDelayControllerImpl2 final : public RenderDelayController { private: static int instance_count_; std::unique_ptr data_dumper_; + const bool use_early_delay_detection_; const int delay_headroom_blocks_; const int hysteresis_limit_1_blocks_; const int hysteresis_limit_2_blocks_; @@ -54,6 +60,7 @@ class RenderDelayControllerImpl2 final : public RenderDelayController { absl::optional delay_samples_; size_t capture_call_counter_ = 0; int delay_change_counter_ = 0; + DelayEstimate::Quality last_delay_estimate_quality_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayControllerImpl2); }; @@ -100,24 +107,29 @@ RenderDelayControllerImpl2::RenderDelayControllerImpl2( int sample_rate_hz) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + use_early_delay_detection_(UseEarlyDelayDetection()), delay_headroom_blocks_( static_cast(config.delay.delay_headroom_blocks)), hysteresis_limit_1_blocks_( static_cast(config.delay.hysteresis_limit_1_blocks)), hysteresis_limit_2_blocks_( static_cast(config.delay.hysteresis_limit_2_blocks)), - delay_estimator_(data_dumper_.get(), config) { + delay_estimator_(data_dumper_.get(), config), + last_delay_estimate_quality_(DelayEstimate::Quality::kCoarse) { RTC_DCHECK(ValidFullBandRate(sample_rate_hz)); delay_estimator_.LogDelayEstimationProperties(sample_rate_hz, 0); } RenderDelayControllerImpl2::~RenderDelayControllerImpl2() = default; -void RenderDelayControllerImpl2::Reset() { +void RenderDelayControllerImpl2::Reset(bool reset_delay_confidence) { delay_ = absl::nullopt; delay_samples_ = absl::nullopt; - delay_estimator_.Reset(false); + delay_estimator_.Reset(reset_delay_confidence); delay_change_counter_ = 0; + if (reset_delay_confidence || true) { + last_delay_estimate_quality_ = DelayEstimate::Quality::kCoarse; + } } void RenderDelayControllerImpl2::LogRenderCall() {} @@ -141,9 +153,6 @@ absl::optional RenderDelayControllerImpl2::GetDelay( } if (delay_samples) { - // TODO(peah): Refactor the rest of the code to assume a kRefined estimate - // quality. - RTC_DCHECK(DelayEstimate::Quality::kRefined == delay_samples->quality); if (!delay_samples_ || delay_samples->delay != delay_samples_->delay) { delay_change_counter_ = 0; } @@ -171,9 +180,14 @@ absl::optional RenderDelayControllerImpl2::GetDelay( if (delay_samples_) { // Compute the render delay buffer delay. + const bool use_hysteresis = + last_delay_estimate_quality_ == DelayEstimate::Quality::kRefined && + delay_samples_->quality == DelayEstimate::Quality::kRefined; delay_ = ComputeBufferDelay(delay_, delay_headroom_blocks_, - hysteresis_limit_1_blocks_, - hysteresis_limit_2_blocks_, *delay_samples_); + use_hysteresis ? hysteresis_limit_1_blocks_ : 0, + use_hysteresis ? hysteresis_limit_2_blocks_ : 0, + *delay_samples_); + last_delay_estimate_quality_ = delay_samples_->quality; } metrics_.Update(delay_samples_ ? absl::optional(delay_samples_->delay) diff --git a/test/fuzzers/audio_processing_configs_fuzzer.cc b/test/fuzzers/audio_processing_configs_fuzzer.cc index a88df8ff07..6bbd0c0cb4 100644 --- a/test/fuzzers/audio_processing_configs_fuzzer.cc +++ b/test/fuzzers/audio_processing_configs_fuzzer.cc @@ -59,6 +59,7 @@ const std::string kFieldTrialNames[] = { "WebRTC-Aec3UseStationarityPropertiesKillSwitch", "WebRTC-Aec3UtilizeShadowFilterOutputKillSwitch", "WebRTC-Aec3ZeroExternalDelayHeadroomKillSwitch", + "WebRTC-Aec3EarlyDelayDetectionKillSwitch", }; std::unique_ptr CreateApm(test::FuzzDataHelper* fuzz_data,