diff --git a/modules/audio_processing/aec3/block_processor.cc b/modules/audio_processing/aec3/block_processor.cc index 24f2c9176d..dafe384bed 100644 --- a/modules/audio_processing/aec3/block_processor.cc +++ b/modules/audio_processing/aec3/block_processor.cc @@ -101,6 +101,7 @@ void BlockProcessorImpl::ProcessCapture( if (!capture_properly_started_) { capture_properly_started_ = true; render_buffer_->Reset(); + delay_controller_->Reset(); } } else { // If no render data has yet arrived, do not process the capture signal. @@ -213,6 +214,7 @@ void BlockProcessorImpl::BufferRender( RenderDelayBuffer::BufferingEvent::kNone); render_properly_started_ = true; + delay_controller_->LogRenderCall(); } void BlockProcessorImpl::UpdateEchoLeakageStatus(bool leakage_detected) { diff --git a/modules/audio_processing/aec3/echo_path_delay_estimator.cc b/modules/audio_processing/aec3/echo_path_delay_estimator.cc index 95796d52ea..d9d753cd43 100644 --- a/modules/audio_processing/aec3/echo_path_delay_estimator.cc +++ b/modules/audio_processing/aec3/echo_path_delay_estimator.cc @@ -43,8 +43,10 @@ EchoPathDelayEstimator::EchoPathDelayEstimator( EchoPathDelayEstimator::~EchoPathDelayEstimator() = default; -void EchoPathDelayEstimator::Reset() { - matched_filter_lag_aggregator_.Reset(); +void EchoPathDelayEstimator::Reset(bool soft_reset) { + if (!soft_reset) { + matched_filter_lag_aggregator_.Reset(); + } matched_filter_.Reset(); } diff --git a/modules/audio_processing/aec3/echo_path_delay_estimator.h b/modules/audio_processing/aec3/echo_path_delay_estimator.h index ef0d4483e1..10fa862fdb 100644 --- a/modules/audio_processing/aec3/echo_path_delay_estimator.h +++ b/modules/audio_processing/aec3/echo_path_delay_estimator.h @@ -33,8 +33,9 @@ class EchoPathDelayEstimator { const EchoCanceller3Config& config); ~EchoPathDelayEstimator(); - // Resets the estimation. - void Reset(); + // Resets the estimation. If the soft-reset is specified, only the matched + // filters are reset. + void Reset(bool soft_reset); // Produce a delay estimate if such is avaliable. rtc::Optional EstimateDelay( diff --git a/modules/audio_processing/aec3/mock/mock_render_delay_controller.h b/modules/audio_processing/aec3/mock/mock_render_delay_controller.h index 4a82ae3292..8fb7a8e900 100644 --- a/modules/audio_processing/aec3/mock/mock_render_delay_controller.h +++ b/modules/audio_processing/aec3/mock/mock_render_delay_controller.h @@ -25,6 +25,7 @@ class MockRenderDelayController : public RenderDelayController { virtual ~MockRenderDelayController() = default; MOCK_METHOD0(Reset, void()); + MOCK_METHOD0(LogRenderCall, void()); MOCK_METHOD2( GetDelay, rtc::Optional(const DownsampledRenderBuffer& render_buffer, diff --git a/modules/audio_processing/aec3/render_delay_controller.cc b/modules/audio_processing/aec3/render_delay_controller.cc index 3bc7d62237..614f0ad65c 100644 --- a/modules/audio_processing/aec3/render_delay_controller.cc +++ b/modules/audio_processing/aec3/render_delay_controller.cc @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -25,6 +26,47 @@ namespace webrtc { namespace { +constexpr int kSkewHistorySizeLog2 = 8; + +// Estimator of API call skew between render and capture. +class SkewEstimator { + public: + // Resets the estimation. + void Reset() { + skew_ = 0; + next_index_ = 0; + sufficient_skew_stored_ = false; + } + + // Updates the skew data for a render call. + void LogRenderCall() { ++skew_; } + + // Updates and computes the skew at a capture call. Returns an optional which + // is non-null if a reliable skew has been found. + rtc::Optional GetSkewFromCapture() { + --skew_; + + skew_history_[next_index_] = skew_; + if (++next_index_ == skew_history_.size()) { + next_index_ = 0; + sufficient_skew_stored_ = true; + } + + if (!sufficient_skew_stored_) { + return rtc::nullopt; + } + + return std::accumulate(skew_history_.begin(), skew_history_.end(), 0) >> + kSkewHistorySizeLog2; + } + + private: + int skew_ = 0; + std::array skew_history_; + size_t next_index_ = 0; + bool sufficient_skew_stored_ = false; +}; + class RenderDelayControllerImpl final : public RenderDelayController { public: RenderDelayControllerImpl(const EchoCanceller3Config& config, @@ -32,6 +74,7 @@ class RenderDelayControllerImpl final : public RenderDelayController { int sample_rate_hz); ~RenderDelayControllerImpl() override; void Reset() override; + void LogRenderCall() override; rtc::Optional GetDelay( const DownsampledRenderBuffer& render_buffer, rtc::ArrayView capture) override; @@ -47,23 +90,27 @@ class RenderDelayControllerImpl final : public RenderDelayController { std::vector delay_buf_; int delay_buf_index_ = 0; RenderDelayControllerMetrics metrics_; + SkewEstimator skew_estimator_; + rtc::Optional delay_samples_; + rtc::Optional skew_; + int delay_change_counter_ = 0; + size_t soft_reset_counter_ = 0; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayControllerImpl); }; -DelayEstimate ComputeNewBufferDelay( +DelayEstimate ComputeBufferDelay( const rtc::Optional& current_delay, int delay_headroom_blocks, int hysteresis_limit_1_blocks, int hysteresis_limit_2_blocks, + int offset_blocks, DelayEstimate estimated_delay) { // The below division is not exact and the truncation is intended. const int echo_path_delay_blocks = estimated_delay.delay >> kBlockSizeLog2; // Compute the buffer delay increase required to achieve the desired latency. - size_t new_delay_blocks = - std::max(echo_path_delay_blocks - delay_headroom_blocks, 0); - - DelayEstimate new_delay(estimated_delay.quality, new_delay_blocks); + size_t new_delay_blocks = std::max( + echo_path_delay_blocks + offset_blocks - delay_headroom_blocks, 0); // Add hysteresis. if (current_delay) { @@ -111,8 +158,17 @@ RenderDelayControllerImpl::~RenderDelayControllerImpl() = default; void RenderDelayControllerImpl::Reset() { delay_ = rtc::nullopt; + delay_samples_ = rtc::nullopt; + skew_ = rtc::nullopt; std::fill(delay_buf_.begin(), delay_buf_.end(), 0.f); - delay_estimator_.Reset(); + delay_estimator_.Reset(false); + skew_estimator_.Reset(); + delay_change_counter_ = 0; + soft_reset_counter_ = 0; +} + +void RenderDelayControllerImpl::LogRenderCall() { + skew_estimator_.LogRenderCall(); } rtc::Optional RenderDelayControllerImpl::GetDelay( @@ -131,24 +187,64 @@ rtc::Optional RenderDelayControllerImpl::GetDelay( delay_buf_.begin() + delay_buf_index_); delay_buf_index_ = (delay_buf_index_ + kBlockSize) % delay_buf_.size(); + // Compute the latest skew update. + rtc::Optional skew = skew_estimator_.GetSkewFromCapture(); + if (delay_samples) { - // Compute and set new render delay buffer delay. - - delay_ = ComputeNewBufferDelay(delay_, delay_headroom_blocks_, - hysteresis_limit_1_blocks_, - hysteresis_limit_2_blocks_, *delay_samples); - - metrics_.Update(static_cast(delay_samples->delay), - delay_ ? delay_->delay : 0); - } else { - metrics_.Update(rtc::nullopt, delay_ ? delay_->delay : 0); + if (!delay_samples_ || delay_samples->delay != delay_samples_->delay) { + delay_change_counter_ = 0; + } + delay_samples_ = delay_samples; } + if (delay_change_counter_ < 2 * kNumBlocksPerSecond) { + ++delay_change_counter_; + // If a new delay estimate is recently obtained, store the skew for that. + skew_ = skew; + } else { + // A reliable skew should have been obtained after 2 seconds. + RTC_DCHECK(skew_); + RTC_DCHECK(skew); + } + + ++soft_reset_counter_; + int offset_blocks = 0; + if (skew_ && skew && delay_samples_ && + delay_samples_->quality == DelayEstimate::Quality::kRefined) { + // Compute the skew offset and add a margin. + offset_blocks = *skew_ - *skew; + if (abs(offset_blocks) <= 1) { + offset_blocks = 0; + } else if (soft_reset_counter_ > 10 * kNumBlocksPerSecond) { + // Soft reset the delay estimator if there is a significant offset + // detected. + delay_estimator_.Reset(true); + soft_reset_counter_ = 0; + } + } + + if (delay_samples_) { + // Compute the render delay buffer delay. + delay_ = ComputeBufferDelay( + delay_, delay_headroom_blocks_, hysteresis_limit_1_blocks_, + hysteresis_limit_2_blocks_, offset_blocks, *delay_samples_); + } + + metrics_.Update(delay_samples_ ? rtc::Optional(delay_samples_->delay) + : rtc::nullopt, + delay_ ? delay_->delay : 0); + data_dumper_->DumpRaw("aec3_render_delay_controller_delay", delay_samples ? delay_samples->delay : 0); data_dumper_->DumpRaw("aec3_render_delay_controller_buffer_delay", delay_ ? delay_->delay : 0); + data_dumper_->DumpRaw("aec3_render_delay_controller_new_skew", + skew ? *skew : 0); + data_dumper_->DumpRaw("aec3_render_delay_controller_old_skew", + skew_ ? *skew_ : 0); + data_dumper_->DumpRaw("aec3_render_delay_controller_offset", offset_blocks); + return delay_; } diff --git a/modules/audio_processing/aec3/render_delay_controller.h b/modules/audio_processing/aec3/render_delay_controller.h index 5b1fc35ee9..842b8fffe5 100644 --- a/modules/audio_processing/aec3/render_delay_controller.h +++ b/modules/audio_processing/aec3/render_delay_controller.h @@ -32,6 +32,9 @@ class RenderDelayController { // Resets the delay controller. virtual void Reset() = 0; + // Logs a render call. + virtual void LogRenderCall() = 0; + // Aligns the render buffer content with the capture signal. virtual rtc::Optional GetDelay( const DownsampledRenderBuffer& render_buffer,