diff --git a/api/audio/echo_canceller3_config.h b/api/audio/echo_canceller3_config.h index e304f15ec5..c19226e9e9 100644 --- a/api/audio/echo_canceller3_config.h +++ b/api/audio/echo_canceller3_config.h @@ -46,6 +46,7 @@ struct RTC_EXPORT EchoCanceller3Config { int initial; int converged; } delay_selection_thresholds = {5, 20}; + bool use_external_delay_estimator = false; } delay; struct Filter { diff --git a/modules/audio_processing/aec3/block_processor.cc b/modules/audio_processing/aec3/block_processor.cc index 2674827dde..0997b1a8d7 100644 --- a/modules/audio_processing/aec3/block_processor.cc +++ b/modules/audio_processing/aec3/block_processor.cc @@ -114,7 +114,8 @@ void BlockProcessorImpl::ProcessCapture( if (!capture_properly_started_) { capture_properly_started_ = true; render_buffer_->Reset(); - delay_controller_->Reset(true); + if (delay_controller_) + delay_controller_->Reset(true); } } else { // If no render data has yet arrived, do not process the capture signal. @@ -129,7 +130,8 @@ void BlockProcessorImpl::ProcessCapture( render_properly_started_) { echo_path_variability.delay_change = EchoPathVariability::DelayAdjustment::kBufferFlush; - delay_controller_->Reset(true); + if (delay_controller_) + delay_controller_->Reset(true); RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun at block " << capture_call_counter_; } @@ -142,35 +144,46 @@ void BlockProcessorImpl::ProcessCapture( render_buffer_->PrepareCaptureProcessing(); // Reset the delay controller at render buffer underrun. if (buffer_event == RenderDelayBuffer::BufferingEvent::kRenderUnderrun) { - delay_controller_->Reset(false); + if (delay_controller_) + delay_controller_->Reset(false); } data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize, &(*capture_block)[0][0], LowestBandRate(sample_rate_hz_), 1); - // Compute and apply the render delay required to achieve proper signal - // alignment. - estimated_delay_ = - delay_controller_->GetDelay(render_buffer_->GetDownsampledRenderBuffer(), - render_buffer_->Delay(), (*capture_block)[0]); + bool has_delay_estimator = !config_.delay.use_external_delay_estimator; + if (has_delay_estimator) { + RTC_DCHECK(delay_controller_); + // Compute and apply the render delay required to achieve proper signal + // alignment. + estimated_delay_ = delay_controller_->GetDelay( + render_buffer_->GetDownsampledRenderBuffer(), render_buffer_->Delay(), + (*capture_block)[0]); - if (estimated_delay_) { - bool delay_change = render_buffer_->SetDelay(estimated_delay_->delay); - if (delay_change) { - RTC_LOG(LS_WARNING) << "Delay changed to " << estimated_delay_->delay - << " at block " << capture_call_counter_; - echo_path_variability.delay_change = - EchoPathVariability::DelayAdjustment::kNewDetectedDelay; + if (estimated_delay_) { + bool delay_change = + render_buffer_->AlignFromDelay(estimated_delay_->delay); + if (delay_change) { + RTC_LOG(LS_WARNING) << "Delay changed to " << estimated_delay_->delay + << " at block " << capture_call_counter_; + echo_path_variability.delay_change = + EchoPathVariability::DelayAdjustment::kNewDetectedDelay; + } } + + echo_path_variability.clock_drift = delay_controller_->HasClockdrift(); + + } else { + render_buffer_->AlignFromExternalDelay(); } - echo_path_variability.clock_drift = delay_controller_->HasClockdrift(); - // Remove the echo from the capture signal. - echo_remover_->ProcessCapture( - echo_path_variability, capture_signal_saturation, estimated_delay_, - render_buffer_->GetRenderBuffer(), capture_block); + if (has_delay_estimator || render_buffer_->HasReceivedBufferDelay()) { + echo_remover_->ProcessCapture( + echo_path_variability, capture_signal_saturation, estimated_delay_, + render_buffer_->GetRenderBuffer(), capture_block); + } // Update the metrics. metrics_.UpdateCapture(false); @@ -193,7 +206,8 @@ void BlockProcessorImpl::BufferRender( RenderDelayBuffer::BufferingEvent::kNone); render_properly_started_ = true; - delay_controller_->LogRenderCall(); + if (delay_controller_) + delay_controller_->LogRenderCall(); } void BlockProcessorImpl::UpdateEchoLeakageStatus(bool leakage_detected) { @@ -217,8 +231,11 @@ BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config, int sample_rate_hz) { std::unique_ptr render_buffer( RenderDelayBuffer::Create(config, NumBandsForRate(sample_rate_hz))); - std::unique_ptr delay_controller( - RenderDelayController::Create(config, sample_rate_hz)); + std::unique_ptr delay_controller; + if (!config.delay.use_external_delay_estimator) { + delay_controller.reset( + RenderDelayController::Create(config, sample_rate_hz)); + } std::unique_ptr echo_remover( EchoRemover::Create(config, sample_rate_hz)); return Create(config, sample_rate_hz, std::move(render_buffer), @@ -229,8 +246,11 @@ BlockProcessor* BlockProcessor::Create( const EchoCanceller3Config& config, int sample_rate_hz, std::unique_ptr render_buffer) { - std::unique_ptr delay_controller( - RenderDelayController::Create(config, sample_rate_hz)); + std::unique_ptr delay_controller; + if (!config.delay.use_external_delay_estimator) { + delay_controller.reset( + RenderDelayController::Create(config, sample_rate_hz)); + } std::unique_ptr echo_remover( EchoRemover::Create(config, sample_rate_hz)); return Create(config, sample_rate_hz, std::move(render_buffer), diff --git a/modules/audio_processing/aec3/block_processor_unittest.cc b/modules/audio_processing/aec3/block_processor_unittest.cc index 439419f5ed..29a25eb2a7 100644 --- a/modules/audio_processing/aec3/block_processor_unittest.cc +++ b/modules/audio_processing/aec3/block_processor_unittest.cc @@ -118,7 +118,7 @@ TEST(BlockProcessor, DISABLED_DelayControllerIntegration) { EXPECT_CALL(*render_delay_buffer_mock, Insert(_)) .Times(kNumBlocks) .WillRepeatedly(Return(RenderDelayBuffer::BufferingEvent::kNone)); - EXPECT_CALL(*render_delay_buffer_mock, SetDelay(kDelayInBlocks)) + EXPECT_CALL(*render_delay_buffer_mock, AlignFromDelay(kDelayInBlocks)) .Times(AtLeast(1)); EXPECT_CALL(*render_delay_buffer_mock, MaxDelay()).WillOnce(Return(30)); EXPECT_CALL(*render_delay_buffer_mock, Delay()) @@ -162,7 +162,7 @@ TEST(BlockProcessor, DISABLED_SubmoduleIntegration) { .WillRepeatedly(Return(RenderDelayBuffer::BufferingEvent::kNone)); EXPECT_CALL(*render_delay_buffer_mock, PrepareCaptureProcessing()) .Times(kNumBlocks); - EXPECT_CALL(*render_delay_buffer_mock, SetDelay(9)).Times(AtLeast(1)); + EXPECT_CALL(*render_delay_buffer_mock, AlignFromDelay(9)).Times(AtLeast(1)); EXPECT_CALL(*render_delay_buffer_mock, Delay()) .Times(kNumBlocks) .WillRepeatedly(Return(0)); diff --git a/modules/audio_processing/aec3/echo_remover_unittest.cc b/modules/audio_processing/aec3/echo_remover_unittest.cc index 82d149a881..92cc8902ef 100644 --- a/modules/audio_processing/aec3/echo_remover_unittest.cc +++ b/modules/audio_processing/aec3/echo_remover_unittest.cc @@ -161,7 +161,7 @@ TEST(EchoRemover, BasicEchoRemoval) { std::unique_ptr remover(EchoRemover::Create(config, rate)); std::unique_ptr render_buffer( RenderDelayBuffer::Create(config, NumBandsForRate(rate))); - render_buffer->SetDelay(delay_samples / kBlockSize); + render_buffer->AlignFromDelay(delay_samples / kBlockSize); std::vector>> delay_buffers(x.size()); for (size_t j = 0; j < x.size(); ++j) { diff --git a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h index ba5e98cb9a..0dd1b9126c 100644 --- a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h +++ b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h @@ -32,7 +32,8 @@ class MockRenderDelayBuffer : public RenderDelayBuffer { RenderDelayBuffer::BufferingEvent( const std::vector>& block)); MOCK_METHOD0(PrepareCaptureProcessing, RenderDelayBuffer::BufferingEvent()); - MOCK_METHOD1(SetDelay, bool(size_t delay)); + MOCK_METHOD1(AlignFromDelay, bool(size_t delay)); + MOCK_METHOD0(AlignFromExternalDelay, void()); MOCK_CONST_METHOD0(Delay, size_t()); MOCK_CONST_METHOD0(MaxDelay, size_t()); MOCK_METHOD0(GetRenderBuffer, RenderBuffer*()); @@ -40,6 +41,7 @@ class MockRenderDelayBuffer : public RenderDelayBuffer { const DownsampledRenderBuffer&()); MOCK_CONST_METHOD1(CausalDelay, bool(size_t delay)); MOCK_METHOD1(SetAudioBufferDelay, void(size_t delay_ms)); + MOCK_METHOD0(HasReceivedBufferDelay, bool()); private: RenderBuffer* FakeGetRenderBuffer() { return &render_buffer_; } diff --git a/modules/audio_processing/aec3/render_delay_buffer.cc b/modules/audio_processing/aec3/render_delay_buffer.cc index fea2d6ed04..0b2e979359 100644 --- a/modules/audio_processing/aec3/render_delay_buffer.cc +++ b/modules/audio_processing/aec3/render_delay_buffer.cc @@ -44,7 +44,8 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer { void Reset() override; BufferingEvent Insert(const std::vector>& block) override; BufferingEvent PrepareCaptureProcessing() override; - bool SetDelay(size_t delay) override; + bool AlignFromDelay(size_t delay) override; + void AlignFromExternalDelay() override; size_t Delay() const override { return ComputeDelay(); } size_t MaxDelay() const override { return blocks_.buffer.size() - 1 - buffer_headroom_; @@ -57,6 +58,7 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer { int BufferLatency() const; void SetAudioBufferDelay(size_t delay_ms) override; + bool HasReceivedBufferDelay() override; private: static int instance_count_; @@ -78,8 +80,8 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer { bool last_call_was_render_ = false; int num_api_calls_in_a_row_ = 0; int max_observed_jitter_ = 1; - size_t capture_call_counter_ = 0; - size_t render_call_counter_ = 0; + int64_t capture_call_counter_ = 0; + int64_t render_call_counter_ = 0; bool render_activity_ = false; size_t render_activity_counter_ = 0; absl::optional external_audio_buffer_delay_; @@ -172,7 +174,7 @@ void RenderDelayBufferImpl::Reset() { // initial delay. Set the render buffer delays to the default delay. ApplyTotalDelay(config_.delay.default_delay); - // Unset the delays which are set by SetDelay. + // Unset the delays which are set by AlignFromDelay. delay_ = absl::nullopt; } } @@ -277,7 +279,8 @@ RenderDelayBufferImpl::PrepareCaptureProcessing() { } // Sets the delay and returns a bool indicating whether the delay was changed. -bool RenderDelayBufferImpl::SetDelay(size_t delay) { +bool RenderDelayBufferImpl::AlignFromDelay(size_t delay) { + RTC_DCHECK(!config_.delay.use_external_delay_estimator); if (!external_audio_buffer_delay_verified_after_reset_ && external_audio_buffer_delay_ && delay_) { int difference = static_cast(delay) - static_cast(*delay_); @@ -312,6 +315,10 @@ void RenderDelayBufferImpl::SetAudioBufferDelay(size_t delay_ms) { external_audio_buffer_delay_ = delay_ms >> ((num_bands_ == 1) ? 1 : 2); } +bool RenderDelayBufferImpl::HasReceivedBufferDelay() { + return external_audio_buffer_delay_.has_value(); +} + // Maps the externally computed delay to the delay used internally. int RenderDelayBufferImpl::MapDelayToTotalDelay( size_t external_delay_blocks) const { @@ -337,6 +344,15 @@ void RenderDelayBufferImpl::ApplyTotalDelay(int delay) { ffts_.read = ffts_.OffsetIndex(ffts_.write, delay); } +void RenderDelayBufferImpl::AlignFromExternalDelay() { + RTC_DCHECK(config_.delay.use_external_delay_estimator); + if (external_audio_buffer_delay_) { + int64_t delay = render_call_counter_ - capture_call_counter_ + + *external_audio_buffer_delay_; + ApplyTotalDelay(delay); + } +} + // Inserts a block into the render buffers. void RenderDelayBufferImpl::InsertBlock( const std::vector>& block, diff --git a/modules/audio_processing/aec3/render_delay_buffer.h b/modules/audio_processing/aec3/render_delay_buffer.h index 6926c678dc..89b3a2a299 100644 --- a/modules/audio_processing/aec3/render_delay_buffer.h +++ b/modules/audio_processing/aec3/render_delay_buffer.h @@ -48,7 +48,10 @@ class RenderDelayBuffer { // Sets the buffer delay and returns a bool indicating whether the delay // changed. - virtual bool SetDelay(size_t delay) = 0; + virtual bool AlignFromDelay(size_t delay) = 0; + + // Sets the buffer delay from the most recently reported external delay. + virtual void AlignFromExternalDelay() = 0; // Gets the buffer delay. virtual size_t Delay() const = 0; @@ -67,6 +70,10 @@ class RenderDelayBuffer { // Provides an optional external estimate of the audio buffer delay. virtual void SetAudioBufferDelay(size_t delay_ms) = 0; + + // Returns whether an external delay estimate has been reported via + // SetAudioBufferDelay. + virtual bool HasReceivedBufferDelay() = 0; }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/render_delay_buffer_unittest.cc b/modules/audio_processing/aec3/render_delay_buffer_unittest.cc index 641a08180c..21b7ffcdc2 100644 --- a/modules/audio_processing/aec3/render_delay_buffer_unittest.cc +++ b/modules/audio_processing/aec3/render_delay_buffer_unittest.cc @@ -70,8 +70,8 @@ TEST(RenderDelayBuffer, AvailableBlock) { delay_buffer->PrepareCaptureProcessing(); } -// Verifies the SetDelay method. -TEST(RenderDelayBuffer, SetDelay) { +// Verifies the AlignFromDelay method. +TEST(RenderDelayBuffer, AlignFromDelay) { EchoCanceller3Config config; std::unique_ptr delay_buffer( RenderDelayBuffer::Create(config, 1)); @@ -80,7 +80,7 @@ TEST(RenderDelayBuffer, SetDelay) { size_t initial_internal_delay = 0; for (size_t delay = initial_internal_delay; delay < initial_internal_delay + 20; ++delay) { - ASSERT_TRUE(delay_buffer->SetDelay(delay)); + ASSERT_TRUE(delay_buffer->AlignFromDelay(delay)); EXPECT_EQ(delay, delay_buffer->Delay()); } } @@ -93,7 +93,7 @@ TEST(RenderDelayBuffer, SetDelay) { TEST(RenderDelayBuffer, DISABLED_WrongDelay) { std::unique_ptr delay_buffer( RenderDelayBuffer::Create(EchoCanceller3Config(), 3)); - EXPECT_DEATH(delay_buffer->SetDelay(21), ""); + EXPECT_DEATH(delay_buffer->AlignFromDelay(21), ""); } // Verifies the check for the number of bands in the inserted blocks. diff --git a/modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc b/modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc index 4e62c9484a..fa0be5cd24 100644 --- a/modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc +++ b/modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc @@ -65,7 +65,7 @@ TestInputs::TestInputs(const EchoCanceller3Config& cfg) : render_delay_buffer_(RenderDelayBuffer::Create(cfg, 1)), H2_(cfg.filter.main.length_blocks), x_(1, std::vector(kBlockSize, 0.f)) { - render_delay_buffer_->SetDelay(4); + render_delay_buffer_->AlignFromDelay(4); render_buffer_ = render_delay_buffer_->GetRenderBuffer(); for (auto& H : H2_) { H.fill(0.f);