From d0fa82055947b76554e9b3dd11fea7c1c61650a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20=C3=85hgren?= Date: Wed, 18 Apr 2018 09:35:13 +0200 Subject: [PATCH] Allow AEC3 to use any externally reported audio buffer delay in AEC3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This CL adds support for using any externally reported audio buffer delay to set the initial alignment in AEC3 which is used before the AEC has been able to detect the delay. Bug: chromium:834182,webrtc:9163 Change-Id: Ic71355f69b7c4d5815b78e49987043441e7908fb Reviewed-on: https://webrtc-review.googlesource.com/70580 Reviewed-by: Gustaf Ullberg Commit-Queue: Gustaf Ullberg Commit-Queue: Per Åhgren Cr-Commit-Position: refs/heads/master@{#22917} --- api/audio/echo_control.h | 3 ++ .../audio_processing/aec3/block_processor.cc | 6 +++ .../audio_processing/aec3/block_processor.h | 3 ++ .../audio_processing/aec3/echo_canceller3.cc | 5 ++ .../audio_processing/aec3/echo_canceller3.h | 2 + .../aec3/echo_canceller3_unittest.cc | 4 ++ .../aec3/mock/mock_block_processor.h | 1 + .../aec3/mock/mock_render_delay_buffer.h | 1 + .../aec3/render_delay_buffer.cc | 53 +++++++++++++++++-- .../aec3/render_delay_buffer.h | 3 ++ .../audio_processing/audio_processing_impl.cc | 5 ++ .../include/mock_audio_processing.h | 1 + 12 files changed, 82 insertions(+), 5 deletions(-) diff --git a/api/audio/echo_control.h b/api/audio/echo_control.h index 021bbf8764..f549f40fbb 100644 --- a/api/audio/echo_control.h +++ b/api/audio/echo_control.h @@ -38,6 +38,9 @@ class EchoControl { // Collect current metrics from the echo controller. virtual Metrics GetMetrics() const = 0; + // Provides an optional external estimate of the audio buffer delay. + virtual void SetAudioBufferDelay(size_t delay_ms) = 0; + virtual ~EchoControl() {} }; diff --git a/modules/audio_processing/aec3/block_processor.cc b/modules/audio_processing/aec3/block_processor.cc index ff11bae8e1..f954be3b63 100644 --- a/modules/audio_processing/aec3/block_processor.cc +++ b/modules/audio_processing/aec3/block_processor.cc @@ -43,6 +43,8 @@ class BlockProcessorImpl final : public BlockProcessor { void GetMetrics(EchoControl::Metrics* metrics) const override; + void SetAudioBufferDelay(size_t delay_ms) override; + private: static int instance_count_; std::unique_ptr data_dumper_; @@ -234,6 +236,10 @@ void BlockProcessorImpl::GetMetrics(EchoControl::Metrics* metrics) const { metrics->delay_ms = delay ? static_cast(*delay) * block_size_ms : 0; } +void BlockProcessorImpl::SetAudioBufferDelay(size_t delay_ms) { + render_buffer_->SetAudioBufferDelay(delay_ms); +} + } // namespace BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config, diff --git a/modules/audio_processing/aec3/block_processor.h b/modules/audio_processing/aec3/block_processor.h index 8687bc2282..a3967ea7e4 100644 --- a/modules/audio_processing/aec3/block_processor.h +++ b/modules/audio_processing/aec3/block_processor.h @@ -42,6 +42,9 @@ class BlockProcessor { // Get current metrics. virtual void GetMetrics(EchoControl::Metrics* metrics) const = 0; + // Provides an optional external estimate of the audio buffer delay. + virtual void SetAudioBufferDelay(size_t delay_ms) = 0; + // Processes a block of capture data. virtual void ProcessCapture( bool echo_path_gain_change, diff --git a/modules/audio_processing/aec3/echo_canceller3.cc b/modules/audio_processing/aec3/echo_canceller3.cc index d478896e71..765e4de073 100644 --- a/modules/audio_processing/aec3/echo_canceller3.cc +++ b/modules/audio_processing/aec3/echo_canceller3.cc @@ -375,6 +375,11 @@ EchoControl::Metrics EchoCanceller3::GetMetrics() const { return metrics; } +void EchoCanceller3::SetAudioBufferDelay(size_t delay_ms) { + RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); + block_processor_->SetAudioBufferDelay(delay_ms); +} + void EchoCanceller3::EmptyRenderQueue() { RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); bool frame_to_buffer = diff --git a/modules/audio_processing/aec3/echo_canceller3.h b/modules/audio_processing/aec3/echo_canceller3.h index 8658814e93..b66b8b1dd1 100644 --- a/modules/audio_processing/aec3/echo_canceller3.h +++ b/modules/audio_processing/aec3/echo_canceller3.h @@ -82,6 +82,8 @@ class EchoCanceller3 : public EchoControl { void ProcessCapture(AudioBuffer* capture, bool level_change) override; // Collect current metrics from the echo canceller. Metrics GetMetrics() const override; + // Provides an optional external estimate of the audio buffer delay. + void SetAudioBufferDelay(size_t delay_ms) override; // Signals whether an external detector has detected echo leakage from the // echo canceller. diff --git a/modules/audio_processing/aec3/echo_canceller3_unittest.cc b/modules/audio_processing/aec3/echo_canceller3_unittest.cc index d4ad4f66f6..f652642130 100644 --- a/modules/audio_processing/aec3/echo_canceller3_unittest.cc +++ b/modules/audio_processing/aec3/echo_canceller3_unittest.cc @@ -104,6 +104,8 @@ class CaptureTransportVerificationProcessor : public BlockProcessor { void GetMetrics(EchoControl::Metrics* metrics) const override {} + void SetAudioBufferDelay(size_t delay_ms) override{}; + private: RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(CaptureTransportVerificationProcessor); }; @@ -132,6 +134,8 @@ class RenderTransportVerificationProcessor : public BlockProcessor { void GetMetrics(EchoControl::Metrics* metrics) const override {} + void SetAudioBufferDelay(size_t delay_ms) override{}; + private: std::deque>> received_render_blocks_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderTransportVerificationProcessor); diff --git a/modules/audio_processing/aec3/mock/mock_block_processor.h b/modules/audio_processing/aec3/mock/mock_block_processor.h index 5fff456185..803bf78e9c 100644 --- a/modules/audio_processing/aec3/mock/mock_block_processor.h +++ b/modules/audio_processing/aec3/mock/mock_block_processor.h @@ -31,6 +31,7 @@ class MockBlockProcessor : public BlockProcessor { void(const std::vector>& block)); MOCK_METHOD1(UpdateEchoLeakageStatus, void(bool leakage_detected)); MOCK_CONST_METHOD1(GetMetrics, void(EchoControl::Metrics* metrics)); + MOCK_METHOD1(SetAudioBufferDelay, void(size_t delay_ms)); }; } // namespace test diff --git a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h index c79cd107f0..00c13f422b 100644 --- a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h +++ b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h @@ -53,6 +53,7 @@ class MockRenderDelayBuffer : public RenderDelayBuffer { MOCK_CONST_METHOD0(GetDownsampledRenderBuffer, const DownsampledRenderBuffer&()); MOCK_CONST_METHOD1(CausalDelay, bool(size_t delay)); + MOCK_METHOD1(SetAudioBufferDelay, void(size_t delay_ms)); private: RenderBuffer* FakeGetRenderBuffer() { return &render_buffer_; } diff --git a/modules/audio_processing/aec3/render_delay_buffer.cc b/modules/audio_processing/aec3/render_delay_buffer.cc index 680e4ec280..5aa432d4c2 100644 --- a/modules/audio_processing/aec3/render_delay_buffer.cc +++ b/modules/audio_processing/aec3/render_delay_buffer.cc @@ -50,6 +50,8 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer { bool CausalDelay(size_t delay) const override; + void SetAudioBufferDelay(size_t delay_ms) override; + private: static int instance_count_; std::unique_ptr data_dumper_; @@ -75,6 +77,8 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer { size_t render_call_counter_ = 0; bool render_activity_ = false; size_t render_activity_counter_ = 0; + rtc::Optional external_audio_buffer_delay_ms_; + bool external_delay_verified_after_reset_ = false; int LowRateBufferOffset() const { return DelayEstimatorOffset(config_) >> 1; } int MapExternalDelayToInternalDelay(size_t external_delay_blocks) const; @@ -197,12 +201,33 @@ void RenderDelayBufferImpl::Reset() { low_rate_.read = low_rate_.OffsetIndex( low_rate_.write, LowRateBufferOffset() * sub_block_size_); - // Set the render buffer delays to the default delay. - ApplyDelay(config_.delay.default_delay); + // Check for any external audio buffer delay and whether it is feasible. + if (external_audio_buffer_delay_ms_) { + constexpr size_t kHeadroom = 5; + size_t external_delay_to_set = 0; + if (*external_audio_buffer_delay_ms_ < kHeadroom) { + external_delay_to_set = 0; + } else { + external_delay_to_set = *external_audio_buffer_delay_ms_ - kHeadroom; + } - // Unset the delays which are set by ApplyConfig. - delay_ = rtc::nullopt; - internal_delay_ = rtc::nullopt; + constexpr size_t kMaxExternalDelay = 170; + external_delay_to_set = std::min(external_delay_to_set, kMaxExternalDelay); + + // When an external delay estimate is available, use that delay as the + // initial render buffer delay. Avoid verifying the set delay. + external_delay_verified_after_reset_ = true; + SetDelay(external_delay_to_set); + external_delay_verified_after_reset_ = false; + } else { + // If an external delay estimate is not available, use that delay as the + // initial delay. Set the render buffer delays to the default delay. + ApplyDelay(config_.delay.default_delay); + + // Unset the delays which are set by SetDelay. + delay_ = rtc::nullopt; + internal_delay_ = rtc::nullopt; + } } // Inserts a new block into the render buffers. @@ -305,6 +330,15 @@ RenderDelayBufferImpl::PrepareCaptureProcessing() { // Sets the delay and returns a bool indicating whether the delay was changed. bool RenderDelayBufferImpl::SetDelay(size_t delay) { + if (!external_delay_verified_after_reset_ && + external_audio_buffer_delay_ms_) { + int delay_difference = static_cast(*external_audio_buffer_delay_ms_) - + static_cast(delay); + RTC_LOG(LS_WARNING) << "Difference between the externally reported delay " + "and the first delay estimate: " + << delay_difference << " ms."; + external_delay_verified_after_reset_ = true; + } if (delay_ && *delay_ == delay) { return false; } @@ -331,6 +365,15 @@ bool RenderDelayBufferImpl::CausalDelay(size_t delay) const { static_cast(config_.delay.min_echo_path_delay_blocks); } +void RenderDelayBufferImpl::SetAudioBufferDelay(size_t delay_ms) { + if (!external_audio_buffer_delay_ms_) { + RTC_LOG(LS_WARNING) + << "Receiving a first reported externally buffer delay of " << delay_ms + << " ms."; + } + external_audio_buffer_delay_ms_ = delay_ms; +} + // Maps the externally computed delay to the delay used internally. int RenderDelayBufferImpl::MapExternalDelayToInternalDelay( size_t external_delay_blocks) const { diff --git a/modules/audio_processing/aec3/render_delay_buffer.h b/modules/audio_processing/aec3/render_delay_buffer.h index 1bccc7dc05..e361628ebc 100644 --- a/modules/audio_processing/aec3/render_delay_buffer.h +++ b/modules/audio_processing/aec3/render_delay_buffer.h @@ -73,6 +73,9 @@ class RenderDelayBuffer { // Returns the maximum non calusal offset that can occur in the delay buffer. static int DelayEstimatorOffset(const EchoCanceller3Config& config); + + // Provides an optional external estimate of the audio buffer delay. + virtual void SetAudioBufferDelay(size_t delay_ms) = 0; }; } // namespace webrtc diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc index ddea435e30..99163525ed 100644 --- a/modules/audio_processing/audio_processing_impl.cc +++ b/modules/audio_processing/audio_processing_impl.cc @@ -1273,6 +1273,11 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { if (private_submodules_->echo_controller) { data_dumper_->DumpRaw("stream_delay", stream_delay_ms()); + if (was_stream_delay_set()) { + private_submodules_->echo_controller->SetAudioBufferDelay( + stream_delay_ms()); + } + private_submodules_->echo_controller->ProcessCapture( capture_buffer, capture_.echo_path_gain_change); } else { diff --git a/modules/audio_processing/include/mock_audio_processing.h b/modules/audio_processing/include/mock_audio_processing.h index 2d49388425..5fa3b1b03a 100644 --- a/modules/audio_processing/include/mock_audio_processing.h +++ b/modules/audio_processing/include/mock_audio_processing.h @@ -121,6 +121,7 @@ class MockEchoControl : public EchoControl { MOCK_METHOD2(ProcessCapture, void(AudioBuffer* capture, bool echo_path_change)); MOCK_CONST_METHOD0(GetMetrics, Metrics()); + MOCK_METHOD1(SetAudioBufferDelay, void(size_t delay_ms)); }; class MockVoiceDetection : public VoiceDetection {