diff --git a/api/audio/echo_control.h b/api/audio/echo_control.h index 021bbf8764..f549f40fbb 100644 --- a/api/audio/echo_control.h +++ b/api/audio/echo_control.h @@ -38,6 +38,9 @@ class EchoControl { // Collect current metrics from the echo controller. virtual Metrics GetMetrics() const = 0; + // Provides an optional external estimate of the audio buffer delay. + virtual void SetAudioBufferDelay(size_t delay_ms) = 0; + virtual ~EchoControl() {} }; diff --git a/modules/audio_processing/aec3/block_processor.cc b/modules/audio_processing/aec3/block_processor.cc index ff11bae8e1..f954be3b63 100644 --- a/modules/audio_processing/aec3/block_processor.cc +++ b/modules/audio_processing/aec3/block_processor.cc @@ -43,6 +43,8 @@ class BlockProcessorImpl final : public BlockProcessor { void GetMetrics(EchoControl::Metrics* metrics) const override; + void SetAudioBufferDelay(size_t delay_ms) override; + private: static int instance_count_; std::unique_ptr data_dumper_; @@ -234,6 +236,10 @@ void BlockProcessorImpl::GetMetrics(EchoControl::Metrics* metrics) const { metrics->delay_ms = delay ? static_cast(*delay) * block_size_ms : 0; } +void BlockProcessorImpl::SetAudioBufferDelay(size_t delay_ms) { + render_buffer_->SetAudioBufferDelay(delay_ms); +} + } // namespace BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config, diff --git a/modules/audio_processing/aec3/block_processor.h b/modules/audio_processing/aec3/block_processor.h index 8687bc2282..a3967ea7e4 100644 --- a/modules/audio_processing/aec3/block_processor.h +++ b/modules/audio_processing/aec3/block_processor.h @@ -42,6 +42,9 @@ class BlockProcessor { // Get current metrics. virtual void GetMetrics(EchoControl::Metrics* metrics) const = 0; + // Provides an optional external estimate of the audio buffer delay. + virtual void SetAudioBufferDelay(size_t delay_ms) = 0; + // Processes a block of capture data. virtual void ProcessCapture( bool echo_path_gain_change, diff --git a/modules/audio_processing/aec3/echo_canceller3.cc b/modules/audio_processing/aec3/echo_canceller3.cc index d478896e71..765e4de073 100644 --- a/modules/audio_processing/aec3/echo_canceller3.cc +++ b/modules/audio_processing/aec3/echo_canceller3.cc @@ -375,6 +375,11 @@ EchoControl::Metrics EchoCanceller3::GetMetrics() const { return metrics; } +void EchoCanceller3::SetAudioBufferDelay(size_t delay_ms) { + RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); + block_processor_->SetAudioBufferDelay(delay_ms); +} + void EchoCanceller3::EmptyRenderQueue() { RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); bool frame_to_buffer = diff --git a/modules/audio_processing/aec3/echo_canceller3.h b/modules/audio_processing/aec3/echo_canceller3.h index 8658814e93..b66b8b1dd1 100644 --- a/modules/audio_processing/aec3/echo_canceller3.h +++ b/modules/audio_processing/aec3/echo_canceller3.h @@ -82,6 +82,8 @@ class EchoCanceller3 : public EchoControl { void ProcessCapture(AudioBuffer* capture, bool level_change) override; // Collect current metrics from the echo canceller. Metrics GetMetrics() const override; + // Provides an optional external estimate of the audio buffer delay. + void SetAudioBufferDelay(size_t delay_ms) override; // Signals whether an external detector has detected echo leakage from the // echo canceller. diff --git a/modules/audio_processing/aec3/echo_canceller3_unittest.cc b/modules/audio_processing/aec3/echo_canceller3_unittest.cc index d4ad4f66f6..f652642130 100644 --- a/modules/audio_processing/aec3/echo_canceller3_unittest.cc +++ b/modules/audio_processing/aec3/echo_canceller3_unittest.cc @@ -104,6 +104,8 @@ class CaptureTransportVerificationProcessor : public BlockProcessor { void GetMetrics(EchoControl::Metrics* metrics) const override {} + void SetAudioBufferDelay(size_t delay_ms) override{}; + private: RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(CaptureTransportVerificationProcessor); }; @@ -132,6 +134,8 @@ class RenderTransportVerificationProcessor : public BlockProcessor { void GetMetrics(EchoControl::Metrics* metrics) const override {} + void SetAudioBufferDelay(size_t delay_ms) override{}; + private: std::deque>> received_render_blocks_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderTransportVerificationProcessor); diff --git a/modules/audio_processing/aec3/mock/mock_block_processor.h b/modules/audio_processing/aec3/mock/mock_block_processor.h index 5fff456185..803bf78e9c 100644 --- a/modules/audio_processing/aec3/mock/mock_block_processor.h +++ b/modules/audio_processing/aec3/mock/mock_block_processor.h @@ -31,6 +31,7 @@ class MockBlockProcessor : public BlockProcessor { void(const std::vector>& block)); MOCK_METHOD1(UpdateEchoLeakageStatus, void(bool leakage_detected)); MOCK_CONST_METHOD1(GetMetrics, void(EchoControl::Metrics* metrics)); + MOCK_METHOD1(SetAudioBufferDelay, void(size_t delay_ms)); }; } // namespace test diff --git a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h index c79cd107f0..00c13f422b 100644 --- a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h +++ b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h @@ -53,6 +53,7 @@ class MockRenderDelayBuffer : public RenderDelayBuffer { MOCK_CONST_METHOD0(GetDownsampledRenderBuffer, const DownsampledRenderBuffer&()); MOCK_CONST_METHOD1(CausalDelay, bool(size_t delay)); + MOCK_METHOD1(SetAudioBufferDelay, void(size_t delay_ms)); private: RenderBuffer* FakeGetRenderBuffer() { return &render_buffer_; } diff --git a/modules/audio_processing/aec3/render_delay_buffer.cc b/modules/audio_processing/aec3/render_delay_buffer.cc index 680e4ec280..5aa432d4c2 100644 --- a/modules/audio_processing/aec3/render_delay_buffer.cc +++ b/modules/audio_processing/aec3/render_delay_buffer.cc @@ -50,6 +50,8 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer { bool CausalDelay(size_t delay) const override; + void SetAudioBufferDelay(size_t delay_ms) override; + private: static int instance_count_; std::unique_ptr data_dumper_; @@ -75,6 +77,8 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer { size_t render_call_counter_ = 0; bool render_activity_ = false; size_t render_activity_counter_ = 0; + rtc::Optional external_audio_buffer_delay_ms_; + bool external_delay_verified_after_reset_ = false; int LowRateBufferOffset() const { return DelayEstimatorOffset(config_) >> 1; } int MapExternalDelayToInternalDelay(size_t external_delay_blocks) const; @@ -197,12 +201,33 @@ void RenderDelayBufferImpl::Reset() { low_rate_.read = low_rate_.OffsetIndex( low_rate_.write, LowRateBufferOffset() * sub_block_size_); - // Set the render buffer delays to the default delay. - ApplyDelay(config_.delay.default_delay); + // Check for any external audio buffer delay and whether it is feasible. + if (external_audio_buffer_delay_ms_) { + constexpr size_t kHeadroom = 5; + size_t external_delay_to_set = 0; + if (*external_audio_buffer_delay_ms_ < kHeadroom) { + external_delay_to_set = 0; + } else { + external_delay_to_set = *external_audio_buffer_delay_ms_ - kHeadroom; + } - // Unset the delays which are set by ApplyConfig. - delay_ = rtc::nullopt; - internal_delay_ = rtc::nullopt; + constexpr size_t kMaxExternalDelay = 170; + external_delay_to_set = std::min(external_delay_to_set, kMaxExternalDelay); + + // When an external delay estimate is available, use that delay as the + // initial render buffer delay. Avoid verifying the set delay. + external_delay_verified_after_reset_ = true; + SetDelay(external_delay_to_set); + external_delay_verified_after_reset_ = false; + } else { + // If an external delay estimate is not available, use that delay as the + // initial delay. Set the render buffer delays to the default delay. + ApplyDelay(config_.delay.default_delay); + + // Unset the delays which are set by SetDelay. + delay_ = rtc::nullopt; + internal_delay_ = rtc::nullopt; + } } // Inserts a new block into the render buffers. @@ -305,6 +330,15 @@ RenderDelayBufferImpl::PrepareCaptureProcessing() { // Sets the delay and returns a bool indicating whether the delay was changed. bool RenderDelayBufferImpl::SetDelay(size_t delay) { + if (!external_delay_verified_after_reset_ && + external_audio_buffer_delay_ms_) { + int delay_difference = static_cast(*external_audio_buffer_delay_ms_) - + static_cast(delay); + RTC_LOG(LS_WARNING) << "Difference between the externally reported delay " + "and the first delay estimate: " + << delay_difference << " ms."; + external_delay_verified_after_reset_ = true; + } if (delay_ && *delay_ == delay) { return false; } @@ -331,6 +365,15 @@ bool RenderDelayBufferImpl::CausalDelay(size_t delay) const { static_cast(config_.delay.min_echo_path_delay_blocks); } +void RenderDelayBufferImpl::SetAudioBufferDelay(size_t delay_ms) { + if (!external_audio_buffer_delay_ms_) { + RTC_LOG(LS_WARNING) + << "Receiving a first reported externally buffer delay of " << delay_ms + << " ms."; + } + external_audio_buffer_delay_ms_ = delay_ms; +} + // Maps the externally computed delay to the delay used internally. int RenderDelayBufferImpl::MapExternalDelayToInternalDelay( size_t external_delay_blocks) const { diff --git a/modules/audio_processing/aec3/render_delay_buffer.h b/modules/audio_processing/aec3/render_delay_buffer.h index 1bccc7dc05..e361628ebc 100644 --- a/modules/audio_processing/aec3/render_delay_buffer.h +++ b/modules/audio_processing/aec3/render_delay_buffer.h @@ -73,6 +73,9 @@ class RenderDelayBuffer { // Returns the maximum non calusal offset that can occur in the delay buffer. static int DelayEstimatorOffset(const EchoCanceller3Config& config); + + // Provides an optional external estimate of the audio buffer delay. + virtual void SetAudioBufferDelay(size_t delay_ms) = 0; }; } // namespace webrtc diff --git a/modules/audio_processing/audio_processing_impl.cc b/modules/audio_processing/audio_processing_impl.cc index ddea435e30..99163525ed 100644 --- a/modules/audio_processing/audio_processing_impl.cc +++ b/modules/audio_processing/audio_processing_impl.cc @@ -1273,6 +1273,11 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() { if (private_submodules_->echo_controller) { data_dumper_->DumpRaw("stream_delay", stream_delay_ms()); + if (was_stream_delay_set()) { + private_submodules_->echo_controller->SetAudioBufferDelay( + stream_delay_ms()); + } + private_submodules_->echo_controller->ProcessCapture( capture_buffer, capture_.echo_path_gain_change); } else { diff --git a/modules/audio_processing/include/mock_audio_processing.h b/modules/audio_processing/include/mock_audio_processing.h index 2d49388425..5fa3b1b03a 100644 --- a/modules/audio_processing/include/mock_audio_processing.h +++ b/modules/audio_processing/include/mock_audio_processing.h @@ -121,6 +121,7 @@ class MockEchoControl : public EchoControl { MOCK_METHOD2(ProcessCapture, void(AudioBuffer* capture, bool echo_path_change)); MOCK_CONST_METHOD0(GetMetrics, Metrics()); + MOCK_METHOD1(SetAudioBufferDelay, void(size_t delay_ms)); }; class MockVoiceDetection : public VoiceDetection {