Allow AEC3 to use any externally reported audio buffer delay in AEC3

This CL adds support for using any externally reported audio buffer
delay to set the initial alignment in AEC3 which is used before the
AEC has been able to detect the delay.

Bug: chromium:834182,webrtc:9163
Change-Id: Ic71355f69b7c4d5815b78e49987043441e7908fb
Reviewed-on: https://webrtc-review.googlesource.com/70580
Reviewed-by: Gustaf Ullberg <gustaf@webrtc.org>
Commit-Queue: Gustaf Ullberg <gustaf@webrtc.org>
Commit-Queue: Per Åhgren <peah@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#22917}

api/audio/echo_control.h

@@ -38,6 +38,9 @@ class EchoControl {
   // Collect current metrics from the echo controller.
   virtual Metrics GetMetrics() const = 0;
 
+  // Provides an optional external estimate of the audio buffer delay.
+  virtual void SetAudioBufferDelay(size_t delay_ms) = 0;
+
   virtual ~EchoControl() {}
 };
 

modules/audio_processing/aec3/block_processor.cc

@@ -43,6 +43,8 @@ class BlockProcessorImpl final : public BlockProcessor {
 
   void GetMetrics(EchoControl::Metrics* metrics) const override;
 
+  void SetAudioBufferDelay(size_t delay_ms) override;
+
  private:
   static int instance_count_;
   std::unique_ptr<ApmDataDumper> data_dumper_;
@@ -234,6 +236,10 @@ void BlockProcessorImpl::GetMetrics(EchoControl::Metrics* metrics) const {
   metrics->delay_ms = delay ? static_cast<int>(*delay) * block_size_ms : 0;
 }
 
+void BlockProcessorImpl::SetAudioBufferDelay(size_t delay_ms) {
+  render_buffer_->SetAudioBufferDelay(delay_ms);
+}
+
 }  // namespace
 
 BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config,

modules/audio_processing/aec3/block_processor.h

@@ -42,6 +42,9 @@ class BlockProcessor {
   // Get current metrics.
   virtual void GetMetrics(EchoControl::Metrics* metrics) const = 0;
 
+  // Provides an optional external estimate of the audio buffer delay.
+  virtual void SetAudioBufferDelay(size_t delay_ms) = 0;
+
   // Processes a block of capture data.
   virtual void ProcessCapture(
       bool echo_path_gain_change,

modules/audio_processing/aec3/echo_canceller3.cc

@@ -375,6 +375,11 @@ EchoControl::Metrics EchoCanceller3::GetMetrics() const {
   return metrics;
 }
 
+void EchoCanceller3::SetAudioBufferDelay(size_t delay_ms) {
+  RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_);
+  block_processor_->SetAudioBufferDelay(delay_ms);
+}
+
 void EchoCanceller3::EmptyRenderQueue() {
   RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_);
   bool frame_to_buffer =

modules/audio_processing/aec3/echo_canceller3.h

@@ -82,6 +82,8 @@ class EchoCanceller3 : public EchoControl {
   void ProcessCapture(AudioBuffer* capture, bool level_change) override;
   // Collect current metrics from the echo canceller.
   Metrics GetMetrics() const override;
+  // Provides an optional external estimate of the audio buffer delay.
+  void SetAudioBufferDelay(size_t delay_ms) override;
 
   // Signals whether an external detector has detected echo leakage from the
   // echo canceller.

modules/audio_processing/aec3/echo_canceller3_unittest.cc

@@ -104,6 +104,8 @@ class CaptureTransportVerificationProcessor : public BlockProcessor {
 
   void GetMetrics(EchoControl::Metrics* metrics) const override {}
 
+  void SetAudioBufferDelay(size_t delay_ms) override{};
+
  private:
   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(CaptureTransportVerificationProcessor);
 };
@@ -132,6 +134,8 @@ class RenderTransportVerificationProcessor : public BlockProcessor {
 
   void GetMetrics(EchoControl::Metrics* metrics) const override {}
 
+  void SetAudioBufferDelay(size_t delay_ms) override{};
+
  private:
   std::deque<std::vector<std::vector<float>>> received_render_blocks_;
   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderTransportVerificationProcessor);

modules/audio_processing/aec3/mock/mock_block_processor.h

@@ -31,6 +31,7 @@ class MockBlockProcessor : public BlockProcessor {
                void(const std::vector<std::vector<float>>& block));
   MOCK_METHOD1(UpdateEchoLeakageStatus, void(bool leakage_detected));
   MOCK_CONST_METHOD1(GetMetrics, void(EchoControl::Metrics* metrics));
+  MOCK_METHOD1(SetAudioBufferDelay, void(size_t delay_ms));
 };
 
 }  // namespace test

modules/audio_processing/aec3/mock/mock_render_delay_buffer.h

@@ -53,6 +53,7 @@ class MockRenderDelayBuffer : public RenderDelayBuffer {
   MOCK_CONST_METHOD0(GetDownsampledRenderBuffer,
                      const DownsampledRenderBuffer&());
   MOCK_CONST_METHOD1(CausalDelay, bool(size_t delay));
+  MOCK_METHOD1(SetAudioBufferDelay, void(size_t delay_ms));
 
  private:
   RenderBuffer* FakeGetRenderBuffer() { return &render_buffer_; }

modules/audio_processing/aec3/render_delay_buffer.cc

@@ -50,6 +50,8 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer {
 
   bool CausalDelay(size_t delay) const override;
 
+  void SetAudioBufferDelay(size_t delay_ms) override;
+
  private:
   static int instance_count_;
   std::unique_ptr<ApmDataDumper> data_dumper_;
@@ -75,6 +77,8 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer {
   size_t render_call_counter_ = 0;
   bool render_activity_ = false;
   size_t render_activity_counter_ = 0;
+  rtc::Optional<size_t> external_audio_buffer_delay_ms_;
+  bool external_delay_verified_after_reset_ = false;
 
   int LowRateBufferOffset() const { return DelayEstimatorOffset(config_) >> 1; }
   int MapExternalDelayToInternalDelay(size_t external_delay_blocks) const;
@@ -197,12 +201,33 @@ void RenderDelayBufferImpl::Reset() {
   low_rate_.read = low_rate_.OffsetIndex(
       low_rate_.write, LowRateBufferOffset() * sub_block_size_);
 
-  // Set the render buffer delays to the default delay.
-  ApplyDelay(config_.delay.default_delay);
+  // Check for any external audio buffer delay and whether it is feasible.
+  if (external_audio_buffer_delay_ms_) {
+    constexpr size_t kHeadroom = 5;
+    size_t external_delay_to_set = 0;
+    if (*external_audio_buffer_delay_ms_ < kHeadroom) {
+      external_delay_to_set = 0;
+    } else {
+      external_delay_to_set = *external_audio_buffer_delay_ms_ - kHeadroom;
+    }
 
-  // Unset the delays which are set by ApplyConfig.
-  delay_ = rtc::nullopt;
-  internal_delay_ = rtc::nullopt;
+    constexpr size_t kMaxExternalDelay = 170;
+    external_delay_to_set = std::min(external_delay_to_set, kMaxExternalDelay);
+
+    // When an external delay estimate is available, use that delay as the
+    // initial render buffer delay. Avoid verifying the set delay.
+    external_delay_verified_after_reset_ = true;
+    SetDelay(external_delay_to_set);
+    external_delay_verified_after_reset_ = false;
+  } else {
+    // If an external delay estimate is not available, use that delay as the
+    // initial delay. Set the render buffer delays to the default delay.
+    ApplyDelay(config_.delay.default_delay);
+
+    // Unset the delays which are set by SetDelay.
+    delay_ = rtc::nullopt;
+    internal_delay_ = rtc::nullopt;
+  }
 }
 
 // Inserts a new block into the render buffers.
@@ -305,6 +330,15 @@ RenderDelayBufferImpl::PrepareCaptureProcessing() {
 
 // Sets the delay and returns a bool indicating whether the delay was changed.
 bool RenderDelayBufferImpl::SetDelay(size_t delay) {
+  if (!external_delay_verified_after_reset_ &&
+      external_audio_buffer_delay_ms_) {
+    int delay_difference = static_cast<int>(*external_audio_buffer_delay_ms_) -
+                           static_cast<int>(delay);
+    RTC_LOG(LS_WARNING) << "Difference between the externally reported delay "
+                           "and the first delay estimate: "
+                        << delay_difference << " ms.";
+    external_delay_verified_after_reset_ = true;
+  }
   if (delay_ && *delay_ == delay) {
     return false;
   }
@@ -331,6 +365,15 @@ bool RenderDelayBufferImpl::CausalDelay(size_t delay) const {
          static_cast<int>(config_.delay.min_echo_path_delay_blocks);
 }
 
+void RenderDelayBufferImpl::SetAudioBufferDelay(size_t delay_ms) {
+  if (!external_audio_buffer_delay_ms_) {
+    RTC_LOG(LS_WARNING)
+        << "Receiving a first reported externally buffer delay of " << delay_ms
+        << " ms.";
+  }
+  external_audio_buffer_delay_ms_ = delay_ms;
+}
+
 // Maps the externally computed delay to the delay used internally.
 int RenderDelayBufferImpl::MapExternalDelayToInternalDelay(
     size_t external_delay_blocks) const {

modules/audio_processing/aec3/render_delay_buffer.h

@@ -73,6 +73,9 @@ class RenderDelayBuffer {
 
   // Returns the maximum non calusal offset that can occur in the delay buffer.
   static int DelayEstimatorOffset(const EchoCanceller3Config& config);
+
+  // Provides an optional external estimate of the audio buffer delay.
+  virtual void SetAudioBufferDelay(size_t delay_ms) = 0;
 };
 
 }  // namespace webrtc

modules/audio_processing/audio_processing_impl.cc

@@ -1273,6 +1273,11 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() {
   if (private_submodules_->echo_controller) {
     data_dumper_->DumpRaw("stream_delay", stream_delay_ms());
 
+    if (was_stream_delay_set()) {
+      private_submodules_->echo_controller->SetAudioBufferDelay(
+          stream_delay_ms());
+    }
+
     private_submodules_->echo_controller->ProcessCapture(
         capture_buffer, capture_.echo_path_gain_change);
   } else {

modules/audio_processing/include/mock_audio_processing.h

@@ -121,6 +121,7 @@ class MockEchoControl : public EchoControl {
   MOCK_METHOD2(ProcessCapture,
                void(AudioBuffer* capture, bool echo_path_change));
   MOCK_CONST_METHOD0(GetMetrics, Metrics());
+  MOCK_METHOD1(SetAudioBufferDelay, void(size_t delay_ms));
 };
 
 class MockVoiceDetection : public VoiceDetection {