Add empty residual echo detector.

This CL does not contain the actual algorithm, but only creates an empty processing component and connects the right signals to it. The algorithm will be added in a follow-up CL.

BUG=webrtc:6525

Review-Url: https://codereview.webrtc.org/2405403003
Cr-Commit-Position: refs/heads/master@{#14820}

webrtc/modules/audio_processing/BUILD.gn

@@ -99,6 +99,8 @@ rtc_static_library("audio_processing") {
     "noise_suppression_impl.cc",
     "noise_suppression_impl.h",
     "render_queue_item_verifier.h",
+    "residual_echo_detector.cc",
+    "residual_echo_detector.h",
     "rms_level.cc",
     "rms_level.h",
     "splitting_filter.cc",

webrtc/modules/audio_processing/audio_processing.gypi

@@ -111,6 +111,8 @@
         'noise_suppression_impl.cc',
         'noise_suppression_impl.h',
         'render_queue_item_verifier.h',
+        'residual_echo_detector.cc',
+        'residual_echo_detector.h',
         'rms_level.cc',
         'rms_level.h',
         'splitting_filter.cc',

webrtc/modules/audio_processing/audio_processing_impl.cc

@@ -35,6 +35,7 @@
 #include "webrtc/modules/audio_processing/level_controller/level_controller.h"
 #include "webrtc/modules/audio_processing/level_estimator_impl.h"
 #include "webrtc/modules/audio_processing/noise_suppression_impl.h"
+#include "webrtc/modules/audio_processing/residual_echo_detector.h"
 #include "webrtc/modules/audio_processing/transient/transient_suppressor.h"
 #include "webrtc/modules/audio_processing/voice_detection_impl.h"
 #include "webrtc/modules/include/module_common_types.h"
@@ -137,6 +138,7 @@ bool AudioProcessingImpl::ApmSubmoduleStates::Update(
     bool high_pass_filter_enabled,
     bool echo_canceller_enabled,
     bool mobile_echo_controller_enabled,
+    bool residual_echo_detector_enabled,
     bool noise_suppressor_enabled,
     bool intelligibility_enhancer_enabled,
     bool beamformer_enabled,
@@ -150,6 +152,8 @@ bool AudioProcessingImpl::ApmSubmoduleStates::Update(
   changed |= (echo_canceller_enabled != echo_canceller_enabled_);
   changed |=
       (mobile_echo_controller_enabled != mobile_echo_controller_enabled_);
+  changed |=
+      (residual_echo_detector_enabled != residual_echo_detector_enabled_);
   changed |= (noise_suppressor_enabled != noise_suppressor_enabled_);
   changed |=
       (intelligibility_enhancer_enabled != intelligibility_enhancer_enabled_);
@@ -165,6 +169,7 @@ bool AudioProcessingImpl::ApmSubmoduleStates::Update(
     high_pass_filter_enabled_ = high_pass_filter_enabled;
     echo_canceller_enabled_ = echo_canceller_enabled;
     mobile_echo_controller_enabled_ = mobile_echo_controller_enabled;
+    residual_echo_detector_enabled_ = residual_echo_detector_enabled;
     noise_suppressor_enabled_ = noise_suppressor_enabled;
     intelligibility_enhancer_enabled_ = intelligibility_enhancer_enabled;
     beamformer_enabled_ = beamformer_enabled;
@@ -239,6 +244,7 @@ struct AudioProcessingImpl::ApmPrivateSubmodules {
   std::unique_ptr<NonlinearBeamformer> beamformer;
   std::unique_ptr<AgcManagerDirect> agc_manager;
   std::unique_ptr<LevelController> level_controller;
+  std::unique_ptr<ResidualEchoDetector> residual_echo_detector;
 };
 
 AudioProcessing* AudioProcessing::Create() {
@@ -304,6 +310,8 @@ AudioProcessingImpl::AudioProcessingImpl(const webrtc::Config& config,
     public_submodules_->gain_control_for_experimental_agc.reset(
         new GainControlForExperimentalAgc(
             public_submodules_->gain_control.get(), &crit_capture_));
+    private_submodules_->residual_echo_detector.reset(
+        new ResidualEchoDetector());
 
     // TODO(peah): Move this creation to happen only when the level controller
     // is enabled.
@@ -470,6 +478,7 @@ int AudioProcessingImpl::InitializeLocked() {
   public_submodules_->voice_detection->Initialize(proc_split_sample_rate_hz());
   public_submodules_->level_estimator->Initialize();
   InitializeLevelController();
+  InitializeResidualEchoDetector();
 
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_dump_.debug_file->is_open()) {
@@ -809,6 +818,18 @@ void AudioProcessingImpl::QueueRenderAudio(AudioBuffer* audio) {
       RTC_DCHECK(result);
     }
   }
+
+  ResidualEchoDetector::PackRenderAudioBuffer(audio, &red_render_queue_buffer_);
+
+  // Insert the samples into the queue.
+  if (!red_render_signal_queue_->Insert(&red_render_queue_buffer_)) {
+    // The data queue is full and needs to be emptied.
+    EmptyQueuedRenderAudio();
+
+    // Retry the insert (should always work).
+    bool result = red_render_signal_queue_->Insert(&red_render_queue_buffer_);
+    RTC_DCHECK(result);
+  }
 }
 
 void AudioProcessingImpl::AllocateRenderQueue() {
@@ -827,6 +848,9 @@ void AudioProcessingImpl::AllocateRenderQueue() {
   const size_t new_agc_render_queue_element_max_size =
       std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerFrame);
 
+  const size_t new_red_render_queue_element_max_size =
+      std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerFrame);
+
   // Reallocate the queues if the queue item sizes are too small to fit the
   // data to put in the queues.
   if (aec_render_queue_element_max_size_ <
@@ -886,6 +910,25 @@ void AudioProcessingImpl::AllocateRenderQueue() {
   } else {
     agc_render_signal_queue_->Clear();
   }
+
+  if (red_render_queue_element_max_size_ <
+      new_red_render_queue_element_max_size) {
+    red_render_queue_element_max_size_ = new_red_render_queue_element_max_size;
+
+    std::vector<float> template_queue_element(
+        red_render_queue_element_max_size_);
+
+    red_render_signal_queue_.reset(
+        new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
+            kMaxNumFramesToBuffer, template_queue_element,
+            RenderQueueItemVerifier<float>(
+                red_render_queue_element_max_size_)));
+
+    red_render_queue_buffer_.resize(red_render_queue_element_max_size_);
+    red_capture_queue_buffer_.resize(red_render_queue_element_max_size_);
+  } else {
+    red_render_signal_queue_->Clear();
+  }
 }
 
 void AudioProcessingImpl::EmptyQueuedRenderAudio() {
@@ -904,6 +947,11 @@ void AudioProcessingImpl::EmptyQueuedRenderAudio() {
     public_submodules_->gain_control->ProcessRenderAudio(
         agc_capture_queue_buffer_);
   }
+
+  while (red_render_signal_queue_->Remove(&red_capture_queue_buffer_)) {
+    private_submodules_->residual_echo_detector->AnalyzeRenderAudio(
+        red_capture_queue_buffer_);
+  }
 }
 
 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) {
@@ -1079,6 +1127,13 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() {
   RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessCaptureAudio(
       capture_buffer, stream_delay_ms()));
 
+  if (config_.residual_echo_detector.enabled) {
+    private_submodules_->residual_echo_detector->AnalyzeCaptureAudio(
+        rtc::ArrayView<const float>(
+            capture_buffer->split_bands_const_f(0)[kBand0To8kHz],
+            capture_buffer->num_frames_per_band()));
+  }
+
   if (capture_nonlocked_.beamformer_enabled) {
     private_submodules_->beamformer->PostFilter(capture_buffer->split_data_f());
   }
@@ -1454,6 +1509,7 @@ bool AudioProcessingImpl::UpdateActiveSubmoduleStates() {
       public_submodules_->high_pass_filter->is_enabled(),
       public_submodules_->echo_cancellation->is_enabled(),
       public_submodules_->echo_control_mobile->is_enabled(),
+      config_.residual_echo_detector.enabled,
       public_submodules_->noise_suppression->is_enabled(),
       capture_nonlocked_.intelligibility_enabled,
       capture_nonlocked_.beamformer_enabled,
@@ -1503,6 +1559,10 @@ void AudioProcessingImpl::InitializeLevelController() {
   private_submodules_->level_controller->Initialize(proc_sample_rate_hz());
 }
 
+void AudioProcessingImpl::InitializeResidualEchoDetector() {
+  private_submodules_->residual_echo_detector->Initialize();
+}
+
 void AudioProcessingImpl::MaybeUpdateHistograms() {
   static const int kMinDiffDelayMs = 60;
 

webrtc/modules/audio_processing/audio_processing_impl.h

@@ -150,6 +150,7 @@ class AudioProcessingImpl : public AudioProcessing {
     bool Update(bool high_pass_filter_enabled,
                 bool echo_canceller_enabled,
                 bool mobile_echo_controller_enabled,
+                bool residual_echo_detector_enabled,
                 bool noise_suppressor_enabled,
                 bool intelligibility_enhancer_enabled,
                 bool beamformer_enabled,
@@ -167,6 +168,7 @@ class AudioProcessingImpl : public AudioProcessing {
     bool high_pass_filter_enabled_ = false;
     bool echo_canceller_enabled_ = false;
     bool mobile_echo_controller_enabled_ = false;
+    bool residual_echo_detector_enabled_ = false;
     bool noise_suppressor_enabled_ = false;
     bool intelligibility_enhancer_enabled_ = false;
     bool beamformer_enabled_ = false;
@@ -234,6 +236,8 @@ class AudioProcessingImpl : public AudioProcessing {
   int InitializeLocked(const ProcessingConfig& config)
       EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
   void InitializeLevelController() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_);
+  void InitializeResidualEchoDetector()
+      EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
 
   void EmptyQueuedRenderAudio();
   void AllocateRenderQueue()
@@ -291,8 +295,7 @@ class AudioProcessingImpl : public AudioProcessing {
 
   // Structs containing the pointers to the submodules.
   std::unique_ptr<ApmPublicSubmodules> public_submodules_;
-  std::unique_ptr<ApmPrivateSubmodules> private_submodules_
-      GUARDED_BY(crit_capture_);
+  std::unique_ptr<ApmPrivateSubmodules> private_submodules_;
 
   // State that is written to while holding both the render and capture locks
   // but can be read without any lock being held.
@@ -386,6 +389,11 @@ class AudioProcessingImpl : public AudioProcessing {
   std::vector<int16_t> agc_render_queue_buffer_ GUARDED_BY(crit_render_);
   std::vector<int16_t> agc_capture_queue_buffer_ GUARDED_BY(crit_capture_);
 
+  size_t red_render_queue_element_max_size_ GUARDED_BY(crit_render_)
+      GUARDED_BY(crit_capture_) = 0;
+  std::vector<float> red_render_queue_buffer_ GUARDED_BY(crit_render_);
+  std::vector<float> red_capture_queue_buffer_ GUARDED_BY(crit_capture_);
+
   // Lock protection not needed.
   std::unique_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>>
       aec_render_signal_queue_;
@@ -395,6 +403,8 @@ class AudioProcessingImpl : public AudioProcessing {
   std::unique_ptr<
       SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
       agc_render_signal_queue_;
+  std::unique_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>>
+      red_render_signal_queue_;
 };
 
 }  // namespace webrtc

webrtc/modules/audio_processing/include/audio_processing.h

@@ -258,6 +258,9 @@ class AudioProcessing {
       // the allowed range is [-100, 0].
       float initial_peak_level_dbfs = -6.0206f;
     } level_controller;
+    struct ResidualEchoDetector {
+      bool enabled = false;
+    } residual_echo_detector;
   };
 
   // TODO(mgraczyk): Remove once all methods that use ChannelLayout are gone.

webrtc/modules/audio_processing/residual_echo_detector.cc

@@ -0,0 +1,52 @@
+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/audio_processing/residual_echo_detector.h"
+
+#include "webrtc/modules/audio_processing/audio_buffer.h"
+
+namespace webrtc {
+
+ResidualEchoDetector::ResidualEchoDetector() = default;
+
+ResidualEchoDetector::~ResidualEchoDetector() = default;
+
+void ResidualEchoDetector::AnalyzeRenderAudio(
+    rtc::ArrayView<const float> render_audio) const {
+  // TODO(ivoc): Add implementation.
+}
+
+void ResidualEchoDetector::AnalyzeCaptureAudio(
+    rtc::ArrayView<const float> render_audio) {
+  // TODO(ivoc): Add implementation.
+}
+
+void ResidualEchoDetector::Initialize() {
+  // TODO(ivoc): Add implementation.
+}
+
+void ResidualEchoDetector::PackRenderAudioBuffer(
+    AudioBuffer* audio,
+    std::vector<float>* packed_buffer) {
+  RTC_DCHECK_GE(160u, audio->num_frames_per_band());
+
+  packed_buffer->clear();
+  packed_buffer->insert(packed_buffer->end(),
+                        audio->split_bands_const_f(0)[kBand0To8kHz],
+                        (audio->split_bands_const_f(0)[kBand0To8kHz] +
+                         audio->num_frames_per_band()));
+}
+
+float ResidualEchoDetector::get_echo_likelihood() const {
+  // TODO(ivoc): Add implementation.
+  return 0.0f;
+}
+
+}  // namespace webrtc

webrtc/modules/audio_processing/residual_echo_detector.h

@@ -0,0 +1,47 @@
+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_RESIDUAL_ECHO_DETECTOR_H_
+#define WEBRTC_MODULES_AUDIO_PROCESSING_RESIDUAL_ECHO_DETECTOR_H_
+
+#include <memory>
+#include <vector>
+
+#include "webrtc/base/array_view.h"
+
+namespace webrtc {
+
+class AudioBuffer;
+class EchoDetector;
+
+class ResidualEchoDetector {
+ public:
+  ResidualEchoDetector();
+  ~ResidualEchoDetector();
+
+  // This function should be called while holding the render lock.
+  void AnalyzeRenderAudio(rtc::ArrayView<const float> render_audio) const;
+
+  // This function should be called while holding the capture lock.
+  void AnalyzeCaptureAudio(rtc::ArrayView<const float> capture_audio);
+
+  // This function should be called while holding the capture lock.
+  void Initialize();
+
+  static void PackRenderAudioBuffer(AudioBuffer* audio,
+                                    std::vector<float>* packed_buffer);
+
+  // This function should be called while holding the capture lock.
+  float get_echo_likelihood() const;
+};
+
+}  // namespace webrtc
+
+#endif  // WEBRTC_MODULES_AUDIO_PROCESSING_RESIDUAL_ECHO_DETECTOR_H_