Introduced the render sample queue for the aec and aecm.

BUG=webrtc:5099

Review URL: https://codereview.webrtc.org/1410833002

Cr-Commit-Position: refs/heads/master@{#10662}

webrtc/modules/audio_processing/audio_processing_impl.cc

@@ -530,6 +530,9 @@ int AudioProcessingImpl::ProcessStream(const float* const* src,
     return kNullPointerError;
   }
 
+  echo_cancellation_->ReadQueuedRenderData();
+  echo_control_mobile_->ReadQueuedRenderData();
+
   ProcessingConfig processing_config = api_format_;
   processing_config.input_stream() = input_config;
   processing_config.output_stream() = output_config;
@@ -571,6 +574,9 @@ int AudioProcessingImpl::ProcessStream(const float* const* src,
 
 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) {
   CriticalSectionScoped crit_scoped(crit_);
+  echo_cancellation_->ReadQueuedRenderData();
+  echo_control_mobile_->ReadQueuedRenderData();
+
   if (!frame) {
     return kNullPointerError;
   }

webrtc/modules/audio_processing/echo_cancellation_impl.cc

@@ -55,6 +55,9 @@ AudioProcessing::Error MapError(int err) {
 }
 }  // namespace
 
+const size_t EchoCancellationImpl::kAllowedValuesOfSamplesPerFrame1;
+const size_t EchoCancellationImpl::kAllowedValuesOfSamplesPerFrame2;
+
 EchoCancellationImpl::EchoCancellationImpl(const AudioProcessing* apm,
                                            CriticalSectionWrapper* crit)
     : ProcessingComponent(),
@@ -68,7 +71,9 @@ EchoCancellationImpl::EchoCancellationImpl(const AudioProcessing* apm,
       stream_has_echo_(false),
       delay_logging_enabled_(false),
       extended_filter_enabled_(false),
-      delay_agnostic_enabled_(false) {
+      delay_agnostic_enabled_(false),
+      render_queue_element_max_size_(0) {
+  AllocateRenderQueue();
 }
 
 EchoCancellationImpl::~EchoCancellationImpl() {}
@@ -85,25 +90,65 @@ int EchoCancellationImpl::ProcessRenderAudio(const AudioBuffer* audio) {
 
   // The ordering convention must be followed to pass to the correct AEC.
   size_t handle_index = 0;
+  render_queue_buffer_.clear();
   for (int i = 0; i < apm_->num_output_channels(); i++) {
     for (int j = 0; j < audio->num_channels(); j++) {
       Handle* my_handle = static_cast<Handle*>(handle(handle_index));
-      err = WebRtcAec_BufferFarend(
-          my_handle,
-          audio->split_bands_const_f(j)[kBand0To8kHz],
+      // Retrieve any error code produced by the buffering of the farend
+      // signal
+      err = WebRtcAec_GetBufferFarendError(
+          my_handle, audio->split_bands_const_f(j)[kBand0To8kHz],
           audio->num_frames_per_band());
 
       if (err != apm_->kNoError) {
         return MapError(err);  // TODO(ajm): warning possible?
       }
 
-      handle_index++;
+      // Buffer the samples in the render queue.
+      render_queue_buffer_.insert(render_queue_buffer_.end(),
+                                  audio->split_bands_const_f(j)[kBand0To8kHz],
+                                  (audio->split_bands_const_f(j)[kBand0To8kHz] +
+                                   audio->num_frames_per_band()));
     }
   }
 
+  // Insert the samples into the queue.
+  if (!render_signal_queue_->Insert(&render_queue_buffer_)) {
+    ReadQueuedRenderData();
+
+    // Retry the insert (should always work).
+    RTC_DCHECK_EQ(render_signal_queue_->Insert(&render_queue_buffer_), true);
+  }
+
   return apm_->kNoError;
 }
 
+// Read chunks of data that were received and queued on the render side from
+// a queue. All the data chunks are buffered into the farend signal of the AEC.
+void EchoCancellationImpl::ReadQueuedRenderData() {
+  if (!is_component_enabled()) {
+    return;
+  }
+
+  while (render_signal_queue_->Remove(&capture_queue_buffer_)) {
+    size_t handle_index = 0;
+    int buffer_index = 0;
+    const int num_frames_per_band =
+        capture_queue_buffer_.size() /
+        (apm_->num_output_channels() * apm_->num_reverse_channels());
+    for (int i = 0; i < apm_->num_output_channels(); i++) {
+      for (int j = 0; j < apm_->num_reverse_channels(); j++) {
+        Handle* my_handle = static_cast<Handle*>(handle(handle_index));
+        WebRtcAec_BufferFarend(my_handle, &capture_queue_buffer_[buffer_index],
+                               num_frames_per_band);
+
+        buffer_index += num_frames_per_band;
+        handle_index++;
+      }
+    }
+  }
+}
+
 int EchoCancellationImpl::ProcessCaptureAudio(AudioBuffer* audio) {
   if (!is_component_enabled()) {
     return apm_->kNoError;
@@ -333,9 +378,37 @@ int EchoCancellationImpl::Initialize() {
     return err;
   }
 
+  AllocateRenderQueue();
+
   return apm_->kNoError;
 }
 
+void EchoCancellationImpl::AllocateRenderQueue() {
+  const size_t max_frame_size = std::max<size_t>(
+      kAllowedValuesOfSamplesPerFrame1, kAllowedValuesOfSamplesPerFrame2);
+
+  const size_t new_render_queue_element_max_size = std::max<size_t>(
+      static_cast<size_t>(1), max_frame_size * num_handles_required());
+
+  // Reallocate the queue if the queue item size is too small to fit the
+  // data to put in the queue.
+  if (new_render_queue_element_max_size > render_queue_element_max_size_) {
+    render_queue_element_max_size_ = new_render_queue_element_max_size;
+
+    std::vector<float> template_queue_element(render_queue_element_max_size_);
+
+    render_signal_queue_.reset(
+        new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
+            kMaxNumFramesToBuffer, template_queue_element,
+            RenderQueueItemVerifier<float>(render_queue_element_max_size_)));
+  } else {
+    render_signal_queue_->Clear();
+  }
+
+  render_queue_buffer_.resize(new_render_queue_element_max_size);
+  capture_queue_buffer_.resize(new_render_queue_element_max_size);
+}
+
 void EchoCancellationImpl::SetExtraOptions(const Config& config) {
   extended_filter_enabled_ = config.Get<ExtendedFilter>().enabled;
   delay_agnostic_enabled_ = config.Get<DelayAgnostic>().enabled;

webrtc/modules/audio_processing/echo_cancellation_impl.h

@@ -11,6 +11,8 @@
 #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_ECHO_CANCELLATION_IMPL_H_
 #define WEBRTC_MODULES_AUDIO_PROCESSING_ECHO_CANCELLATION_IMPL_H_
 
+#include "webrtc/base/scoped_ptr.h"
+#include "webrtc/common_audio/swap_queue.h"
 #include "webrtc/modules/audio_processing/include/audio_processing.h"
 #include "webrtc/modules/audio_processing/processing_component.h"
 
@@ -42,7 +44,16 @@ class EchoCancellationImpl : public EchoCancellation,
   bool is_delay_agnostic_enabled() const;
   bool is_extended_filter_enabled() const;
 
+  // Reads render side data that has been queued on the render call.
+  void ReadQueuedRenderData();
+
  private:
+  static const size_t kAllowedValuesOfSamplesPerFrame1 = 80;
+  static const size_t kAllowedValuesOfSamplesPerFrame2 = 160;
+  // TODO(peah): Decrease this once we properly handle hugely unbalanced
+  // reverse and forward call numbers.
+  static const size_t kMaxNumFramesToBuffer = 100;
+
   // EchoCancellation implementation.
   int Enable(bool enable) override;
   int enable_drift_compensation(bool enable) override;
@@ -68,6 +79,8 @@ class EchoCancellationImpl : public EchoCancellation,
   int num_handles_required() const override;
   int GetHandleError(void* handle) const override;
 
+  void AllocateRenderQueue();
+
   const AudioProcessing* apm_;
   CriticalSectionWrapper* crit_;
   bool drift_compensation_enabled_;
@@ -79,6 +92,12 @@ class EchoCancellationImpl : public EchoCancellation,
   bool delay_logging_enabled_;
   bool extended_filter_enabled_;
   bool delay_agnostic_enabled_;
+
+  size_t render_queue_element_max_size_;
+  std::vector<float> render_queue_buffer_;
+  std::vector<float> capture_queue_buffer_;
+  rtc::scoped_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>>
+      render_signal_queue_;
 };
 
 }  // namespace webrtc

webrtc/modules/audio_processing/echo_control_mobile_impl.cc

@@ -40,20 +40,42 @@ int16_t MapSetting(EchoControlMobile::RoutingMode mode) {
   return -1;
 }
 
+AudioProcessing::Error MapError(int err) {
+  switch (err) {
+    case AECM_UNSUPPORTED_FUNCTION_ERROR:
+      return AudioProcessing::kUnsupportedFunctionError;
+    case AECM_NULL_POINTER_ERROR:
+      return AudioProcessing::kNullPointerError;
+    case AECM_BAD_PARAMETER_ERROR:
+      return AudioProcessing::kBadParameterError;
+    case AECM_BAD_PARAMETER_WARNING:
+      return AudioProcessing::kBadStreamParameterWarning;
+    default:
+      // AECM_UNSPECIFIED_ERROR
+      // AECM_UNINITIALIZED_ERROR
+      return AudioProcessing::kUnspecifiedError;
+  }
+}
 }  // namespace
 
+const size_t EchoControlMobileImpl::kAllowedValuesOfSamplesPerFrame1;
+const size_t EchoControlMobileImpl::kAllowedValuesOfSamplesPerFrame2;
+
 size_t EchoControlMobile::echo_path_size_bytes() {
     return WebRtcAecm_echo_path_size_bytes();
 }
 
 EchoControlMobileImpl::EchoControlMobileImpl(const AudioProcessing* apm,
                                              CriticalSectionWrapper* crit)
-  : ProcessingComponent(),
-    apm_(apm),
-    crit_(crit),
-    routing_mode_(kSpeakerphone),
-    comfort_noise_enabled_(true),
-    external_echo_path_(NULL) {}
+    : ProcessingComponent(),
+      apm_(apm),
+      crit_(crit),
+      routing_mode_(kSpeakerphone),
+      comfort_noise_enabled_(true),
+      external_echo_path_(NULL),
+      render_queue_element_max_size_(0) {
+  AllocateRenderQueue();
+}
 
 EchoControlMobileImpl::~EchoControlMobileImpl() {
     if (external_echo_path_ != NULL) {
@@ -74,25 +96,64 @@ int EchoControlMobileImpl::ProcessRenderAudio(const AudioBuffer* audio) {
 
   // The ordering convention must be followed to pass to the correct AECM.
   size_t handle_index = 0;
+  render_queue_buffer_.clear();
   for (int i = 0; i < apm_->num_output_channels(); i++) {
     for (int j = 0; j < audio->num_channels(); j++) {
       Handle* my_handle = static_cast<Handle*>(handle(handle_index));
-      err = WebRtcAecm_BufferFarend(
-          my_handle,
-          audio->split_bands_const(j)[kBand0To8kHz],
+      err = WebRtcAecm_GetBufferFarendError(
+          my_handle, audio->split_bands_const(j)[kBand0To8kHz],
           audio->num_frames_per_band());
 
-      if (err != apm_->kNoError) {
-        return GetHandleError(my_handle);  // TODO(ajm): warning possible?
-      }
+      if (err != apm_->kNoError)
+        return MapError(err);  // TODO(ajm): warning possible?);
+
+      // Buffer the samples in the render queue.
+      render_queue_buffer_.insert(render_queue_buffer_.end(),
+                                  audio->split_bands_const(j)[kBand0To8kHz],
+                                  (audio->split_bands_const(j)[kBand0To8kHz] +
+                                   audio->num_frames_per_band()));
 
       handle_index++;
     }
   }
 
+  // Insert the samples into the queue.
+  if (!render_signal_queue_->Insert(&render_queue_buffer_)) {
+    ReadQueuedRenderData();
+
+    // Retry the insert (should always work).
+    RTC_DCHECK_EQ(render_signal_queue_->Insert(&render_queue_buffer_), true);
+  }
+
   return apm_->kNoError;
 }
 
+// Read chunks of data that were received and queued on the render side from
+// a queue. All the data chunks are buffered into the farend signal of the AEC.
+void EchoControlMobileImpl::ReadQueuedRenderData() {
+  if (!is_component_enabled()) {
+    return;
+  }
+
+  while (render_signal_queue_->Remove(&capture_queue_buffer_)) {
+    size_t handle_index = 0;
+    int buffer_index = 0;
+    const int num_frames_per_band =
+        capture_queue_buffer_.size() /
+        (apm_->num_output_channels() * apm_->num_reverse_channels());
+    for (int i = 0; i < apm_->num_output_channels(); i++) {
+      for (int j = 0; j < apm_->num_reverse_channels(); j++) {
+        Handle* my_handle = static_cast<Handle*>(handle(handle_index));
+        WebRtcAecm_BufferFarend(my_handle, &capture_queue_buffer_[buffer_index],
+                                num_frames_per_band);
+
+        buffer_index += num_frames_per_band;
+        handle_index++;
+      }
+    }
+  }
+}
+
 int EchoControlMobileImpl::ProcessCaptureAudio(AudioBuffer* audio) {
   if (!is_component_enabled()) {
     return apm_->kNoError;
@@ -128,9 +189,8 @@ int EchoControlMobileImpl::ProcessCaptureAudio(AudioBuffer* audio) {
           audio->num_frames_per_band(),
           apm_->stream_delay_ms());
 
-      if (err != apm_->kNoError) {
-        return GetHandleError(my_handle);  // TODO(ajm): warning possible?
-      }
+      if (err != apm_->kNoError)
+        return MapError(err);
 
       handle_index++;
     }
@@ -213,9 +273,9 @@ int EchoControlMobileImpl::GetEchoPath(void* echo_path,
 
   // Get the echo path from the first channel
   Handle* my_handle = static_cast<Handle*>(handle(0));
-  if (WebRtcAecm_GetEchoPath(my_handle, echo_path, size_bytes) != 0) {
-      return GetHandleError(my_handle);
-  }
+  int32_t err = WebRtcAecm_GetEchoPath(my_handle, echo_path, size_bytes);
+  if (err != 0)
+    return MapError(err);
 
   return apm_->kNoError;
 }
@@ -230,7 +290,39 @@ int EchoControlMobileImpl::Initialize() {
     return apm_->kBadSampleRateError;
   }
 
-  return ProcessingComponent::Initialize();
+  int err = ProcessingComponent::Initialize();
+  if (err != apm_->kNoError) {
+    return err;
+  }
+
+  AllocateRenderQueue();
+
+  return apm_->kNoError;
+}
+
+void EchoControlMobileImpl::AllocateRenderQueue() {
+  const size_t max_frame_size = std::max<size_t>(
+      kAllowedValuesOfSamplesPerFrame1, kAllowedValuesOfSamplesPerFrame2);
+  const size_t new_render_queue_element_max_size = std::max<size_t>(
+      static_cast<size_t>(1), max_frame_size * num_handles_required());
+
+  // Reallocate the queue if the queue item size is too small to fit the
+  // data to put in the queue.
+  if (new_render_queue_element_max_size > render_queue_element_max_size_) {
+    render_queue_element_max_size_ = new_render_queue_element_max_size;
+
+    std::vector<int16_t> template_queue_element(render_queue_element_max_size_);
+
+    render_signal_queue_.reset(
+        new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
+            kMaxNumFramesToBuffer, template_queue_element,
+            RenderQueueItemVerifier<int16_t>(render_queue_element_max_size_)));
+  } else {
+    render_signal_queue_->Clear();
+  }
+
+  render_queue_buffer_.resize(new_render_queue_element_max_size);
+  capture_queue_buffer_.resize(new_render_queue_element_max_size);
 }
 
 void* EchoControlMobileImpl::CreateHandle() const {

webrtc/modules/audio_processing/echo_control_mobile_impl.h

@@ -11,6 +11,8 @@
 #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_ECHO_CONTROL_MOBILE_IMPL_H_
 #define WEBRTC_MODULES_AUDIO_PROCESSING_ECHO_CONTROL_MOBILE_IMPL_H_
 
+#include "webrtc/base/scoped_ptr.h"
+#include "webrtc/common_audio/swap_queue.h"
 #include "webrtc/modules/audio_processing/include/audio_processing.h"
 #include "webrtc/modules/audio_processing/processing_component.h"
 
@@ -37,7 +39,16 @@ class EchoControlMobileImpl : public EchoControlMobile,
   // ProcessingComponent implementation.
   int Initialize() override;
 
+  // Reads render side data that has been queued on the render call.
+  void ReadQueuedRenderData();
+
  private:
+  static const size_t kAllowedValuesOfSamplesPerFrame1 = 80;
+  static const size_t kAllowedValuesOfSamplesPerFrame2 = 160;
+  // TODO(peah): Decrease this once we properly handle hugely unbalanced
+  // reverse and forward call numbers.
+  static const size_t kMaxNumFramesToBuffer = 100;
+
   // EchoControlMobile implementation.
   int Enable(bool enable) override;
   int set_routing_mode(RoutingMode mode) override;
@@ -53,11 +64,20 @@ class EchoControlMobileImpl : public EchoControlMobile,
   int num_handles_required() const override;
   int GetHandleError(void* handle) const override;
 
+  void AllocateRenderQueue();
+
   const AudioProcessing* apm_;
   CriticalSectionWrapper* crit_;
   RoutingMode routing_mode_;
   bool comfort_noise_enabled_;
   unsigned char* external_echo_path_;
+
+  size_t render_queue_element_max_size_;
+  std::vector<int16_t> render_queue_buffer_;
+  std::vector<int16_t> capture_queue_buffer_;
+  rtc::scoped_ptr<
+      SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
+      render_signal_queue_;
 };
 }  // namespace webrtc
 

webrtc/modules/audio_processing/processing_component.h

@@ -17,6 +17,22 @@
 
 namespace webrtc {
 
+// Functor to use when supplying a verifier function for the queue item
+// verifcation.
+template <typename T>
+class RenderQueueItemVerifier {
+ public:
+  explicit RenderQueueItemVerifier(size_t minimum_capacity)
+      : minimum_capacity_(minimum_capacity) {}
+
+  bool operator()(const std::vector<T>& v) const {
+    return v.capacity() >= minimum_capacity_;
+  }
+
+ private:
+  size_t minimum_capacity_;
+};
+
 class ProcessingComponent {
  public:
   ProcessingComponent();