From 764e364933da2a6ba1629b066d0a897cd5e5c10c Mon Sep 17 00:00:00 2001 From: peah Date: Sat, 22 Oct 2016 05:04:30 -0700 Subject: [PATCH] Several subcomponents inside APM copy render audio from the render side to the capture side using the same pattern. Currently this is done independently for the submodules. This CL moves the the AEC functionality for this into APM. BUG=webrtc:5298, webrtc:6540 Review-Url: https://codereview.webrtc.org/2427553003 Cr-Commit-Position: refs/heads/master@{#14726} --- .../audio_processing/audio_processing_impl.cc | 69 +++++++- .../audio_processing/audio_processing_impl.h | 17 ++ .../echo_cancellation_bit_exact_unittest.cc | 7 +- .../echo_cancellation_impl.cc | 151 +++++------------- .../audio_processing/echo_cancellation_impl.h | 26 +-- 5 files changed, 137 insertions(+), 133 deletions(-) diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc index c892b7886c..93e43f7add 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.cc +++ b/webrtc/modules/audio_processing/audio_processing_impl.cc @@ -119,6 +119,13 @@ int FindNativeProcessRateToUse(int minimum_rate, bool band_splitting_required) { return uppermost_native_rate; } +// Maximum length that a frame of samples can have. +static const size_t kMaxAllowedValuesOfSamplesPerFrame = 160; +// Maximum number of frames to buffer in the render queue. +// TODO(peah): Decrease this once we properly handle hugely unbalanced +// reverse and forward call numbers. +static const size_t kMaxNumFramesToBuffer = 100; + } // namespace // Throughout webrtc, it's assumed that success is represented by zero. @@ -430,9 +437,12 @@ int AudioProcessingImpl::InitializeLocked() { public_submodules_->gain_control->Initialize(num_proc_channels(), proc_sample_rate_hz()); + public_submodules_->echo_cancellation->Initialize( proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(), num_proc_channels()); + AllocateRenderQueue(); + public_submodules_->echo_control_mobile->Initialize( proc_split_sample_rate_hz(), num_reverse_channels(), num_output_channels()); @@ -697,7 +707,7 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, // that retrieves the render side data. This function accesses apm // getters that need the capture lock held when being called. rtc::CritScope cs_capture(&crit_capture_); - public_submodules_->echo_cancellation->ReadQueuedRenderData(); + EmptyQueuedRenderAudio(); public_submodules_->echo_control_mobile->ReadQueuedRenderData(); public_submodules_->gain_control->ReadQueuedRenderData(); @@ -757,6 +767,58 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, return kNoError; } +void AudioProcessingImpl::QueueRenderAudio(const AudioBuffer* audio) { + EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(), + num_reverse_channels(), + &render_queue_buffer_); + + RTC_DCHECK_GE(160u, audio->num_frames_per_band()); + + // Insert the samples into the queue. + if (!render_signal_queue_->Insert(&render_queue_buffer_)) { + // The data queue is full and needs to be emptied. + EmptyQueuedRenderAudio(); + + // Retry the insert (should always work). + bool result = render_signal_queue_->Insert(&render_queue_buffer_); + RTC_DCHECK(result); + } +} + +void AudioProcessingImpl::AllocateRenderQueue() { + const size_t new_render_queue_element_max_size = + std::max(static_cast(1), + kMaxAllowedValuesOfSamplesPerFrame * + EchoCancellationImpl::NumCancellersRequired( + num_output_channels(), num_reverse_channels())); + + // Reallocate the queue if the queue item size is too small to fit the + // data to put in the queue. + if (render_queue_element_max_size_ < new_render_queue_element_max_size) { + render_queue_element_max_size_ = new_render_queue_element_max_size; + + std::vector template_queue_element(render_queue_element_max_size_); + + render_signal_queue_.reset( + new SwapQueue, RenderQueueItemVerifier>( + kMaxNumFramesToBuffer, template_queue_element, + RenderQueueItemVerifier(render_queue_element_max_size_))); + + render_queue_buffer_.resize(render_queue_element_max_size_); + capture_queue_buffer_.resize(render_queue_element_max_size_); + } else { + render_signal_queue_->Clear(); + } +} + +void AudioProcessingImpl::EmptyQueuedRenderAudio() { + rtc::CritScope cs_capture(&crit_capture_); + while (render_signal_queue_->Remove(&capture_queue_buffer_)) { + public_submodules_->echo_cancellation->ProcessRenderAudio( + capture_queue_buffer_); + } +} + int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_AudioFrame"); { @@ -767,7 +829,7 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { // public_submodules_->echo_control_mobile->is_enabled() aquires this lock // as well. rtc::CritScope cs_capture(&crit_capture_); - public_submodules_->echo_cancellation->ReadQueuedRenderData(); + EmptyQueuedRenderAudio(); public_submodules_->echo_control_mobile->ReadQueuedRenderData(); public_submodules_->gain_control->ReadQueuedRenderData(); } @@ -1130,8 +1192,7 @@ int AudioProcessingImpl::ProcessRenderStreamLocked() { } #endif - RETURN_ON_ERR( - public_submodules_->echo_cancellation->ProcessRenderAudio(render_buffer)); + QueueRenderAudio(render_buffer); RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessRenderAudio( render_buffer)); if (!constants_.use_experimental_agc) { diff --git a/webrtc/modules/audio_processing/audio_processing_impl.h b/webrtc/modules/audio_processing/audio_processing_impl.h index 21bc588198..d6af64d22f 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.h +++ b/webrtc/modules/audio_processing/audio_processing_impl.h @@ -19,9 +19,11 @@ #include "webrtc/base/criticalsection.h" #include "webrtc/base/gtest_prod_util.h" #include "webrtc/base/ignore_wundef.h" +#include "webrtc/base/swap_queue.h" #include "webrtc/base/thread_annotations.h" #include "webrtc/modules/audio_processing/audio_buffer.h" #include "webrtc/modules/audio_processing/include/audio_processing.h" +#include "webrtc/modules/audio_processing/render_queue_item_verifier.h" #include "webrtc/system_wrappers/include/file_wrapper.h" #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP @@ -233,6 +235,12 @@ class AudioProcessingImpl : public AudioProcessing { EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); void InitializeLevelController() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + void EmptyQueuedRenderAudio(); + void AllocateRenderQueue() + EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); + void QueueRenderAudio(const AudioBuffer* audio) + EXCLUSIVE_LOCKS_REQUIRED(crit_render_); + // Capture-side exclusive methods possibly running APM in a multi-threaded // manner that are called with the render lock already acquired. int ProcessCaptureStreamLocked() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); @@ -362,6 +370,15 @@ class AudioProcessingImpl : public AudioProcessing { std::unique_ptr render_converter; std::unique_ptr render_audio; } render_ GUARDED_BY(crit_render_); + + size_t render_queue_element_max_size_ GUARDED_BY(crit_render_) + GUARDED_BY(crit_capture_) = 0; + std::vector render_queue_buffer_ GUARDED_BY(crit_render_); + std::vector capture_queue_buffer_ GUARDED_BY(crit_capture_); + + // Lock protection not needed. + std::unique_ptr, RenderQueueItemVerifier>> + render_signal_queue_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc b/webrtc/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc index 6b0fc0178d..93eaa04b9f 100644 --- a/webrtc/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc +++ b/webrtc/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc @@ -49,8 +49,11 @@ void ProcessOneFrame(int sample_rate_hz, capture_audio_buffer->SplitIntoFrequencyBands(); } - echo_canceller->ProcessRenderAudio(render_audio_buffer); - echo_canceller->ReadQueuedRenderData(); + std::vector render_audio; + EchoCancellationImpl::PackRenderAudioBuffer( + render_audio_buffer, 1, render_audio_buffer->num_channels(), + &render_audio); + echo_canceller->ProcessRenderAudio(render_audio); if (drift_compensation_enabled) { static_cast(echo_canceller) diff --git a/webrtc/modules/audio_processing/echo_cancellation_impl.cc b/webrtc/modules/audio_processing/echo_cancellation_impl.cc index 18c815dc00..e06b2faa84 100644 --- a/webrtc/modules/audio_processing/echo_cancellation_impl.cc +++ b/webrtc/modules/audio_processing/echo_cancellation_impl.cc @@ -49,12 +49,6 @@ AudioProcessing::Error MapError(int err) { } } -// Maximum length that a frame of samples can have. -static const size_t kMaxAllowedValuesOfSamplesPerFrame = 160; -// Maximum number of frames to buffer in the render queue. -// TODO(peah): Decrease this once we properly handle hugely unbalanced -// reverse and forward call numbers. -static const size_t kMaxNumFramesToBuffer = 100; } // namespace struct EchoCancellationImpl::StreamProperties { @@ -113,94 +107,38 @@ EchoCancellationImpl::EchoCancellationImpl(rtc::CriticalSection* crit_render, delay_logging_enabled_(false), extended_filter_enabled_(false), delay_agnostic_enabled_(false), - aec3_enabled_(false), - render_queue_element_max_size_(0) { + aec3_enabled_(false) { RTC_DCHECK(crit_render); RTC_DCHECK(crit_capture); } -EchoCancellationImpl::~EchoCancellationImpl() {} +EchoCancellationImpl::~EchoCancellationImpl() = default; -int EchoCancellationImpl::ProcessRenderAudio(const AudioBuffer* audio) { - rtc::CritScope cs_render(crit_render_); - if (!enabled_) { - return AudioProcessing::kNoError; - } - - RTC_DCHECK(stream_properties_); - RTC_DCHECK_GE(160u, audio->num_frames_per_band()); - RTC_DCHECK_EQ(audio->num_channels(), - stream_properties_->num_reverse_channels); - RTC_DCHECK_GE(cancellers_.size(), stream_properties_->num_output_channels * - audio->num_channels()); - - int err = AudioProcessing::kNoError; - - // The ordering convention must be followed to pass to the correct AEC. - size_t handle_index = 0; - render_queue_buffer_.clear(); - for (size_t i = 0; i < stream_properties_->num_output_channels; i++) { - for (size_t j = 0; j < audio->num_channels(); j++) { - // Retrieve any error code produced by the buffering of the farend - // signal. - err = WebRtcAec_GetBufferFarendError( - cancellers_[handle_index++]->state(), - audio->split_bands_const_f(j)[kBand0To8kHz], - audio->num_frames_per_band()); - - if (err != AudioProcessing::kNoError) { - return MapError(err); // TODO(ajm): warning possible? - } - - // Buffer the samples in the render queue. - render_queue_buffer_.insert(render_queue_buffer_.end(), - audio->split_bands_const_f(j)[kBand0To8kHz], - (audio->split_bands_const_f(j)[kBand0To8kHz] + - audio->num_frames_per_band())); - } - } - - // Insert the samples into the queue. - if (!render_signal_queue_->Insert(&render_queue_buffer_)) { - // The data queue is full and needs to be emptied. - ReadQueuedRenderData(); - - // Retry the insert (should always work). - bool result = render_signal_queue_->Insert(&render_queue_buffer_); - RTC_DCHECK(result); - } - - return AudioProcessing::kNoError; -} - -// Read chunks of data that were received and queued on the render side from -// a queue. All the data chunks are buffered into the farend signal of the AEC. -void EchoCancellationImpl::ReadQueuedRenderData() { +void EchoCancellationImpl::ProcessRenderAudio( + rtc::ArrayView packed_render_audio) { rtc::CritScope cs_capture(crit_capture_); if (!enabled_) { return; } RTC_DCHECK(stream_properties_); - while (render_signal_queue_->Remove(&capture_queue_buffer_)) { - size_t handle_index = 0; - size_t buffer_index = 0; - const size_t num_frames_per_band = - capture_queue_buffer_.size() / - (stream_properties_->num_output_channels * - stream_properties_->num_reverse_channels); - for (size_t i = 0; i < stream_properties_->num_output_channels; i++) { - for (size_t j = 0; j < stream_properties_->num_reverse_channels; j++) { - WebRtcAec_BufferFarend(cancellers_[handle_index++]->state(), - &capture_queue_buffer_[buffer_index], - num_frames_per_band); + size_t handle_index = 0; + size_t buffer_index = 0; + const size_t num_frames_per_band = + packed_render_audio.size() / (stream_properties_->num_output_channels * + stream_properties_->num_reverse_channels); + for (size_t i = 0; i < stream_properties_->num_output_channels; i++) { + for (size_t j = 0; j < stream_properties_->num_reverse_channels; j++) { + WebRtcAec_BufferFarend(cancellers_[handle_index++]->state(), + &packed_render_audio[buffer_index], + num_frames_per_band); - buffer_index += num_frames_per_band; - } + buffer_index += num_frames_per_band; } } } + int EchoCancellationImpl::ProcessCaptureAudio(AudioBuffer* audio, int stream_delay_ms) { rtc::CritScope cs_capture(crit_capture_); @@ -480,9 +418,12 @@ void EchoCancellationImpl::Initialize(int sample_rate_hz, return; } - if (NumCancellersRequired() > cancellers_.size()) { + const size_t num_cancellers_required = + NumCancellersRequired(stream_properties_->num_output_channels, + stream_properties_->num_reverse_channels); + if (num_cancellers_required > cancellers_.size()) { const size_t cancellers_old_size = cancellers_.size(); - cancellers_.resize(NumCancellersRequired()); + cancellers_.resize(num_cancellers_required); for (size_t i = cancellers_old_size; i < cancellers_.size(); ++i) { cancellers_[i].reset(new Canceller()); @@ -494,8 +435,6 @@ void EchoCancellationImpl::Initialize(int sample_rate_hz, } Configure(); - - AllocateRenderQueue(); } int EchoCancellationImpl::GetSystemDelayInSamples() const { @@ -506,30 +445,24 @@ int EchoCancellationImpl::GetSystemDelayInSamples() const { WebRtcAec_aec_core(cancellers_[0]->state())); } -void EchoCancellationImpl::AllocateRenderQueue() { - const size_t new_render_queue_element_max_size = std::max( - static_cast(1), - kMaxAllowedValuesOfSamplesPerFrame * NumCancellersRequired()); +void EchoCancellationImpl::PackRenderAudioBuffer( + const AudioBuffer* audio, + size_t num_output_channels, + size_t num_channels, + std::vector* packed_buffer) { + RTC_DCHECK_GE(160u, audio->num_frames_per_band()); + RTC_DCHECK_EQ(num_channels, audio->num_channels()); - rtc::CritScope cs_render(crit_render_); - rtc::CritScope cs_capture(crit_capture_); - - // Reallocate the queue if the queue item size is too small to fit the - // data to put in the queue. - if (render_queue_element_max_size_ < new_render_queue_element_max_size) { - render_queue_element_max_size_ = new_render_queue_element_max_size; - - std::vector template_queue_element(render_queue_element_max_size_); - - render_signal_queue_.reset( - new SwapQueue, RenderQueueItemVerifier>( - kMaxNumFramesToBuffer, template_queue_element, - RenderQueueItemVerifier(render_queue_element_max_size_))); - - render_queue_buffer_.resize(render_queue_element_max_size_); - capture_queue_buffer_.resize(render_queue_element_max_size_); - } else { - render_signal_queue_->Clear(); + packed_buffer->clear(); + // The ordering convention must be followed to pass the correct data. + for (size_t i = 0; i < num_output_channels; i++) { + for (size_t j = 0; j < audio->num_channels(); j++) { + // Buffer the samples in the render queue. + packed_buffer->insert(packed_buffer->end(), + audio->split_bands_const_f(j)[kBand0To8kHz], + (audio->split_bands_const_f(j)[kBand0To8kHz] + + audio->num_frames_per_band())); + } } } @@ -573,10 +506,10 @@ int EchoCancellationImpl::Configure() { return error; } -size_t EchoCancellationImpl::NumCancellersRequired() const { - RTC_DCHECK(stream_properties_); - return stream_properties_->num_output_channels * - stream_properties_->num_reverse_channels; +size_t EchoCancellationImpl::NumCancellersRequired( + size_t num_output_channels, + size_t num_reverse_channels) { + return num_output_channels * num_reverse_channels; } } // namespace webrtc diff --git a/webrtc/modules/audio_processing/echo_cancellation_impl.h b/webrtc/modules/audio_processing/echo_cancellation_impl.h index bab5e54f4b..8a4ea71846 100644 --- a/webrtc/modules/audio_processing/echo_cancellation_impl.h +++ b/webrtc/modules/audio_processing/echo_cancellation_impl.h @@ -16,9 +16,7 @@ #include "webrtc/base/constructormagic.h" #include "webrtc/base/criticalsection.h" -#include "webrtc/base/swap_queue.h" #include "webrtc/modules/audio_processing/include/audio_processing.h" -#include "webrtc/modules/audio_processing/render_queue_item_verifier.h" namespace webrtc { @@ -30,7 +28,7 @@ class EchoCancellationImpl : public EchoCancellation { rtc::CriticalSection* crit_capture); ~EchoCancellationImpl() override; - int ProcessRenderAudio(const AudioBuffer* audio); + void ProcessRenderAudio(rtc::ArrayView packed_render_audio); int ProcessCaptureAudio(AudioBuffer* audio, int stream_delay_ms); // EchoCancellation implementation. @@ -50,13 +48,16 @@ class EchoCancellationImpl : public EchoCancellation { std::string GetExperimentsDescription(); bool is_refined_adaptive_filter_enabled() const; - // Reads render side data that has been queued on the render call. - // Called holding the capture lock. - void ReadQueuedRenderData(); - // Returns the system delay of the first AEC component. int GetSystemDelayInSamples() const; + static void PackRenderAudioBuffer(const AudioBuffer* audio, + size_t num_output_channels, + size_t num_channels, + std::vector* packed_buffer); + static size_t NumCancellersRequired(size_t num_output_channels, + size_t num_reverse_channels); + private: class Canceller; struct StreamProperties; @@ -79,8 +80,6 @@ class EchoCancellationImpl : public EchoCancellation { struct AecCore* aec_core() const override; - size_t NumCancellersRequired() const; - void AllocateRenderQueue(); int Configure(); @@ -100,15 +99,6 @@ class EchoCancellationImpl : public EchoCancellation { bool aec3_enabled_ GUARDED_BY(crit_capture_); bool refined_adaptive_filter_enabled_ GUARDED_BY(crit_capture_) = false; - size_t render_queue_element_max_size_ GUARDED_BY(crit_render_) - GUARDED_BY(crit_capture_); - std::vector render_queue_buffer_ GUARDED_BY(crit_render_); - std::vector capture_queue_buffer_ GUARDED_BY(crit_capture_); - - // Lock protection not needed. - std::unique_ptr, RenderQueueItemVerifier>> - render_signal_queue_; - std::vector> cancellers_; std::unique_ptr stream_properties_;