diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc index c892b7886c..93e43f7add 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.cc +++ b/webrtc/modules/audio_processing/audio_processing_impl.cc @@ -119,6 +119,13 @@ int FindNativeProcessRateToUse(int minimum_rate, bool band_splitting_required) { return uppermost_native_rate; } +// Maximum length that a frame of samples can have. +static const size_t kMaxAllowedValuesOfSamplesPerFrame = 160; +// Maximum number of frames to buffer in the render queue. +// TODO(peah): Decrease this once we properly handle hugely unbalanced +// reverse and forward call numbers. +static const size_t kMaxNumFramesToBuffer = 100; + } // namespace // Throughout webrtc, it's assumed that success is represented by zero. @@ -430,9 +437,12 @@ int AudioProcessingImpl::InitializeLocked() { public_submodules_->gain_control->Initialize(num_proc_channels(), proc_sample_rate_hz()); + public_submodules_->echo_cancellation->Initialize( proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(), num_proc_channels()); + AllocateRenderQueue(); + public_submodules_->echo_control_mobile->Initialize( proc_split_sample_rate_hz(), num_reverse_channels(), num_output_channels()); @@ -697,7 +707,7 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, // that retrieves the render side data. This function accesses apm // getters that need the capture lock held when being called. rtc::CritScope cs_capture(&crit_capture_); - public_submodules_->echo_cancellation->ReadQueuedRenderData(); + EmptyQueuedRenderAudio(); public_submodules_->echo_control_mobile->ReadQueuedRenderData(); public_submodules_->gain_control->ReadQueuedRenderData(); @@ -757,6 +767,58 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, return kNoError; } +void AudioProcessingImpl::QueueRenderAudio(const AudioBuffer* audio) { + EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(), + num_reverse_channels(), + &render_queue_buffer_); + + RTC_DCHECK_GE(160u, audio->num_frames_per_band()); + + // Insert the samples into the queue. + if (!render_signal_queue_->Insert(&render_queue_buffer_)) { + // The data queue is full and needs to be emptied. + EmptyQueuedRenderAudio(); + + // Retry the insert (should always work). + bool result = render_signal_queue_->Insert(&render_queue_buffer_); + RTC_DCHECK(result); + } +} + +void AudioProcessingImpl::AllocateRenderQueue() { + const size_t new_render_queue_element_max_size = + std::max(static_cast(1), + kMaxAllowedValuesOfSamplesPerFrame * + EchoCancellationImpl::NumCancellersRequired( + num_output_channels(), num_reverse_channels())); + + // Reallocate the queue if the queue item size is too small to fit the + // data to put in the queue. + if (render_queue_element_max_size_ < new_render_queue_element_max_size) { + render_queue_element_max_size_ = new_render_queue_element_max_size; + + std::vector template_queue_element(render_queue_element_max_size_); + + render_signal_queue_.reset( + new SwapQueue, RenderQueueItemVerifier>( + kMaxNumFramesToBuffer, template_queue_element, + RenderQueueItemVerifier(render_queue_element_max_size_))); + + render_queue_buffer_.resize(render_queue_element_max_size_); + capture_queue_buffer_.resize(render_queue_element_max_size_); + } else { + render_signal_queue_->Clear(); + } +} + +void AudioProcessingImpl::EmptyQueuedRenderAudio() { + rtc::CritScope cs_capture(&crit_capture_); + while (render_signal_queue_->Remove(&capture_queue_buffer_)) { + public_submodules_->echo_cancellation->ProcessRenderAudio( + capture_queue_buffer_); + } +} + int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_AudioFrame"); { @@ -767,7 +829,7 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { // public_submodules_->echo_control_mobile->is_enabled() aquires this lock // as well. rtc::CritScope cs_capture(&crit_capture_); - public_submodules_->echo_cancellation->ReadQueuedRenderData(); + EmptyQueuedRenderAudio(); public_submodules_->echo_control_mobile->ReadQueuedRenderData(); public_submodules_->gain_control->ReadQueuedRenderData(); } @@ -1130,8 +1192,7 @@ int AudioProcessingImpl::ProcessRenderStreamLocked() { } #endif - RETURN_ON_ERR( - public_submodules_->echo_cancellation->ProcessRenderAudio(render_buffer)); + QueueRenderAudio(render_buffer); RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessRenderAudio( render_buffer)); if (!constants_.use_experimental_agc) { diff --git a/webrtc/modules/audio_processing/audio_processing_impl.h b/webrtc/modules/audio_processing/audio_processing_impl.h index 21bc588198..d6af64d22f 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.h +++ b/webrtc/modules/audio_processing/audio_processing_impl.h @@ -19,9 +19,11 @@ #include "webrtc/base/criticalsection.h" #include "webrtc/base/gtest_prod_util.h" #include "webrtc/base/ignore_wundef.h" +#include "webrtc/base/swap_queue.h" #include "webrtc/base/thread_annotations.h" #include "webrtc/modules/audio_processing/audio_buffer.h" #include "webrtc/modules/audio_processing/include/audio_processing.h" +#include "webrtc/modules/audio_processing/render_queue_item_verifier.h" #include "webrtc/system_wrappers/include/file_wrapper.h" #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP @@ -233,6 +235,12 @@ class AudioProcessingImpl : public AudioProcessing { EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); void InitializeLevelController() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); + void EmptyQueuedRenderAudio(); + void AllocateRenderQueue() + EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); + void QueueRenderAudio(const AudioBuffer* audio) + EXCLUSIVE_LOCKS_REQUIRED(crit_render_); + // Capture-side exclusive methods possibly running APM in a multi-threaded // manner that are called with the render lock already acquired. int ProcessCaptureStreamLocked() EXCLUSIVE_LOCKS_REQUIRED(crit_capture_); @@ -362,6 +370,15 @@ class AudioProcessingImpl : public AudioProcessing { std::unique_ptr render_converter; std::unique_ptr render_audio; } render_ GUARDED_BY(crit_render_); + + size_t render_queue_element_max_size_ GUARDED_BY(crit_render_) + GUARDED_BY(crit_capture_) = 0; + std::vector render_queue_buffer_ GUARDED_BY(crit_render_); + std::vector capture_queue_buffer_ GUARDED_BY(crit_capture_); + + // Lock protection not needed. + std::unique_ptr, RenderQueueItemVerifier>> + render_signal_queue_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc b/webrtc/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc index 6b0fc0178d..93eaa04b9f 100644 --- a/webrtc/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc +++ b/webrtc/modules/audio_processing/echo_cancellation_bit_exact_unittest.cc @@ -49,8 +49,11 @@ void ProcessOneFrame(int sample_rate_hz, capture_audio_buffer->SplitIntoFrequencyBands(); } - echo_canceller->ProcessRenderAudio(render_audio_buffer); - echo_canceller->ReadQueuedRenderData(); + std::vector render_audio; + EchoCancellationImpl::PackRenderAudioBuffer( + render_audio_buffer, 1, render_audio_buffer->num_channels(), + &render_audio); + echo_canceller->ProcessRenderAudio(render_audio); if (drift_compensation_enabled) { static_cast(echo_canceller) diff --git a/webrtc/modules/audio_processing/echo_cancellation_impl.cc b/webrtc/modules/audio_processing/echo_cancellation_impl.cc index 18c815dc00..e06b2faa84 100644 --- a/webrtc/modules/audio_processing/echo_cancellation_impl.cc +++ b/webrtc/modules/audio_processing/echo_cancellation_impl.cc @@ -49,12 +49,6 @@ AudioProcessing::Error MapError(int err) { } } -// Maximum length that a frame of samples can have. -static const size_t kMaxAllowedValuesOfSamplesPerFrame = 160; -// Maximum number of frames to buffer in the render queue. -// TODO(peah): Decrease this once we properly handle hugely unbalanced -// reverse and forward call numbers. -static const size_t kMaxNumFramesToBuffer = 100; } // namespace struct EchoCancellationImpl::StreamProperties { @@ -113,94 +107,38 @@ EchoCancellationImpl::EchoCancellationImpl(rtc::CriticalSection* crit_render, delay_logging_enabled_(false), extended_filter_enabled_(false), delay_agnostic_enabled_(false), - aec3_enabled_(false), - render_queue_element_max_size_(0) { + aec3_enabled_(false) { RTC_DCHECK(crit_render); RTC_DCHECK(crit_capture); } -EchoCancellationImpl::~EchoCancellationImpl() {} +EchoCancellationImpl::~EchoCancellationImpl() = default; -int EchoCancellationImpl::ProcessRenderAudio(const AudioBuffer* audio) { - rtc::CritScope cs_render(crit_render_); - if (!enabled_) { - return AudioProcessing::kNoError; - } - - RTC_DCHECK(stream_properties_); - RTC_DCHECK_GE(160u, audio->num_frames_per_band()); - RTC_DCHECK_EQ(audio->num_channels(), - stream_properties_->num_reverse_channels); - RTC_DCHECK_GE(cancellers_.size(), stream_properties_->num_output_channels * - audio->num_channels()); - - int err = AudioProcessing::kNoError; - - // The ordering convention must be followed to pass to the correct AEC. - size_t handle_index = 0; - render_queue_buffer_.clear(); - for (size_t i = 0; i < stream_properties_->num_output_channels; i++) { - for (size_t j = 0; j < audio->num_channels(); j++) { - // Retrieve any error code produced by the buffering of the farend - // signal. - err = WebRtcAec_GetBufferFarendError( - cancellers_[handle_index++]->state(), - audio->split_bands_const_f(j)[kBand0To8kHz], - audio->num_frames_per_band()); - - if (err != AudioProcessing::kNoError) { - return MapError(err); // TODO(ajm): warning possible? - } - - // Buffer the samples in the render queue. - render_queue_buffer_.insert(render_queue_buffer_.end(), - audio->split_bands_const_f(j)[kBand0To8kHz], - (audio->split_bands_const_f(j)[kBand0To8kHz] + - audio->num_frames_per_band())); - } - } - - // Insert the samples into the queue. - if (!render_signal_queue_->Insert(&render_queue_buffer_)) { - // The data queue is full and needs to be emptied. - ReadQueuedRenderData(); - - // Retry the insert (should always work). - bool result = render_signal_queue_->Insert(&render_queue_buffer_); - RTC_DCHECK(result); - } - - return AudioProcessing::kNoError; -} - -// Read chunks of data that were received and queued on the render side from -// a queue. All the data chunks are buffered into the farend signal of the AEC. -void EchoCancellationImpl::ReadQueuedRenderData() { +void EchoCancellationImpl::ProcessRenderAudio( + rtc::ArrayView packed_render_audio) { rtc::CritScope cs_capture(crit_capture_); if (!enabled_) { return; } RTC_DCHECK(stream_properties_); - while (render_signal_queue_->Remove(&capture_queue_buffer_)) { - size_t handle_index = 0; - size_t buffer_index = 0; - const size_t num_frames_per_band = - capture_queue_buffer_.size() / - (stream_properties_->num_output_channels * - stream_properties_->num_reverse_channels); - for (size_t i = 0; i < stream_properties_->num_output_channels; i++) { - for (size_t j = 0; j < stream_properties_->num_reverse_channels; j++) { - WebRtcAec_BufferFarend(cancellers_[handle_index++]->state(), - &capture_queue_buffer_[buffer_index], - num_frames_per_band); + size_t handle_index = 0; + size_t buffer_index = 0; + const size_t num_frames_per_band = + packed_render_audio.size() / (stream_properties_->num_output_channels * + stream_properties_->num_reverse_channels); + for (size_t i = 0; i < stream_properties_->num_output_channels; i++) { + for (size_t j = 0; j < stream_properties_->num_reverse_channels; j++) { + WebRtcAec_BufferFarend(cancellers_[handle_index++]->state(), + &packed_render_audio[buffer_index], + num_frames_per_band); - buffer_index += num_frames_per_band; - } + buffer_index += num_frames_per_band; } } } + int EchoCancellationImpl::ProcessCaptureAudio(AudioBuffer* audio, int stream_delay_ms) { rtc::CritScope cs_capture(crit_capture_); @@ -480,9 +418,12 @@ void EchoCancellationImpl::Initialize(int sample_rate_hz, return; } - if (NumCancellersRequired() > cancellers_.size()) { + const size_t num_cancellers_required = + NumCancellersRequired(stream_properties_->num_output_channels, + stream_properties_->num_reverse_channels); + if (num_cancellers_required > cancellers_.size()) { const size_t cancellers_old_size = cancellers_.size(); - cancellers_.resize(NumCancellersRequired()); + cancellers_.resize(num_cancellers_required); for (size_t i = cancellers_old_size; i < cancellers_.size(); ++i) { cancellers_[i].reset(new Canceller()); @@ -494,8 +435,6 @@ void EchoCancellationImpl::Initialize(int sample_rate_hz, } Configure(); - - AllocateRenderQueue(); } int EchoCancellationImpl::GetSystemDelayInSamples() const { @@ -506,30 +445,24 @@ int EchoCancellationImpl::GetSystemDelayInSamples() const { WebRtcAec_aec_core(cancellers_[0]->state())); } -void EchoCancellationImpl::AllocateRenderQueue() { - const size_t new_render_queue_element_max_size = std::max( - static_cast(1), - kMaxAllowedValuesOfSamplesPerFrame * NumCancellersRequired()); +void EchoCancellationImpl::PackRenderAudioBuffer( + const AudioBuffer* audio, + size_t num_output_channels, + size_t num_channels, + std::vector* packed_buffer) { + RTC_DCHECK_GE(160u, audio->num_frames_per_band()); + RTC_DCHECK_EQ(num_channels, audio->num_channels()); - rtc::CritScope cs_render(crit_render_); - rtc::CritScope cs_capture(crit_capture_); - - // Reallocate the queue if the queue item size is too small to fit the - // data to put in the queue. - if (render_queue_element_max_size_ < new_render_queue_element_max_size) { - render_queue_element_max_size_ = new_render_queue_element_max_size; - - std::vector template_queue_element(render_queue_element_max_size_); - - render_signal_queue_.reset( - new SwapQueue, RenderQueueItemVerifier>( - kMaxNumFramesToBuffer, template_queue_element, - RenderQueueItemVerifier(render_queue_element_max_size_))); - - render_queue_buffer_.resize(render_queue_element_max_size_); - capture_queue_buffer_.resize(render_queue_element_max_size_); - } else { - render_signal_queue_->Clear(); + packed_buffer->clear(); + // The ordering convention must be followed to pass the correct data. + for (size_t i = 0; i < num_output_channels; i++) { + for (size_t j = 0; j < audio->num_channels(); j++) { + // Buffer the samples in the render queue. + packed_buffer->insert(packed_buffer->end(), + audio->split_bands_const_f(j)[kBand0To8kHz], + (audio->split_bands_const_f(j)[kBand0To8kHz] + + audio->num_frames_per_band())); + } } } @@ -573,10 +506,10 @@ int EchoCancellationImpl::Configure() { return error; } -size_t EchoCancellationImpl::NumCancellersRequired() const { - RTC_DCHECK(stream_properties_); - return stream_properties_->num_output_channels * - stream_properties_->num_reverse_channels; +size_t EchoCancellationImpl::NumCancellersRequired( + size_t num_output_channels, + size_t num_reverse_channels) { + return num_output_channels * num_reverse_channels; } } // namespace webrtc diff --git a/webrtc/modules/audio_processing/echo_cancellation_impl.h b/webrtc/modules/audio_processing/echo_cancellation_impl.h index bab5e54f4b..8a4ea71846 100644 --- a/webrtc/modules/audio_processing/echo_cancellation_impl.h +++ b/webrtc/modules/audio_processing/echo_cancellation_impl.h @@ -16,9 +16,7 @@ #include "webrtc/base/constructormagic.h" #include "webrtc/base/criticalsection.h" -#include "webrtc/base/swap_queue.h" #include "webrtc/modules/audio_processing/include/audio_processing.h" -#include "webrtc/modules/audio_processing/render_queue_item_verifier.h" namespace webrtc { @@ -30,7 +28,7 @@ class EchoCancellationImpl : public EchoCancellation { rtc::CriticalSection* crit_capture); ~EchoCancellationImpl() override; - int ProcessRenderAudio(const AudioBuffer* audio); + void ProcessRenderAudio(rtc::ArrayView packed_render_audio); int ProcessCaptureAudio(AudioBuffer* audio, int stream_delay_ms); // EchoCancellation implementation. @@ -50,13 +48,16 @@ class EchoCancellationImpl : public EchoCancellation { std::string GetExperimentsDescription(); bool is_refined_adaptive_filter_enabled() const; - // Reads render side data that has been queued on the render call. - // Called holding the capture lock. - void ReadQueuedRenderData(); - // Returns the system delay of the first AEC component. int GetSystemDelayInSamples() const; + static void PackRenderAudioBuffer(const AudioBuffer* audio, + size_t num_output_channels, + size_t num_channels, + std::vector* packed_buffer); + static size_t NumCancellersRequired(size_t num_output_channels, + size_t num_reverse_channels); + private: class Canceller; struct StreamProperties; @@ -79,8 +80,6 @@ class EchoCancellationImpl : public EchoCancellation { struct AecCore* aec_core() const override; - size_t NumCancellersRequired() const; - void AllocateRenderQueue(); int Configure(); @@ -100,15 +99,6 @@ class EchoCancellationImpl : public EchoCancellation { bool aec3_enabled_ GUARDED_BY(crit_capture_); bool refined_adaptive_filter_enabled_ GUARDED_BY(crit_capture_) = false; - size_t render_queue_element_max_size_ GUARDED_BY(crit_render_) - GUARDED_BY(crit_capture_); - std::vector render_queue_buffer_ GUARDED_BY(crit_render_); - std::vector capture_queue_buffer_ GUARDED_BY(crit_capture_); - - // Lock protection not needed. - std::unique_ptr, RenderQueueItemVerifier>> - render_signal_queue_; - std::vector> cancellers_; std::unique_ptr stream_properties_;