From 701d628f5f392a80436bd53cb118800b7845cb9d Mon Sep 17 00:00:00 2001 From: peah Date: Tue, 25 Oct 2016 05:42:20 -0700 Subject: [PATCH] Moved the AGC render sample queue into the audio processing module Several subcomponents inside APM copy render audio from the render side to the capture side using the same pattern. Currently this is done independently for the submodules. This CL moves the the AGC functionality for this into APM. BUG=webrtc:5298, webrtc:6540 Review-Url: https://codereview.webrtc.org/2444283002 Cr-Commit-Position: refs/heads/master@{#14770} --- .../audio_processing/audio_processing_impl.cc | 113 +++++++++++------- .../audio_processing/audio_processing_impl.h | 26 ++-- .../audio_processing/gain_control_impl.cc | 107 +++-------------- .../audio_processing/gain_control_impl.h | 19 +-- .../audio_processing/gain_control_unittest.cc | 5 +- 5 files changed, 114 insertions(+), 156 deletions(-) diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc index ace788efa0..12b4c94904 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.cc +++ b/webrtc/modules/audio_processing/audio_processing_impl.cc @@ -708,7 +708,6 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, // getters that need the capture lock held when being called. rtc::CritScope cs_capture(&crit_capture_); EmptyQueuedRenderAudio(); - public_submodules_->gain_control->ReadQueuedRenderData(); if (!src || !dest) { return kNullPointerError; @@ -766,106 +765,143 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, return kNoError; } -void AudioProcessingImpl::QueueRenderAudio(const AudioBuffer* audio) { +void AudioProcessingImpl::QueueRenderAudio(AudioBuffer* audio) { EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(), num_reverse_channels(), - &float_render_queue_buffer_); + &aec_render_queue_buffer_); RTC_DCHECK_GE(160u, audio->num_frames_per_band()); // Insert the samples into the queue. - if (!float_render_signal_queue_->Insert(&float_render_queue_buffer_)) { + if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) { // The data queue is full and needs to be emptied. EmptyQueuedRenderAudio(); // Retry the insert (should always work). - bool result = - float_render_signal_queue_->Insert(&float_render_queue_buffer_); + bool result = aec_render_signal_queue_->Insert(&aec_render_queue_buffer_); RTC_DCHECK(result); } EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(), num_reverse_channels(), - &int16_render_queue_buffer_); + &aecm_render_queue_buffer_); // Insert the samples into the queue. - if (!int16_render_signal_queue_->Insert(&int16_render_queue_buffer_)) { + if (!aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_)) { // The data queue is full and needs to be emptied. EmptyQueuedRenderAudio(); // Retry the insert (should always work). - bool result = - int16_render_signal_queue_->Insert(&int16_render_queue_buffer_); + bool result = aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_); RTC_DCHECK(result); } + + if (!constants_.use_experimental_agc) { + GainControlImpl::PackRenderAudioBuffer(audio, &agc_render_queue_buffer_); + // Insert the samples into the queue. + if (!agc_render_signal_queue_->Insert(&agc_render_queue_buffer_)) { + // The data queue is full and needs to be emptied. + EmptyQueuedRenderAudio(); + + // Retry the insert (should always work). + bool result = agc_render_signal_queue_->Insert(&agc_render_queue_buffer_); + RTC_DCHECK(result); + } + } } void AudioProcessingImpl::AllocateRenderQueue() { - const size_t new_float_render_queue_element_max_size = + const size_t new_aec_render_queue_element_max_size = std::max(static_cast(1), kMaxAllowedValuesOfSamplesPerFrame * EchoCancellationImpl::NumCancellersRequired( num_output_channels(), num_reverse_channels())); - const size_t new_int16_render_queue_element_max_size = + const size_t new_aecm_render_queue_element_max_size = std::max(static_cast(1), kMaxAllowedValuesOfSamplesPerFrame * EchoControlMobileImpl::NumCancellersRequired( num_output_channels(), num_reverse_channels())); + const size_t new_agc_render_queue_element_max_size = + std::max(static_cast(1), kMaxAllowedValuesOfSamplesPerFrame); + // Reallocate the queues if the queue item sizes are too small to fit the // data to put in the queues. - if (float_render_queue_element_max_size_ < - new_float_render_queue_element_max_size) { - float_render_queue_element_max_size_ = - new_float_render_queue_element_max_size; + if (aec_render_queue_element_max_size_ < + new_aec_render_queue_element_max_size) { + aec_render_queue_element_max_size_ = new_aec_render_queue_element_max_size; std::vector template_queue_element( - float_render_queue_element_max_size_); + aec_render_queue_element_max_size_); - float_render_signal_queue_.reset( + aec_render_signal_queue_.reset( new SwapQueue, RenderQueueItemVerifier>( kMaxNumFramesToBuffer, template_queue_element, RenderQueueItemVerifier( - float_render_queue_element_max_size_))); + aec_render_queue_element_max_size_))); - float_render_queue_buffer_.resize(float_render_queue_element_max_size_); - float_capture_queue_buffer_.resize(float_render_queue_element_max_size_); + aec_render_queue_buffer_.resize(aec_render_queue_element_max_size_); + aec_capture_queue_buffer_.resize(aec_render_queue_element_max_size_); } else { - float_render_signal_queue_->Clear(); + aec_render_signal_queue_->Clear(); } - if (int16_render_queue_element_max_size_ < - new_int16_render_queue_element_max_size) { - int16_render_queue_element_max_size_ = - new_int16_render_queue_element_max_size; + if (aecm_render_queue_element_max_size_ < + new_aecm_render_queue_element_max_size) { + aecm_render_queue_element_max_size_ = + new_aecm_render_queue_element_max_size; std::vector template_queue_element( - int16_render_queue_element_max_size_); + aecm_render_queue_element_max_size_); - int16_render_signal_queue_.reset( + aecm_render_signal_queue_.reset( new SwapQueue, RenderQueueItemVerifier>( kMaxNumFramesToBuffer, template_queue_element, RenderQueueItemVerifier( - int16_render_queue_element_max_size_))); + aecm_render_queue_element_max_size_))); - int16_render_queue_buffer_.resize(int16_render_queue_element_max_size_); - int16_capture_queue_buffer_.resize(int16_render_queue_element_max_size_); + aecm_render_queue_buffer_.resize(aecm_render_queue_element_max_size_); + aecm_capture_queue_buffer_.resize(aecm_render_queue_element_max_size_); } else { - int16_render_signal_queue_->Clear(); + aecm_render_signal_queue_->Clear(); + } + + if (agc_render_queue_element_max_size_ < + new_agc_render_queue_element_max_size) { + agc_render_queue_element_max_size_ = new_agc_render_queue_element_max_size; + + std::vector template_queue_element( + agc_render_queue_element_max_size_); + + agc_render_signal_queue_.reset( + new SwapQueue, RenderQueueItemVerifier>( + kMaxNumFramesToBuffer, template_queue_element, + RenderQueueItemVerifier( + agc_render_queue_element_max_size_))); + + agc_render_queue_buffer_.resize(agc_render_queue_element_max_size_); + agc_capture_queue_buffer_.resize(agc_render_queue_element_max_size_); + } else { + agc_render_signal_queue_->Clear(); } } void AudioProcessingImpl::EmptyQueuedRenderAudio() { rtc::CritScope cs_capture(&crit_capture_); - while (float_render_signal_queue_->Remove(&float_capture_queue_buffer_)) { + while (aec_render_signal_queue_->Remove(&aec_capture_queue_buffer_)) { public_submodules_->echo_cancellation->ProcessRenderAudio( - float_capture_queue_buffer_); + aec_capture_queue_buffer_); } - while (int16_render_signal_queue_->Remove(&int16_capture_queue_buffer_)) { + while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) { public_submodules_->echo_control_mobile->ProcessRenderAudio( - int16_capture_queue_buffer_); + aecm_capture_queue_buffer_); + } + + while (agc_render_signal_queue_->Remove(&agc_capture_queue_buffer_)) { + public_submodules_->gain_control->ProcessRenderAudio( + agc_capture_queue_buffer_); } } @@ -880,7 +916,6 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { // as well. rtc::CritScope cs_capture(&crit_capture_); EmptyQueuedRenderAudio(); - public_submodules_->gain_control->ReadQueuedRenderData(); } if (!frame) { @@ -1242,10 +1277,6 @@ int AudioProcessingImpl::ProcessRenderStreamLocked() { #endif QueueRenderAudio(render_buffer); - if (!constants_.use_experimental_agc) { - RETURN_ON_ERR( - public_submodules_->gain_control->ProcessRenderAudio(render_buffer)); - } if (submodule_states_.RenderMultiBandProcessingActive() && SampleRateSupportsMultiBand( diff --git a/webrtc/modules/audio_processing/audio_processing_impl.h b/webrtc/modules/audio_processing/audio_processing_impl.h index 817b6b2fe8..aec4ed7086 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.h +++ b/webrtc/modules/audio_processing/audio_processing_impl.h @@ -238,7 +238,7 @@ class AudioProcessingImpl : public AudioProcessing { void EmptyQueuedRenderAudio(); void AllocateRenderQueue() EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); - void QueueRenderAudio(const AudioBuffer* audio) + void QueueRenderAudio(AudioBuffer* audio) EXCLUSIVE_LOCKS_REQUIRED(crit_render_); // Capture-side exclusive methods possibly running APM in a multi-threaded @@ -371,22 +371,30 @@ class AudioProcessingImpl : public AudioProcessing { std::unique_ptr render_audio; } render_ GUARDED_BY(crit_render_); - size_t float_render_queue_element_max_size_ GUARDED_BY(crit_render_) + size_t aec_render_queue_element_max_size_ GUARDED_BY(crit_render_) GUARDED_BY(crit_capture_) = 0; - std::vector float_render_queue_buffer_ GUARDED_BY(crit_render_); - std::vector float_capture_queue_buffer_ GUARDED_BY(crit_capture_); + std::vector aec_render_queue_buffer_ GUARDED_BY(crit_render_); + std::vector aec_capture_queue_buffer_ GUARDED_BY(crit_capture_); - size_t int16_render_queue_element_max_size_ GUARDED_BY(crit_render_) + size_t aecm_render_queue_element_max_size_ GUARDED_BY(crit_render_) GUARDED_BY(crit_capture_) = 0; - std::vector int16_render_queue_buffer_ GUARDED_BY(crit_render_); - std::vector int16_capture_queue_buffer_ GUARDED_BY(crit_capture_); + std::vector aecm_render_queue_buffer_ GUARDED_BY(crit_render_); + std::vector aecm_capture_queue_buffer_ GUARDED_BY(crit_capture_); + + size_t agc_render_queue_element_max_size_ GUARDED_BY(crit_render_) + GUARDED_BY(crit_capture_) = 0; + std::vector agc_render_queue_buffer_ GUARDED_BY(crit_render_); + std::vector agc_capture_queue_buffer_ GUARDED_BY(crit_capture_); // Lock protection not needed. std::unique_ptr, RenderQueueItemVerifier>> - float_render_signal_queue_; + aec_render_signal_queue_; std::unique_ptr< SwapQueue, RenderQueueItemVerifier>> - int16_render_signal_queue_; + aecm_render_signal_queue_; + std::unique_ptr< + SwapQueue, RenderQueueItemVerifier>> + agc_render_signal_queue_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/gain_control_impl.cc b/webrtc/modules/audio_processing/gain_control_impl.cc index aa4316de28..8f707fc3b5 100644 --- a/webrtc/modules/audio_processing/gain_control_impl.cc +++ b/webrtc/modules/audio_processing/gain_control_impl.cc @@ -33,13 +33,6 @@ int16_t MapSetting(GainControl::Mode mode) { return -1; } -// Maximum length that a frame of samples can have. -static const size_t kMaxAllowedValuesOfSamplesPerFrame = 160; -// Maximum number of frames to buffer in the render queue. -// TODO(peah): Decrease this once we properly handle hugely unbalanced -// reverse and forward call numbers. -static const size_t kMaxNumFramesToBuffer = 100; - } // namespace class GainControlImpl::GainController { @@ -103,74 +96,37 @@ GainControlImpl::GainControlImpl(rtc::CriticalSection* crit_render, compression_gain_db_(9), analog_capture_level_(0), was_analog_level_set_(false), - stream_is_saturated_(false), - render_queue_element_max_size_(0) { + stream_is_saturated_(false) { RTC_DCHECK(crit_render); RTC_DCHECK(crit_capture); } GainControlImpl::~GainControlImpl() {} -int GainControlImpl::ProcessRenderAudio(AudioBuffer* audio) { - rtc::CritScope cs(crit_render_); - if (!enabled_) { - return AudioProcessing::kNoError; - } - - RTC_DCHECK_GE(160u, audio->num_frames_per_band()); - - render_queue_buffer_.resize(0); - for (auto& gain_controller : gain_controllers_) { - int err = WebRtcAgc_GetAddFarendError(gain_controller->state(), - audio->num_frames_per_band()); - - if (err != AudioProcessing::kNoError) { - return AudioProcessing::kUnspecifiedError; - } - - // Buffer the samples in the render queue. - render_queue_buffer_.insert( - render_queue_buffer_.end(), audio->mixed_low_pass_data(), - (audio->mixed_low_pass_data() + audio->num_frames_per_band())); - } - - // Insert the samples into the queue. - if (!render_signal_queue_->Insert(&render_queue_buffer_)) { - // The data queue is full and needs to be emptied. - ReadQueuedRenderData(); - - // Retry the insert (should always work). - RTC_DCHECK_EQ(render_signal_queue_->Insert(&render_queue_buffer_), true); - } - - return AudioProcessing::kNoError; -} - -// Read chunks of data that were received and queued on the render side from -// a queue. All the data chunks are buffered into the farend signal of the AGC. -void GainControlImpl::ReadQueuedRenderData() { - rtc::CritScope cs(crit_capture_); - +void GainControlImpl::ProcessRenderAudio( + rtc::ArrayView packed_render_audio) { + rtc::CritScope cs_capture(crit_capture_); if (!enabled_) { return; } - while (render_signal_queue_->Remove(&capture_queue_buffer_)) { - size_t buffer_index = 0; - RTC_DCHECK(num_proc_channels_); - RTC_DCHECK_LT(0ul, *num_proc_channels_); - const size_t num_frames_per_band = - capture_queue_buffer_.size() / (*num_proc_channels_); - for (auto& gain_controller : gain_controllers_) { - WebRtcAgc_AddFarend(gain_controller->state(), - &capture_queue_buffer_[buffer_index], - num_frames_per_band); - - buffer_index += num_frames_per_band; - } + for (auto& gain_controller : gain_controllers_) { + WebRtcAgc_AddFarend(gain_controller->state(), packed_render_audio.data(), + packed_render_audio.size()); } } +void GainControlImpl::PackRenderAudioBuffer( + AudioBuffer* audio, + std::vector* packed_buffer) { + RTC_DCHECK_GE(160u, audio->num_frames_per_band()); + + packed_buffer->clear(); + packed_buffer->insert( + packed_buffer->end(), audio->mixed_low_pass_data(), + (audio->mixed_low_pass_data() + audio->num_frames_per_band())); +} + int GainControlImpl::AnalyzeCaptureAudio(AudioBuffer* audio) { rtc::CritScope cs(crit_capture_); @@ -447,33 +403,6 @@ void GainControlImpl::Initialize(size_t num_proc_channels, int sample_rate_hz) { } Configure(); - - AllocateRenderQueue(); -} - -void GainControlImpl::AllocateRenderQueue() { - rtc::CritScope cs_render(crit_render_); - rtc::CritScope cs_capture(crit_capture_); - - RTC_DCHECK(num_proc_channels_); - const size_t new_render_queue_element_max_size = std::max( - static_cast(1), - kMaxAllowedValuesOfSamplesPerFrame * (*num_proc_channels_)); - - if (render_queue_element_max_size_ < new_render_queue_element_max_size) { - render_queue_element_max_size_ = new_render_queue_element_max_size; - std::vector template_queue_element(render_queue_element_max_size_); - - render_signal_queue_.reset( - new SwapQueue, RenderQueueItemVerifier>( - kMaxNumFramesToBuffer, template_queue_element, - RenderQueueItemVerifier(render_queue_element_max_size_))); - - render_queue_buffer_.resize(render_queue_element_max_size_); - capture_queue_buffer_.resize(render_queue_element_max_size_); - } else { - render_signal_queue_->Clear(); - } } int GainControlImpl::Configure() { diff --git a/webrtc/modules/audio_processing/gain_control_impl.h b/webrtc/modules/audio_processing/gain_control_impl.h index 1af6d7b246..812b88cb8d 100644 --- a/webrtc/modules/audio_processing/gain_control_impl.h +++ b/webrtc/modules/audio_processing/gain_control_impl.h @@ -31,21 +31,21 @@ class GainControlImpl : public GainControl { rtc::CriticalSection* crit_capture); ~GainControlImpl() override; - int ProcessRenderAudio(AudioBuffer* audio); + void ProcessRenderAudio(rtc::ArrayView packed_render_audio); int AnalyzeCaptureAudio(AudioBuffer* audio); int ProcessCaptureAudio(AudioBuffer* audio, bool stream_has_echo); void Initialize(size_t num_proc_channels, int sample_rate_hz); + static void PackRenderAudioBuffer(AudioBuffer* audio, + std::vector* packed_buffer); + // GainControl implementation. bool is_enabled() const override; int stream_analog_level() override; bool is_limiter_enabled() const override; Mode mode() const override; - // Reads render side data that has been queued on the render call. - void ReadQueuedRenderData(); - int compression_gain_db() const override; private: @@ -64,7 +64,6 @@ class GainControlImpl : public GainControl { int analog_level_maximum() const override; bool stream_is_saturated() const override; - void AllocateRenderQueue(); int Configure(); rtc::CriticalSection* const crit_render_ ACQUIRED_BEFORE(crit_capture_); @@ -82,16 +81,6 @@ class GainControlImpl : public GainControl { bool was_analog_level_set_ GUARDED_BY(crit_capture_); bool stream_is_saturated_ GUARDED_BY(crit_capture_); - size_t render_queue_element_max_size_ GUARDED_BY(crit_render_) - GUARDED_BY(crit_capture_); - std::vector render_queue_buffer_ GUARDED_BY(crit_render_); - std::vector capture_queue_buffer_ GUARDED_BY(crit_capture_); - - // Lock protection not needed. - std::unique_ptr< - SwapQueue, RenderQueueItemVerifier>> - render_signal_queue_; - std::vector> gain_controllers_; rtc::Optional num_proc_channels_ GUARDED_BY(crit_capture_); diff --git a/webrtc/modules/audio_processing/gain_control_unittest.cc b/webrtc/modules/audio_processing/gain_control_unittest.cc index 989771ac69..1c89d76f52 100644 --- a/webrtc/modules/audio_processing/gain_control_unittest.cc +++ b/webrtc/modules/audio_processing/gain_control_unittest.cc @@ -30,8 +30,9 @@ void ProcessOneFrame(int sample_rate_hz, capture_audio_buffer->SplitIntoFrequencyBands(); } - gain_controller->ProcessRenderAudio(render_audio_buffer); - gain_controller->ReadQueuedRenderData(); + std::vector render_audio; + GainControlImpl::PackRenderAudioBuffer(render_audio_buffer, &render_audio); + gain_controller->ProcessRenderAudio(render_audio); gain_controller->AnalyzeCaptureAudio(capture_audio_buffer); gain_controller->ProcessCaptureAudio(capture_audio_buffer, false);