diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc index ace788efa0..12b4c94904 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.cc +++ b/webrtc/modules/audio_processing/audio_processing_impl.cc @@ -708,7 +708,6 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, // getters that need the capture lock held when being called. rtc::CritScope cs_capture(&crit_capture_); EmptyQueuedRenderAudio(); - public_submodules_->gain_control->ReadQueuedRenderData(); if (!src || !dest) { return kNullPointerError; @@ -766,106 +765,143 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, return kNoError; } -void AudioProcessingImpl::QueueRenderAudio(const AudioBuffer* audio) { +void AudioProcessingImpl::QueueRenderAudio(AudioBuffer* audio) { EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(), num_reverse_channels(), - &float_render_queue_buffer_); + &aec_render_queue_buffer_); RTC_DCHECK_GE(160u, audio->num_frames_per_band()); // Insert the samples into the queue. - if (!float_render_signal_queue_->Insert(&float_render_queue_buffer_)) { + if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) { // The data queue is full and needs to be emptied. EmptyQueuedRenderAudio(); // Retry the insert (should always work). - bool result = - float_render_signal_queue_->Insert(&float_render_queue_buffer_); + bool result = aec_render_signal_queue_->Insert(&aec_render_queue_buffer_); RTC_DCHECK(result); } EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(), num_reverse_channels(), - &int16_render_queue_buffer_); + &aecm_render_queue_buffer_); // Insert the samples into the queue. - if (!int16_render_signal_queue_->Insert(&int16_render_queue_buffer_)) { + if (!aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_)) { // The data queue is full and needs to be emptied. EmptyQueuedRenderAudio(); // Retry the insert (should always work). - bool result = - int16_render_signal_queue_->Insert(&int16_render_queue_buffer_); + bool result = aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_); RTC_DCHECK(result); } + + if (!constants_.use_experimental_agc) { + GainControlImpl::PackRenderAudioBuffer(audio, &agc_render_queue_buffer_); + // Insert the samples into the queue. + if (!agc_render_signal_queue_->Insert(&agc_render_queue_buffer_)) { + // The data queue is full and needs to be emptied. + EmptyQueuedRenderAudio(); + + // Retry the insert (should always work). + bool result = agc_render_signal_queue_->Insert(&agc_render_queue_buffer_); + RTC_DCHECK(result); + } + } } void AudioProcessingImpl::AllocateRenderQueue() { - const size_t new_float_render_queue_element_max_size = + const size_t new_aec_render_queue_element_max_size = std::max(static_cast(1), kMaxAllowedValuesOfSamplesPerFrame * EchoCancellationImpl::NumCancellersRequired( num_output_channels(), num_reverse_channels())); - const size_t new_int16_render_queue_element_max_size = + const size_t new_aecm_render_queue_element_max_size = std::max(static_cast(1), kMaxAllowedValuesOfSamplesPerFrame * EchoControlMobileImpl::NumCancellersRequired( num_output_channels(), num_reverse_channels())); + const size_t new_agc_render_queue_element_max_size = + std::max(static_cast(1), kMaxAllowedValuesOfSamplesPerFrame); + // Reallocate the queues if the queue item sizes are too small to fit the // data to put in the queues. - if (float_render_queue_element_max_size_ < - new_float_render_queue_element_max_size) { - float_render_queue_element_max_size_ = - new_float_render_queue_element_max_size; + if (aec_render_queue_element_max_size_ < + new_aec_render_queue_element_max_size) { + aec_render_queue_element_max_size_ = new_aec_render_queue_element_max_size; std::vector template_queue_element( - float_render_queue_element_max_size_); + aec_render_queue_element_max_size_); - float_render_signal_queue_.reset( + aec_render_signal_queue_.reset( new SwapQueue, RenderQueueItemVerifier>( kMaxNumFramesToBuffer, template_queue_element, RenderQueueItemVerifier( - float_render_queue_element_max_size_))); + aec_render_queue_element_max_size_))); - float_render_queue_buffer_.resize(float_render_queue_element_max_size_); - float_capture_queue_buffer_.resize(float_render_queue_element_max_size_); + aec_render_queue_buffer_.resize(aec_render_queue_element_max_size_); + aec_capture_queue_buffer_.resize(aec_render_queue_element_max_size_); } else { - float_render_signal_queue_->Clear(); + aec_render_signal_queue_->Clear(); } - if (int16_render_queue_element_max_size_ < - new_int16_render_queue_element_max_size) { - int16_render_queue_element_max_size_ = - new_int16_render_queue_element_max_size; + if (aecm_render_queue_element_max_size_ < + new_aecm_render_queue_element_max_size) { + aecm_render_queue_element_max_size_ = + new_aecm_render_queue_element_max_size; std::vector template_queue_element( - int16_render_queue_element_max_size_); + aecm_render_queue_element_max_size_); - int16_render_signal_queue_.reset( + aecm_render_signal_queue_.reset( new SwapQueue, RenderQueueItemVerifier>( kMaxNumFramesToBuffer, template_queue_element, RenderQueueItemVerifier( - int16_render_queue_element_max_size_))); + aecm_render_queue_element_max_size_))); - int16_render_queue_buffer_.resize(int16_render_queue_element_max_size_); - int16_capture_queue_buffer_.resize(int16_render_queue_element_max_size_); + aecm_render_queue_buffer_.resize(aecm_render_queue_element_max_size_); + aecm_capture_queue_buffer_.resize(aecm_render_queue_element_max_size_); } else { - int16_render_signal_queue_->Clear(); + aecm_render_signal_queue_->Clear(); + } + + if (agc_render_queue_element_max_size_ < + new_agc_render_queue_element_max_size) { + agc_render_queue_element_max_size_ = new_agc_render_queue_element_max_size; + + std::vector template_queue_element( + agc_render_queue_element_max_size_); + + agc_render_signal_queue_.reset( + new SwapQueue, RenderQueueItemVerifier>( + kMaxNumFramesToBuffer, template_queue_element, + RenderQueueItemVerifier( + agc_render_queue_element_max_size_))); + + agc_render_queue_buffer_.resize(agc_render_queue_element_max_size_); + agc_capture_queue_buffer_.resize(agc_render_queue_element_max_size_); + } else { + agc_render_signal_queue_->Clear(); } } void AudioProcessingImpl::EmptyQueuedRenderAudio() { rtc::CritScope cs_capture(&crit_capture_); - while (float_render_signal_queue_->Remove(&float_capture_queue_buffer_)) { + while (aec_render_signal_queue_->Remove(&aec_capture_queue_buffer_)) { public_submodules_->echo_cancellation->ProcessRenderAudio( - float_capture_queue_buffer_); + aec_capture_queue_buffer_); } - while (int16_render_signal_queue_->Remove(&int16_capture_queue_buffer_)) { + while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) { public_submodules_->echo_control_mobile->ProcessRenderAudio( - int16_capture_queue_buffer_); + aecm_capture_queue_buffer_); + } + + while (agc_render_signal_queue_->Remove(&agc_capture_queue_buffer_)) { + public_submodules_->gain_control->ProcessRenderAudio( + agc_capture_queue_buffer_); } } @@ -880,7 +916,6 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { // as well. rtc::CritScope cs_capture(&crit_capture_); EmptyQueuedRenderAudio(); - public_submodules_->gain_control->ReadQueuedRenderData(); } if (!frame) { @@ -1242,10 +1277,6 @@ int AudioProcessingImpl::ProcessRenderStreamLocked() { #endif QueueRenderAudio(render_buffer); - if (!constants_.use_experimental_agc) { - RETURN_ON_ERR( - public_submodules_->gain_control->ProcessRenderAudio(render_buffer)); - } if (submodule_states_.RenderMultiBandProcessingActive() && SampleRateSupportsMultiBand( diff --git a/webrtc/modules/audio_processing/audio_processing_impl.h b/webrtc/modules/audio_processing/audio_processing_impl.h index 817b6b2fe8..aec4ed7086 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.h +++ b/webrtc/modules/audio_processing/audio_processing_impl.h @@ -238,7 +238,7 @@ class AudioProcessingImpl : public AudioProcessing { void EmptyQueuedRenderAudio(); void AllocateRenderQueue() EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_); - void QueueRenderAudio(const AudioBuffer* audio) + void QueueRenderAudio(AudioBuffer* audio) EXCLUSIVE_LOCKS_REQUIRED(crit_render_); // Capture-side exclusive methods possibly running APM in a multi-threaded @@ -371,22 +371,30 @@ class AudioProcessingImpl : public AudioProcessing { std::unique_ptr render_audio; } render_ GUARDED_BY(crit_render_); - size_t float_render_queue_element_max_size_ GUARDED_BY(crit_render_) + size_t aec_render_queue_element_max_size_ GUARDED_BY(crit_render_) GUARDED_BY(crit_capture_) = 0; - std::vector float_render_queue_buffer_ GUARDED_BY(crit_render_); - std::vector float_capture_queue_buffer_ GUARDED_BY(crit_capture_); + std::vector aec_render_queue_buffer_ GUARDED_BY(crit_render_); + std::vector aec_capture_queue_buffer_ GUARDED_BY(crit_capture_); - size_t int16_render_queue_element_max_size_ GUARDED_BY(crit_render_) + size_t aecm_render_queue_element_max_size_ GUARDED_BY(crit_render_) GUARDED_BY(crit_capture_) = 0; - std::vector int16_render_queue_buffer_ GUARDED_BY(crit_render_); - std::vector int16_capture_queue_buffer_ GUARDED_BY(crit_capture_); + std::vector aecm_render_queue_buffer_ GUARDED_BY(crit_render_); + std::vector aecm_capture_queue_buffer_ GUARDED_BY(crit_capture_); + + size_t agc_render_queue_element_max_size_ GUARDED_BY(crit_render_) + GUARDED_BY(crit_capture_) = 0; + std::vector agc_render_queue_buffer_ GUARDED_BY(crit_render_); + std::vector agc_capture_queue_buffer_ GUARDED_BY(crit_capture_); // Lock protection not needed. std::unique_ptr, RenderQueueItemVerifier>> - float_render_signal_queue_; + aec_render_signal_queue_; std::unique_ptr< SwapQueue, RenderQueueItemVerifier>> - int16_render_signal_queue_; + aecm_render_signal_queue_; + std::unique_ptr< + SwapQueue, RenderQueueItemVerifier>> + agc_render_signal_queue_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/gain_control_impl.cc b/webrtc/modules/audio_processing/gain_control_impl.cc index aa4316de28..8f707fc3b5 100644 --- a/webrtc/modules/audio_processing/gain_control_impl.cc +++ b/webrtc/modules/audio_processing/gain_control_impl.cc @@ -33,13 +33,6 @@ int16_t MapSetting(GainControl::Mode mode) { return -1; } -// Maximum length that a frame of samples can have. -static const size_t kMaxAllowedValuesOfSamplesPerFrame = 160; -// Maximum number of frames to buffer in the render queue. -// TODO(peah): Decrease this once we properly handle hugely unbalanced -// reverse and forward call numbers. -static const size_t kMaxNumFramesToBuffer = 100; - } // namespace class GainControlImpl::GainController { @@ -103,74 +96,37 @@ GainControlImpl::GainControlImpl(rtc::CriticalSection* crit_render, compression_gain_db_(9), analog_capture_level_(0), was_analog_level_set_(false), - stream_is_saturated_(false), - render_queue_element_max_size_(0) { + stream_is_saturated_(false) { RTC_DCHECK(crit_render); RTC_DCHECK(crit_capture); } GainControlImpl::~GainControlImpl() {} -int GainControlImpl::ProcessRenderAudio(AudioBuffer* audio) { - rtc::CritScope cs(crit_render_); - if (!enabled_) { - return AudioProcessing::kNoError; - } - - RTC_DCHECK_GE(160u, audio->num_frames_per_band()); - - render_queue_buffer_.resize(0); - for (auto& gain_controller : gain_controllers_) { - int err = WebRtcAgc_GetAddFarendError(gain_controller->state(), - audio->num_frames_per_band()); - - if (err != AudioProcessing::kNoError) { - return AudioProcessing::kUnspecifiedError; - } - - // Buffer the samples in the render queue. - render_queue_buffer_.insert( - render_queue_buffer_.end(), audio->mixed_low_pass_data(), - (audio->mixed_low_pass_data() + audio->num_frames_per_band())); - } - - // Insert the samples into the queue. - if (!render_signal_queue_->Insert(&render_queue_buffer_)) { - // The data queue is full and needs to be emptied. - ReadQueuedRenderData(); - - // Retry the insert (should always work). - RTC_DCHECK_EQ(render_signal_queue_->Insert(&render_queue_buffer_), true); - } - - return AudioProcessing::kNoError; -} - -// Read chunks of data that were received and queued on the render side from -// a queue. All the data chunks are buffered into the farend signal of the AGC. -void GainControlImpl::ReadQueuedRenderData() { - rtc::CritScope cs(crit_capture_); - +void GainControlImpl::ProcessRenderAudio( + rtc::ArrayView packed_render_audio) { + rtc::CritScope cs_capture(crit_capture_); if (!enabled_) { return; } - while (render_signal_queue_->Remove(&capture_queue_buffer_)) { - size_t buffer_index = 0; - RTC_DCHECK(num_proc_channels_); - RTC_DCHECK_LT(0ul, *num_proc_channels_); - const size_t num_frames_per_band = - capture_queue_buffer_.size() / (*num_proc_channels_); - for (auto& gain_controller : gain_controllers_) { - WebRtcAgc_AddFarend(gain_controller->state(), - &capture_queue_buffer_[buffer_index], - num_frames_per_band); - - buffer_index += num_frames_per_band; - } + for (auto& gain_controller : gain_controllers_) { + WebRtcAgc_AddFarend(gain_controller->state(), packed_render_audio.data(), + packed_render_audio.size()); } } +void GainControlImpl::PackRenderAudioBuffer( + AudioBuffer* audio, + std::vector* packed_buffer) { + RTC_DCHECK_GE(160u, audio->num_frames_per_band()); + + packed_buffer->clear(); + packed_buffer->insert( + packed_buffer->end(), audio->mixed_low_pass_data(), + (audio->mixed_low_pass_data() + audio->num_frames_per_band())); +} + int GainControlImpl::AnalyzeCaptureAudio(AudioBuffer* audio) { rtc::CritScope cs(crit_capture_); @@ -447,33 +403,6 @@ void GainControlImpl::Initialize(size_t num_proc_channels, int sample_rate_hz) { } Configure(); - - AllocateRenderQueue(); -} - -void GainControlImpl::AllocateRenderQueue() { - rtc::CritScope cs_render(crit_render_); - rtc::CritScope cs_capture(crit_capture_); - - RTC_DCHECK(num_proc_channels_); - const size_t new_render_queue_element_max_size = std::max( - static_cast(1), - kMaxAllowedValuesOfSamplesPerFrame * (*num_proc_channels_)); - - if (render_queue_element_max_size_ < new_render_queue_element_max_size) { - render_queue_element_max_size_ = new_render_queue_element_max_size; - std::vector template_queue_element(render_queue_element_max_size_); - - render_signal_queue_.reset( - new SwapQueue, RenderQueueItemVerifier>( - kMaxNumFramesToBuffer, template_queue_element, - RenderQueueItemVerifier(render_queue_element_max_size_))); - - render_queue_buffer_.resize(render_queue_element_max_size_); - capture_queue_buffer_.resize(render_queue_element_max_size_); - } else { - render_signal_queue_->Clear(); - } } int GainControlImpl::Configure() { diff --git a/webrtc/modules/audio_processing/gain_control_impl.h b/webrtc/modules/audio_processing/gain_control_impl.h index 1af6d7b246..812b88cb8d 100644 --- a/webrtc/modules/audio_processing/gain_control_impl.h +++ b/webrtc/modules/audio_processing/gain_control_impl.h @@ -31,21 +31,21 @@ class GainControlImpl : public GainControl { rtc::CriticalSection* crit_capture); ~GainControlImpl() override; - int ProcessRenderAudio(AudioBuffer* audio); + void ProcessRenderAudio(rtc::ArrayView packed_render_audio); int AnalyzeCaptureAudio(AudioBuffer* audio); int ProcessCaptureAudio(AudioBuffer* audio, bool stream_has_echo); void Initialize(size_t num_proc_channels, int sample_rate_hz); + static void PackRenderAudioBuffer(AudioBuffer* audio, + std::vector* packed_buffer); + // GainControl implementation. bool is_enabled() const override; int stream_analog_level() override; bool is_limiter_enabled() const override; Mode mode() const override; - // Reads render side data that has been queued on the render call. - void ReadQueuedRenderData(); - int compression_gain_db() const override; private: @@ -64,7 +64,6 @@ class GainControlImpl : public GainControl { int analog_level_maximum() const override; bool stream_is_saturated() const override; - void AllocateRenderQueue(); int Configure(); rtc::CriticalSection* const crit_render_ ACQUIRED_BEFORE(crit_capture_); @@ -82,16 +81,6 @@ class GainControlImpl : public GainControl { bool was_analog_level_set_ GUARDED_BY(crit_capture_); bool stream_is_saturated_ GUARDED_BY(crit_capture_); - size_t render_queue_element_max_size_ GUARDED_BY(crit_render_) - GUARDED_BY(crit_capture_); - std::vector render_queue_buffer_ GUARDED_BY(crit_render_); - std::vector capture_queue_buffer_ GUARDED_BY(crit_capture_); - - // Lock protection not needed. - std::unique_ptr< - SwapQueue, RenderQueueItemVerifier>> - render_signal_queue_; - std::vector> gain_controllers_; rtc::Optional num_proc_channels_ GUARDED_BY(crit_capture_); diff --git a/webrtc/modules/audio_processing/gain_control_unittest.cc b/webrtc/modules/audio_processing/gain_control_unittest.cc index 989771ac69..1c89d76f52 100644 --- a/webrtc/modules/audio_processing/gain_control_unittest.cc +++ b/webrtc/modules/audio_processing/gain_control_unittest.cc @@ -30,8 +30,9 @@ void ProcessOneFrame(int sample_rate_hz, capture_audio_buffer->SplitIntoFrequencyBands(); } - gain_controller->ProcessRenderAudio(render_audio_buffer); - gain_controller->ReadQueuedRenderData(); + std::vector render_audio; + GainControlImpl::PackRenderAudioBuffer(render_audio_buffer, &render_audio); + gain_controller->ProcessRenderAudio(render_audio); gain_controller->AnalyzeCaptureAudio(capture_audio_buffer); gain_controller->ProcessCaptureAudio(capture_audio_buffer, false);