diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc index 93e43f7add..ace788efa0 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.cc +++ b/webrtc/modules/audio_processing/audio_processing_impl.cc @@ -708,7 +708,6 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, // getters that need the capture lock held when being called. rtc::CritScope cs_capture(&crit_capture_); EmptyQueuedRenderAudio(); - public_submodules_->echo_control_mobile->ReadQueuedRenderData(); public_submodules_->gain_control->ReadQueuedRenderData(); if (!src || !dest) { @@ -770,52 +769,103 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, void AudioProcessingImpl::QueueRenderAudio(const AudioBuffer* audio) { EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(), num_reverse_channels(), - &render_queue_buffer_); + &float_render_queue_buffer_); RTC_DCHECK_GE(160u, audio->num_frames_per_band()); // Insert the samples into the queue. - if (!render_signal_queue_->Insert(&render_queue_buffer_)) { + if (!float_render_signal_queue_->Insert(&float_render_queue_buffer_)) { // The data queue is full and needs to be emptied. EmptyQueuedRenderAudio(); // Retry the insert (should always work). - bool result = render_signal_queue_->Insert(&render_queue_buffer_); + bool result = + float_render_signal_queue_->Insert(&float_render_queue_buffer_); + RTC_DCHECK(result); + } + + EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(), + num_reverse_channels(), + &int16_render_queue_buffer_); + + // Insert the samples into the queue. + if (!int16_render_signal_queue_->Insert(&int16_render_queue_buffer_)) { + // The data queue is full and needs to be emptied. + EmptyQueuedRenderAudio(); + + // Retry the insert (should always work). + bool result = + int16_render_signal_queue_->Insert(&int16_render_queue_buffer_); RTC_DCHECK(result); } } void AudioProcessingImpl::AllocateRenderQueue() { - const size_t new_render_queue_element_max_size = + const size_t new_float_render_queue_element_max_size = std::max(static_cast(1), kMaxAllowedValuesOfSamplesPerFrame * EchoCancellationImpl::NumCancellersRequired( num_output_channels(), num_reverse_channels())); - // Reallocate the queue if the queue item size is too small to fit the - // data to put in the queue. - if (render_queue_element_max_size_ < new_render_queue_element_max_size) { - render_queue_element_max_size_ = new_render_queue_element_max_size; + const size_t new_int16_render_queue_element_max_size = + std::max(static_cast(1), + kMaxAllowedValuesOfSamplesPerFrame * + EchoControlMobileImpl::NumCancellersRequired( + num_output_channels(), num_reverse_channels())); - std::vector template_queue_element(render_queue_element_max_size_); + // Reallocate the queues if the queue item sizes are too small to fit the + // data to put in the queues. + if (float_render_queue_element_max_size_ < + new_float_render_queue_element_max_size) { + float_render_queue_element_max_size_ = + new_float_render_queue_element_max_size; - render_signal_queue_.reset( + std::vector template_queue_element( + float_render_queue_element_max_size_); + + float_render_signal_queue_.reset( new SwapQueue, RenderQueueItemVerifier>( kMaxNumFramesToBuffer, template_queue_element, - RenderQueueItemVerifier(render_queue_element_max_size_))); + RenderQueueItemVerifier( + float_render_queue_element_max_size_))); - render_queue_buffer_.resize(render_queue_element_max_size_); - capture_queue_buffer_.resize(render_queue_element_max_size_); + float_render_queue_buffer_.resize(float_render_queue_element_max_size_); + float_capture_queue_buffer_.resize(float_render_queue_element_max_size_); } else { - render_signal_queue_->Clear(); + float_render_signal_queue_->Clear(); + } + + if (int16_render_queue_element_max_size_ < + new_int16_render_queue_element_max_size) { + int16_render_queue_element_max_size_ = + new_int16_render_queue_element_max_size; + + std::vector template_queue_element( + int16_render_queue_element_max_size_); + + int16_render_signal_queue_.reset( + new SwapQueue, RenderQueueItemVerifier>( + kMaxNumFramesToBuffer, template_queue_element, + RenderQueueItemVerifier( + int16_render_queue_element_max_size_))); + + int16_render_queue_buffer_.resize(int16_render_queue_element_max_size_); + int16_capture_queue_buffer_.resize(int16_render_queue_element_max_size_); + } else { + int16_render_signal_queue_->Clear(); } } void AudioProcessingImpl::EmptyQueuedRenderAudio() { rtc::CritScope cs_capture(&crit_capture_); - while (render_signal_queue_->Remove(&capture_queue_buffer_)) { + while (float_render_signal_queue_->Remove(&float_capture_queue_buffer_)) { public_submodules_->echo_cancellation->ProcessRenderAudio( - capture_queue_buffer_); + float_capture_queue_buffer_); + } + + while (int16_render_signal_queue_->Remove(&int16_capture_queue_buffer_)) { + public_submodules_->echo_control_mobile->ProcessRenderAudio( + int16_capture_queue_buffer_); } } @@ -830,7 +880,6 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { // as well. rtc::CritScope cs_capture(&crit_capture_); EmptyQueuedRenderAudio(); - public_submodules_->echo_control_mobile->ReadQueuedRenderData(); public_submodules_->gain_control->ReadQueuedRenderData(); } @@ -1193,8 +1242,6 @@ int AudioProcessingImpl::ProcessRenderStreamLocked() { #endif QueueRenderAudio(render_buffer); - RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessRenderAudio( - render_buffer)); if (!constants_.use_experimental_agc) { RETURN_ON_ERR( public_submodules_->gain_control->ProcessRenderAudio(render_buffer)); diff --git a/webrtc/modules/audio_processing/audio_processing_impl.h b/webrtc/modules/audio_processing/audio_processing_impl.h index d6af64d22f..817b6b2fe8 100644 --- a/webrtc/modules/audio_processing/audio_processing_impl.h +++ b/webrtc/modules/audio_processing/audio_processing_impl.h @@ -371,14 +371,22 @@ class AudioProcessingImpl : public AudioProcessing { std::unique_ptr render_audio; } render_ GUARDED_BY(crit_render_); - size_t render_queue_element_max_size_ GUARDED_BY(crit_render_) + size_t float_render_queue_element_max_size_ GUARDED_BY(crit_render_) GUARDED_BY(crit_capture_) = 0; - std::vector render_queue_buffer_ GUARDED_BY(crit_render_); - std::vector capture_queue_buffer_ GUARDED_BY(crit_capture_); + std::vector float_render_queue_buffer_ GUARDED_BY(crit_render_); + std::vector float_capture_queue_buffer_ GUARDED_BY(crit_capture_); + + size_t int16_render_queue_element_max_size_ GUARDED_BY(crit_render_) + GUARDED_BY(crit_capture_) = 0; + std::vector int16_render_queue_buffer_ GUARDED_BY(crit_render_); + std::vector int16_capture_queue_buffer_ GUARDED_BY(crit_capture_); // Lock protection not needed. std::unique_ptr, RenderQueueItemVerifier>> - render_signal_queue_; + float_render_signal_queue_; + std::unique_ptr< + SwapQueue, RenderQueueItemVerifier>> + int16_render_signal_queue_; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/echo_control_mobile_impl.cc b/webrtc/modules/audio_processing/echo_control_mobile_impl.cc index 0c62648e70..cfc4249804 100644 --- a/webrtc/modules/audio_processing/echo_control_mobile_impl.cc +++ b/webrtc/modules/audio_processing/echo_control_mobile_impl.cc @@ -53,12 +53,6 @@ AudioProcessing::Error MapError(int err) { return AudioProcessing::kUnspecifiedError; } } -// Maximum length that a frame of samples can have. -static const size_t kMaxAllowedValuesOfSamplesPerFrame = 160; -// Maximum number of frames to buffer in the render queue. -// TODO(peah): Decrease this once we properly handle hugely unbalanced -// reverse and forward call numbers. -static const size_t kMaxNumFramesToBuffer = 100; } // namespace size_t EchoControlMobile::echo_path_size_bytes() { @@ -120,8 +114,7 @@ EchoControlMobileImpl::EchoControlMobileImpl(rtc::CriticalSection* crit_render, crit_capture_(crit_capture), routing_mode_(kSpeakerphone), comfort_noise_enabled_(true), - external_echo_path_(NULL), - render_queue_element_max_size_(0) { + external_echo_path_(NULL) { RTC_DCHECK(crit_render); RTC_DCHECK(crit_capture); } @@ -133,80 +126,59 @@ EchoControlMobileImpl::~EchoControlMobileImpl() { } } -int EchoControlMobileImpl::ProcessRenderAudio(const AudioBuffer* audio) { - rtc::CritScope cs_render(crit_render_); - if (!enabled_) { - return AudioProcessing::kNoError; - } - - RTC_DCHECK(stream_properties_); - RTC_DCHECK_GE(160u, audio->num_frames_per_band()); - RTC_DCHECK_EQ(audio->num_channels(), - stream_properties_->num_reverse_channels); - RTC_DCHECK_GE(cancellers_.size(), stream_properties_->num_output_channels * - audio->num_channels()); - - int err = AudioProcessing::kNoError; - // The ordering convention must be followed to pass to the correct AECM. - render_queue_buffer_.clear(); - int render_channel = 0; - for (auto& canceller : cancellers_) { - err = WebRtcAecm_GetBufferFarendError( - canceller->state(), - audio->split_bands_const(render_channel)[kBand0To8kHz], - audio->num_frames_per_band()); - - if (err != AudioProcessing::kNoError) - return MapError(err); // TODO(ajm): warning possible?); - - // Buffer the samples in the render queue. - render_queue_buffer_.insert( - render_queue_buffer_.end(), - audio->split_bands_const(render_channel)[kBand0To8kHz], - (audio->split_bands_const(render_channel)[kBand0To8kHz] + - audio->num_frames_per_band())); - render_channel = (render_channel + 1) % audio->num_channels(); - } - - // Insert the samples into the queue. - if (!render_signal_queue_->Insert(&render_queue_buffer_)) { - // The data queue is full and needs to be emptied. - ReadQueuedRenderData(); - - // Retry the insert (should always work). - bool result = render_signal_queue_->Insert(&render_queue_buffer_); - RTC_DCHECK(result); - } - - return AudioProcessing::kNoError; -} - -// Read chunks of data that were received and queued on the render side from -// a queue. All the data chunks are buffered into the farend signal of the AEC. -void EchoControlMobileImpl::ReadQueuedRenderData() { +void EchoControlMobileImpl::ProcessRenderAudio( + rtc::ArrayView packed_render_audio) { rtc::CritScope cs_capture(crit_capture_); - RTC_DCHECK(stream_properties_); - if (!enabled_) { return; } - while (render_signal_queue_->Remove(&capture_queue_buffer_)) { - size_t buffer_index = 0; - size_t num_frames_per_band = capture_queue_buffer_.size() / - (stream_properties_->num_output_channels * - stream_properties_->num_reverse_channels); + RTC_DCHECK(stream_properties_); - for (auto& canceller : cancellers_) { - WebRtcAecm_BufferFarend(canceller->state(), - &capture_queue_buffer_[buffer_index], - num_frames_per_band); + size_t buffer_index = 0; + size_t num_frames_per_band = + packed_render_audio.size() / (stream_properties_->num_output_channels * + stream_properties_->num_reverse_channels); - buffer_index += num_frames_per_band; + for (auto& canceller : cancellers_) { + WebRtcAecm_BufferFarend(canceller->state(), + &packed_render_audio[buffer_index], + num_frames_per_band); + + buffer_index += num_frames_per_band; + } +} + +void EchoControlMobileImpl::PackRenderAudioBuffer( + const AudioBuffer* audio, + size_t num_output_channels, + size_t num_channels, + std::vector* packed_buffer) { + RTC_DCHECK_GE(160u, audio->num_frames_per_band()); + RTC_DCHECK_EQ(num_channels, audio->num_channels()); + + // The ordering convention must be followed to pass to the correct AECM. + packed_buffer->clear(); + int render_channel = 0; + for (size_t i = 0; i < num_output_channels; i++) { + for (size_t j = 0; j < audio->num_channels(); j++) { + // Buffer the samples in the render queue. + packed_buffer->insert( + packed_buffer->end(), + audio->split_bands_const(render_channel)[kBand0To8kHz], + (audio->split_bands_const(render_channel)[kBand0To8kHz] + + audio->num_frames_per_band())); + render_channel = (render_channel + 1) % audio->num_channels(); } } } +size_t EchoControlMobileImpl::NumCancellersRequired( + size_t num_output_channels, + size_t num_reverse_channels) { + return num_output_channels * num_reverse_channels; +} + int EchoControlMobileImpl::ProcessCaptureAudio(AudioBuffer* audio, int stream_delay_ms) { rtc::CritScope cs_capture(crit_capture_); @@ -385,7 +357,10 @@ void EchoControlMobileImpl::Initialize(int sample_rate_hz, LOG(LS_ERROR) << "AECM only supports 16 kHz or lower sample rates"; } - cancellers_.resize(num_handles_required()); + cancellers_.resize( + NumCancellersRequired(stream_properties_->num_output_channels, + stream_properties_->num_reverse_channels)); + for (auto& canceller : cancellers_) { if (!canceller) { canceller.reset(new Canceller()); @@ -395,35 +370,6 @@ void EchoControlMobileImpl::Initialize(int sample_rate_hz, } Configure(); - - AllocateRenderQueue(); -} - -void EchoControlMobileImpl::AllocateRenderQueue() { - const size_t new_render_queue_element_max_size = std::max( - static_cast(1), - kMaxAllowedValuesOfSamplesPerFrame * num_handles_required()); - - rtc::CritScope cs_render(crit_render_); - rtc::CritScope cs_capture(crit_capture_); - - // Reallocate the queue if the queue item size is too small to fit the - // data to put in the queue. - if (render_queue_element_max_size_ < new_render_queue_element_max_size) { - render_queue_element_max_size_ = new_render_queue_element_max_size; - - std::vector template_queue_element(render_queue_element_max_size_); - - render_signal_queue_.reset( - new SwapQueue, RenderQueueItemVerifier>( - kMaxNumFramesToBuffer, template_queue_element, - RenderQueueItemVerifier(render_queue_element_max_size_))); - - render_queue_buffer_.resize(render_queue_element_max_size_); - capture_queue_buffer_.resize(render_queue_element_max_size_); - } else { - render_signal_queue_->Clear(); - } } int EchoControlMobileImpl::Configure() { @@ -442,9 +388,4 @@ int EchoControlMobileImpl::Configure() { return error; } -size_t EchoControlMobileImpl::num_handles_required() const { - RTC_DCHECK(stream_properties_); - return stream_properties_->num_output_channels * - stream_properties_->num_reverse_channels; -} } // namespace webrtc diff --git a/webrtc/modules/audio_processing/echo_control_mobile_impl.h b/webrtc/modules/audio_processing/echo_control_mobile_impl.h index dc1b72c559..571de36add 100644 --- a/webrtc/modules/audio_processing/echo_control_mobile_impl.h +++ b/webrtc/modules/audio_processing/echo_control_mobile_impl.h @@ -31,7 +31,7 @@ class EchoControlMobileImpl : public EchoControlMobile { ~EchoControlMobileImpl() override; - int ProcessRenderAudio(const AudioBuffer* audio); + void ProcessRenderAudio(rtc::ArrayView packed_render_audio); int ProcessCaptureAudio(AudioBuffer* audio, int stream_delay_ms); // EchoControlMobile implementation. @@ -43,8 +43,13 @@ class EchoControlMobileImpl : public EchoControlMobile { size_t num_reverse_channels, size_t num_output_channels); - // Reads render side data that has been queued on the render call. - void ReadQueuedRenderData(); + static void PackRenderAudioBuffer(const AudioBuffer* audio, + size_t num_output_channels, + size_t num_channels, + std::vector* packed_buffer); + + static size_t NumCancellersRequired(size_t num_output_channels, + size_t num_reverse_channels); private: class Canceller; @@ -57,9 +62,6 @@ class EchoControlMobileImpl : public EchoControlMobile { int SetEchoPath(const void* echo_path, size_t size_bytes) override; int GetEchoPath(void* echo_path, size_t size_bytes) const override; - size_t num_handles_required() const; - - void AllocateRenderQueue(); int Configure(); rtc::CriticalSection* const crit_render_ ACQUIRED_BEFORE(crit_capture_); @@ -72,17 +74,6 @@ class EchoControlMobileImpl : public EchoControlMobile { unsigned char* external_echo_path_ GUARDED_BY(crit_render_) GUARDED_BY(crit_capture_); - size_t render_queue_element_max_size_ GUARDED_BY(crit_render_) - GUARDED_BY(crit_capture_); - - std::vector render_queue_buffer_ GUARDED_BY(crit_render_); - std::vector capture_queue_buffer_ GUARDED_BY(crit_capture_); - - // Lock protection not needed. - std::unique_ptr< - SwapQueue, RenderQueueItemVerifier>> - render_signal_queue_; - std::vector> cancellers_; std::unique_ptr stream_properties_; diff --git a/webrtc/modules/audio_processing/echo_control_mobile_unittest.cc b/webrtc/modules/audio_processing/echo_control_mobile_unittest.cc index b743c100c6..253421a13c 100644 --- a/webrtc/modules/audio_processing/echo_control_mobile_unittest.cc +++ b/webrtc/modules/audio_processing/echo_control_mobile_unittest.cc @@ -45,8 +45,12 @@ void ProcessOneFrame(int sample_rate_hz, capture_audio_buffer->SplitIntoFrequencyBands(); } - echo_control_mobile->ProcessRenderAudio(render_audio_buffer); - echo_control_mobile->ReadQueuedRenderData(); + std::vector render_audio; + EchoControlMobileImpl::PackRenderAudioBuffer( + render_audio_buffer, 1, render_audio_buffer->num_channels(), + &render_audio); + echo_control_mobile->ProcessRenderAudio(render_audio); + echo_control_mobile->ProcessCaptureAudio(capture_audio_buffer, stream_delay_ms);