Moved the AGC render sample queue into the audio processing module

Several subcomponents inside APM copy render audio from
the render side to the capture side using the same
pattern. Currently this is done independently for the
submodules.

This CL moves the the AGC functionality for this into
APM.

BUG=webrtc:5298, webrtc:6540

Review-Url: https://codereview.webrtc.org/2444283002
Cr-Commit-Position: refs/heads/master@{#14770}
This commit is contained in:
peah 2016-10-25 05:42:20 -07:00 committed by Commit bot
parent d8872c5907
commit 701d628f5f
5 changed files with 114 additions and 156 deletions

View File

@ -708,7 +708,6 @@ int AudioProcessingImpl::ProcessStream(const float* const* src,
// getters that need the capture lock held when being called.
rtc::CritScope cs_capture(&crit_capture_);
EmptyQueuedRenderAudio();
public_submodules_->gain_control->ReadQueuedRenderData();
if (!src || !dest) {
return kNullPointerError;
@ -766,106 +765,143 @@ int AudioProcessingImpl::ProcessStream(const float* const* src,
return kNoError;
}
void AudioProcessingImpl::QueueRenderAudio(const AudioBuffer* audio) {
void AudioProcessingImpl::QueueRenderAudio(AudioBuffer* audio) {
EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(),
num_reverse_channels(),
&float_render_queue_buffer_);
&aec_render_queue_buffer_);
RTC_DCHECK_GE(160u, audio->num_frames_per_band());
// Insert the samples into the queue.
if (!float_render_signal_queue_->Insert(&float_render_queue_buffer_)) {
if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) {
// The data queue is full and needs to be emptied.
EmptyQueuedRenderAudio();
// Retry the insert (should always work).
bool result =
float_render_signal_queue_->Insert(&float_render_queue_buffer_);
bool result = aec_render_signal_queue_->Insert(&aec_render_queue_buffer_);
RTC_DCHECK(result);
}
EchoControlMobileImpl::PackRenderAudioBuffer(audio, num_output_channels(),
num_reverse_channels(),
&int16_render_queue_buffer_);
&aecm_render_queue_buffer_);
// Insert the samples into the queue.
if (!int16_render_signal_queue_->Insert(&int16_render_queue_buffer_)) {
if (!aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_)) {
// The data queue is full and needs to be emptied.
EmptyQueuedRenderAudio();
// Retry the insert (should always work).
bool result =
int16_render_signal_queue_->Insert(&int16_render_queue_buffer_);
bool result = aecm_render_signal_queue_->Insert(&aecm_render_queue_buffer_);
RTC_DCHECK(result);
}
if (!constants_.use_experimental_agc) {
GainControlImpl::PackRenderAudioBuffer(audio, &agc_render_queue_buffer_);
// Insert the samples into the queue.
if (!agc_render_signal_queue_->Insert(&agc_render_queue_buffer_)) {
// The data queue is full and needs to be emptied.
EmptyQueuedRenderAudio();
// Retry the insert (should always work).
bool result = agc_render_signal_queue_->Insert(&agc_render_queue_buffer_);
RTC_DCHECK(result);
}
}
}
void AudioProcessingImpl::AllocateRenderQueue() {
const size_t new_float_render_queue_element_max_size =
const size_t new_aec_render_queue_element_max_size =
std::max(static_cast<size_t>(1),
kMaxAllowedValuesOfSamplesPerFrame *
EchoCancellationImpl::NumCancellersRequired(
num_output_channels(), num_reverse_channels()));
const size_t new_int16_render_queue_element_max_size =
const size_t new_aecm_render_queue_element_max_size =
std::max(static_cast<size_t>(1),
kMaxAllowedValuesOfSamplesPerFrame *
EchoControlMobileImpl::NumCancellersRequired(
num_output_channels(), num_reverse_channels()));
const size_t new_agc_render_queue_element_max_size =
std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerFrame);
// Reallocate the queues if the queue item sizes are too small to fit the
// data to put in the queues.
if (float_render_queue_element_max_size_ <
new_float_render_queue_element_max_size) {
float_render_queue_element_max_size_ =
new_float_render_queue_element_max_size;
if (aec_render_queue_element_max_size_ <
new_aec_render_queue_element_max_size) {
aec_render_queue_element_max_size_ = new_aec_render_queue_element_max_size;
std::vector<float> template_queue_element(
float_render_queue_element_max_size_);
aec_render_queue_element_max_size_);
float_render_signal_queue_.reset(
aec_render_signal_queue_.reset(
new SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>(
kMaxNumFramesToBuffer, template_queue_element,
RenderQueueItemVerifier<float>(
float_render_queue_element_max_size_)));
aec_render_queue_element_max_size_)));
float_render_queue_buffer_.resize(float_render_queue_element_max_size_);
float_capture_queue_buffer_.resize(float_render_queue_element_max_size_);
aec_render_queue_buffer_.resize(aec_render_queue_element_max_size_);
aec_capture_queue_buffer_.resize(aec_render_queue_element_max_size_);
} else {
float_render_signal_queue_->Clear();
aec_render_signal_queue_->Clear();
}
if (int16_render_queue_element_max_size_ <
new_int16_render_queue_element_max_size) {
int16_render_queue_element_max_size_ =
new_int16_render_queue_element_max_size;
if (aecm_render_queue_element_max_size_ <
new_aecm_render_queue_element_max_size) {
aecm_render_queue_element_max_size_ =
new_aecm_render_queue_element_max_size;
std::vector<int16_t> template_queue_element(
int16_render_queue_element_max_size_);
aecm_render_queue_element_max_size_);
int16_render_signal_queue_.reset(
aecm_render_signal_queue_.reset(
new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
kMaxNumFramesToBuffer, template_queue_element,
RenderQueueItemVerifier<int16_t>(
int16_render_queue_element_max_size_)));
aecm_render_queue_element_max_size_)));
int16_render_queue_buffer_.resize(int16_render_queue_element_max_size_);
int16_capture_queue_buffer_.resize(int16_render_queue_element_max_size_);
aecm_render_queue_buffer_.resize(aecm_render_queue_element_max_size_);
aecm_capture_queue_buffer_.resize(aecm_render_queue_element_max_size_);
} else {
int16_render_signal_queue_->Clear();
aecm_render_signal_queue_->Clear();
}
if (agc_render_queue_element_max_size_ <
new_agc_render_queue_element_max_size) {
agc_render_queue_element_max_size_ = new_agc_render_queue_element_max_size;
std::vector<int16_t> template_queue_element(
agc_render_queue_element_max_size_);
agc_render_signal_queue_.reset(
new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
kMaxNumFramesToBuffer, template_queue_element,
RenderQueueItemVerifier<int16_t>(
agc_render_queue_element_max_size_)));
agc_render_queue_buffer_.resize(agc_render_queue_element_max_size_);
agc_capture_queue_buffer_.resize(agc_render_queue_element_max_size_);
} else {
agc_render_signal_queue_->Clear();
}
}
void AudioProcessingImpl::EmptyQueuedRenderAudio() {
rtc::CritScope cs_capture(&crit_capture_);
while (float_render_signal_queue_->Remove(&float_capture_queue_buffer_)) {
while (aec_render_signal_queue_->Remove(&aec_capture_queue_buffer_)) {
public_submodules_->echo_cancellation->ProcessRenderAudio(
float_capture_queue_buffer_);
aec_capture_queue_buffer_);
}
while (int16_render_signal_queue_->Remove(&int16_capture_queue_buffer_)) {
while (aecm_render_signal_queue_->Remove(&aecm_capture_queue_buffer_)) {
public_submodules_->echo_control_mobile->ProcessRenderAudio(
int16_capture_queue_buffer_);
aecm_capture_queue_buffer_);
}
while (agc_render_signal_queue_->Remove(&agc_capture_queue_buffer_)) {
public_submodules_->gain_control->ProcessRenderAudio(
agc_capture_queue_buffer_);
}
}
@ -880,7 +916,6 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) {
// as well.
rtc::CritScope cs_capture(&crit_capture_);
EmptyQueuedRenderAudio();
public_submodules_->gain_control->ReadQueuedRenderData();
}
if (!frame) {
@ -1242,10 +1277,6 @@ int AudioProcessingImpl::ProcessRenderStreamLocked() {
#endif
QueueRenderAudio(render_buffer);
if (!constants_.use_experimental_agc) {
RETURN_ON_ERR(
public_submodules_->gain_control->ProcessRenderAudio(render_buffer));
}
if (submodule_states_.RenderMultiBandProcessingActive() &&
SampleRateSupportsMultiBand(

View File

@ -238,7 +238,7 @@ class AudioProcessingImpl : public AudioProcessing {
void EmptyQueuedRenderAudio();
void AllocateRenderQueue()
EXCLUSIVE_LOCKS_REQUIRED(crit_render_, crit_capture_);
void QueueRenderAudio(const AudioBuffer* audio)
void QueueRenderAudio(AudioBuffer* audio)
EXCLUSIVE_LOCKS_REQUIRED(crit_render_);
// Capture-side exclusive methods possibly running APM in a multi-threaded
@ -371,22 +371,30 @@ class AudioProcessingImpl : public AudioProcessing {
std::unique_ptr<AudioBuffer> render_audio;
} render_ GUARDED_BY(crit_render_);
size_t float_render_queue_element_max_size_ GUARDED_BY(crit_render_)
size_t aec_render_queue_element_max_size_ GUARDED_BY(crit_render_)
GUARDED_BY(crit_capture_) = 0;
std::vector<float> float_render_queue_buffer_ GUARDED_BY(crit_render_);
std::vector<float> float_capture_queue_buffer_ GUARDED_BY(crit_capture_);
std::vector<float> aec_render_queue_buffer_ GUARDED_BY(crit_render_);
std::vector<float> aec_capture_queue_buffer_ GUARDED_BY(crit_capture_);
size_t int16_render_queue_element_max_size_ GUARDED_BY(crit_render_)
size_t aecm_render_queue_element_max_size_ GUARDED_BY(crit_render_)
GUARDED_BY(crit_capture_) = 0;
std::vector<int16_t> int16_render_queue_buffer_ GUARDED_BY(crit_render_);
std::vector<int16_t> int16_capture_queue_buffer_ GUARDED_BY(crit_capture_);
std::vector<int16_t> aecm_render_queue_buffer_ GUARDED_BY(crit_render_);
std::vector<int16_t> aecm_capture_queue_buffer_ GUARDED_BY(crit_capture_);
size_t agc_render_queue_element_max_size_ GUARDED_BY(crit_render_)
GUARDED_BY(crit_capture_) = 0;
std::vector<int16_t> agc_render_queue_buffer_ GUARDED_BY(crit_render_);
std::vector<int16_t> agc_capture_queue_buffer_ GUARDED_BY(crit_capture_);
// Lock protection not needed.
std::unique_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>>
float_render_signal_queue_;
aec_render_signal_queue_;
std::unique_ptr<
SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
int16_render_signal_queue_;
aecm_render_signal_queue_;
std::unique_ptr<
SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
agc_render_signal_queue_;
};
} // namespace webrtc

View File

@ -33,13 +33,6 @@ int16_t MapSetting(GainControl::Mode mode) {
return -1;
}
// Maximum length that a frame of samples can have.
static const size_t kMaxAllowedValuesOfSamplesPerFrame = 160;
// Maximum number of frames to buffer in the render queue.
// TODO(peah): Decrease this once we properly handle hugely unbalanced
// reverse and forward call numbers.
static const size_t kMaxNumFramesToBuffer = 100;
} // namespace
class GainControlImpl::GainController {
@ -103,74 +96,37 @@ GainControlImpl::GainControlImpl(rtc::CriticalSection* crit_render,
compression_gain_db_(9),
analog_capture_level_(0),
was_analog_level_set_(false),
stream_is_saturated_(false),
render_queue_element_max_size_(0) {
stream_is_saturated_(false) {
RTC_DCHECK(crit_render);
RTC_DCHECK(crit_capture);
}
GainControlImpl::~GainControlImpl() {}
int GainControlImpl::ProcessRenderAudio(AudioBuffer* audio) {
rtc::CritScope cs(crit_render_);
if (!enabled_) {
return AudioProcessing::kNoError;
}
RTC_DCHECK_GE(160u, audio->num_frames_per_band());
render_queue_buffer_.resize(0);
for (auto& gain_controller : gain_controllers_) {
int err = WebRtcAgc_GetAddFarendError(gain_controller->state(),
audio->num_frames_per_band());
if (err != AudioProcessing::kNoError) {
return AudioProcessing::kUnspecifiedError;
}
// Buffer the samples in the render queue.
render_queue_buffer_.insert(
render_queue_buffer_.end(), audio->mixed_low_pass_data(),
(audio->mixed_low_pass_data() + audio->num_frames_per_band()));
}
// Insert the samples into the queue.
if (!render_signal_queue_->Insert(&render_queue_buffer_)) {
// The data queue is full and needs to be emptied.
ReadQueuedRenderData();
// Retry the insert (should always work).
RTC_DCHECK_EQ(render_signal_queue_->Insert(&render_queue_buffer_), true);
}
return AudioProcessing::kNoError;
}
// Read chunks of data that were received and queued on the render side from
// a queue. All the data chunks are buffered into the farend signal of the AGC.
void GainControlImpl::ReadQueuedRenderData() {
rtc::CritScope cs(crit_capture_);
void GainControlImpl::ProcessRenderAudio(
rtc::ArrayView<const int16_t> packed_render_audio) {
rtc::CritScope cs_capture(crit_capture_);
if (!enabled_) {
return;
}
while (render_signal_queue_->Remove(&capture_queue_buffer_)) {
size_t buffer_index = 0;
RTC_DCHECK(num_proc_channels_);
RTC_DCHECK_LT(0ul, *num_proc_channels_);
const size_t num_frames_per_band =
capture_queue_buffer_.size() / (*num_proc_channels_);
for (auto& gain_controller : gain_controllers_) {
WebRtcAgc_AddFarend(gain_controller->state(),
&capture_queue_buffer_[buffer_index],
num_frames_per_band);
buffer_index += num_frames_per_band;
}
for (auto& gain_controller : gain_controllers_) {
WebRtcAgc_AddFarend(gain_controller->state(), packed_render_audio.data(),
packed_render_audio.size());
}
}
void GainControlImpl::PackRenderAudioBuffer(
AudioBuffer* audio,
std::vector<int16_t>* packed_buffer) {
RTC_DCHECK_GE(160u, audio->num_frames_per_band());
packed_buffer->clear();
packed_buffer->insert(
packed_buffer->end(), audio->mixed_low_pass_data(),
(audio->mixed_low_pass_data() + audio->num_frames_per_band()));
}
int GainControlImpl::AnalyzeCaptureAudio(AudioBuffer* audio) {
rtc::CritScope cs(crit_capture_);
@ -447,33 +403,6 @@ void GainControlImpl::Initialize(size_t num_proc_channels, int sample_rate_hz) {
}
Configure();
AllocateRenderQueue();
}
void GainControlImpl::AllocateRenderQueue() {
rtc::CritScope cs_render(crit_render_);
rtc::CritScope cs_capture(crit_capture_);
RTC_DCHECK(num_proc_channels_);
const size_t new_render_queue_element_max_size = std::max<size_t>(
static_cast<size_t>(1),
kMaxAllowedValuesOfSamplesPerFrame * (*num_proc_channels_));
if (render_queue_element_max_size_ < new_render_queue_element_max_size) {
render_queue_element_max_size_ = new_render_queue_element_max_size;
std::vector<int16_t> template_queue_element(render_queue_element_max_size_);
render_signal_queue_.reset(
new SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>(
kMaxNumFramesToBuffer, template_queue_element,
RenderQueueItemVerifier<int16_t>(render_queue_element_max_size_)));
render_queue_buffer_.resize(render_queue_element_max_size_);
capture_queue_buffer_.resize(render_queue_element_max_size_);
} else {
render_signal_queue_->Clear();
}
}
int GainControlImpl::Configure() {

View File

@ -31,21 +31,21 @@ class GainControlImpl : public GainControl {
rtc::CriticalSection* crit_capture);
~GainControlImpl() override;
int ProcessRenderAudio(AudioBuffer* audio);
void ProcessRenderAudio(rtc::ArrayView<const int16_t> packed_render_audio);
int AnalyzeCaptureAudio(AudioBuffer* audio);
int ProcessCaptureAudio(AudioBuffer* audio, bool stream_has_echo);
void Initialize(size_t num_proc_channels, int sample_rate_hz);
static void PackRenderAudioBuffer(AudioBuffer* audio,
std::vector<int16_t>* packed_buffer);
// GainControl implementation.
bool is_enabled() const override;
int stream_analog_level() override;
bool is_limiter_enabled() const override;
Mode mode() const override;
// Reads render side data that has been queued on the render call.
void ReadQueuedRenderData();
int compression_gain_db() const override;
private:
@ -64,7 +64,6 @@ class GainControlImpl : public GainControl {
int analog_level_maximum() const override;
bool stream_is_saturated() const override;
void AllocateRenderQueue();
int Configure();
rtc::CriticalSection* const crit_render_ ACQUIRED_BEFORE(crit_capture_);
@ -82,16 +81,6 @@ class GainControlImpl : public GainControl {
bool was_analog_level_set_ GUARDED_BY(crit_capture_);
bool stream_is_saturated_ GUARDED_BY(crit_capture_);
size_t render_queue_element_max_size_ GUARDED_BY(crit_render_)
GUARDED_BY(crit_capture_);
std::vector<int16_t> render_queue_buffer_ GUARDED_BY(crit_render_);
std::vector<int16_t> capture_queue_buffer_ GUARDED_BY(crit_capture_);
// Lock protection not needed.
std::unique_ptr<
SwapQueue<std::vector<int16_t>, RenderQueueItemVerifier<int16_t>>>
render_signal_queue_;
std::vector<std::unique_ptr<GainController>> gain_controllers_;
rtc::Optional<size_t> num_proc_channels_ GUARDED_BY(crit_capture_);

View File

@ -30,8 +30,9 @@ void ProcessOneFrame(int sample_rate_hz,
capture_audio_buffer->SplitIntoFrequencyBands();
}
gain_controller->ProcessRenderAudio(render_audio_buffer);
gain_controller->ReadQueuedRenderData();
std::vector<int16_t> render_audio;
GainControlImpl::PackRenderAudioBuffer(render_audio_buffer, &render_audio);
gain_controller->ProcessRenderAudio(render_audio);
gain_controller->AnalyzeCaptureAudio(capture_audio_buffer);
gain_controller->ProcessCaptureAudio(capture_audio_buffer, false);