admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AEC3: 'Block' class

Browse Source 
This change adds a Block class to reduce the need for std::vector<std::vector<std::vector<float>>>. This make the code
easier to read and less error prone.

It also enables future changes to the underlying data structure of a
block. For instance, the data of all bands and channels could be stored
in a single vector.

The change has been verified to be bit-exact.

Bug: webrtc:14089
Change-Id: Ied9a78124c0bbafe0e912017aef91f7c311de2ae
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/262252
Reviewed-by: Per Åhgren <peah@webrtc.org>
Commit-Queue: Gustaf Ullberg <gustaf@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#36968}
This commit is contained in:
Gustaf Ullberg 2022-05-23 10:39:53 +02:00 committed by WebRTC LUCI CQ
parent 742714870a
commit d3ead1a942
62 changed files with 663 additions and 937 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/audio_processing/aec3/BUILD.gn
View File

@ -22,6 +22,7 @@ rtc_library("aec3") {
"alignment_mixer.h",
"api_call_jitter_metrics.cc",
"api_call_jitter_metrics.h",
"block.h",
"block_buffer.cc",
"block_delay_buffer.cc",
"block_delay_buffer.h",
@ -182,6 +183,7 @@ rtc_source_set("aec3_fft") {
rtc_source_set("render_buffer") {
sources = [
"block.h",
"block_buffer.h",
"fft_buffer.h",
"render_buffer.h",

45
modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc
View File

@ -73,10 +73,7 @@ TEST_P(AdaptiveFirFilterOneTwoFourEightRenderChannels,
RenderDelayBuffer::Create(EchoCanceller3Config(), kSampleRateHz,
num_render_channels));
Random random_generator(42U);
std::vector<std::vector<std::vector<float>>> x(
kNumBands,
std::vector<std::vector<float>>(num_render_channels,
std::vector<float>(kBlockSize, 0.f)));
Block x(kNumBands, num_render_channels);
FftData S_C;
FftData S_Neon;
FftData G;
@ -92,10 +89,10 @@ TEST_P(AdaptiveFirFilterOneTwoFourEightRenderChannels,
}
}
for (size_t k = 0; k < 30; ++k) {
for (size_t band = 0; band < x.size(); ++band) {
for (size_t ch = 0; ch < x[band].size(); ++ch) {
RandomizeSampleVector(&random_generator, x[band][ch]);
for (int k = 0; k < 30; ++k) {
for (int band = 0; band < x.NumBands(); ++band) {
for (int ch = 0; ch < x.NumChannels(); ++ch) {
RandomizeSampleVector(&random_generator, x.View(band, ch));
}
}
render_delay_buffer->Insert(x);
@ -186,10 +183,7 @@ TEST_P(AdaptiveFirFilterOneTwoFourEightRenderChannels,
RenderDelayBuffer::Create(EchoCanceller3Config(), kSampleRateHz,
num_render_channels));
Random random_generator(42U);
std::vector<std::vector<std::vector<float>>> x(
kNumBands,
std::vector<std::vector<float>>(num_render_channels,
std::vector<float>(kBlockSize, 0.f)));
Block x(kNumBands, num_render_channels);
FftData S_C;
FftData S_Sse2;
FftData G;
@ -206,9 +200,9 @@ TEST_P(AdaptiveFirFilterOneTwoFourEightRenderChannels,
}
for (size_t k = 0; k < 500; ++k) {
for (size_t band = 0; band < x.size(); ++band) {
for (size_t ch = 0; ch < x[band].size(); ++ch) {
RandomizeSampleVector(&random_generator, x[band][ch]);
for (int band = 0; band < x.NumBands(); ++band) {
for (int ch = 0; ch < x.NumChannels(); ++ch) {
RandomizeSampleVector(&random_generator, x.View(band, ch));
}
}
render_delay_buffer->Insert(x);
@ -261,10 +255,7 @@ TEST_P(AdaptiveFirFilterOneTwoFourEightRenderChannels,
RenderDelayBuffer::Create(EchoCanceller3Config(), kSampleRateHz,
num_render_channels));
Random random_generator(42U);
std::vector<std::vector<std::vector<float>>> x(
kNumBands,
std::vector<std::vector<float>>(num_render_channels,
std::vector<float>(kBlockSize, 0.f)));
Block x(kNumBands, num_render_channels);
FftData S_C;
FftData S_Avx2;
FftData G;
@ -281,9 +272,9 @@ TEST_P(AdaptiveFirFilterOneTwoFourEightRenderChannels,
}
for (size_t k = 0; k < 500; ++k) {
for (size_t band = 0; band < x.size(); ++band) {
for (size_t ch = 0; ch < x[band].size(); ++ch) {
RandomizeSampleVector(&random_generator, x[band][ch]);
for (int band = 0; band < x.NumBands(); ++band) {
for (int ch = 0; ch < x.NumChannels(); ++ch) {
RandomizeSampleVector(&random_generator, x.View(band, ch));
}
}
render_delay_buffer->Insert(x);
@ -488,9 +479,7 @@ TEST_P(AdaptiveFirFilterMultiChannel, FilterAndAdapt) {
CoarseFilterUpdateGain gain(config.filter.coarse,
config.filter.config_change_duration_blocks);
Random random_generator(42U);
std::vector<std::vector<std::vector<float>>> x(
kNumBands, std::vector<std::vector<float>>(
num_render_channels, std::vector<float>(kBlockSize, 0.f)));
Block x(kNumBands, num_render_channels);
std::vector<float> n(kBlockSize, 0.f);
std::vector<float> y(kBlockSize, 0.f);
AecState aec_state(EchoCanceller3Config{}, num_capture_channels);
@ -540,9 +529,9 @@ TEST_P(AdaptiveFirFilterMultiChannel, FilterAndAdapt) {
for (size_t j = 0; j < num_blocks_to_process; ++j) {
std::fill(y.begin(), y.end(), 0.f);
for (size_t ch = 0; ch < num_render_channels; ++ch) {
RandomizeSampleVector(&random_generator, x[0][ch]);
RandomizeSampleVector(&random_generator, x.View(/*band=*/0, ch));
std::array<float, kBlockSize> y_channel;
delay_buffer[ch].Delay(x[0][ch], y_channel);
delay_buffer[ch].Delay(x.View(/*band=*/0, ch), y_channel);
for (size_t k = 0; k < y.size(); ++k) {
y[k] += y_channel[k] / num_render_channels;
}
@ -555,7 +544,7 @@ TEST_P(AdaptiveFirFilterMultiChannel, FilterAndAdapt) {
}
for (size_t ch = 0; ch < num_render_channels; ++ch) {
x_hp_filter[ch]->Process(x[0][ch]);
x_hp_filter[ch]->Process(x.View(/*band=*/0, ch));
}
y_hp_filter.Process(y);

20
modules/audio_processing/aec3/aec_state.cc
View File

@ -206,15 +206,16 @@ void AecState::Update(
strong_not_saturated_render_blocks_);
}
const std::vector<std::vector<float>>& aligned_render_block =
render_buffer.Block(-delay_state_.MinDirectPathFilterDelay())[0];
const Block& aligned_render_block =
render_buffer.Block(-delay_state_.MinDirectPathFilterDelay());
// Update render counters.
bool active_render = false;
for (size_t ch = 0; ch < aligned_render_block.size(); ++ch) {
const float render_energy = std::inner_product(
aligned_render_block[ch].begin(), aligned_render_block[ch].end(),
aligned_render_block[ch].begin(), 0.f);
for (int ch = 0; ch < aligned_render_block.NumChannels(); ++ch) {
const float render_energy =
std::inner_product(aligned_render_block.begin(/*block=*/0, ch),
aligned_render_block.end(/*block=*/0, ch),
aligned_render_block.begin(/*block=*/0, ch), 0.f);
if (render_energy > (config_.render_levels.active_render_limit *
config_.render_levels.active_render_limit) *
kFftLengthBy2) {
@ -446,7 +447,7 @@ void AecState::FilteringQualityAnalyzer::Update(
}
void AecState::SaturationDetector::Update(
rtc::ArrayView<const std::vector<float>> x,
const Block& x,
bool saturated_capture,
bool usable_linear_estimate,
rtc::ArrayView<const SubtractorOutput> subtractor_output,
@ -466,8 +467,9 @@ void AecState::SaturationDetector::Update(
}
} else {
float max_sample = 0.f;
for (auto& channel : x) {
for (float sample : channel) {
for (int ch = 0; ch < x.NumChannels(); ++ch) {
rtc::ArrayView<const float, kBlockSize> x_ch = x.View(/*band=*/0, ch);
for (float sample : x_ch) {
max_sample = std::max(max_sample, fabsf(sample));
}
}

2
modules/audio_processing/aec3/aec_state.h
View File

@ -272,7 +272,7 @@ class AecState {
bool SaturatedEcho() const { return saturated_echo_; }
// Updates the detection decision based on new data.
void Update(rtc::ArrayView<const std::vector<float>> x,
void Update(const Block& x,
bool saturated_capture,
bool usable_linear_estimate,
rtc::ArrayView<const SubtractorOutput> subtractor_output,

28
modules/audio_processing/aec3/aec_state_unittest.cc
View File

@ -35,9 +35,7 @@ void RunNormalUsageTest(size_t num_render_channels,
std::vector<std::array<float, kFftLengthBy2Plus1>> E2_refined(
num_capture_channels);
std::vector<std::array<float, kFftLengthBy2Plus1>> Y2(num_capture_channels);
std::vector<std::vector<std::vector<float>>> x(
kNumBands, std::vector<std::vector<float>>(
num_render_channels, std::vector<float>(kBlockSize, 0.f)));
Block x(kNumBands, num_render_channels);
EchoPathVariability echo_path_variability(
false, EchoPathVariability::DelayAdjustment::kNone, false);
std::vector<std::array<float, kBlockSize>> y(num_capture_channels);
@ -52,9 +50,9 @@ void RunNormalUsageTest(size_t num_render_channels,
}
Aec3Fft fft;
std::vector<std::vector<std::array<float, kFftLengthBy2Plus1>>>
converged_filter_frequency_response(
num_capture_channels,
std::vector<std::array<float, kFftLengthBy2Plus1>>(10));
converged_filter_frequency_response(
num_capture_channels,
std::vector<std::array<float, kFftLengthBy2Plus1>>(10));
for (auto& v_ch : converged_filter_frequency_response) {
for (auto& v : v_ch) {
v.fill(0.01f);
@ -72,7 +70,7 @@ void RunNormalUsageTest(size_t num_render_channels,
// Verify that linear AEC usability is true when the filter is converged
for (size_t band = 0; band < kNumBands; ++band) {
for (size_t ch = 0; ch < num_render_channels; ++ch) {
std::fill(x[band][ch].begin(), x[band][ch].end(), 101.f);
std::fill(x.begin(band, ch), x.end(band, ch), 101.f);
}
}
for (int k = 0; k < 3000; ++k) {
@ -100,7 +98,7 @@ void RunNormalUsageTest(size_t num_render_channels,
// Verify that the active render detection works as intended.
for (size_t ch = 0; ch < num_render_channels; ++ch) {
std::fill(x[0][ch].begin(), x[0][ch].end(), 101.f);
std::fill(x.begin(0, ch), x.end(0, ch), 101.f);
}
render_delay_buffer->Insert(x);
for (size_t ch = 0; ch < num_capture_channels; ++ch) {
@ -125,14 +123,14 @@ void RunNormalUsageTest(size_t num_render_channels,
EXPECT_TRUE(state.ActiveRender());
// Verify that the ERL is properly estimated
for (auto& band : x) {
for (auto& channel : band) {
channel = std::vector<float>(kBlockSize, 0.f);
for (int band = 0; band < x.NumBands(); ++band) {
for (int channel = 0; channel < x.NumChannels(); ++channel) {
std::fill(x.begin(band, channel), x.end(band, channel), 0.0f);
}
}
for (size_t ch = 0; ch < num_render_channels; ++ch) {
x[0][ch][0] = 5000.f;
x.View(/*band=*/0, ch)[0] = 5000.f;
}
for (size_t k = 0;
k < render_delay_buffer->GetRenderBuffer()->GetFftBuffer().size(); ++k) {
@ -266,9 +264,9 @@ TEST(AecState, ConvergedFilterDelay) {
y.fill(0.f);
std::vector<std::vector<std::array<float, kFftLengthBy2Plus1>>>
frequency_response(
kNumCaptureChannels,
std::vector<std::array<float, kFftLengthBy2Plus1>>(kFilterLengthBlocks));
frequency_response(kNumCaptureChannels,
std::vector<std::array<float, kFftLengthBy2Plus1>>(
kFilterLengthBlocks));
for (auto& v_ch : frequency_response) {
for (auto& v : v_ch) {
v.fill(0.01f);

27
modules/audio_processing/aec3/alignment_mixer.cc
View File

@ -63,9 +63,10 @@ AlignmentMixer::AlignmentMixer(size_t num_channels,
}
}
void AlignmentMixer::ProduceOutput(rtc::ArrayView<const std::vector<float>> x,
void AlignmentMixer::ProduceOutput(const Block& x,
rtc::ArrayView<float, kBlockSize> y) {
RTC_DCHECK_EQ(x.size(), num_channels_);
RTC_DCHECK_EQ(x.NumChannels(), num_channels_);
if (selection_variant_ == MixingVariant::kDownmix) {
Downmix(x, y);
return;
@ -73,18 +74,20 @@ void AlignmentMixer::ProduceOutput(rtc::ArrayView<const std::vector<float>> x,
int ch = selection_variant_ == MixingVariant::kFixed ? 0 : SelectChannel(x);
RTC_DCHECK_GE(x.size(), ch);
std::copy(x[ch].begin(), x[ch].end(), y.begin());
RTC_DCHECK_GT(x.NumChannels(), ch);
std::copy(x.begin(/*band=*/0, ch), x.end(/*band=*/0, ch), y.begin());
}
void AlignmentMixer::Downmix(rtc::ArrayView<const std::vector<float>> x,
void AlignmentMixer::Downmix(const Block& x,
rtc::ArrayView<float, kBlockSize> y) const {
RTC_DCHECK_EQ(x.size(), num_channels_);
RTC_DCHECK_EQ(x.NumChannels(), num_channels_);
RTC_DCHECK_GE(num_channels_, 2);
std::copy(x[0].begin(), x[0].end(), y.begin());
std::memcpy(&y[0], x.View(/*band=*/0, /*channel=*/0).data(),
kBlockSize * sizeof(y[0]));
for (size_t ch = 1; ch < num_channels_; ++ch) {
const auto x_ch = x.View(/*band=*/0, ch);
for (size_t i = 0; i < kBlockSize; ++i) {
y[i] += x[ch][i];
y[i] += x_ch[i];
}
}
@ -93,8 +96,8 @@ void AlignmentMixer::Downmix(rtc::ArrayView<const std::vector<float>> x,
}
}
int AlignmentMixer::SelectChannel(rtc::ArrayView<const std::vector<float>> x) {
RTC_DCHECK_EQ(x.size(), num_channels_);
int AlignmentMixer::SelectChannel(const Block& x) {
RTC_DCHECK_EQ(x.NumChannels(), num_channels_);
RTC_DCHECK_GE(num_channels_, 2);
RTC_DCHECK_EQ(cumulative_energies_.size(), num_channels_);
@ -112,10 +115,10 @@ int AlignmentMixer::SelectChannel(rtc::ArrayView<const std::vector<float>> x) {
++block_counter_;
for (int ch = 0; ch < num_ch_to_analyze; ++ch) {
RTC_DCHECK_EQ(x[ch].size(), kBlockSize);
float x2_sum = 0.f;
rtc::ArrayView<const float, kBlockSize> x_ch = x.View(/*band=*/0, ch);
for (size_t i = 0; i < kBlockSize; ++i) {
x2_sum += x[ch][i] * x[ch][i];
x2_sum += x_ch[i] * x_ch[i];
}
if (ch < 2 && x2_sum > excitation_energy_threshold_) {

9
modules/audio_processing/aec3/alignment_mixer.h
View File

@ -16,6 +16,7 @@
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/block.h"
namespace webrtc {
@ -33,8 +34,7 @@ class AlignmentMixer {
float excitation_limit,
bool prefer_first_two_channels);
void ProduceOutput(rtc::ArrayView<const std::vector<float>> x,
rtc::ArrayView<float, kBlockSize> y);
void ProduceOutput(const Block& x, rtc::ArrayView<float, kBlockSize> y);
enum class MixingVariant { kDownmix, kAdaptive, kFixed };
@ -49,9 +49,8 @@ class AlignmentMixer {
int selected_channel_ = 0;
size_t block_counter_ = 0;
void Downmix(rtc::ArrayView<const std::vector<float>> x,
rtc::ArrayView<float, kBlockSize> y) const;
int SelectChannel(rtc::ArrayView<const std::vector<float>> x);
void Downmix(const Block& x, rtc::ArrayView<float, kBlockSize> y) const;
int SelectChannel(const Block& x);
};
} // namespace webrtc

30
modules/audio_processing/aec3/alignment_mixer_unittest.cc
View File

@ -60,12 +60,9 @@ TEST(AlignmentMixer, GeneralAdaptiveMode) {
/*adaptive_selection*/ true, excitation_limit,
prefer_first_two_channels);
std::vector<std::vector<float>> x(
num_channels, std::vector<float>(kBlockSize, 0.f));
Block x(
/*num_bands=*/1, num_channels);
if (initial_silence) {
for (int ch = 0; ch < num_channels; ++ch) {
std::fill(x[ch].begin(), x[ch].end(), 0.f);
}
std::array<float, kBlockSize> y;
for (int frame = 0; frame < 10 * kNumBlocksPerSecond; ++frame) {
am.ProduceOutput(x, y);
@ -82,7 +79,8 @@ TEST(AlignmentMixer, GeneralAdaptiveMode) {
for (int ch = 0; ch < num_channels; ++ch) {
float scaling =
ch == strongest_ch ? kStrongestSignalScaling : 1.f;
std::fill(x[ch].begin(), x[ch].end(),
auto x_ch = x.View(/*band=*/0, ch);
std::fill(x_ch.begin(), x_ch.end(),
channel_value(frame, ch) * scaling);
}
@ -92,13 +90,15 @@ TEST(AlignmentMixer, GeneralAdaptiveMode) {
if (frame > 1 * kNumBlocksPerSecond) {
if (!prefer_first_two_channels || huge_activity_threshold) {
EXPECT_THAT(y, AllOf(Each(x[strongest_ch][0])));
EXPECT_THAT(y,
AllOf(Each(x.View(/*band=*/0, strongest_ch)[0])));
} else {
bool left_or_right_chosen;
for (int ch = 0; ch < 2; ++ch) {
left_or_right_chosen = true;
const auto x_ch = x.View(/*band=*/0, ch);
for (size_t k = 0; k < kBlockSize; ++k) {
if (y[k] != x[ch][k]) {
if (y[k] != x_ch[k]) {
left_or_right_chosen = false;
break;
}
@ -124,14 +124,14 @@ TEST(AlignmentMixer, DownmixMode) {
/*adaptive_selection*/ false, /*excitation_limit*/ 1.f,
/*prefer_first_two_channels*/ false);
std::vector<std::vector<float>> x(num_channels,
std::vector<float>(kBlockSize, 0.f));
Block x(/*num_bands=*/1, num_channels);
const auto channel_value = [](int frame_index, int channel_index) {
return static_cast<float>(frame_index + channel_index);
};
for (int frame = 0; frame < 10; ++frame) {
for (int ch = 0; ch < num_channels; ++ch) {
std::fill(x[ch].begin(), x[ch].end(), channel_value(frame, ch));
auto x_ch = x.View(/*band=*/0, ch);
std::fill(x_ch.begin(), x_ch.end(), channel_value(frame, ch));
}
std::array<float, kBlockSize> y;
@ -155,20 +155,20 @@ TEST(AlignmentMixer, FixedMode) {
/*adaptive_selection*/ false, /*excitation_limit*/ 1.f,
/*prefer_first_two_channels*/ false);
std::vector<std::vector<float>> x(num_channels,
std::vector<float>(kBlockSize, 0.f));
Block x(/*num_band=*/1, num_channels);
const auto channel_value = [](int frame_index, int channel_index) {
return static_cast<float>(frame_index + channel_index);
};
for (int frame = 0; frame < 10; ++frame) {
for (int ch = 0; ch < num_channels; ++ch) {
std::fill(x[ch].begin(), x[ch].end(), channel_value(frame, ch));
auto x_ch = x.View(/*band=*/0, ch);
std::fill(x_ch.begin(), x_ch.end(), channel_value(frame, ch));
}
std::array<float, kBlockSize> y;
y.fill(-1.f);
am.ProduceOutput(x, y);
EXPECT_THAT(y, AllOf(Each(x[0][0])));
EXPECT_THAT(y, AllOf(Each(x.View(/*band=*/0, /*channel=*/0)[0])));
}
}
}

73
modules/audio_processing/aec3/block.h Normal file
View File

@ -0,0 +1,73 @@
/*
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_PROCESSING_AEC3_BLOCK_H_
#define MODULES_AUDIO_PROCESSING_AEC3_BLOCK_H_
#include <array>
#include <vector>
#include "api/array_view.h"
#include "modules/audio_processing/aec3/aec3_common.h"
namespace webrtc {
class Block {
public:
Block(int num_bands, int num_channels, float default_value = 0.0f)
: data_(num_bands,
std::vector<std::array<float, kBlockSize>>(
num_channels,
std::array<float, kBlockSize>({default_value}))) {}
// Returns the number of bands.
int NumBands() const { return data_.size(); }
// Returns the number of channels.
int NumChannels() const { return data_[0].size(); }
// Modifies the number of channels.
void SetNumChannels(int num_channels) {
for (std::vector<std::array<float, kBlockSize>>& block_band : data_) {
block_band.resize(num_channels, std::array<float, kBlockSize>({0.0f}));
}
}
// Iterators for accessing the data.
auto begin(int band, int channel) { return data_[band][channel].begin(); }
auto begin(int band, int channel) const {
return data_[band][channel].begin();
}
auto end(int band, int channel) { return data_[band][channel].end(); }
auto end(int band, int channel) const { return data_[band][channel].end(); }
// Access data via ArrayView.
rtc::ArrayView<float, kBlockSize> View(int band, int channel) {
return rtc::ArrayView<float, kBlockSize>(data_[band][channel].data(),
kBlockSize);
}
rtc::ArrayView<const float, kBlockSize> View(int band, int channel) const {
return rtc::ArrayView<const float, kBlockSize>(data_[band][channel].data(),
kBlockSize);
}
// Lets two Blocks swap audio data.
void Swap(Block& b) { data_.swap(b.data_); }
private:
std::vector<std::vector<std::array<float, kBlockSize>>> data_;
};
} // namespace webrtc
#endif // MODULES_AUDIO_PROCESSING_AEC3_BLOCK_H_

20
modules/audio_processing/aec3/block_buffer.cc
View File

@ -14,25 +14,9 @@
namespace webrtc {
BlockBuffer::BlockBuffer(size_t size,
size_t num_bands,
size_t num_channels,
size_t frame_length)
BlockBuffer::BlockBuffer(size_t size, size_t num_bands, size_t num_channels)
: size(static_cast<int>(size)),
buffer(size,
std::vector<std::vector<std::vector<float>>>(
num_bands,
std::vector<std::vector<float>>(
num_channels,
std::vector<float>(frame_length, 0.f)))) {
for (auto& block : buffer) {
for (auto& band : block) {
for (auto& channel : band) {
std::fill(channel.begin(), channel.end(), 0.f);
}
}
}
}
buffer(size, Block(num_bands, num_channels)) {}
BlockBuffer::~BlockBuffer() = default;

8
modules/audio_processing/aec3/block_buffer.h
View File

@ -15,6 +15,7 @@
#include <vector>
#include "modules/audio_processing/aec3/block.h"
#include "rtc_base/checks.h"
namespace webrtc {
@ -22,10 +23,7 @@ namespace webrtc {
// Struct for bundling a circular buffer of two dimensional vector objects
// together with the read and write indices.
struct BlockBuffer {
BlockBuffer(size_t size,
size_t num_bands,
size_t num_channels,
size_t frame_length);
BlockBuffer(size_t size, size_t num_bands, size_t num_channels);
~BlockBuffer();
int IncIndex(int index) const {
@ -52,7 +50,7 @@ struct BlockBuffer {
void DecReadIndex() { read = DecIndex(read); }
const int size;
std::vector<std::vector<std::vector<std::vector<float>>>> buffer;
std::vector<Block> buffer;
int write = 0;
int read = 0;
};

27
modules/audio_processing/aec3/block_framer.cc
View File

@ -34,35 +34,32 @@ BlockFramer::~BlockFramer() = default;
// samples for InsertBlockAndExtractSubFrame to produce a frame. In order to
// achieve this, the InsertBlockAndExtractSubFrame and InsertBlock methods need
// to be called in the correct order.
void BlockFramer::InsertBlock(
const std::vector<std::vector<std::vector<float>>>& block) {
RTC_DCHECK_EQ(num_bands_, block.size());
void BlockFramer::InsertBlock(const Block& block) {
RTC_DCHECK_EQ(num_bands_, block.NumBands());
RTC_DCHECK_EQ(num_channels_, block.NumChannels());
for (size_t band = 0; band < num_bands_; ++band) {
RTC_DCHECK_EQ(num_channels_, block[band].size());
for (size_t channel = 0; channel < num_channels_; ++channel) {
RTC_DCHECK_EQ(kBlockSize, block[band][channel].size());
RTC_DCHECK_EQ(0, buffer_[band][channel].size());
buffer_[band][channel].insert(buffer_[band][channel].begin(),
block[band][channel].begin(),
block[band][channel].end());
block.begin(band, channel),
block.end(band, channel));
}
}
}
void BlockFramer::InsertBlockAndExtractSubFrame(
const std::vector<std::vector<std::vector<float>>>& block,
const Block& block,
std::vector<std::vector<rtc::ArrayView<float>>>* sub_frame) {
RTC_DCHECK(sub_frame);
RTC_DCHECK_EQ(num_bands_, block.size());
RTC_DCHECK_EQ(num_bands_, block.NumBands());
RTC_DCHECK_EQ(num_channels_, block.NumChannels());
RTC_DCHECK_EQ(num_bands_, sub_frame->size());
for (size_t band = 0; band < num_bands_; ++band) {
RTC_DCHECK_EQ(num_channels_, block[band].size());
RTC_DCHECK_EQ(num_channels_, (*sub_frame)[0].size());
for (size_t channel = 0; channel < num_channels_; ++channel) {
RTC_DCHECK_LE(kSubFrameLength,
buffer_[band][channel].size() + kBlockSize);
RTC_DCHECK_EQ(kBlockSize, block[band][channel].size());
RTC_DCHECK_GE(kBlockSize, buffer_[band][channel].size());
RTC_DCHECK_EQ(kSubFrameLength, (*sub_frame)[band][channel].size());
@ -71,14 +68,14 @@ void BlockFramer::InsertBlockAndExtractSubFrame(
std::copy(buffer_[band][channel].begin(), buffer_[band][channel].end(),
(*sub_frame)[band][channel].begin());
std::copy(
block[band][channel].begin(),
block[band][channel].begin() + samples_to_frame,
block.begin(band, channel),
block.begin(band, channel) + samples_to_frame,
(*sub_frame)[band][channel].begin() + buffer_[band][channel].size());
buffer_[band][channel].clear();
buffer_[band][channel].insert(
buffer_[band][channel].begin(),
block[band][channel].begin() + samples_to_frame,
block[band][channel].end());
block.begin(band, channel) + samples_to_frame,
block.end(band, channel));
}
}
}

5
modules/audio_processing/aec3/block_framer.h
View File

@ -15,6 +15,7 @@
#include "api/array_view.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/block.h"
namespace webrtc {
@ -32,10 +33,10 @@ class BlockFramer {
BlockFramer& operator=(const BlockFramer&) = delete;
// Adds a 64 sample block into the data that will form the next output frame.
void InsertBlock(const std::vector<std::vector<std::vector<float>>>& block);
void InsertBlock(const Block& block);
// Adds a 64 sample block and extracts an 80 sample subframe.
void InsertBlockAndExtractSubFrame(
const std::vector<std::vector<std::vector<float>>>& block,
const Block& block,
std::vector<std::vector<rtc::ArrayView<float>>>* sub_frame);
private:

95
modules/audio_processing/aec3/block_framer_unittest.cc
View File

@ -64,14 +64,13 @@ bool VerifySubFrame(
return true;
}
void FillBlock(size_t block_counter,
std::vector<std::vector<std::vector<float>>>* block) {
for (size_t band = 0; band < block->size(); ++band) {
for (size_t channel = 0; channel < (*block)[band].size(); ++channel) {
for (size_t sample = 0; sample < (*block)[band][channel].size();
++sample) {
(*block)[band][channel][sample] = ComputeSampleValue(
block_counter, kBlockSize, band, channel, sample, 0);
void FillBlock(size_t block_counter, Block* block) {
for (int band = 0; band < block->NumBands(); ++band) {
for (int channel = 0; channel < block->NumChannels(); ++channel) {
auto b = block->View(band, channel);
for (size_t sample = 0; sample < kBlockSize; ++sample) {
b[sample] = ComputeSampleValue(block_counter, kBlockSize, band, channel,
sample, 0);
}
}
}
@ -82,9 +81,7 @@ void RunFramerTest(int sample_rate_hz, size_t num_channels) {
constexpr size_t kNumSubFramesToProcess = 10;
const size_t num_bands = NumBandsForRate(sample_rate_hz);
std::vector<std::vector<std::vector<float>>> block(
num_bands, std::vector<std::vector<float>>(
num_channels, std::vector<float>(kBlockSize, 0.f)));
Block block(num_bands, num_channels);
std::vector<std::vector<std::vector<float>>> output_sub_frame(
num_bands, std::vector<std::vector<float>>(
num_channels, std::vector<float>(kSubFrameLength, 0.f)));
@ -117,16 +114,12 @@ void RunWronglySizedInsertAndExtractParametersTest(
size_t correct_num_channels,
size_t num_block_bands,
size_t num_block_channels,
size_t block_length,
size_t num_sub_frame_bands,
size_t num_sub_frame_channels,
size_t sub_frame_length) {
const size_t correct_num_bands = NumBandsForRate(sample_rate_hz);
std::vector<std::vector<std::vector<float>>> block(
num_block_bands,
std::vector<std::vector<float>>(num_block_channels,
std::vector<float>(block_length, 0.f)));
Block block(num_block_bands, num_block_channels);
std::vector<std::vector<std::vector<float>>> output_sub_frame(
num_sub_frame_bands,
std::vector<std::vector<float>>(
@ -145,18 +138,11 @@ void RunWronglySizedInsertAndExtractParametersTest(
void RunWronglySizedInsertParameterTest(int sample_rate_hz,
size_t correct_num_channels,
size_t num_block_bands,
size_t num_block_channels,
size_t block_length) {
size_t num_block_channels) {
const size_t correct_num_bands = NumBandsForRate(sample_rate_hz);
std::vector<std::vector<std::vector<float>>> correct_block(
correct_num_bands,
std::vector<std::vector<float>>(correct_num_channels,
std::vector<float>(kBlockSize, 0.f)));
std::vector<std::vector<std::vector<float>>> wrong_block(
num_block_bands,
std::vector<std::vector<float>>(num_block_channels,
std::vector<float>(block_length, 0.f)));
Block correct_block(correct_num_bands, correct_num_channels);
Block wrong_block(num_block_bands, num_block_channels);
std::vector<std::vector<std::vector<float>>> output_sub_frame(
correct_num_bands,
std::vector<std::vector<float>>(
@ -183,10 +169,7 @@ void RunWronglyInsertOrderTest(int sample_rate_hz,
size_t num_preceeding_api_calls) {
const size_t correct_num_bands = NumBandsForRate(sample_rate_hz);
std::vector<std::vector<std::vector<float>>> block(
correct_num_bands,
std::vector<std::vector<float>>(num_channels,
std::vector<float>(kBlockSize, 0.f)));
Block block(correct_num_bands, num_channels);
std::vector<std::vector<std::vector<float>>> output_sub_frame(
correct_num_bands,
std::vector<std::vector<float>>(
@ -223,7 +206,7 @@ TEST(BlockFramerDeathTest,
const size_t wrong_num_bands = (correct_num_bands % 3) + 1;
RunWronglySizedInsertAndExtractParametersTest(
rate, correct_num_channels, wrong_num_bands, correct_num_channels,
kBlockSize, correct_num_bands, correct_num_channels, kSubFrameLength);
correct_num_bands, correct_num_channels, kSubFrameLength);
}
}
}
@ -237,7 +220,7 @@ TEST(BlockFramerDeathTest,
const size_t wrong_num_channels = correct_num_channels + 1;
RunWronglySizedInsertAndExtractParametersTest(
rate, correct_num_channels, correct_num_bands, wrong_num_channels,
kBlockSize, correct_num_bands, correct_num_channels, kSubFrameLength);
correct_num_bands, correct_num_channels, kSubFrameLength);
}
}
}
@ -251,7 +234,7 @@ TEST(BlockFramerDeathTest,
const size_t wrong_num_bands = (correct_num_bands % 3) + 1;
RunWronglySizedInsertAndExtractParametersTest(
rate, correct_num_channels, correct_num_bands, correct_num_channels,
kBlockSize, wrong_num_bands, correct_num_channels, kSubFrameLength);
wrong_num_bands, correct_num_channels, kSubFrameLength);
}
}
}
@ -265,21 +248,7 @@ TEST(BlockFramerDeathTest,
const size_t wrong_num_channels = correct_num_channels + 1;
RunWronglySizedInsertAndExtractParametersTest(
rate, correct_num_channels, correct_num_bands, correct_num_channels,
kBlockSize, correct_num_bands, wrong_num_channels, kSubFrameLength);
}
}
}
TEST(BlockFramerDeathTest,
WrongNumberOfSamplesInBlockForInsertBlockAndExtractSubFrame) {
for (auto rate : {16000, 32000, 48000}) {
for (auto correct_num_channels : {1, 2, 8}) {
SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels));
const size_t correct_num_bands = NumBandsForRate(rate);
RunWronglySizedInsertAndExtractParametersTest(
rate, correct_num_channels, correct_num_bands, correct_num_channels,
kBlockSize - 1, correct_num_bands, correct_num_channels,
kSubFrameLength);
correct_num_bands, wrong_num_channels, kSubFrameLength);
}
}
}
@ -292,8 +261,7 @@ TEST(BlockFramerDeathTest,
const size_t correct_num_bands = NumBandsForRate(rate);
RunWronglySizedInsertAndExtractParametersTest(
rate, correct_num_channels, correct_num_bands, correct_num_channels,
kBlockSize, correct_num_bands, correct_num_channels,
kSubFrameLength - 1);
correct_num_bands, correct_num_channels, kSubFrameLength - 1);
}
}
@ -304,8 +272,7 @@ TEST(BlockFramerDeathTest, WrongNumberOfBandsInBlockForInsertBlock) {
const size_t correct_num_bands = NumBandsForRate(rate);
const size_t wrong_num_bands = (correct_num_bands % 3) + 1;
RunWronglySizedInsertParameterTest(rate, correct_num_channels,
wrong_num_bands, correct_num_channels,
kBlockSize);
wrong_num_bands, correct_num_channels);
}
}
}
@ -317,20 +284,7 @@ TEST(BlockFramerDeathTest, WrongNumberOfChannelsInBlockForInsertBlock) {
const size_t correct_num_bands = NumBandsForRate(rate);
const size_t wrong_num_channels = correct_num_channels + 1;
RunWronglySizedInsertParameterTest(rate, correct_num_channels,
correct_num_bands, wrong_num_channels,
kBlockSize);
}
}
}
TEST(BlockFramerDeathTest, WrongNumberOfSamplesInBlockForInsertBlock) {
for (auto rate : {16000, 32000, 48000}) {
for (auto correct_num_channels : {1, 2, 8}) {
SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels));
const size_t correct_num_bands = NumBandsForRate(rate);
RunWronglySizedInsertParameterTest(rate, correct_num_channels,
correct_num_bands,
correct_num_channels, kBlockSize - 1);
correct_num_bands, wrong_num_channels);
}
}
}
@ -364,12 +318,9 @@ TEST(BlockFramerDeathTest, ZeroNumberOfBandsParameter) {
// Verifies that the verification for null sub_frame pointer works.
TEST(BlockFramerDeathTest, NullSubFrameParameter) {
EXPECT_DEATH(BlockFramer(1, 1).InsertBlockAndExtractSubFrame(
std::vector<std::vector<std::vector<float>>>(
1, std::vector<std::vector<float>>(
1, std::vector<float>(kBlockSize, 0.f))),
nullptr),
"");
EXPECT_DEATH(
BlockFramer(1, 1).InsertBlockAndExtractSubFrame(Block(1, 1), nullptr),
"");
}
#endif

50
modules/audio_processing/aec3/block_processor.cc
View File

@ -49,14 +49,12 @@ class BlockProcessorImpl final : public BlockProcessor {
~BlockProcessorImpl() override;
void ProcessCapture(
bool echo_path_gain_change,
bool capture_signal_saturation,
std::vector<std::vector<std::vector<float>>>* linear_output,
std::vector<std::vector<std::vector<float>>>* capture_block) override;
void ProcessCapture(bool echo_path_gain_change,
bool capture_signal_saturation,
Block* linear_output,
Block* capture_block) override;
void BufferRender(
const std::vector<std::vector<std::vector<float>>>& block) override;
void BufferRender(const Block& block) override;
void UpdateEchoLeakageStatus(bool leakage_detected) override;
@ -104,21 +102,20 @@ BlockProcessorImpl::BlockProcessorImpl(
BlockProcessorImpl::~BlockProcessorImpl() = default;
void BlockProcessorImpl::ProcessCapture(
bool echo_path_gain_change,
bool capture_signal_saturation,
std::vector<std::vector<std::vector<float>>>* linear_output,
std::vector<std::vector<std::vector<float>>>* capture_block) {
void BlockProcessorImpl::ProcessCapture(bool echo_path_gain_change,
bool capture_signal_saturation,
Block* linear_output,
Block* capture_block) {
RTC_DCHECK(capture_block);
RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), capture_block->size());
RTC_DCHECK_EQ(kBlockSize, (*capture_block)[0][0].size());
RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), capture_block->NumBands());
capture_call_counter_++;
data_dumper_->DumpRaw("aec3_processblock_call_order",
static_cast<int>(BlockProcessorApiCall::kCapture));
data_dumper_->DumpWav("aec3_processblock_capture_input", kBlockSize,
&(*capture_block)[0][0][0], 16000, 1);
data_dumper_->DumpWav("aec3_processblock_capture_input",
capture_block->View(/*band=*/0, /*channel=*/0), 16000,
1);
if (render_properly_started_) {
if (!capture_properly_started_) {
@ -159,8 +156,9 @@ void BlockProcessorImpl::ProcessCapture(
delay_controller_->Reset(false);
}
data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize,
&(*capture_block)[0][0][0], 16000, 1);
data_dumper_->DumpWav("aec3_processblock_capture_input2",
capture_block->View(/*band=*/0, /*channel=*/0), 16000,
1);
bool has_delay_estimator = !config_.delay.use_external_delay_estimator;
if (has_delay_estimator) {
@ -169,7 +167,7 @@ void BlockProcessorImpl::ProcessCapture(
// alignment.
estimated_delay_ = delay_controller_->GetDelay(
render_buffer_->GetDownsampledRenderBuffer(), render_buffer_->Delay(),
(*capture_block)[0]);
*capture_block);
if (estimated_delay_) {
bool delay_change =
@ -202,16 +200,14 @@ void BlockProcessorImpl::ProcessCapture(
metrics_.UpdateCapture(false);
}
void BlockProcessorImpl::BufferRender(
const std::vector<std::vector<std::vector<float>>>& block) {
RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), block.size());
RTC_DCHECK_EQ(kBlockSize, block[0][0].size());
void BlockProcessorImpl::BufferRender(const Block& block) {
RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), block.NumBands());
data_dumper_->DumpRaw("aec3_processblock_call_order",
static_cast<int>(BlockProcessorApiCall::kRender));
data_dumper_->DumpWav("aec3_processblock_render_input", kBlockSize,
&block[0][0][0], 16000, 1);
data_dumper_->DumpWav("aec3_processblock_render_input2", kBlockSize,
&block[0][0][0], 16000, 1);
data_dumper_->DumpWav("aec3_processblock_render_input",
block.View(/*band=*/0, /*channel=*/0), 16000, 1);
data_dumper_->DumpWav("aec3_processblock_render_input2",
block.View(/*band=*/0, /*channel=*/0), 16000, 1);
render_event_ = render_buffer_->Insert(block);

13
modules/audio_processing/aec3/block_processor.h
View File

@ -18,6 +18,7 @@
#include "api/audio/echo_canceller3_config.h"
#include "api/audio/echo_control.h"
#include "modules/audio_processing/aec3/block.h"
#include "modules/audio_processing/aec3/echo_remover.h"
#include "modules/audio_processing/aec3/render_delay_buffer.h"
#include "modules/audio_processing/aec3/render_delay_controller.h"
@ -56,15 +57,13 @@ class BlockProcessor {
virtual void SetAudioBufferDelay(int delay_ms) = 0;
// Processes a block of capture data.
virtual void ProcessCapture(
bool echo_path_gain_change,
bool capture_signal_saturation,
std::vector<std::vector<std::vector<float>>>* linear_output,
std::vector<std::vector<std::vector<float>>>* capture_block) = 0;
virtual void ProcessCapture(bool echo_path_gain_change,
bool capture_signal_saturation,
Block* linear_output,
Block* capture_block) = 0;
// Buffers a block of render data supplied by a FrameBlocker object.
virtual void BufferRender(
const std::vector<std::vector<std::vector<float>>>& render_block) = 0;
virtual void BufferRender(const Block& render_block) = 0;
// Reports whether echo leakage has been detected in the echo canceller
// output.

105
modules/audio_processing/aec3/block_processor_unittest.cc
View File

@ -43,10 +43,7 @@ void RunBasicSetupAndApiCallTest(int sample_rate_hz, int num_iterations) {
std::unique_ptr<BlockProcessor> block_processor(
BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz,
kNumRenderChannels, kNumCaptureChannels));
std::vector<std::vector<std::vector<float>>> block(
NumBandsForRate(sample_rate_hz),
std::vector<std::vector<float>>(kNumRenderChannels,
std::vector<float>(kBlockSize, 1000.f)));
Block block(NumBandsForRate(sample_rate_hz), kNumRenderChannels, 1000.f);
for (int k = 0; k < num_iterations; ++k) {
block_processor->BufferRender(block);
block_processor->ProcessCapture(false, false, nullptr, &block);
@ -62,30 +59,11 @@ void RunRenderBlockSizeVerificationTest(int sample_rate_hz) {
std::unique_ptr<BlockProcessor> block_processor(
BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz,
kNumRenderChannels, kNumCaptureChannels));
std::vector<std::vector<std::vector<float>>> block(
NumBandsForRate(sample_rate_hz),
std::vector<std::vector<float>>(kNumRenderChannels,
std::vector<float>(kBlockSize - 1, 0.f)));
Block block(NumBandsForRate(sample_rate_hz), kNumRenderChannels);
EXPECT_DEATH(block_processor->BufferRender(block), "");
}
void RunCaptureBlockSizeVerificationTest(int sample_rate_hz) {
constexpr size_t kNumRenderChannels = 1;
constexpr size_t kNumCaptureChannels = 1;
std::unique_ptr<BlockProcessor> block_processor(
BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz,
kNumRenderChannels, kNumCaptureChannels));
std::vector<std::vector<std::vector<float>>> block(
NumBandsForRate(sample_rate_hz),
std::vector<std::vector<float>>(kNumRenderChannels,
std::vector<float>(kBlockSize - 1, 0.f)));
EXPECT_DEATH(block_processor->ProcessCapture(false, false, nullptr, &block),
"");
}
void RunRenderNumBandsVerificationTest(int sample_rate_hz) {
constexpr size_t kNumRenderChannels = 1;
constexpr size_t kNumCaptureChannels = 1;
@ -96,10 +74,7 @@ void RunRenderNumBandsVerificationTest(int sample_rate_hz) {
std::unique_ptr<BlockProcessor> block_processor(
BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz,
kNumRenderChannels, kNumCaptureChannels));
std::vector<std::vector<std::vector<float>>> block(
wrong_num_bands,
std::vector<std::vector<float>>(kNumRenderChannels,
std::vector<float>(kBlockSize, 0.f)));
Block block(wrong_num_bands, kNumRenderChannels);
EXPECT_DEATH(block_processor->BufferRender(block), "");
}
@ -114,10 +89,7 @@ void RunCaptureNumBandsVerificationTest(int sample_rate_hz) {
std::unique_ptr<BlockProcessor> block_processor(
BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz,
kNumRenderChannels, kNumCaptureChannels));
std::vector<std::vector<std::vector<float>>> block(
wrong_num_bands,
std::vector<std::vector<float>>(kNumRenderChannels,
std::vector<float>(kBlockSize, 0.f)));
Block block(wrong_num_bands, kNumRenderChannels);
EXPECT_DEATH(block_processor->ProcessCapture(false, false, nullptr, &block),
"");
@ -170,18 +142,14 @@ TEST(BlockProcessor, DISABLED_DelayControllerIntegration) {
EchoCanceller3Config(), rate, kNumRenderChannels, kNumCaptureChannels,
std::move(render_delay_buffer_mock)));
std::vector<std::vector<std::vector<float>>> render_block(
NumBandsForRate(rate),
std::vector<std::vector<float>>(kNumRenderChannels,
std::vector<float>(kBlockSize, 0.f)));
std::vector<std::vector<std::vector<float>>> capture_block(
NumBandsForRate(rate),
std::vector<std::vector<float>>(kNumCaptureChannels,
std::vector<float>(kBlockSize, 0.f)));
Block render_block(NumBandsForRate(rate), kNumRenderChannels);
Block capture_block(NumBandsForRate(rate), kNumCaptureChannels);
DelayBuffer<float> signal_delay_buffer(kDelayInSamples);
for (size_t k = 0; k < kNumBlocks; ++k) {
RandomizeSampleVector(&random_generator, render_block[0][0]);
signal_delay_buffer.Delay(render_block[0][0], capture_block[0][0]);
RandomizeSampleVector(&random_generator,
render_block.View(/*band=*/0, /*capture=*/0));
signal_delay_buffer.Delay(render_block.View(/*band=*/0, /*capture=*/0),
capture_block.View(/*band=*/0, /*capture=*/0));
block_processor->BufferRender(render_block);
block_processor->ProcessCapture(false, false, nullptr, &capture_block);
}
@ -228,18 +196,14 @@ TEST(BlockProcessor, DISABLED_SubmoduleIntegration) {
std::move(render_delay_buffer_mock),
std::move(render_delay_controller_mock), std::move(echo_remover_mock)));
std::vector<std::vector<std::vector<float>>> render_block(
NumBandsForRate(rate),
std::vector<std::vector<float>>(kNumRenderChannels,
std::vector<float>(kBlockSize, 0.f)));
std::vector<std::vector<std::vector<float>>> capture_block(
NumBandsForRate(rate),
std::vector<std::vector<float>>(kNumCaptureChannels,
std::vector<float>(kBlockSize, 0.f)));
Block render_block(NumBandsForRate(rate), kNumRenderChannels);
Block capture_block(NumBandsForRate(rate), kNumCaptureChannels);
DelayBuffer<float> signal_delay_buffer(640);
for (size_t k = 0; k < kNumBlocks; ++k) {
RandomizeSampleVector(&random_generator, render_block[0][0]);
signal_delay_buffer.Delay(render_block[0][0], capture_block[0][0]);
RandomizeSampleVector(&random_generator,
render_block.View(/*band=*/0, /*capture=*/0));
signal_delay_buffer.Delay(render_block.View(/*band=*/0, /*capture=*/0),
capture_block.View(/*band=*/0, /*capture=*/0));
block_processor->BufferRender(render_block);
block_processor->ProcessCapture(false, false, nullptr, &capture_block);
block_processor->UpdateEchoLeakageStatus(false);
@ -268,13 +232,6 @@ TEST(BlockProcessorDeathTest, DISABLED_VerifyRenderBlockSizeCheck) {
}
}
TEST(BlockProcessorDeathTest, VerifyCaptureBlockSizeCheck) {
for (auto rate : {16000, 32000, 48000}) {
SCOPED_TRACE(ProduceDebugText(rate));
RunCaptureBlockSizeVerificationTest(rate);
}
}
TEST(BlockProcessorDeathTest, VerifyRenderNumBandsCheck) {
for (auto rate : {16000, 32000, 48000}) {
SCOPED_TRACE(ProduceDebugText(rate));
@ -333,14 +290,8 @@ TEST(BlockProcessor, ExternalDelayAppliedCorrectlyWithInitialCaptureCalls) {
std::move(delay_buffer), /*delay_controller=*/nullptr,
std::move(echo_remover_mock)));
std::vector<std::vector<std::vector<float>>> render_block(
NumBandsForRate(kSampleRateHz),
std::vector<std::vector<float>>(kNumRenderChannels,
std::vector<float>(kBlockSize, 0.f)));
std::vector<std::vector<std::vector<float>>> capture_block(
NumBandsForRate(kSampleRateHz),
std::vector<std::vector<float>>(kNumCaptureChannels,
std::vector<float>(kBlockSize, 0.f)));
Block render_block(NumBandsForRate(kSampleRateHz), kNumRenderChannels);
Block capture_block(NumBandsForRate(kSampleRateHz), kNumCaptureChannels);
// Process...
// - 10 capture calls, where no render data is available,
@ -354,11 +305,12 @@ TEST(BlockProcessor, ExternalDelayAppliedCorrectlyWithInitialCaptureCalls) {
int render_call_counter = 0;
for (size_t k = 0; k < 10; ++k) {
FillSampleVector(++capture_call_counter, kDelayInBlocks,
capture_block[0][0]);
capture_block.View(/*band=*/0, /*capture=*/0));
block_processor->ProcessCapture(false, false, nullptr, &capture_block);
}
for (size_t k = 0; k < 10; ++k) {
FillSampleVector(++render_call_counter, 0, render_block[0][0]);
FillSampleVector(++render_call_counter, 0,
render_block.View(/*band=*/0, /*capture=*/0));
block_processor->BufferRender(render_block);
}
@ -367,19 +319,22 @@ TEST(BlockProcessor, ExternalDelayAppliedCorrectlyWithInitialCaptureCalls) {
[](EchoPathVariability /*echo_path_variability*/,
bool /*capture_signal_saturation*/,
const absl::optional<DelayEstimate>& /*external_delay*/,
RenderBuffer* render_buffer,
std::vector<std::vector<std::vector<float>>>* /*linear_output*/,
std::vector<std::vector<std::vector<float>>>* capture) {
RenderBuffer* render_buffer, Block* /*linear_output*/,
Block* capture) {
const auto& render = render_buffer->Block(0);
const auto render_view = render.View(/*band=*/0, /*channel=*/0);
const auto capture_view = capture->View(/*band=*/0, /*channel=*/0);
for (size_t i = 0; i < kBlockSize; ++i) {
EXPECT_FLOAT_EQ(render[0][0][i], (*capture)[0][0][i]);
EXPECT_FLOAT_EQ(render_view[i], capture_view[i]);
}
});
FillSampleVector(++capture_call_counter, kDelayInBlocks, capture_block[0][0]);
FillSampleVector(++capture_call_counter, kDelayInBlocks,
capture_block.View(/*band=*/0, /*capture=*/0));
block_processor->ProcessCapture(false, false, nullptr, &capture_block);
FillSampleVector(++capture_call_counter, kDelayInBlocks, capture_block[0][0]);
FillSampleVector(++capture_call_counter, kDelayInBlocks,
capture_block.View(/*band=*/0, /*capture=*/0));
block_processor->ProcessCapture(false, false, nullptr, &capture_block);
}

13
modules/audio_processing/aec3/coarse_filter_update_gain_unittest.cc
View File

@ -59,10 +59,7 @@ void RunFilterUpdateTest(int num_blocks_to_process,
CoarseFilterUpdateGain coarse_gain(
config.filter.coarse, config.filter.config_change_duration_blocks);
Random random_generator(42U);
std::vector<std::vector<std::vector<float>>> x(
NumBandsForRate(kSampleRateHz),
std::vector<std::vector<float>>(num_render_channels,
std::vector<float>(kBlockSize, 0.f)));
Block x(NumBandsForRate(kSampleRateHz), num_render_channels);
std::array<float, kBlockSize> y;
RenderSignalAnalyzer render_signal_analyzer(config);
std::array<float, kFftLength> s;
@ -81,12 +78,12 @@ void RunFilterUpdateTest(int num_blocks_to_process,
k) != blocks_with_saturation.end();
// Create the render signal.
for (size_t band = 0; band < x.size(); ++band) {
for (size_t channel = 0; channel < x[band].size(); ++channel) {
RandomizeSampleVector(&random_generator, x[band][channel]);
for (int band = 0; band < x.NumBands(); ++band) {
for (int channel = 0; channel < x.NumChannels(); ++channel) {
RandomizeSampleVector(&random_generator, x.View(band, channel));
}
}
delay_buffer.Delay(x[0][0], y);
delay_buffer.Delay(x.View(/*band=*/0, /*channel*/ 0), y);
render_delay_buffer->Insert(x);
if (k == 0) {

5
modules/audio_processing/aec3/echo_audibility.cc
View File

@ -88,7 +88,7 @@ void EchoAudibility::UpdateRenderNoiseEstimator(
bool EchoAudibility::IsRenderTooLow(const BlockBuffer& block_buffer) {
const int num_render_channels =
static_cast<int>(block_buffer.buffer[0][0].size());
static_cast<int>(block_buffer.buffer[0].NumChannels());
bool too_low = false;
const int render_block_write_current = block_buffer.write;
if (render_block_write_current == render_block_write_prev_) {
@ -98,7 +98,8 @@ bool EchoAudibility::IsRenderTooLow(const BlockBuffer& block_buffer) {
idx = block_buffer.IncIndex(idx)) {
float max_abs_over_channels = 0.f;
for (int ch = 0; ch < num_render_channels; ++ch) {
auto block = block_buffer.buffer[idx][0][ch];
rtc::ArrayView<const float, kBlockSize> block =
block_buffer.buffer[idx].View(/*band=*/0, /*channel=*/ch);
auto r = std::minmax_element(block.cbegin(), block.cend());
float max_abs_channel =
std::max(std::fabs(*r.first), std::fabs(*r.second));

64
modules/audio_processing/aec3/echo_canceller3.cc
View File

@ -154,10 +154,10 @@ void ProcessCaptureFrameContent(
BlockFramer* linear_output_framer,
BlockFramer* output_framer,
BlockProcessor* block_processor,
std::vector<std::vector<std::vector<float>>>* linear_output_block,
Block* linear_output_block,
std::vector<std::vector<rtc::ArrayView<float>>>*
linear_output_sub_frame_view,
std::vector<std::vector<std::vector<float>>>* capture_block,
Block* capture_block,
std::vector<std::vector<rtc::ArrayView<float>>>* capture_sub_frame_view) {
FillSubFrameView(capture, sub_frame_index, capture_sub_frame_view);
@ -171,10 +171,10 @@ void ProcessCaptureFrameContent(
capture_blocker->InsertSubFrameAndExtractBlock(*capture_sub_frame_view,
capture_block);
block_processor->ProcessCapture(/*echo_path_gain_change=*/level_change ||
aec_reference_is_downmixed_stereo,
saturated_microphone_signal,
linear_output_block, capture_block);
block_processor->ProcessCapture(
/*echo_path_gain_change=*/level_change ||
aec_reference_is_downmixed_stereo,
saturated_microphone_signal, linear_output_block, capture_block);
output_framer->InsertBlockAndExtractSubFrame(*capture_block,
capture_sub_frame_view);
@ -185,16 +185,15 @@ void ProcessCaptureFrameContent(
}
}
void ProcessRemainingCaptureFrameContent(
bool level_change,
bool aec_reference_is_downmixed_stereo,
bool saturated_microphone_signal,
FrameBlocker* capture_blocker,
BlockFramer* linear_output_framer,
BlockFramer* output_framer,
BlockProcessor* block_processor,
std::vector<std::vector<std::vector<float>>>* linear_output_block,
std::vector<std::vector<std::vector<float>>>* block) {
void ProcessRemainingCaptureFrameContent(bool level_change,
bool aec_reference_is_downmixed_stereo,
bool saturated_microphone_signal,
FrameBlocker* capture_blocker,
BlockFramer* linear_output_framer,
BlockFramer* output_framer,
BlockProcessor* block_processor,
Block* linear_output_block,
Block* block) {
if (!capture_blocker->IsBlockAvailable()) {
return;
}
@ -218,7 +217,7 @@ void BufferRenderFrameContent(
size_t sub_frame_index,
FrameBlocker* render_blocker,
BlockProcessor* block_processor,
std::vector<std::vector<std::vector<float>>>* block,
Block* block,
std::vector<std::vector<rtc::ArrayView<float>>>* sub_frame_view) {
FillSubFrameView(proper_downmix_needed, render_frame, sub_frame_index,
sub_frame_view);
@ -226,10 +225,9 @@ void BufferRenderFrameContent(
block_processor->BufferRender(*block);
}
void BufferRemainingRenderFrameContent(
FrameBlocker* render_blocker,
BlockProcessor* block_processor,
std::vector<std::vector<std::vector<float>>>* block) {
void BufferRemainingRenderFrameContent(FrameBlocker* render_blocker,
BlockProcessor* block_processor,
Block* block) {
if (!render_blocker->IsBlockAvailable()) {
return;
}
@ -753,14 +751,8 @@ EchoCanceller3::EchoCanceller3(
std::vector<std::vector<float>>(
num_render_input_channels_,
std::vector<float>(AudioBuffer::kSplitBandSize, 0.f))),
render_block_(
num_bands_,
std::vector<std::vector<float>>(num_render_input_channels_,
std::vector<float>(kBlockSize, 0.f))),
capture_block_(
num_bands_,
std::vector<std::vector<float>>(num_capture_channels_,
std::vector<float>(kBlockSize, 0.f))),
render_block_(num_bands_, num_render_input_channels_),
capture_block_(num_bands_, num_capture_channels_),
capture_sub_frame_view_(
num_bands_,
std::vector<rtc::ArrayView<float>>(num_capture_channels_)) {
@ -780,11 +772,10 @@ EchoCanceller3::EchoCanceller3(
RTC_DCHECK_GE(kMaxNumBands, num_bands_);
if (config_selector_.active_config().filter.export_linear_aec_output) {
linear_output_framer_.reset(new BlockFramer(1, num_capture_channels_));
linear_output_framer_.reset(
new BlockFramer(/*num_bands=*/1, num_capture_channels_));
linear_output_block_ =
std::make_unique<std::vector<std::vector<std::vector<float>>>>(
1, std::vector<std::vector<float>>(
num_capture_channels_, std::vector<float>(kBlockSize, 0.f)));
std::make_unique<Block>(/*num_bands=*/1, num_capture_channels_),
linear_output_sub_frame_view_ =
std::vector<std::vector<rtc::ArrayView<float>>>(
1, std::vector<rtc::ArrayView<float>>(num_capture_channels_));
@ -810,12 +801,7 @@ void EchoCanceller3::Initialize() {
config_selector_.Update(
multichannel_content_detector_.IsProperMultiChannelContentDetected());
for (std::vector<std::vector<float>>& block_band : render_block_) {
block_band.resize(num_render_channels_to_aec_);
for (std::vector<float>& block_channel : block_band) {
block_channel.resize(kBlockSize, 0.0f);
}
}
render_block_.SetNumChannels(num_render_channels_to_aec_);
render_blocker_.reset(
new FrameBlocker(num_bands_, num_render_channels_to_aec_));

8
modules/audio_processing/aec3/echo_canceller3.h
View File

@ -210,12 +210,10 @@ class EchoCanceller3 : public EchoControl {
RTC_GUARDED_BY(capture_race_checker_);
bool saturated_microphone_signal_ RTC_GUARDED_BY(capture_race_checker_) =
false;
std::vector<std::vector<std::vector<float>>> render_block_
RTC_GUARDED_BY(capture_race_checker_);
std::unique_ptr<std::vector<std::vector<std::vector<float>>>>
linear_output_block_ RTC_GUARDED_BY(capture_race_checker_);
std::vector<std::vector<std::vector<float>>> capture_block_
Block render_block_ RTC_GUARDED_BY(capture_race_checker_);
std::unique_ptr<Block> linear_output_block_
RTC_GUARDED_BY(capture_race_checker_);
Block capture_block_ RTC_GUARDED_BY(capture_race_checker_);
std::vector<std::vector<rtc::ArrayView<float>>> render_sub_frame_view_
RTC_GUARDED_BY(capture_race_checker_);
std::vector<std::vector<rtc::ArrayView<float>>> linear_output_sub_frame_view_

32
modules/audio_processing/aec3/echo_canceller3_unittest.cc
View File

@ -117,14 +117,12 @@ class CaptureTransportVerificationProcessor : public BlockProcessor {
~CaptureTransportVerificationProcessor() override = default;
void ProcessCapture(
bool level_change,
bool saturated_microphone_signal,
std::vector<std::vector<std::vector<float>>>* linear_output,
std::vector<std::vector<std::vector<float>>>* capture_block) override {}
void ProcessCapture(bool level_change,
bool saturated_microphone_signal,
Block* linear_output,
Block* capture_block) override {}
void BufferRender(
const std::vector<std::vector<std::vector<float>>>& block) override {}
void BufferRender(const Block& block) override {}
void UpdateEchoLeakageStatus(bool leakage_detected) override {}
@ -149,19 +147,16 @@ class RenderTransportVerificationProcessor : public BlockProcessor {
~RenderTransportVerificationProcessor() override = default;
void ProcessCapture(
bool level_change,
bool saturated_microphone_signal,
std::vector<std::vector<std::vector<float>>>* linear_output,
std::vector<std::vector<std::vector<float>>>* capture_block) override {
std::vector<std::vector<std::vector<float>>> render_block =
received_render_blocks_.front();
void ProcessCapture(bool level_change,
bool saturated_microphone_signal,
Block* linear_output,
Block* capture_block) override {
Block render_block = received_render_blocks_.front();
received_render_blocks_.pop_front();
capture_block->swap(render_block);
capture_block->Swap(render_block);
}
void BufferRender(
const std::vector<std::vector<std::vector<float>>>& block) override {
void BufferRender(const Block& block) override {
received_render_blocks_.push_back(block);
}
@ -174,8 +169,7 @@ class RenderTransportVerificationProcessor : public BlockProcessor {
void SetCaptureOutputUsage(bool capture_output_used) {}
private:
std::deque<std::vector<std::vector<std::vector<float>>>>
received_render_blocks_;
std::deque<Block> received_render_blocks_;
};
std::string ProduceDebugText(int sample_rate_hz) {

4
modules/audio_processing/aec3/echo_path_delay_estimator.cc
View File

@ -59,9 +59,7 @@ void EchoPathDelayEstimator::Reset(bool reset_delay_confidence) {
absl::optional<DelayEstimate> EchoPathDelayEstimator::EstimateDelay(
const DownsampledRenderBuffer& render_buffer,
const std::vector<std::vector<float>>& capture) {
RTC_DCHECK_EQ(kBlockSize, capture[0].size());
const Block& capture) {
std::array<float, kBlockSize> downsampled_capture_data;
rtc::ArrayView<float> downsampled_capture(downsampled_capture_data.data(),
sub_block_size_);

3
modules/audio_processing/aec3/echo_path_delay_estimator.h
View File

@ -16,6 +16,7 @@
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "modules/audio_processing/aec3/alignment_mixer.h"
#include "modules/audio_processing/aec3/block.h"
#include "modules/audio_processing/aec3/clockdrift_detector.h"
#include "modules/audio_processing/aec3/decimator.h"
#include "modules/audio_processing/aec3/delay_estimate.h"
@ -46,7 +47,7 @@ class EchoPathDelayEstimator {
// Produce a delay estimate if such is avaliable.
absl::optional<DelayEstimate> EstimateDelay(
const DownsampledRenderBuffer& render_buffer,
const std::vector<std::vector<float>>& capture);
const Block& capture);
// Log delay estimator properties.
void LogDelayEstimationProperties(int sample_rate_hz, size_t shift) const {

54
modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc
View File

@ -54,11 +54,8 @@ TEST_P(EchoPathDelayEstimatorMultiChannel, BasicApiCalls) {
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create(config, kSampleRateHz, num_render_channels));
EchoPathDelayEstimator estimator(&data_dumper, config, num_capture_channels);
std::vector<std::vector<std::vector<float>>> render(
kNumBands, std::vector<std::vector<float>>(
num_render_channels, std::vector<float>(kBlockSize)));
std::vector<std::vector<float>> capture(num_capture_channels,
std::vector<float>(kBlockSize));
Block render(kNumBands, num_render_channels);
Block capture(/*num_bands=*/1, num_capture_channels);
for (size_t k = 0; k < 100; ++k) {
render_delay_buffer->Insert(render);
estimator.EstimateDelay(render_delay_buffer->GetDownsampledRenderBuffer(),
@ -75,11 +72,8 @@ TEST(EchoPathDelayEstimator, DelayEstimation) {
constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz);
Random random_generator(42U);
std::vector<std::vector<std::vector<float>>> render(
kNumBands, std::vector<std::vector<float>>(
kNumRenderChannels, std::vector<float>(kBlockSize)));
std::vector<std::vector<float>> capture(kNumCaptureChannels,
std::vector<float>(kBlockSize));
Block render(kNumBands, kNumRenderChannels);
Block capture(/*num_bands=*/1, kNumCaptureChannels);
ApmDataDumper data_dumper(0);
constexpr size_t kDownSamplingFactors[] = {2, 4, 8};
for (auto down_sampling_factor : kDownSamplingFactors) {
@ -96,8 +90,10 @@ TEST(EchoPathDelayEstimator, DelayEstimation) {
absl::optional<DelayEstimate> estimated_delay_samples;
for (size_t k = 0; k < (500 + (delay_samples) / kBlockSize); ++k) {
RandomizeSampleVector(&random_generator, render[0][0]);
signal_delay_buffer.Delay(render[0][0], capture[0]);
RandomizeSampleVector(&random_generator,
render.View(/*band=*/0, /*channel=*/0));
signal_delay_buffer.Delay(render.View(/*band=*/0, /*channel=*/0),
capture.View(/*band=*/0, /*channel=*/0));
render_delay_buffer->Insert(render);
if (k == 0) {
@ -137,22 +133,22 @@ TEST(EchoPathDelayEstimator, NoDelayEstimatesForLowLevelRenderSignals) {
constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz);
Random random_generator(42U);
EchoCanceller3Config config;
std::vector<std::vector<std::vector<float>>> render(
kNumBands, std::vector<std::vector<float>>(
kNumRenderChannels, std::vector<float>(kBlockSize)));
std::vector<std::vector<float>> capture(kNumCaptureChannels,
std::vector<float>(kBlockSize));
Block render(kNumBands, kNumRenderChannels);
Block capture(/*num_bands=*/1, kNumCaptureChannels);
ApmDataDumper data_dumper(0);
EchoPathDelayEstimator estimator(&data_dumper, config, kNumCaptureChannels);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create(EchoCanceller3Config(), kSampleRateHz,
kNumRenderChannels));
for (size_t k = 0; k < 100; ++k) {
RandomizeSampleVector(&random_generator, render[0][0]);
for (auto& render_k : render[0][0]) {
RandomizeSampleVector(&random_generator,
render.View(/*band=*/0, /*channel=*/0));
for (auto& render_k : render.View(/*band=*/0, /*channel=*/0)) {
render_k *= 100.f / 32767.f;
}
std::copy(render[0][0].begin(), render[0][0].end(), capture[0].begin());
std::copy(render.begin(/*band=*/0, /*channel=*/0),
render.end(/*band=*/0, /*channel=*/0),
capture.begin(/*band*/ 0, /*channel=*/0));
render_delay_buffer->Insert(render);
render_delay_buffer->PrepareCaptureProcessing();
EXPECT_FALSE(estimator.EstimateDelay(
@ -171,23 +167,7 @@ TEST(EchoPathDelayEstimatorDeathTest, DISABLED_WrongRenderBlockSize) {
EchoPathDelayEstimator estimator(&data_dumper, config, 1);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create(config, 48000, 1));
std::vector<std::vector<float>> capture(1, std::vector<float>(kBlockSize));
EXPECT_DEATH(estimator.EstimateDelay(
render_delay_buffer->GetDownsampledRenderBuffer(), capture),
"");
}
// Verifies the check for the capture blocksize.
// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH
// tests on test bots has been fixed.
TEST(EchoPathDelayEstimatorDeathTest, WrongCaptureBlockSize) {
ApmDataDumper data_dumper(0);
EchoCanceller3Config config;
EchoPathDelayEstimator estimator(&data_dumper, config, 1);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create(config, 48000, 1));
std::vector<std::vector<float>> capture(1,
std::vector<float>(kBlockSize - 1));
Block capture(/*num_bands=*/1, /*num_channels=*/1);
EXPECT_DEATH(estimator.EstimateDelay(
render_delay_buffer->GetDownsampledRenderBuffer(), capture),
"");

69
modules/audio_processing/aec3/echo_remover.cc
View File

@ -118,13 +118,12 @@ class EchoRemoverImpl final : public EchoRemover {
// Removes the echo from a block of samples from the capture signal. The
// supplied render signal is assumed to be pre-aligned with the capture
// signal.
void ProcessCapture(
EchoPathVariability echo_path_variability,
bool capture_signal_saturation,
const absl::optional<DelayEstimate>& external_delay,
RenderBuffer* render_buffer,
std::vector<std::vector<std::vector<float>>>* linear_output,
std::vector<std::vector<std::vector<float>>>* capture) override;
void ProcessCapture(EchoPathVariability echo_path_variability,
bool capture_signal_saturation,
const absl::optional<DelayEstimate>& external_delay,
RenderBuffer* render_buffer,
Block* linear_output,
Block* capture) override;
// Updates the status on whether echo leakage is detected in the output of the
// echo remover.
@ -243,20 +242,17 @@ void EchoRemoverImpl::ProcessCapture(
bool capture_signal_saturation,
const absl::optional<DelayEstimate>& external_delay,
RenderBuffer* render_buffer,
std::vector<std::vector<std::vector<float>>>* linear_output,
std::vector<std::vector<std::vector<float>>>* capture) {
Block* linear_output,
Block* capture) {
++block_counter_;
const std::vector<std::vector<std::vector<float>>>& x =
render_buffer->Block(0);
std::vector<std::vector<std::vector<float>>>* y = capture;
const Block& x = render_buffer->Block(0);
Block* y = capture;
RTC_DCHECK(render_buffer);
RTC_DCHECK(y);
RTC_DCHECK_EQ(x.size(), NumBandsForRate(sample_rate_hz_));
RTC_DCHECK_EQ(y->size(), NumBandsForRate(sample_rate_hz_));
RTC_DCHECK_EQ(x[0].size(), num_render_channels_);
RTC_DCHECK_EQ((*y)[0].size(), num_capture_channels_);
RTC_DCHECK_EQ(x[0][0].size(), kBlockSize);
RTC_DCHECK_EQ((*y)[0][0].size(), kBlockSize);
RTC_DCHECK_EQ(x.NumBands(), NumBandsForRate(sample_rate_hz_));
RTC_DCHECK_EQ(y->NumBands(), NumBandsForRate(sample_rate_hz_));
RTC_DCHECK_EQ(x.NumChannels(), num_render_channels_);
RTC_DCHECK_EQ(y->NumChannels(), num_capture_channels_);
// Stack allocated data to use when the number of channels is low.
std::array<std::array<float, kFftLengthBy2>, kMaxNumChannelsOnStack> e_stack;
@ -321,12 +317,14 @@ void EchoRemoverImpl::ProcessCapture(
subtractor_output_heap_.data(), num_capture_channels_);
}
data_dumper_->DumpWav("aec3_echo_remover_capture_input", kBlockSize,
&(*y)[0][0][0], 16000, 1);
data_dumper_->DumpWav("aec3_echo_remover_render_input", kBlockSize,
&x[0][0][0], 16000, 1);
data_dumper_->DumpRaw("aec3_echo_remover_capture_input", (*y)[0][0]);
data_dumper_->DumpRaw("aec3_echo_remover_render_input", x[0][0]);
data_dumper_->DumpWav("aec3_echo_remover_capture_input",
y->View(/*band=*/0, /*channel=*/0), 16000, 1);
data_dumper_->DumpWav("aec3_echo_remover_render_input",
x.View(/*band=*/0, /*channel=*/0), 16000, 1);
data_dumper_->DumpRaw("aec3_echo_remover_capture_input",
y->View(/*band=*/0, /*channel=*/0));
data_dumper_->DumpRaw("aec3_echo_remover_render_input",
x.View(/*band=*/0, /*channel=*/0));
aec_state_.UpdateCaptureSaturation(capture_signal_saturation);
@ -369,13 +367,13 @@ void EchoRemoverImpl::ProcessCapture(
}
// Perform linear echo cancellation.
subtractor_.Process(*render_buffer, (*y)[0], render_signal_analyzer_,
aec_state_, subtractor_output);
subtractor_.Process(*render_buffer, *y, render_signal_analyzer_, aec_state_,
subtractor_output);
// Compute spectra.
for (size_t ch = 0; ch < num_capture_channels_; ++ch) {
FormLinearFilterOutput(subtractor_output[ch], e[ch]);
WindowedPaddedFft(fft_, (*y)[0][ch], y_old_[ch], &Y[ch]);
WindowedPaddedFft(fft_, y->View(/*band=*/0, ch), y_old_[ch], &Y[ch]);
WindowedPaddedFft(fft_, e[ch], e_old_[ch], &E[ch]);
LinearEchoPower(E[ch], Y[ch], &S2_linear[ch]);
Y[ch].Spectrum(optimization_, Y2[ch]);
@ -384,11 +382,11 @@ void EchoRemoverImpl::ProcessCapture(
// Optionally return the linear filter output.
if (linear_output) {
RTC_DCHECK_GE(1, linear_output->size());
RTC_DCHECK_EQ(num_capture_channels_, linear_output[0].size());
RTC_DCHECK_GE(1, linear_output->NumBands());
RTC_DCHECK_EQ(num_capture_channels_, linear_output->NumChannels());
for (size_t ch = 0; ch < num_capture_channels_; ++ch) {
RTC_DCHECK_EQ(kBlockSize, (*linear_output)[0][ch].size());
std::copy(e[ch].begin(), e[ch].end(), (*linear_output)[0][ch].begin());
std::copy(e[ch].begin(), e[ch].end(),
linear_output->begin(/*band=*/0, ch));
}
}
@ -400,8 +398,8 @@ void EchoRemoverImpl::ProcessCapture(
// Choose the linear output.
const auto& Y_fft = aec_state_.UseLinearFilterOutput() ? E : Y;
data_dumper_->DumpWav("aec3_output_linear", kBlockSize, &(*y)[0][0][0], 16000,
1);
data_dumper_->DumpWav("aec3_output_linear",
y->View(/*band=*/0, /*channel=*/0), 16000, 1);
data_dumper_->DumpWav("aec3_output_linear2", kBlockSize, &e[0][0], 16000, 1);
// Estimate the comfort noise.
@ -455,13 +453,12 @@ void EchoRemoverImpl::ProcessCapture(
// Debug outputs for the purpose of development and analysis.
data_dumper_->DumpWav("aec3_echo_estimate", kBlockSize,
&subtractor_output[0].s_refined[0], 16000, 1);
data_dumper_->DumpRaw("aec3_output", (*y)[0][0]);
data_dumper_->DumpRaw("aec3_output", y->View(/*band=*/0, /*channel=*/0));
data_dumper_->DumpRaw("aec3_narrow_render",
render_signal_analyzer_.NarrowPeakBand() ? 1 : 0);
data_dumper_->DumpRaw("aec3_N2", cng_.NoiseSpectrum()[0]);
data_dumper_->DumpRaw("aec3_suppressor_gain", G);
data_dumper_->DumpWav("aec3_output",
rtc::ArrayView<const float>(&(*y)[0][0][0], kBlockSize),
data_dumper_->DumpWav("aec3_output", y->View(/*band=*/0, /*channel=*/0),
16000, 1);
data_dumper_->DumpRaw("aec3_using_subtractor_output[0]",
aec_state_.UseLinearFilterOutput() ? 1 : 0);

5
modules/audio_processing/aec3/echo_remover.h
View File

@ -16,6 +16,7 @@
#include "absl/types/optional.h"
#include "api/audio/echo_canceller3_config.h"
#include "api/audio/echo_control.h"
#include "modules/audio_processing/aec3/block.h"
#include "modules/audio_processing/aec3/delay_estimate.h"
#include "modules/audio_processing/aec3/echo_path_variability.h"
#include "modules/audio_processing/aec3/render_buffer.h"
@ -42,8 +43,8 @@ class EchoRemover {
bool capture_signal_saturation,
const absl::optional<DelayEstimate>& external_delay,
RenderBuffer* render_buffer,
std::vector<std::vector<std::vector<float>>>* linear_output,
std::vector<std::vector<std::vector<float>>>* capture) = 0;
Block* linear_output,
Block* capture) = 0;
// Updates the status on whether echo leakage is detected in the output of the
// echo remover.

79
modules/audio_processing/aec3/echo_remover_unittest.cc
View File

@ -62,14 +62,8 @@ TEST_P(EchoRemoverMultiChannel, BasicApiCalls) {
std::unique_ptr<RenderDelayBuffer> render_buffer(RenderDelayBuffer::Create(
EchoCanceller3Config(), rate, num_render_channels));
std::vector<std::vector<std::vector<float>>> render(
NumBandsForRate(rate),
std::vector<std::vector<float>>(num_render_channels,
std::vector<float>(kBlockSize, 0.f)));
std::vector<std::vector<std::vector<float>>> capture(
NumBandsForRate(rate),
std::vector<std::vector<float>>(num_capture_channels,
std::vector<float>(kBlockSize, 0.f)));
Block render(NumBandsForRate(rate), num_render_channels);
Block capture(NumBandsForRate(rate), num_capture_channels);
for (size_t k = 0; k < 100; ++k) {
EchoPathVariability echo_path_variability(
k % 3 == 0 ? true : false,
@ -97,27 +91,6 @@ TEST(EchoRemoverDeathTest, DISABLED_WrongSampleRate) {
"");
}
// Verifies the check for the capture block size.
TEST(EchoRemoverDeathTest, WrongCaptureBlockSize) {
absl::optional<DelayEstimate> delay_estimate;
for (auto rate : {16000, 32000, 48000}) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<EchoRemover> remover(
EchoRemover::Create(EchoCanceller3Config(), rate, 1, 1));
std::unique_ptr<RenderDelayBuffer> render_buffer(
RenderDelayBuffer::Create(EchoCanceller3Config(), rate, 1));
std::vector<std::vector<std::vector<float>>> capture(
NumBandsForRate(rate), std::vector<std::vector<float>>(
1, std::vector<float>(kBlockSize - 1, 0.f)));
EchoPathVariability echo_path_variability(
false, EchoPathVariability::DelayAdjustment::kNone, false);
EXPECT_DEATH(remover->ProcessCapture(
echo_path_variability, false, delay_estimate,
render_buffer->GetRenderBuffer(), nullptr, &capture),
"");
}
}
// Verifies the check for the number of capture bands.
// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH
// tests on test bots has been fixed.c
@ -129,10 +102,7 @@ TEST(EchoRemoverDeathTest, DISABLED_WrongCaptureNumBands) {
EchoRemover::Create(EchoCanceller3Config(), rate, 1, 1));
std::unique_ptr<RenderDelayBuffer> render_buffer(
RenderDelayBuffer::Create(EchoCanceller3Config(), rate, 1));
std::vector<std::vector<std::vector<float>>> capture(
NumBandsForRate(rate == 48000 ? 16000 : rate + 16000),
std::vector<std::vector<float>>(1,
std::vector<float>(kBlockSize, 0.f)));
Block capture(NumBandsForRate(rate == 48000 ? 16000 : rate + 16000), 1);
EchoPathVariability echo_path_variability(
false, EchoPathVariability::DelayAdjustment::kNone, false);
EXPECT_DEATH(remover->ProcessCapture(
@ -167,14 +137,8 @@ TEST(EchoRemover, BasicEchoRemoval) {
absl::optional<DelayEstimate> delay_estimate;
for (size_t num_channels : {1, 2, 4}) {
for (auto rate : {16000, 32000, 48000}) {
std::vector<std::vector<std::vector<float>>> x(
NumBandsForRate(rate),
std::vector<std::vector<float>>(num_channels,
std::vector<float>(kBlockSize, 0.f)));
std::vector<std::vector<std::vector<float>>> y(
NumBandsForRate(rate),
std::vector<std::vector<float>>(num_channels,
std::vector<float>(kBlockSize, 0.f)));
Block x(NumBandsForRate(rate), num_channels);
Block y(NumBandsForRate(rate), num_channels);
EchoPathVariability echo_path_variability(
false, EchoPathVariability::DelayAdjustment::kNone, false);
for (size_t delay_samples : {0, 64, 150, 200, 301}) {
@ -187,13 +151,13 @@ TEST(EchoRemover, BasicEchoRemoval) {
render_buffer->AlignFromDelay(delay_samples / kBlockSize);
std::vector<std::vector<std::unique_ptr<DelayBuffer<float>>>>
delay_buffers(x.size());
delay_buffers(x.NumBands());
for (size_t band = 0; band < delay_buffers.size(); ++band) {
delay_buffers[band].resize(x[0].size());
delay_buffers[band].resize(x.NumChannels());
}
for (size_t band = 0; band < x.size(); ++band) {
for (size_t channel = 0; channel < x[0].size(); ++channel) {
for (int band = 0; band < x.NumBands(); ++band) {
for (int channel = 0; channel < x.NumChannels(); ++channel) {
delay_buffers[band][channel].reset(
new DelayBuffer<float>(delay_samples));
}
@ -204,22 +168,23 @@ TEST(EchoRemover, BasicEchoRemoval) {
for (int k = 0; k < kNumBlocksToProcess; ++k) {
const bool silence = k < 100 || (k % 100 >= 10);
for (size_t band = 0; band < x.size(); ++band) {
for (size_t channel = 0; channel < x[0].size(); ++channel) {
for (int band = 0; band < x.NumBands(); ++band) {
for (int channel = 0; channel < x.NumChannels(); ++channel) {
if (silence) {
std::fill(x[band][channel].begin(), x[band][channel].end(),
0.f);
std::fill(x.begin(band, channel), x.end(band, channel), 0.f);
} else {
RandomizeSampleVector(&random_generator, x[band][channel]);
RandomizeSampleVector(&random_generator, x.View(band, channel));
}
delay_buffers[band][channel]->Delay(x[band][channel],
y[band][channel]);
delay_buffers[band][channel]->Delay(x.View(band, channel),
y.View(band, channel));
}
}
if (k > kNumBlocksToProcess / 2) {
input_energy = std::inner_product(y[0][0].begin(), y[0][0].end(),
y[0][0].begin(), input_energy);
input_energy = std::inner_product(
y.begin(/*band=*/0, /*channel=*/0),
y.end(/*band=*/0, /*channel=*/0),
y.begin(/*band=*/0, /*channel=*/0), input_energy);
}
render_buffer->Insert(x);
@ -230,8 +195,10 @@ TEST(EchoRemover, BasicEchoRemoval) {
&y);
if (k > kNumBlocksToProcess / 2) {
output_energy = std::inner_product(y[0][0].begin(), y[0][0].end(),
y[0][0].begin(), output_energy);
output_energy = std::inner_product(
y.begin(/*band=*/0, /*channel=*/0),
y.end(/*band=*/0, /*channel=*/0),
y.begin(/*band=*/0, /*channel=*/0), output_energy);
}
}
EXPECT_GT(input_energy, 10.f * output_energy);

26
modules/audio_processing/aec3/erle_estimator_unittest.cc
View File

@ -60,7 +60,7 @@ void VerifyErleGreaterOrEqual(
}
}
void FormFarendTimeFrame(std::vector<std::vector<std::vector<float>>>* x) {
void FormFarendTimeFrame(Block* x) {
const std::array<float, kBlockSize> frame = {
7459.88, 17209.6, 17383, 20768.9, 16816.7, 18386.3, 4492.83, 9675.85,
6665.52, 14808.6, 9342.3, 7483.28, 19261.7, 4145.98, 1622.18, 13475.2,
@ -70,10 +70,10 @@ void FormFarendTimeFrame(std::vector<std::vector<std::vector<float>>>* x) {
11405, 15031.4, 14541.6, 19765.5, 18346.3, 19350.2, 3157.47, 18095.8,
1743.68, 21328.2, 19727.5, 7295.16, 10332.4, 11055.5, 20107.4, 14708.4,
12416.2, 16434, 2454.69, 9840.8, 6867.23, 1615.75, 6059.9, 8394.19};
for (size_t band = 0; band < x->size(); ++band) {
for (size_t channel = 0; channel < (*x)[band].size(); ++channel) {
RTC_DCHECK_GE((*x)[band][channel].size(), frame.size());
std::copy(frame.begin(), frame.end(), (*x)[band][channel].begin());
for (int band = 0; band < x->NumBands(); ++band) {
for (int channel = 0; channel < x->NumChannels(); ++channel) {
RTC_DCHECK_GE(kBlockSize, frame.size());
std::copy(frame.begin(), frame.end(), x->begin(band, channel));
}
}
}
@ -104,13 +104,13 @@ void FormFarendFrame(const RenderBuffer& render_buffer,
}
void FormNearendFrame(
std::vector<std::vector<std::vector<float>>>* x,
Block* x,
std::array<float, kFftLengthBy2Plus1>* X2,
rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>> E2,
rtc::ArrayView<std::array<float, kFftLengthBy2Plus1>> Y2) {
for (size_t band = 0; band < x->size(); ++band) {
for (size_t ch = 0; ch < (*x)[band].size(); ++ch) {
std::fill((*x)[band][ch].begin(), (*x)[band][ch].end(), 0.f);
for (int band = 0; band < x->NumBands(); ++band) {
for (int ch = 0; ch < x->NumChannels(); ++ch) {
std::fill(x->begin(band, ch), x->end(band, ch), 0.f);
}
}
@ -162,9 +162,7 @@ TEST_P(ErleEstimatorMultiChannel, VerifyErleIncreaseAndHold) {
EchoCanceller3Config config;
config.erle.onset_detection = true;
std::vector<std::vector<std::vector<float>>> x(
kNumBands, std::vector<std::vector<float>>(
num_render_channels, std::vector<float>(kBlockSize, 0.f)));
Block x(kNumBands, num_render_channels);
std::vector<std::vector<std::array<float, kFftLengthBy2Plus1>>>
filter_frequency_response(
config.filter.refined.length_blocks,
@ -227,9 +225,7 @@ TEST_P(ErleEstimatorMultiChannel, VerifyErleTrackingOnOnsets) {
std::vector<bool> converged_filters(num_capture_channels, true);
EchoCanceller3Config config;
config.erle.onset_detection = true;
std::vector<std::vector<std::vector<float>>> x(
kNumBands, std::vector<std::vector<float>>(
num_render_channels, std::vector<float>(kBlockSize, 0.f)));
Block x(kNumBands, num_render_channels);
std::vector<std::vector<std::array<float, kFftLengthBy2Plus1>>>
filter_frequency_response(
config.filter.refined.length_blocks,

8
modules/audio_processing/aec3/filter_analyzer.cc
View File

@ -131,7 +131,7 @@ void FilterAnalyzer::AnalyzeRegion(
st_ch.consistent_estimate = st_ch.consistent_filter_detector.Detect(
h_highpass_[ch], region_,
render_buffer.Block(-filter_delays_blocks_[ch])[0], st_ch.peak_index,
render_buffer.Block(-filter_delays_blocks_[ch]), st_ch.peak_index,
filter_delays_blocks_[ch]);
}
}
@ -224,7 +224,7 @@ void FilterAnalyzer::ConsistentFilterDetector::Reset() {
bool FilterAnalyzer::ConsistentFilterDetector::Detect(
rtc::ArrayView<const float> filter_to_analyze,
const FilterRegion& region,
rtc::ArrayView<const std::vector<float>> x_block,
const Block& x_block,
size_t peak_index,
int delay_blocks) {
if (region.start_sample_ == 0) {
@ -265,7 +265,9 @@ bool FilterAnalyzer::ConsistentFilterDetector::Detect(
if (significant_peak_) {
bool active_render_block = false;
for (auto& x_channel : x_block) {
for (int ch = 0; ch < x_block.NumChannels(); ++ch) {
rtc::ArrayView<const float, kBlockSize> x_channel =
x_block.View(/*band=*/0, ch);
const float x_energy = std::inner_product(
x_channel.begin(), x_channel.end(), x_channel.begin(), 0.f);
if (x_energy > active_render_threshold_) {

3
modules/audio_processing/aec3/filter_analyzer.h
View File

@ -20,6 +20,7 @@
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/block.h"
namespace webrtc {
@ -93,7 +94,7 @@ class FilterAnalyzer {
void Reset();
bool Detect(rtc::ArrayView<const float> filter_to_analyze,
const FilterRegion& region,
rtc::ArrayView<const std::vector<float>> x_block,
const Block& x_block,
size_t peak_index,
int delay_blocks);

34
modules/audio_processing/aec3/frame_blocker.cc
View File

@ -33,26 +33,22 @@ FrameBlocker::~FrameBlocker() = default;
void FrameBlocker::InsertSubFrameAndExtractBlock(
const std::vector<std::vector<rtc::ArrayView<float>>>& sub_frame,
std::vector<std::vector<std::vector<float>>>* block) {
Block* block) {
RTC_DCHECK(block);
RTC_DCHECK_EQ(num_bands_, block->size());
RTC_DCHECK_EQ(num_bands_, block->NumBands());
RTC_DCHECK_EQ(num_bands_, sub_frame.size());
for (size_t band = 0; band < num_bands_; ++band) {
RTC_DCHECK_EQ(num_channels_, (*block)[band].size());
RTC_DCHECK_EQ(num_channels_, block->NumChannels());
RTC_DCHECK_EQ(num_channels_, sub_frame[band].size());
for (size_t channel = 0; channel < num_channels_; ++channel) {
RTC_DCHECK_GE(kBlockSize - 16, buffer_[band][channel].size());
RTC_DCHECK_EQ(kBlockSize, (*block)[band][channel].size());
RTC_DCHECK_EQ(kSubFrameLength, sub_frame[band][channel].size());
const int samples_to_block = kBlockSize - buffer_[band][channel].size();
(*block)[band][channel].clear();
(*block)[band][channel].insert((*block)[band][channel].begin(),
buffer_[band][channel].begin(),
buffer_[band][channel].end());
(*block)[band][channel].insert(
(*block)[band][channel].begin() + buffer_[band][channel].size(),
sub_frame[band][channel].begin(),
sub_frame[band][channel].begin() + samples_to_block);
std::copy(buffer_[band][channel].begin(), buffer_[band][channel].end(),
block->begin(band, channel));
std::copy(sub_frame[band][channel].begin(),
sub_frame[band][channel].begin() + samples_to_block,
block->begin(band, channel) + kBlockSize - samples_to_block);
buffer_[band][channel].clear();
buffer_[band][channel].insert(
buffer_[band][channel].begin(),
@ -66,20 +62,16 @@ bool FrameBlocker::IsBlockAvailable() const {
return kBlockSize == buffer_[0][0].size();
}
void FrameBlocker::ExtractBlock(
std::vector<std::vector<std::vector<float>>>* block) {
void FrameBlocker::ExtractBlock(Block* block) {
RTC_DCHECK(block);
RTC_DCHECK_EQ(num_bands_, block->size());
RTC_DCHECK_EQ(num_bands_, block->NumBands());
RTC_DCHECK_EQ(num_channels_, block->NumChannels());
RTC_DCHECK(IsBlockAvailable());
for (size_t band = 0; band < num_bands_; ++band) {
RTC_DCHECK_EQ(num_channels_, (*block)[band].size());
for (size_t channel = 0; channel < num_channels_; ++channel) {
RTC_DCHECK_EQ(kBlockSize, buffer_[band][channel].size());
RTC_DCHECK_EQ(kBlockSize, (*block)[band][channel].size());
(*block)[band][channel].clear();
(*block)[band][channel].insert((*block)[band][channel].begin(),
buffer_[band][channel].begin(),
buffer_[band][channel].end());
std::copy(buffer_[band][channel].begin(), buffer_[band][channel].end(),
block->begin(band, channel));
buffer_[band][channel].clear();
}
}

5
modules/audio_processing/aec3/frame_blocker.h
View File

@ -17,6 +17,7 @@
#include "api/array_view.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/block.h"
namespace webrtc {
@ -33,12 +34,12 @@ class FrameBlocker {
// extracts one 64 sample multiband block.
void InsertSubFrameAndExtractBlock(
const std::vector<std::vector<rtc::ArrayView<float>>>& sub_frame,
std::vector<std::vector<std::vector<float>>>* block);
Block* block);
// Reports whether a multiband block of 64 samples is available for
// extraction.
bool IsBlockAvailable() const;
// Extracts a multiband block of 64 samples.
void ExtractBlock(std::vector<std::vector<std::vector<float>>>* block);
void ExtractBlock(Block* block);
private:
const size_t num_bands_;

91
modules/audio_processing/aec3/frame_blocker_unittest.cc
View File

@ -86,15 +86,14 @@ bool VerifySubFrame(
return true;
}
bool VerifyBlock(size_t block_counter,
int offset,
const std::vector<std::vector<std::vector<float>>>& block) {
for (size_t band = 0; band < block.size(); ++band) {
for (size_t channel = 0; channel < block[band].size(); ++channel) {
for (size_t sample = 0; sample < block[band][channel].size(); ++sample) {
bool VerifyBlock(size_t block_counter, int offset, const Block& block) {
for (int band = 0; band < block.NumBands(); ++band) {
for (int channel = 0; channel < block.NumChannels(); ++channel) {
for (size_t sample = 0; sample < kBlockSize; ++sample) {
auto it = block.begin(band, channel) + sample;
const float reference_value = ComputeSampleValue(
block_counter, kBlockSize, band, channel, sample, offset);
if (reference_value != block[band][channel][sample]) {
if (reference_value != *it) {
return false;
}
}
@ -108,9 +107,7 @@ void RunBlockerTest(int sample_rate_hz, size_t num_channels) {
constexpr size_t kNumSubFramesToProcess = 20;
const size_t num_bands = NumBandsForRate(sample_rate_hz);
std::vector<std::vector<std::vector<float>>> block(
num_bands, std::vector<std::vector<float>>(
num_channels, std::vector<float>(kBlockSize, 0.f)));
Block block(num_bands, num_channels);
std::vector<std::vector<std::vector<float>>> input_sub_frame(
num_bands, std::vector<std::vector<float>>(
num_channels, std::vector<float>(kSubFrameLength, 0.f)));
@ -145,9 +142,7 @@ void RunBlockerAndFramerTest(int sample_rate_hz, size_t num_channels) {
const size_t kNumSubFramesToProcess = 20;
const size_t num_bands = NumBandsForRate(sample_rate_hz);
std::vector<std::vector<std::vector<float>>> block(
num_bands, std::vector<std::vector<float>>(
num_channels, std::vector<float>(kBlockSize, 0.f)));
Block block(num_bands, num_channels);
std::vector<std::vector<std::vector<float>>> input_sub_frame(
num_bands, std::vector<std::vector<float>>(
num_channels, std::vector<float>(kSubFrameLength, 0.f)));
@ -194,16 +189,12 @@ void RunWronglySizedInsertAndExtractParametersTest(
size_t correct_num_channels,
size_t num_block_bands,
size_t num_block_channels,
size_t block_length,
size_t num_sub_frame_bands,
size_t num_sub_frame_channels,
size_t sub_frame_length) {
const size_t correct_num_bands = NumBandsForRate(sample_rate_hz);
std::vector<std::vector<std::vector<float>>> block(
num_block_bands,
std::vector<std::vector<float>>(num_block_channels,
std::vector<float>(block_length, 0.f)));
Block block(num_block_bands, num_block_channels);
std::vector<std::vector<std::vector<float>>> input_sub_frame(
num_sub_frame_bands,
std::vector<std::vector<float>>(
@ -222,18 +213,11 @@ void RunWronglySizedInsertAndExtractParametersTest(
void RunWronglySizedExtractParameterTest(int sample_rate_hz,
size_t correct_num_channels,
size_t num_block_bands,
size_t num_block_channels,
size_t block_length) {
size_t num_block_channels) {
const size_t correct_num_bands = NumBandsForRate(sample_rate_hz);
std::vector<std::vector<std::vector<float>>> correct_block(
correct_num_bands,
std::vector<std::vector<float>>(correct_num_channels,
std::vector<float>(kBlockSize, 0.f)));
std::vector<std::vector<std::vector<float>>> wrong_block(
num_block_bands,
std::vector<std::vector<float>>(num_block_channels,
std::vector<float>(block_length, 0.f)));
Block correct_block(correct_num_bands, correct_num_channels);
Block wrong_block(num_block_bands, num_block_channels);
std::vector<std::vector<std::vector<float>>> input_sub_frame(
correct_num_bands,
std::vector<std::vector<float>>(
@ -259,9 +243,7 @@ void RunWrongExtractOrderTest(int sample_rate_hz,
size_t num_preceeding_api_calls) {
const size_t num_bands = NumBandsForRate(sample_rate_hz);
std::vector<std::vector<std::vector<float>>> block(
num_bands, std::vector<std::vector<float>>(
num_channels, std::vector<float>(kBlockSize, 0.f)));
Block block(num_bands, num_channels);
std::vector<std::vector<std::vector<float>>> input_sub_frame(
num_bands, std::vector<std::vector<float>>(
num_channels, std::vector<float>(kSubFrameLength, 0.f)));
@ -296,7 +278,7 @@ TEST(FrameBlockerDeathTest,
const size_t wrong_num_bands = (correct_num_bands % 3) + 1;
RunWronglySizedInsertAndExtractParametersTest(
rate, correct_num_channels, wrong_num_bands, correct_num_channels,
kBlockSize, correct_num_bands, correct_num_channels, kSubFrameLength);
correct_num_bands, correct_num_channels, kSubFrameLength);
}
}
}
@ -310,7 +292,7 @@ TEST(FrameBlockerDeathTest,
const size_t wrong_num_channels = correct_num_channels + 1;
RunWronglySizedInsertAndExtractParametersTest(
rate, correct_num_channels, correct_num_bands, wrong_num_channels,
kBlockSize, correct_num_bands, correct_num_channels, kSubFrameLength);
correct_num_bands, correct_num_channels, kSubFrameLength);
}
}
}
@ -324,7 +306,7 @@ TEST(FrameBlockerDeathTest,
const size_t wrong_num_bands = (correct_num_bands % 3) + 1;
RunWronglySizedInsertAndExtractParametersTest(
rate, correct_num_channels, correct_num_bands, correct_num_channels,
kBlockSize, wrong_num_bands, correct_num_channels, kSubFrameLength);
wrong_num_bands, correct_num_channels, kSubFrameLength);
}
}
}
@ -338,21 +320,7 @@ TEST(FrameBlockerDeathTest,
const size_t wrong_num_channels = correct_num_channels + 1;
RunWronglySizedInsertAndExtractParametersTest(
rate, correct_num_channels, correct_num_bands, wrong_num_channels,
kBlockSize, correct_num_bands, wrong_num_channels, kSubFrameLength);
}
}
}
TEST(FrameBlockerDeathTest,
WrongNumberOfSamplesInBlockForInsertSubFrameAndExtractBlock) {
for (auto rate : {16000, 32000, 48000}) {
for (size_t correct_num_channels : {1, 2, 4, 8}) {
SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels));
const size_t correct_num_bands = NumBandsForRate(rate);
RunWronglySizedInsertAndExtractParametersTest(
rate, correct_num_channels, correct_num_bands, correct_num_channels,
kBlockSize - 1, correct_num_bands, correct_num_channels,
kSubFrameLength);
correct_num_bands, wrong_num_channels, kSubFrameLength);
}
}
}
@ -365,8 +333,7 @@ TEST(FrameBlockerDeathTest,
const size_t correct_num_bands = NumBandsForRate(rate);
RunWronglySizedInsertAndExtractParametersTest(
rate, correct_num_channels, correct_num_bands, correct_num_channels,
kBlockSize, correct_num_bands, correct_num_channels,
kSubFrameLength - 1);
correct_num_bands, correct_num_channels, kSubFrameLength - 1);
}
}
}
@ -377,9 +344,8 @@ TEST(FrameBlockerDeathTest, WrongNumberOfBandsInBlockForExtractBlock) {
SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels));
const size_t correct_num_bands = NumBandsForRate(rate);
const size_t wrong_num_bands = (correct_num_bands % 3) + 1;
RunWronglySizedExtractParameterTest(rate, correct_num_channels,
wrong_num_bands, correct_num_channels,
kBlockSize);
RunWronglySizedExtractParameterTest(
rate, correct_num_channels, wrong_num_bands, correct_num_channels);
}
}
}
@ -390,21 +356,8 @@ TEST(FrameBlockerDeathTest, WrongNumberOfChannelsInBlockForExtractBlock) {
SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels));
const size_t correct_num_bands = NumBandsForRate(rate);
const size_t wrong_num_channels = correct_num_channels + 1;
RunWronglySizedExtractParameterTest(rate, correct_num_channels,
correct_num_bands, wrong_num_channels,
kBlockSize);
}
}
}
TEST(FrameBlockerDeathTest, WrongNumberOfSamplesInBlockForExtractBlock) {
for (auto rate : {16000, 32000, 48000}) {
for (size_t correct_num_channels : {1, 2, 4, 8}) {
SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels));
const size_t correct_num_bands = NumBandsForRate(rate);
RunWronglySizedExtractParameterTest(rate, correct_num_channels,
correct_num_bands,
correct_num_channels, kBlockSize - 1);
RunWronglySizedExtractParameterTest(
rate, correct_num_channels, correct_num_bands, wrong_num_channels);
}
}
}

17
modules/audio_processing/aec3/matched_filter_unittest.cc
View File

@ -210,9 +210,7 @@ TEST(MatchedFilter, LagEstimation) {
for (auto down_sampling_factor : kDownSamplingFactors) {
const size_t sub_block_size = kBlockSize / down_sampling_factor;
std::vector<std::vector<std::vector<float>>> render(
kNumBands, std::vector<std::vector<float>>(
kNumChannels, std::vector<float>(kBlockSize, 0.f)));
Block render(kNumBands, kNumChannels);
std::vector<std::vector<float>> capture(
1, std::vector<float>(kBlockSize, 0.f));
ApmDataDumper data_dumper(0);
@ -238,10 +236,12 @@ TEST(MatchedFilter, LagEstimation) {
for (size_t k = 0; k < (600 + delay_samples / sub_block_size); ++k) {
for (size_t band = 0; band < kNumBands; ++band) {
for (size_t channel = 0; channel < kNumChannels; ++channel) {
RandomizeSampleVector(&random_generator, render[band][channel]);
RandomizeSampleVector(&random_generator,
render.View(band, channel));
}
}
signal_delay_buffer.Delay(render[0][0], capture[0]);
signal_delay_buffer.Delay(render.View(/*band=*/0, /*channel=*/0),
capture[0]);
render_delay_buffer->Insert(render);
if (k == 0) {
@ -328,9 +328,7 @@ TEST(MatchedFilter, LagNotReliableForUncorrelatedRenderAndCapture) {
config.delay.num_filters = kNumMatchedFilters;
const size_t sub_block_size = kBlockSize / down_sampling_factor;
std::vector<std::vector<std::vector<float>>> render(
kNumBands, std::vector<std::vector<float>>(
kNumChannels, std::vector<float>(kBlockSize, 0.f)));
Block render(kNumBands, kNumChannels);
std::array<float, kBlockSize> capture_data;
rtc::ArrayView<float> capture(capture_data.data(), sub_block_size);
std::fill(capture.begin(), capture.end(), 0.f);
@ -346,7 +344,8 @@ TEST(MatchedFilter, LagNotReliableForUncorrelatedRenderAndCapture) {
// Analyze the correlation between render and capture.
for (size_t k = 0; k < 100; ++k) {
RandomizeSampleVector(&random_generator, render[0][0]);
RandomizeSampleVector(&random_generator,
render.View(/*band=*/0, /*channel=*/0));
RandomizeSampleVector(&random_generator, capture);
render_delay_buffer->Insert(render);
filter.Update(render_delay_buffer->GetDownsampledRenderBuffer(), capture,

9
modules/audio_processing/aec3/mock/mock_block_processor.h
View File

@ -28,13 +28,10 @@ class MockBlockProcessor : public BlockProcessor {
ProcessCapture,
(bool level_change,
bool saturated_microphone_signal,
std::vector<std::vector<std::vector<float>>>* linear_output,
std::vector<std::vector<std::vector<float>>>* capture_block),
(override));
MOCK_METHOD(void,
BufferRender,
(const std::vector<std::vector<std::vector<float>>>& block),
Block* linear_output,
Block* capture_block),
(override));
MOCK_METHOD(void, BufferRender, (const Block& block), (override));
MOCK_METHOD(void,
UpdateEchoLeakageStatus,
(bool leakage_detected),

4
modules/audio_processing/aec3/mock/mock_echo_remover.h
View File

@ -33,8 +33,8 @@ class MockEchoRemover : public EchoRemover {
bool capture_signal_saturation,
const absl::optional<DelayEstimate>& delay_estimate,
RenderBuffer* render_buffer,
std::vector<std::vector<std::vector<float>>>* linear_output,
std::vector<std::vector<std::vector<float>>>* capture),
Block* linear_output,
Block* capture),
(override));
MOCK_METHOD(void,
UpdateEchoLeakageStatus,

3
modules/audio_processing/aec3/mock/mock_render_delay_buffer.cc
View File

@ -17,8 +17,7 @@ MockRenderDelayBuffer::MockRenderDelayBuffer(int sample_rate_hz,
size_t num_channels)
: block_buffer_(GetRenderDelayBufferSize(4, 4, 12),
NumBandsForRate(sample_rate_hz),
num_channels,
kBlockSize),
num_channels),
spectrum_buffer_(block_buffer_.buffer.size(), num_channels),
fft_buffer_(block_buffer_.buffer.size(), num_channels),
render_buffer_(&block_buffer_, &spectrum_buffer_, &fft_buffer_),

2
modules/audio_processing/aec3/mock/mock_render_delay_buffer.h
View File

@ -30,7 +30,7 @@ class MockRenderDelayBuffer : public RenderDelayBuffer {
MOCK_METHOD(void, Reset, (), (override));
MOCK_METHOD(RenderDelayBuffer::BufferingEvent,
Insert,
(const std::vector<std::vector<std::vector<float>>>& block),
(const Block& block),
(override));
MOCK_METHOD(void, HandleSkippedCaptureProcessing, (), (override));
MOCK_METHOD(RenderDelayBuffer::BufferingEvent,

2
modules/audio_processing/aec3/mock/mock_render_delay_controller.h
View File

@ -31,7 +31,7 @@ class MockRenderDelayController : public RenderDelayController {
GetDelay,
(const DownsampledRenderBuffer& render_buffer,
size_t render_delay_buffer_delay,
const std::vector<std::vector<float>>& capture),
const Block& capture),
(override));
MOCK_METHOD(bool, HasClockdrift, (), (const, override));
};

18
modules/audio_processing/aec3/refined_filter_update_gain_unittest.cc
View File

@ -84,9 +84,7 @@ void RunFilterUpdateTest(int num_blocks_to_process,
RefinedFilterUpdateGain refined_gain(
config.filter.refined, config.filter.config_change_duration_blocks);
Random random_generator(42U);
std::vector<std::vector<std::vector<float>>> x(
kNumBands, std::vector<std::vector<float>>(
kNumRenderChannels, std::vector<float>(kBlockSize, 0.f)));
Block x(kNumBands, kNumRenderChannels);
std::vector<float> y(kBlockSize, 0.f);
config.delay.default_delay = 1;
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
@ -131,19 +129,19 @@ void RunFilterUpdateTest(int num_blocks_to_process,
// Create the render signal.
if (use_silent_render_in_second_half && k > num_blocks_to_process / 2) {
for (size_t band = 0; band < x.size(); ++band) {
for (size_t channel = 0; channel < x[band].size(); ++channel) {
std::fill(x[band][channel].begin(), x[band][channel].end(), 0.f);
for (int band = 0; band < x.NumBands(); ++band) {
for (int channel = 0; channel < x.NumChannels(); ++channel) {
std::fill(x.begin(band, channel), x.end(band, channel), 0.f);
}
}
} else {
for (size_t band = 0; band < x.size(); ++band) {
for (size_t channel = 0; channel < x[band].size(); ++channel) {
RandomizeSampleVector(&random_generator, x[band][channel]);
for (int band = 0; band < x.NumChannels(); ++band) {
for (int channel = 0; channel < x.NumChannels(); ++channel) {
RandomizeSampleVector(&random_generator, x.View(band, channel));
}
}
}
delay_buffer.Delay(x[0][0], y);
delay_buffer.Delay(x.View(/*band=*/0, /*channel=*/0), y);
render_delay_buffer->Insert(x);
if (k == 0) {

3
modules/audio_processing/aec3/render_buffer.h
View File

@ -40,8 +40,7 @@ class RenderBuffer {
~RenderBuffer();
// Get a block.
const std::vector<std::vector<std::vector<float>>>& Block(
int buffer_offset_blocks) const {
const Block& Block(int buffer_offset_blocks) const {
int position =
block_buffer_->OffsetIndex(block_buffer_->read, buffer_offset_blocks);
return block_buffer_->buffer[position];

4
modules/audio_processing/aec3/render_buffer_unittest.cc
View File

@ -22,7 +22,7 @@ namespace webrtc {
// Verifies the check for non-null fft buffer.
TEST(RenderBufferDeathTest, NullExternalFftBuffer) {
BlockBuffer block_buffer(10, 3, 1, kBlockSize);
BlockBuffer block_buffer(10, 3, 1);
SpectrumBuffer spectrum_buffer(10, 1);
EXPECT_DEATH(RenderBuffer(&block_buffer, &spectrum_buffer, nullptr), "");
}
@ -30,7 +30,7 @@ TEST(RenderBufferDeathTest, NullExternalFftBuffer) {
// Verifies the check for non-null spectrum buffer.
TEST(RenderBufferDeathTest, NullExternalSpectrumBuffer) {
FftBuffer fft_buffer(10, 1);
BlockBuffer block_buffer(10, 3, 1, kBlockSize);
BlockBuffer block_buffer(10, 3, 1);
EXPECT_DEATH(RenderBuffer(&block_buffer, nullptr, &fft_buffer), "");
}

49
modules/audio_processing/aec3/render_delay_buffer.cc
View File

@ -54,8 +54,7 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer {
~RenderDelayBufferImpl() override;
void Reset() override;
BufferingEvent Insert(
const std::vector<std::vector<std::vector<float>>>& block) override;
BufferingEvent Insert(const Block& block) override;
BufferingEvent PrepareCaptureProcessing() override;
void HandleSkippedCaptureProcessing() override;
bool AlignFromDelay(size_t delay) override;
@ -110,8 +109,7 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer {
int MapDelayToTotalDelay(size_t delay) const;
int ComputeDelay() const;
void ApplyTotalDelay(int delay);
void InsertBlock(const std::vector<std::vector<std::vector<float>>>& block,
int previous_write);
void InsertBlock(const Block& block, int previous_write);
bool DetectActiveRender(rtc::ArrayView<const float> x) const;
bool DetectExcessRenderBlocks();
void IncrementWriteIndices();
@ -145,8 +143,7 @@ RenderDelayBufferImpl::RenderDelayBufferImpl(const EchoCanceller3Config& config,
config.delay.num_filters,
config.filter.refined.length_blocks),
NumBandsForRate(sample_rate_hz),
num_render_channels,
kBlockSize),
num_render_channels),
spectra_(blocks_.buffer.size(), num_render_channels),
ffts_(blocks_.buffer.size(), num_render_channels),
delay_(config_.delay.default_delay),
@ -161,7 +158,7 @@ RenderDelayBufferImpl::RenderDelayBufferImpl(const EchoCanceller3Config& config,
RTC_DCHECK_EQ(blocks_.buffer.size(), ffts_.buffer.size());
RTC_DCHECK_EQ(spectra_.buffer.size(), ffts_.buffer.size());
for (size_t i = 0; i < blocks_.buffer.size(); ++i) {
RTC_DCHECK_EQ(blocks_.buffer[i][0].size(), ffts_.buffer[i].size());
RTC_DCHECK_EQ(blocks_.buffer[i].NumChannels(), ffts_.buffer[i].size());
RTC_DCHECK_EQ(spectra_.buffer[i].size(), ffts_.buffer[i].size());
}
@ -211,7 +208,7 @@ void RenderDelayBufferImpl::Reset() {
// Inserts a new block into the render buffers.
RenderDelayBuffer::BufferingEvent RenderDelayBufferImpl::Insert(
const std::vector<std::vector<std::vector<float>>>& block) {
const Block& block) {
++render_call_counter_;
if (delay_) {
if (!last_call_was_render_) {
@ -239,7 +236,8 @@ RenderDelayBuffer::BufferingEvent RenderDelayBufferImpl::Insert(
// Detect and update render activity.
if (!render_activity_) {
render_activity_counter_ += DetectActiveRender(block[0][0]) ? 1 : 0;
render_activity_counter_ +=
DetectActiveRender(block.View(/*band=*/0, /*channel=*/0)) ? 1 : 0;
render_activity_ = render_activity_counter_ >= 20;
}
@ -394,46 +392,45 @@ void RenderDelayBufferImpl::AlignFromExternalDelay() {
}
// Inserts a block into the render buffers.
void RenderDelayBufferImpl::InsertBlock(
const std::vector<std::vector<std::vector<float>>>& block,
int previous_write) {
void RenderDelayBufferImpl::InsertBlock(const Block& block,
int previous_write) {
auto& b = blocks_;
auto& lr = low_rate_;
auto& ds = render_ds_;
auto& f = ffts_;
auto& s = spectra_;
const size_t num_bands = b.buffer[b.write].size();
const size_t num_render_channels = b.buffer[b.write][0].size();
RTC_DCHECK_EQ(block.size(), b.buffer[b.write].size());
const size_t num_bands = b.buffer[b.write].NumBands();
const size_t num_render_channels = b.buffer[b.write].NumChannels();
RTC_DCHECK_EQ(block.NumBands(), num_bands);
RTC_DCHECK_EQ(block.NumChannels(), num_render_channels);
for (size_t band = 0; band < num_bands; ++band) {
RTC_DCHECK_EQ(block[band].size(), num_render_channels);
RTC_DCHECK_EQ(b.buffer[b.write][band].size(), num_render_channels);
for (size_t ch = 0; ch < num_render_channels; ++ch) {
RTC_DCHECK_EQ(block[band][ch].size(), b.buffer[b.write][band][ch].size());
std::copy(block[band][ch].begin(), block[band][ch].end(),
b.buffer[b.write][band][ch].begin());
std::copy(block.begin(band, ch), block.end(band, ch),
b.buffer[b.write].begin(band, ch));
}
}
if (render_linear_amplitude_gain_ != 1.f) {
for (size_t band = 0; band < num_bands; ++band) {
for (size_t ch = 0; ch < num_render_channels; ++ch) {
for (size_t k = 0; k < 64; ++k) {
b.buffer[b.write][band][ch][k] *= render_linear_amplitude_gain_;
rtc::ArrayView<float, kBlockSize> b_view =
b.buffer[b.write].View(band, ch);
for (float& sample : b_view) {
sample *= render_linear_amplitude_gain_;
}
}
}
}
std::array<float, kBlockSize> downmixed_render;
render_mixer_.ProduceOutput(b.buffer[b.write][0], downmixed_render);
render_mixer_.ProduceOutput(b.buffer[b.write], downmixed_render);
render_decimator_.Decimate(downmixed_render, ds);
data_dumper_->DumpWav("aec3_render_decimator_output", ds.size(), ds.data(),
16000 / down_sampling_factor_, 1);
std::copy(ds.rbegin(), ds.rend(), lr.buffer.begin() + lr.write);
for (size_t channel = 0; channel < b.buffer[b.write][0].size(); ++channel) {
fft_.PaddedFft(b.buffer[b.write][0][channel],
b.buffer[previous_write][0][channel],
for (int channel = 0; channel < b.buffer[b.write].NumChannels(); ++channel) {
fft_.PaddedFft(b.buffer[b.write].View(/*band=*/0, channel),
b.buffer[previous_write].View(/*band=*/0, channel),
&f.buffer[f.write][channel]);
f.buffer[f.write][channel].Spectrum(optimization_,
s.buffer[s.write][channel]);

4
modules/audio_processing/aec3/render_delay_buffer.h
View File

@ -16,6 +16,7 @@
#include <vector>
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/block.h"
#include "modules/audio_processing/aec3/downsampled_render_buffer.h"
#include "modules/audio_processing/aec3/render_buffer.h"
@ -41,8 +42,7 @@ class RenderDelayBuffer {
virtual void Reset() = 0;
// Inserts a block into the buffer.
virtual BufferingEvent Insert(
const std::vector<std::vector<std::vector<float>>>& block) = 0;
virtual BufferingEvent Insert(const Block& block) = 0;
// Updates the buffers one step based on the specified buffer delay. Returns
// an enum indicating whether there was a special event that occurred.

36
modules/audio_processing/aec3/render_delay_buffer_unittest.cc
View File

@ -40,10 +40,7 @@ TEST(RenderDelayBuffer, BufferOverflow) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<RenderDelayBuffer> delay_buffer(
RenderDelayBuffer::Create(config, rate, num_channels));
std::vector<std::vector<std::vector<float>>> block_to_insert(
NumBandsForRate(rate),
std::vector<std::vector<float>>(num_channels,
std::vector<float>(kBlockSize, 0.f)));
Block block_to_insert(NumBandsForRate(rate), num_channels);
for (size_t k = 0; k < 10; ++k) {
EXPECT_EQ(RenderDelayBuffer::BufferingEvent::kNone,
delay_buffer->Insert(block_to_insert));
@ -69,9 +66,7 @@ TEST(RenderDelayBuffer, AvailableBlock) {
constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz);
std::unique_ptr<RenderDelayBuffer> delay_buffer(RenderDelayBuffer::Create(
EchoCanceller3Config(), kSampleRateHz, kNumChannels));
std::vector<std::vector<std::vector<float>>> input_block(
kNumBands, std::vector<std::vector<float>>(
kNumChannels, std::vector<float>(kBlockSize, 1.f)));
Block input_block(kNumBands, kNumChannels, 1.0f);
EXPECT_EQ(RenderDelayBuffer::BufferingEvent::kNone,
delay_buffer->Insert(input_block));
delay_buffer->PrepareCaptureProcessing();
@ -110,10 +105,8 @@ TEST(RenderDelayBufferDeathTest, WrongNumberOfBands) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<RenderDelayBuffer> delay_buffer(RenderDelayBuffer::Create(
EchoCanceller3Config(), rate, num_channels));
std::vector<std::vector<std::vector<float>>> block_to_insert(
NumBandsForRate(rate < 48000 ? rate + 16000 : 16000),
std::vector<std::vector<float>>(num_channels,
std::vector<float>(kBlockSize, 0.f)));
Block block_to_insert(
NumBandsForRate(rate < 48000 ? rate + 16000 : 16000), num_channels);
EXPECT_DEATH(delay_buffer->Insert(block_to_insert), "");
}
}
@ -126,26 +119,7 @@ TEST(RenderDelayBufferDeathTest, WrongNumberOfChannels) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<RenderDelayBuffer> delay_buffer(RenderDelayBuffer::Create(
EchoCanceller3Config(), rate, num_channels));
std::vector<std::vector<std::vector<float>>> block_to_insert(
NumBandsForRate(rate),
std::vector<std::vector<float>>(num_channels + 1,
std::vector<float>(kBlockSize, 0.f)));
EXPECT_DEATH(delay_buffer->Insert(block_to_insert), "");
}
}
}
// Verifies the check of the length of the inserted blocks.
TEST(RenderDelayBufferDeathTest, WrongBlockLength) {
for (auto rate : {16000, 32000, 48000}) {
for (size_t num_channels : {1, 2, 8}) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<RenderDelayBuffer> delay_buffer(RenderDelayBuffer::Create(
EchoCanceller3Config(), rate, num_channels));
std::vector<std::vector<std::vector<float>>> block_to_insert(
NumBandsForRate(rate),
std::vector<std::vector<float>>(
num_channels, std::vector<float>(kBlockSize - 1, 0.f)));
Block block_to_insert(NumBandsForRate(rate), num_channels + 1);
EXPECT_DEATH(delay_buffer->Insert(block_to_insert), "");
}
}

5
modules/audio_processing/aec3/render_delay_controller.cc
View File

@ -47,7 +47,7 @@ class RenderDelayControllerImpl final : public RenderDelayController {
absl::optional<DelayEstimate> GetDelay(
const DownsampledRenderBuffer& render_buffer,
size_t render_delay_buffer_delay,
const std::vector<std::vector<float>>& capture) override;
const Block& capture) override;
bool HasClockdrift() const override;
private:
@ -124,8 +124,7 @@ void RenderDelayControllerImpl::LogRenderCall() {}
absl::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
const DownsampledRenderBuffer& render_buffer,
size_t render_delay_buffer_delay,
const std::vector<std::vector<float>>& capture) {
RTC_DCHECK_EQ(kBlockSize, capture[0].size());
const Block& capture) {
++capture_call_counter_;
auto delay_samples = delay_estimator_.EstimateDelay(render_buffer, capture);

3
modules/audio_processing/aec3/render_delay_controller.h
View File

@ -14,6 +14,7 @@
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/block.h"
#include "modules/audio_processing/aec3/delay_estimate.h"
#include "modules/audio_processing/aec3/downsampled_render_buffer.h"
#include "modules/audio_processing/aec3/render_delay_buffer.h"
@ -40,7 +41,7 @@ class RenderDelayController {
virtual absl::optional<DelayEstimate> GetDelay(
const DownsampledRenderBuffer& render_buffer,
size_t render_delay_buffer_delay,
const std::vector<std::vector<float>>& capture) = 0;
const Block& capture) = 0;
// Returns true if clockdrift has been detected.
virtual bool HasClockdrift() const = 0;

89
modules/audio_processing/aec3/render_delay_controller_unittest.cc
View File

@ -53,8 +53,7 @@ constexpr size_t kDownSamplingFactors[] = {2, 4, 8};
// TODO(bugs.webrtc.org/11161): Re-enable tests.
TEST(RenderDelayController, DISABLED_NoRenderSignal) {
for (size_t num_render_channels : {1, 2, 8}) {
std::vector<std::vector<float>> block(1,
std::vector<float>(kBlockSize, 0.f));
Block block(/*num_bands=1*/ 1, /*num_channels=*/1);
EchoCanceller3Config config;
for (size_t num_matched_filters = 4; num_matched_filters <= 10;
num_matched_filters++) {
@ -85,8 +84,7 @@ TEST(RenderDelayController, DISABLED_NoRenderSignal) {
TEST(RenderDelayController, DISABLED_BasicApiCalls) {
for (size_t num_capture_channels : {1, 2, 4}) {
for (size_t num_render_channels : {1, 2, 8}) {
std::vector<std::vector<float>> capture_block(
num_capture_channels, std::vector<float>(kBlockSize, 0.f));
Block capture_block(/*num_bands=*/1, num_capture_channels);
absl::optional<DelayEstimate> delay_blocks;
for (size_t num_matched_filters = 4; num_matched_filters <= 10;
num_matched_filters++) {
@ -98,10 +96,7 @@ TEST(RenderDelayController, DISABLED_BasicApiCalls) {
config.delay.capture_alignment_mixing.adaptive_selection = false;
for (auto rate : {16000, 32000, 48000}) {
std::vector<std::vector<std::vector<float>>> render_block(
NumBandsForRate(rate),
std::vector<std::vector<float>>(
num_render_channels, std::vector<float>(kBlockSize, 0.f)));
Block render_block(NumBandsForRate(rate), num_render_channels);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create(config, rate, num_render_channels));
std::unique_ptr<RenderDelayController> delay_controller(
@ -130,8 +125,7 @@ TEST(RenderDelayController, DISABLED_BasicApiCalls) {
TEST(RenderDelayController, DISABLED_Alignment) {
Random random_generator(42U);
for (size_t num_capture_channels : {1, 2, 4}) {
std::vector<std::vector<float>> capture_block(
num_capture_channels, std::vector<float>(kBlockSize, 0.f));
Block capture_block(/*num_bands=*/1, num_capture_channels);
for (size_t num_matched_filters = 4; num_matched_filters <= 10;
num_matched_filters++) {
for (auto down_sampling_factor : kDownSamplingFactors) {
@ -143,10 +137,7 @@ TEST(RenderDelayController, DISABLED_Alignment) {
for (size_t num_render_channels : {1, 2, 8}) {
for (auto rate : {16000, 32000, 48000}) {
std::vector<std::vector<std::vector<float>>> render_block(
NumBandsForRate(rate),
std::vector<std::vector<float>>(
num_render_channels, std::vector<float>(kBlockSize, 0.f)));
Block render_block(NumBandsForRate(rate), num_render_channels);
for (size_t delay_samples : {15, 50, 150, 200, 800, 4000}) {
absl::optional<DelayEstimate> delay_blocks;
@ -160,14 +151,16 @@ TEST(RenderDelayController, DISABLED_Alignment) {
num_capture_channels));
DelayBuffer<float> signal_delay_buffer(delay_samples);
for (size_t k = 0; k < (400 + delay_samples / kBlockSize); ++k) {
for (size_t band = 0; band < render_block.size(); ++band) {
for (size_t channel = 0; channel < render_block[band].size();
for (int band = 0; band < render_block.NumBands(); ++band) {
for (int channel = 0; channel < render_block.NumChannels();
++channel) {
RandomizeSampleVector(&random_generator,
render_block[band][channel]);
render_block.View(band, channel));
}
}
signal_delay_buffer.Delay(render_block[0][0], capture_block[0]);
signal_delay_buffer.Delay(
render_block.View(/*band=*/0, /*channel=*/0),
capture_block.View(/*band=*/0, /*channel=*/0));
render_delay_buffer->Insert(render_block);
render_delay_buffer->PrepareCaptureProcessing();
delay_blocks = delay_controller->GetDelay(
@ -206,14 +199,8 @@ TEST(RenderDelayController, DISABLED_NonCausalAlignment) {
config.delay.capture_alignment_mixing.downmix = false;
config.delay.capture_alignment_mixing.adaptive_selection = false;
for (auto rate : {16000, 32000, 48000}) {
std::vector<std::vector<std::vector<float>>> render_block(
NumBandsForRate(rate),
std::vector<std::vector<float>>(
num_render_channels, std::vector<float>(kBlockSize, 0.f)));
std::vector<std::vector<std::vector<float>>> capture_block(
NumBandsForRate(rate),
std::vector<std::vector<float>>(
num_capture_channels, std::vector<float>(kBlockSize, 0.f)));
Block render_block(NumBandsForRate(rate), num_render_channels);
Block capture_block(NumBandsForRate(rate), num_capture_channels);
for (int delay_samples : {-15, -50, -150, -200}) {
absl::optional<DelayEstimate> delay_blocks;
@ -229,14 +216,17 @@ TEST(RenderDelayController, DISABLED_NonCausalAlignment) {
for (int k = 0;
k < (400 - delay_samples / static_cast<int>(kBlockSize));
++k) {
RandomizeSampleVector(&random_generator, capture_block[0][0]);
signal_delay_buffer.Delay(capture_block[0][0],
render_block[0][0]);
RandomizeSampleVector(
&random_generator,
capture_block.View(/*band=*/0, /*channel=*/0));
signal_delay_buffer.Delay(
capture_block.View(/*band=*/0, /*channel=*/0),
render_block.View(/*band=*/0, /*channel=*/0));
render_delay_buffer->Insert(render_block);
render_delay_buffer->PrepareCaptureProcessing();
delay_blocks = delay_controller->GetDelay(
render_delay_buffer->GetDownsampledRenderBuffer(),
render_delay_buffer->Delay(), capture_block[0]);
render_delay_buffer->Delay(), capture_block);
}
ASSERT_FALSE(delay_blocks);
@ -255,8 +245,8 @@ TEST(RenderDelayController, DISABLED_AlignmentWithJitter) {
Random random_generator(42U);
for (size_t num_capture_channels : {1, 2, 4}) {
for (size_t num_render_channels : {1, 2, 8}) {
std::vector<std::vector<float>> capture_block(
num_capture_channels, std::vector<float>(kBlockSize, 0.f));
Block capture_block(
/*num_bands=*/1, num_capture_channels);
for (size_t num_matched_filters = 4; num_matched_filters <= 10;
num_matched_filters++) {
for (auto down_sampling_factor : kDownSamplingFactors) {
@ -267,10 +257,7 @@ TEST(RenderDelayController, DISABLED_AlignmentWithJitter) {
config.delay.capture_alignment_mixing.adaptive_selection = false;
for (auto rate : {16000, 32000, 48000}) {
std::vector<std::vector<std::vector<float>>> render_block(
NumBandsForRate(rate),
std::vector<std::vector<float>>(
num_render_channels, std::vector<float>(kBlockSize, 0.f)));
Block render_block(NumBandsForRate(rate), num_render_channels);
for (size_t delay_samples : {15, 50, 300, 800}) {
absl::optional<DelayEstimate> delay_blocks;
SCOPED_TRACE(ProduceDebugText(rate, delay_samples,
@ -287,12 +274,14 @@ TEST(RenderDelayController, DISABLED_AlignmentWithJitter) {
kMaxTestJitterBlocks +
1;
++j) {
std::vector<std::vector<std::vector<float>>>
capture_block_buffer;
std::vector<Block> capture_block_buffer;
for (size_t k = 0; k < (kMaxTestJitterBlocks - 1); ++k) {
RandomizeSampleVector(&random_generator, render_block[0][0]);
signal_delay_buffer.Delay(render_block[0][0],
capture_block[0]);
RandomizeSampleVector(
&random_generator,
render_block.View(/*band=*/0, /*channel=*/0));
signal_delay_buffer.Delay(
render_block.View(/*band=*/0, /*channel=*/0),
capture_block.View(/*band=*/0, /*channel=*/0));
capture_block_buffer.push_back(capture_block);
render_delay_buffer->Insert(render_block);
}
@ -324,24 +313,6 @@ TEST(RenderDelayController, DISABLED_AlignmentWithJitter) {
#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID)
// Verifies the check for the capture signal block size.
TEST(RenderDelayControllerDeathTest, WrongCaptureSize) {
std::vector<std::vector<float>> block(
1, std::vector<float>(kBlockSize - 1, 0.f));
EchoCanceller3Config config;
for (auto rate : {16000, 32000, 48000}) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create(config, rate, 1));
EXPECT_DEATH(
std::unique_ptr<RenderDelayController>(
RenderDelayController::Create(EchoCanceller3Config(), rate, 1))
->GetDelay(render_delay_buffer->GetDownsampledRenderBuffer(),
render_delay_buffer->Delay(), block),
"");
}
}
// Verifies the check for correct sample rate.
// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH
// tests on test bots has been fixed.

16
modules/audio_processing/aec3/render_signal_analyzer.cc
View File

@ -66,10 +66,9 @@ void IdentifyStrongNarrowBandComponent(const RenderBuffer& render_buffer,
*narrow_peak_band = absl::nullopt;
}
const std::vector<std::vector<std::vector<float>>>& x_latest =
render_buffer.Block(0);
const Block& x_latest = render_buffer.Block(0);
float max_peak_level = 0.f;
for (size_t channel = 0; channel < x_latest[0].size(); ++channel) {
for (int channel = 0; channel < x_latest.NumChannels(); ++channel) {
rtc::ArrayView<const float, kFftLengthBy2Plus1> X2_latest =
render_buffer.Spectrum(0)[channel];
@ -90,13 +89,14 @@ void IdentifyStrongNarrowBandComponent(const RenderBuffer& render_buffer,
}
// Assess the render signal strength.
auto result0 = std::minmax_element(x_latest[0][channel].begin(),
x_latest[0][channel].end());
auto result0 = std::minmax_element(x_latest.begin(/*band=*/0, channel),
x_latest.end(/*band=*/0, channel));
float max_abs = std::max(fabs(*result0.first), fabs(*result0.second));
if (x_latest.size() > 1) {
const auto result1 = std::minmax_element(x_latest[1][channel].begin(),
x_latest[1][channel].end());
if (x_latest.NumBands() > 1) {
const auto result1 =
std::minmax_element(x_latest.begin(/*band=*/1, channel),
x_latest.end(/*band=*/1, channel));
max_abs =
std::max(max_abs, static_cast<float>(std::max(
fabs(*result1.first), fabs(*result1.second))));

26
modules/audio_processing/aec3/render_signal_analyzer_unittest.cc
View File

@ -36,18 +36,18 @@ void ProduceSinusoidInNoise(int sample_rate_hz,
float sinusoidal_frequency_hz,
Random* random_generator,
size_t* sample_counter,
std::vector<std::vector<std::vector<float>>>* x) {
Block* x) {
// Fill x with low-amplitude noise.
for (auto& band : *x) {
for (auto& channel : band) {
RandomizeSampleVector(random_generator, channel,
for (int band = 0; band < x->NumBands(); ++band) {
for (int channel = 0; channel < x->NumChannels(); ++channel) {
RandomizeSampleVector(random_generator, x->View(band, channel),
/*amplitude=*/500.f);
}
}
// Produce a sinusoid of the specified frequency in the specified channel.
for (size_t k = *sample_counter, j = 0; k < (*sample_counter + kBlockSize);
++k, ++j) {
(*x)[0][sinusoid_channel][j] +=
x->View(/*band=*/0, sinusoid_channel)[j] +=
32000.f *
std::sin(2.f * kPi * sinusoidal_frequency_hz * k / sample_rate_hz);
}
@ -59,9 +59,7 @@ void RunNarrowBandDetectionTest(size_t num_channels) {
Random random_generator(42U);
constexpr int kSampleRateHz = 48000;
constexpr size_t kNumBands = NumBandsForRate(kSampleRateHz);
std::vector<std::vector<std::vector<float>>> x(
kNumBands, std::vector<std::vector<float>>(
num_channels, std::vector<float>(kBlockSize, 0.f)));
Block x(kNumBands, num_channels);
std::array<float, kBlockSize> x_old;
Aec3Fft fft;
EchoCanceller3Config config;
@ -130,19 +128,17 @@ TEST(RenderSignalAnalyzer, NoFalseDetectionOfNarrowBands) {
SCOPED_TRACE(ProduceDebugText(num_channels));
RenderSignalAnalyzer analyzer(EchoCanceller3Config{});
Random random_generator(42U);
std::vector<std::vector<std::vector<float>>> x(
3, std::vector<std::vector<float>>(
num_channels, std::vector<float>(kBlockSize, 0.f)));
Block x(3, num_channels);
std::array<float, kBlockSize> x_old;
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create(EchoCanceller3Config(), 48000, num_channels));
std::array<float, kFftLengthBy2Plus1> mask;
x_old.fill(0.f);
for (size_t k = 0; k < 100; ++k) {
for (auto& band : x) {
for (auto& channel : band) {
RandomizeSampleVector(&random_generator, channel);
for (int k = 0; k < 100; ++k) {
for (int band = 0; band < x.NumBands(); ++band) {
for (int channel = 0; channel < x.NumChannels(); ++channel) {
RandomizeSampleVector(&random_generator, x.View(band, channel));
}
}

10
modules/audio_processing/aec3/residual_echo_estimator_unittest.cc
View File

@ -47,10 +47,7 @@ class ResidualEchoEstimatorTest {
Y2_(num_capture_channels_),
R2_(num_capture_channels_),
R2_unbounded_(num_capture_channels_),
x_(kNumBands,
std::vector<std::vector<float>>(
num_render_channels_,
std::vector<float>(kBlockSize, 0.0f))),
x_(kNumBands, num_render_channels_),
H2_(num_capture_channels_,
std::vector<std::array<float, kFftLengthBy2Plus1>>(10)),
h_(num_capture_channels_,
@ -84,7 +81,8 @@ class ResidualEchoEstimatorTest {
}
void RunOneFrame(bool dominant_nearend) {
RandomizeSampleVector(&random_generator_, x_[0][0]);
RandomizeSampleVector(&random_generator_,
x_.View(/*band=*/0, /*channel=*/0));
render_delay_buffer_->Insert(x_);
if (first_frame_) {
render_delay_buffer_->Reset();
@ -116,7 +114,7 @@ class ResidualEchoEstimatorTest {
std::vector<std::array<float, kFftLengthBy2Plus1>> Y2_;
std::vector<std::array<float, kFftLengthBy2Plus1>> R2_;
std::vector<std::array<float, kFftLengthBy2Plus1>> R2_unbounded_;
std::vector<std::vector<std::vector<float>>> x_;
Block x_;
std::vector<std::vector<std::array<float, kFftLengthBy2Plus1>>> H2_;
std::vector<std::vector<float>> h_;
Random random_generator_;

19
modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc
View File

@ -24,7 +24,7 @@ namespace webrtc {
namespace {
void GetActiveFrame(std::vector<std::vector<std::vector<float>>>* x) {
void GetActiveFrame(Block* x) {
const std::array<float, kBlockSize> frame = {
7459.88, 17209.6, 17383, 20768.9, 16816.7, 18386.3, 4492.83, 9675.85,
6665.52, 14808.6, 9342.3, 7483.28, 19261.7, 4145.98, 1622.18, 13475.2,
@ -34,10 +34,10 @@ void GetActiveFrame(std::vector<std::vector<std::vector<float>>>* x) {
11405, 15031.4, 14541.6, 19765.5, 18346.3, 19350.2, 3157.47, 18095.8,
1743.68, 21328.2, 19727.5, 7295.16, 10332.4, 11055.5, 20107.4, 14708.4,
12416.2, 16434, 2454.69, 9840.8, 6867.23, 1615.75, 6059.9, 8394.19};
for (size_t band = 0; band < x->size(); ++band) {
for (size_t channel = 0; channel < (*x)[band].size(); ++channel) {
RTC_DCHECK_GE((*x)[band][channel].size(), frame.size());
std::copy(frame.begin(), frame.end(), (*x)[band][channel].begin());
for (int band = 0; band < x->NumBands(); ++band) {
for (int channel = 0; channel < x->NumChannels(); ++channel) {
RTC_DCHECK_GE(kBlockSize, frame.size());
std::copy(frame.begin(), frame.end(), x->begin(band, channel));
}
}
}
@ -74,7 +74,7 @@ class TestInputs {
std::vector<std::array<float, kFftLengthBy2Plus1>> Y2_;
std::vector<std::array<float, kFftLengthBy2Plus1>> E2_;
std::vector<std::vector<std::array<float, kFftLengthBy2Plus1>>> H2_;
std::vector<std::vector<std::vector<float>>> x_;
Block x_;
std::vector<bool> converged_filters_;
};
@ -88,9 +88,7 @@ TestInputs::TestInputs(const EchoCanceller3Config& cfg,
H2_(num_capture_channels,
std::vector<std::array<float, kFftLengthBy2Plus1>>(
cfg.filter.refined.length_blocks)),
x_(1,
std::vector<std::vector<float>>(num_render_channels,
std::vector<float>(kBlockSize, 0.f))),
x_(1, num_render_channels),
converged_filters_(num_capture_channels, true) {
render_delay_buffer_->AlignFromDelay(4);
render_buffer_ = render_delay_buffer_->GetRenderBuffer();
@ -108,7 +106,8 @@ TestInputs::~TestInputs() = default;
void TestInputs::Update() {
if (n_ % 2 == 0) {
std::fill(x_[0][0].begin(), x_[0][0].end(), 0.f);
std::fill(x_.begin(/*band=*/0, /*channel=*/0),
x_.end(/*band=*/0, /*channel=*/0), 0.f);
} else {
GetActiveFrame(&x_);
}

7
modules/audio_processing/aec3/subtractor.cc
View File

@ -176,11 +176,11 @@ void Subtractor::ExitInitialState() {
}
void Subtractor::Process(const RenderBuffer& render_buffer,
const std::vector<std::vector<float>>& capture,
const Block& capture,
const RenderSignalAnalyzer& render_signal_analyzer,
const AecState& aec_state,
rtc::ArrayView<SubtractorOutput> outputs) {
RTC_DCHECK_EQ(num_capture_channels_, capture.size());
RTC_DCHECK_EQ(num_capture_channels_, capture.NumChannels());
// Compute the render powers.
const bool same_filter_sizes = refined_filters_[0]->SizePartitions() ==
@ -204,9 +204,8 @@ void Subtractor::Process(const RenderBuffer& render_buffer,
// Process all capture channels
for (size_t ch = 0; ch < num_capture_channels_; ++ch) {
RTC_DCHECK_EQ(kBlockSize, capture[ch].size());
SubtractorOutput& output = outputs[ch];
rtc::ArrayView<const float> y = capture[ch];
rtc::ArrayView<const float> y = capture.View(/*band=*/0, ch);
FftData& E_refined = output.E_refined;
FftData E_coarse;
std::array<float, kBlockSize>& e_refined = output.e_refined;

3
modules/audio_processing/aec3/subtractor.h
View File

@ -23,6 +23,7 @@
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/aec3_fft.h"
#include "modules/audio_processing/aec3/aec_state.h"
#include "modules/audio_processing/aec3/block.h"
#include "modules/audio_processing/aec3/coarse_filter_update_gain.h"
#include "modules/audio_processing/aec3/echo_path_variability.h"
#include "modules/audio_processing/aec3/refined_filter_update_gain.h"
@ -48,7 +49,7 @@ class Subtractor {
// Performs the echo subtraction.
void Process(const RenderBuffer& render_buffer,
const std::vector<std::vector<float>>& capture,
const Block& capture,
const RenderSignalAnalyzer& render_signal_analyzer,
const AecState& aec_state,
rtc::ArrayView<SubtractorOutput> outputs);

45
modules/audio_processing/aec3/subtractor_unittest.cc
View File

@ -45,11 +45,8 @@ std::vector<float> RunSubtractorTest(
Subtractor subtractor(config, num_render_channels, num_capture_channels,
&data_dumper, DetectOptimization());
absl::optional<DelayEstimate> delay_estimate;
std::vector<std::vector<std::vector<float>>> x(
kNumBands, std::vector<std::vector<float>>(
num_render_channels, std::vector<float>(kBlockSize, 0.f)));
std::vector<std::vector<float>> y(num_capture_channels,
std::vector<float>(kBlockSize, 0.f));
Block x(kNumBands, num_render_channels);
Block y(/*num_bands=*/1, num_capture_channels);
std::array<float, kBlockSize> x_old;
std::vector<SubtractorOutput> output(num_capture_channels);
config.delay.default_delay = 1;
@ -101,32 +98,34 @@ std::vector<float> RunSubtractorTest(
for (int k = 0; k < num_blocks_to_process; ++k) {
for (size_t render_ch = 0; render_ch < num_render_channels; ++render_ch) {
RandomizeSampleVector(&random_generator, x[0][render_ch]);
RandomizeSampleVector(&random_generator, x.View(/*band=*/0, render_ch));
}
if (uncorrelated_inputs) {
for (size_t capture_ch = 0; capture_ch < num_capture_channels;
++capture_ch) {
RandomizeSampleVector(&random_generator, y[capture_ch]);
RandomizeSampleVector(&random_generator,
y.View(/*band=*/0, capture_ch));
}
} else {
for (size_t capture_ch = 0; capture_ch < num_capture_channels;
++capture_ch) {
rtc::ArrayView<float> y_view = y.View(/*band=*/0, capture_ch);
for (size_t render_ch = 0; render_ch < num_render_channels;
++render_ch) {
std::array<float, kBlockSize> y_channel;
delay_buffer[capture_ch][render_ch]->Delay(x[0][render_ch],
y_channel);
for (size_t k = 0; k < y.size(); ++k) {
y[capture_ch][k] += y_channel[k] / num_render_channels;
delay_buffer[capture_ch][render_ch]->Delay(
x.View(/*band=*/0, render_ch), y_channel);
for (size_t k = 0; k < kBlockSize; ++k) {
y_view[k] += y_channel[k] / num_render_channels;
}
}
}
}
for (size_t ch = 0; ch < num_render_channels; ++ch) {
x_hp_filter[ch]->Process(x[0][ch]);
x_hp_filter[ch]->Process(x.View(/*band=*/0, ch));
}
for (size_t ch = 0; ch < num_capture_channels; ++ch) {
y_hp_filter[ch]->Process(y[ch]);
y_hp_filter[ch]->Process(y.View(/*band=*/0, ch));
}
render_delay_buffer->Insert(x);
@ -162,7 +161,8 @@ std::vector<float> RunSubtractorTest(
output[ch].e_refined.begin(), output[ch].e_refined.end(),
output[ch].e_refined.begin(), 0.f);
const float y_power =
std::inner_product(y[ch].begin(), y[ch].end(), y[ch].begin(), 0.f);
std::inner_product(y.begin(/*band=*/0, ch), y.end(/*band=*/0, ch),
y.begin(/*band=*/0, ch), 0.f);
if (y_power == 0.f) {
ADD_FAILURE();
results[ch] = -1.f;
@ -195,23 +195,6 @@ TEST(SubtractorDeathTest, NullDataDumper) {
"");
}
// Verifies the check for the capture signal size.
TEST(Subtractor, WrongCaptureSize) {
ApmDataDumper data_dumper(42);
EchoCanceller3Config config;
Subtractor subtractor(config, 1, 1, &data_dumper, DetectOptimization());
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create(config, 48000, 1));
RenderSignalAnalyzer render_signal_analyzer(config);
std::vector<std::vector<float>> y(1, std::vector<float>(kBlockSize - 1, 0.f));
std::array<SubtractorOutput, 1> output;
EXPECT_DEATH(
subtractor.Process(*render_delay_buffer->GetRenderBuffer(), y,
render_signal_analyzer, AecState(config, 1), output),
"");
}
#endif
// Verifies that the subtractor is able to converge on correlated data.

22
modules/audio_processing/aec3/suppression_filter.cc
View File

@ -86,9 +86,9 @@ void SuppressionFilter::ApplyGain(
const std::array<float, kFftLengthBy2Plus1>& suppression_gain,
float high_bands_gain,
rtc::ArrayView<const FftData> E_lowest_band,
std::vector<std::vector<std::vector<float>>>* e) {
Block* e) {
RTC_DCHECK(e);
RTC_DCHECK_EQ(e->size(), NumBandsForRate(sample_rate_hz_));
RTC_DCHECK_EQ(e->NumBands(), NumBandsForRate(sample_rate_hz_));
// Comfort noise gain is sqrt(1-g^2), where g is the suppression gain.
std::array<float, kFftLengthBy2Plus1> noise_gain;
@ -121,7 +121,7 @@ void SuppressionFilter::ApplyGain(
constexpr float kIfftNormalization = 2.f / kFftLength;
fft_.Ifft(E, &e_extended);
float* e0 = (*e)[0][ch].data();
auto e0 = e->View(/*band=*/0, ch);
float* e0_old = e_output_old_[0][ch].data();
// Window and add the first half of e_extended with the second half of
@ -138,20 +138,20 @@ void SuppressionFilter::ApplyGain(
std::begin(e_output_old_[0][ch]));
// Apply suppression gain to upper bands.
for (size_t b = 1; b < e->size(); ++b) {
float* e_band = (*e)[b][ch].data();
for (int b = 1; b < e->NumBands(); ++b) {
auto e_band = e->View(b, ch);
for (size_t i = 0; i < kFftLengthBy2; ++i) {
e_band[i] *= high_bands_gain;
}
}
// Add comfort noise to band 1.
if (e->size() > 1) {
if (e->NumBands() > 1) {
E.Assign(comfort_noise_high_band[ch]);
std::array<float, kFftLength> time_domain_high_band_noise;
fft_.Ifft(E, &time_domain_high_band_noise);
float* e1 = (*e)[1][ch].data();
auto e1 = e->View(/*band=*/1, ch);
const float gain = high_bands_noise_scaling * kIfftNormalization;
for (size_t i = 0; i < kFftLengthBy2; ++i) {
e1[i] += time_domain_high_band_noise[i] * gain;
@ -159,8 +159,8 @@ void SuppressionFilter::ApplyGain(
}
// Delay upper bands to match the delay of the filter bank.
for (size_t b = 1; b < e->size(); ++b) {
float* e_band = (*e)[b][ch].data();
for (int b = 1; b < e->NumBands(); ++b) {
auto e_band = e->View(b, ch);
float* e_band_old = e_output_old_[b][ch].data();
for (size_t i = 0; i < kFftLengthBy2; ++i) {
std::swap(e_band[i], e_band_old[i]);
@ -168,8 +168,8 @@ void SuppressionFilter::ApplyGain(
}
// Clamp output of all bands.
for (size_t b = 0; b < e->size(); ++b) {
float* e_band = (*e)[b][ch].data();
for (int b = 0; b < e->NumBands(); ++b) {
auto e_band = e->View(b, ch);
for (size_t i = 0; i < kFftLengthBy2; ++i) {
e_band[i] = rtc::SafeClamp(e_band[i], -32768.f, 32767.f);
}

3
modules/audio_processing/aec3/suppression_filter.h
View File

@ -16,6 +16,7 @@
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/aec3_fft.h"
#include "modules/audio_processing/aec3/block.h"
#include "modules/audio_processing/aec3/fft_data.h"
namespace webrtc {
@ -35,7 +36,7 @@ class SuppressionFilter {
const std::array<float, kFftLengthBy2Plus1>& suppression_gain,
float high_bands_gain,
rtc::ArrayView<const FftData> E_lowest_band,
std::vector<std::vector<std::vector<float>>>* e);
Block* e);
private:
const Aec3Optimization optimization_;

90
modules/audio_processing/aec3/suppression_filter_unittest.cc
View File

@ -26,21 +26,21 @@ constexpr float kPi = 3.141592f;
void ProduceSinusoid(int sample_rate_hz,
float sinusoidal_frequency_hz,
size_t* sample_counter,
std::vector<std::vector<std::vector<float>>>* x) {
Block* x) {
// Produce a sinusoid of the specified frequency.
for (size_t k = *sample_counter, j = 0; k < (*sample_counter + kBlockSize);
++k, ++j) {
for (size_t channel = 0; channel < (*x)[0].size(); ++channel) {
(*x)[0][channel][j] =
for (int channel = 0; channel < x->NumChannels(); ++channel) {
x->View(/*band=*/0, channel)[j] =
32767.f *
std::sin(2.f * kPi * sinusoidal_frequency_hz * k / sample_rate_hz);
}
}
*sample_counter = *sample_counter + kBlockSize;
for (size_t band = 1; band < x->size(); ++band) {
for (size_t channel = 0; channel < (*x)[band].size(); ++channel) {
std::fill((*x)[band][channel].begin(), (*x)[band][channel].end(), 0.f);
for (int band = 1; band < x->NumBands(); ++band) {
for (int channel = 0; channel < x->NumChannels(); ++channel) {
std::fill(x->begin(band, channel), x->end(band, channel), 0.f);
}
}
}
@ -84,21 +84,23 @@ TEST(SuppressionFilter, ComfortNoiseInUnityGain) {
cn_high_bands[0].re.fill(1.f);
cn_high_bands[0].im.fill(1.f);
std::vector<std::vector<std::vector<float>>> e(
3,
std::vector<std::vector<float>>(1, std::vector<float>(kBlockSize, 0.f)));
std::vector<std::vector<std::vector<float>>> e_ref = e;
Block e(3, kBlockSize);
Block e_ref = e;
std::vector<FftData> E(1);
fft.PaddedFft(e[0][0], e_old_, Aec3Fft::Window::kSqrtHanning, &E[0]);
std::copy(e[0][0].begin(), e[0][0].end(), e_old_.begin());
fft.PaddedFft(e.View(/*band=*/0, /*channel=*/0), e_old_,
Aec3Fft::Window::kSqrtHanning, &E[0]);
std::copy(e.begin(/*band=*/0, /*channel=*/0),
e.end(/*band=*/0, /*channel=*/0), e_old_.begin());
filter.ApplyGain(cn, cn_high_bands, gain, 1.f, E, &e);
for (size_t band = 0; band < e.size(); ++band) {
for (size_t channel = 0; channel < e[band].size(); ++channel) {
for (size_t sample = 0; sample < e[band][channel].size(); ++sample) {
EXPECT_EQ(e_ref[band][channel][sample], e[band][channel][sample]);
for (int band = 0; band < e.NumBands(); ++band) {
for (int channel = 0; channel < e.NumChannels(); ++channel) {
const auto e_view = e.View(band, channel);
const auto e_ref_view = e_ref.View(band, channel);
for (size_t sample = 0; sample < e_view.size(); ++sample) {
EXPECT_EQ(e_ref_view[sample], e_view[sample]);
}
}
}
@ -116,9 +118,7 @@ TEST(SuppressionFilter, SignalSuppression) {
std::array<float, kFftLengthBy2> e_old_;
Aec3Fft fft;
std::array<float, kFftLengthBy2Plus1> gain;
std::vector<std::vector<std::vector<float>>> e(
kNumBands, std::vector<std::vector<float>>(
kNumChannels, std::vector<float>(kBlockSize, 0.f)));
Block e(kNumBands, kNumChannels);
e_old_.fill(0.f);
gain.fill(1.f);
@ -135,16 +135,20 @@ TEST(SuppressionFilter, SignalSuppression) {
float e0_output = 0.f;
for (size_t k = 0; k < 100; ++k) {
ProduceSinusoid(16000, 16000 * 40 / kFftLengthBy2 / 2, &sample_counter, &e);
e0_input = std::inner_product(e[0][0].begin(), e[0][0].end(),
e[0][0].begin(), e0_input);
e0_input = std::inner_product(e.begin(/*band=*/0, /*channel=*/0),
e.end(/*band=*/0, /*channel=*/0),
e.begin(/*band=*/0, /*channel=*/0), e0_input);
std::vector<FftData> E(1);
fft.PaddedFft(e[0][0], e_old_, Aec3Fft::Window::kSqrtHanning, &E[0]);
std::copy(e[0][0].begin(), e[0][0].end(), e_old_.begin());
fft.PaddedFft(e.View(/*band=*/0, /*channel=*/0), e_old_,
Aec3Fft::Window::kSqrtHanning, &E[0]);
std::copy(e.begin(/*band=*/0, /*channel=*/0),
e.end(/*band=*/0, /*channel=*/0), e_old_.begin());
filter.ApplyGain(cn, cn_high_bands, gain, 1.f, E, &e);
e0_output = std::inner_product(e[0][0].begin(), e[0][0].end(),
e[0][0].begin(), e0_output);
e0_output = std::inner_product(
e.begin(/*band=*/0, /*channel=*/0), e.end(/*band=*/0, /*channel=*/0),
e.begin(/*band=*/0, /*channel=*/0), e0_output);
}
EXPECT_LT(e0_output, e0_input / 1000.f);
@ -163,9 +167,7 @@ TEST(SuppressionFilter, SignalTransparency) {
Aec3Fft fft;
std::vector<FftData> cn_high_bands(1);
std::array<float, kFftLengthBy2Plus1> gain;
std::vector<std::vector<std::vector<float>>> e(
kNumBands, std::vector<std::vector<float>>(
kNumChannels, std::vector<float>(kBlockSize, 0.f)));
Block e(kNumBands, kNumChannels);
e_old_.fill(0.f);
gain.fill(1.f);
std::for_each(gain.begin() + 30, gain.end(), [](float& a) { a = 0.f; });
@ -181,16 +183,20 @@ TEST(SuppressionFilter, SignalTransparency) {
float e0_output = 0.f;
for (size_t k = 0; k < 100; ++k) {
ProduceSinusoid(16000, 16000 * 10 / kFftLengthBy2 / 2, &sample_counter, &e);
e0_input = std::inner_product(e[0][0].begin(), e[0][0].end(),
e[0][0].begin(), e0_input);
e0_input = std::inner_product(e.begin(/*band=*/0, /*channel=*/0),
e.end(/*band=*/0, /*channel=*/0),
e.begin(/*band=*/0, /*channel=*/0), e0_input);
std::vector<FftData> E(1);
fft.PaddedFft(e[0][0], e_old_, Aec3Fft::Window::kSqrtHanning, &E[0]);
std::copy(e[0][0].begin(), e[0][0].end(), e_old_.begin());
fft.PaddedFft(e.View(/*band=*/0, /*channel=*/0), e_old_,
Aec3Fft::Window::kSqrtHanning, &E[0]);
std::copy(e.begin(/*band=*/0, /*channel=*/0),
e.end(/*band=*/0, /*channel=*/0), e_old_.begin());
filter.ApplyGain(cn, cn_high_bands, gain, 1.f, E, &e);
e0_output = std::inner_product(e[0][0].begin(), e[0][0].end(),
e[0][0].begin(), e0_output);
e0_output = std::inner_product(
e.begin(/*band=*/0, /*channel=*/0), e.end(/*band=*/0, /*channel=*/0),
e.begin(/*band=*/0, /*channel=*/0), e0_output);
}
EXPECT_LT(0.9f * e0_input, e0_output);
@ -208,9 +214,7 @@ TEST(SuppressionFilter, Delay) {
std::array<float, kFftLengthBy2> e_old_;
Aec3Fft fft;
std::array<float, kFftLengthBy2Plus1> gain;
std::vector<std::vector<std::vector<float>>> e(
kNumBands, std::vector<std::vector<float>>(
kNumChannels, std::vector<float>(kBlockSize, 0.f)));
Block e(kNumBands, kNumChannels);
gain.fill(1.f);
@ -222,23 +226,27 @@ TEST(SuppressionFilter, Delay) {
for (size_t k = 0; k < 100; ++k) {
for (size_t band = 0; band < kNumBands; ++band) {
for (size_t channel = 0; channel < kNumChannels; ++channel) {
auto e_view = e.View(band, channel);
for (size_t sample = 0; sample < kBlockSize; ++sample) {
e[band][channel][sample] = k * kBlockSize + sample + channel;
e_view[sample] = k * kBlockSize + sample + channel;
}
}
}
std::vector<FftData> E(1);
fft.PaddedFft(e[0][0], e_old_, Aec3Fft::Window::kSqrtHanning, &E[0]);
std::copy(e[0][0].begin(), e[0][0].end(), e_old_.begin());
fft.PaddedFft(e.View(/*band=*/0, /*channel=*/0), e_old_,
Aec3Fft::Window::kSqrtHanning, &E[0]);
std::copy(e.begin(/*band=*/0, /*channel=*/0),
e.end(/*band=*/0, /*channel=*/0), e_old_.begin());
filter.ApplyGain(cn, cn_high_bands, gain, 1.f, E, &e);
if (k > 2) {
for (size_t band = 0; band < kNumBands; ++band) {
for (size_t channel = 0; channel < kNumChannels; ++channel) {
const auto e_view = e.View(band, channel);
for (size_t sample = 0; sample < kBlockSize; ++sample) {
EXPECT_NEAR(k * kBlockSize + sample - kBlockSize + channel,
e[band][channel][sample], 0.01);
e_view[sample], 0.01);
}
}
}

34
modules/audio_processing/aec3/suppression_gain.cc
View File

@ -110,13 +110,13 @@ float SuppressionGain::UpperBandsGain(
comfort_noise_spectrum,
const absl::optional<int>& narrow_peak_band,
bool saturated_echo,
const std::vector<std::vector<std::vector<float>>>& render,
const Block& render,
const std::array<float, kFftLengthBy2Plus1>& low_band_gain) const {
RTC_DCHECK_LT(0, render.size());
if (render.size() == 1) {
RTC_DCHECK_LT(0, render.NumBands());
if (render.NumBands() == 1) {
return 1.f;
}
const size_t num_render_channels = render[0].size();
const int num_render_channels = render.NumChannels();
if (narrow_peak_band &&
(*narrow_peak_band > static_cast<int>(kFftLengthBy2Plus1 - 10))) {
@ -135,16 +135,19 @@ float SuppressionGain::UpperBandsGain(
// Compute the upper and lower band energies.
const auto sum_of_squares = [](float a, float b) { return a + b * b; };
float low_band_energy = 0.f;
for (size_t ch = 0; ch < num_render_channels; ++ch) {
for (int ch = 0; ch < num_render_channels; ++ch) {
// TODO(bugs.webrtc.org/14108): Fix the computation below to compute the
// energy for each channel.
const float channel_energy = std::accumulate(
render[0][0].begin(), render[0][0].end(), 0.f, sum_of_squares);
render.begin(/*band=0*/ 0, /*channel=*/0),
render.end(/*band=0*/ 0, /*channel=*/0), 0.f, sum_of_squares);
low_band_energy = std::max(low_band_energy, channel_energy);
}
float high_band_energy = 0.f;
for (size_t k = 1; k < render.size(); ++k) {
for (size_t ch = 0; ch < num_render_channels; ++ch) {
for (int k = 1; k < render.NumBands(); ++k) {
for (int ch = 0; ch < num_render_channels; ++ch) {
const float energy = std::accumulate(
render[k][ch].begin(), render[k][ch].end(), 0.f, sum_of_squares);
render.begin(k, ch), render.end(k, ch), 0.f, sum_of_squares);
high_band_energy = std::max(high_band_energy, energy);
}
}
@ -372,7 +375,7 @@ void SuppressionGain::GetGain(
comfort_noise_spectrum,
const RenderSignalAnalyzer& render_signal_analyzer,
const AecState& aec_state,
const std::vector<std::vector<std::vector<float>>>& render,
const Block& render,
bool clock_drift,
float* high_bands_gain,
std::array<float, kFftLengthBy2Plus1>* low_band_gain) {
@ -417,20 +420,17 @@ void SuppressionGain::SetInitialState(bool state) {
// Detects when the render signal can be considered to have low power and
// consist of stationary noise.
bool SuppressionGain::LowNoiseRenderDetector::Detect(
const std::vector<std::vector<std::vector<float>>>& render) {
bool SuppressionGain::LowNoiseRenderDetector::Detect(const Block& render) {
float x2_sum = 0.f;
float x2_max = 0.f;
for (const auto& x_ch : render[0]) {
for (const auto& x_k : x_ch) {
for (int ch = 0; ch < render.NumChannels(); ++ch) {
for (float x_k : render.View(/*band=*/0, ch)) {
const float x2 = x_k * x_k;
x2_sum += x2;
x2_max = std::max(x2_max, x2);
}
}
const size_t num_render_channels = render[0].size();
x2_sum = x2_sum / num_render_channels;
;
x2_sum = x2_sum / render.NumChannels();
constexpr float kThreshold = 50.f * 50.f * 64.f;
const bool low_noise_render =

6
modules/audio_processing/aec3/suppression_gain.h
View File

@ -51,7 +51,7 @@ class SuppressionGain {
comfort_noise_spectrum,
const RenderSignalAnalyzer& render_signal_analyzer,
const AecState& aec_state,
const std::vector<std::vector<std::vector<float>>>& render,
const Block& render,
bool clock_drift,
float* high_bands_gain,
std::array<float, kFftLengthBy2Plus1>* low_band_gain);
@ -71,7 +71,7 @@ class SuppressionGain {
comfort_noise_spectrum,
const absl::optional<int>& narrow_peak_band,
bool saturated_echo,
const std::vector<std::vector<std::vector<float>>>& render,
const Block& render,
const std::array<float, kFftLengthBy2Plus1>& low_band_gain) const;
void GainToNoAudibleEcho(const std::array<float, kFftLengthBy2Plus1>& nearend,
@ -100,7 +100,7 @@ class SuppressionGain {
class LowNoiseRenderDetector {
public:
bool Detect(const std::vector<std::vector<std::vector<float>>>& render);
bool Detect(const Block& render);
private:
float average_power_ = 32768.f * 32768.f;

9
modules/audio_processing/aec3/suppression_gain_unittest.cc
View File

@ -47,10 +47,7 @@ TEST(SuppressionGainDeathTest, NullOutputGains) {
SuppressionGain(EchoCanceller3Config{}, DetectOptimization(), 16000, 1)
.GetGain(E2, S2, R2, R2_unbounded, N2,
RenderSignalAnalyzer((EchoCanceller3Config{})), aec_state,
std::vector<std::vector<std::vector<float>>>(
3, std::vector<std::vector<float>>(
1, std::vector<float>(kBlockSize, 0.0f))),
false, &high_bands_gain, nullptr),
Block(3, 1), false, &high_bands_gain, nullptr),
"");
}
@ -76,9 +73,7 @@ TEST(SuppressionGain, BasicGainComputation) {
std::vector<std::array<float, kFftLengthBy2Plus1>> N2(kNumCaptureChannels);
std::array<float, kFftLengthBy2Plus1> g;
std::vector<SubtractorOutput> output(kNumCaptureChannels);
std::vector<std::vector<std::vector<float>>> x(
kNumBands, std::vector<std::vector<float>>(
kNumRenderChannels, std::vector<float>(kBlockSize, 0.0f)));
Block x(kNumBands, kNumRenderChannels);
EchoCanceller3Config config;
AecState aec_state(config, kNumCaptureChannels);
ApmDataDumper data_dumper(42);