Inform the AEC3 echo remover about the status of the estimated delay

This CL adds functionality for passing the information about the
estimated delay to the echo remover in AEC3.
The CL also adds information about how long ago the delay changed,
and how long ago the delay estimate was updated.

Bug: webrtc:8671
Change-Id: If274ffe0465eb550f3e186d0599c6dc6fef7f5e8
Reviewed-on: https://webrtc-review.googlesource.com/55261
Reviewed-by: Gustaf Ullberg <gustaf@webrtc.org>
Commit-Queue: Per Åhgren <peah@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#22137}
This commit is contained in:
Per Åhgren 2018-02-21 08:46:03 +01:00 committed by Commit Bot
parent bbfccfd9e0
commit 3ab308f869
16 changed files with 92 additions and 46 deletions

View File

@ -321,6 +321,7 @@ TEST(AdaptiveFirFilter, FilterAndAdapt) {
std::vector<float> y(kBlockSize, 0.f);
AecState aec_state(EchoCanceller3Config{});
RenderSignalAnalyzer render_signal_analyzer;
rtc::Optional<DelayEstimate> delay_estimate;
std::vector<float> e(kBlockSize, 0.f);
std::array<float, kFftLength> s_scratch;
std::array<float, kBlockSize> s;
@ -387,7 +388,7 @@ TEST(AdaptiveFirFilter, FilterAndAdapt) {
filter.Adapt(*render_buffer, G);
aec_state.HandleEchoPathChange(EchoPathVariability(
false, EchoPathVariability::DelayAdjustment::kNone, false));
aec_state.Update(filter.FilterFrequencyResponse(),
aec_state.Update(delay_estimate, filter.FilterFrequencyResponse(),
filter.FilterImpulseResponse(), true, *render_buffer,
E2_main, Y2, s, false);
}

View File

@ -110,6 +110,7 @@ void AecState::HandleEchoPathChange(
}
void AecState::Update(
const rtc::Optional<DelayEstimate>& delay_estimate,
const std::vector<std::array<float, kFftLengthBy2Plus1>>&
adaptive_filter_frequency_response,
const std::vector<float>& adaptive_filter_impulse_response,

View File

@ -19,6 +19,7 @@
#include "api/audio/echo_canceller3_config.h"
#include "api/optional.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/delay_estimate.h"
#include "modules/audio_processing/aec3/echo_path_variability.h"
#include "modules/audio_processing/aec3/erl_estimator.h"
#include "modules/audio_processing/aec3/erle_estimator.h"
@ -107,7 +108,8 @@ class AecState {
bool InitialState() const { return initial_state_; }
// Updates the aec state.
void Update(const std::vector<std::array<float, kFftLengthBy2Plus1>>&
void Update(const rtc::Optional<DelayEstimate>& delay_estimate,
const std::vector<std::array<float, kFftLengthBy2Plus1>>&
adaptive_filter_frequency_response,
const std::vector<float>& adaptive_filter_impulse_response,
bool converged_filter,

View File

@ -22,6 +22,7 @@ TEST(AecState, NormalUsage) {
ApmDataDumper data_dumper(42);
EchoCanceller3Config config;
AecState state(config);
rtc::Optional<DelayEstimate> delay_estimate;
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create(config, 3));
std::array<float, kFftLengthBy2Plus1> E2_main = {};
@ -47,17 +48,18 @@ TEST(AecState, NormalUsage) {
GetTimeDomainLength(config.filter.main.length_blocks), 0.f);
// Verify that linear AEC usability is false when the filter is diverged.
state.Update(diverged_filter_frequency_response, impulse_response, true,
*render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s, false);
state.Update(delay_estimate, diverged_filter_frequency_response,
impulse_response, true, *render_delay_buffer->GetRenderBuffer(),
E2_main, Y2, s, false);
EXPECT_FALSE(state.UsableLinearEstimate());
// Verify that linear AEC usability is true when the filter is converged
std::fill(x[0].begin(), x[0].end(), 101.f);
for (int k = 0; k < 3000; ++k) {
render_delay_buffer->Insert(x);
state.Update(converged_filter_frequency_response, impulse_response, true,
*render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s,
false);
state.Update(
delay_estimate, converged_filter_frequency_response, impulse_response,
true, *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s, false);
}
EXPECT_TRUE(state.UsableLinearEstimate());
@ -65,8 +67,9 @@ TEST(AecState, NormalUsage) {
// reported
state.HandleEchoPathChange(EchoPathVariability(
true, EchoPathVariability::DelayAdjustment::kNone, false));
state.Update(converged_filter_frequency_response, impulse_response, true,
*render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s, false);
state.Update(delay_estimate, converged_filter_frequency_response,
impulse_response, true, *render_delay_buffer->GetRenderBuffer(),
E2_main, Y2, s, false);
EXPECT_FALSE(state.UsableLinearEstimate());
// Verify that the active render detection works as intended.
@ -74,25 +77,28 @@ TEST(AecState, NormalUsage) {
render_delay_buffer->Insert(x);
state.HandleEchoPathChange(EchoPathVariability(
true, EchoPathVariability::DelayAdjustment::kNewDetectedDelay, false));
state.Update(converged_filter_frequency_response, impulse_response, true,
*render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s, false);
state.Update(delay_estimate, converged_filter_frequency_response,
impulse_response, true, *render_delay_buffer->GetRenderBuffer(),
E2_main, Y2, s, false);
EXPECT_FALSE(state.ActiveRender());
for (int k = 0; k < 1000; ++k) {
render_delay_buffer->Insert(x);
state.Update(converged_filter_frequency_response, impulse_response, true,
*render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s,
false);
state.Update(
delay_estimate, converged_filter_frequency_response, impulse_response,
true, *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s, false);
}
EXPECT_TRUE(state.ActiveRender());
// Verify that echo leakage is properly reported.
state.Update(converged_filter_frequency_response, impulse_response, true,
*render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s, false);
state.Update(delay_estimate, converged_filter_frequency_response,
impulse_response, true, *render_delay_buffer->GetRenderBuffer(),
E2_main, Y2, s, false);
EXPECT_FALSE(state.EchoLeakageDetected());
state.Update(converged_filter_frequency_response, impulse_response, true,
*render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s, true);
state.Update(delay_estimate, converged_filter_frequency_response,
impulse_response, true, *render_delay_buffer->GetRenderBuffer(),
E2_main, Y2, s, true);
EXPECT_TRUE(state.EchoLeakageDetected());
// Verify that the ERL is properly estimated
@ -112,9 +118,9 @@ TEST(AecState, NormalUsage) {
Y2.fill(10.f * 10000.f * 10000.f);
for (size_t k = 0; k < 1000; ++k) {
state.Update(converged_filter_frequency_response, impulse_response, true,
*render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s,
false);
state.Update(
delay_estimate, converged_filter_frequency_response, impulse_response,
true, *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s, false);
}
ASSERT_TRUE(state.UsableLinearEstimate());
@ -129,9 +135,9 @@ TEST(AecState, NormalUsage) {
E2_main.fill(1.f * 10000.f * 10000.f);
Y2.fill(10.f * E2_main[0]);
for (size_t k = 0; k < 1000; ++k) {
state.Update(converged_filter_frequency_response, impulse_response, true,
*render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s,
false);
state.Update(
delay_estimate, converged_filter_frequency_response, impulse_response,
true, *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s, false);
}
ASSERT_TRUE(state.UsableLinearEstimate());
{
@ -150,9 +156,9 @@ TEST(AecState, NormalUsage) {
E2_main.fill(1.f * 10000.f * 10000.f);
Y2.fill(5.f * E2_main[0]);
for (size_t k = 0; k < 1000; ++k) {
state.Update(converged_filter_frequency_response, impulse_response, true,
*render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s,
false);
state.Update(
delay_estimate, converged_filter_frequency_response, impulse_response,
true, *render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s, false);
}
ASSERT_TRUE(state.UsableLinearEstimate());
@ -177,6 +183,7 @@ TEST(AecState, ConvergedFilterDelay) {
AecState state(config);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create(config, 3));
rtc::Optional<DelayEstimate> delay_estimate;
std::array<float, kFftLengthBy2Plus1> E2_main;
std::array<float, kFftLengthBy2Plus1> Y2;
std::array<float, kBlockSize> x;
@ -200,7 +207,7 @@ TEST(AecState, ConvergedFilterDelay) {
frequency_response[k].fill(100.f);
frequency_response[k][0] = 0.f;
state.HandleEchoPathChange(echo_path_variability);
state.Update(frequency_response, impulse_response, true,
state.Update(delay_estimate, frequency_response, impulse_response, true,
*render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s,
false);
if (k != (kFilterLength - 1)) {

View File

@ -188,7 +188,7 @@ void BlockProcessorImpl::ProcessCapture(
// Remove the echo from the capture signal.
echo_remover_->ProcessCapture(
echo_path_variability, capture_signal_saturation,
echo_path_variability, capture_signal_saturation, estimated_delay_,
render_buffer_->GetRenderBuffer(), capture_block);
// Update the metrics.

View File

@ -168,7 +168,7 @@ TEST(BlockProcessor, DISABLED_SubmoduleIntegration) {
.WillRepeatedly(Return(0));
EXPECT_CALL(*render_delay_controller_mock, GetDelay(_, _))
.Times(kNumBlocks);
EXPECT_CALL(*echo_remover_mock, ProcessCapture(_, _, _, _))
EXPECT_CALL(*echo_remover_mock, ProcessCapture(_, _, _, _, _))
.Times(kNumBlocks);
EXPECT_CALL(*echo_remover_mock, UpdateEchoLeakageStatus(_))
.Times(kNumBlocks);

View File

@ -22,6 +22,8 @@ struct DelayEstimate {
Quality quality;
size_t delay;
size_t blocks_since_last_change = 0;
size_t blocks_since_last_update = 0;
};
} // namespace webrtc

View File

@ -60,6 +60,7 @@ class EchoRemoverImpl final : public EchoRemover {
// signal.
void ProcessCapture(const EchoPathVariability& echo_path_variability,
bool capture_signal_saturation,
const rtc::Optional<DelayEstimate>& delay_estimate,
RenderBuffer* render_buffer,
std::vector<std::vector<float>>* capture) override;
@ -122,6 +123,7 @@ void EchoRemoverImpl::GetMetrics(EchoControl::Metrics* metrics) const {
void EchoRemoverImpl::ProcessCapture(
const EchoPathVariability& echo_path_variability,
bool capture_signal_saturation,
const rtc::Optional<DelayEstimate>& delay_estimate,
RenderBuffer* render_buffer,
std::vector<std::vector<float>>* capture) {
const std::vector<std::vector<float>>& x = render_buffer->Block(0);
@ -182,7 +184,7 @@ void EchoRemoverImpl::ProcessCapture(
Y.Spectrum(optimization_, Y2);
// Update the AEC state information.
aec_state_.Update(subtractor_.FilterFrequencyResponse(),
aec_state_.Update(delay_estimate, subtractor_.FilterFrequencyResponse(),
subtractor_.FilterImpulseResponse(),
subtractor_.ConvergedFilter(), *render_buffer, E2_main, Y2,
subtractor_output.s_main, echo_leakage_detected_);

View File

@ -16,6 +16,7 @@
#include "api/audio/echo_canceller3_config.h"
#include "api/audio/echo_control.h"
#include "api/optional.h"
#include "modules/audio_processing/aec3/delay_estimate.h"
#include "modules/audio_processing/aec3/echo_path_variability.h"
#include "modules/audio_processing/aec3/render_buffer.h"
@ -37,6 +38,7 @@ class EchoRemover {
virtual void ProcessCapture(
const EchoPathVariability& echo_path_variability,
bool capture_signal_saturation,
const rtc::Optional<DelayEstimate>& delay_estimate,
RenderBuffer* render_buffer,
std::vector<std::vector<float>>* capture) = 0;

View File

@ -43,6 +43,7 @@ std::string ProduceDebugText(int sample_rate_hz, int delay) {
// Verifies the basic API call sequence
TEST(EchoRemover, BasicApiCalls) {
rtc::Optional<DelayEstimate> delay_estimate;
for (auto rate : {8000, 16000, 32000, 48000}) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<EchoRemover> remover(
@ -64,7 +65,8 @@ TEST(EchoRemover, BasicApiCalls) {
render_buffer->PrepareCaptureProcessing();
remover->ProcessCapture(echo_path_variability, k % 2 == 0 ? true : false,
render_buffer->GetRenderBuffer(), &capture);
delay_estimate, render_buffer->GetRenderBuffer(),
&capture);
}
}
}
@ -82,6 +84,7 @@ TEST(EchoRemover, DISABLED_WrongSampleRate) {
// Verifies the check for the capture block size.
TEST(EchoRemover, WrongCaptureBlockSize) {
rtc::Optional<DelayEstimate> delay_estimate;
for (auto rate : {8000, 16000, 32000, 48000}) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<EchoRemover> remover(
@ -93,7 +96,7 @@ TEST(EchoRemover, WrongCaptureBlockSize) {
EchoPathVariability echo_path_variability(
false, EchoPathVariability::DelayAdjustment::kNone, false);
EXPECT_DEATH(
remover->ProcessCapture(echo_path_variability, false,
remover->ProcessCapture(echo_path_variability, false, delay_estimate,
render_buffer->GetRenderBuffer(), &capture),
"");
}
@ -103,6 +106,7 @@ TEST(EchoRemover, WrongCaptureBlockSize) {
// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH
// tests on test bots has been fixed.c
TEST(EchoRemover, DISABLED_WrongCaptureNumBands) {
rtc::Optional<DelayEstimate> delay_estimate;
for (auto rate : {16000, 32000, 48000}) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<EchoRemover> remover(
@ -115,7 +119,7 @@ TEST(EchoRemover, DISABLED_WrongCaptureNumBands) {
EchoPathVariability echo_path_variability(
false, EchoPathVariability::DelayAdjustment::kNone, false);
EXPECT_DEATH(
remover->ProcessCapture(echo_path_variability, false,
remover->ProcessCapture(echo_path_variability, false, delay_estimate,
render_buffer->GetRenderBuffer(), &capture),
"");
}
@ -123,6 +127,7 @@ TEST(EchoRemover, DISABLED_WrongCaptureNumBands) {
// Verifies the check for non-null capture block.
TEST(EchoRemover, NullCapture) {
rtc::Optional<DelayEstimate> delay_estimate;
std::unique_ptr<EchoRemover> remover(
EchoRemover::Create(EchoCanceller3Config(), 8000));
std::unique_ptr<RenderDelayBuffer> render_buffer(
@ -130,7 +135,7 @@ TEST(EchoRemover, NullCapture) {
EchoPathVariability echo_path_variability(
false, EchoPathVariability::DelayAdjustment::kNone, false);
EXPECT_DEATH(
remover->ProcessCapture(echo_path_variability, false,
remover->ProcessCapture(echo_path_variability, false, delay_estimate,
render_buffer->GetRenderBuffer(), nullptr),
"");
}
@ -142,6 +147,7 @@ TEST(EchoRemover, NullCapture) {
TEST(EchoRemover, BasicEchoRemoval) {
constexpr int kNumBlocksToProcess = 500;
Random random_generator(42U);
rtc::Optional<DelayEstimate> delay_estimate;
for (auto rate : {8000, 16000, 32000, 48000}) {
std::vector<std::vector<float>> x(NumBandsForRate(rate),
std::vector<float>(kBlockSize, 0.f));
@ -187,7 +193,7 @@ TEST(EchoRemover, BasicEchoRemoval) {
render_buffer->Insert(x);
render_buffer->PrepareCaptureProcessing();
remover->ProcessCapture(echo_path_variability, false,
remover->ProcessCapture(echo_path_variability, false, delay_estimate,
render_buffer->GetRenderBuffer(), &y);
if (k > kNumBlocksToProcess / 2) {

View File

@ -62,6 +62,7 @@ void RunFilterUpdateTest(int num_blocks_to_process,
RenderDelayBuffer::Create(config, 3));
AecState aec_state(config);
RenderSignalAnalyzer render_signal_analyzer;
rtc::Optional<DelayEstimate> delay_estimate;
std::array<float, kFftLength> s_scratch;
std::array<float, kBlockSize> s;
FftData S;
@ -154,7 +155,7 @@ void RunFilterUpdateTest(int num_blocks_to_process,
// Update the delay.
aec_state.HandleEchoPathChange(EchoPathVariability(
false, EchoPathVariability::DelayAdjustment::kNone, false));
aec_state.Update(main_filter.FilterFrequencyResponse(),
aec_state.Update(delay_estimate, main_filter.FilterFrequencyResponse(),
main_filter.FilterImpulseResponse(), true,
*render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s,
false);

View File

@ -26,9 +26,10 @@ class MockEchoRemover : public EchoRemover {
public:
virtual ~MockEchoRemover() = default;
MOCK_METHOD4(ProcessCapture,
MOCK_METHOD5(ProcessCapture,
void(const EchoPathVariability& echo_path_variability,
bool capture_signal_saturation,
const rtc::Optional<DelayEstimate>& delay_estimate,
RenderBuffer* render_buffer,
std::vector<std::vector<float>>* capture));

View File

@ -130,7 +130,9 @@ DelayEstimate ComputeBufferDelay(
}
}
return DelayEstimate(estimated_delay.quality, new_delay_blocks);
DelayEstimate new_delay = estimated_delay;
new_delay.delay = new_delay_blocks;
return new_delay;
}
int RenderDelayControllerImpl::instance_count_ = 0;
@ -194,7 +196,22 @@ rtc::Optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
if (!delay_samples_ || delay_samples->delay != delay_samples_->delay) {
delay_change_counter_ = 0;
}
delay_samples_ = delay_samples;
if (delay_samples_) {
delay_samples_->blocks_since_last_change =
delay_samples_->delay == delay_samples->delay
? delay_samples_->blocks_since_last_change + 1
: 0;
delay_samples_->blocks_since_last_update = 0;
delay_samples_->delay = delay_samples->delay;
delay_samples_->quality = delay_samples->quality;
} else {
delay_samples_ = delay_samples;
}
} else {
if (delay_samples_) {
++delay_samples_->blocks_since_last_change;
++delay_samples_->blocks_since_last_update;
}
}
if (delay_change_counter_ < 2 * kNumBlocksPerSecond) {

View File

@ -63,6 +63,7 @@ TEST(ResidualEchoEstimator, DISABLED_BasicTest) {
Random random_generator(42U);
std::array<float, kBlockSize> s;
Aec3Fft fft;
rtc::Optional<DelayEstimate> delay_estimate;
for (auto& H2_k : H2) {
H2_k.fill(0.01f);
@ -92,8 +93,9 @@ TEST(ResidualEchoEstimator, DISABLED_BasicTest) {
render_delay_buffer->PrepareCaptureProcessing();
aec_state.HandleEchoPathChange(echo_path_variability);
aec_state.Update(H2, h, true, *render_delay_buffer->GetRenderBuffer(),
E2_main, Y2, s, false);
aec_state.Update(delay_estimate, H2, h, true,
*render_delay_buffer->GetRenderBuffer(), E2_main, Y2, s,
false);
estimator.Estimate(aec_state, *render_delay_buffer->GetRenderBuffer(),
S2_linear, Y2, &R2);

View File

@ -33,6 +33,7 @@ float RunSubtractorTest(int num_blocks_to_process,
config.filter.main.length_blocks = config.filter.shadow.length_blocks =
filter_length_blocks;
Subtractor subtractor(config, &data_dumper, DetectOptimization());
rtc::Optional<DelayEstimate> delay_estimate;
std::vector<std::vector<float>> x(3, std::vector<float>(kBlockSize, 0.f));
std::vector<float> y(kBlockSize, 0.f);
std::array<float, kBlockSize> x_old;
@ -82,7 +83,7 @@ float RunSubtractorTest(int num_blocks_to_process,
aec_state.HandleEchoPathChange(EchoPathVariability(
false, EchoPathVariability::DelayAdjustment::kNone, false));
aec_state.Update(subtractor.FilterFrequencyResponse(),
aec_state.Update(delay_estimate, subtractor.FilterFrequencyResponse(),
subtractor.FilterImpulseResponse(),
subtractor.ConvergedFilter(),
*render_delay_buffer->GetRenderBuffer(), E2_main, Y2,

View File

@ -63,6 +63,7 @@ TEST(SuppressionGain, BasicGainComputation) {
Subtractor subtractor(config, &data_dumper, DetectOptimization());
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create(config, 3));
rtc::Optional<DelayEstimate> delay_estimate;
// Ensure that a strong noise is detected to mask any echoes.
E2.fill(10.f);
@ -73,14 +74,14 @@ TEST(SuppressionGain, BasicGainComputation) {
// Ensure that the gain is no longer forced to zero.
for (int k = 0; k <= kNumBlocksPerSecond / 5 + 1; ++k) {
aec_state.Update(subtractor.FilterFrequencyResponse(),
aec_state.Update(delay_estimate, subtractor.FilterFrequencyResponse(),
subtractor.FilterImpulseResponse(),
subtractor.ConvergedFilter(),
*render_delay_buffer->GetRenderBuffer(), E2, Y2, s, false);
}
for (int k = 0; k < 100; ++k) {
aec_state.Update(subtractor.FilterFrequencyResponse(),
aec_state.Update(delay_estimate, subtractor.FilterFrequencyResponse(),
subtractor.FilterImpulseResponse(),
subtractor.ConvergedFilter(),
*render_delay_buffer->GetRenderBuffer(), E2, Y2, s, false);
@ -96,7 +97,7 @@ TEST(SuppressionGain, BasicGainComputation) {
R2.fill(0.1f);
N2.fill(0.f);
for (int k = 0; k < 100; ++k) {
aec_state.Update(subtractor.FilterFrequencyResponse(),
aec_state.Update(delay_estimate, subtractor.FilterFrequencyResponse(),
subtractor.FilterImpulseResponse(),
subtractor.ConvergedFilter(),
*render_delay_buffer->GetRenderBuffer(), E2, Y2, s, false);