diff --git a/modules/audio_processing/BUILD.gn b/modules/audio_processing/BUILD.gn index fdc4783edf..6f09165525 100644 --- a/modules/audio_processing/BUILD.gn +++ b/modules/audio_processing/BUILD.gn @@ -62,6 +62,8 @@ rtc_static_library("audio_processing") { "aec3/erl_estimator.h", "aec3/erle_estimator.cc", "aec3/erle_estimator.h", + "aec3/fft_buffer.cc", + "aec3/fft_buffer.h", "aec3/fft_data.h", "aec3/frame_blocker.cc", "aec3/frame_blocker.h", @@ -71,6 +73,8 @@ rtc_static_library("audio_processing") { "aec3/matched_filter.h", "aec3/matched_filter_lag_aggregator.cc", "aec3/matched_filter_lag_aggregator.h", + "aec3/matrix_buffer.cc", + "aec3/matrix_buffer.h", "aec3/output_selector.cc", "aec3/output_selector.h", "aec3/render_buffer.cc", @@ -94,6 +98,8 @@ rtc_static_library("audio_processing") { "aec3/suppression_filter.h", "aec3/suppression_gain.cc", "aec3/suppression_gain.h", + "aec3/vector_buffer.cc", + "aec3/vector_buffer.h", "aec3/vector_math.h", "aecm/aecm_core.cc", "aecm/aecm_core.h", diff --git a/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc b/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc index 11d7e02b41..b3a93a700e 100644 --- a/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc +++ b/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc @@ -21,6 +21,7 @@ #include "modules/audio_processing/aec3/aec3_fft.h" #include "modules/audio_processing/aec3/aec_state.h" #include "modules/audio_processing/aec3/cascaded_biquad_filter.h" +#include "modules/audio_processing/aec3/render_delay_buffer.h" #include "modules/audio_processing/aec3/render_signal_analyzer.h" #include "modules/audio_processing/aec3/shadow_filter_update_gain.h" #include "modules/audio_processing/logging/apm_data_dumper.h" @@ -47,8 +48,8 @@ std::string ProduceDebugText(size_t delay) { // Verifies that the optimized methods for filter adaptation are similar to // their reference counterparts. TEST(AdaptiveFirFilter, FilterAdaptationNeonOptimizations) { - RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 12, - std::vector(1, 12)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(EchoCanceller3Config(), 3)); Random random_generator(42U); std::vector> x(3, std::vector(kBlockSize, 0.f)); FftData S_C; @@ -66,8 +67,13 @@ TEST(AdaptiveFirFilter, FilterAdaptationNeonOptimizations) { for (size_t k = 0; k < 30; ++k) { RandomizeSampleVector(&random_generator, x[0]); - render_buffer.Insert(x); + render_delay_buffer->Insert(x); + if (k == 0) { + render_delay_buffer->Reset(); + } + render_delay_buffer->PrepareCaptureCall(); } + const auto& render_buffer = render_delay_buffer->GetRenderBuffer(); for (size_t j = 0; j < G.re.size(); ++j) { G.re[j] = j / 10001.f; @@ -153,8 +159,8 @@ TEST(AdaptiveFirFilter, UpdateErlNeonOptimization) { TEST(AdaptiveFirFilter, FilterAdaptationSse2Optimizations) { bool use_sse2 = (WebRtc_GetCPUInfo(kSSE2) != 0); if (use_sse2) { - RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 12, - std::vector(1, 12)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(EchoCanceller3Config(), 3)); Random random_generator(42U); std::vector> x(3, std::vector(kBlockSize, 0.f)); FftData S_C; @@ -172,7 +178,12 @@ TEST(AdaptiveFirFilter, FilterAdaptationSse2Optimizations) { for (size_t k = 0; k < 500; ++k) { RandomizeSampleVector(&random_generator, x[0]); - render_buffer.Insert(x); + render_delay_buffer->Insert(x); + if (k == 0) { + render_delay_buffer->Reset(); + } + render_delay_buffer->PrepareCaptureCall(); + const auto& render_buffer = render_delay_buffer->GetRenderBuffer(); ApplyFilter_SSE2(render_buffer, H_SSE2, &S_SSE2); ApplyFilter(render_buffer, H_C, &S_C); @@ -264,10 +275,10 @@ TEST(AdaptiveFirFilter, NullDataDumper) { TEST(AdaptiveFirFilter, NullFilterOutput) { ApmDataDumper data_dumper(42); AdaptiveFirFilter filter(9, DetectOptimization(), &data_dumper); - RenderBuffer render_buffer(Aec3Optimization::kNone, 3, - filter.SizePartitions(), - std::vector(1, filter.SizePartitions())); - EXPECT_DEATH(filter.Filter(render_buffer, nullptr), ""); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(EchoCanceller3Config(), 3)); + EXPECT_DEATH(filter.Filter(render_delay_buffer->GetRenderBuffer(), nullptr), + ""); } #endif @@ -295,11 +306,13 @@ TEST(AdaptiveFirFilter, FilterSize) { TEST(AdaptiveFirFilter, FilterAndAdapt) { constexpr size_t kNumBlocksToProcess = 500; ApmDataDumper data_dumper(42); - AdaptiveFirFilter filter(9, DetectOptimization(), &data_dumper); + AdaptiveFirFilter filter(kAdaptiveFilterLength, DetectOptimization(), + &data_dumper); Aec3Fft fft; - RenderBuffer render_buffer(Aec3Optimization::kNone, 3, - filter.SizePartitions(), - std::vector(1, filter.SizePartitions())); + EchoCanceller3Config config; + config.delay.min_echo_path_delay_blocks = 0; + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, 3)); ShadowFilterUpdateGain gain; Random random_generator(42U); std::vector> x(3, std::vector(kBlockSize, 0.f)); @@ -345,7 +358,13 @@ TEST(AdaptiveFirFilter, FilterAndAdapt) { x_hp_filter.Process(x[0]); y_hp_filter.Process(y); - render_buffer.Insert(x); + render_delay_buffer->Insert(x); + if (k == 0) { + render_delay_buffer->Reset(); + } + render_delay_buffer->PrepareCaptureCall(); + const auto& render_buffer = render_delay_buffer->GetRenderBuffer(); + render_signal_analyzer.Update(render_buffer, aec_state.FilterDelay()); filter.Filter(render_buffer, &S); @@ -363,7 +382,8 @@ TEST(AdaptiveFirFilter, FilterAndAdapt) { gain.Compute(render_buffer, render_signal_analyzer, E, filter.SizePartitions(), false, &G); filter.Adapt(render_buffer, G); - aec_state.HandleEchoPathChange(EchoPathVariability(false, false)); + aec_state.HandleEchoPathChange(EchoPathVariability( + false, EchoPathVariability::DelayAdjustment::kNone, false)); aec_state.Update(filter.FilterFrequencyResponse(), filter.FilterImpulseResponse(), true, rtc::nullopt, render_buffer, E2_main, Y2, x[0], s, false); diff --git a/modules/audio_processing/aec3/aec3_common.h b/modules/audio_processing/aec3/aec3_common.h index 3e80442981..aa6ffd9bd6 100644 --- a/modules/audio_processing/aec3/aec3_common.h +++ b/modules/audio_processing/aec3/aec3_common.h @@ -42,6 +42,7 @@ constexpr int kAdaptiveFilterLength = 12; constexpr int kUnknownDelayRenderWindowSize = 30; constexpr int kAdaptiveFilterTimeDomainLength = kAdaptiveFilterLength * kFftLengthBy2; +constexpr int kRenderTransferQueueSizeFrames = 100; constexpr size_t kMaxNumBands = 3; constexpr size_t kSubFrameLength = 80; @@ -52,11 +53,6 @@ constexpr size_t kMatchedFilterWindowSizeSubBlocks = 32; constexpr size_t kMatchedFilterAlignmentShiftSizeSubBlocks = kMatchedFilterWindowSizeSubBlocks * 3 / 4; -constexpr size_t kMinEchoPathDelayBlocks = 5; -constexpr size_t kMaxApiCallsJitterBlocks = 26; -constexpr size_t kRenderTransferQueueSize = kMaxApiCallsJitterBlocks / 2; -static_assert(2 * kRenderTransferQueueSize >= kMaxApiCallsJitterBlocks, - "Requirement to ensure buffer overflow detection"); constexpr size_t kEchoPathChangeConvergenceBlocks = 2 * kNumBlocksPerSecond; @@ -83,9 +79,8 @@ constexpr size_t GetDownSampledBufferSize(size_t down_sampling_factor, constexpr size_t GetRenderDelayBufferSize(size_t down_sampling_factor, size_t num_matched_filters) { - return (3 * - GetDownSampledBufferSize(down_sampling_factor, num_matched_filters)) / - (4 * kBlockSize / down_sampling_factor); + return GetDownSampledBufferSize(down_sampling_factor, num_matched_filters) / + (kBlockSize / down_sampling_factor); } // Detects what kind of optimizations to use for the code. diff --git a/modules/audio_processing/aec3/aec_state.cc b/modules/audio_processing/aec3/aec_state.cc index 9349e475dd..58859d837b 100644 --- a/modules/audio_processing/aec3/aec_state.cc +++ b/modules/audio_processing/aec3/aec_state.cc @@ -64,28 +64,45 @@ AecState::~AecState() = default; void AecState::HandleEchoPathChange( const EchoPathVariability& echo_path_variability) { - if (echo_path_variability.AudioPathChanged()) { + const auto full_reset = [&]() { blocks_since_last_saturation_ = kUnknownDelayRenderWindowSize + 1; usable_linear_estimate_ = false; echo_leakage_detected_ = false; capture_signal_saturation_ = false; echo_saturation_ = false; max_render_.fill(0.f); + force_zero_gain_counter_ = 0; + blocks_with_filter_adaptation_ = 0; + blocks_with_strong_render_ = 0; + initial_state_ = true; + capture_block_counter_ = 0; + linear_echo_estimate_ = false; + sufficient_filter_updates_ = false; + render_received_ = false; + force_zero_gain_ = true; + }; - if (echo_path_variability.delay_change) { - force_zero_gain_counter_ = 0; - blocks_with_filter_adaptation_ = 0; - blocks_with_strong_render_ = 0; - initial_state_ = true; - linear_echo_estimate_ = false; - sufficient_filter_updates_ = false; - render_received_ = false; - force_zero_gain_ = true; - capture_block_counter_ = 0; - } - if (echo_path_variability.gain_change) { - capture_block_counter_ = kNumBlocksPerSecond; - } + // TODO(peah): Refine the reset scheme according to the type of gain and + // delay adjustment. + if (echo_path_variability.gain_change) { + full_reset(); + } + + if (echo_path_variability.delay_change != + EchoPathVariability::DelayAdjustment::kBufferReadjustment) { + full_reset(); + } else if (echo_path_variability.delay_change != + EchoPathVariability::DelayAdjustment::kBufferFlush) { + full_reset(); + + } else if (echo_path_variability.delay_change != + EchoPathVariability::DelayAdjustment::kDelayReset) { + full_reset(); + } else if (echo_path_variability.delay_change != + EchoPathVariability::DelayAdjustment::kNewDetectedDelay) { + full_reset(); + } else if (echo_path_variability.gain_change) { + capture_block_counter_ = kNumBlocksPerSecond; } } diff --git a/modules/audio_processing/aec3/aec_state_unittest.cc b/modules/audio_processing/aec3/aec_state_unittest.cc index 34b877b4f9..fe09086740 100644 --- a/modules/audio_processing/aec3/aec_state_unittest.cc +++ b/modules/audio_processing/aec3/aec_state_unittest.cc @@ -10,6 +10,8 @@ #include "modules/audio_processing/aec3/aec_state.h" +#include "modules/audio_processing/aec3/aec3_fft.h" +#include "modules/audio_processing/aec3/render_delay_buffer.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "test/gtest.h" @@ -18,14 +20,17 @@ namespace webrtc { // Verify the general functionality of AecState TEST(AecState, NormalUsage) { ApmDataDumper data_dumper(42); - AecState state(EchoCanceller3Config{}); - RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 30, - std::vector(1, 30)); + EchoCanceller3Config config; + AecState state(config); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, 3)); std::array E2_main = {}; std::array Y2 = {}; std::vector> x(3, std::vector(kBlockSize, 0.f)); - EchoPathVariability echo_path_variability(false, false); + EchoPathVariability echo_path_variability( + false, EchoPathVariability::DelayAdjustment::kNone, false); std::array s; + Aec3Fft fft; s.fill(100.f); std::vector> @@ -44,44 +49,53 @@ TEST(AecState, NormalUsage) { // Verify that linear AEC usability is false when the filter is diverged and // there is no external delay reported. state.Update(diverged_filter_frequency_response, impulse_response, true, - rtc::nullopt, render_buffer, E2_main, Y2, x[0], s, false); + rtc::nullopt, render_delay_buffer->GetRenderBuffer(), E2_main, + Y2, x[0], s, false); EXPECT_FALSE(state.UsableLinearEstimate()); // Verify that linear AEC usability is true when the filter is converged std::fill(x[0].begin(), x[0].end(), 101.f); for (int k = 0; k < 3000; ++k) { state.Update(converged_filter_frequency_response, impulse_response, true, 2, - render_buffer, E2_main, Y2, x[0], s, false); + render_delay_buffer->GetRenderBuffer(), E2_main, Y2, x[0], s, + false); } EXPECT_TRUE(state.UsableLinearEstimate()); // Verify that linear AEC usability becomes false after an echo path change is // reported - state.HandleEchoPathChange(EchoPathVariability(true, false)); + state.HandleEchoPathChange(EchoPathVariability( + true, EchoPathVariability::DelayAdjustment::kNone, false)); state.Update(converged_filter_frequency_response, impulse_response, true, 2, - render_buffer, E2_main, Y2, x[0], s, false); + render_delay_buffer->GetRenderBuffer(), E2_main, Y2, x[0], s, + false); EXPECT_FALSE(state.UsableLinearEstimate()); // Verify that the active render detection works as intended. std::fill(x[0].begin(), x[0].end(), 101.f); - state.HandleEchoPathChange(EchoPathVariability(true, true)); + state.HandleEchoPathChange(EchoPathVariability( + true, EchoPathVariability::DelayAdjustment::kNewDetectedDelay, false)); state.Update(converged_filter_frequency_response, impulse_response, true, 2, - render_buffer, E2_main, Y2, x[0], s, false); + render_delay_buffer->GetRenderBuffer(), E2_main, Y2, x[0], s, + false); EXPECT_FALSE(state.ActiveRender()); for (int k = 0; k < 1000; ++k) { state.Update(converged_filter_frequency_response, impulse_response, true, 2, - render_buffer, E2_main, Y2, x[0], s, false); + render_delay_buffer->GetRenderBuffer(), E2_main, Y2, x[0], s, + false); } EXPECT_TRUE(state.ActiveRender()); // Verify that echo leakage is properly reported. state.Update(converged_filter_frequency_response, impulse_response, true, 2, - render_buffer, E2_main, Y2, x[0], s, false); + render_delay_buffer->GetRenderBuffer(), E2_main, Y2, x[0], s, + false); EXPECT_FALSE(state.EchoLeakageDetected()); state.Update(converged_filter_frequency_response, impulse_response, true, 2, - render_buffer, E2_main, Y2, x[0], s, true); + render_delay_buffer->GetRenderBuffer(), E2_main, Y2, x[0], s, + true); EXPECT_TRUE(state.EchoLeakageDetected()); // Verify that the ERL is properly estimated @@ -90,14 +104,20 @@ TEST(AecState, NormalUsage) { } x[0][0] = 5000.f; - for (size_t k = 0; k < render_buffer.Buffer().size(); ++k) { - render_buffer.Insert(x); + for (size_t k = 0; k < render_delay_buffer->GetRenderBuffer().Buffer().size(); + ++k) { + render_delay_buffer->Insert(x); + if (k == 0) { + render_delay_buffer->Reset(); + } + render_delay_buffer->PrepareCaptureCall(); } Y2.fill(10.f * 10000.f * 10000.f); for (size_t k = 0; k < 1000; ++k) { state.Update(converged_filter_frequency_response, impulse_response, true, 2, - render_buffer, E2_main, Y2, x[0], s, false); + render_delay_buffer->GetRenderBuffer(), E2_main, Y2, x[0], s, + false); } ASSERT_TRUE(state.UsableLinearEstimate()); @@ -113,7 +133,8 @@ TEST(AecState, NormalUsage) { Y2.fill(10.f * E2_main[0]); for (size_t k = 0; k < 1000; ++k) { state.Update(converged_filter_frequency_response, impulse_response, true, 2, - render_buffer, E2_main, Y2, x[0], s, false); + render_delay_buffer->GetRenderBuffer(), E2_main, Y2, x[0], s, + false); } ASSERT_TRUE(state.UsableLinearEstimate()); { @@ -133,7 +154,8 @@ TEST(AecState, NormalUsage) { Y2.fill(5.f * E2_main[0]); for (size_t k = 0; k < 1000; ++k) { state.Update(converged_filter_frequency_response, impulse_response, true, 2, - render_buffer, E2_main, Y2, x[0], s, false); + render_delay_buffer->GetRenderBuffer(), E2_main, Y2, x[0], s, + false); } ASSERT_TRUE(state.UsableLinearEstimate()); @@ -154,13 +176,15 @@ TEST(AecState, NormalUsage) { // Verifies the delay for a converged filter is correctly identified. TEST(AecState, ConvergedFilterDelay) { constexpr int kFilterLength = 10; - AecState state(EchoCanceller3Config{}); - RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 30, - std::vector(1, 30)); + EchoCanceller3Config config; + AecState state(config); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, 3)); std::array E2_main; std::array Y2; std::array x; - EchoPathVariability echo_path_variability(false, false); + EchoPathVariability echo_path_variability( + false, EchoPathVariability::DelayAdjustment::kNone, false); std::array s; s.fill(100.f); x.fill(0.f); @@ -180,7 +204,8 @@ TEST(AecState, ConvergedFilterDelay) { frequency_response[k][0] = 0.f; state.HandleEchoPathChange(echo_path_variability); state.Update(frequency_response, impulse_response, true, rtc::nullopt, - render_buffer, E2_main, Y2, x, s, false); + render_delay_buffer->GetRenderBuffer(), E2_main, Y2, x, s, + false); EXPECT_TRUE(k == (kFilterLength - 1) || state.FilterDelay()); if (k != (kFilterLength - 1)) { EXPECT_EQ(k, state.FilterDelay()); @@ -190,7 +215,10 @@ TEST(AecState, ConvergedFilterDelay) { // Verify that the externally reported delay is properly reported and converted. TEST(AecState, ExternalDelay) { - AecState state(EchoCanceller3Config{}); + EchoCanceller3Config config; + AecState state(config); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, 3)); std::array E2_main; std::array E2_shadow; std::array Y2; @@ -201,8 +229,7 @@ TEST(AecState, ExternalDelay) { E2_shadow.fill(0.f); Y2.fill(0.f); x.fill(0.f); - RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 30, - std::vector(1, 30)); + std::vector> frequency_response( kAdaptiveFilterLength); for (auto& v : frequency_response) { @@ -213,18 +240,22 @@ TEST(AecState, ExternalDelay) { impulse_response.fill(0.f); for (size_t k = 0; k < frequency_response.size() - 1; ++k) { - state.HandleEchoPathChange(EchoPathVariability(false, false)); + state.HandleEchoPathChange(EchoPathVariability( + false, EchoPathVariability::DelayAdjustment::kNone, false)); state.Update(frequency_response, impulse_response, true, k * kBlockSize + 5, - render_buffer, E2_main, Y2, x, s, false); + render_delay_buffer->GetRenderBuffer(), E2_main, Y2, x, s, + false); EXPECT_TRUE(state.ExternalDelay()); EXPECT_EQ(k, state.ExternalDelay()); } // Verify that the externally reported delay is properly unset when it is no // longer present. - state.HandleEchoPathChange(EchoPathVariability(false, false)); + state.HandleEchoPathChange(EchoPathVariability( + false, EchoPathVariability::DelayAdjustment::kNone, false)); state.Update(frequency_response, impulse_response, true, rtc::nullopt, - render_buffer, E2_main, Y2, x, s, false); + render_delay_buffer->GetRenderBuffer(), E2_main, Y2, x, s, + false); EXPECT_FALSE(state.ExternalDelay()); } diff --git a/modules/audio_processing/aec3/block_processor.cc b/modules/audio_processing/aec3/block_processor.cc index f0b963087c..f12e7eed3c 100644 --- a/modules/audio_processing/aec3/block_processor.cc +++ b/modules/audio_processing/aec3/block_processor.cc @@ -25,7 +25,8 @@ enum class BlockProcessorApiCall { kCapture, kRender }; class BlockProcessorImpl final : public BlockProcessor { public: - BlockProcessorImpl(int sample_rate_hz, + BlockProcessorImpl(const EchoCanceller3Config& config, + int sample_rate_hz, std::unique_ptr render_buffer, std::unique_ptr delay_controller, std::unique_ptr echo_remover); @@ -44,31 +45,35 @@ class BlockProcessorImpl final : public BlockProcessor { private: static int instance_count_; - bool no_capture_data_received_ = true; - bool no_render_data_received_ = true; std::unique_ptr data_dumper_; + const EchoCanceller3Config config_; + bool capture_properly_started_ = false; + bool render_properly_started_ = false; const size_t sample_rate_hz_; std::unique_ptr render_buffer_; std::unique_ptr delay_controller_; std::unique_ptr echo_remover_; BlockProcessorMetrics metrics_; - bool render_buffer_overrun_occurred_ = false; + RenderDelayBuffer::BufferingEvent render_event_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(BlockProcessorImpl); }; int BlockProcessorImpl::instance_count_ = 0; BlockProcessorImpl::BlockProcessorImpl( + const EchoCanceller3Config& config, int sample_rate_hz, std::unique_ptr render_buffer, std::unique_ptr delay_controller, std::unique_ptr echo_remover) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + config_(config), sample_rate_hz_(sample_rate_hz), render_buffer_(std::move(render_buffer)), delay_controller_(std::move(delay_controller)), - echo_remover_(std::move(echo_remover)) { + echo_remover_(std::move(echo_remover)), + render_event_(RenderDelayBuffer::BufferingEvent::kNone) { RTC_DCHECK(ValidFullBandRate(sample_rate_hz_)); } @@ -87,67 +92,93 @@ void BlockProcessorImpl::ProcessCapture( &(*capture_block)[0][0], LowestBandRate(sample_rate_hz_), 1); - // Do not start processing until render data has been buffered as that will - // cause the buffers to be wrongly aligned. - no_capture_data_received_ = false; - if (no_render_data_received_) { + if (render_properly_started_) { + if (!capture_properly_started_) { + capture_properly_started_ = true; + render_buffer_->Reset(); + } + } else { + // If no render data has yet arrived, do not process the capture signal. return; } + EchoPathVariability echo_path_variability( + echo_path_gain_change, EchoPathVariability::DelayAdjustment::kNone, + false); + + if (render_event_ == RenderDelayBuffer::BufferingEvent::kRenderOverrun && + render_properly_started_) { + echo_path_variability.delay_change = + EchoPathVariability::DelayAdjustment::kBufferFlush; + delay_controller_->Reset(); + render_buffer_->Reset(); + RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun."; + } + + // Update the render buffers with any newly arrived render blocks and prepare + // the render buffers for reading the render data corresponding to the current + // capture block. + render_event_ = render_buffer_->PrepareCaptureCall(); + RTC_DCHECK(RenderDelayBuffer::BufferingEvent::kRenderOverrun != + render_event_); + if (render_event_ == RenderDelayBuffer::BufferingEvent::kRenderUnderrun) { + echo_path_variability.delay_change = + EchoPathVariability::DelayAdjustment::kBufferReadjustment; + delay_controller_->Reset(); + render_buffer_->Reset(); + } else if (render_event_ == RenderDelayBuffer::BufferingEvent::kApiCallSkew) { + // There have been too many render calls in a row. Reset to avoid noncausal + // echo. + echo_path_variability.delay_change = + EchoPathVariability::DelayAdjustment::kDelayReset; + delay_controller_->Reset(); + render_buffer_->Reset(); + capture_properly_started_ = false; + render_properly_started_ = false; + } + data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize, &(*capture_block)[0][0], LowestBandRate(sample_rate_hz_), 1); - bool render_buffer_underrun = false; - if (render_buffer_overrun_occurred_) { - // Reset the render buffers and the alignment functionality when there has - // been a render buffer overrun as the buffer alignment may be noncausal. - delay_controller_->Reset(); - render_buffer_->Reset(); - RTC_LOG(LS_WARNING) << "Reset due to detected render buffer overrun."; - } - - // Update the render buffers with new render data, filling the buffers with - // empty blocks when there is no render data available. - render_buffer_underrun = !render_buffer_->UpdateBuffers(); - if (render_buffer_underrun) { - RTC_LOG(LS_WARNING) << "Render API jitter buffer underrun."; - } - // Compute and and apply the render delay required to achieve proper signal // alignment. - const size_t old_delay = render_buffer_->Delay(); - const size_t new_delay = delay_controller_->GetDelay( + const size_t estimated_delay = delay_controller_->GetDelay( render_buffer_->GetDownsampledRenderBuffer(), (*capture_block)[0]); + const size_t new_delay = + std::min(render_buffer_->MaxDelay(), estimated_delay); - bool delay_change; - if (new_delay >= kMinEchoPathDelayBlocks) { + bool delay_change = render_buffer_->Delay() != new_delay; + if (delay_change && new_delay >= config_.delay.min_echo_path_delay_blocks) { + echo_path_variability.delay_change = + EchoPathVariability::DelayAdjustment::kNewDetectedDelay; render_buffer_->SetDelay(new_delay); - const size_t achieved_delay = render_buffer_->Delay(); - delay_change = old_delay != achieved_delay || old_delay != new_delay || - render_buffer_overrun_occurred_; - - // Inform the delay controller of the actually set delay to allow it to - // properly react to a non-feasible delay. - delay_controller_->SetDelay(achieved_delay); - } else { + RTC_DCHECK_EQ(render_buffer_->Delay(), new_delay); + delay_controller_->SetDelay(new_delay); + } else if (delay_change && + new_delay < config_.delay.min_echo_path_delay_blocks) { + // A noncausal delay has been detected. This can only happen if there is + // clockdrift, an audio pipeline issue has occurred or the specified minimum + // delay is too short. Perform a full reset. + echo_path_variability.delay_change = + EchoPathVariability::DelayAdjustment::kDelayReset; delay_controller_->Reset(); render_buffer_->Reset(); - delay_change = true; + capture_properly_started_ = false; + render_properly_started_ = false; RTC_LOG(LS_WARNING) << "Reset due to noncausal delay."; } // Remove the echo from the capture signal. echo_remover_->ProcessCapture( - delay_controller_->AlignmentHeadroomSamples(), - EchoPathVariability(echo_path_gain_change, delay_change), + delay_controller_->AlignmentHeadroomSamples(), echo_path_variability, capture_signal_saturation, render_buffer_->GetRenderBuffer(), capture_block); // Update the metrics. - metrics_.UpdateCapture(render_buffer_underrun); + metrics_.UpdateCapture(false); - render_buffer_overrun_occurred_ = false; + render_event_ = RenderDelayBuffer::BufferingEvent::kNone; } void BlockProcessorImpl::BufferRender( @@ -158,23 +189,15 @@ void BlockProcessorImpl::BufferRender( static_cast(BlockProcessorApiCall::kRender)); data_dumper_->DumpWav("aec3_processblock_render_input", kBlockSize, &block[0][0], LowestBandRate(sample_rate_hz_), 1); - - no_render_data_received_ = false; - - // Do not start buffer render data until capture data has been received as - // that data may give a false alignment. - if (no_capture_data_received_) { - return; - } - data_dumper_->DumpWav("aec3_processblock_render_input2", kBlockSize, &block[0][0], LowestBandRate(sample_rate_hz_), 1); - // Buffer the render data. - render_buffer_overrun_occurred_ = !render_buffer_->Insert(block); + render_event_ = render_buffer_->Insert(block); - // Update the metrics. - metrics_.UpdateRender(render_buffer_overrun_occurred_); + metrics_.UpdateRender(render_event_ != + RenderDelayBuffer::BufferingEvent::kNone); + + render_properly_started_ = true; } void BlockProcessorImpl::UpdateEchoLeakageStatus(bool leakage_detected) { @@ -191,12 +214,8 @@ void BlockProcessorImpl::GetMetrics(EchoControl::Metrics* metrics) const { BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config, int sample_rate_hz) { - std::unique_ptr render_buffer(RenderDelayBuffer::Create( - NumBandsForRate(sample_rate_hz), config.delay.down_sampling_factor, - GetDownSampledBufferSize(config.delay.down_sampling_factor, - config.delay.num_filters), - GetRenderDelayBufferSize(config.delay.down_sampling_factor, - config.delay.num_filters))); + std::unique_ptr render_buffer( + RenderDelayBuffer::Create(config, NumBandsForRate(sample_rate_hz))); std::unique_ptr delay_controller( RenderDelayController::Create(config, sample_rate_hz)); std::unique_ptr echo_remover( @@ -223,9 +242,9 @@ BlockProcessor* BlockProcessor::Create( std::unique_ptr render_buffer, std::unique_ptr delay_controller, std::unique_ptr echo_remover) { - return new BlockProcessorImpl(sample_rate_hz, std::move(render_buffer), - std::move(delay_controller), - std::move(echo_remover)); + return new BlockProcessorImpl( + config, sample_rate_hz, std::move(render_buffer), + std::move(delay_controller), std::move(echo_remover)); } } // namespace webrtc diff --git a/modules/audio_processing/aec3/block_processor_unittest.cc b/modules/audio_processing/aec3/block_processor_unittest.cc index 18d1f65ce3..1d97002ecf 100644 --- a/modules/audio_processing/aec3/block_processor_unittest.cc +++ b/modules/audio_processing/aec3/block_processor_unittest.cc @@ -115,7 +115,7 @@ TEST(BlockProcessor, DISABLED_DelayControllerIntegration) { new StrictMock(rate)); EXPECT_CALL(*render_delay_buffer_mock, Insert(_)) .Times(kNumBlocks) - .WillRepeatedly(Return(true)); + .WillRepeatedly(Return(RenderDelayBuffer::BufferingEvent::kNone)); EXPECT_CALL(*render_delay_buffer_mock, IsBlockAvailable()) .Times(kNumBlocks) .WillRepeatedly(Return(true)); @@ -160,11 +160,12 @@ TEST(BlockProcessor, DISABLED_SubmoduleIntegration) { EXPECT_CALL(*render_delay_buffer_mock, Insert(_)) .Times(kNumBlocks - 1) - .WillRepeatedly(Return(true)); + .WillRepeatedly(Return(RenderDelayBuffer::BufferingEvent::kNone)); EXPECT_CALL(*render_delay_buffer_mock, IsBlockAvailable()) .Times(kNumBlocks) .WillRepeatedly(Return(true)); - EXPECT_CALL(*render_delay_buffer_mock, UpdateBuffers()).Times(kNumBlocks); + EXPECT_CALL(*render_delay_buffer_mock, PrepareCaptureCall()) + .Times(kNumBlocks); EXPECT_CALL(*render_delay_buffer_mock, SetDelay(9)).Times(AtLeast(1)); EXPECT_CALL(*render_delay_buffer_mock, Delay()) .Times(kNumBlocks) diff --git a/modules/audio_processing/aec3/comfort_noise_generator_unittest.cc b/modules/audio_processing/aec3/comfort_noise_generator_unittest.cc index 46da3eca46..d7e9407dff 100644 --- a/modules/audio_processing/aec3/comfort_noise_generator_unittest.cc +++ b/modules/audio_processing/aec3/comfort_noise_generator_unittest.cc @@ -24,7 +24,7 @@ namespace { float Power(const FftData& N) { std::array N2; - N.Spectrum(Aec3Optimization::kNone, &N2); + N.Spectrum(Aec3Optimization::kNone, N2); return std::accumulate(N2.begin(), N2.end(), 0.f) / N2.size(); } diff --git a/modules/audio_processing/aec3/downsampled_render_buffer.cc b/modules/audio_processing/aec3/downsampled_render_buffer.cc index efc733b182..cd6ae48649 100644 --- a/modules/audio_processing/aec3/downsampled_render_buffer.cc +++ b/modules/audio_processing/aec3/downsampled_render_buffer.cc @@ -13,7 +13,9 @@ namespace webrtc { DownsampledRenderBuffer::DownsampledRenderBuffer(size_t downsampled_buffer_size) - : buffer(downsampled_buffer_size, 0.f) {} + : size(downsampled_buffer_size), buffer(downsampled_buffer_size, 0.f) { + std::fill(buffer.begin(), buffer.end(), 0.f); +} DownsampledRenderBuffer::~DownsampledRenderBuffer() = default; diff --git a/modules/audio_processing/aec3/downsampled_render_buffer.h b/modules/audio_processing/aec3/downsampled_render_buffer.h index 531852a0c9..c656846960 100644 --- a/modules/audio_processing/aec3/downsampled_render_buffer.h +++ b/modules/audio_processing/aec3/downsampled_render_buffer.h @@ -14,6 +14,7 @@ #include #include "modules/audio_processing/aec3/aec3_common.h" +#include "rtc_base/checks.h" namespace webrtc { @@ -21,8 +22,31 @@ namespace webrtc { struct DownsampledRenderBuffer { explicit DownsampledRenderBuffer(size_t downsampled_buffer_size); ~DownsampledRenderBuffer(); + + size_t IncIndex(size_t index) { + return index < (buffer.size() - 1) ? index + 1 : 0; + } + + size_t DecIndex(size_t index) { + return index > 0 ? index - 1 : buffer.size() - 1; + } + + size_t OffsetIndex(size_t index, int offset) { + RTC_DCHECK_GE(buffer.size(), offset); + return (buffer.size() + index + offset) % buffer.size(); + } + + void UpdateWriteIndex(int offset) { write = OffsetIndex(write, offset); } + void IncWriteIndex() { write = IncIndex(write); } + void DecWriteIndex() { write = DecIndex(write); } + void UpdateReadIndex(int offset) { read = OffsetIndex(read, offset); } + void IncReadIndex() { read = IncIndex(read); } + void DecReadIndex() { read = DecIndex(read); } + + size_t size; std::vector buffer; - int position = 0; + int write = 0; + int read = 0; }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/echo_canceller3.cc b/modules/audio_processing/aec3/echo_canceller3.cc index 491faa01fb..100ed27160 100644 --- a/modules/audio_processing/aec3/echo_canceller3.cc +++ b/modules/audio_processing/aec3/echo_canceller3.cc @@ -203,11 +203,13 @@ int EchoCanceller3::instance_count_ = 0; EchoCanceller3::EchoCanceller3(const EchoCanceller3Config& config, int sample_rate_hz, bool use_highpass_filter) - : EchoCanceller3(sample_rate_hz, + : EchoCanceller3(config, + sample_rate_hz, use_highpass_filter, std::unique_ptr( BlockProcessor::Create(config, sample_rate_hz))) {} -EchoCanceller3::EchoCanceller3(int sample_rate_hz, +EchoCanceller3::EchoCanceller3(const EchoCanceller3Config& config, + int sample_rate_hz, bool use_highpass_filter, std::unique_ptr block_processor) : data_dumper_( @@ -219,7 +221,7 @@ EchoCanceller3::EchoCanceller3(int sample_rate_hz, capture_blocker_(num_bands_), render_blocker_(num_bands_), render_transfer_queue_( - kRenderTransferQueueSize, + kRenderTransferQueueSizeFrames, std::vector>( num_bands_, std::vector(frame_length_, 0.f)), diff --git a/modules/audio_processing/aec3/echo_canceller3.h b/modules/audio_processing/aec3/echo_canceller3.h index 475bacb723..d136c1e011 100644 --- a/modules/audio_processing/aec3/echo_canceller3.h +++ b/modules/audio_processing/aec3/echo_canceller3.h @@ -67,7 +67,8 @@ class EchoCanceller3 : public EchoControl { int sample_rate_hz, bool use_highpass_filter); // Testing c-tor that is used only for testing purposes. - EchoCanceller3(int sample_rate_hz, + EchoCanceller3(const EchoCanceller3Config& config, + int sample_rate_hz, bool use_highpass_filter, std::unique_ptr block_processor); ~EchoCanceller3() override; diff --git a/modules/audio_processing/aec3/echo_canceller3_unittest.cc b/modules/audio_processing/aec3/echo_canceller3_unittest.cc index 75de48b547..d54295f90a 100644 --- a/modules/audio_processing/aec3/echo_canceller3_unittest.cc +++ b/modules/audio_processing/aec3/echo_canceller3_unittest.cc @@ -160,7 +160,7 @@ class EchoCanceller3Tester { // output. void RunCaptureTransportVerificationTest() { EchoCanceller3 aec3( - sample_rate_hz_, false, + EchoCanceller3Config(), sample_rate_hz_, false, std::unique_ptr( new CaptureTransportVerificationProcessor(num_bands_))); @@ -185,7 +185,7 @@ class EchoCanceller3Tester { // block processor. void RunRenderTransportVerificationTest() { EchoCanceller3 aec3( - sample_rate_hz_, false, + EchoCanceller3Config(), sample_rate_hz_, false, std::unique_ptr( new RenderTransportVerificationProcessor(num_bands_))); @@ -249,7 +249,7 @@ class EchoCanceller3Tester { break; } - EchoCanceller3 aec3(sample_rate_hz_, false, + EchoCanceller3 aec3(EchoCanceller3Config(), sample_rate_hz_, false, std::move(block_processor_mock)); for (size_t frame_index = 0; frame_index < kNumFramesToProcess; @@ -331,7 +331,7 @@ class EchoCanceller3Tester { } break; } - EchoCanceller3 aec3(sample_rate_hz_, false, + EchoCanceller3 aec3(EchoCanceller3Config(), sample_rate_hz_, false, std::move(block_processor_mock)); for (size_t frame_index = 0; frame_index < kNumFramesToProcess; @@ -420,7 +420,7 @@ class EchoCanceller3Tester { } break; } - EchoCanceller3 aec3(sample_rate_hz_, false, + EchoCanceller3 aec3(EchoCanceller3Config(), sample_rate_hz_, false, std::move(block_processor_mock)); for (size_t frame_index = 0; frame_index < kNumFramesToProcess; ++frame_index) { @@ -458,12 +458,13 @@ class EchoCanceller3Tester { // This test verifies that the swapqueue is able to handle jitter in the // capture and render API calls. void RunRenderSwapQueueVerificationTest() { + const EchoCanceller3Config config; EchoCanceller3 aec3( - sample_rate_hz_, false, + config, sample_rate_hz_, false, std::unique_ptr( new RenderTransportVerificationProcessor(num_bands_))); - for (size_t frame_index = 0; frame_index < kRenderTransferQueueSize; + for (size_t frame_index = 0; frame_index < kRenderTransferQueueSizeFrames; ++frame_index) { if (sample_rate_hz_ > 16000) { render_buffer_.SplitIntoFrequencyBands(); @@ -478,7 +479,7 @@ class EchoCanceller3Tester { aec3.AnalyzeRender(&render_buffer_); } - for (size_t frame_index = 0; frame_index < kRenderTransferQueueSize; + for (size_t frame_index = 0; frame_index < kRenderTransferQueueSizeFrames; ++frame_index) { aec3.AnalyzeCapture(&capture_buffer_); if (sample_rate_hz_ > 16000) { diff --git a/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc b/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc index 2dbdb1ccf4..e51287b053 100644 --- a/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc +++ b/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc @@ -16,6 +16,7 @@ #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/render_delay_buffer.h" +#include "modules/audio_processing/include/audio_processing.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "modules/audio_processing/test/echo_canceller_test_tools.h" #include "rtc_base/random.h" @@ -24,9 +25,10 @@ namespace webrtc { namespace { -std::string ProduceDebugText(size_t delay) { +std::string ProduceDebugText(size_t delay, size_t down_sampling_factor) { std::ostringstream ss; ss << "Delay: " << delay; + ss << ", Down sampling factor: " << down_sampling_factor; return ss.str(); } @@ -37,12 +39,7 @@ TEST(EchoPathDelayEstimator, BasicApiCalls) { ApmDataDumper data_dumper(0); EchoCanceller3Config config; std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create( - 3, config.delay.down_sampling_factor, - GetDownSampledBufferSize(config.delay.down_sampling_factor, - config.delay.num_filters), - GetRenderDelayBufferSize(config.delay.down_sampling_factor, - config.delay.num_filters))); + RenderDelayBuffer::Create(config, 3)); EchoPathDelayEstimator estimator(&data_dumper, config); std::vector> render(3, std::vector(kBlockSize)); std::vector capture(kBlockSize); @@ -66,14 +63,22 @@ TEST(EchoPathDelayEstimator, DelayEstimation) { config.delay.down_sampling_factor = down_sampling_factor; config.delay.num_filters = 10; for (size_t delay_samples : {30, 64, 150, 200, 800, 4000}) { - SCOPED_TRACE(ProduceDebugText(delay_samples)); + SCOPED_TRACE(ProduceDebugText(delay_samples, down_sampling_factor)); + + config.delay.min_echo_path_delay_blocks = 0; + while ((config.delay.min_echo_path_delay_blocks + 1) * kBlockSize < + delay_samples && + config.delay.min_echo_path_delay_blocks + 1 <= 5) { + ++config.delay.min_echo_path_delay_blocks; + } + + const int delay_estimate_offset = + std::max(std::min(config.delay.api_call_jitter_blocks, + config.delay.min_echo_path_delay_blocks) - + 1, + 0); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create( - 3, config.delay.down_sampling_factor, - GetDownSampledBufferSize(config.delay.down_sampling_factor, - config.delay.num_filters), - GetRenderDelayBufferSize(config.delay.down_sampling_factor, - config.delay.num_filters))); + RenderDelayBuffer::Create(config, 3)); DelayBuffer signal_delay_buffer(delay_samples); EchoPathDelayEstimator estimator(&data_dumper, config); @@ -82,21 +87,29 @@ TEST(EchoPathDelayEstimator, DelayEstimation) { RandomizeSampleVector(&random_generator, render[0]); signal_delay_buffer.Delay(render[0], capture); render_delay_buffer->Insert(render); - render_delay_buffer->UpdateBuffers(); + + if (k == 0) { + render_delay_buffer->Reset(); + } + + render_delay_buffer->PrepareCaptureCall(); estimated_delay_samples = estimator.EstimateDelay( render_delay_buffer->GetDownsampledRenderBuffer(), capture); } + if (estimated_delay_samples) { // Due to the internal down-sampling done inside the delay estimator // the estimated delay cannot be expected to be exact to the true delay. - EXPECT_NEAR(delay_samples, *estimated_delay_samples, - config.delay.down_sampling_factor); + EXPECT_NEAR( + delay_samples, + *estimated_delay_samples + delay_estimate_offset * kBlockSize, + config.delay.down_sampling_factor); } else { ADD_FAILURE(); } - } } } +} // Verifies that the delay estimator does not produce delay estimates too // quickly. @@ -107,19 +120,14 @@ TEST(EchoPathDelayEstimator, NoInitialDelayestimates) { std::vector capture(kBlockSize); ApmDataDumper data_dumper(0); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create( - 3, config.delay.down_sampling_factor, - GetDownSampledBufferSize(config.delay.down_sampling_factor, - config.delay.num_filters), - GetRenderDelayBufferSize(config.delay.down_sampling_factor, - config.delay.num_filters))); + RenderDelayBuffer::Create(config, 3)); EchoPathDelayEstimator estimator(&data_dumper, config); for (size_t k = 0; k < 19; ++k) { RandomizeSampleVector(&random_generator, render[0]); std::copy(render[0].begin(), render[0].end(), capture.begin()); render_delay_buffer->Insert(render); - render_delay_buffer->UpdateBuffers(); + render_delay_buffer->PrepareCaptureCall(); EXPECT_FALSE(estimator.EstimateDelay( render_delay_buffer->GetDownsampledRenderBuffer(), capture)); } @@ -135,12 +143,7 @@ TEST(EchoPathDelayEstimator, NoDelayEstimatesForLowLevelRenderSignals) { ApmDataDumper data_dumper(0); EchoPathDelayEstimator estimator(&data_dumper, config); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create( - 3, config.delay.down_sampling_factor, - GetDownSampledBufferSize(config.delay.down_sampling_factor, - config.delay.num_filters), - GetRenderDelayBufferSize(config.delay.down_sampling_factor, - config.delay.num_filters))); + RenderDelayBuffer::Create(EchoCanceller3Config(), 3)); for (size_t k = 0; k < 100; ++k) { RandomizeSampleVector(&random_generator, render[0]); for (auto& render_k : render[0]) { @@ -148,7 +151,7 @@ TEST(EchoPathDelayEstimator, NoDelayEstimatesForLowLevelRenderSignals) { } std::copy(render[0].begin(), render[0].end(), capture.begin()); render_delay_buffer->Insert(render); - render_delay_buffer->UpdateBuffers(); + render_delay_buffer->PrepareCaptureCall(); EXPECT_FALSE(estimator.EstimateDelay( render_delay_buffer->GetDownsampledRenderBuffer(), capture)); } @@ -164,17 +167,12 @@ TEST(EchoPathDelayEstimator, NoDelayEstimatesForUncorrelatedSignals) { ApmDataDumper data_dumper(0); EchoPathDelayEstimator estimator(&data_dumper, config); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create( - 3, config.delay.down_sampling_factor, - GetDownSampledBufferSize(config.delay.down_sampling_factor, - config.delay.num_filters), - GetRenderDelayBufferSize(config.delay.down_sampling_factor, - config.delay.num_filters))); + RenderDelayBuffer::Create(config, 3)); for (size_t k = 0; k < 100; ++k) { RandomizeSampleVector(&random_generator, render[0]); RandomizeSampleVector(&random_generator, capture); render_delay_buffer->Insert(render); - render_delay_buffer->UpdateBuffers(); + render_delay_buffer->PrepareCaptureCall(); EXPECT_FALSE(estimator.EstimateDelay( render_delay_buffer->GetDownsampledRenderBuffer(), capture)); } @@ -190,12 +188,7 @@ TEST(EchoPathDelayEstimator, DISABLED_WrongRenderBlockSize) { EchoCanceller3Config config; EchoPathDelayEstimator estimator(&data_dumper, config); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create( - 3, config.delay.down_sampling_factor, - GetDownSampledBufferSize(config.delay.down_sampling_factor, - config.delay.num_filters), - GetRenderDelayBufferSize(config.delay.down_sampling_factor, - config.delay.num_filters))); + RenderDelayBuffer::Create(config, 3)); std::vector capture(kBlockSize); EXPECT_DEATH(estimator.EstimateDelay( render_delay_buffer->GetDownsampledRenderBuffer(), capture), @@ -210,12 +203,7 @@ TEST(EchoPathDelayEstimator, WrongCaptureBlockSize) { EchoCanceller3Config config; EchoPathDelayEstimator estimator(&data_dumper, config); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create( - 3, config.delay.down_sampling_factor, - GetDownSampledBufferSize(config.delay.down_sampling_factor, - config.delay.num_filters), - GetRenderDelayBufferSize(config.delay.down_sampling_factor, - config.delay.num_filters))); + RenderDelayBuffer::Create(config, 3)); std::vector capture(std::vector(kBlockSize - 1)); EXPECT_DEATH(estimator.EstimateDelay( render_delay_buffer->GetDownsampledRenderBuffer(), capture), diff --git a/modules/audio_processing/aec3/echo_path_variability.cc b/modules/audio_processing/aec3/echo_path_variability.cc index f63a83006e..0ae9cff98e 100644 --- a/modules/audio_processing/aec3/echo_path_variability.cc +++ b/modules/audio_processing/aec3/echo_path_variability.cc @@ -12,7 +12,11 @@ namespace webrtc { -EchoPathVariability::EchoPathVariability(bool gain_change, bool delay_change) - : gain_change(gain_change), delay_change(delay_change) {} +EchoPathVariability::EchoPathVariability(bool gain_change, + DelayAdjustment delay_change, + bool clock_drift) + : gain_change(gain_change), + delay_change(delay_change), + clock_drift(clock_drift) {} } // namespace webrtc diff --git a/modules/audio_processing/aec3/echo_path_variability.h b/modules/audio_processing/aec3/echo_path_variability.h index 55915d5b68..adf0d7a4ad 100644 --- a/modules/audio_processing/aec3/echo_path_variability.h +++ b/modules/audio_processing/aec3/echo_path_variability.h @@ -14,11 +14,24 @@ namespace webrtc { struct EchoPathVariability { - EchoPathVariability(bool gain_change, bool delay_change); + enum class DelayAdjustment { + kNone, + kBufferReadjustment, + kBufferFlush, + kDelayReset, + kNewDetectedDelay + }; - bool AudioPathChanged() const { return gain_change || delay_change; } + EchoPathVariability(bool gain_change, + DelayAdjustment delay_change, + bool clock_drift); + + bool AudioPathChanged() const { + return gain_change || delay_change != DelayAdjustment::kNone; + } bool gain_change; - bool delay_change; + DelayAdjustment delay_change; + bool clock_drift; }; } // namespace webrtc diff --git a/modules/audio_processing/aec3/echo_path_variability_unittest.cc b/modules/audio_processing/aec3/echo_path_variability_unittest.cc index 9a1df78885..b1795edb6f 100644 --- a/modules/audio_processing/aec3/echo_path_variability_unittest.cc +++ b/modules/audio_processing/aec3/echo_path_variability_unittest.cc @@ -15,25 +15,35 @@ namespace webrtc { TEST(EchoPathVariability, CorrectBehavior) { // Test correct passing and reporting of the gain change information. - EchoPathVariability v(true, true); + EchoPathVariability v( + true, EchoPathVariability::DelayAdjustment::kNewDetectedDelay, false); EXPECT_TRUE(v.gain_change); - EXPECT_TRUE(v.delay_change); + EXPECT_TRUE(v.delay_change == + EchoPathVariability::DelayAdjustment::kNewDetectedDelay); EXPECT_TRUE(v.AudioPathChanged()); + EXPECT_FALSE(v.clock_drift); - v = EchoPathVariability(true, false); + v = EchoPathVariability(true, EchoPathVariability::DelayAdjustment::kNone, + false); EXPECT_TRUE(v.gain_change); - EXPECT_FALSE(v.delay_change); + EXPECT_TRUE(v.delay_change == EchoPathVariability::DelayAdjustment::kNone); EXPECT_TRUE(v.AudioPathChanged()); + EXPECT_FALSE(v.clock_drift); - v = EchoPathVariability(false, true); + v = EchoPathVariability( + false, EchoPathVariability::DelayAdjustment::kNewDetectedDelay, false); EXPECT_FALSE(v.gain_change); - EXPECT_TRUE(v.delay_change); + EXPECT_TRUE(v.delay_change == + EchoPathVariability::DelayAdjustment::kNewDetectedDelay); EXPECT_TRUE(v.AudioPathChanged()); + EXPECT_FALSE(v.clock_drift); - v = EchoPathVariability(false, false); + v = EchoPathVariability(false, EchoPathVariability::DelayAdjustment::kNone, + false); EXPECT_FALSE(v.gain_change); - EXPECT_FALSE(v.delay_change); + EXPECT_TRUE(v.delay_change == EchoPathVariability::DelayAdjustment::kNone); EXPECT_FALSE(v.AudioPathChanged()); + EXPECT_FALSE(v.clock_drift); } } // namespace webrtc diff --git a/modules/audio_processing/aec3/echo_remover.cc b/modules/audio_processing/aec3/echo_remover.cc index 9adcec51f4..fc72207f2e 100644 --- a/modules/audio_processing/aec3/echo_remover.cc +++ b/modules/audio_processing/aec3/echo_remover.cc @@ -174,7 +174,7 @@ void EchoRemoverImpl::ProcessCapture( // Compute spectra. fft_.ZeroPaddedFft(y0, &Y); LinearEchoPower(E_main, Y, &S2_linear); - Y.Spectrum(optimization_, &Y2); + Y.Spectrum(optimization_, Y2); // Update the AEC state information. aec_state_.Update(subtractor_.FilterFrequencyResponse(), diff --git a/modules/audio_processing/aec3/echo_remover_metrics_unittest.cc b/modules/audio_processing/aec3/echo_remover_metrics_unittest.cc index e20971048b..2b30a74541 100644 --- a/modules/audio_processing/aec3/echo_remover_metrics_unittest.cc +++ b/modules/audio_processing/aec3/echo_remover_metrics_unittest.cc @@ -65,7 +65,7 @@ TEST(TransformDbMetricForReporting, DbFsScaling) { Aec3Fft fft; x.fill(1000.f); fft.ZeroPaddedFft(x, &X); - X.Spectrum(Aec3Optimization::kNone, &X2); + X.Spectrum(Aec3Optimization::kNone, X2); float offset = -10.f * log10(32768.f * 32768.f); EXPECT_NEAR(offset, -90.3f, 0.1f); diff --git a/modules/audio_processing/aec3/echo_remover_unittest.cc b/modules/audio_processing/aec3/echo_remover_unittest.cc index 24b50e8c95..f035f4f4d1 100644 --- a/modules/audio_processing/aec3/echo_remover_unittest.cc +++ b/modules/audio_processing/aec3/echo_remover_unittest.cc @@ -39,9 +39,6 @@ std::string ProduceDebugText(int sample_rate_hz, int delay) { return ss.str(); } -constexpr size_t kDownSamplingFactor = 4; -constexpr size_t kNumMatchedFilters = 4; - } // namespace // Verifies the basic API call sequence @@ -51,22 +48,23 @@ TEST(EchoRemover, BasicApiCalls) { std::unique_ptr remover( EchoRemover::Create(EchoCanceller3Config(), rate)); std::unique_ptr render_buffer(RenderDelayBuffer::Create( - NumBandsForRate(rate), kDownSamplingFactor, - GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), - GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + EchoCanceller3Config(), NumBandsForRate(rate))); std::vector> render(NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); std::vector> capture( NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); for (size_t k = 0; k < 100; ++k) { - EchoPathVariability echo_path_variability(k % 3 == 0 ? true : false, - k % 5 == 0 ? true : false); + EchoPathVariability echo_path_variability( + k % 3 == 0 ? true : false, + k % 5 == 0 ? EchoPathVariability::DelayAdjustment::kNewDetectedDelay + : EchoPathVariability::DelayAdjustment::kNone, + false); rtc::Optional echo_path_delay_samples = (k % 6 == 0 ? rtc::Optional(k * 10) : rtc::nullopt); render_buffer->Insert(render); - render_buffer->UpdateBuffers(); + render_buffer->PrepareCaptureCall(); remover->ProcessCapture(echo_path_delay_samples, echo_path_variability, k % 2 == 0 ? true : false, render_buffer->GetRenderBuffer(), &capture); @@ -92,12 +90,11 @@ TEST(EchoRemover, WrongCaptureBlockSize) { std::unique_ptr remover( EchoRemover::Create(EchoCanceller3Config(), rate)); std::unique_ptr render_buffer(RenderDelayBuffer::Create( - NumBandsForRate(rate), kDownSamplingFactor, - GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), - GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + EchoCanceller3Config(), NumBandsForRate(rate))); std::vector> capture( NumBandsForRate(rate), std::vector(kBlockSize - 1, 0.f)); - EchoPathVariability echo_path_variability(false, false); + EchoPathVariability echo_path_variability( + false, EchoPathVariability::DelayAdjustment::kNone, false); rtc::Optional echo_path_delay_samples; EXPECT_DEATH(remover->ProcessCapture( echo_path_delay_samples, echo_path_variability, false, @@ -115,13 +112,12 @@ TEST(EchoRemover, DISABLED_WrongCaptureNumBands) { std::unique_ptr remover( EchoRemover::Create(EchoCanceller3Config(), rate)); std::unique_ptr render_buffer(RenderDelayBuffer::Create( - NumBandsForRate(rate), kDownSamplingFactor, - GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), - GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + EchoCanceller3Config(), NumBandsForRate(rate))); std::vector> capture( NumBandsForRate(rate == 48000 ? 16000 : rate + 16000), std::vector(kBlockSize, 0.f)); - EchoPathVariability echo_path_variability(false, false); + EchoPathVariability echo_path_variability( + false, EchoPathVariability::DelayAdjustment::kNone, false); rtc::Optional echo_path_delay_samples; EXPECT_DEATH(remover->ProcessCapture( echo_path_delay_samples, echo_path_variability, false, @@ -134,11 +130,10 @@ TEST(EchoRemover, DISABLED_WrongCaptureNumBands) { TEST(EchoRemover, NullCapture) { std::unique_ptr remover( EchoRemover::Create(EchoCanceller3Config(), 8000)); - std::unique_ptr render_buffer(RenderDelayBuffer::Create( - 3, kDownSamplingFactor, - GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), - GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); - EchoPathVariability echo_path_variability(false, false); + std::unique_ptr render_buffer( + RenderDelayBuffer::Create(EchoCanceller3Config(), 3)); + EchoPathVariability echo_path_variability( + false, EchoPathVariability::DelayAdjustment::kNone, false); rtc::Optional echo_path_delay_samples; EXPECT_DEATH( remover->ProcessCapture(echo_path_delay_samples, echo_path_variability, @@ -158,17 +153,18 @@ TEST(EchoRemover, BasicEchoRemoval) { std::vector(kBlockSize, 0.f)); std::vector> y(NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); - EchoPathVariability echo_path_variability(false, false); + EchoPathVariability echo_path_variability( + false, EchoPathVariability::DelayAdjustment::kNone, false); for (size_t delay_samples : {0, 64, 150, 200, 301}) { SCOPED_TRACE(ProduceDebugText(rate, delay_samples)); - std::unique_ptr remover( - EchoRemover::Create(EchoCanceller3Config(), rate)); + EchoCanceller3Config config; + config.delay.min_echo_path_delay_blocks = 0; + std::unique_ptr remover(EchoRemover::Create(config, rate)); std::unique_ptr render_buffer( - RenderDelayBuffer::Create( - NumBandsForRate(rate), kDownSamplingFactor, - GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), - GetRenderDelayBufferSize(kDownSamplingFactor, - kNumMatchedFilters))); + RenderDelayBuffer::Create(config, NumBandsForRate(rate))); + if (delay_samples != render_buffer->Delay() * kBlockSize) { + render_buffer->SetDelay(delay_samples / kBlockSize); + } std::vector>> delay_buffers(x.size()); for (size_t j = 0; j < x.size(); ++j) { delay_buffers[j].reset(new DelayBuffer(delay_samples)); @@ -196,7 +192,7 @@ TEST(EchoRemover, BasicEchoRemoval) { } render_buffer->Insert(x); - render_buffer->UpdateBuffers(); + render_buffer->PrepareCaptureCall(); remover->ProcessCapture(delay_samples, echo_path_variability, false, render_buffer->GetRenderBuffer(), &y); diff --git a/modules/audio_processing/aec3/erl_estimator.cc b/modules/audio_processing/aec3/erl_estimator.cc index 3f12ba41a1..b2849db18a 100644 --- a/modules/audio_processing/aec3/erl_estimator.cc +++ b/modules/audio_processing/aec3/erl_estimator.cc @@ -31,9 +31,10 @@ ErlEstimator::ErlEstimator() { ErlEstimator::~ErlEstimator() = default; -void ErlEstimator::Update( - const std::array& render_spectrum, - const std::array& capture_spectrum) { +void ErlEstimator::Update(rtc::ArrayView render_spectrum, + rtc::ArrayView capture_spectrum) { + RTC_DCHECK_EQ(kFftLengthBy2Plus1, render_spectrum.size()); + RTC_DCHECK_EQ(kFftLengthBy2Plus1, capture_spectrum.size()); const auto& X2 = render_spectrum; const auto& Y2 = capture_spectrum; diff --git a/modules/audio_processing/aec3/erl_estimator.h b/modules/audio_processing/aec3/erl_estimator.h index 24b3f4b104..215c22ea17 100644 --- a/modules/audio_processing/aec3/erl_estimator.h +++ b/modules/audio_processing/aec3/erl_estimator.h @@ -13,6 +13,7 @@ #include +#include "api/array_view.h" #include "modules/audio_processing/aec3/aec3_common.h" #include "rtc_base/constructormagic.h" @@ -25,8 +26,8 @@ class ErlEstimator { ~ErlEstimator(); // Updates the ERL estimate. - void Update(const std::array& render_spectrum, - const std::array& capture_spectrum); + void Update(rtc::ArrayView render_spectrum, + rtc::ArrayView capture_spectrum); // Returns the most recent ERL estimate. const std::array& Erl() const { return erl_; } diff --git a/modules/audio_processing/aec3/erle_estimator.cc b/modules/audio_processing/aec3/erle_estimator.cc index 385e6dd7e6..0e4cbe1469 100644 --- a/modules/audio_processing/aec3/erle_estimator.cc +++ b/modules/audio_processing/aec3/erle_estimator.cc @@ -31,10 +31,12 @@ ErleEstimator::ErleEstimator(float min_erle, ErleEstimator::~ErleEstimator() = default; -void ErleEstimator::Update( - const std::array& render_spectrum, - const std::array& capture_spectrum, - const std::array& subtractor_spectrum) { +void ErleEstimator::Update(rtc::ArrayView render_spectrum, + rtc::ArrayView capture_spectrum, + rtc::ArrayView subtractor_spectrum) { + RTC_DCHECK_EQ(kFftLengthBy2Plus1, render_spectrum.size()); + RTC_DCHECK_EQ(kFftLengthBy2Plus1, capture_spectrum.size()); + RTC_DCHECK_EQ(kFftLengthBy2Plus1, subtractor_spectrum.size()); const auto& X2 = render_spectrum; const auto& Y2 = capture_spectrum; const auto& E2 = subtractor_spectrum; diff --git a/modules/audio_processing/aec3/erle_estimator.h b/modules/audio_processing/aec3/erle_estimator.h index d88b11bbb8..cb9fce6e14 100644 --- a/modules/audio_processing/aec3/erle_estimator.h +++ b/modules/audio_processing/aec3/erle_estimator.h @@ -13,6 +13,7 @@ #include +#include "api/array_view.h" #include "modules/audio_processing/aec3/aec3_common.h" #include "rtc_base/constructormagic.h" @@ -25,9 +26,9 @@ class ErleEstimator { ~ErleEstimator(); // Updates the ERLE estimate. - void Update(const std::array& render_spectrum, - const std::array& capture_spectrum, - const std::array& subtractor_spectrum); + void Update(rtc::ArrayView render_spectrum, + rtc::ArrayView capture_spectrum, + rtc::ArrayView subtractor_spectrum); // Returns the most recent ERLE estimate. const std::array& Erle() const { return erle_; } diff --git a/modules/audio_processing/aec3/fft_buffer.cc b/modules/audio_processing/aec3/fft_buffer.cc new file mode 100644 index 0000000000..133d77bec2 --- /dev/null +++ b/modules/audio_processing/aec3/fft_buffer.cc @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/fft_buffer.h" + +namespace webrtc { + +FftBuffer::FftBuffer(size_t size) : buffer(size) { + for (auto& b : buffer) { + b.Clear(); + } +} + +FftBuffer::~FftBuffer() = default; + +} // namespace webrtc diff --git a/modules/audio_processing/aec3/fft_buffer.h b/modules/audio_processing/aec3/fft_buffer.h new file mode 100644 index 0000000000..42fbb4a623 --- /dev/null +++ b/modules/audio_processing/aec3/fft_buffer.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_FFT_BUFFER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_FFT_BUFFER_H_ + +#include + +#include "modules/audio_processing/aec3/fft_data.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +// Struct for bundling a circular buffer of FftData objects together with the +// read and write indices. +struct FftBuffer { + explicit FftBuffer(size_t size); + ~FftBuffer(); + + size_t IncIndex(size_t index) { + return index < buffer.size() - 1 ? index + 1 : 0; + } + + size_t DecIndex(size_t index) { + return index > 0 ? index - 1 : buffer.size() - 1; + } + + size_t OffsetIndex(size_t index, int offset) { + RTC_DCHECK_GE(buffer.size(), offset); + return (buffer.size() + index + offset) % buffer.size(); + } + + void UpdateWriteIndex(int offset) { write = OffsetIndex(write, offset); } + void IncWriteIndex() { write = IncIndex(write); } + void DecWriteIndex() { write = DecIndex(write); } + void UpdateReadIndex(int offset) { read = OffsetIndex(read, offset); } + void IncReadIndex() { read = IncIndex(read); } + void DecReadIndex() { read = DecIndex(read); } + + std::vector buffer; + size_t write = 0; + size_t read = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_FFT_BUFFER_H_ diff --git a/modules/audio_processing/aec3/fft_data.h b/modules/audio_processing/aec3/fft_data.h index a5c51bf342..59511b56fe 100644 --- a/modules/audio_processing/aec3/fft_data.h +++ b/modules/audio_processing/aec3/fft_data.h @@ -40,8 +40,8 @@ struct FftData { // Computes the power spectrum of the data. void Spectrum(Aec3Optimization optimization, - std::array* power_spectrum) const { - RTC_DCHECK(power_spectrum); + rtc::ArrayView power_spectrum) const { + RTC_DCHECK_EQ(kFftLengthBy2Plus1, power_spectrum.size()); switch (optimization) { #if defined(WEBRTC_ARCH_X86_FAMILY) case Aec3Optimization::kSse2: { @@ -53,16 +53,14 @@ struct FftData { const __m128 ii = _mm_mul_ps(i, i); const __m128 rr = _mm_mul_ps(r, r); const __m128 rrii = _mm_add_ps(rr, ii); - _mm_storeu_ps(&(*power_spectrum)[k], rrii); + _mm_storeu_ps(&power_spectrum[k], rrii); } - (*power_spectrum)[kFftLengthBy2] = - re[kFftLengthBy2] * re[kFftLengthBy2] + - im[kFftLengthBy2] * im[kFftLengthBy2]; + power_spectrum[kFftLengthBy2] = re[kFftLengthBy2] * re[kFftLengthBy2] + + im[kFftLengthBy2] * im[kFftLengthBy2]; } break; #endif default: - std::transform(re.begin(), re.end(), im.begin(), - power_spectrum->begin(), + std::transform(re.begin(), re.end(), im.begin(), power_spectrum.begin(), [](float a, float b) { return a * a + b * b; }); } } diff --git a/modules/audio_processing/aec3/fft_data_unittest.cc b/modules/audio_processing/aec3/fft_data_unittest.cc index d969744c04..8fc5ca7c76 100644 --- a/modules/audio_processing/aec3/fft_data_unittest.cc +++ b/modules/audio_processing/aec3/fft_data_unittest.cc @@ -34,8 +34,8 @@ TEST(FftData, TestOptimizations) { std::array spectrum; std::array spectrum_sse2; - x.Spectrum(Aec3Optimization::kNone, &spectrum); - x.Spectrum(Aec3Optimization::kSse2, &spectrum_sse2); + x.Spectrum(Aec3Optimization::kNone, spectrum); + x.Spectrum(Aec3Optimization::kSse2, spectrum_sse2); EXPECT_EQ(spectrum, spectrum_sse2); } } @@ -102,7 +102,7 @@ TEST(FftData, Spectrum) { } std::array spectrum; - x.Spectrum(Aec3Optimization::kNone, &spectrum); + x.Spectrum(Aec3Optimization::kNone, spectrum); EXPECT_EQ(x.re[0] * x.re[0], spectrum[0]); EXPECT_EQ(x.re[spectrum.size() - 1] * x.re[spectrum.size() - 1], diff --git a/modules/audio_processing/aec3/main_filter_update_gain.cc b/modules/audio_processing/aec3/main_filter_update_gain.cc index 1dd2a20b27..45253cd927 100644 --- a/modules/audio_processing/aec3/main_filter_update_gain.cc +++ b/modules/audio_processing/aec3/main_filter_update_gain.cc @@ -37,7 +37,9 @@ MainFilterUpdateGain::MainFilterUpdateGain() MainFilterUpdateGain::~MainFilterUpdateGain() {} -void MainFilterUpdateGain::HandleEchoPathChange() { +void MainFilterUpdateGain::HandleEchoPathChange( + const EchoPathVariability& echo_path_variability) { + // TODO(peah): Add even-specific behavior. H_error_.fill(kHErrorInitial); poor_excitation_counter_ = kPoorExcitationCounterInitial; call_counter_ = 0; @@ -57,7 +59,7 @@ void MainFilterUpdateGain::Compute( const auto& E2_shadow = subtractor_output.E2_shadow; FftData* G = gain_fft; const size_t size_partitions = filter.SizePartitions(); - const auto& X2 = render_buffer.SpectralSum(size_partitions); + auto X2 = render_buffer.SpectralSum(size_partitions); const auto& erl = filter.Erl(); ++call_counter_; diff --git a/modules/audio_processing/aec3/main_filter_update_gain.h b/modules/audio_processing/aec3/main_filter_update_gain.h index 756a5d0e5d..92ec02afd0 100644 --- a/modules/audio_processing/aec3/main_filter_update_gain.h +++ b/modules/audio_processing/aec3/main_filter_update_gain.h @@ -16,7 +16,7 @@ #include "modules/audio_processing/aec3/adaptive_fir_filter.h" #include "modules/audio_processing/aec3/aec3_common.h" -#include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/aec3/echo_path_variability.h" #include "modules/audio_processing/aec3/render_signal_analyzer.h" #include "modules/audio_processing/aec3/subtractor_output.h" #include "rtc_base/constructormagic.h" @@ -32,7 +32,7 @@ class MainFilterUpdateGain { ~MainFilterUpdateGain(); // Takes action in the case of a known echo path change. - void HandleEchoPathChange(); + void HandleEchoPathChange(const EchoPathVariability& echo_path_variability); // Computes the gain. void Compute(const RenderBuffer& render_buffer, diff --git a/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc b/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc index 203731a929..b59273db42 100644 --- a/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc +++ b/modules/audio_processing/aec3/main_filter_update_gain_unittest.cc @@ -16,7 +16,7 @@ #include "modules/audio_processing/aec3/adaptive_fir_filter.h" #include "modules/audio_processing/aec3/aec_state.h" -#include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/aec3/render_delay_buffer.h" #include "modules/audio_processing/aec3/render_signal_analyzer.h" #include "modules/audio_processing/aec3/shadow_filter_update_gain.h" #include "modules/audio_processing/aec3/subtractor_output.h" @@ -40,12 +40,9 @@ void RunFilterUpdateTest(int num_blocks_to_process, std::array* y_last_block, FftData* G_last_block) { ApmDataDumper data_dumper(42); - AdaptiveFirFilter main_filter(9, DetectOptimization(), &data_dumper); - AdaptiveFirFilter shadow_filter(9, DetectOptimization(), &data_dumper); + AdaptiveFirFilter main_filter(12, DetectOptimization(), &data_dumper); + AdaptiveFirFilter shadow_filter(12, DetectOptimization(), &data_dumper); Aec3Fft fft; - RenderBuffer render_buffer( - Aec3Optimization::kNone, 3, main_filter.SizePartitions(), - std::vector(1, main_filter.SizePartitions())); std::array x_old; x_old.fill(0.f); ShadowFilterUpdateGain shadow_gain; @@ -53,7 +50,11 @@ void RunFilterUpdateTest(int num_blocks_to_process, Random random_generator(42U); std::vector> x(3, std::vector(kBlockSize, 0.f)); std::vector y(kBlockSize, 0.f); - AecState aec_state(EchoCanceller3Config{}); + EchoCanceller3Config config; + config.delay.min_echo_path_delay_blocks = 0; + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, 3)); + AecState aec_state(config); RenderSignalAnalyzer render_signal_analyzer; std::array s_scratch; std::array s; @@ -92,11 +93,18 @@ void RunFilterUpdateTest(int num_blocks_to_process, RandomizeSampleVector(&random_generator, x[0]); } delay_buffer.Delay(x[0], y); - render_buffer.Insert(x); - render_signal_analyzer.Update(render_buffer, aec_state.FilterDelay()); + + render_delay_buffer->Insert(x); + if (k == 0) { + render_delay_buffer->Reset(); + } + render_delay_buffer->PrepareCaptureCall(); + + render_signal_analyzer.Update(render_delay_buffer->GetRenderBuffer(), + aec_state.FilterDelay()); // Apply the main filter. - main_filter.Filter(render_buffer, &S); + main_filter.Filter(render_delay_buffer->GetRenderBuffer(), &S); fft.Ifft(S, &s_scratch); std::transform(y.begin(), y.end(), s_scratch.begin() + kFftLengthBy2, e_main.begin(), @@ -109,7 +117,7 @@ void RunFilterUpdateTest(int num_blocks_to_process, } // Apply the shadow filter. - shadow_filter.Filter(render_buffer, &S); + shadow_filter.Filter(render_delay_buffer->GetRenderBuffer(), &S); fft.Ifft(S, &s_scratch); std::transform(y.begin(), y.end(), s_scratch.begin() + kFftLengthBy2, e_shadow.begin(), @@ -119,24 +127,28 @@ void RunFilterUpdateTest(int num_blocks_to_process, fft.ZeroPaddedFft(e_shadow, &E_shadow); // Compute spectra for future use. - E_main.Spectrum(Aec3Optimization::kNone, &output.E2_main); - E_shadow.Spectrum(Aec3Optimization::kNone, &output.E2_shadow); + E_main.Spectrum(Aec3Optimization::kNone, output.E2_main); + E_shadow.Spectrum(Aec3Optimization::kNone, output.E2_shadow); // Adapt the shadow filter. - shadow_gain.Compute(render_buffer, render_signal_analyzer, E_shadow, + shadow_gain.Compute(render_delay_buffer->GetRenderBuffer(), + render_signal_analyzer, E_shadow, shadow_filter.SizePartitions(), saturation, &G); - shadow_filter.Adapt(render_buffer, G); + shadow_filter.Adapt(render_delay_buffer->GetRenderBuffer(), G); // Adapt the main filter - main_gain.Compute(render_buffer, render_signal_analyzer, output, - main_filter, saturation, &G); - main_filter.Adapt(render_buffer, G); + main_gain.Compute(render_delay_buffer->GetRenderBuffer(), + render_signal_analyzer, output, main_filter, saturation, + &G); + main_filter.Adapt(render_delay_buffer->GetRenderBuffer(), G); // Update the delay. - aec_state.HandleEchoPathChange(EchoPathVariability(false, false)); + aec_state.HandleEchoPathChange(EchoPathVariability( + false, EchoPathVariability::DelayAdjustment::kNone, false)); aec_state.Update(main_filter.FilterFrequencyResponse(), main_filter.FilterImpulseResponse(), true, rtc::nullopt, - render_buffer, E2_main, Y2, x[0], s, false); + render_delay_buffer->GetRenderBuffer(), E2_main, Y2, x[0], + s, false); } std::copy(e_main.begin(), e_main.end(), e_last_block->begin()); @@ -158,16 +170,16 @@ std::string ProduceDebugText(size_t delay) { // Verifies that the check for non-null output gain parameter works. TEST(MainFilterUpdateGain, NullDataOutputGain) { ApmDataDumper data_dumper(42); - AdaptiveFirFilter filter(9, DetectOptimization(), &data_dumper); - RenderBuffer render_buffer(Aec3Optimization::kNone, 3, - filter.SizePartitions(), - std::vector(1, filter.SizePartitions())); + AdaptiveFirFilter filter(kAdaptiveFilterLength, DetectOptimization(), + &data_dumper); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(EchoCanceller3Config(), 3)); RenderSignalAnalyzer analyzer; SubtractorOutput output; MainFilterUpdateGain gain; - EXPECT_DEATH( - gain.Compute(render_buffer, analyzer, output, filter, false, nullptr), - ""); + EXPECT_DEATH(gain.Compute(render_delay_buffer->GetRenderBuffer(), analyzer, + output, filter, false, nullptr), + ""); } #endif @@ -214,9 +226,9 @@ TEST(MainFilterUpdateGain, DecreasingGain) { RunFilterUpdateTest(300, 65, blocks_with_echo_path_changes, blocks_with_saturation, false, &e, &y, &G_c); - G_a.Spectrum(Aec3Optimization::kNone, &G_a_power); - G_b.Spectrum(Aec3Optimization::kNone, &G_b_power); - G_c.Spectrum(Aec3Optimization::kNone, &G_c_power); + G_a.Spectrum(Aec3Optimization::kNone, G_a_power); + G_b.Spectrum(Aec3Optimization::kNone, G_b_power); + G_c.Spectrum(Aec3Optimization::kNone, G_c_power); EXPECT_GT(std::accumulate(G_a_power.begin(), G_a_power.end(), 0.), std::accumulate(G_b_power.begin(), G_b_power.end(), 0.)); @@ -256,8 +268,8 @@ TEST(MainFilterUpdateGain, SaturationBehavior) { RunFilterUpdateTest(201, 65, blocks_with_echo_path_changes, blocks_with_saturation, false, &e, &y, &G_b); - G_a.Spectrum(Aec3Optimization::kNone, &G_a_power); - G_b.Spectrum(Aec3Optimization::kNone, &G_b_power); + G_a.Spectrum(Aec3Optimization::kNone, G_a_power); + G_b.Spectrum(Aec3Optimization::kNone, G_b_power); EXPECT_LT(std::accumulate(G_a_power.begin(), G_a_power.end(), 0.), std::accumulate(G_b_power.begin(), G_b_power.end(), 0.)); @@ -276,13 +288,13 @@ TEST(MainFilterUpdateGain, EchoPathChangeBehavior) { std::array G_a_power; std::array G_b_power; - RunFilterUpdateTest(99, 65, blocks_with_echo_path_changes, - blocks_with_saturation, false, &e, &y, &G_a); RunFilterUpdateTest(100, 65, blocks_with_echo_path_changes, + blocks_with_saturation, false, &e, &y, &G_a); + RunFilterUpdateTest(101, 65, blocks_with_echo_path_changes, blocks_with_saturation, false, &e, &y, &G_b); - G_a.Spectrum(Aec3Optimization::kNone, &G_a_power); - G_b.Spectrum(Aec3Optimization::kNone, &G_b_power); + G_a.Spectrum(Aec3Optimization::kNone, G_a_power); + G_b.Spectrum(Aec3Optimization::kNone, G_b_power); EXPECT_LT(std::accumulate(G_a_power.begin(), G_a_power.end(), 0.), std::accumulate(G_b_power.begin(), G_b_power.end(), 0.)); diff --git a/modules/audio_processing/aec3/matched_filter.cc b/modules/audio_processing/aec3/matched_filter.cc index d5e6a28bf7..52bb3711fd 100644 --- a/modules/audio_processing/aec3/matched_filter.cc +++ b/modules/audio_processing/aec3/matched_filter.cc @@ -338,7 +338,7 @@ void MatchedFilter::Update(const DownsampledRenderBuffer& render_buffer, bool filters_updated = false; size_t x_start_index = - (render_buffer.position + alignment_shift + sub_block_size_ - 1) % + (render_buffer.read + alignment_shift + sub_block_size_ - 1) % render_buffer.buffer.size(); switch (optimization_) { diff --git a/modules/audio_processing/aec3/matched_filter_unittest.cc b/modules/audio_processing/aec3/matched_filter_unittest.cc index 06004190ae..aec68821d9 100644 --- a/modules/audio_processing/aec3/matched_filter_unittest.cc +++ b/modules/audio_processing/aec3/matched_filter_unittest.cc @@ -151,20 +151,24 @@ TEST(MatchedFilter, LagEstimation) { MatchedFilter filter(&data_dumper, DetectOptimization(), sub_block_size, kWindowSizeSubBlocks, kNumMatchedFilters, kAlignmentShiftSubBlocks, 150); + EchoCanceller3Config config; + config.delay.down_sampling_factor = down_sampling_factor; + config.delay.num_filters = kNumMatchedFilters; + config.delay.min_echo_path_delay_blocks = 0; + std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create( - 3, down_sampling_factor, - GetDownSampledBufferSize(down_sampling_factor, - kNumMatchedFilters), - GetRenderDelayBufferSize(down_sampling_factor, - kNumMatchedFilters))); + RenderDelayBuffer::Create(config, 3)); // Analyze the correlation between render and capture. - for (size_t k = 0; k < (300 + delay_samples / sub_block_size); ++k) { + for (size_t k = 0; k < (600 + delay_samples / sub_block_size); ++k) { RandomizeSampleVector(&random_generator, render[0]); signal_delay_buffer.Delay(render[0], capture); render_delay_buffer->Insert(render); - render_delay_buffer->UpdateBuffers(); + if (k == 0) { + render_delay_buffer->Reset(); + } + + render_delay_buffer->PrepareCaptureCall(); std::array downsampled_capture_data; rtc::ArrayView downsampled_capture( downsampled_capture_data.data(), sub_block_size); @@ -179,7 +183,7 @@ TEST(MatchedFilter, LagEstimation) { // Find which lag estimate should be the most accurate. rtc::Optional expected_most_accurate_lag_estimate; size_t alignment_shift_sub_blocks = 0; - for (size_t k = 0; k < kNumMatchedFilters; ++k) { + for (size_t k = 0; k < config.delay.num_filters; ++k) { if ((alignment_shift_sub_blocks + 3 * kWindowSizeSubBlocks / 4) * sub_block_size > delay_samples) { @@ -236,6 +240,9 @@ TEST(MatchedFilter, LagEstimation) { TEST(MatchedFilter, LagNotReliableForUncorrelatedRenderAndCapture) { Random random_generator(42U); for (auto down_sampling_factor : kDownSamplingFactors) { + EchoCanceller3Config config; + config.delay.down_sampling_factor = down_sampling_factor; + config.delay.num_filters = kNumMatchedFilters; const size_t sub_block_size = kBlockSize / down_sampling_factor; std::vector> render(3, @@ -245,11 +252,7 @@ TEST(MatchedFilter, LagNotReliableForUncorrelatedRenderAndCapture) { std::fill(capture.begin(), capture.end(), 0.f); ApmDataDumper data_dumper(0); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create( - 3, down_sampling_factor, - GetDownSampledBufferSize(down_sampling_factor, kNumMatchedFilters), - GetRenderDelayBufferSize(down_sampling_factor, - kNumMatchedFilters))); + RenderDelayBuffer::Create(config, 3)); MatchedFilter filter(&data_dumper, DetectOptimization(), sub_block_size, kWindowSizeSubBlocks, kNumMatchedFilters, kAlignmentShiftSubBlocks, 150); @@ -289,11 +292,7 @@ TEST(MatchedFilter, LagNotUpdatedForLowLevelRender) { kWindowSizeSubBlocks, kNumMatchedFilters, kAlignmentShiftSubBlocks, 150); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create( - 3, down_sampling_factor, - GetDownSampledBufferSize(down_sampling_factor, kNumMatchedFilters), - GetRenderDelayBufferSize(down_sampling_factor, - kNumMatchedFilters))); + RenderDelayBuffer::Create(EchoCanceller3Config(), 3)); Decimator capture_decimator(down_sampling_factor); // Analyze the correlation between render and capture. diff --git a/modules/audio_processing/aec3/matrix_buffer.cc b/modules/audio_processing/aec3/matrix_buffer.cc new file mode 100644 index 0000000000..23a5ad4d64 --- /dev/null +++ b/modules/audio_processing/aec3/matrix_buffer.cc @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/matrix_buffer.h" + +#include "modules/audio_processing/aec3/aec3_common.h" + +namespace webrtc { + +MatrixBuffer::MatrixBuffer(size_t size, size_t height, size_t width) + : size(size), + buffer(size, + std::vector>(height, + std::vector(width, 0.f))) { + for (auto& c : buffer) { + for (auto& b : c) { + std::fill(b.begin(), b.end(), 0.f); + } + } +} + +MatrixBuffer::~MatrixBuffer() = default; + +} // namespace webrtc diff --git a/modules/audio_processing/aec3/matrix_buffer.h b/modules/audio_processing/aec3/matrix_buffer.h new file mode 100644 index 0000000000..67980bab0c --- /dev/null +++ b/modules/audio_processing/aec3/matrix_buffer.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_MATRIX_BUFFER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_MATRIX_BUFFER_H_ + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +// Struct for bundling a circular buffer of two dimensional vector objects +// together with the read and write indices. +struct MatrixBuffer { + MatrixBuffer(size_t size, size_t height, size_t width); + ~MatrixBuffer(); + + size_t IncIndex(size_t index) { + return index < buffer.size() - 1 ? index + 1 : 0; + } + + size_t DecIndex(size_t index) { + return index > 0 ? index - 1 : buffer.size() - 1; + } + + size_t OffsetIndex(size_t index, int offset) { + RTC_DCHECK_GE(buffer.size(), offset); + return (buffer.size() + index + offset) % buffer.size(); + } + + void UpdateWriteIndex(int offset) { write = OffsetIndex(write, offset); } + void IncWriteIndex() { write = IncIndex(write); } + void DecWriteIndex() { write = DecIndex(write); } + void UpdateReadIndex(int offset) { read = OffsetIndex(read, offset); } + void IncReadIndex() { read = IncIndex(read); } + void DecReadIndex() { read = DecIndex(read); } + + size_t size; + std::vector>> buffer; + size_t write = 0; + size_t read = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_MATRIX_BUFFER_H_ diff --git a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h index 6b5870901d..02ac6316ca 100644 --- a/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h +++ b/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h @@ -25,10 +25,15 @@ namespace test { class MockRenderDelayBuffer : public RenderDelayBuffer { public: explicit MockRenderDelayBuffer(int sample_rate_hz) - : render_buffer_(Aec3Optimization::kNone, - NumBandsForRate(sample_rate_hz), - GetRenderDelayBufferSize(4, 4), - std::vector(1, kAdaptiveFilterLength)), + : block_buffer_(GetRenderDelayBufferSize(4, 4), + NumBandsForRate(sample_rate_hz), + kBlockSize), + spectrum_buffer_(block_buffer_.buffer.size(), kFftLengthBy2Plus1), + fft_buffer_(block_buffer_.buffer.size()), + render_buffer_(kAdaptiveFilterLength, + &block_buffer_, + &spectrum_buffer_, + &fft_buffer_), downsampled_render_buffer_(GetDownSampledBufferSize(4, 4)) { ON_CALL(*this, GetRenderBuffer()) .WillByDefault( @@ -40,11 +45,14 @@ class MockRenderDelayBuffer : public RenderDelayBuffer { virtual ~MockRenderDelayBuffer() = default; MOCK_METHOD0(Reset, void()); - MOCK_METHOD1(Insert, bool(const std::vector>& block)); - MOCK_METHOD0(UpdateBuffers, bool()); + MOCK_METHOD1(Insert, + RenderDelayBuffer::BufferingEvent( + const std::vector>& block)); + MOCK_METHOD0(PrepareCaptureCall, RenderDelayBuffer::BufferingEvent()); MOCK_METHOD1(SetDelay, void(size_t delay)); MOCK_CONST_METHOD0(Delay, size_t()); MOCK_CONST_METHOD0(MaxDelay, size_t()); + MOCK_CONST_METHOD0(MaxApiJitter, size_t()); MOCK_CONST_METHOD0(IsBlockAvailable, bool()); MOCK_CONST_METHOD0(GetRenderBuffer, const RenderBuffer&()); MOCK_CONST_METHOD0(GetDownsampledRenderBuffer, @@ -55,6 +63,9 @@ class MockRenderDelayBuffer : public RenderDelayBuffer { const DownsampledRenderBuffer& FakeGetDownsampledRenderBuffer() const { return downsampled_render_buffer_; } + MatrixBuffer block_buffer_; + VectorBuffer spectrum_buffer_; + FftBuffer fft_buffer_; RenderBuffer render_buffer_; DownsampledRenderBuffer downsampled_render_buffer_; }; diff --git a/modules/audio_processing/aec3/render_buffer.cc b/modules/audio_processing/aec3/render_buffer.cc index fa86ea6b36..60b50cea01 100644 --- a/modules/audio_processing/aec3/render_buffer.cc +++ b/modules/audio_processing/aec3/render_buffer.cc @@ -17,21 +17,18 @@ namespace webrtc { -RenderBuffer::RenderBuffer(Aec3Optimization optimization, - size_t num_bands, - size_t num_partitions, - const std::vector num_ffts_for_spectral_sums) - : optimization_(optimization), - fft_buffer_(num_partitions), - spectrum_buffer_(num_partitions, std::array()), - spectral_sums_(num_ffts_for_spectral_sums.size(), - std::array()), - last_block_(num_bands, std::vector(kBlockSize, 0.f)), - fft_() { - // Current implementation only allows a maximum of one spectral sum lengths. - RTC_DCHECK_EQ(1, num_ffts_for_spectral_sums.size()); - spectral_sums_length_ = num_ffts_for_spectral_sums[0]; - RTC_DCHECK_GE(fft_buffer_.size(), spectral_sums_length_); +RenderBuffer::RenderBuffer(size_t num_ffts_for_spectral_sums, + MatrixBuffer* block_buffer, + VectorBuffer* spectrum_buffer, + FftBuffer* fft_buffer) + : block_buffer_(block_buffer), + spectrum_buffer_(spectrum_buffer), + fft_buffer_(fft_buffer), + spectral_sums_length_(num_ffts_for_spectral_sums) { + RTC_DCHECK(block_buffer_); + RTC_DCHECK(spectrum_buffer_); + RTC_DCHECK(fft_buffer_); + RTC_DCHECK_GE(fft_buffer_->buffer.size(), spectral_sums_length_); Clear(); } @@ -39,56 +36,17 @@ RenderBuffer::RenderBuffer(Aec3Optimization optimization, RenderBuffer::~RenderBuffer() = default; void RenderBuffer::Clear() { - position_ = 0; - for (auto& sum : spectral_sums_) { - sum.fill(0.f); - } - - for (auto& spectrum : spectrum_buffer_) { - spectrum.fill(0.f); - } - - for (auto& fft : fft_buffer_) { - fft.Clear(); - } - - for (auto& b : last_block_) { - std::fill(b.begin(), b.end(), 0.f); - } + spectral_sums_.fill(0.f); } -void RenderBuffer::Insert(const std::vector>& block) { - // Compute the FFT of the data in the lowest band. - FftData X; - fft_.PaddedFft(block[0], last_block_[0], &X); - - // Copy the last render frame. - RTC_DCHECK_EQ(last_block_.size(), block.size()); - for (size_t k = 0; k < block.size(); ++k) { - RTC_DCHECK_EQ(last_block_[k].size(), block[k].size()); - std::copy(block[k].begin(), block[k].end(), last_block_[k].begin()); - } - - // Insert X into the buffer. - position_ = position_ > 0 ? position_ - 1 : fft_buffer_.size() - 1; - fft_buffer_[position_].Assign(X); - - // Compute and insert the spectrum for the FFT into the spectrum buffer. - X.Spectrum(optimization_, &spectrum_buffer_[position_]); - - // Pre-compute and cache the spectral sums. - std::copy(spectrum_buffer_[position_].begin(), - spectrum_buffer_[position_].end(), spectral_sums_[0].begin()); - size_t position = (position_ + 1) % fft_buffer_.size(); - for (size_t j = 1; j < spectral_sums_length_; ++j) { - const std::array& spectrum = - spectrum_buffer_[position]; - - for (size_t k = 0; k < spectral_sums_[0].size(); ++k) { - spectral_sums_[0][k] += spectrum[k]; +void RenderBuffer::UpdateSpectralSum() { + std::fill(spectral_sums_.begin(), spectral_sums_.end(), 0.f); + size_t position = spectrum_buffer_->read; + for (size_t j = 0; j < spectral_sums_length_; ++j) { + for (size_t k = 0; k < spectral_sums_.size(); ++k) { + spectral_sums_[k] += spectrum_buffer_->buffer[position][k]; } - - position = position < (fft_buffer_.size() - 1) ? position + 1 : 0; + position = spectrum_buffer_->IncIndex(position); } } diff --git a/modules/audio_processing/aec3/render_buffer.h b/modules/audio_processing/aec3/render_buffer.h index 3288ff36ad..aa132b872d 100644 --- a/modules/audio_processing/aec3/render_buffer.h +++ b/modules/audio_processing/aec3/render_buffer.h @@ -11,12 +11,14 @@ #ifndef MODULES_AUDIO_PROCESSING_AEC3_RENDER_BUFFER_H_ #define MODULES_AUDIO_PROCESSING_AEC3_RENDER_BUFFER_H_ +#include #include -#include #include "api/array_view.h" -#include "modules/audio_processing/aec3/aec3_fft.h" +#include "modules/audio_processing/aec3/fft_buffer.h" #include "modules/audio_processing/aec3/fft_data.h" +#include "modules/audio_processing/aec3/matrix_buffer.h" +#include "modules/audio_processing/aec3/vector_buffer.h" #include "rtc_base/constructormagic.h" namespace webrtc { @@ -24,55 +26,48 @@ namespace webrtc { // Provides a buffer of the render data for the echo remover. class RenderBuffer { public: - // The constructor takes, besides from the other parameters, a vector - // containing the number of FFTs that will be included in the spectral sums in - // the call to SpectralSum. - RenderBuffer(Aec3Optimization optimization, - size_t num_bands, - size_t size, - const std::vector num_ffts_for_spectral_sums); + RenderBuffer(size_t num_ffts_for_spectral_sums, + MatrixBuffer* block_buffer, + VectorBuffer* spectrum_buffer, + FftBuffer* fft_buffer); ~RenderBuffer(); // Clears the buffer. void Clear(); // Insert a block into the buffer. - void Insert(const std::vector>& block); + void UpdateSpectralSum(); // Gets the last inserted block. const std::vector>& MostRecentBlock() const { - return last_block_; + return block_buffer_->buffer[block_buffer_->read]; } - // Get the spectrum from one of the FFTs in the buffer - const std::array& Spectrum( - size_t buffer_offset_ffts) const { - return spectrum_buffer_[(position_ + buffer_offset_ffts) % - fft_buffer_.size()]; + // Get the spectrum from one of the FFTs in the buffer. + rtc::ArrayView Spectrum(size_t buffer_offset_ffts) const { + size_t position = spectrum_buffer_->OffsetIndex(spectrum_buffer_->read, + buffer_offset_ffts); + return spectrum_buffer_->buffer[position]; } // Returns the sum of the spectrums for a certain number of FFTs. - const std::array& SpectralSum( - size_t num_ffts) const { + rtc::ArrayView SpectralSum(size_t num_ffts) const { RTC_DCHECK_EQ(spectral_sums_length_, num_ffts); - return spectral_sums_[0]; + return spectral_sums_; } // Returns the circular buffer. - rtc::ArrayView Buffer() const { return fft_buffer_; } + rtc::ArrayView Buffer() const { return fft_buffer_->buffer; } - // Returns the current position in the circular buffer - size_t Position() const { return position_; } + // Returns the current position in the circular buffer. + size_t Position() const { return fft_buffer_->read; } private: - const Aec3Optimization optimization_; - std::vector fft_buffer_; - std::vector> spectrum_buffer_; - size_t spectral_sums_length_; - std::vector> spectral_sums_; - size_t position_ = 0; - std::vector> last_block_; - const Aec3Fft fft_; + const MatrixBuffer* const block_buffer_; + VectorBuffer* spectrum_buffer_; + const FftBuffer* const fft_buffer_; + const size_t spectral_sums_length_; + std::array spectral_sums_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderBuffer); }; diff --git a/modules/audio_processing/aec3/render_buffer_unittest.cc b/modules/audio_processing/aec3/render_buffer_unittest.cc index 1498f4ea25..989cb277e9 100644 --- a/modules/audio_processing/aec3/render_buffer_unittest.cc +++ b/modules/audio_processing/aec3/render_buffer_unittest.cc @@ -20,25 +20,25 @@ namespace webrtc { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) -// Verifies the check for the provided numbers of Ffts to include in the -// spectral sum. -TEST(RenderBuffer, TooLargeNumberOfSpectralSums) { - EXPECT_DEATH( - RenderBuffer(Aec3Optimization::kNone, 3, 1, std::vector(2, 1)), - ""); +// Verifies the check for non-null fft buffer. +TEST(RenderBuffer, NullExternalFftBuffer) { + MatrixBuffer block_buffer(10, 3, kBlockSize); + VectorBuffer spectrum_buffer(10, kFftLengthBy2Plus1); + EXPECT_DEATH(RenderBuffer(1, &block_buffer, &spectrum_buffer, nullptr), ""); } -TEST(RenderBuffer, TooSmallNumberOfSpectralSums) { - EXPECT_DEATH( - RenderBuffer(Aec3Optimization::kNone, 3, 1, std::vector()), ""); +// Verifies the check for non-null spectrum buffer. +TEST(RenderBuffer, NullExternalSpectrumBuffer) { + FftBuffer fft_buffer(10); + MatrixBuffer block_buffer(10, 3, kBlockSize); + EXPECT_DEATH(RenderBuffer(1, &block_buffer, nullptr, &fft_buffer), ""); } -// Verifies the feasibility check for the provided number of Ffts to include in -// the spectral. -TEST(RenderBuffer, FeasibleNumberOfFftsInSum) { - EXPECT_DEATH( - RenderBuffer(Aec3Optimization::kNone, 3, 1, std::vector(1, 2)), - ""); +// Verifies the check for non-null block buffer. +TEST(RenderBuffer, NullExternalBlockBuffer) { + FftBuffer fft_buffer(10); + VectorBuffer spectrum_buffer(10, kFftLengthBy2Plus1); + EXPECT_DEATH(RenderBuffer(1, nullptr, &spectrum_buffer, &fft_buffer), ""); } #endif diff --git a/modules/audio_processing/aec3/render_delay_buffer.cc b/modules/audio_processing/aec3/render_delay_buffer.cc index d2ead63b02..9640131885 100644 --- a/modules/audio_processing/aec3/render_delay_buffer.cc +++ b/modules/audio_processing/aec3/render_delay_buffer.cc @@ -14,9 +14,12 @@ #include #include "modules/audio_processing/aec3/aec3_common.h" +#include "modules/audio_processing/aec3/aec3_fft.h" #include "modules/audio_processing/aec3/block_processor.h" #include "modules/audio_processing/aec3/decimator.h" +#include "modules/audio_processing/aec3/fft_buffer.h" #include "modules/audio_processing/aec3/fft_data.h" +#include "modules/audio_processing/aec3/matrix_buffer.h" #include "rtc_base/atomicops.h" #include "rtc_base/checks.h" #include "rtc_base/constructormagic.h" @@ -25,191 +28,161 @@ namespace webrtc { namespace { -class ApiCallJitterBuffer { - public: - explicit ApiCallJitterBuffer(size_t num_bands) { - buffer_.fill(std::vector>( - num_bands, std::vector(kBlockSize, 0.f))); - } - - ~ApiCallJitterBuffer() = default; - - void Reset() { - size_ = 0; - last_insert_index_ = 0; - } - - void Insert(const std::vector>& block) { - RTC_DCHECK_LT(size_, buffer_.size()); - last_insert_index_ = (last_insert_index_ + 1) % buffer_.size(); - RTC_DCHECK_EQ(buffer_[last_insert_index_].size(), block.size()); - RTC_DCHECK_EQ(buffer_[last_insert_index_][0].size(), block[0].size()); - for (size_t k = 0; k < block.size(); ++k) { - std::copy(block[k].begin(), block[k].end(), - buffer_[last_insert_index_][k].begin()); - } - ++size_; - } - - void Remove(std::vector>* block) { - RTC_DCHECK_LT(0, size_); - --size_; - const size_t extract_index = - (last_insert_index_ - size_ + buffer_.size()) % buffer_.size(); - for (size_t k = 0; k < block->size(); ++k) { - std::copy(buffer_[extract_index][k].begin(), - buffer_[extract_index][k].end(), (*block)[k].begin()); - } - } - - size_t Size() const { return size_; } - bool Full() const { return size_ >= (buffer_.size()); } - bool Empty() const { return size_ == 0; } - - private: - std::array>, kMaxApiCallsJitterBlocks> buffer_; - size_t size_ = 0; - int last_insert_index_ = 0; -}; +constexpr int kBufferHeadroom = kAdaptiveFilterLength; class RenderDelayBufferImpl final : public RenderDelayBuffer { public: - RenderDelayBufferImpl(size_t num_bands, - size_t down_sampling_factor, - size_t downsampled_render_buffer_size, - size_t render_delay_buffer_size); + RenderDelayBufferImpl(const EchoCanceller3Config& config, size_t num_bands); ~RenderDelayBufferImpl() override; void Reset() override; - bool Insert(const std::vector>& block) override; - bool UpdateBuffers() override; + BufferingEvent Insert(const std::vector>& block) override; + BufferingEvent PrepareCaptureCall() override; void SetDelay(size_t delay) override; size_t Delay() const override { return delay_; } - - const RenderBuffer& GetRenderBuffer() const override { return fft_buffer_; } + size_t MaxDelay() const override { + return blocks_.buffer.size() - 1 - kBufferHeadroom; + } + size_t MaxApiJitter() const override { return max_api_jitter_; } + const RenderBuffer& GetRenderBuffer() const override { + return echo_remover_buffer_; + } const DownsampledRenderBuffer& GetDownsampledRenderBuffer() const override { - return downsampled_render_buffer_; + return low_rate_; } private: static int instance_count_; std::unique_ptr data_dumper_; const Aec3Optimization optimization_; - const size_t down_sampling_factor_; - const size_t sub_block_size_; - std::vector>> buffer_; - size_t delay_ = 0; - size_t last_insert_index_ = 0; - RenderBuffer fft_buffer_; - DownsampledRenderBuffer downsampled_render_buffer_; + const size_t api_call_jitter_blocks_; + const size_t min_echo_path_delay_blocks_; + const int sub_block_size_; + MatrixBuffer blocks_; + VectorBuffer spectra_; + FftBuffer ffts_; + size_t delay_; + int max_api_jitter_ = 0; + int render_surplus_ = 0; + bool first_reset_occurred_ = false; + RenderBuffer echo_remover_buffer_; + DownsampledRenderBuffer low_rate_; Decimator render_decimator_; - ApiCallJitterBuffer api_call_jitter_buffer_; const std::vector> zero_block_; + const Aec3Fft fft_; + size_t capture_call_counter_ = 0; + std::vector render_ds_; + int render_calls_in_a_row_ = 0; + + void UpdateBuffersWithLatestBlock(size_t previous_write); + void IncreaseRead(); + void IncreaseInsert(); + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayBufferImpl); }; int RenderDelayBufferImpl::instance_count_ = 0; -RenderDelayBufferImpl::RenderDelayBufferImpl( - size_t num_bands, - size_t down_sampling_factor, - size_t downsampled_render_buffer_size, - size_t render_delay_buffer_size) +RenderDelayBufferImpl::RenderDelayBufferImpl(const EchoCanceller3Config& config, + size_t num_bands) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), optimization_(DetectOptimization()), - down_sampling_factor_(down_sampling_factor), - sub_block_size_(down_sampling_factor_ > 0 - ? kBlockSize / down_sampling_factor - : kBlockSize), - buffer_( - render_delay_buffer_size, - std::vector>(num_bands, - std::vector(kBlockSize, 0.f))), - fft_buffer_( - optimization_, - num_bands, - std::max(kUnknownDelayRenderWindowSize, kAdaptiveFilterLength), - std::vector(1, kAdaptiveFilterLength)), - downsampled_render_buffer_(downsampled_render_buffer_size), - render_decimator_(down_sampling_factor_), - api_call_jitter_buffer_(num_bands), - zero_block_(num_bands, std::vector(kBlockSize, 0.f)) { - RTC_DCHECK_LT(buffer_.size(), downsampled_render_buffer_.buffer.size()); + api_call_jitter_blocks_(config.delay.api_call_jitter_blocks), + min_echo_path_delay_blocks_(config.delay.min_echo_path_delay_blocks), + sub_block_size_( + static_cast(config.delay.down_sampling_factor > 0 + ? kBlockSize / config.delay.down_sampling_factor + : kBlockSize)), + blocks_(GetRenderDelayBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters), + num_bands, + kBlockSize), + spectra_(blocks_.buffer.size(), kFftLengthBy2Plus1), + ffts_(blocks_.buffer.size()), + delay_(min_echo_path_delay_blocks_), + echo_remover_buffer_(kAdaptiveFilterLength, &blocks_, &spectra_, &ffts_), + low_rate_(GetDownSampledBufferSize(config.delay.down_sampling_factor, + config.delay.num_filters)), + render_decimator_(config.delay.down_sampling_factor), + zero_block_(num_bands, std::vector(kBlockSize, 0.f)), + fft_(), + render_ds_(sub_block_size_, 0.f) { + RTC_DCHECK_EQ(blocks_.buffer.size(), ffts_.buffer.size()); + RTC_DCHECK_EQ(spectra_.buffer.size(), ffts_.buffer.size()); + Reset(); + first_reset_occurred_ = false; } RenderDelayBufferImpl::~RenderDelayBufferImpl() = default; void RenderDelayBufferImpl::Reset() { - // Empty all data in the buffers. - delay_ = 0; - last_insert_index_ = 0; - downsampled_render_buffer_.position = 0; - std::fill(downsampled_render_buffer_.buffer.begin(), - downsampled_render_buffer_.buffer.end(), 0.f); - fft_buffer_.Clear(); - api_call_jitter_buffer_.Reset(); - for (auto& c : buffer_) { - for (auto& b : c) { - std::fill(b.begin(), b.end(), 0.f); - } - } + delay_ = min_echo_path_delay_blocks_; + const int offset1 = std::max( + std::min(api_call_jitter_blocks_, min_echo_path_delay_blocks_), 1); + const int offset2 = static_cast(delay_ + offset1); + const int offset3 = offset1 * sub_block_size_; + low_rate_.read = low_rate_.OffsetIndex(low_rate_.write, offset3); + blocks_.read = blocks_.OffsetIndex(blocks_.write, -offset2); + spectra_.read = spectra_.OffsetIndex(spectra_.write, offset2); + ffts_.read = ffts_.OffsetIndex(ffts_.write, offset2); + render_surplus_ = 0; + first_reset_occurred_ = true; } -bool RenderDelayBufferImpl::Insert( +RenderDelayBuffer::BufferingEvent RenderDelayBufferImpl::Insert( const std::vector>& block) { - RTC_DCHECK_EQ(block.size(), buffer_[0].size()); - RTC_DCHECK_EQ(block[0].size(), buffer_[0][0].size()); + RTC_DCHECK_EQ(block.size(), blocks_.buffer[0].size()); + RTC_DCHECK_EQ(block[0].size(), blocks_.buffer[0][0].size()); + BufferingEvent event = BufferingEvent::kNone; - if (api_call_jitter_buffer_.Full()) { - // Report buffer overrun and let the caller handle the overrun. - return false; + ++render_surplus_; + if (first_reset_occurred_) { + ++render_calls_in_a_row_; + max_api_jitter_ = std::max(max_api_jitter_, render_calls_in_a_row_); } - api_call_jitter_buffer_.Insert(block); - return true; + const size_t previous_write = blocks_.write; + IncreaseInsert(); + + if (low_rate_.read == low_rate_.write || blocks_.read == blocks_.write) { + // Render overrun due to more render data being inserted than read. Discard + // the oldest render data. + event = BufferingEvent::kRenderOverrun; + IncreaseRead(); + } + + for (size_t k = 0; k < block.size(); ++k) { + std::copy(block[k].begin(), block[k].end(), + blocks_.buffer[blocks_.write][k].begin()); + } + + UpdateBuffersWithLatestBlock(previous_write); + return event; } -bool RenderDelayBufferImpl::UpdateBuffers() { - bool underrun = true; - // Update the buffers with a new block if such is available, otherwise insert - // a block of silence. - if (api_call_jitter_buffer_.Size() > 0) { - last_insert_index_ = (last_insert_index_ + 1) % buffer_.size(); - api_call_jitter_buffer_.Remove(&buffer_[last_insert_index_]); - underrun = false; - } +RenderDelayBuffer::BufferingEvent RenderDelayBufferImpl::PrepareCaptureCall() { + BufferingEvent event = BufferingEvent::kNone; + render_calls_in_a_row_ = 0; - downsampled_render_buffer_.position = - (downsampled_render_buffer_.position - sub_block_size_ + - downsampled_render_buffer_.buffer.size()) % - downsampled_render_buffer_.buffer.size(); - - rtc::ArrayView input( - underrun ? zero_block_[0].data() : buffer_[last_insert_index_][0].data(), - kBlockSize); - rtc::ArrayView output(downsampled_render_buffer_.buffer.data() + - downsampled_render_buffer_.position, - sub_block_size_); - data_dumper_->DumpWav("aec3_render_decimator_input", input.size(), - input.data(), 16000, 1); - render_decimator_.Decimate(input, output); - data_dumper_->DumpWav("aec3_render_decimator_output", output.size(), - output.data(), 16000 / down_sampling_factor_, 1); - for (size_t k = 0; k < output.size() / 2; ++k) { - float tmp = output[k]; - output[k] = output[output.size() - 1 - k]; - output[output.size() - 1 - k] = tmp; - } - - if (underrun) { - fft_buffer_.Insert(zero_block_); + if (low_rate_.read == low_rate_.write || blocks_.read == blocks_.write) { + event = BufferingEvent::kRenderUnderrun; } else { - fft_buffer_.Insert(buffer_[(last_insert_index_ - delay_ + buffer_.size()) % - buffer_.size()]); + IncreaseRead(); } - return !underrun; + --render_surplus_; + + echo_remover_buffer_.UpdateSpectralSum(); + + if (render_surplus_ >= static_cast(api_call_jitter_blocks_)) { + event = BufferingEvent::kApiCallSkew; + RTC_LOG(LS_WARNING) << "Api call skew detected at " << capture_call_counter_ + << "."; + } + + ++capture_call_counter_; + return event; } void RenderDelayBufferImpl::SetDelay(size_t delay) { @@ -217,37 +190,51 @@ void RenderDelayBufferImpl::SetDelay(size_t delay) { return; } - // If there is a new delay set, clear the fft buffer. - fft_buffer_.Clear(); - - if ((buffer_.size() - 1) < delay) { - // If the desired delay is larger than the delay buffer, shorten the delay - // buffer size to achieve the desired alignment with the available buffer - // size. - downsampled_render_buffer_.position = - (downsampled_render_buffer_.position + - sub_block_size_ * (delay - (buffer_.size() - 1))) % - downsampled_render_buffer_.buffer.size(); - - last_insert_index_ = - (last_insert_index_ - (delay - (buffer_.size() - 1)) + buffer_.size()) % - buffer_.size(); - delay_ = buffer_.size() - 1; - } else { - delay_ = delay; + const int delta_delay = static_cast(delay_) - static_cast(delay); + delay_ = delay; + if (delay_ > MaxDelay()) { + delay_ = std::min(MaxDelay(), delay); + RTC_NOTREACHED(); } + + // Recompute the read indices according to the set delay. + blocks_.UpdateReadIndex(delta_delay); + spectra_.UpdateReadIndex(-delta_delay); + ffts_.UpdateReadIndex(-delta_delay); } +void RenderDelayBufferImpl::UpdateBuffersWithLatestBlock( + size_t previous_write) { + render_decimator_.Decimate(blocks_.buffer[blocks_.write][0], render_ds_); + std::copy(render_ds_.rbegin(), render_ds_.rend(), + low_rate_.buffer.begin() + low_rate_.write); + + fft_.PaddedFft(blocks_.buffer[blocks_.write][0], + blocks_.buffer[previous_write][0], &ffts_.buffer[ffts_.write]); + + ffts_.buffer[ffts_.write].Spectrum(optimization_, + spectra_.buffer[spectra_.write]); +}; + +void RenderDelayBufferImpl::IncreaseRead() { + low_rate_.UpdateReadIndex(-sub_block_size_); + blocks_.IncReadIndex(); + spectra_.DecReadIndex(); + ffts_.DecReadIndex(); +}; + +void RenderDelayBufferImpl::IncreaseInsert() { + low_rate_.UpdateWriteIndex(-sub_block_size_); + blocks_.IncWriteIndex(); + spectra_.DecWriteIndex(); + ffts_.DecWriteIndex(); +}; + } // namespace -RenderDelayBuffer* RenderDelayBuffer::Create( - size_t num_bands, - size_t down_sampling_factor, - size_t downsampled_render_buffer_size, - size_t render_delay_buffer_size) { - return new RenderDelayBufferImpl(num_bands, down_sampling_factor, - downsampled_render_buffer_size, - render_delay_buffer_size); +RenderDelayBuffer* RenderDelayBuffer::Create(const EchoCanceller3Config& config, + size_t num_bands) { + return new RenderDelayBufferImpl(config, num_bands); } } // namespace webrtc diff --git a/modules/audio_processing/aec3/render_delay_buffer.h b/modules/audio_processing/aec3/render_delay_buffer.h index 8f5de40752..0261e841c8 100644 --- a/modules/audio_processing/aec3/render_delay_buffer.h +++ b/modules/audio_processing/aec3/render_delay_buffer.h @@ -20,6 +20,7 @@ #include "modules/audio_processing/aec3/downsampled_render_buffer.h" #include "modules/audio_processing/aec3/fft_data.h" #include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/include/audio_processing.h" namespace webrtc { @@ -27,22 +28,28 @@ namespace webrtc { // extracted with a specified delay. class RenderDelayBuffer { public: - static RenderDelayBuffer* Create(size_t num_bands, - size_t down_sampling_factor, - size_t downsampled_render_buffer_size, - size_t render_delay_buffer_size); + enum class BufferingEvent { + kNone, + kRenderUnderrun, + kRenderOverrun, + kApiCallSkew, + kRenderDataLost + }; + + static RenderDelayBuffer* Create(const EchoCanceller3Config& config, + size_t num_bands); virtual ~RenderDelayBuffer() = default; - // Resets the buffer data. + // Resets the buffer alignment. virtual void Reset() = 0; - // Inserts a block into the buffer and returns true if the insert is - // successful. - virtual bool Insert(const std::vector>& block) = 0; + // Inserts a block into the buffer. + virtual BufferingEvent Insert( + const std::vector>& block) = 0; // Updates the buffers one step based on the specified buffer delay. Returns - // true if there was no overrun, otherwise returns false. - virtual bool UpdateBuffers() = 0; + // an enum indicating whether there was a special event that occurred. + virtual BufferingEvent PrepareCaptureCall() = 0; // Sets the buffer delay. virtual void SetDelay(size_t delay) = 0; @@ -50,6 +57,12 @@ class RenderDelayBuffer { // Gets the buffer delay. virtual size_t Delay() const = 0; + // Gets the buffer delay. + virtual size_t MaxDelay() const = 0; + + // Gets the observed jitter in the render and capture call sequence. + virtual size_t MaxApiJitter() const = 0; + // Returns the render buffer for the echo remover. virtual const RenderBuffer& GetRenderBuffer() const = 0; diff --git a/modules/audio_processing/aec3/render_delay_buffer_unittest.cc b/modules/audio_processing/aec3/render_delay_buffer_unittest.cc index 3e0abea753..9b99a8e9b3 100644 --- a/modules/audio_processing/aec3/render_delay_buffer_unittest.cc +++ b/modules/audio_processing/aec3/render_delay_buffer_unittest.cc @@ -30,49 +30,50 @@ std::string ProduceDebugText(int sample_rate_hz) { return ss.str(); } -constexpr size_t kDownSamplingFactor = 4; -constexpr size_t kNumMatchedFilters = 4; - } // namespace // Verifies that the buffer overflow is correctly reported. TEST(RenderDelayBuffer, BufferOverflow) { + const EchoCanceller3Config config; for (auto rate : {8000, 16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); - std::unique_ptr delay_buffer(RenderDelayBuffer::Create( - NumBandsForRate(rate), kDownSamplingFactor, - GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), - GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + std::unique_ptr delay_buffer( + RenderDelayBuffer::Create(config, NumBandsForRate(rate))); std::vector> block_to_insert( NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); - for (size_t k = 0; k < kMaxApiCallsJitterBlocks; ++k) { - EXPECT_TRUE(delay_buffer->Insert(block_to_insert)); + for (size_t k = 0; k < 10; ++k) { + EXPECT_EQ(RenderDelayBuffer::BufferingEvent::kNone, + delay_buffer->Insert(block_to_insert)); } - EXPECT_FALSE(delay_buffer->Insert(block_to_insert)); + for (size_t k = 0; k < 1000; ++k) { + delay_buffer->Insert(block_to_insert); + } + + EXPECT_EQ(RenderDelayBuffer::BufferingEvent::kRenderOverrun, + delay_buffer->Insert(block_to_insert)); } } // Verifies that the check for available block works. TEST(RenderDelayBuffer, AvailableBlock) { constexpr size_t kNumBands = 1; - std::unique_ptr delay_buffer(RenderDelayBuffer::Create( - kNumBands, kDownSamplingFactor, - GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), - GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + std::unique_ptr delay_buffer( + RenderDelayBuffer::Create(EchoCanceller3Config(), kNumBands)); std::vector> input_block( kNumBands, std::vector(kBlockSize, 1.f)); - EXPECT_TRUE(delay_buffer->Insert(input_block)); - delay_buffer->UpdateBuffers(); + EXPECT_EQ(RenderDelayBuffer::BufferingEvent::kNone, + delay_buffer->Insert(input_block)); + delay_buffer->PrepareCaptureCall(); } // Verifies the SetDelay method. TEST(RenderDelayBuffer, SetDelay) { - std::unique_ptr delay_buffer(RenderDelayBuffer::Create( - 1, kDownSamplingFactor, - GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), - GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); - EXPECT_EQ(0u, delay_buffer->Delay()); - for (size_t delay = 0; delay < 20; ++delay) { + EchoCanceller3Config config; + std::unique_ptr delay_buffer( + RenderDelayBuffer::Create(config, 1)); + EXPECT_EQ(config.delay.min_echo_path_delay_blocks, delay_buffer->Delay()); + for (size_t delay = config.delay.min_echo_path_delay_blocks + 1; delay < 20; + ++delay) { delay_buffer->SetDelay(delay); EXPECT_EQ(delay, delay_buffer->Delay()); } @@ -84,10 +85,8 @@ TEST(RenderDelayBuffer, SetDelay) { // TODO(peah): Re-enable the test once the issue with memory leaks during DEATH // tests on test bots has been fixed. TEST(RenderDelayBuffer, DISABLED_WrongDelay) { - std::unique_ptr delay_buffer(RenderDelayBuffer::Create( - 3, kDownSamplingFactor, - GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), - GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + std::unique_ptr delay_buffer( + RenderDelayBuffer::Create(EchoCanceller3Config(), 3)); EXPECT_DEATH(delay_buffer->SetDelay(21), ""); } @@ -96,9 +95,7 @@ TEST(RenderDelayBuffer, WrongNumberOfBands) { for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); std::unique_ptr delay_buffer(RenderDelayBuffer::Create( - NumBandsForRate(rate), kDownSamplingFactor, - GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), - GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + EchoCanceller3Config(), NumBandsForRate(rate))); std::vector> block_to_insert( NumBandsForRate(rate < 48000 ? rate + 16000 : 16000), std::vector(kBlockSize, 0.f)); @@ -110,10 +107,8 @@ TEST(RenderDelayBuffer, WrongNumberOfBands) { TEST(RenderDelayBuffer, WrongBlockLength) { for (auto rate : {8000, 16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); - std::unique_ptr delay_buffer(RenderDelayBuffer::Create( - 3, kDownSamplingFactor, - GetDownSampledBufferSize(kDownSamplingFactor, kNumMatchedFilters), - GetRenderDelayBufferSize(kDownSamplingFactor, kNumMatchedFilters))); + std::unique_ptr delay_buffer( + RenderDelayBuffer::Create(EchoCanceller3Config(), 3)); std::vector> block_to_insert( NumBandsForRate(rate), std::vector(kBlockSize - 1, 0.f)); EXPECT_DEATH(delay_buffer->Insert(block_to_insert), ""); diff --git a/modules/audio_processing/aec3/render_delay_controller.cc b/modules/audio_processing/aec3/render_delay_controller.cc index 2c1f263ee7..05121f26f0 100644 --- a/modules/audio_processing/aec3/render_delay_controller.cc +++ b/modules/audio_processing/aec3/render_delay_controller.cc @@ -41,8 +41,10 @@ class RenderDelayControllerImpl final : public RenderDelayController { private: static int instance_count_; std::unique_ptr data_dumper_; + const size_t min_echo_path_delay_; const size_t default_delay_; size_t delay_; + EchoPathDelayEstimator delay_estimator_; size_t blocks_since_last_delay_estimate_ = 300000; int echo_path_delay_samples_; size_t align_call_counter_ = 0; @@ -50,7 +52,6 @@ class RenderDelayControllerImpl final : public RenderDelayController { std::vector capture_delay_buffer_; int capture_delay_buffer_index_ = 0; RenderDelayControllerMetrics metrics_; - EchoPathDelayEstimator delay_estimator_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayControllerImpl); }; @@ -78,12 +79,15 @@ RenderDelayControllerImpl::RenderDelayControllerImpl( int sample_rate_hz) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + min_echo_path_delay_(config.delay.min_echo_path_delay_blocks), default_delay_( - std::max(config.delay.default_delay, kMinEchoPathDelayBlocks)), + std::max(config.delay.default_delay, min_echo_path_delay_)), delay_(default_delay_), + delay_estimator_(data_dumper_.get(), config), echo_path_delay_samples_(default_delay_ * kBlockSize), - capture_delay_buffer_(kBlockSize * (kMaxApiCallsJitterBlocks + 2), 0.f), - delay_estimator_(data_dumper_.get(), config) { + capture_delay_buffer_( + kBlockSize * (config.delay.api_call_jitter_blocks + 2), + 0.f) { RTC_DCHECK(ValidFullBandRate(sample_rate_hz)); delay_estimator_.LogDelayEstimationProperties(sample_rate_hz, capture_delay_buffer_.size()); diff --git a/modules/audio_processing/aec3/render_delay_controller_unittest.cc b/modules/audio_processing/aec3/render_delay_controller_unittest.cc index 2e36d22484..0f5432b728 100644 --- a/modules/audio_processing/aec3/render_delay_controller_unittest.cc +++ b/modules/audio_processing/aec3/render_delay_controller_unittest.cc @@ -47,22 +47,20 @@ constexpr size_t kDownSamplingFactors[] = {2, 4, 8}; // Verifies the output of GetDelay when there are no AnalyzeRender calls. TEST(RenderDelayController, NoRenderSignal) { std::vector block(kBlockSize, 0.f); + EchoCanceller3Config config; for (size_t num_matched_filters = 4; num_matched_filters == 10; num_matched_filters++) { for (auto down_sampling_factor : kDownSamplingFactors) { + config.delay.down_sampling_factor = down_sampling_factor; + config.delay.num_filters = num_matched_filters; for (auto rate : {8000, 16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); std::unique_ptr delay_buffer( - RenderDelayBuffer::Create( - NumBandsForRate(rate), down_sampling_factor, - GetDownSampledBufferSize(down_sampling_factor, - num_matched_filters), - GetRenderDelayBufferSize(down_sampling_factor, - num_matched_filters))); + RenderDelayBuffer::Create(config, NumBandsForRate(rate))); std::unique_ptr delay_controller( - RenderDelayController::Create(EchoCanceller3Config(), rate)); + RenderDelayController::Create(config, rate)); for (size_t k = 0; k < 100; ++k) { - EXPECT_EQ(kMinEchoPathDelayBlocks, + EXPECT_EQ(config.delay.min_echo_path_delay_blocks, delay_controller->GetDelay( delay_buffer->GetDownsampledRenderBuffer(), block)); } @@ -78,26 +76,24 @@ TEST(RenderDelayController, BasicApiCalls) { for (size_t num_matched_filters = 4; num_matched_filters == 10; num_matched_filters++) { for (auto down_sampling_factor : kDownSamplingFactors) { + EchoCanceller3Config config; + config.delay.down_sampling_factor = down_sampling_factor; + config.delay.num_filters = num_matched_filters; for (auto rate : {8000, 16000, 32000, 48000}) { std::vector> render_block( NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create( - NumBandsForRate(rate), down_sampling_factor, - GetDownSampledBufferSize(down_sampling_factor, - num_matched_filters), - GetRenderDelayBufferSize(down_sampling_factor, - num_matched_filters))); + RenderDelayBuffer::Create(config, NumBandsForRate(rate))); std::unique_ptr delay_controller( RenderDelayController::Create(EchoCanceller3Config(), rate)); for (size_t k = 0; k < 10; ++k) { render_delay_buffer->Insert(render_block); - render_delay_buffer->UpdateBuffers(); + render_delay_buffer->PrepareCaptureCall(); delay_blocks = delay_controller->GetDelay( render_delay_buffer->GetDownsampledRenderBuffer(), capture_block); } EXPECT_FALSE(delay_controller->AlignmentHeadroomSamples()); - EXPECT_EQ(kMinEchoPathDelayBlocks, delay_blocks); + EXPECT_EQ(config.delay.min_echo_path_delay_blocks, delay_blocks); } } } @@ -112,6 +108,10 @@ TEST(RenderDelayController, Alignment) { for (size_t num_matched_filters = 4; num_matched_filters == 10; num_matched_filters++) { for (auto down_sampling_factor : kDownSamplingFactors) { + EchoCanceller3Config config; + config.delay.down_sampling_factor = down_sampling_factor; + config.delay.num_filters = num_matched_filters; + for (auto rate : {8000, 16000, 32000, 48000}) { std::vector> render_block( NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); @@ -119,20 +119,15 @@ TEST(RenderDelayController, Alignment) { for (size_t delay_samples : {15, 50, 150, 200, 800, 4000}) { SCOPED_TRACE(ProduceDebugText(rate, delay_samples)); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create( - NumBandsForRate(rate), down_sampling_factor, - GetDownSampledBufferSize(down_sampling_factor, - num_matched_filters), - GetRenderDelayBufferSize(down_sampling_factor, - num_matched_filters))); + RenderDelayBuffer::Create(config, NumBandsForRate(rate))); std::unique_ptr delay_controller( - RenderDelayController::Create(EchoCanceller3Config(), rate)); + RenderDelayController::Create(config, rate)); DelayBuffer signal_delay_buffer(delay_samples); for (size_t k = 0; k < (400 + delay_samples / kBlockSize); ++k) { RandomizeSampleVector(&random_generator, render_block[0]); signal_delay_buffer.Delay(render_block[0], capture_block); render_delay_buffer->Insert(render_block); - render_delay_buffer->UpdateBuffers(); + render_delay_buffer->PrepareCaptureCall(); delay_blocks = delay_controller->GetDelay( render_delay_buffer->GetDownsampledRenderBuffer(), capture_block); @@ -164,6 +159,9 @@ TEST(RenderDelayController, NonCausalAlignment) { for (size_t num_matched_filters = 4; num_matched_filters == 10; num_matched_filters++) { for (auto down_sampling_factor : kDownSamplingFactors) { + EchoCanceller3Config config; + config.delay.down_sampling_factor = down_sampling_factor; + config.delay.num_filters = num_matched_filters; for (auto rate : {8000, 16000, 32000, 48000}) { std::vector> render_block( NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); @@ -173,12 +171,7 @@ TEST(RenderDelayController, NonCausalAlignment) { for (int delay_samples : {-15, -50, -150, -200}) { SCOPED_TRACE(ProduceDebugText(rate, -delay_samples)); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create( - NumBandsForRate(rate), down_sampling_factor, - GetDownSampledBufferSize(down_sampling_factor, - num_matched_filters), - GetRenderDelayBufferSize(down_sampling_factor, - num_matched_filters))); + RenderDelayBuffer::Create(config, NumBandsForRate(rate))); std::unique_ptr delay_controller( RenderDelayController::Create(EchoCanceller3Config(), rate)); DelayBuffer signal_delay_buffer(-delay_samples); @@ -187,7 +180,7 @@ TEST(RenderDelayController, NonCausalAlignment) { RandomizeSampleVector(&random_generator, capture_block[0]); signal_delay_buffer.Delay(capture_block[0], render_block[0]); render_delay_buffer->Insert(render_block); - render_delay_buffer->UpdateBuffers(); + render_delay_buffer->PrepareCaptureCall(); delay_blocks = delay_controller->GetDelay( render_delay_buffer->GetDownsampledRenderBuffer(), capture_block[0]); @@ -212,6 +205,9 @@ TEST(RenderDelayController, AlignmentWithJitter) { for (size_t num_matched_filters = 4; num_matched_filters == 10; num_matched_filters++) { for (auto down_sampling_factor : kDownSamplingFactors) { + EchoCanceller3Config config; + config.delay.down_sampling_factor = down_sampling_factor; + config.delay.num_filters = num_matched_filters; for (auto rate : {8000, 16000, 32000, 48000}) { std::vector> render_block( NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); @@ -219,28 +215,25 @@ TEST(RenderDelayController, AlignmentWithJitter) { size_t delay_blocks = 0; SCOPED_TRACE(ProduceDebugText(rate, delay_samples)); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create( - NumBandsForRate(rate), down_sampling_factor, - GetDownSampledBufferSize(down_sampling_factor, - num_matched_filters), - GetRenderDelayBufferSize(down_sampling_factor, - num_matched_filters))); + RenderDelayBuffer::Create(config, NumBandsForRate(rate))); std::unique_ptr delay_controller( - RenderDelayController::Create(EchoCanceller3Config(), rate)); + RenderDelayController::Create(config, rate)); DelayBuffer signal_delay_buffer(delay_samples); for (size_t j = 0; j < (1000 + delay_samples / kBlockSize) / - kMaxApiCallsJitterBlocks + + config.delay.api_call_jitter_blocks + 1; ++j) { std::vector> capture_block_buffer; - for (size_t k = 0; k < (kMaxApiCallsJitterBlocks - 1); ++k) { + for (size_t k = 0; k < (config.delay.api_call_jitter_blocks - 1); + ++k) { RandomizeSampleVector(&random_generator, render_block[0]); signal_delay_buffer.Delay(render_block[0], capture_block); capture_block_buffer.push_back(capture_block); render_delay_buffer->Insert(render_block); } - for (size_t k = 0; k < (kMaxApiCallsJitterBlocks - 1); ++k) { - render_delay_buffer->UpdateBuffers(); + for (size_t k = 0; k < (config.delay.api_call_jitter_blocks - 1); + ++k) { + render_delay_buffer->PrepareCaptureCall(); delay_blocks = delay_controller->GetDelay( render_delay_buffer->GetDownsampledRenderBuffer(), capture_block_buffer[k]); @@ -275,17 +268,16 @@ TEST(RenderDelayController, InitialHeadroom) { for (size_t num_matched_filters = 4; num_matched_filters == 10; num_matched_filters++) { for (auto down_sampling_factor : kDownSamplingFactors) { + EchoCanceller3Config config; + config.delay.down_sampling_factor = down_sampling_factor; + config.delay.num_filters = num_matched_filters; for (auto rate : {8000, 16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create( - NumBandsForRate(rate), down_sampling_factor, - GetDownSampledBufferSize(down_sampling_factor, - num_matched_filters), - GetRenderDelayBufferSize(down_sampling_factor, - num_matched_filters))); + RenderDelayBuffer::Create(config, NumBandsForRate(rate))); + std::unique_ptr delay_controller( - RenderDelayController::Create(EchoCanceller3Config(), rate)); + RenderDelayController::Create(config, rate)); EXPECT_FALSE(delay_controller->AlignmentHeadroomSamples()); } } @@ -297,12 +289,11 @@ TEST(RenderDelayController, InitialHeadroom) { // Verifies the check for the capture signal block size. TEST(RenderDelayController, WrongCaptureSize) { std::vector block(kBlockSize - 1, 0.f); + EchoCanceller3Config config; for (auto rate : {8000, 16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(NumBandsForRate(rate), 4, - GetDownSampledBufferSize(4, 4), - GetRenderDelayBufferSize(4, 4))); + RenderDelayBuffer::Create(config, NumBandsForRate(rate))); EXPECT_DEATH( std::unique_ptr( RenderDelayController::Create(EchoCanceller3Config(), rate)) @@ -318,10 +309,9 @@ TEST(RenderDelayController, WrongCaptureSize) { TEST(RenderDelayController, DISABLED_WrongSampleRate) { for (auto rate : {-1, 0, 8001, 16001}) { SCOPED_TRACE(ProduceDebugText(rate)); + EchoCanceller3Config config; std::unique_ptr render_delay_buffer( - RenderDelayBuffer::Create(NumBandsForRate(rate), 4, - GetDownSampledBufferSize(4, 4), - GetRenderDelayBufferSize(4, 4))); + RenderDelayBuffer::Create(config, NumBandsForRate(rate))); EXPECT_DEATH( std::unique_ptr( RenderDelayController::Create(EchoCanceller3Config(), rate)), diff --git a/modules/audio_processing/aec3/render_signal_analyzer.cc b/modules/audio_processing/aec3/render_signal_analyzer.cc index 22aa352320..b5a3ce1ecf 100644 --- a/modules/audio_processing/aec3/render_signal_analyzer.cc +++ b/modules/audio_processing/aec3/render_signal_analyzer.cc @@ -30,8 +30,8 @@ void IdentifySmallNarrowBandRegions( return; } - const std::array& X2 = - render_buffer.Spectrum(*delay_partitions); + rtc::ArrayView X2 = render_buffer.Spectrum(*delay_partitions); + RTC_DCHECK_EQ(kFftLengthBy2Plus1, X2.size()); for (size_t k = 1; k < (X2.size() - 1); ++k) { (*narrow_band_counters)[k - 1] = X2[k] > 3 * std::max(X2[k - 1], X2[k + 1]) diff --git a/modules/audio_processing/aec3/render_signal_analyzer_unittest.cc b/modules/audio_processing/aec3/render_signal_analyzer_unittest.cc index 7e01f3fc44..eba28cf9c6 100644 --- a/modules/audio_processing/aec3/render_signal_analyzer_unittest.cc +++ b/modules/audio_processing/aec3/render_signal_analyzer_unittest.cc @@ -18,7 +18,7 @@ #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/aec3_fft.h" #include "modules/audio_processing/aec3/fft_data.h" -#include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/aec3/render_delay_buffer.h" #include "modules/audio_processing/test/echo_canceller_test_tools.h" #include "rtc_base/random.h" #include "test/gtest.h" @@ -58,18 +58,22 @@ TEST(RenderSignalAnalyzer, NoFalseDetectionOfNarrowBands) { Random random_generator(42U); std::vector> x(3, std::vector(kBlockSize, 0.f)); std::array x_old; - FftData X; - Aec3Fft fft; - RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 1, - std::vector(1, 1)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(EchoCanceller3Config(), 3)); std::array mask; x_old.fill(0.f); for (size_t k = 0; k < 100; ++k) { RandomizeSampleVector(&random_generator, x[0]); - fft.PaddedFft(x[0], x_old, &X); - render_buffer.Insert(x); - analyzer.Update(render_buffer, 0); + + render_delay_buffer->Insert(x); + if (k == 0) { + render_delay_buffer->Reset(); + } + render_delay_buffer->PrepareCaptureCall(); + + analyzer.Update(render_delay_buffer->GetRenderBuffer(), + rtc::Optional(0)); } mask.fill(1.f); @@ -86,8 +90,11 @@ TEST(RenderSignalAnalyzer, NarrowBandDetection) { std::vector> x(3, std::vector(kBlockSize, 0.f)); std::array x_old; Aec3Fft fft; - RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 1, - std::vector(1, 1)); + EchoCanceller3Config config; + config.delay.min_echo_path_delay_blocks = 0; + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, 3)); + std::array mask; x_old.fill(0.f); constexpr int kSinusFrequencyBin = 32; @@ -97,9 +104,15 @@ TEST(RenderSignalAnalyzer, NarrowBandDetection) { for (size_t k = 0; k < 100; ++k) { ProduceSinusoid(16000, 16000 / 2 * kSinusFrequencyBin / kFftLengthBy2, &sample_counter, x[0]); - render_buffer.Insert(x); - analyzer.Update(render_buffer, known_delay ? rtc::Optional(0) - : rtc::nullopt); + + render_delay_buffer->Insert(x); + if (k == 0) { + render_delay_buffer->Reset(); + } + render_delay_buffer->PrepareCaptureCall(); + + analyzer.Update(render_delay_buffer->GetRenderBuffer(), + known_delay ? rtc::Optional(0) : rtc::nullopt); } }; diff --git a/modules/audio_processing/aec3/residual_echo_estimator_unittest.cc b/modules/audio_processing/aec3/residual_echo_estimator_unittest.cc index b85bc1d936..92669c873d 100644 --- a/modules/audio_processing/aec3/residual_echo_estimator_unittest.cc +++ b/modules/audio_processing/aec3/residual_echo_estimator_unittest.cc @@ -12,6 +12,7 @@ #include "modules/audio_processing/aec3/aec3_fft.h" #include "modules/audio_processing/aec3/aec_state.h" +#include "modules/audio_processing/aec3/render_delay_buffer.h" #include "modules/audio_processing/include/audio_processing.h" #include "modules/audio_processing/test/echo_canceller_test_tools.h" #include "rtc_base/random.h" @@ -23,38 +24,43 @@ namespace webrtc { // Verifies that the check for non-null output residual echo power works. TEST(ResidualEchoEstimator, NullResidualEchoPowerOutput) { - AecState aec_state(EchoCanceller3Config{}); - RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 10, - std::vector(1, 10)); + EchoCanceller3Config config; + AecState aec_state(config); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, 3)); std::vector> H2; std::array S2_linear; std::array Y2; EXPECT_DEATH(ResidualEchoEstimator(EchoCanceller3Config{}) - .Estimate(aec_state, render_buffer, S2_linear, Y2, nullptr), + .Estimate(aec_state, render_delay_buffer->GetRenderBuffer(), + S2_linear, Y2, nullptr), ""); } #endif -TEST(ResidualEchoEstimator, BasicTest) { - ResidualEchoEstimator estimator(EchoCanceller3Config{}); +// TODO(peah): This test is broken in the sense that it not at all tests what it +// seems to test. Enable the test once that is adressed. +TEST(ResidualEchoEstimator, DISABLED_BasicTest) { EchoCanceller3Config config; config.ep_strength.default_len = 0.f; + config.delay.min_echo_path_delay_blocks = 0; + ResidualEchoEstimator estimator(config); AecState aec_state(config); - RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 10, - std::vector(1, 10)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, 3)); + std::array E2_main; std::array E2_shadow; std::array S2_linear; std::array S2_fallback; std::array Y2; std::array R2; - EchoPathVariability echo_path_variability(false, false); + EchoPathVariability echo_path_variability( + false, EchoPathVariability::DelayAdjustment::kNone, false); std::vector> x(3, std::vector(kBlockSize, 0.f)); std::vector> H2(10); Random random_generator(42U); - FftData X; - std::array x_old; std::array s; Aec3Fft fft; @@ -76,17 +82,21 @@ TEST(ResidualEchoEstimator, BasicTest) { S2_fallback.fill(kLevel); Y2.fill(kLevel); - for (int k = 0; k < 2000; ++k) { + for (int k = 0; k < 1993; ++k) { RandomizeSampleVector(&random_generator, x[0]); std::for_each(x[0].begin(), x[0].end(), [](float& a) { a /= 30.f; }); - fft.PaddedFft(x[0], x_old, &X); - render_buffer.Insert(x); + render_delay_buffer->Insert(x); + if (k == 0) { + render_delay_buffer->Reset(); + } + render_delay_buffer->PrepareCaptureCall(); aec_state.HandleEchoPathChange(echo_path_variability); - aec_state.Update(H2, h, true, 2, render_buffer, E2_main, Y2, x[0], s, - false); + aec_state.Update(H2, h, true, 2, render_delay_buffer->GetRenderBuffer(), + E2_main, Y2, x[0], s, false); - estimator.Estimate(aec_state, render_buffer, S2_linear, Y2, &R2); + estimator.Estimate(aec_state, render_delay_buffer->GetRenderBuffer(), + S2_linear, Y2, &R2); } std::for_each(R2.begin(), R2.end(), [&](float a) { EXPECT_NEAR(kLevel, a, 0.1f); }); diff --git a/modules/audio_processing/aec3/shadow_filter_update_gain.cc b/modules/audio_processing/aec3/shadow_filter_update_gain.cc index db393a78b2..464a0261d9 100644 --- a/modules/audio_processing/aec3/shadow_filter_update_gain.cc +++ b/modules/audio_processing/aec3/shadow_filter_update_gain.cc @@ -51,7 +51,7 @@ void ShadowFilterUpdateGain::Compute( constexpr float kNoiseGatePower = 220075344.f; constexpr float kMuFixed = .5f; std::array mu; - const auto& X2 = render_buffer.SpectralSum(size_partitions); + auto X2 = render_buffer.SpectralSum(size_partitions); std::transform(X2.begin(), X2.end(), mu.begin(), [&](float a) { return a > kNoiseGatePower ? kMuFixed / a : 0.f; }); diff --git a/modules/audio_processing/aec3/shadow_filter_update_gain_unittest.cc b/modules/audio_processing/aec3/shadow_filter_update_gain_unittest.cc index b89fc718ac..41645ae8cd 100644 --- a/modules/audio_processing/aec3/shadow_filter_update_gain_unittest.cc +++ b/modules/audio_processing/aec3/shadow_filter_update_gain_unittest.cc @@ -18,6 +18,7 @@ #include "modules/audio_processing/aec3/adaptive_fir_filter.h" #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/aec_state.h" +#include "modules/audio_processing/aec3/render_delay_buffer.h" #include "modules/audio_processing/test/echo_canceller_test_tools.h" #include "rtc_base/numerics/safe_minmax.h" #include "rtc_base/random.h" @@ -35,19 +36,22 @@ void RunFilterUpdateTest(int num_blocks_to_process, std::array* y_last_block, FftData* G_last_block) { ApmDataDumper data_dumper(42); - AdaptiveFirFilter main_filter(9, DetectOptimization(), &data_dumper); - AdaptiveFirFilter shadow_filter(9, DetectOptimization(), &data_dumper); + AdaptiveFirFilter main_filter(12, DetectOptimization(), &data_dumper); + AdaptiveFirFilter shadow_filter(12, DetectOptimization(), &data_dumper); Aec3Fft fft; - RenderBuffer render_buffer( - Aec3Optimization::kNone, 3, main_filter.SizePartitions(), - std::vector(1, main_filter.SizePartitions())); + + EchoCanceller3Config config; + config.delay.min_echo_path_delay_blocks = 0; + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, 3)); + std::array x_old; x_old.fill(0.f); ShadowFilterUpdateGain shadow_gain; Random random_generator(42U); std::vector> x(3, std::vector(kBlockSize, 0.f)); std::vector y(kBlockSize, 0.f); - AecState aec_state(EchoCanceller3Config{}); + AecState aec_state(config); RenderSignalAnalyzer render_signal_analyzer; std::array s; FftData S; @@ -67,10 +71,17 @@ void RunFilterUpdateTest(int num_blocks_to_process, // Create the render signal. RandomizeSampleVector(&random_generator, x[0]); delay_buffer.Delay(x[0], y); - render_buffer.Insert(x); - render_signal_analyzer.Update(render_buffer, delay_samples / kBlockSize); - shadow_filter.Filter(render_buffer, &S); + render_delay_buffer->Insert(x); + if (k == 0) { + render_delay_buffer->Reset(); + } + render_delay_buffer->PrepareCaptureCall(); + + render_signal_analyzer.Update(render_delay_buffer->GetRenderBuffer(), + delay_samples / kBlockSize); + + shadow_filter.Filter(render_delay_buffer->GetRenderBuffer(), &S); fft.Ifft(S, &s); std::transform(y.begin(), y.end(), s.begin() + kFftLengthBy2, e_shadow.begin(), @@ -79,9 +90,10 @@ void RunFilterUpdateTest(int num_blocks_to_process, [](float& a) { a = rtc::SafeClamp(a, -32768.f, 32767.f); }); fft.ZeroPaddedFft(e_shadow, &E_shadow); - shadow_gain.Compute(render_buffer, render_signal_analyzer, E_shadow, + shadow_gain.Compute(render_delay_buffer->GetRenderBuffer(), + render_signal_analyzer, E_shadow, shadow_filter.SizePartitions(), saturation, &G); - shadow_filter.Adapt(render_buffer, G); + shadow_filter.Adapt(render_delay_buffer->GetRenderBuffer(), G); } std::copy(e_shadow.begin(), e_shadow.end(), e_last_block->begin()); @@ -103,8 +115,10 @@ std::string ProduceDebugText(size_t delay) { // Verifies that the check for non-null output gain parameter works. TEST(ShadowFilterUpdateGain, NullDataOutputGain) { ApmDataDumper data_dumper(42); - RenderBuffer render_buffer(Aec3Optimization::kNone, 3, 1, - std::vector(1, 1)); + FftBuffer fft_buffer(1); + MatrixBuffer block_buffer(fft_buffer.buffer.size(), 3, kBlockSize); + VectorBuffer spectrum_buffer(fft_buffer.buffer.size(), kFftLengthBy2Plus1); + RenderBuffer render_buffer(1, &block_buffer, &spectrum_buffer, &fft_buffer); RenderSignalAnalyzer analyzer; FftData E; ShadowFilterUpdateGain gain; @@ -151,9 +165,9 @@ TEST(ShadowFilterUpdateGain, DecreasingGain) { RunFilterUpdateTest(200, 65, blocks_with_saturation, &e, &y, &G_b); RunFilterUpdateTest(300, 65, blocks_with_saturation, &e, &y, &G_c); - G_a.Spectrum(Aec3Optimization::kNone, &G_a_power); - G_b.Spectrum(Aec3Optimization::kNone, &G_b_power); - G_c.Spectrum(Aec3Optimization::kNone, &G_c_power); + G_a.Spectrum(Aec3Optimization::kNone, G_a_power); + G_b.Spectrum(Aec3Optimization::kNone, G_b_power); + G_c.Spectrum(Aec3Optimization::kNone, G_c_power); EXPECT_GT(std::accumulate(G_a_power.begin(), G_a_power.end(), 0.), std::accumulate(G_b_power.begin(), G_b_power.end(), 0.)); diff --git a/modules/audio_processing/aec3/subtractor.cc b/modules/audio_processing/aec3/subtractor.cc index b374f49cc3..3c99b989a4 100644 --- a/modules/audio_processing/aec3/subtractor.cc +++ b/modules/audio_processing/aec3/subtractor.cc @@ -59,14 +59,28 @@ Subtractor::~Subtractor() = default; void Subtractor::HandleEchoPathChange( const EchoPathVariability& echo_path_variability) { - use_shadow_filter_frequency_response_ = false; - if (echo_path_variability.delay_change) { + const auto full_reset = [&]() { + use_shadow_filter_frequency_response_ = false; main_filter_.HandleEchoPathChange(); shadow_filter_.HandleEchoPathChange(); - G_main_.HandleEchoPathChange(); + G_main_.HandleEchoPathChange(echo_path_variability); G_shadow_.HandleEchoPathChange(); converged_filter_ = false; converged_filter_counter_ = 0; + }; + + // TODO(peah): Add delay-change specific reset behavior. + if ((echo_path_variability.delay_change == + EchoPathVariability::DelayAdjustment::kBufferFlush) || + (echo_path_variability.delay_change == + EchoPathVariability::DelayAdjustment::kDelayReset)) { + full_reset(); + } else if (echo_path_variability.delay_change == + EchoPathVariability::DelayAdjustment::kNewDetectedDelay) { + full_reset(); + } else if (echo_path_variability.delay_change == + EchoPathVariability::DelayAdjustment::kBufferReadjustment) { + full_reset(); } } @@ -120,8 +134,8 @@ void Subtractor::Process(const RenderBuffer& render_buffer, } // Compute spectra for future use. - E_main.Spectrum(optimization_, &output->E2_main); - E_shadow.Spectrum(optimization_, &output->E2_shadow); + E_main.Spectrum(optimization_, output->E2_main); + E_shadow.Spectrum(optimization_, output->E2_shadow); // Update the main filter. G_main_.Compute(render_buffer, render_signal_analyzer, *output, main_filter_, diff --git a/modules/audio_processing/aec3/subtractor_unittest.cc b/modules/audio_processing/aec3/subtractor_unittest.cc index b10421b428..bd8c7777c9 100644 --- a/modules/audio_processing/aec3/subtractor_unittest.cc +++ b/modules/audio_processing/aec3/subtractor_unittest.cc @@ -15,6 +15,7 @@ #include #include "modules/audio_processing/aec3/aec_state.h" +#include "modules/audio_processing/aec3/render_delay_buffer.h" #include "modules/audio_processing/test/echo_canceller_test_tools.h" #include "rtc_base/random.h" #include "test/gtest.h" @@ -32,8 +33,10 @@ float RunSubtractorTest(int num_blocks_to_process, std::vector y(kBlockSize, 0.f); std::array x_old; SubtractorOutput output; - RenderBuffer render_buffer(Aec3Optimization::kNone, 3, kAdaptiveFilterLength, - std::vector(1, kAdaptiveFilterLength)); + EchoCanceller3Config config; + config.delay.min_echo_path_delay_blocks = 0; + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, 3)); RenderSignalAnalyzer render_signal_analyzer; Random random_generator(42U); Aec3Fft fft; @@ -54,23 +57,33 @@ float RunSubtractorTest(int num_blocks_to_process, } else { delay_buffer.Delay(x[0], y); } - render_buffer.Insert(x); - render_signal_analyzer.Update(render_buffer, aec_state.FilterDelay()); + render_delay_buffer->Insert(x); + if (k == 0) { + render_delay_buffer->Reset(); + } + render_delay_buffer->PrepareCaptureCall(); + render_signal_analyzer.Update(render_delay_buffer->GetRenderBuffer(), + aec_state.FilterDelay()); // Handle echo path changes. if (std::find(blocks_with_echo_path_changes.begin(), blocks_with_echo_path_changes.end(), k) != blocks_with_echo_path_changes.end()) { - subtractor.HandleEchoPathChange(EchoPathVariability(true, true)); + subtractor.HandleEchoPathChange(EchoPathVariability( + true, EchoPathVariability::DelayAdjustment::kNewDetectedDelay, + false)); } - subtractor.Process(render_buffer, y, render_signal_analyzer, aec_state, - &output); + subtractor.Process(render_delay_buffer->GetRenderBuffer(), y, + render_signal_analyzer, aec_state, &output); - aec_state.HandleEchoPathChange(EchoPathVariability(false, false)); + aec_state.HandleEchoPathChange(EchoPathVariability( + false, EchoPathVariability::DelayAdjustment::kNone, false)); aec_state.Update(subtractor.FilterFrequencyResponse(), subtractor.FilterImpulseResponse(), - subtractor.ConvergedFilter(), delay_samples / kBlockSize, - render_buffer, E2_main, Y2, x[0], output.s_main, false); + subtractor.ConvergedFilter(), + rtc::Optional(delay_samples / kBlockSize), + render_delay_buffer->GetRenderBuffer(), E2_main, Y2, x[0], + output.s_main, false); } const float output_power = std::inner_product( @@ -104,12 +117,13 @@ TEST(Subtractor, NullDataDumper) { TEST(Subtractor, DISABLED_NullOutput) { ApmDataDumper data_dumper(42); Subtractor subtractor(&data_dumper, DetectOptimization()); - RenderBuffer render_buffer(Aec3Optimization::kNone, 3, kAdaptiveFilterLength, - std::vector(1, kAdaptiveFilterLength)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(EchoCanceller3Config(), 3)); RenderSignalAnalyzer render_signal_analyzer; std::vector y(kBlockSize, 0.f); - EXPECT_DEATH(subtractor.Process(render_buffer, y, render_signal_analyzer, + EXPECT_DEATH(subtractor.Process(render_delay_buffer->GetRenderBuffer(), y, + render_signal_analyzer, AecState(EchoCanceller3Config{}), nullptr), ""); } @@ -118,13 +132,14 @@ TEST(Subtractor, DISABLED_NullOutput) { TEST(Subtractor, WrongCaptureSize) { ApmDataDumper data_dumper(42); Subtractor subtractor(&data_dumper, DetectOptimization()); - RenderBuffer render_buffer(Aec3Optimization::kNone, 3, kAdaptiveFilterLength, - std::vector(1, kAdaptiveFilterLength)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(EchoCanceller3Config(), 3)); RenderSignalAnalyzer render_signal_analyzer; std::vector y(kBlockSize - 1, 0.f); SubtractorOutput output; - EXPECT_DEATH(subtractor.Process(render_buffer, y, render_signal_analyzer, + EXPECT_DEATH(subtractor.Process(render_delay_buffer->GetRenderBuffer(), y, + render_signal_analyzer, AecState(EchoCanceller3Config{}), &output), ""); } diff --git a/modules/audio_processing/aec3/suppression_gain_unittest.cc b/modules/audio_processing/aec3/suppression_gain_unittest.cc index 9fee6a24a9..2b5e702e31 100644 --- a/modules/audio_processing/aec3/suppression_gain_unittest.cc +++ b/modules/audio_processing/aec3/suppression_gain_unittest.cc @@ -11,7 +11,7 @@ #include "modules/audio_processing/aec3/suppression_gain.h" #include "modules/audio_processing/aec3/aec_state.h" -#include "modules/audio_processing/aec3/render_buffer.h" +#include "modules/audio_processing/aec3/render_delay_buffer.h" #include "modules/audio_processing/aec3/subtractor.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/checks.h" @@ -57,13 +57,12 @@ TEST(SuppressionGain, BasicGainComputation) { std::array g; std::array s; std::vector> x(1, std::vector(kBlockSize, 0.f)); - AecState aec_state(EchoCanceller3Config{}); + EchoCanceller3Config config; + AecState aec_state(config); ApmDataDumper data_dumper(42); Subtractor subtractor(&data_dumper, DetectOptimization()); - RenderBuffer render_buffer( - DetectOptimization(), 1, - std::max(kUnknownDelayRenderWindowSize, kAdaptiveFilterLength), - std::vector(1, kAdaptiveFilterLength)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(config, 3)); // Verify the functionality for forcing a zero gain. E2.fill(1000000000.f); @@ -72,7 +71,8 @@ TEST(SuppressionGain, BasicGainComputation) { s.fill(10.f); aec_state.Update( subtractor.FilterFrequencyResponse(), subtractor.FilterImpulseResponse(), - subtractor.ConvergedFilter(), 10, render_buffer, E2, Y2, x[0], s, false); + subtractor.ConvergedFilter(), 10, render_delay_buffer->GetRenderBuffer(), + E2, Y2, x[0], s, false); suppression_gain.GetGain(E2, R2, N2, analyzer, aec_state, x, &high_bands_gain, &g); std::for_each(g.begin(), g.end(), [](float a) { EXPECT_FLOAT_EQ(0.f, a); }); @@ -85,17 +85,17 @@ TEST(SuppressionGain, BasicGainComputation) { N2.fill(100.f); // Ensure that the gain is no longer forced to zero. for (int k = 0; k <= kNumBlocksPerSecond / 5 + 1; ++k) { - aec_state.Update(subtractor.FilterFrequencyResponse(), - subtractor.FilterImpulseResponse(), - subtractor.ConvergedFilter(), 10, render_buffer, E2, Y2, - x[0], s, false); + aec_state.Update( + subtractor.FilterFrequencyResponse(), + subtractor.FilterImpulseResponse(), subtractor.ConvergedFilter(), 10, + render_delay_buffer->GetRenderBuffer(), E2, Y2, x[0], s, false); } for (int k = 0; k < 100; ++k) { - aec_state.Update(subtractor.FilterFrequencyResponse(), - subtractor.FilterImpulseResponse(), - subtractor.ConvergedFilter(), 10, render_buffer, E2, Y2, - x[0], s, false); + aec_state.Update( + subtractor.FilterFrequencyResponse(), + subtractor.FilterImpulseResponse(), subtractor.ConvergedFilter(), 10, + render_delay_buffer->GetRenderBuffer(), E2, Y2, x[0], s, false); suppression_gain.GetGain(E2, R2, N2, analyzer, aec_state, x, &high_bands_gain, &g); } @@ -108,10 +108,10 @@ TEST(SuppressionGain, BasicGainComputation) { R2.fill(0.1f); N2.fill(0.f); for (int k = 0; k < 100; ++k) { - aec_state.Update(subtractor.FilterFrequencyResponse(), - subtractor.FilterImpulseResponse(), - subtractor.ConvergedFilter(), 10, render_buffer, E2, Y2, - x[0], s, false); + aec_state.Update( + subtractor.FilterFrequencyResponse(), + subtractor.FilterImpulseResponse(), subtractor.ConvergedFilter(), 10, + render_delay_buffer->GetRenderBuffer(), E2, Y2, x[0], s, false); suppression_gain.GetGain(E2, R2, N2, analyzer, aec_state, x, &high_bands_gain, &g); } diff --git a/modules/audio_processing/aec3/vector_buffer.cc b/modules/audio_processing/aec3/vector_buffer.cc new file mode 100644 index 0000000000..5fd664644b --- /dev/null +++ b/modules/audio_processing/aec3/vector_buffer.cc @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_processing/aec3/vector_buffer.h" + +#include "modules/audio_processing/aec3/aec3_common.h" + +namespace webrtc { + +VectorBuffer::VectorBuffer(size_t size, size_t height) + : size(size), buffer(size, std::vector(height, 0.f)) { + for (auto& c : buffer) { + std::fill(c.begin(), c.end(), 0.f); + } +} + +VectorBuffer::~VectorBuffer() = default; + +} // namespace webrtc diff --git a/modules/audio_processing/aec3/vector_buffer.h b/modules/audio_processing/aec3/vector_buffer.h new file mode 100644 index 0000000000..9dce1ef65b --- /dev/null +++ b/modules/audio_processing/aec3/vector_buffer.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_PROCESSING_AEC3_VECTOR_BUFFER_H_ +#define MODULES_AUDIO_PROCESSING_AEC3_VECTOR_BUFFER_H_ + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { + +// Struct for bundling a circular buffer of one dimensional vector objects +// together with the read and write indices. +struct VectorBuffer { + VectorBuffer(size_t size, size_t height); + ~VectorBuffer(); + + size_t IncIndex(size_t index) { + return index < buffer.size() - 1 ? index + 1 : 0; + } + + size_t DecIndex(size_t index) { + return index > 0 ? index - 1 : buffer.size() - 1; + } + + size_t OffsetIndex(size_t index, int offset) { + RTC_DCHECK_GE(buffer.size(), offset); + return (buffer.size() + index + offset) % buffer.size(); + } + + void UpdateWriteIndex(int offset) { write = OffsetIndex(write, offset); } + void IncWriteIndex() { write = IncIndex(write); } + void DecWriteIndex() { write = DecIndex(write); } + void UpdateReadIndex(int offset) { read = OffsetIndex(read, offset); } + void IncReadIndex() { read = IncIndex(read); } + void DecReadIndex() { read = DecIndex(read); } + + size_t size; + std::vector> buffer; + size_t write = 0; + size_t read = 0; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_PROCESSING_AEC3_VECTOR_BUFFER_H_ diff --git a/modules/audio_processing/include/audio_processing.h b/modules/audio_processing/include/audio_processing.h index d3a1ef507c..a639851d27 100644 --- a/modules/audio_processing/include/audio_processing.h +++ b/modules/audio_processing/include/audio_processing.h @@ -1154,6 +1154,8 @@ struct EchoCanceller3Config { size_t default_delay = 5; size_t down_sampling_factor = 4; size_t num_filters = 4; + size_t api_call_jitter_blocks = 26; + size_t min_echo_path_delay_blocks = 5; } delay; struct Erle {