From 69221db53413bdca8cb1bacff11b3651f283dd95 Mon Sep 17 00:00:00 2001 From: peah Date: Fri, 27 Jan 2017 03:28:19 -0800 Subject: [PATCH] Adding second layer of the echo canceller 3 functionality. This CL adds code to the BlockProcessor, which basically constitutes the second layer in echo canceller 3. The CL includes two incomplete classes (EchoRemover and EchoPathDelayEstimator) which will be completed in upcoming CLs. Because of this, some of the unittests are disabled until those are added. BUG=webrtc:6018 Review-Url: https://codereview.webrtc.org/2611223003 Cr-Commit-Position: refs/heads/master@{#16319} --- webrtc/modules/audio_processing/BUILD.gn | 19 ++ .../audio_processing/aec3/block_processor.cc | 113 +++++-- .../audio_processing/aec3/block_processor.h | 24 +- .../aec3/block_processor_unittest.cc | 134 +++++++- .../audio_processing/aec3/echo_canceller3.cc | 58 ++-- .../audio_processing/aec3/echo_canceller3.h | 8 +- .../aec3/echo_canceller3_unittest.cc | 47 +-- .../aec3/echo_path_delay_estimator.cc | 37 ++ .../aec3/echo_path_delay_estimator.h | 35 ++ .../echo_path_delay_estimator_unittest.cc | 88 +++++ .../aec3/echo_path_variability.h | 27 ++ .../audio_processing/aec3/echo_remover.cc | 74 ++++ .../audio_processing/aec3/echo_remover.h | 44 +++ .../aec3/echo_remover_unittest.cc | 158 +++++++++ .../aec3/mock/mock_block_processor.h | 4 +- .../aec3/mock/mock_echo_remover.h | 41 +++ .../aec3/mock/mock_render_delay_buffer.h | 51 +++ .../aec3/mock/mock_render_delay_controller.h | 34 ++ .../aec3/render_delay_buffer.cc | 105 ++++++ .../aec3/render_delay_buffer.h | 57 ++++ .../aec3/render_delay_buffer_unittest.cc | 319 ++++++++++++++++++ .../aec3/render_delay_controller.cc | 175 ++++++++++ .../aec3/render_delay_controller.h | 40 +++ .../aec3/render_delay_controller_unittest.cc | 264 +++++++++++++++ .../logging/apm_data_dumper.h | 73 +++- .../test/echo_canceller_test_tools.cc | 40 +++ .../test/echo_canceller_test_tools.h | 44 +++ .../echo_canceller_test_tools_unittest.cc | 71 ++++ 28 files changed, 2091 insertions(+), 93 deletions(-) create mode 100644 webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.cc create mode 100644 webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.h create mode 100644 webrtc/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc create mode 100644 webrtc/modules/audio_processing/aec3/echo_path_variability.h create mode 100644 webrtc/modules/audio_processing/aec3/echo_remover.cc create mode 100644 webrtc/modules/audio_processing/aec3/echo_remover.h create mode 100644 webrtc/modules/audio_processing/aec3/echo_remover_unittest.cc create mode 100644 webrtc/modules/audio_processing/aec3/mock/mock_echo_remover.h create mode 100644 webrtc/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h create mode 100644 webrtc/modules/audio_processing/aec3/mock/mock_render_delay_controller.h create mode 100644 webrtc/modules/audio_processing/aec3/render_delay_buffer.cc create mode 100644 webrtc/modules/audio_processing/aec3/render_delay_buffer.h create mode 100644 webrtc/modules/audio_processing/aec3/render_delay_buffer_unittest.cc create mode 100644 webrtc/modules/audio_processing/aec3/render_delay_controller.cc create mode 100644 webrtc/modules/audio_processing/aec3/render_delay_controller.h create mode 100644 webrtc/modules/audio_processing/aec3/render_delay_controller_unittest.cc create mode 100644 webrtc/modules/audio_processing/test/echo_canceller_test_tools.cc create mode 100644 webrtc/modules/audio_processing/test/echo_canceller_test_tools.h create mode 100644 webrtc/modules/audio_processing/test/echo_canceller_test_tools_unittest.cc diff --git a/webrtc/modules/audio_processing/BUILD.gn b/webrtc/modules/audio_processing/BUILD.gn index 4dcf7d90d1..dfedbee11c 100644 --- a/webrtc/modules/audio_processing/BUILD.gn +++ b/webrtc/modules/audio_processing/BUILD.gn @@ -35,8 +35,17 @@ rtc_static_library("audio_processing") { "aec3/cascaded_biquad_filter.h", "aec3/echo_canceller3.cc", "aec3/echo_canceller3.h", + "aec3/echo_path_delay_estimator.cc", + "aec3/echo_path_delay_estimator.h", + "aec3/echo_path_variability.h", + "aec3/echo_remover.cc", + "aec3/echo_remover.h", "aec3/frame_blocker.cc", "aec3/frame_blocker.h", + "aec3/render_delay_buffer.cc", + "aec3/render_delay_buffer.h", + "aec3/render_delay_controller.cc", + "aec3/render_delay_controller.h", "aecm/aecm_core.cc", "aecm/aecm_core.h", "aecm/echo_control_mobile.cc", @@ -511,8 +520,15 @@ if (rtc_include_tests) { "aec3/block_processor_unittest.cc", "aec3/cascaded_biquad_filter_unittest.cc", "aec3/echo_canceller3_unittest.cc", + "aec3/echo_path_delay_estimator_unittest.cc", + "aec3/echo_remover_unittest.cc", "aec3/frame_blocker_unittest.cc", "aec3/mock/mock_block_processor.h", + "aec3/mock/mock_echo_remover.h", + "aec3/mock/mock_render_delay_buffer.h", + "aec3/mock/mock_render_delay_controller.h", + "aec3/render_delay_buffer_unittest.cc", + "aec3/render_delay_controller_unittest.cc", "audio_processing_impl_locking_unittest.cc", "audio_processing_impl_unittest.cc", "audio_processing_unittest.cc", @@ -535,6 +551,9 @@ if (rtc_include_tests) { "test/debug_dump_replayer.cc", "test/debug_dump_replayer.h", "test/debug_dump_test.cc", + "test/echo_canceller_test_tools.cc", + "test/echo_canceller_test_tools.h", + "test/echo_canceller_test_tools_unittest.cc", "test/test_utils.h", "voice_detection_unittest.cc", ] diff --git a/webrtc/modules/audio_processing/aec3/block_processor.cc b/webrtc/modules/audio_processing/aec3/block_processor.cc index 066e2f71ac..1f50a84854 100644 --- a/webrtc/modules/audio_processing/aec3/block_processor.cc +++ b/webrtc/modules/audio_processing/aec3/block_processor.cc @@ -10,64 +10,139 @@ #include "webrtc/modules/audio_processing/aec3/block_processor.h" #include "webrtc/base/atomicops.h" +#include "webrtc/base/constructormagic.h" #include "webrtc/base/optional.h" #include "webrtc/modules/audio_processing/aec3/aec3_constants.h" +#include "webrtc/modules/audio_processing/aec3/echo_path_variability.h" +#include "webrtc/modules/audio_processing/logging/apm_data_dumper.h" +#include "webrtc/system_wrappers/include/logging.h" namespace webrtc { namespace { class BlockProcessorImpl final : public BlockProcessor { public: - explicit BlockProcessorImpl(int sample_rate_hz); + BlockProcessorImpl(int sample_rate_hz, + std::unique_ptr render_buffer, + std::unique_ptr delay_controller, + std::unique_ptr echo_remover); + ~BlockProcessorImpl() override; - void ProcessCapture(bool known_echo_path_change, - bool saturated_microphone_signal, + void ProcessCapture(bool echo_path_gain_change, + bool capture_signal_saturation, std::vector>* capture_block) override; bool BufferRender(std::vector>* block) override; - void ReportEchoLeakage(bool leakage_detected) override; + void UpdateEchoLeakageStatus(bool leakage_detected) override; private: - const size_t sample_rate_hz_; static int instance_count_; std::unique_ptr data_dumper_; + const size_t sample_rate_hz_; + std::unique_ptr render_buffer_; + std::unique_ptr delay_controller_; + std::unique_ptr echo_remover_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(BlockProcessorImpl); }; +constexpr size_t kRenderBufferSize = 250; int BlockProcessorImpl::instance_count_ = 0; +constexpr size_t kMaxApiJitter = 30; -BlockProcessorImpl::BlockProcessorImpl(int sample_rate_hz) - : sample_rate_hz_(sample_rate_hz), - data_dumper_( - new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))) {} +BlockProcessorImpl::BlockProcessorImpl( + int sample_rate_hz, + std::unique_ptr render_buffer, + std::unique_ptr delay_controller, + std::unique_ptr echo_remover) + : data_dumper_( + new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + sample_rate_hz_(sample_rate_hz), + render_buffer_(std::move(render_buffer)), + delay_controller_(std::move(delay_controller)), + echo_remover_(std::move(echo_remover)) {} BlockProcessorImpl::~BlockProcessorImpl() = default; void BlockProcessorImpl::ProcessCapture( - bool known_echo_path_change, - bool saturated_microphone_signal, + bool echo_path_gain_change, + bool capture_signal_saturation, std::vector>* capture_block) { RTC_DCHECK(capture_block); RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), capture_block->size()); RTC_DCHECK_EQ(kBlockSize, (*capture_block)[0].size()); + + const size_t delay = delay_controller_->GetDelay((*capture_block)[0]); + const bool render_delay_change = delay != render_buffer_->Delay(); + + if (render_delay_change) { + render_buffer_->SetDelay(delay); + } + + if (render_buffer_->IsBlockAvailable()) { + auto& render_block = render_buffer_->GetNext(); + echo_remover_->ProcessBlock( + delay_controller_->AlignmentHeadroomSamples(), + EchoPathVariability(echo_path_gain_change, render_delay_change), + capture_signal_saturation, render_block, capture_block); + } else { + LOG(LS_INFO) << "AEC3 empty render buffer"; + } } -bool BlockProcessorImpl::BufferRender( - std::vector>* render_block) { - RTC_DCHECK(render_block); - RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), render_block->size()); - RTC_DCHECK_EQ(kBlockSize, (*render_block)[0].size()); - return false; +bool BlockProcessorImpl::BufferRender(std::vector>* block) { + RTC_DCHECK(block); + RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), block->size()); + RTC_DCHECK_EQ(kBlockSize, (*block)[0].size()); + + const bool delay_controller_overrun = + !delay_controller_->AnalyzeRender((*block)[0]); + const bool render_buffer_overrun = !render_buffer_->Insert(block); + if (delay_controller_overrun || render_buffer_overrun) { + LOG(LS_INFO) << "AEC3 buffer overrrun"; + return false; + } + + return true; } -void BlockProcessorImpl::ReportEchoLeakage(bool leakage_detected) {} +void BlockProcessorImpl::UpdateEchoLeakageStatus(bool leakage_detected) { + echo_remover_->UpdateEchoLeakageStatus(leakage_detected); +} } // namespace BlockProcessor* BlockProcessor::Create(int sample_rate_hz) { - return new BlockProcessorImpl(sample_rate_hz); + std::unique_ptr render_buffer(RenderDelayBuffer::Create( + kRenderBufferSize, NumBandsForRate(sample_rate_hz), kMaxApiJitter)); + std::unique_ptr delay_controller( + RenderDelayController::Create(sample_rate_hz, *render_buffer)); + std::unique_ptr echo_remover( + EchoRemover::Create(sample_rate_hz)); + return Create(sample_rate_hz, std::move(render_buffer), + std::move(delay_controller), std::move(echo_remover)); +} + +BlockProcessor* BlockProcessor::Create( + int sample_rate_hz, + std::unique_ptr render_buffer) { + std::unique_ptr delay_controller( + RenderDelayController::Create(sample_rate_hz, *render_buffer)); + std::unique_ptr echo_remover( + EchoRemover::Create(sample_rate_hz)); + return Create(sample_rate_hz, std::move(render_buffer), + std::move(delay_controller), std::move(echo_remover)); +} + +BlockProcessor* BlockProcessor::Create( + int sample_rate_hz, + std::unique_ptr render_buffer, + std::unique_ptr delay_controller, + std::unique_ptr echo_remover) { + return new BlockProcessorImpl(sample_rate_hz, std::move(render_buffer), + std::move(delay_controller), + std::move(echo_remover)); } } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/block_processor.h b/webrtc/modules/audio_processing/aec3/block_processor.h index b84b18df81..12a994e609 100644 --- a/webrtc/modules/audio_processing/aec3/block_processor.h +++ b/webrtc/modules/audio_processing/aec3/block_processor.h @@ -14,8 +14,9 @@ #include #include -#include "webrtc/base/constructormagic.h" -#include "webrtc/modules/audio_processing/logging/apm_data_dumper.h" +#include "webrtc/modules/audio_processing/aec3/echo_remover.h" +#include "webrtc/modules/audio_processing/aec3/render_delay_buffer.h" +#include "webrtc/modules/audio_processing/aec3/render_delay_controller.h" namespace webrtc { @@ -23,20 +24,31 @@ namespace webrtc { class BlockProcessor { public: static BlockProcessor* Create(int sample_rate_hz); + // Only used for testing purposes. + static BlockProcessor* Create( + int sample_rate_hz, + std::unique_ptr render_buffer); + static BlockProcessor* Create( + int sample_rate_hz, + std::unique_ptr render_buffer, + std::unique_ptr delay_controller, + std::unique_ptr echo_remover); + virtual ~BlockProcessor() = default; // Processes a block of capture data. virtual void ProcessCapture( - bool known_echo_path_change, - bool saturated_microphone_signal, + bool echo_path_gain_change, + bool capture_signal_saturation, std::vector>* capture_block) = 0; - // Buffers a block of render data supplied by a FrameBlocker object. + // Buffers a block of render data supplied by a FrameBlocker object. Returns a + // bool indicating the success of the buffering. virtual bool BufferRender(std::vector>* render_block) = 0; // Reports whether echo leakage has been detected in the echo canceller // output. - virtual void ReportEchoLeakage(bool leakage_detected) = 0; + virtual void UpdateEchoLeakageStatus(bool leakage_detected) = 0; }; } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/block_processor_unittest.cc b/webrtc/modules/audio_processing/aec3/block_processor_unittest.cc index 6e22b41879..b5d6a1432f 100644 --- a/webrtc/modules/audio_processing/aec3/block_processor_unittest.cc +++ b/webrtc/modules/audio_processing/aec3/block_processor_unittest.cc @@ -15,12 +15,24 @@ #include #include +#include "webrtc/base/checks.h" +#include "webrtc/base/random.h" #include "webrtc/modules/audio_processing/aec3/aec3_constants.h" +#include "webrtc/modules/audio_processing/aec3/mock/mock_echo_remover.h" +#include "webrtc/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h" +#include "webrtc/modules/audio_processing/aec3/mock/mock_render_delay_controller.h" +#include "webrtc/modules/audio_processing/test/echo_canceller_test_tools.h" +#include "webrtc/test/gmock.h" #include "webrtc/test/gtest.h" namespace webrtc { namespace { +using testing::AtLeast; +using testing::Return; +using testing::StrictMock; +using testing::_; + // Verifies that the basic BlockProcessor functionality works and that the API // methods are callable. void RunBasicSetupAndApiCallTest(int sample_rate_hz) { @@ -29,9 +41,9 @@ void RunBasicSetupAndApiCallTest(int sample_rate_hz) { std::vector> block(NumBandsForRate(sample_rate_hz), std::vector(kBlockSize, 0.f)); - EXPECT_FALSE(block_processor->BufferRender(&block)); + EXPECT_TRUE(block_processor->BufferRender(&block)); block_processor->ProcessCapture(false, false, &block); - block_processor->ReportEchoLeakage(false); + block_processor->UpdateEchoLeakageStatus(false); } #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) @@ -86,6 +98,110 @@ std::string ProduceDebugText(int sample_rate_hz) { } // namespace +// Verifies that the delay controller functionality is properly integrated with +// the render delay buffer inside block processor. +// TODO(peah): Activate the unittest once the required code has been landed. +TEST(BlockProcessor, DISABLED_DelayControllerIntegration) { + constexpr size_t kNumBlocks = 310; + constexpr size_t kDelayInSamples = 640; + constexpr size_t kDelayHeadroom = 1; + constexpr size_t kDelayInBlocks = + kDelayInSamples / kBlockSize - kDelayHeadroom; + Random random_generator(42U); + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr> + render_delay_buffer_mock( + new StrictMock(rate)); + EXPECT_CALL(*render_delay_buffer_mock, Insert(_)) + .Times(kNumBlocks) + .WillRepeatedly(Return(true)); + EXPECT_CALL(*render_delay_buffer_mock, IsBlockAvailable()) + .Times(kNumBlocks) + .WillRepeatedly(Return(true)); + EXPECT_CALL(*render_delay_buffer_mock, SetDelay(kDelayInBlocks)) + .Times(AtLeast(1)); + EXPECT_CALL(*render_delay_buffer_mock, MaxDelay()).WillOnce(Return(30)); + EXPECT_CALL(*render_delay_buffer_mock, MaxApiJitter()).WillOnce(Return(30)); + EXPECT_CALL(*render_delay_buffer_mock, Delay()) + .Times(kNumBlocks + 1) + .WillRepeatedly(Return(0)); + std::unique_ptr block_processor( + BlockProcessor::Create(rate, std::move(render_delay_buffer_mock))); + + std::vector> render_block( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + std::vector> capture_block( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + DelayBuffer signal_delay_buffer(kDelayInSamples); + for (size_t k = 0; k < kNumBlocks; ++k) { + RandomizeSampleVector(&random_generator, render_block[0]); + signal_delay_buffer.Delay(render_block[0], capture_block[0]); + EXPECT_TRUE(block_processor->BufferRender(&render_block)); + block_processor->ProcessCapture(false, false, &capture_block); + } + } +} + +// Verifies that BlockProcessor submodules are called in a proper manner. +TEST(BlockProcessor, SubmoduleIntegration) { + constexpr size_t kNumBlocks = 310; + Random random_generator(42U); + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr> + render_delay_buffer_mock( + new StrictMock(rate)); + std::unique_ptr< + testing::StrictMock> + render_delay_controller_mock( + new StrictMock()); + std::unique_ptr> + echo_remover_mock(new StrictMock()); + + EXPECT_CALL(*render_delay_buffer_mock, Insert(_)) + .Times(kNumBlocks) + .WillRepeatedly(Return(true)); + EXPECT_CALL(*render_delay_buffer_mock, IsBlockAvailable()) + .Times(kNumBlocks) + .WillRepeatedly(Return(true)); + EXPECT_CALL(*render_delay_buffer_mock, GetNext()).Times(kNumBlocks); + EXPECT_CALL(*render_delay_buffer_mock, SetDelay(9)).Times(AtLeast(1)); + EXPECT_CALL(*render_delay_buffer_mock, Delay()) + .Times(kNumBlocks) + .WillRepeatedly(Return(0)); + EXPECT_CALL(*render_delay_controller_mock, GetDelay(_)) + .Times(kNumBlocks) + .WillRepeatedly(Return(9)); + EXPECT_CALL(*render_delay_controller_mock, AnalyzeRender(_)) + .Times(kNumBlocks) + .WillRepeatedly(Return(true)); + EXPECT_CALL(*render_delay_controller_mock, AlignmentHeadroomSamples()) + .Times(kNumBlocks); + EXPECT_CALL(*echo_remover_mock, ProcessBlock(_, _, _, _, _)) + .Times(kNumBlocks); + EXPECT_CALL(*echo_remover_mock, UpdateEchoLeakageStatus(_)) + .Times(kNumBlocks); + + std::unique_ptr block_processor(BlockProcessor::Create( + rate, std::move(render_delay_buffer_mock), + std::move(render_delay_controller_mock), std::move(echo_remover_mock))); + + std::vector> render_block( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + std::vector> capture_block( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + DelayBuffer signal_delay_buffer(640); + for (size_t k = 0; k < kNumBlocks; ++k) { + RandomizeSampleVector(&random_generator, render_block[0]); + signal_delay_buffer.Delay(render_block[0], capture_block[0]); + EXPECT_TRUE(block_processor->BufferRender(&render_block)); + block_processor->ProcessCapture(false, false, &capture_block); + block_processor->UpdateEchoLeakageStatus(false); + } + } +} + TEST(BlockProcessor, BasicSetupAndApiCalls) { for (auto rate : {8000, 16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); @@ -94,30 +210,28 @@ TEST(BlockProcessor, BasicSetupAndApiCalls) { } #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) -// TODO(peah): Enable all DEATH tests below, or add suppressions, once clarity -// has been reached on why they fail on the trybots. -TEST(BlockProcessor, DISABLED_VerifyRenderBlockSizeCheck) { +TEST(BlockProcessor, VerifyRenderBlockSizeCheck) { for (auto rate : {8000, 16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); RunRenderBlockSizeVerificationTest(rate); } } -TEST(BlockProcessor, DISABLED_VerifyCaptureBlockSizeCheck) { +TEST(BlockProcessor, VerifyCaptureBlockSizeCheck) { for (auto rate : {8000, 16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); RunCaptureBlockSizeVerificationTest(rate); } } -TEST(BlockProcessor, DISABLED_VerifyRenderNumBandsCheck) { +TEST(BlockProcessor, VerifyRenderNumBandsCheck) { for (auto rate : {8000, 16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); RunRenderNumBandsVerificationTest(rate); } } -TEST(BlockProcessor, DISABLED_VerifyCaptureNumBandsCheck) { +TEST(BlockProcessor, VerifyCaptureNumBandsCheck) { for (auto rate : {8000, 16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); RunCaptureNumBandsVerificationTest(rate); @@ -125,14 +239,14 @@ TEST(BlockProcessor, DISABLED_VerifyCaptureNumBandsCheck) { } // Verifiers that the verification for null ProcessCapture input works. -TEST(BlockProcessor, DISABLED_NullProcessCaptureParameter) { +TEST(BlockProcessor, NullProcessCaptureParameter) { EXPECT_DEATH(std::unique_ptr(BlockProcessor::Create(8000)) ->ProcessCapture(false, false, nullptr), ""); } // Verifiers that the verification for null BufferRender input works. -TEST(BlockProcessor, DISABLED_NullBufferRenderParameter) { +TEST(BlockProcessor, NullBufferRenderParameter) { EXPECT_DEATH(std::unique_ptr(BlockProcessor::Create(8000)) ->BufferRender(nullptr), ""); diff --git a/webrtc/modules/audio_processing/aec3/echo_canceller3.cc b/webrtc/modules/audio_processing/aec3/echo_canceller3.cc index b409e60350..ec7b55a215 100644 --- a/webrtc/modules/audio_processing/aec3/echo_canceller3.cc +++ b/webrtc/modules/audio_processing/aec3/echo_canceller3.cc @@ -53,7 +53,7 @@ void FillSubFrameView(std::vector>* frame, void ProcessCaptureFrameContent( AudioBuffer* capture, - bool known_echo_path_change, + bool level_change, bool saturated_microphone_signal, size_t sub_frame_index, FrameBlocker* capture_blocker, @@ -63,13 +63,13 @@ void ProcessCaptureFrameContent( std::vector>* sub_frame_view) { FillSubFrameView(capture, sub_frame_index, sub_frame_view); capture_blocker->InsertSubFrameAndExtractBlock(*sub_frame_view, block); - block_processor->ProcessCapture(known_echo_path_change, - saturated_microphone_signal, block); + block_processor->ProcessCapture(level_change, saturated_microphone_signal, + block); output_framer->InsertBlockAndExtractSubFrame(*block, sub_frame_view); } void ProcessRemainingCaptureFrameContent( - bool known_echo_path_change, + bool level_change, bool saturated_microphone_signal, FrameBlocker* capture_blocker, BlockFramer* output_framer, @@ -80,8 +80,8 @@ void ProcessRemainingCaptureFrameContent( } capture_blocker->ExtractBlock(block); - block_processor->ProcessCapture(known_echo_path_change, - saturated_microphone_signal, block); + block_processor->ProcessCapture(level_change, saturated_microphone_signal, + block); output_framer->InsertBlock(*block); } @@ -101,7 +101,7 @@ bool BufferRemainingRenderFrameContent(FrameBlocker* render_blocker, BlockProcessor* block_processor, std::vector>* block) { if (!render_blocker->IsBlockAvailable()) { - return false; + return true; } render_blocker->ExtractBlock(block); return block_processor->BufferRender(block); @@ -275,8 +275,7 @@ void EchoCanceller3::AnalyzeCapture(AudioBuffer* capture) { } } -void EchoCanceller3::ProcessCapture(AudioBuffer* capture, - bool known_echo_path_change) { +void EchoCanceller3::ProcessCapture(AudioBuffer* capture, bool level_change) { RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); RTC_DCHECK(capture); RTC_DCHECK_EQ(1u, capture->num_channels()); @@ -289,27 +288,26 @@ void EchoCanceller3::ProcessCapture(AudioBuffer* capture, data_dumper_->DumpWav("aec3_capture_input", capture_lower_band, LowestBandRate(sample_rate_hz_), 1); - const bool render_buffer_overrun = EmptyRenderQueue(); - RTC_DCHECK(!render_buffer_overrun); + const bool successful_buffering = EmptyRenderQueue(); + RTC_DCHECK(successful_buffering); if (capture_highpass_filter_) { capture_highpass_filter_->Process(capture_lower_band); } - ProcessCaptureFrameContent(capture, known_echo_path_change, - saturated_microphone_signal_, 0, &capture_blocker_, - &output_framer_, block_processor_.get(), &block_, - &sub_frame_view_); + ProcessCaptureFrameContent( + capture, level_change, saturated_microphone_signal_, 0, &capture_blocker_, + &output_framer_, block_processor_.get(), &block_, &sub_frame_view_); if (sample_rate_hz_ != 8000) { ProcessCaptureFrameContent( - capture, known_echo_path_change, saturated_microphone_signal_, 1, + capture, level_change, saturated_microphone_signal_, 1, &capture_blocker_, &output_framer_, block_processor_.get(), &block_, &sub_frame_view_); } ProcessRemainingCaptureFrameContent( - known_echo_path_change, saturated_microphone_signal_, &capture_blocker_, + level_change, saturated_microphone_signal_, &capture_blocker_, &output_framer_, block_processor_.get(), &block_); data_dumper_->DumpWav("aec3_capture_output", frame_length_, @@ -332,27 +330,33 @@ bool EchoCanceller3::Validate( bool EchoCanceller3::EmptyRenderQueue() { RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); - bool render_buffer_overrun = false; + bool successful_buffering = true; bool frame_to_buffer = render_transfer_queue_.Remove(&render_queue_output_frame_); while (frame_to_buffer) { - render_buffer_overrun |= BufferRenderFrameContent( - &render_queue_output_frame_, 0, &render_blocker_, - block_processor_.get(), &block_, &sub_frame_view_); + successful_buffering = + BufferRenderFrameContent(&render_queue_output_frame_, 0, + &render_blocker_, block_processor_.get(), + &block_, &sub_frame_view_) && + successful_buffering; if (sample_rate_hz_ != 8000) { - render_buffer_overrun |= BufferRenderFrameContent( - &render_queue_output_frame_, 1, &render_blocker_, - block_processor_.get(), &block_, &sub_frame_view_); + successful_buffering = + BufferRenderFrameContent(&render_queue_output_frame_, 1, + &render_blocker_, block_processor_.get(), + &block_, &sub_frame_view_) && + successful_buffering; } - render_buffer_overrun |= BufferRemainingRenderFrameContent( - &render_blocker_, block_processor_.get(), &block_); + successful_buffering = + BufferRemainingRenderFrameContent(&render_blocker_, + block_processor_.get(), &block_) && + successful_buffering; frame_to_buffer = render_transfer_queue_.Remove(&render_queue_output_frame_); } - return render_buffer_overrun; + return successful_buffering; } } // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/echo_canceller3.h b/webrtc/modules/audio_processing/aec3/echo_canceller3.h index 57714b2b06..f1a7f38b3a 100644 --- a/webrtc/modules/audio_processing/aec3/echo_canceller3.h +++ b/webrtc/modules/audio_processing/aec3/echo_canceller3.h @@ -76,15 +76,15 @@ class EchoCanceller3 { void AnalyzeCapture(AudioBuffer* capture); // Processes the split-band domain capture signal in order to remove any echo // present in the signal. - void ProcessCapture(AudioBuffer* capture, bool known_echo_path_change); + void ProcessCapture(AudioBuffer* capture, bool level_change); // Signals whether an external detector has detected echo leakage from the // echo canceller. // Note that in the case echo leakage has been flagged, it should be unflagged // once it is no longer occurring. - void ReportEchoLeakage(bool leakage_detected) { + void UpdateEchoLeakageStatus(bool leakage_detected) { RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); - block_processor_->ReportEchoLeakage(leakage_detected); + block_processor_->UpdateEchoLeakageStatus(leakage_detected); } // Validates a config. @@ -96,6 +96,8 @@ class EchoCanceller3 { private: class RenderWriter; + // Empties the render SwapQueue. A bool is returned that indicates the success + // of the operation. bool EmptyRenderQueue(); rtc::RaceChecker capture_race_checker_; diff --git a/webrtc/modules/audio_processing/aec3/echo_canceller3_unittest.cc b/webrtc/modules/audio_processing/aec3/echo_canceller3_unittest.cc index 8baa45606f..afe429d1d2 100644 --- a/webrtc/modules/audio_processing/aec3/echo_canceller3_unittest.cc +++ b/webrtc/modules/audio_processing/aec3/echo_canceller3_unittest.cc @@ -28,6 +28,7 @@ namespace webrtc { namespace { +using testing::Return; using testing::StrictMock; using testing::_; @@ -82,16 +83,16 @@ class CaptureTransportVerificationProcessor : public BlockProcessor { explicit CaptureTransportVerificationProcessor(size_t num_bands) {} ~CaptureTransportVerificationProcessor() override = default; - void ProcessCapture(bool known_echo_path_change, + void ProcessCapture(bool level_change, bool saturated_microphone_signal, std::vector>* capture_block) override { } bool BufferRender(std::vector>* block) override { - return false; + return true; } - void ReportEchoLeakage(bool leakage_detected) override {} + void UpdateEchoLeakageStatus(bool leakage_detected) override {} private: RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(CaptureTransportVerificationProcessor); @@ -104,7 +105,7 @@ class RenderTransportVerificationProcessor : public BlockProcessor { explicit RenderTransportVerificationProcessor(size_t num_bands) {} ~RenderTransportVerificationProcessor() override = default; - void ProcessCapture(bool known_echo_path_change, + void ProcessCapture(bool level_change, bool saturated_microphone_signal, std::vector>* capture_block) override { std::vector> render_block = @@ -115,10 +116,10 @@ class RenderTransportVerificationProcessor : public BlockProcessor { bool BufferRender(std::vector>* block) override { received_render_blocks_.push_back(*block); - return false; + return true; } - void ReportEchoLeakage(bool leakage_detected) override {} + void UpdateEchoLeakageStatus(bool leakage_detected) override {} private: std::deque>> received_render_blocks_; @@ -217,8 +218,9 @@ class EchoCanceller3Tester { block_processor_mock( new StrictMock()); EXPECT_CALL(*block_processor_mock, BufferRender(_)) - .Times(expected_num_block_to_process); - EXPECT_CALL(*block_processor_mock, ReportEchoLeakage(_)).Times(0); + .Times(expected_num_block_to_process) + .WillRepeatedly(Return(true)); + EXPECT_CALL(*block_processor_mock, UpdateEchoLeakageStatus(_)).Times(0); switch (echo_path_change_test_variant) { case EchoPathChangeTestVariant::kNone: @@ -294,24 +296,28 @@ class EchoCanceller3Tester { block_processor_mock( new StrictMock()); EXPECT_CALL(*block_processor_mock, BufferRender(_)) - .Times(expected_num_block_to_process); + .Times(expected_num_block_to_process) + .WillRepeatedly(Return(true)); EXPECT_CALL(*block_processor_mock, ProcessCapture(_, _, _)) .Times(expected_num_block_to_process); switch (leakage_report_variant) { case EchoLeakageTestVariant::kNone: - EXPECT_CALL(*block_processor_mock, ReportEchoLeakage(_)).Times(0); + EXPECT_CALL(*block_processor_mock, UpdateEchoLeakageStatus(_)).Times(0); break; case EchoLeakageTestVariant::kFalseSticky: - EXPECT_CALL(*block_processor_mock, ReportEchoLeakage(false)).Times(1); + EXPECT_CALL(*block_processor_mock, UpdateEchoLeakageStatus(false)) + .Times(1); break; case EchoLeakageTestVariant::kTrueSticky: - EXPECT_CALL(*block_processor_mock, ReportEchoLeakage(true)).Times(1); + EXPECT_CALL(*block_processor_mock, UpdateEchoLeakageStatus(true)) + .Times(1); break; case EchoLeakageTestVariant::kTrueNonSticky: { testing::InSequence s; - EXPECT_CALL(*block_processor_mock, ReportEchoLeakage(true)).Times(1); - EXPECT_CALL(*block_processor_mock, ReportEchoLeakage(false)) + EXPECT_CALL(*block_processor_mock, UpdateEchoLeakageStatus(true)) + .Times(1); + EXPECT_CALL(*block_processor_mock, UpdateEchoLeakageStatus(false)) .Times(kNumFramesToProcess - 1); } break; } @@ -326,19 +332,19 @@ class EchoCanceller3Tester { break; case EchoLeakageTestVariant::kFalseSticky: if (frame_index == 0) { - aec3.ReportEchoLeakage(false); + aec3.UpdateEchoLeakageStatus(false); } break; case EchoLeakageTestVariant::kTrueSticky: if (frame_index == 0) { - aec3.ReportEchoLeakage(true); + aec3.UpdateEchoLeakageStatus(true); } break; case EchoLeakageTestVariant::kTrueNonSticky: if (frame_index == 0) { - aec3.ReportEchoLeakage(true); + aec3.UpdateEchoLeakageStatus(true); } else { - aec3.ReportEchoLeakage(false); + aec3.UpdateEchoLeakageStatus(false); } break; } @@ -381,8 +387,9 @@ class EchoCanceller3Tester { block_processor_mock( new StrictMock()); EXPECT_CALL(*block_processor_mock, BufferRender(_)) - .Times(expected_num_block_to_process); - EXPECT_CALL(*block_processor_mock, ReportEchoLeakage(_)).Times(0); + .Times(expected_num_block_to_process) + .WillRepeatedly(Return(true)); + EXPECT_CALL(*block_processor_mock, UpdateEchoLeakageStatus(_)).Times(0); switch (saturation_variant) { case SaturationTestVariant::kNone: diff --git a/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.cc b/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.cc new file mode 100644 index 0000000000..3ae8f879ef --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.cc @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.h" + +#include "webrtc/base/checks.h" +#include "webrtc/modules/audio_processing/aec3/aec3_constants.h" +#include "webrtc/modules/audio_processing/logging/apm_data_dumper.h" + +namespace webrtc { + +// TODO(peah): Add functionality. +EchoPathDelayEstimator::EchoPathDelayEstimator(ApmDataDumper* data_dumper, + int sample_rate_hz) { + RTC_DCHECK(data_dumper); + RTC_DCHECK(sample_rate_hz == 8000 || sample_rate_hz == 16000 || + sample_rate_hz == 32000 || sample_rate_hz == 48000); +} + +EchoPathDelayEstimator::~EchoPathDelayEstimator() = default; + +// TODO(peah): Add functionality. +rtc::Optional EchoPathDelayEstimator::EstimateDelay( + rtc::ArrayView render, + rtc::ArrayView capture) { + RTC_DCHECK_EQ(render.size(), kBlockSize); + RTC_DCHECK_EQ(capture.size(), kBlockSize); + return rtc::Optional(); +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.h b/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.h new file mode 100644 index 0000000000..ae8ab4318e --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_ECHO_PATH_DELAY_ESTIMATOR_H_ +#define WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_ECHO_PATH_DELAY_ESTIMATOR_H_ + +#include + +#include "webrtc/base/constructormagic.h" +#include "webrtc/base/optional.h" + +namespace webrtc { + +class ApmDataDumper; + +class EchoPathDelayEstimator { + public: + EchoPathDelayEstimator(ApmDataDumper* data_dumper, int sample_rate_hz); + ~EchoPathDelayEstimator(); + rtc::Optional EstimateDelay(rtc::ArrayView render, + rtc::ArrayView capture); + + private: + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(EchoPathDelayEstimator); +}; +} // namespace webrtc + +#endif // WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_ECHO_PATH_DELAY_ESTIMATOR_H_ diff --git a/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc b/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc new file mode 100644 index 0000000000..c3146ed7b8 --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.h" + +#include +#include + +#include "webrtc/modules/audio_processing/aec3/aec3_constants.h" +#include "webrtc/modules/audio_processing/logging/apm_data_dumper.h" +#include "webrtc/test/gtest.h" + +namespace webrtc { +namespace { + +std::string ProduceDebugText(int sample_rate_hz) { + std::ostringstream ss; + ss << "Sample rate: " << sample_rate_hz; + return ss.str(); +} + +} // namespace + +// Verifies that the basic API calls work. +TEST(EchoPathDelayEstimator, BasicApiCalls) { + for (auto rate : {8000, 16000, 32000, 48000}) { + ProduceDebugText(rate); + ApmDataDumper data_dumper(0); + EchoPathDelayEstimator estimator(&data_dumper, rate); + std::vector render(kBlockSize, 0.f); + std::vector capture(kBlockSize, 0.f); + for (size_t k = 0; k < 100; ++k) { + estimator.EstimateDelay(render, capture); + } + } +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies the check for correct sample rate. +TEST(EchoPathDelayEstimator, WrongSampleRate) { + ApmDataDumper data_dumper(0); + EXPECT_DEATH(EchoPathDelayEstimator remover(&data_dumper, 8001), ""); +} + +// Verifies the check for the render blocksize. +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed. +TEST(EchoPathDelayEstimator, DISABLED_WrongRenderBlockSize) { + for (auto rate : {8000, 16000, 32000, 48000}) { + ProduceDebugText(rate); + ApmDataDumper data_dumper(0); + EchoPathDelayEstimator estimator(&data_dumper, rate); + std::vector render(kBlockSize - 1, 0.f); + std::vector capture(kBlockSize, 0.f); + EXPECT_DEATH(estimator.EstimateDelay(render, capture), ""); + } +} + +// Verifies the check for the capture blocksize. +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed. +TEST(EchoPathDelayEstimator, DISABLED_WrongCaptureBlockSize) { + for (auto rate : {8000, 16000, 32000, 48000}) { + ProduceDebugText(rate); + ApmDataDumper data_dumper(0); + EchoPathDelayEstimator estimator(&data_dumper, rate); + std::vector render(kBlockSize, 0.f); + std::vector capture(kBlockSize - 1, 0.f); + EXPECT_DEATH(estimator.EstimateDelay(render, capture), ""); + } +} + +// Verifies the check for non-null data dumper. +TEST(EchoPathDelayEstimator, NullDataDumper) { + EXPECT_DEATH(EchoPathDelayEstimator(nullptr, 8000), ""); +} + +#endif + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/echo_path_variability.h b/webrtc/modules/audio_processing/aec3/echo_path_variability.h new file mode 100644 index 0000000000..070887964d --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/echo_path_variability.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_ECHO_PATH_VARIABILITY_H_ +#define WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_ECHO_PATH_VARIABILITY_H_ + +namespace webrtc { + +struct EchoPathVariability { + EchoPathVariability(bool gain_change, bool delay_change) + : gain_change(gain_change), delay_change(delay_change) {} + + bool AudioPathChanged() const { return gain_change || delay_change; } + bool gain_change; + bool delay_change; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_ECHO_PATH_VARIABILITY_H_ diff --git a/webrtc/modules/audio_processing/aec3/echo_remover.cc b/webrtc/modules/audio_processing/aec3/echo_remover.cc new file mode 100644 index 0000000000..2ae5525b4a --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/echo_remover.cc @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "webrtc/modules/audio_processing/aec3/echo_remover.h" + +#include +#include + +#include "webrtc/base/constructormagic.h" +#include "webrtc/base/checks.h" +#include "webrtc/base/optional.h" +#include "webrtc/modules/audio_processing/aec3/aec3_constants.h" + +namespace webrtc { + +namespace { +class EchoRemoverImpl final : public EchoRemover { + public: + explicit EchoRemoverImpl(int sample_rate_hz); + ~EchoRemoverImpl() override; + + void ProcessBlock(const rtc::Optional& echo_path_delay_samples, + const EchoPathVariability& echo_path_variability, + bool capture_signal_saturation, + const std::vector>& render, + std::vector>* capture) override; + + void UpdateEchoLeakageStatus(bool leakage_detected) override; + + private: + const int sample_rate_hz_; + + RTC_DISALLOW_COPY_AND_ASSIGN(EchoRemoverImpl); +}; + +// TODO(peah): Add functionality. +EchoRemoverImpl::EchoRemoverImpl(int sample_rate_hz) + : sample_rate_hz_(sample_rate_hz) { + RTC_DCHECK(sample_rate_hz == 8000 || sample_rate_hz == 16000 || + sample_rate_hz == 32000 || sample_rate_hz == 48000); +} + +EchoRemoverImpl::~EchoRemoverImpl() = default; + +// TODO(peah): Add functionality. +void EchoRemoverImpl::ProcessBlock( + const rtc::Optional& echo_path_delay_samples, + const EchoPathVariability& echo_path_variability, + bool capture_signal_saturation, + const std::vector>& render, + std::vector>* capture) { + RTC_DCHECK(capture); + RTC_DCHECK_EQ(render.size(), NumBandsForRate(sample_rate_hz_)); + RTC_DCHECK_EQ(capture->size(), NumBandsForRate(sample_rate_hz_)); + RTC_DCHECK_EQ(render[0].size(), kBlockSize); + RTC_DCHECK_EQ((*capture)[0].size(), kBlockSize); +} + +// TODO(peah): Add functionality. +void EchoRemoverImpl::UpdateEchoLeakageStatus(bool leakage_detected) {} + +} // namespace + +EchoRemover* EchoRemover::Create(int sample_rate_hz) { + return new EchoRemoverImpl(sample_rate_hz); +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/echo_remover.h b/webrtc/modules/audio_processing/aec3/echo_remover.h new file mode 100644 index 0000000000..f7ac50cec7 --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/echo_remover.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_ECHO_REMOVER_H_ +#define WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_ECHO_REMOVER_H_ + +#include + +#include "webrtc/base/optional.h" +#include "webrtc/modules/audio_processing/aec3/echo_path_variability.h" + +namespace webrtc { + +// Class for removing the echo from the capture signal. +class EchoRemover { + public: + static EchoRemover* Create(int sample_rate_hz); + virtual ~EchoRemover() = default; + + // Removes the echo from a block of samples from the capture signal. The + // supplied render signal is assumed to be pre-aligned with the capture + // signal. + virtual void ProcessBlock( + const rtc::Optional& echo_path_delay_samples, + const EchoPathVariability& echo_path_variability, + bool capture_signal_saturation, + const std::vector>& render, + std::vector>* capture) = 0; + + // Updates the status on whether echo leakage is detected in the output of the + // echo remover. + virtual void UpdateEchoLeakageStatus(bool leakage_detected) = 0; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_ECHO_REMOVER_H_ diff --git a/webrtc/modules/audio_processing/aec3/echo_remover_unittest.cc b/webrtc/modules/audio_processing/aec3/echo_remover_unittest.cc new file mode 100644 index 0000000000..1c019937e5 --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/echo_remover_unittest.cc @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_processing/aec3/echo_remover.h" + +#include +#include +#include + +#include "webrtc/modules/audio_processing/aec3/aec3_constants.h" +#include "webrtc/modules/audio_processing/logging/apm_data_dumper.h" +#include "webrtc/test/gtest.h" + +namespace webrtc { +namespace { + +std::string ProduceDebugText(int sample_rate_hz) { + std::ostringstream ss; + ss << "Sample rate: " << sample_rate_hz; + return ss.str(); +} + +} // namespace + +// Verifies the basic API call sequence +TEST(EchoRemover, BasicApiCalls) { + for (auto rate : {8000, 16000, 32000, 48000}) { + ProduceDebugText(rate); + std::unique_ptr remover(EchoRemover::Create(rate)); + + std::vector> render(NumBandsForRate(rate), + std::vector(kBlockSize, 0.f)); + std::vector> capture( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + for (size_t k = 0; k < 100; ++k) { + EchoPathVariability echo_path_variability(k % 3 == 0 ? true : false, + k % 5 == 0 ? true : false); + rtc::Optional echo_path_delay_samples = + (k % 6 == 0 ? rtc::Optional(k * 10) + : rtc::Optional()); + remover->ProcessBlock(echo_path_delay_samples, echo_path_variability, + k % 2 == 0 ? true : false, render, &capture); + remover->UpdateEchoLeakageStatus(k % 7 == 0 ? true : false); + } + } +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies the check for the samplerate. +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed. +TEST(EchoRemover, DISABLED_WrongSampleRate) { + EXPECT_DEATH(std::unique_ptr(EchoRemover::Create(8001)), ""); +} + +// Verifies the check for the render block size. +TEST(EchoRemover, WrongRenderBlockSize) { + for (auto rate : {8000, 16000, 32000, 48000}) { + ProduceDebugText(rate); + std::unique_ptr remover(EchoRemover::Create(rate)); + + std::vector> render( + NumBandsForRate(rate), std::vector(kBlockSize - 1, 0.f)); + std::vector> capture( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + EchoPathVariability echo_path_variability(false, false); + rtc::Optional echo_path_delay_samples; + EXPECT_DEATH( + remover->ProcessBlock(echo_path_delay_samples, echo_path_variability, + false, render, &capture), + ""); + } +} + +// Verifies the check for the capture block size. +TEST(EchoRemover, WrongCaptureBlockSize) { + for (auto rate : {8000, 16000, 32000, 48000}) { + ProduceDebugText(rate); + std::unique_ptr remover(EchoRemover::Create(rate)); + + std::vector> render(NumBandsForRate(rate), + std::vector(kBlockSize, 0.f)); + std::vector> capture( + NumBandsForRate(rate), std::vector(kBlockSize - 1, 0.f)); + EchoPathVariability echo_path_variability(false, false); + rtc::Optional echo_path_delay_samples; + EXPECT_DEATH( + remover->ProcessBlock(echo_path_delay_samples, echo_path_variability, + false, render, &capture), + ""); + } +} + +// Verifies the check for the number of render bands. +TEST(EchoRemover, WrongRenderNumBands) { + for (auto rate : {16000, 32000, 48000}) { + ProduceDebugText(rate); + std::unique_ptr remover(EchoRemover::Create(rate)); + + std::vector> render( + NumBandsForRate(rate == 48000 ? 16000 : rate + 16000), + std::vector(kBlockSize, 0.f)); + std::vector> capture( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + EchoPathVariability echo_path_variability(false, false); + rtc::Optional echo_path_delay_samples; + EXPECT_DEATH( + remover->ProcessBlock(echo_path_delay_samples, echo_path_variability, + false, render, &capture), + ""); + } +} + +// Verifies the check for the number of capture bands. +TEST(EchoRemover, WrongCaptureNumBands) { + for (auto rate : {16000, 32000, 48000}) { + ProduceDebugText(rate); + std::unique_ptr remover(EchoRemover::Create(rate)); + + std::vector> render(NumBandsForRate(rate), + std::vector(kBlockSize, 0.f)); + std::vector> capture( + NumBandsForRate(rate == 48000 ? 16000 : rate + 16000), + std::vector(kBlockSize, 0.f)); + EchoPathVariability echo_path_variability(false, false); + rtc::Optional echo_path_delay_samples; + EXPECT_DEATH( + remover->ProcessBlock(echo_path_delay_samples, echo_path_variability, + false, render, &capture), + ""); + } +} + +// Verifies the check for non-null capture block. +TEST(EchoRemover, NullCapture) { + std::unique_ptr remover(EchoRemover::Create(8000)); + + std::vector> render(NumBandsForRate(8000), + std::vector(kBlockSize, 0.f)); + EchoPathVariability echo_path_variability(false, false); + rtc::Optional echo_path_delay_samples; + EXPECT_DEATH( + remover->ProcessBlock(echo_path_delay_samples, echo_path_variability, + false, render, nullptr), + ""); +} + +#endif + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/mock/mock_block_processor.h b/webrtc/modules/audio_processing/aec3/mock/mock_block_processor.h index d73b095e26..60dab63ea9 100644 --- a/webrtc/modules/audio_processing/aec3/mock/mock_block_processor.h +++ b/webrtc/modules/audio_processing/aec3/mock/mock_block_processor.h @@ -24,11 +24,11 @@ class MockBlockProcessor : public BlockProcessor { virtual ~MockBlockProcessor() {} MOCK_METHOD3(ProcessCapture, - void(bool known_echo_path_change, + void(bool level_change, bool saturated_microphone_signal, std::vector>* capture_block)); MOCK_METHOD1(BufferRender, bool(std::vector>* block)); - MOCK_METHOD1(ReportEchoLeakage, void(bool leakage_detected)); + MOCK_METHOD1(UpdateEchoLeakageStatus, void(bool leakage_detected)); }; } // namespace test diff --git a/webrtc/modules/audio_processing/aec3/mock/mock_echo_remover.h b/webrtc/modules/audio_processing/aec3/mock/mock_echo_remover.h new file mode 100644 index 0000000000..6cde439ed6 --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/mock/mock_echo_remover.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_ECHO_REMOVER_H_ +#define WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_ECHO_REMOVER_H_ + +#include + +#include "webrtc/base/optional.h" +#include "webrtc/modules/audio_processing/aec3/echo_path_variability.h" +#include "webrtc/modules/audio_processing/aec3/echo_remover.h" +#include "webrtc/test/gmock.h" + +namespace webrtc { +namespace test { + +class MockEchoRemover : public EchoRemover { + public: + virtual ~MockEchoRemover() = default; + + MOCK_METHOD5(ProcessBlock, + void(const rtc::Optional& echo_path_delay_samples, + const EchoPathVariability& echo_path_variability, + bool capture_signal_saturation, + const std::vector>& render, + std::vector>* capture)); + + MOCK_METHOD1(UpdateEchoLeakageStatus, void(bool leakage_detected)); +}; + +} // namespace test +} // namespace webrtc + +#endif // WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_ECHO_REMOVER_H_ diff --git a/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h b/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h new file mode 100644 index 0000000000..3b17dbe2fe --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_RENDER_DELAY_BUFFER_H_ +#define WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_RENDER_DELAY_BUFFER_H_ + +#include + +#include "webrtc/modules/audio_processing/aec3/aec3_constants.h" +#include "webrtc/modules/audio_processing/aec3/render_delay_buffer.h" +#include "webrtc/test/gmock.h" + +namespace webrtc { +namespace test { + +class MockRenderDelayBuffer : public RenderDelayBuffer { + public: + explicit MockRenderDelayBuffer(int sample_rate_hz) + : block_(std::vector>( + NumBandsForRate(sample_rate_hz), + std::vector(kBlockSize, 0.f))) { + ON_CALL(*this, GetNext()) + .WillByDefault( + testing::Invoke(this, &MockRenderDelayBuffer::FakeGetNext)); + } + virtual ~MockRenderDelayBuffer() = default; + + MOCK_METHOD1(Insert, bool(std::vector>* block)); + MOCK_METHOD0(GetNext, const std::vector>&()); + MOCK_METHOD1(SetDelay, void(size_t delay)); + MOCK_CONST_METHOD0(Delay, size_t()); + MOCK_CONST_METHOD0(MaxDelay, size_t()); + MOCK_CONST_METHOD0(IsBlockAvailable, bool()); + MOCK_CONST_METHOD0(MaxApiJitter, size_t()); + + private: + const std::vector>& FakeGetNext() const { return block_; } + std::vector> block_; +}; + +} // namespace test +} // namespace webrtc + +#endif // WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_RENDER_DELAY_BUFFER_H_ diff --git a/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_controller.h b/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_controller.h new file mode 100644 index 0000000000..5af2e84bda --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_controller.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_RENDER_DELAY_CONTROLLER_H_ +#define WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_RENDER_DELAY_CONTROLLER_H_ + +#include "webrtc/base/array_view.h" +#include "webrtc/base/optional.h" +#include "webrtc/modules/audio_processing/aec3/render_delay_controller.h" +#include "webrtc/test/gmock.h" + +namespace webrtc { +namespace test { + +class MockRenderDelayController : public RenderDelayController { + public: + virtual ~MockRenderDelayController() = default; + + MOCK_METHOD1(GetDelay, size_t(rtc::ArrayView capture)); + MOCK_METHOD1(AnalyzeRender, bool(rtc::ArrayView capture)); + MOCK_CONST_METHOD0(AlignmentHeadroomSamples, rtc::Optional()); +}; + +} // namespace test +} // namespace webrtc + +#endif // WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_MOCK_MOCK_RENDER_DELAY_CONTROLLER_H_ diff --git a/webrtc/modules/audio_processing/aec3/render_delay_buffer.cc b/webrtc/modules/audio_processing/aec3/render_delay_buffer.cc new file mode 100644 index 0000000000..8fc6b926b1 --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/render_delay_buffer.cc @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_processing/aec3/render_delay_buffer.h" + +#include +#include + +#include "webrtc/base/checks.h" +#include "webrtc/base/constructormagic.h" +#include "webrtc/modules/audio_processing/aec3/aec3_constants.h" +#include "webrtc/system_wrappers/include/logging.h" + +namespace webrtc { +namespace { + +class RenderDelayBufferImpl final : public RenderDelayBuffer { + public: + RenderDelayBufferImpl(size_t size_blocks, + size_t num_bands, + size_t max_api_jitter_blocks); + ~RenderDelayBufferImpl() override; + + bool Insert(std::vector>* block) override; + const std::vector>& GetNext() override; + void SetDelay(size_t delay) override; + size_t Delay() const override { return delay_; } + size_t MaxDelay() const override { + return buffer_.size() - max_api_jitter_blocks_; + } + bool IsBlockAvailable() const override { return insert_surplus_ > 0; } + size_t MaxApiJitter() const override { return max_api_jitter_blocks_; } + + private: + const size_t max_api_jitter_blocks_; + std::vector>> buffer_; + size_t last_insert_index_ = 0; + size_t delay_ = 0; + size_t insert_surplus_ = 0; + + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayBufferImpl); +}; + +RenderDelayBufferImpl::RenderDelayBufferImpl(size_t size_blocks, + size_t num_bands, + size_t max_api_jitter_blocks) + : max_api_jitter_blocks_(max_api_jitter_blocks), + buffer_(size_blocks + max_api_jitter_blocks_, + std::vector>( + num_bands, + std::vector(kBlockSize, 0.f))) {} + +RenderDelayBufferImpl::~RenderDelayBufferImpl() = default; + +bool RenderDelayBufferImpl::Insert(std::vector>* block) { + RTC_DCHECK_EQ(block->size(), buffer_[0].size()); + RTC_DCHECK_EQ((*block)[0].size(), buffer_[0][0].size()); + + if (insert_surplus_ == max_api_jitter_blocks_) { + return false; + } + last_insert_index_ = (last_insert_index_ + 1) % buffer_.size(); + block->swap(buffer_[last_insert_index_]); + + ++insert_surplus_; + + return true; +} + +const std::vector>& RenderDelayBufferImpl::GetNext() { + RTC_DCHECK(IsBlockAvailable()); + const size_t extract_index_ = + (last_insert_index_ - delay_ - insert_surplus_ + 1 + buffer_.size()) % + buffer_.size(); + RTC_DCHECK_LE(0, extract_index_); + RTC_DCHECK_GT(buffer_.size(), extract_index_); + + RTC_DCHECK_LT(0, insert_surplus_); + --insert_surplus_; + + return buffer_[extract_index_]; +} + +void RenderDelayBufferImpl::SetDelay(size_t delay) { + RTC_DCHECK_GE(MaxDelay(), delay); + delay_ = delay; +} + +} // namespace + +RenderDelayBuffer* RenderDelayBuffer::Create(size_t size_blocks, + size_t num_bands, + size_t max_api_jitter_blocks) { + return new RenderDelayBufferImpl(size_blocks, num_bands, + max_api_jitter_blocks); +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/render_delay_buffer.h b/webrtc/modules/audio_processing/aec3/render_delay_buffer.h new file mode 100644 index 0000000000..0fe2c9e677 --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/render_delay_buffer.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_RENDER_DELAY_BUFFER_H_ +#define WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_RENDER_DELAY_BUFFER_H_ + +#include +#include + +namespace webrtc { + +// Class for buffering the incoming render blocks such that these may be +// extracted with a specified delay. +class RenderDelayBuffer { + public: + static RenderDelayBuffer* Create(size_t size_blocks, + size_t num_bands, + size_t max_api_jitter_blocks); + virtual ~RenderDelayBuffer() = default; + + // Swaps a block into the buffer (the content of block is destroyed) and + // returns true if the insert is successful. + virtual bool Insert(std::vector>* block) = 0; + + // Gets a reference to the next block (having the specified buffer delay) to + // read in the buffer. This method can only be called if a block is + // available which means that that must be checked prior to the call using + // the method IsBlockAvailable(). + virtual const std::vector>& GetNext() = 0; + + // Sets the buffer delay. The delay set must be lower than the delay reported + // by MaxDelay(). + virtual void SetDelay(size_t delay) = 0; + + // Gets the buffer delay. + virtual size_t Delay() const = 0; + + // Returns the maximum allowed buffer delay increase. + virtual size_t MaxDelay() const = 0; + + // Returns whether a block is available for reading. + virtual bool IsBlockAvailable() const = 0; + + // Returns the maximum allowed api call jitter in blocks. + virtual size_t MaxApiJitter() const = 0; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_RENDER_DELAY_BUFFER_H_ diff --git a/webrtc/modules/audio_processing/aec3/render_delay_buffer_unittest.cc b/webrtc/modules/audio_processing/aec3/render_delay_buffer_unittest.cc new file mode 100644 index 0000000000..448ba0b355 --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/render_delay_buffer_unittest.cc @@ -0,0 +1,319 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_processing/aec3/render_delay_buffer.h" + +#include +#include +#include +#include + +#include "webrtc/base/array_view.h" +#include "webrtc/base/random.h" +#include "webrtc/modules/audio_processing/aec3/aec3_constants.h" +#include "webrtc/modules/audio_processing/logging/apm_data_dumper.h" +#include "webrtc/test/gtest.h" + +namespace webrtc { +namespace { + +std::string ProduceDebugText(int sample_rate_hz) { + std::ostringstream ss; + ss << "Sample rate: " << sample_rate_hz; + return ss.str(); +} + +std::string ProduceDebugText(int sample_rate_hz, size_t delay) { + std::ostringstream ss; + ss << "Sample rate: " << sample_rate_hz; + ss << ", Delay: " << delay; + return ss.str(); +} + +constexpr size_t kMaxApiCallJitter = 30; + +} // namespace + +// Verifies that the basic swap in the insert call works. +TEST(RenderDelayBuffer, InsertSwap) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + 250, NumBandsForRate(rate), kMaxApiCallJitter)); + for (size_t k = 0; k < 10; ++k) { + std::vector> block_to_insert( + NumBandsForRate(rate), std::vector(kBlockSize, k + 1)); + std::vector> reference_block = block_to_insert; + + EXPECT_TRUE(delay_buffer->Insert(&block_to_insert)); + EXPECT_NE(reference_block, block_to_insert); + } + } +} + +// Verifies that the buffer passes the blocks in a bitexact manner when the +// delay is zero. +TEST(RenderDelayBuffer, BasicBitexactness) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + 20, NumBandsForRate(rate), kMaxApiCallJitter)); + for (size_t k = 0; k < 200; ++k) { + std::vector> block_to_insert( + NumBandsForRate(rate), std::vector(kBlockSize, k)); + std::vector> reference_block = block_to_insert; + EXPECT_TRUE(delay_buffer->Insert(&block_to_insert)); + ASSERT_TRUE(delay_buffer->IsBlockAvailable()); + const std::vector>& output_block = + delay_buffer->GetNext(); + EXPECT_EQ(reference_block, output_block); + } + } +} + +// Verifies that the buffer passes the blocks in a bitexact manner when the +// delay is non-zero. +TEST(RenderDelayBuffer, BitexactnessWithNonZeroDelay) { + constexpr size_t kMaxDelay = 200; + for (auto rate : {8000, 16000, 32000, 48000}) { + for (size_t delay = 0; delay < kMaxDelay; ++delay) { + SCOPED_TRACE(ProduceDebugText(rate, delay)); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + 20 + kMaxDelay, NumBandsForRate(rate), kMaxApiCallJitter)); + delay_buffer->SetDelay(delay); + for (size_t k = 0; k < 200 + delay; ++k) { + std::vector> block_to_insert( + NumBandsForRate(rate), std::vector(kBlockSize, k)); + EXPECT_TRUE(delay_buffer->Insert(&block_to_insert)); + ASSERT_TRUE(delay_buffer->IsBlockAvailable()); + const std::vector>& output_block = + delay_buffer->GetNext(); + if (k >= delay) { + std::vector> reference_block( + NumBandsForRate(rate), std::vector(kBlockSize, k - delay)); + EXPECT_EQ(reference_block, output_block); + } + } + } + } +} + +// Verifies that the buffer passes the blocks in a bitexact manner when the +// delay is zero and there is jitter in the Insert and GetNext calls. +TEST(RenderDelayBuffer, BasicBitexactnessWithJitter) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + 20, NumBandsForRate(rate), kMaxApiCallJitter)); + for (size_t k = 0; k < kMaxApiCallJitter; ++k) { + std::vector> block_to_insert( + NumBandsForRate(rate), std::vector(kBlockSize, k)); + EXPECT_TRUE(delay_buffer->Insert(&block_to_insert)); + } + + for (size_t k = 0; k < kMaxApiCallJitter; ++k) { + std::vector> reference_block( + NumBandsForRate(rate), std::vector(kBlockSize, k)); + ASSERT_TRUE(delay_buffer->IsBlockAvailable()); + const std::vector>& output_block = + delay_buffer->GetNext(); + EXPECT_EQ(reference_block, output_block); + } + EXPECT_FALSE(delay_buffer->IsBlockAvailable()); + } +} + +// Verifies that the buffer passes the blocks in a bitexact manner when the +// delay is non-zero and there is jitter in the Insert and GetNext calls. +TEST(RenderDelayBuffer, BitexactnessWithNonZeroDelayAndJitter) { + constexpr size_t kMaxDelay = 200; + for (auto rate : {8000, 16000, 32000, 48000}) { + for (size_t delay = 0; delay < kMaxDelay; ++delay) { + SCOPED_TRACE(ProduceDebugText(rate, delay)); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + 20 + kMaxDelay, NumBandsForRate(rate), kMaxApiCallJitter)); + delay_buffer->SetDelay(delay); + for (size_t j = 0; j < 10; ++j) { + for (size_t k = 0; k < kMaxApiCallJitter; ++k) { + const size_t block_value = k + j * kMaxApiCallJitter; + std::vector> block_to_insert( + NumBandsForRate(rate), + std::vector(kBlockSize, block_value)); + EXPECT_TRUE(delay_buffer->Insert(&block_to_insert)); + } + + for (size_t k = 0; k < kMaxApiCallJitter; ++k) { + ASSERT_TRUE(delay_buffer->IsBlockAvailable()); + const std::vector>& output_block = + delay_buffer->GetNext(); + const size_t block_value = k + j * kMaxApiCallJitter; + if (block_value >= delay) { + std::vector> reference_block( + NumBandsForRate(rate), + std::vector(kBlockSize, block_value - delay)); + EXPECT_EQ(reference_block, output_block); + } + } + } + } + } +} + +// Verifies that no blocks present in the buffer are lost when the buffer is +// overflowed. +TEST(RenderDelayBuffer, BufferOverflowBitexactness) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + 20, NumBandsForRate(rate), kMaxApiCallJitter)); + for (size_t k = 0; k < kMaxApiCallJitter; ++k) { + std::vector> block_to_insert( + NumBandsForRate(rate), std::vector(kBlockSize, k)); + EXPECT_TRUE(delay_buffer->Insert(&block_to_insert)); + } + + std::vector> block_to_insert( + NumBandsForRate(rate), + std::vector(kBlockSize, kMaxApiCallJitter + 1)); + auto block_to_insert_copy = block_to_insert; + EXPECT_FALSE(delay_buffer->Insert(&block_to_insert)); + EXPECT_EQ(block_to_insert_copy, block_to_insert); + + for (size_t k = 0; k < kMaxApiCallJitter; ++k) { + std::vector> reference_block( + NumBandsForRate(rate), std::vector(kBlockSize, k)); + ASSERT_TRUE(delay_buffer->IsBlockAvailable()); + const std::vector>& output_block = + delay_buffer->GetNext(); + EXPECT_EQ(reference_block, output_block); + } + EXPECT_FALSE(delay_buffer->IsBlockAvailable()); + } +} + +// Verifies that the buffer overflow is correctly reported. +TEST(RenderDelayBuffer, BufferOverflow) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + 20, NumBandsForRate(rate), kMaxApiCallJitter)); + std::vector> block_to_insert( + NumBandsForRate(rate), std::vector(kBlockSize, 0.f)); + for (size_t k = 0; k < kMaxApiCallJitter; ++k) { + EXPECT_TRUE(delay_buffer->Insert(&block_to_insert)); + } + EXPECT_FALSE(delay_buffer->Insert(&block_to_insert)); + } +} + +// Verifies that the check for available block works. +TEST(RenderDelayBuffer, AvailableBlock) { + constexpr size_t kNumBands = 1; + std::unique_ptr delay_buffer( + RenderDelayBuffer::Create(20, kNumBands, kMaxApiCallJitter)); + EXPECT_FALSE(delay_buffer->IsBlockAvailable()); + std::vector> input_block( + kNumBands, std::vector(kBlockSize, 1.f)); + EXPECT_TRUE(delay_buffer->Insert(&input_block)); + ASSERT_TRUE(delay_buffer->IsBlockAvailable()); + delay_buffer->GetNext(); + EXPECT_FALSE(delay_buffer->IsBlockAvailable()); +} + +// Verifies that the maximum delay is computed correctly. +TEST(RenderDelayBuffer, MaxDelay) { + for (size_t max_delay = 1; max_delay < 20; ++max_delay) { + std::unique_ptr delay_buffer( + RenderDelayBuffer::Create(max_delay, 1, kMaxApiCallJitter)); + EXPECT_EQ(max_delay, delay_buffer->MaxDelay()); + } +} + +// Verifies the SetDelay method. +TEST(RenderDelayBuffer, SetDelay) { + std::unique_ptr delay_buffer( + RenderDelayBuffer::Create(20, 1, kMaxApiCallJitter)); + EXPECT_EQ(0u, delay_buffer->Delay()); + for (size_t delay = 0; delay < 20; ++delay) { + delay_buffer->SetDelay(delay); + EXPECT_EQ(delay, delay_buffer->Delay()); + } +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies the check for null insert. +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed. +TEST(RenderDelayBuffer, DISABLED_NullPointerInInsert) { + std::unique_ptr delay_buffer( + RenderDelayBuffer::Create(20, 1, kMaxApiCallJitter)); + EXPECT_DEATH(delay_buffer->Insert(nullptr), ""); +} + +// Verifies the check for feasible delay. +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed. +TEST(RenderDelayBuffer, DISABLED_WrongDelay) { + std::unique_ptr delay_buffer( + RenderDelayBuffer::Create(20, 1, kMaxApiCallJitter)); + EXPECT_DEATH(delay_buffer->SetDelay(21), ""); +} + +// Verifies the check for the number of bands in the inserted blocks. +TEST(RenderDelayBuffer, WrongNumberOfBands) { + for (auto rate : {16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + 20, NumBandsForRate(rate), kMaxApiCallJitter)); + std::vector> block_to_insert( + NumBandsForRate(rate < 48000 ? rate + 16000 : 16000), + std::vector(kBlockSize, 0.f)); + EXPECT_DEATH(delay_buffer->Insert(&block_to_insert), ""); + } +} + +// Verifies the check of the length of the inserted blocks. +TEST(RenderDelayBuffer, WrongBlockLength) { + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + 20, NumBandsForRate(rate), kMaxApiCallJitter)); + std::vector> block_to_insert( + NumBandsForRate(rate), std::vector(kBlockSize - 1, 0.f)); + EXPECT_DEATH(delay_buffer->Insert(&block_to_insert), ""); + } +} + +// Verifies the behavior when getting a block from an empty buffer. +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed. +TEST(RenderDelayBuffer, DISABLED_GetNextWithNoAvailableBlockVariant1) { + std::unique_ptr delay_buffer( + RenderDelayBuffer::Create(20, 1, kMaxApiCallJitter)); + EXPECT_DEATH(delay_buffer->GetNext(), ""); +} + +// Verifies the behavior when getting a block from an empty buffer. +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed. +TEST(RenderDelayBuffer, DISABLED_GetNextWithNoAvailableBlockVariant2) { + std::unique_ptr delay_buffer( + RenderDelayBuffer::Create(20, 1, kMaxApiCallJitter)); + std::vector> input_block( + 1, std::vector(kBlockSize, 1.f)); + EXPECT_TRUE(delay_buffer->Insert(&input_block)); + delay_buffer->GetNext(); + EXPECT_DEATH(delay_buffer->GetNext(), ""); +} + +#endif + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/render_delay_controller.cc b/webrtc/modules/audio_processing/aec3/render_delay_controller.cc new file mode 100644 index 0000000000..e18701e5ea --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/render_delay_controller.cc @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "webrtc/modules/audio_processing/aec3/render_delay_controller.h" + +#include +#include +#include +#include + +#include "webrtc/base/atomicops.h" +#include "webrtc/base/constructormagic.h" +#include "webrtc/modules/audio_processing/aec3/aec3_constants.h" +#include "webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.h" +#include "webrtc/system_wrappers/include/logging.h" + +namespace webrtc { + +namespace { + +class RenderBuffer { + public: + explicit RenderBuffer(size_t size) + : buffer_(size, std::vector(kBlockSize, 0.f)) {} + ~RenderBuffer() = default; + + bool Insert(rtc::ArrayView v) { + if (size_ >= buffer_.size() - 1) { + return false; + } + + last_insert_index_ = (last_insert_index_ + 1) % buffer_.size(); + RTC_DCHECK_EQ(buffer_[last_insert_index_].size(), v.size()); + + buffer_[last_insert_index_].clear(); + buffer_[last_insert_index_].insert(buffer_[last_insert_index_].begin(), + v.begin(), v.end()); + ++size_; + return true; + } + rtc::ArrayView Get() { + RTC_DCHECK_LT(0, size_); + --size_; + return buffer_[(last_insert_index_ - size_ + buffer_.size()) % + buffer_.size()]; + } + + size_t Size() { return size_; } + + private: + std::vector> buffer_; + size_t size_ = 0; + int last_insert_index_ = 0; +}; + +class RenderDelayControllerImpl final : public RenderDelayController { + public: + RenderDelayControllerImpl(int sample_rate_hz, + const RenderDelayBuffer& render_delay_buffer); + ~RenderDelayControllerImpl() override; + size_t GetDelay(rtc::ArrayView capture) override; + bool AnalyzeRender(rtc::ArrayView render) override; + rtc::Optional AlignmentHeadroomSamples() const override { + return headroom_samples_; + } + + private: + static int instance_count_; + std::unique_ptr data_dumper_; + const size_t max_delay_; + size_t delay_; + RenderBuffer render_buffer_; + EchoPathDelayEstimator delay_estimator_; + size_t blocks_since_last_delay_estimate_ = 300000; + int echo_path_delay_samples_ = 0; + size_t align_call_counter_ = 0; + rtc::Optional headroom_samples_; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayControllerImpl); +}; + +size_t ComputeNewBufferDelay(size_t current_delay, + size_t max_delay, + size_t echo_path_delay_samples) { + // The below division is not exact and the truncation is intended. + const int echo_path_delay_blocks = echo_path_delay_samples / kBlockSize; + constexpr int kDelayHeadroomBlocks = 1; + + // Compute the buffer delay increase required to achieve the desired latency. + size_t new_delay = std::max(echo_path_delay_blocks - kDelayHeadroomBlocks, 0); + + // Add hysteresis. + if (new_delay == current_delay + 1 || new_delay + 1 == current_delay) { + new_delay = current_delay; + } + + // Limit the delay to what is possible. + new_delay = std::min(new_delay, max_delay); + + return new_delay; +} + +int RenderDelayControllerImpl::instance_count_ = 0; + +RenderDelayControllerImpl::RenderDelayControllerImpl( + int sample_rate_hz, + const RenderDelayBuffer& render_delay_buffer) + : data_dumper_( + new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), + max_delay_(render_delay_buffer.MaxDelay()), + delay_(render_delay_buffer.Delay()), + render_buffer_(render_delay_buffer.MaxApiJitter() + 1), + delay_estimator_(data_dumper_.get(), sample_rate_hz) { + RTC_DCHECK(sample_rate_hz == 8000 || sample_rate_hz == 16000 || + sample_rate_hz == 32000 || sample_rate_hz == 48000); +} + +RenderDelayControllerImpl::~RenderDelayControllerImpl() = default; + +size_t RenderDelayControllerImpl::GetDelay( + rtc::ArrayView capture) { + RTC_DCHECK_EQ(kBlockSize, capture.size()); + if (render_buffer_.Size() == 0) { + return delay_; + } + + ++align_call_counter_; + rtc::ArrayView render = render_buffer_.Get(); + rtc::Optional echo_path_delay_samples = + delay_estimator_.EstimateDelay(render, capture); + if (echo_path_delay_samples) { + echo_path_delay_samples_ = *echo_path_delay_samples; + + // Compute and set new render delay buffer delay. + const size_t new_delay = + ComputeNewBufferDelay(delay_, max_delay_, echo_path_delay_samples_); + if (new_delay != delay_ && align_call_counter_ > 250) { + delay_ = new_delay; + } + + // Update render delay buffer headroom. + blocks_since_last_delay_estimate_ = 0; + const int headroom = echo_path_delay_samples_ - delay_ * kBlockSize; + RTC_DCHECK_LE(0, headroom); + headroom_samples_ = rtc::Optional(headroom); + } else if (++blocks_since_last_delay_estimate_ > 25000) { + headroom_samples_ = rtc::Optional(); + } + + data_dumper_->DumpRaw("aec3_render_delay_controller_delay", 1, + &echo_path_delay_samples_); + data_dumper_->DumpRaw("aec3_render_delay_controller_buffer_delay", delay_); + + return delay_; +} + +bool RenderDelayControllerImpl::AnalyzeRender( + rtc::ArrayView render) { + return render_buffer_.Insert(render); +} + +} // namespace + +RenderDelayController* RenderDelayController::Create( + int sample_rate_hz, + const RenderDelayBuffer& render_delay_buffer) { + return new RenderDelayControllerImpl(sample_rate_hz, render_delay_buffer); +} + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/aec3/render_delay_controller.h b/webrtc/modules/audio_processing/aec3/render_delay_controller.h new file mode 100644 index 0000000000..b22a5fde16 --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/render_delay_controller.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_RENDER_DELAY_CONTROLLER_H_ +#define WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_RENDER_DELAY_CONTROLLER_H_ + +#include "webrtc/base/array_view.h" +#include "webrtc/base/optional.h" +#include "webrtc/modules/audio_processing/aec3/render_delay_buffer.h" +#include "webrtc/modules/audio_processing/logging/apm_data_dumper.h" + +namespace webrtc { + +// Class for aligning the render and capture signal using a RenderDelayBuffer. +class RenderDelayController { + public: + static RenderDelayController* Create( + int sample_rate_hz, + const RenderDelayBuffer& render_delay_buffer); + virtual ~RenderDelayController() = default; + + // Aligns the render buffer content with the capture signal. + virtual size_t GetDelay(rtc::ArrayView capture) = 0; + + // Analyzes the render signal and returns false if there is a buffer overrun. + virtual bool AnalyzeRender(rtc::ArrayView render) = 0; + + // Returns an approximate value for the headroom in the buffer alignment. + virtual rtc::Optional AlignmentHeadroomSamples() const = 0; +}; +} // namespace webrtc + +#endif // WEBRTC_MODULES_AUDIO_PROCESSING_AEC3_RENDER_DELAY_CONTROLLER_H_ diff --git a/webrtc/modules/audio_processing/aec3/render_delay_controller_unittest.cc b/webrtc/modules/audio_processing/aec3/render_delay_controller_unittest.cc new file mode 100644 index 0000000000..145f3a3042 --- /dev/null +++ b/webrtc/modules/audio_processing/aec3/render_delay_controller_unittest.cc @@ -0,0 +1,264 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_processing/aec3/render_delay_controller.h" + +#include +#include +#include +#include +#include + +#include "webrtc/base/random.h" +#include "webrtc/modules/audio_processing/aec3/aec3_constants.h" +#include "webrtc/modules/audio_processing/aec3/render_delay_buffer.h" +#include "webrtc/modules/audio_processing/logging/apm_data_dumper.h" +#include "webrtc/modules/audio_processing/test/echo_canceller_test_tools.h" +#include "webrtc/test/gtest.h" + +namespace webrtc { +namespace { + +std::string ProduceDebugText(int sample_rate_hz) { + std::ostringstream ss; + ss << "Sample rate: " << sample_rate_hz; + return ss.str(); +} + +std::string ProduceDebugText(int sample_rate_hz, size_t delay) { + std::ostringstream ss(ProduceDebugText(sample_rate_hz)); + ss << ", Delay: " << delay; + return ss.str(); +} + +std::string ProduceDebugText(int sample_rate_hz, + size_t delay, + size_t max_jitter) { + std::ostringstream ss(ProduceDebugText(sample_rate_hz, delay)); + ss << ", Max Api call jitter: " << max_jitter; + return ss.str(); +} + +constexpr size_t kMaxApiCallJitter = 30; + +} // namespace + +// Verifies the output of GetDelay when there are no AnalyzeRender calls. +TEST(RenderDelayController, NoRenderSignal) { + std::vector block(kBlockSize, 0.f); + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + 250, NumBandsForRate(rate), kMaxApiCallJitter)); + std::unique_ptr delay_controller( + RenderDelayController::Create(rate, *delay_buffer)); + for (size_t k = 0; k < 100; ++k) { + EXPECT_EQ(0u, delay_controller->GetDelay(block)); + } + } +} + +// Verifies the behavior when there are too many AnalyzeRender calls. +TEST(RenderDelayController, RenderOverflow) { + std::vector block(kBlockSize, 0.f); + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr delay_buffer(RenderDelayBuffer::Create( + 250, NumBandsForRate(rate), kMaxApiCallJitter)); + std::unique_ptr delay_controller( + RenderDelayController::Create(rate, *delay_buffer)); + for (size_t k = 0; k < kMaxApiCallJitter; ++k) { + EXPECT_TRUE(delay_controller->AnalyzeRender(block)); + } + EXPECT_FALSE(delay_controller->AnalyzeRender(block)); + delay_controller->GetDelay(block); + EXPECT_TRUE(delay_controller->AnalyzeRender(block)); + } +} + +// Verifies the basic API call sequence. +TEST(RenderDelayController, BasicApiCalls) { + std::vector render_block(kBlockSize, 0.f); + std::vector capture_block(kBlockSize, 0.f); + size_t delay_blocks = 0; + for (auto rate : {8000, 16000, 32000, 48000}) { + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(50, NumBandsForRate(rate), + kMaxApiCallJitter)); + std::unique_ptr delay_controller( + RenderDelayController::Create(rate, *render_delay_buffer)); + for (size_t k = 0; k < 10; ++k) { + EXPECT_TRUE(delay_controller->AnalyzeRender(render_block)); + delay_blocks = delay_controller->GetDelay(capture_block); + } + EXPECT_FALSE(delay_controller->AlignmentHeadroomSamples()); + EXPECT_EQ(0u, delay_blocks); + } +} + +// Verifies that the RenderDelayController is able to align the signals for +// simple timeshifts between the signals. +// TODO(peah): Activate the unittest once the required code has been landed. +TEST(RenderDelayController, DISABLED_Alignment) { + Random random_generator(42U); + std::vector render_block(kBlockSize, 0.f); + std::vector capture_block(kBlockSize, 0.f); + size_t delay_blocks = 0; + for (auto rate : {8000, 16000, 32000, 48000}) { + for (size_t delay_samples : {0, 50, 150, 200, 800, 4000}) { + SCOPED_TRACE(ProduceDebugText(rate, delay_samples)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(250, NumBandsForRate(rate), + kMaxApiCallJitter)); + std::unique_ptr delay_controller( + RenderDelayController::Create(rate, *render_delay_buffer)); + DelayBuffer signal_delay_buffer(delay_samples); + for (size_t k = 0; k < (300 + delay_samples / kBlockSize); ++k) { + RandomizeSampleVector(&random_generator, render_block); + signal_delay_buffer.Delay(render_block, capture_block); + EXPECT_TRUE(delay_controller->AnalyzeRender(render_block)); + delay_blocks = delay_controller->GetDelay(capture_block); + } + + constexpr int kDelayHeadroomBlocks = 1; + size_t expected_delay_blocks = + std::max(0, static_cast(delay_samples / kBlockSize) - + kDelayHeadroomBlocks); + if (expected_delay_blocks < 2) { + expected_delay_blocks = 0; + } + + EXPECT_EQ(expected_delay_blocks, delay_blocks); + + const rtc::Optional headroom_samples = + delay_controller->AlignmentHeadroomSamples(); + ASSERT_TRUE(headroom_samples); + EXPECT_NEAR(delay_samples - delay_blocks * kBlockSize, *headroom_samples, + 4); + } + } +} + +// Verifies that the RenderDelayController is able to align the signals for +// simple timeshifts between the signals when there is jitter in the API calls. +// TODO(peah): Activate the unittest once the required code has been landed. +TEST(RenderDelayController, DISABLED_AlignmentWithJitter) { + Random random_generator(42U); + std::vector render_block(kBlockSize, 0.f); + std::vector capture_block(kBlockSize, 0.f); + for (auto rate : {8000, 16000, 32000, 48000}) { + for (size_t delay_samples : {0, 50, 800}) { + for (size_t max_jitter : {1, 9, 20}) { + size_t delay_blocks = 0; + SCOPED_TRACE(ProduceDebugText(rate, delay_samples, max_jitter)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(250, NumBandsForRate(rate), max_jitter)); + std::unique_ptr delay_controller( + RenderDelayController::Create(rate, *render_delay_buffer)); + DelayBuffer signal_delay_buffer(delay_samples); + for (size_t j = 0; + j < (300 + delay_samples / kBlockSize) / max_jitter + 1; ++j) { + std::vector> capture_block_buffer; + for (size_t k = 0; k < max_jitter; ++k) { + RandomizeSampleVector(&random_generator, render_block); + signal_delay_buffer.Delay(render_block, capture_block); + capture_block_buffer.push_back(capture_block); + EXPECT_TRUE(delay_controller->AnalyzeRender(render_block)); + } + for (size_t k = 0; k < max_jitter; ++k) { + delay_blocks = delay_controller->GetDelay(capture_block_buffer[k]); + } + } + + constexpr int kDelayHeadroomBlocks = 1; + size_t expected_delay_blocks = + std::max(0, static_cast(delay_samples / kBlockSize) - + kDelayHeadroomBlocks); + if (expected_delay_blocks < 2) { + expected_delay_blocks = 0; + } + + EXPECT_EQ(expected_delay_blocks, delay_blocks); + + const rtc::Optional headroom_samples = + delay_controller->AlignmentHeadroomSamples(); + ASSERT_TRUE(headroom_samples); + EXPECT_NEAR(delay_samples - delay_blocks * kBlockSize, + *headroom_samples, 4); + } + } + } +} + +// Verifies the initial value for the AlignmentHeadroomSamples. +TEST(RenderDelayController, InitialHeadroom) { + std::vector render_block(kBlockSize, 0.f); + std::vector capture_block(kBlockSize, 0.f); + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(250, NumBandsForRate(rate), + kMaxApiCallJitter)); + std::unique_ptr delay_controller( + RenderDelayController::Create(rate, *render_delay_buffer)); + EXPECT_FALSE(delay_controller->AlignmentHeadroomSamples()); + } +} + +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +// Verifies the check for the capture signal block size. +TEST(RenderDelayController, WrongCaptureSize) { + std::vector block(kBlockSize - 1, 0.f); + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(250, NumBandsForRate(rate), + kMaxApiCallJitter)); + EXPECT_DEATH(std::unique_ptr( + RenderDelayController::Create(rate, *render_delay_buffer)) + ->GetDelay(block), + ""); + } +} + +// Verifies the check for the render signal block size. +// TODO(peah): Re-enable the test once the issue with memory leaks during DEATH +// tests on test bots has been fixed. +TEST(RenderDelayController, DISABLED_WrongRenderSize) { + std::vector block(kBlockSize - 1, 0.f); + for (auto rate : {8000, 16000, 32000, 48000}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(250, NumBandsForRate(rate), + kMaxApiCallJitter)); + EXPECT_DEATH(std::unique_ptr( + RenderDelayController::Create(rate, *render_delay_buffer)) + ->AnalyzeRender(block), + ""); + } +} + +// Verifies the check for correct sample rate. +TEST(RenderDelayController, WrongSampleRate) { + for (auto rate : {-1, 0, 8001, 16001}) { + SCOPED_TRACE(ProduceDebugText(rate)); + std::unique_ptr render_delay_buffer( + RenderDelayBuffer::Create(10, NumBandsForRate(rate), + kMaxApiCallJitter)); + EXPECT_DEATH(std::unique_ptr( + RenderDelayController::Create(rate, *render_delay_buffer)), + ""); + } +} + +#endif + +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/logging/apm_data_dumper.h b/webrtc/modules/audio_processing/logging/apm_data_dumper.h index 691c4cec5b..e342fab365 100644 --- a/webrtc/modules/audio_processing/logging/apm_data_dumper.h +++ b/webrtc/modules/audio_processing/logging/apm_data_dumper.h @@ -57,7 +57,34 @@ class ApmDataDumper { // Methods for performing dumping of data of various types into // various formats. - void DumpRaw(const char* name, int v_length, const float* v) { + void DumpRaw(const char* name, double v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + FILE* file = GetRawFile(name); + fwrite(&v, sizeof(v), 1, file); +#endif + } + + void DumpRaw(const char* name, size_t v_length, const double* v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + FILE* file = GetRawFile(name); + fwrite(v, sizeof(v[0]), v_length, file); +#endif + } + + void DumpRaw(const char* name, rtc::ArrayView v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + DumpRaw(name, v.size(), v.data()); +#endif + } + + void DumpRaw(const char* name, float v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + FILE* file = GetRawFile(name); + fwrite(&v, sizeof(v), 1, file); +#endif + } + + void DumpRaw(const char* name, size_t v_length, const float* v) { #if WEBRTC_APM_DEBUG_DUMP == 1 FILE* file = GetRawFile(name); fwrite(v, sizeof(v[0]), v_length, file); @@ -70,10 +97,16 @@ class ApmDataDumper { #endif } - void DumpRaw(const char* name, int v_length, const bool* v) { + void DumpRaw(const char* name, bool v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + DumpRaw(name, static_cast(v)); +#endif + } + + void DumpRaw(const char* name, size_t v_length, const bool* v) { #if WEBRTC_APM_DEBUG_DUMP == 1 FILE* file = GetRawFile(name); - for (int k = 0; k < v_length; ++k) { + for (size_t k = 0; k < v_length; ++k) { int16_t value = static_cast(v[k]); fwrite(&value, sizeof(value), 1, file); } @@ -86,7 +119,14 @@ class ApmDataDumper { #endif } - void DumpRaw(const char* name, int v_length, const int16_t* v) { + void DumpRaw(const char* name, int16_t v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + FILE* file = GetRawFile(name); + fwrite(&v, sizeof(v), 1, file); +#endif + } + + void DumpRaw(const char* name, size_t v_length, const int16_t* v) { #if WEBRTC_APM_DEBUG_DUMP == 1 FILE* file = GetRawFile(name); fwrite(v, sizeof(v[0]), v_length, file); @@ -99,7 +139,28 @@ class ApmDataDumper { #endif } - void DumpRaw(const char* name, int v_length, const int32_t* v) { + void DumpRaw(const char* name, int32_t v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + FILE* file = GetRawFile(name); + fwrite(&v, sizeof(v), 1, file); +#endif + } + + void DumpRaw(const char* name, size_t v_length, const int32_t* v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + FILE* file = GetRawFile(name); + fwrite(v, sizeof(v[0]), v_length, file); +#endif + } + + void DumpRaw(const char* name, size_t v) { +#if WEBRTC_APM_DEBUG_DUMP == 1 + FILE* file = GetRawFile(name); + fwrite(&v, sizeof(v), 1, file); +#endif + } + + void DumpRaw(const char* name, size_t v_length, const size_t* v) { #if WEBRTC_APM_DEBUG_DUMP == 1 FILE* file = GetRawFile(name); fwrite(v, sizeof(v[0]), v_length, file); @@ -113,7 +174,7 @@ class ApmDataDumper { } void DumpWav(const char* name, - int v_length, + size_t v_length, const float* v, int sample_rate_hz, int num_channels) { diff --git a/webrtc/modules/audio_processing/test/echo_canceller_test_tools.cc b/webrtc/modules/audio_processing/test/echo_canceller_test_tools.cc new file mode 100644 index 0000000000..59f1da9dea --- /dev/null +++ b/webrtc/modules/audio_processing/test/echo_canceller_test_tools.cc @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_processing/test/echo_canceller_test_tools.h" + +#include "webrtc/base/checks.h" + +namespace webrtc { + +void RandomizeSampleVector(Random* random_generator, rtc::ArrayView v) { + for (auto& v_k : v) { + v_k = 2 * 32767.f * random_generator->Rand() - 32767.f; + } +} + +template +void DelayBuffer::Delay(rtc::ArrayView x, + rtc::ArrayView x_delayed) { + RTC_DCHECK_EQ(x.size(), x_delayed.size()); + if (buffer_.size() == 0) { + std::copy(x.begin(), x.end(), x_delayed.begin()); + } else { + for (size_t k = 0; k < x.size(); ++k) { + x_delayed[k] = buffer_[next_insert_index_]; + buffer_[next_insert_index_] = x[k]; + next_insert_index_ = (next_insert_index_ + 1) % buffer_.size(); + } + } +} + +template class DelayBuffer; +template class DelayBuffer; +} // namespace webrtc diff --git a/webrtc/modules/audio_processing/test/echo_canceller_test_tools.h b/webrtc/modules/audio_processing/test/echo_canceller_test_tools.h new file mode 100644 index 0000000000..59e1cadebb --- /dev/null +++ b/webrtc/modules/audio_processing/test/echo_canceller_test_tools.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_AUDIO_PROCESSING_TEST_ECHO_CANCELLER_TEST_TOOLS_H_ +#define WEBRTC_MODULES_AUDIO_PROCESSING_TEST_ECHO_CANCELLER_TEST_TOOLS_H_ + +#include +#include + +#include "webrtc/base/array_view.h" +#include "webrtc/base/constructormagic.h" +#include "webrtc/base/random.h" + +namespace webrtc { + +// Randomizes the elements in a vector with values -32767.f:32767.f. +void RandomizeSampleVector(Random* random_generator, rtc::ArrayView v); + +// Class for delaying a signal a fixed number of samples. +template +class DelayBuffer { + public: + explicit DelayBuffer(size_t delay) : buffer_(delay) {} + ~DelayBuffer() = default; + + // Produces a delayed signal copy of x. + void Delay(rtc::ArrayView x, rtc::ArrayView x_delayed); + + private: + std::vector buffer_; + size_t next_insert_index_ = 0; + RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(DelayBuffer); +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_AUDIO_PROCESSING_TEST_ECHO_CANCELLER_TEST_TOOLS_H_ diff --git a/webrtc/modules/audio_processing/test/echo_canceller_test_tools_unittest.cc b/webrtc/modules/audio_processing/test/echo_canceller_test_tools_unittest.cc new file mode 100644 index 0000000000..18223fd480 --- /dev/null +++ b/webrtc/modules/audio_processing/test/echo_canceller_test_tools_unittest.cc @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/audio_processing/test/echo_canceller_test_tools.h" + +#include + +#include "webrtc/base/array_view.h" +#include "webrtc/base/checks.h" +#include "webrtc/base/random.h" +#include "webrtc/test/gtest.h" + +namespace webrtc { + +TEST(EchoCancellerTestTools, FloatDelayBuffer) { + constexpr size_t kDelay = 10; + DelayBuffer delay_buffer(kDelay); + std::vector v(1000, 0.f); + for (size_t k = 0; k < v.size(); ++k) { + v[k] = k; + } + std::vector v_delayed = v; + constexpr size_t kBlockSize = 50; + for (size_t k = 0; k < rtc::CheckedDivExact(v.size(), kBlockSize); ++k) { + delay_buffer.Delay( + rtc::ArrayView(&v[k * kBlockSize], kBlockSize), + rtc::ArrayView(&v_delayed[k * kBlockSize], kBlockSize)); + } + for (size_t k = kDelay; k < v.size(); ++k) { + EXPECT_EQ(v[k - kDelay], v_delayed[k]); + } +} + +TEST(EchoCancellerTestTools, IntDelayBuffer) { + constexpr size_t kDelay = 10; + DelayBuffer delay_buffer(kDelay); + std::vector v(1000, 0); + for (size_t k = 0; k < v.size(); ++k) { + v[k] = k; + } + std::vector v_delayed = v; + const size_t kBlockSize = 50; + for (size_t k = 0; k < rtc::CheckedDivExact(v.size(), kBlockSize); ++k) { + delay_buffer.Delay( + rtc::ArrayView(&v[k * kBlockSize], kBlockSize), + rtc::ArrayView(&v_delayed[k * kBlockSize], kBlockSize)); + } + for (size_t k = kDelay; k < v.size(); ++k) { + EXPECT_EQ(v[k - kDelay], v_delayed[k]); + } +} + +TEST(EchoCancellerTestTools, RandomizeSampleVector) { + Random random_generator(42U); + std::vector v(50, 0.f); + std::vector v_ref = v; + RandomizeSampleVector(&random_generator, v); + EXPECT_NE(v, v_ref); + v_ref = v; + RandomizeSampleVector(&random_generator, v); + EXPECT_NE(v, v_ref); +} + +} // namespace webrtc