admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AEC3: Remove legacy render buffering

Browse Source 
This CL removes the legacy, no longer used, render buffering code. It
also removes four unused parameters from the AEC3 config. The change
is tested for bit-exactness.

Bug: webrtc:8671
Change-Id: I2bb6cb7a1097863f228767d757d551c00593bb00
Reviewed-on: https://webrtc-review.googlesource.com/c/119701
Reviewed-by: Henrik Lundin <henrik.lundin@webrtc.org>
Reviewed-by: Per Åhgren <peah@webrtc.org>
Commit-Queue: Gustaf Ullberg <gustaf@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26399}
This commit is contained in:
Gustaf Ullberg 2019-01-24 16:00:57 +01:00 committed by Commit Bot
parent 8a40edd802
commit e47433f017
31 changed files with 263 additions and 1422 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/audio/echo_canceller3_config.cc
View File

@ -94,12 +94,9 @@ bool EchoCanceller3Config::Validate(EchoCanceller3Config* config) {
}
res = res & Limit(&c->delay.default_delay, 0, 5000);
res = res & Limit(&c->delay.num_filters, 0, 5000);
res = res & Limit(&c->delay.api_call_jitter_blocks, 1, 5000);
res = res & Limit(&c->delay.min_echo_path_delay_blocks, 0, 5000);
res = res & Limit(&c->delay.delay_headroom_blocks, 0, 5000);
res = res & Limit(&c->delay.hysteresis_limit_1_blocks, 0, 5000);
res = res & Limit(&c->delay.hysteresis_limit_2_blocks, 0, 5000);
res = res & Limit(&c->delay.skew_hysteresis_blocks, 0, 5000);
res = res & Limit(&c->delay.fixed_capture_delay_samples, 0, 5000);
res = res & Limit(&c->delay.delay_estimate_smoothing, 0.f, 1.f);
res = res & Limit(&c->delay.delay_candidate_detection_threshold, 0.f, 1.f);

4
api/audio/echo_canceller3_config.h
View File

@ -27,7 +27,6 @@ struct RTC_EXPORT EchoCanceller3Config {
EchoCanceller3Config(const EchoCanceller3Config& e);
struct Buffering {
bool use_new_render_buffering = true;
size_t excess_render_detection_interval_blocks = 250;
size_t max_allowed_excess_render_blocks = 8;
} buffering;
@ -38,12 +37,9 @@ struct RTC_EXPORT EchoCanceller3Config {
size_t default_delay = 5;
size_t down_sampling_factor = 4;
size_t num_filters = 5;
size_t api_call_jitter_blocks = 26;
size_t min_echo_path_delay_blocks = 0;
size_t delay_headroom_blocks = 2;
size_t hysteresis_limit_1_blocks = 1;
size_t hysteresis_limit_2_blocks = 1;
size_t skew_hysteresis_blocks = 3;
size_t fixed_capture_delay_samples = 0;
float delay_estimate_smoothing = 0.7f;
float delay_candidate_detection_threshold = 0.2f;

14
api/audio/echo_canceller3_config_json.cc
View File

@ -138,8 +138,6 @@ void Aec3ConfigFromJsonString(absl::string_view json_string,
Json::Value section;
if (rtc::GetValueFromJsonObject(root, "buffering", &section)) {
ReadParam(section, "use_new_render_buffering",
&cfg.buffering.use_new_render_buffering);
ReadParam(section, "excess_render_detection_interval_blocks",
&cfg.buffering.excess_render_detection_interval_blocks);
ReadParam(section, "max_allowed_excess_render_blocks",
@ -150,18 +148,12 @@ void Aec3ConfigFromJsonString(absl::string_view json_string,
ReadParam(section, "default_delay", &cfg.delay.default_delay);
ReadParam(section, "down_sampling_factor", &cfg.delay.down_sampling_factor);
ReadParam(section, "num_filters", &cfg.delay.num_filters);
ReadParam(section, "api_call_jitter_blocks",
&cfg.delay.api_call_jitter_blocks);
ReadParam(section, "min_echo_path_delay_blocks",
&cfg.delay.min_echo_path_delay_blocks);
ReadParam(section, "delay_headroom_blocks",
&cfg.delay.delay_headroom_blocks);
ReadParam(section, "hysteresis_limit_1_blocks",
&cfg.delay.hysteresis_limit_1_blocks);
ReadParam(section, "hysteresis_limit_2_blocks",
&cfg.delay.hysteresis_limit_2_blocks);
ReadParam(section, "skew_hysteresis_blocks",
&cfg.delay.skew_hysteresis_blocks);
ReadParam(section, "fixed_capture_delay_samples",
&cfg.delay.fixed_capture_delay_samples);
ReadParam(section, "delay_estimate_smoothing",
@ -354,18 +346,12 @@ std::string Aec3ConfigToJsonString(const EchoCanceller3Config& config) {
ost << "\"down_sampling_factor\": " << config.delay.down_sampling_factor
<< ",";
ost << "\"num_filters\": " << config.delay.num_filters << ",";
ost << "\"api_call_jitter_blocks\": " << config.delay.api_call_jitter_blocks
<< ",";
ost << "\"min_echo_path_delay_blocks\": "
<< config.delay.min_echo_path_delay_blocks << ",";
ost << "\"delay_headroom_blocks\": " << config.delay.delay_headroom_blocks
<< ",";
ost << "\"hysteresis_limit_1_blocks\": "
<< config.delay.hysteresis_limit_1_blocks << ",";
ost << "\"hysteresis_limit_2_blocks\": "
<< config.delay.hysteresis_limit_2_blocks << ",";
ost << "\"skew_hysteresis_blocks\": " << config.delay.skew_hysteresis_blocks
<< ",";
ost << "\"fixed_capture_delay_samples\": "
<< config.delay.fixed_capture_delay_samples << ",";
ost << "\"delay_estimate_smoothing\": "

3
modules/audio_processing/aec3/BUILD.gn
View File

@ -28,7 +28,6 @@ rtc_static_library("aec3") {
"block_framer.h",
"block_processor.cc",
"block_processor.h",
"block_processor2.cc",
"block_processor_metrics.cc",
"block_processor_metrics.h",
"cascaded_biquad_filter.cc",
@ -81,10 +80,8 @@ rtc_static_library("aec3") {
"render_buffer.h",
"render_delay_buffer.cc",
"render_delay_buffer.h",
"render_delay_buffer2.cc",
"render_delay_controller.cc",
"render_delay_controller.h",
"render_delay_controller2.cc",
"render_delay_controller_metrics.cc",
"render_delay_controller_metrics.h",
"render_reverb_model.cc",

1
modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc
View File

@ -317,7 +317,6 @@ TEST(AdaptiveFirFilter, FilterAndAdapt) {
config.filter.config_change_duration_blocks,
DetectOptimization(), &data_dumper);
Aec3Fft fft;
config.delay.min_echo_path_delay_blocks = 0;
config.delay.default_delay = 1;
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create(config, 3));

4
modules/audio_processing/aec3/aec_state_unittest.cc
View File

@ -25,7 +25,7 @@ TEST(AecState, NormalUsage) {
absl::optional<DelayEstimate> delay_estimate =
DelayEstimate(DelayEstimate::Quality::kRefined, 10);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
std::array<float, kFftLengthBy2Plus1> E2_main = {};
std::array<float, kFftLengthBy2Plus1> Y2 = {};
std::vector<std::vector<float>> x(3, std::vector<float>(kBlockSize, 0.f));
@ -179,7 +179,7 @@ TEST(AecState, ConvergedFilterDelay) {
EchoCanceller3Config config;
AecState state(config);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
absl::optional<DelayEstimate> delay_estimate;
std::array<float, kFftLengthBy2Plus1> E2_main;
std::array<float, kFftLengthBy2Plus1> Y2;

86
modules/audio_processing/aec3/block_processor.cc
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
@ -7,19 +7,25 @@
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_processing/aec3/block_processor.h"
#include <stddef.h>
#include <memory>
#include <utility>
#include <vector>
#include "absl/types/optional.h"
#include "api/audio/echo_canceller3_config.h"
#include "api/audio/echo_control.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/block_processor.h"
#include "modules/audio_processing/aec3/block_processor_metrics.h"
#include "modules/audio_processing/aec3/delay_estimate.h"
#include "modules/audio_processing/aec3/echo_path_variability.h"
#include "modules/audio_processing/aec3/echo_remover.h"
#include "modules/audio_processing/aec3/render_delay_buffer.h"
#include "modules/audio_processing/aec3/render_delay_controller.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/atomic_ops.h"
#include "rtc_base/checks.h"
#include "rtc_base/constructor_magic.h"
#include "rtc_base/logging.h"
namespace webrtc {
@ -35,6 +41,8 @@ class BlockProcessorImpl final : public BlockProcessor {
std::unique_ptr<RenderDelayController> delay_controller,
std::unique_ptr<EchoRemover> echo_remover);
BlockProcessorImpl() = delete;
~BlockProcessorImpl() override;
void ProcessCapture(bool echo_path_gain_change,
@ -64,7 +72,6 @@ class BlockProcessorImpl final : public BlockProcessor {
size_t capture_call_counter_ = 0;
absl::optional<DelayEstimate> estimated_delay_;
absl::optional<int> echo_remover_delay_;
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(BlockProcessorImpl);
};
int BlockProcessorImpl::instance_count_ = 0;
@ -127,35 +134,16 @@ void BlockProcessorImpl::ProcessCapture(
RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun at block "
<< capture_call_counter_;
}
render_event_ = RenderDelayBuffer::BufferingEvent::kNone;
// Update the render buffers with any newly arrived render blocks and prepare
// the render buffers for reading the render data corresponding to the current
// capture block.
render_event_ = render_buffer_->PrepareCaptureProcessing();
RTC_DCHECK(RenderDelayBuffer::BufferingEvent::kRenderOverrun !=
render_event_);
if (render_event_ == RenderDelayBuffer::BufferingEvent::kRenderUnderrun) {
if (estimated_delay_ &&
estimated_delay_->quality == DelayEstimate::Quality::kRefined) {
echo_path_variability.delay_change =
EchoPathVariability::DelayAdjustment::kDelayReset;
delay_controller_->Reset(true);
capture_properly_started_ = false;
render_properly_started_ = false;
RTC_LOG(LS_WARNING) << "Reset due to render buffer underrun at block "
<< capture_call_counter_;
}
} else if (render_event_ == RenderDelayBuffer::BufferingEvent::kApiCallSkew) {
// There have been too many render calls in a row. Reset to avoid noncausal
// echo.
echo_path_variability.delay_change =
EchoPathVariability::DelayAdjustment::kDelayReset;
delay_controller_->Reset(true);
capture_properly_started_ = false;
render_properly_started_ = false;
RTC_LOG(LS_WARNING) << "Reset due to render buffer api skew at block "
<< capture_call_counter_;
RenderDelayBuffer::BufferingEvent buffer_event =
render_buffer_->PrepareCaptureProcessing();
// Reset the delay controller at render buffer underrun.
if (buffer_event == RenderDelayBuffer::BufferingEvent::kRenderUnderrun) {
delay_controller_->Reset(false);
}
data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize,
@ -169,28 +157,12 @@ void BlockProcessorImpl::ProcessCapture(
echo_remover_delay_, (*capture_block)[0]);
if (estimated_delay_) {
if (render_buffer_->CausalDelay(estimated_delay_->delay)) {
bool delay_change = render_buffer_->SetDelay(estimated_delay_->delay);
if (delay_change) {
RTC_LOG(LS_WARNING) << "Delay changed to " << estimated_delay_->delay
<< " at block " << capture_call_counter_;
echo_path_variability.delay_change =
EchoPathVariability::DelayAdjustment::kNewDetectedDelay;
}
} else {
// A noncausal delay has been detected. This can only happen if there is
// clockdrift, an audio pipeline issue has occurred, an unreliable delay
// estimate is used or the specified minimum delay is too short.
if (estimated_delay_->quality == DelayEstimate::Quality::kRefined) {
echo_path_variability.delay_change =
EchoPathVariability::DelayAdjustment::kDelayReset;
delay_controller_->Reset(true);
render_buffer_->Reset();
capture_properly_started_ = false;
render_properly_started_ = false;
RTC_LOG(LS_WARNING) << "Reset due to noncausal delay at block "
<< capture_call_counter_;
}
bool delay_change = render_buffer_->SetDelay(estimated_delay_->delay);
if (delay_change) {
RTC_LOG(LS_WARNING) << "Delay changed to " << estimated_delay_->delay
<< " at block " << capture_call_counter_;
echo_path_variability.delay_change =
EchoPathVariability::DelayAdjustment::kNewDetectedDelay;
}
}
@ -207,8 +179,6 @@ void BlockProcessorImpl::ProcessCapture(
// Update the metrics.
metrics_.UpdateCapture(false);
render_event_ = RenderDelayBuffer::BufferingEvent::kNone;
}
void BlockProcessorImpl::BufferRender(
@ -253,9 +223,7 @@ BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config,
std::unique_ptr<RenderDelayBuffer> render_buffer(
RenderDelayBuffer::Create(config, NumBandsForRate(sample_rate_hz)));
std::unique_ptr<RenderDelayController> delay_controller(
RenderDelayController::Create(
config, RenderDelayBuffer::DelayEstimatorOffset(config),
sample_rate_hz));
RenderDelayController::Create(config, sample_rate_hz));
std::unique_ptr<EchoRemover> echo_remover(
EchoRemover::Create(config, sample_rate_hz));
return Create(config, sample_rate_hz, std::move(render_buffer),
@ -267,9 +235,7 @@ BlockProcessor* BlockProcessor::Create(
int sample_rate_hz,
std::unique_ptr<RenderDelayBuffer> render_buffer) {
std::unique_ptr<RenderDelayController> delay_controller(
RenderDelayController::Create(
config, RenderDelayBuffer::DelayEstimatorOffset(config),
sample_rate_hz));
RenderDelayController::Create(config, sample_rate_hz));
std::unique_ptr<EchoRemover> echo_remover(
EchoRemover::Create(config, sample_rate_hz));
return Create(config, sample_rate_hz, std::move(render_buffer),

14
modules/audio_processing/aec3/block_processor.h
View File

@ -26,33 +26,19 @@ namespace webrtc {
// Class for performing echo cancellation on 64 sample blocks of audio data.
class BlockProcessor {
public:
// Create a block processor with the legacy render buffering.
static BlockProcessor* Create(const EchoCanceller3Config& config,
int sample_rate_hz);
// Create a block processor with the new render buffering.
static BlockProcessor* Create2(const EchoCanceller3Config& config,
int sample_rate_hz);
// Only used for testing purposes.
static BlockProcessor* Create(
const EchoCanceller3Config& config,
int sample_rate_hz,
std::unique_ptr<RenderDelayBuffer> render_buffer);
static BlockProcessor* Create2(
const EchoCanceller3Config& config,
int sample_rate_hz,
std::unique_ptr<RenderDelayBuffer> render_buffer);
static BlockProcessor* Create(
const EchoCanceller3Config& config,
int sample_rate_hz,
std::unique_ptr<RenderDelayBuffer> render_buffer,
std::unique_ptr<RenderDelayController> delay_controller,
std::unique_ptr<EchoRemover> echo_remover);
static BlockProcessor* Create2(
const EchoCanceller3Config& config,
int sample_rate_hz,
std::unique_ptr<RenderDelayBuffer> render_buffer,
std::unique_ptr<RenderDelayController> delay_controller,
std::unique_ptr<EchoRemover> echo_remover);
virtual ~BlockProcessor() = default;

256
modules/audio_processing/aec3/block_processor2.cc
View File

@ -1,256 +0,0 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <stddef.h>
#include <memory>
#include <utility>
#include <vector>
#include "absl/types/optional.h"
#include "api/audio/echo_canceller3_config.h"
#include "api/audio/echo_control.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/block_processor.h"
#include "modules/audio_processing/aec3/block_processor_metrics.h"
#include "modules/audio_processing/aec3/delay_estimate.h"
#include "modules/audio_processing/aec3/echo_path_variability.h"
#include "modules/audio_processing/aec3/echo_remover.h"
#include "modules/audio_processing/aec3/render_delay_buffer.h"
#include "modules/audio_processing/aec3/render_delay_controller.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/atomic_ops.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
namespace webrtc {
namespace {
enum class BlockProcessorApiCall { kCapture, kRender };
class BlockProcessorImpl2 final : public BlockProcessor {
public:
BlockProcessorImpl2(const EchoCanceller3Config& config,
int sample_rate_hz,
std::unique_ptr<RenderDelayBuffer> render_buffer,
std::unique_ptr<RenderDelayController> delay_controller,
std::unique_ptr<EchoRemover> echo_remover);
BlockProcessorImpl2() = delete;
~BlockProcessorImpl2() override;
void ProcessCapture(bool echo_path_gain_change,
bool capture_signal_saturation,
std::vector<std::vector<float>>* capture_block) override;
void BufferRender(const std::vector<std::vector<float>>& block) override;
void UpdateEchoLeakageStatus(bool leakage_detected) override;
void GetMetrics(EchoControl::Metrics* metrics) const override;
void SetAudioBufferDelay(size_t delay_ms) override;
private:
static int instance_count_;
std::unique_ptr<ApmDataDumper> data_dumper_;
const EchoCanceller3Config config_;
bool capture_properly_started_ = false;
bool render_properly_started_ = false;
const size_t sample_rate_hz_;
std::unique_ptr<RenderDelayBuffer> render_buffer_;
std::unique_ptr<RenderDelayController> delay_controller_;
std::unique_ptr<EchoRemover> echo_remover_;
BlockProcessorMetrics metrics_;
RenderDelayBuffer::BufferingEvent render_event_;
size_t capture_call_counter_ = 0;
absl::optional<DelayEstimate> estimated_delay_;
absl::optional<int> echo_remover_delay_;
};
int BlockProcessorImpl2::instance_count_ = 0;
BlockProcessorImpl2::BlockProcessorImpl2(
const EchoCanceller3Config& config,
int sample_rate_hz,
std::unique_ptr<RenderDelayBuffer> render_buffer,
std::unique_ptr<RenderDelayController> delay_controller,
std::unique_ptr<EchoRemover> echo_remover)
: data_dumper_(
new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
config_(config),
sample_rate_hz_(sample_rate_hz),
render_buffer_(std::move(render_buffer)),
delay_controller_(std::move(delay_controller)),
echo_remover_(std::move(echo_remover)),
render_event_(RenderDelayBuffer::BufferingEvent::kNone) {
RTC_DCHECK(ValidFullBandRate(sample_rate_hz_));
}
BlockProcessorImpl2::~BlockProcessorImpl2() = default;
void BlockProcessorImpl2::ProcessCapture(
bool echo_path_gain_change,
bool capture_signal_saturation,
std::vector<std::vector<float>>* capture_block) {
RTC_DCHECK(capture_block);
RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), capture_block->size());
RTC_DCHECK_EQ(kBlockSize, (*capture_block)[0].size());
capture_call_counter_++;
data_dumper_->DumpRaw("aec3_processblock_call_order",
static_cast<int>(BlockProcessorApiCall::kCapture));
data_dumper_->DumpWav("aec3_processblock_capture_input", kBlockSize,
&(*capture_block)[0][0],
LowestBandRate(sample_rate_hz_), 1);
if (render_properly_started_) {
if (!capture_properly_started_) {
capture_properly_started_ = true;
render_buffer_->Reset();
delay_controller_->Reset(true);
}
} else {
// If no render data has yet arrived, do not process the capture signal.
return;
}
EchoPathVariability echo_path_variability(
echo_path_gain_change, EchoPathVariability::DelayAdjustment::kNone,
false);
if (render_event_ == RenderDelayBuffer::BufferingEvent::kRenderOverrun &&
render_properly_started_) {
echo_path_variability.delay_change =
EchoPathVariability::DelayAdjustment::kBufferFlush;
delay_controller_->Reset(true);
RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun at block "
<< capture_call_counter_;
}
render_event_ = RenderDelayBuffer::BufferingEvent::kNone;
// Update the render buffers with any newly arrived render blocks and prepare
// the render buffers for reading the render data corresponding to the current
// capture block.
RenderDelayBuffer::BufferingEvent buffer_event =
render_buffer_->PrepareCaptureProcessing();
// Reset the delay controller at render buffer underrun.
if (buffer_event == RenderDelayBuffer::BufferingEvent::kRenderUnderrun) {
delay_controller_->Reset(false);
}
data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize,
&(*capture_block)[0][0],
LowestBandRate(sample_rate_hz_), 1);
// Compute and and apply the render delay required to achieve proper signal
// alignment.
estimated_delay_ = delay_controller_->GetDelay(
render_buffer_->GetDownsampledRenderBuffer(), render_buffer_->Delay(),
echo_remover_delay_, (*capture_block)[0]);
if (estimated_delay_) {
bool delay_change = render_buffer_->SetDelay(estimated_delay_->delay);
if (delay_change) {
RTC_LOG(LS_WARNING) << "Delay changed to " << estimated_delay_->delay
<< " at block " << capture_call_counter_;
echo_path_variability.delay_change =
EchoPathVariability::DelayAdjustment::kNewDetectedDelay;
}
}
echo_path_variability.clock_drift = delay_controller_->HasClockdrift();
// Remove the echo from the capture signal.
echo_remover_->ProcessCapture(
echo_path_variability, capture_signal_saturation, estimated_delay_,
render_buffer_->GetRenderBuffer(), capture_block);
// Check to see if a refined delay estimate has been obtained from the echo
// remover.
echo_remover_delay_ = echo_remover_->Delay();
// Update the metrics.
metrics_.UpdateCapture(false);
}
void BlockProcessorImpl2::BufferRender(
const std::vector<std::vector<float>>& block) {
RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), block.size());
RTC_DCHECK_EQ(kBlockSize, block[0].size());
data_dumper_->DumpRaw("aec3_processblock_call_order",
static_cast<int>(BlockProcessorApiCall::kRender));
data_dumper_->DumpWav("aec3_processblock_render_input", kBlockSize,
&block[0][0], LowestBandRate(sample_rate_hz_), 1);
data_dumper_->DumpWav("aec3_processblock_render_input2", kBlockSize,
&block[0][0], LowestBandRate(sample_rate_hz_), 1);
render_event_ = render_buffer_->Insert(block);
metrics_.UpdateRender(render_event_ !=
RenderDelayBuffer::BufferingEvent::kNone);
render_properly_started_ = true;
delay_controller_->LogRenderCall();
}
void BlockProcessorImpl2::UpdateEchoLeakageStatus(bool leakage_detected) {
echo_remover_->UpdateEchoLeakageStatus(leakage_detected);
}
void BlockProcessorImpl2::GetMetrics(EchoControl::Metrics* metrics) const {
echo_remover_->GetMetrics(metrics);
const int block_size_ms = sample_rate_hz_ == 8000 ? 8 : 4;
absl::optional<size_t> delay = render_buffer_->Delay();
metrics->delay_ms = delay ? static_cast<int>(*delay) * block_size_ms : 0;
}
void BlockProcessorImpl2::SetAudioBufferDelay(size_t delay_ms) {
render_buffer_->SetAudioBufferDelay(delay_ms);
}
} // namespace
BlockProcessor* BlockProcessor::Create2(const EchoCanceller3Config& config,
int sample_rate_hz) {
std::unique_ptr<RenderDelayBuffer> render_buffer(
RenderDelayBuffer::Create2(config, NumBandsForRate(sample_rate_hz)));
std::unique_ptr<RenderDelayController> delay_controller(
RenderDelayController::Create2(config, sample_rate_hz));
std::unique_ptr<EchoRemover> echo_remover(
EchoRemover::Create(config, sample_rate_hz));
return Create2(config, sample_rate_hz, std::move(render_buffer),
std::move(delay_controller), std::move(echo_remover));
}
BlockProcessor* BlockProcessor::Create2(
const EchoCanceller3Config& config,
int sample_rate_hz,
std::unique_ptr<RenderDelayBuffer> render_buffer) {
std::unique_ptr<RenderDelayController> delay_controller(
RenderDelayController::Create2(config, sample_rate_hz));
std::unique_ptr<EchoRemover> echo_remover(
EchoRemover::Create(config, sample_rate_hz));
return Create2(config, sample_rate_hz, std::move(render_buffer),
std::move(delay_controller), std::move(echo_remover));
}
BlockProcessor* BlockProcessor::Create2(
const EchoCanceller3Config& config,
int sample_rate_hz,
std::unique_ptr<RenderDelayBuffer> render_buffer,
std::unique_ptr<RenderDelayController> delay_controller,
std::unique_ptr<EchoRemover> echo_remover) {
return new BlockProcessorImpl2(
config, sample_rate_hz, std::move(render_buffer),
std::move(delay_controller), std::move(echo_remover));
}
} // namespace webrtc

18
modules/audio_processing/aec3/block_processor_unittest.cc
View File

@ -37,7 +37,7 @@ using testing::_;
// methods are callable.
void RunBasicSetupAndApiCallTest(int sample_rate_hz, int num_iterations) {
std::unique_ptr<BlockProcessor> block_processor(
BlockProcessor::Create2(EchoCanceller3Config(), sample_rate_hz));
BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz));
std::vector<std::vector<float>> block(NumBandsForRate(sample_rate_hz),
std::vector<float>(kBlockSize, 1000.f));
@ -51,7 +51,7 @@ void RunBasicSetupAndApiCallTest(int sample_rate_hz, int num_iterations) {
#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID)
void RunRenderBlockSizeVerificationTest(int sample_rate_hz) {
std::unique_ptr<BlockProcessor> block_processor(
BlockProcessor::Create2(EchoCanceller3Config(), sample_rate_hz));
BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz));
std::vector<std::vector<float>> block(
NumBandsForRate(sample_rate_hz), std::vector<float>(kBlockSize - 1, 0.f));
@ -60,7 +60,7 @@ void RunRenderBlockSizeVerificationTest(int sample_rate_hz) {
void RunCaptureBlockSizeVerificationTest(int sample_rate_hz) {
std::unique_ptr<BlockProcessor> block_processor(
BlockProcessor::Create2(EchoCanceller3Config(), sample_rate_hz));
BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz));
std::vector<std::vector<float>> block(
NumBandsForRate(sample_rate_hz), std::vector<float>(kBlockSize - 1, 0.f));
@ -72,7 +72,7 @@ void RunRenderNumBandsVerificationTest(int sample_rate_hz) {
? NumBandsForRate(sample_rate_hz) + 1
: 1;
std::unique_ptr<BlockProcessor> block_processor(
BlockProcessor::Create2(EchoCanceller3Config(), sample_rate_hz));
BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz));
std::vector<std::vector<float>> block(wrong_num_bands,
std::vector<float>(kBlockSize, 0.f));
@ -84,7 +84,7 @@ void RunCaptureNumBandsVerificationTest(int sample_rate_hz) {
? NumBandsForRate(sample_rate_hz) + 1
: 1;
std::unique_ptr<BlockProcessor> block_processor(
BlockProcessor::Create2(EchoCanceller3Config(), sample_rate_hz));
BlockProcessor::Create(EchoCanceller3Config(), sample_rate_hz));
std::vector<std::vector<float>> block(wrong_num_bands,
std::vector<float>(kBlockSize, 0.f));
@ -124,7 +124,7 @@ TEST(BlockProcessor, DISABLED_DelayControllerIntegration) {
EXPECT_CALL(*render_delay_buffer_mock, Delay())
.Times(kNumBlocks + 1)
.WillRepeatedly(Return(0));
std::unique_ptr<BlockProcessor> block_processor(BlockProcessor::Create2(
std::unique_ptr<BlockProcessor> block_processor(BlockProcessor::Create(
EchoCanceller3Config(), rate, std::move(render_delay_buffer_mock)));
std::vector<std::vector<float>> render_block(
@ -173,7 +173,7 @@ TEST(BlockProcessor, DISABLED_SubmoduleIntegration) {
EXPECT_CALL(*echo_remover_mock, UpdateEchoLeakageStatus(_))
.Times(kNumBlocks);
std::unique_ptr<BlockProcessor> block_processor(BlockProcessor::Create2(
std::unique_ptr<BlockProcessor> block_processor(BlockProcessor::Create(
EchoCanceller3Config(), rate, std::move(render_delay_buffer_mock),
std::move(render_delay_controller_mock), std::move(echo_remover_mock)));
@ -239,7 +239,7 @@ TEST(BlockProcessor, VerifyCaptureNumBandsCheck) {
// Verifiers that the verification for null ProcessCapture input works.
TEST(BlockProcessor, NullProcessCaptureParameter) {
EXPECT_DEATH(std::unique_ptr<BlockProcessor>(
BlockProcessor::Create2(EchoCanceller3Config(), 8000))
BlockProcessor::Create(EchoCanceller3Config(), 8000))
->ProcessCapture(false, false, nullptr),
"");
}
@ -249,7 +249,7 @@ TEST(BlockProcessor, NullProcessCaptureParameter) {
// tests on test bots has been fixed.
TEST(BlockProcessor, DISABLED_WrongSampleRate) {
EXPECT_DEATH(std::unique_ptr<BlockProcessor>(
BlockProcessor::Create2(EchoCanceller3Config(), 8001)),
BlockProcessor::Create(EchoCanceller3Config(), 8001)),
"");
}

27
modules/audio_processing/aec3/echo_canceller3.cc
View File

@ -76,10 +76,6 @@ bool ActivateStationarityPropertiesAtInit() {
return field_trial::IsEnabled("WebRTC-Aec3UseStationarityPropertiesAtInit");
}
bool EnableNewRenderBuffering() {
return !field_trial::IsEnabled("WebRTC-Aec3NewRenderBufferingKillSwitch");
}
bool UseEarlyDelayDetection() {
return !field_trial::IsEnabled("WebRTC-Aec3EarlyDelayDetectionKillSwitch");
}
@ -98,13 +94,6 @@ EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) {
std::min(adjusted_cfg.delay.num_filters, static_cast<size_t>(5));
}
bool use_new_render_buffering =
EnableNewRenderBuffering() && config.buffering.use_new_render_buffering;
// Old render buffering needs one more filter to cover the same delay.
if (!use_new_render_buffering) {
adjusted_cfg.delay.num_filters += 1;
}
if (EnableReverbBasedOnRender() == false) {
adjusted_cfg.ep_strength.reverb_based_on_render = false;
}
@ -347,16 +336,12 @@ int EchoCanceller3::instance_count_ = 0;
EchoCanceller3::EchoCanceller3(const EchoCanceller3Config& config,
int sample_rate_hz,
bool use_highpass_filter)
: EchoCanceller3(AdjustConfig(config),
sample_rate_hz,
use_highpass_filter,
std::unique_ptr<BlockProcessor>(
EnableNewRenderBuffering() &&
config.buffering.use_new_render_buffering
? BlockProcessor::Create2(AdjustConfig(config),
sample_rate_hz)
: BlockProcessor::Create(AdjustConfig(config),
sample_rate_hz))) {}
: EchoCanceller3(
AdjustConfig(config),
sample_rate_hz,
use_highpass_filter,
std::unique_ptr<BlockProcessor>(
BlockProcessor::Create(AdjustConfig(config), sample_rate_hz))) {}
EchoCanceller3::EchoCanceller3(const EchoCanceller3Config& config,
int sample_rate_hz,
bool use_highpass_filter,

10
modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc
View File

@ -39,7 +39,7 @@ TEST(EchoPathDelayEstimator, BasicApiCalls) {
ApmDataDumper data_dumper(0);
EchoCanceller3Config config;
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
EchoPathDelayEstimator estimator(&data_dumper, config);
std::vector<std::vector<float>> render(3, std::vector<float>(kBlockSize));
std::vector<float> capture(kBlockSize);
@ -65,7 +65,7 @@ TEST(EchoPathDelayEstimator, DelayEstimation) {
for (size_t delay_samples : {30, 64, 150, 200, 800, 4000}) {
SCOPED_TRACE(ProduceDebugText(delay_samples, down_sampling_factor));
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
DelayBuffer<float> signal_delay_buffer(delay_samples);
EchoPathDelayEstimator estimator(&data_dumper, config);
@ -113,7 +113,7 @@ TEST(EchoPathDelayEstimator, NoDelayEstimatesForLowLevelRenderSignals) {
ApmDataDumper data_dumper(0);
EchoPathDelayEstimator estimator(&data_dumper, config);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(EchoCanceller3Config(), 3));
RenderDelayBuffer::Create(EchoCanceller3Config(), 3));
for (size_t k = 0; k < 100; ++k) {
RandomizeSampleVector(&random_generator, render[0]);
for (auto& render_k : render[0]) {
@ -137,7 +137,7 @@ TEST(EchoPathDelayEstimator, DISABLED_WrongRenderBlockSize) {
EchoCanceller3Config config;
EchoPathDelayEstimator estimator(&data_dumper, config);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
std::vector<float> capture(kBlockSize);
EXPECT_DEATH(estimator.EstimateDelay(
render_delay_buffer->GetDownsampledRenderBuffer(), capture),
@ -152,7 +152,7 @@ TEST(EchoPathDelayEstimator, WrongCaptureBlockSize) {
EchoCanceller3Config config;
EchoPathDelayEstimator estimator(&data_dumper, config);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
std::vector<float> capture(std::vector<float>(kBlockSize - 1));
EXPECT_DEATH(estimator.EstimateDelay(
render_delay_buffer->GetDownsampledRenderBuffer(), capture),

11
modules/audio_processing/aec3/echo_remover_unittest.cc
View File

@ -48,7 +48,7 @@ TEST(EchoRemover, BasicApiCalls) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<EchoRemover> remover(
EchoRemover::Create(EchoCanceller3Config(), rate));
std::unique_ptr<RenderDelayBuffer> render_buffer(RenderDelayBuffer::Create2(
std::unique_ptr<RenderDelayBuffer> render_buffer(RenderDelayBuffer::Create(
EchoCanceller3Config(), NumBandsForRate(rate)));
std::vector<std::vector<float>> render(NumBandsForRate(rate),
@ -89,7 +89,7 @@ TEST(EchoRemover, WrongCaptureBlockSize) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<EchoRemover> remover(
EchoRemover::Create(EchoCanceller3Config(), rate));
std::unique_ptr<RenderDelayBuffer> render_buffer(RenderDelayBuffer::Create2(
std::unique_ptr<RenderDelayBuffer> render_buffer(RenderDelayBuffer::Create(
EchoCanceller3Config(), NumBandsForRate(rate)));
std::vector<std::vector<float>> capture(
NumBandsForRate(rate), std::vector<float>(kBlockSize - 1, 0.f));
@ -111,7 +111,7 @@ TEST(EchoRemover, DISABLED_WrongCaptureNumBands) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<EchoRemover> remover(
EchoRemover::Create(EchoCanceller3Config(), rate));
std::unique_ptr<RenderDelayBuffer> render_buffer(RenderDelayBuffer::Create2(
std::unique_ptr<RenderDelayBuffer> render_buffer(RenderDelayBuffer::Create(
EchoCanceller3Config(), NumBandsForRate(rate)));
std::vector<std::vector<float>> capture(
NumBandsForRate(rate == 48000 ? 16000 : rate + 16000),
@ -131,7 +131,7 @@ TEST(EchoRemover, NullCapture) {
std::unique_ptr<EchoRemover> remover(
EchoRemover::Create(EchoCanceller3Config(), 8000));
std::unique_ptr<RenderDelayBuffer> render_buffer(
RenderDelayBuffer::Create2(EchoCanceller3Config(), 3));
RenderDelayBuffer::Create(EchoCanceller3Config(), 3));
EchoPathVariability echo_path_variability(
false, EchoPathVariability::DelayAdjustment::kNone, false);
EXPECT_DEATH(
@ -158,10 +158,9 @@ TEST(EchoRemover, BasicEchoRemoval) {
for (size_t delay_samples : {0, 64, 150, 200, 301}) {
SCOPED_TRACE(ProduceDebugText(rate, delay_samples));
EchoCanceller3Config config;
config.delay.min_echo_path_delay_blocks = 0;
std::unique_ptr<EchoRemover> remover(EchoRemover::Create(config, rate));
std::unique_ptr<RenderDelayBuffer> render_buffer(
RenderDelayBuffer::Create2(config, NumBandsForRate(rate)));
RenderDelayBuffer::Create(config, NumBandsForRate(rate)));
render_buffer->SetDelay(delay_samples / kBlockSize);
std::vector<std::unique_ptr<DelayBuffer<float>>> delay_buffers(x.size());

4
modules/audio_processing/aec3/erle_estimator_unittest.cc
View File

@ -108,7 +108,7 @@ TEST(ErleEstimator, VerifyErleIncreaseAndHold) {
std::vector<std::array<float, kFftLengthBy2Plus1>> filter_frequency_response(
config.filter.main.length_blocks);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
GetFilterFreq(filter_frequency_response, config.delay.delay_headroom_blocks);
@ -152,7 +152,7 @@ TEST(ErleEstimator, VerifyErleTrackingOnOnsets) {
config.filter.main.length_blocks);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
GetFilterFreq(filter_frequency_response, config.delay.delay_headroom_blocks);

3
modules/audio_processing/aec3/main_filter_update_gain_unittest.cc
View File

@ -63,10 +63,9 @@ void RunFilterUpdateTest(int num_blocks_to_process,
Random random_generator(42U);
std::vector<std::vector<float>> x(3, std::vector<float>(kBlockSize, 0.f));
std::vector<float> y(kBlockSize, 0.f);
config.delay.min_echo_path_delay_blocks = 0;
config.delay.default_delay = 1;
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
AecState aec_state(config);
RenderSignalAnalyzer render_signal_analyzer(config);
absl::optional<DelayEstimate> delay_estimate;

8
modules/audio_processing/aec3/matched_filter_unittest.cc
View File

@ -153,8 +153,6 @@ TEST(MatchedFilter, LagEstimation) {
EchoCanceller3Config config;
config.delay.down_sampling_factor = down_sampling_factor;
config.delay.num_filters = kNumMatchedFilters;
config.delay.min_echo_path_delay_blocks = 0;
config.delay.api_call_jitter_blocks = 0;
Decimator capture_decimator(down_sampling_factor);
DelayBuffer<float> signal_delay_buffer(down_sampling_factor *
delay_samples);
@ -165,7 +163,7 @@ TEST(MatchedFilter, LagEstimation) {
config.delay.delay_candidate_detection_threshold);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
// Analyze the correlation between render and capture.
for (size_t k = 0; k < (600 + delay_samples / sub_block_size); ++k) {
@ -261,7 +259,7 @@ TEST(MatchedFilter, LagNotReliableForUncorrelatedRenderAndCapture) {
std::fill(capture.begin(), capture.end(), 0.f);
ApmDataDumper data_dumper(0);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
MatchedFilter filter(&data_dumper, DetectOptimization(), sub_block_size,
kWindowSizeSubBlocks, kNumMatchedFilters,
kAlignmentShiftSubBlocks, 150,
@ -306,7 +304,7 @@ TEST(MatchedFilter, LagNotUpdatedForLowLevelRender) {
config.delay.delay_estimate_smoothing,
config.delay.delay_candidate_detection_threshold);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(EchoCanceller3Config(), 3));
RenderDelayBuffer::Create(EchoCanceller3Config(), 3));
Decimator capture_decimator(down_sampling_factor);
// Analyze the correlation between render and capture.

319
modules/audio_processing/aec3/render_delay_buffer.cc
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
@ -8,47 +8,44 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_processing/aec3/render_delay_buffer.h"
#include <stdlib.h>
#include <string.h>
#include <algorithm>
#include <memory>
#include <numeric>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/aec3_fft.h"
#include "modules/audio_processing/aec3/decimator.h"
#include "modules/audio_processing/aec3/downsampled_render_buffer.h"
#include "modules/audio_processing/aec3/fft_buffer.h"
#include "modules/audio_processing/aec3/fft_data.h"
#include "modules/audio_processing/aec3/matrix_buffer.h"
#include "modules/audio_processing/aec3/render_buffer.h"
#include "modules/audio_processing/aec3/render_delay_buffer.h"
#include "modules/audio_processing/aec3/vector_buffer.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/atomic_ops.h"
#include "rtc_base/checks.h"
#include "rtc_base/constructor_magic.h"
#include "rtc_base/logging.h"
#include "system_wrappers/include/field_trial.h"
namespace webrtc {
namespace {
bool EnableZeroExternalDelayHeadroom() {
return !field_trial::IsEnabled(
"WebRTC-Aec3ZeroExternalDelayHeadroomKillSwitch");
}
class RenderDelayBufferImpl final : public RenderDelayBuffer {
public:
RenderDelayBufferImpl(const EchoCanceller3Config& config, size_t num_bands);
RenderDelayBufferImpl() = delete;
~RenderDelayBufferImpl() override;
void Reset() override;
BufferingEvent Insert(const std::vector<std::vector<float>>& block) override;
BufferingEvent PrepareCaptureProcessing() override;
bool SetDelay(size_t delay) override;
size_t Delay() const override { return MapInternalDelayToExternalDelay(); }
size_t Delay() const override { return ComputeDelay(); }
size_t MaxDelay() const override {
return blocks_.buffer.size() - 1 - buffer_headroom_;
}
@ -58,8 +55,7 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer {
return low_rate_;
}
bool CausalDelay(size_t delay) const override;
int BufferLatency() const;
void SetAudioBufferDelay(size_t delay_ms) override;
private:
@ -68,17 +64,14 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer {
const Aec3Optimization optimization_;
const EchoCanceller3Config config_;
size_t down_sampling_factor_;
const bool use_zero_external_delay_headroom_;
const int sub_block_size_;
MatrixBuffer blocks_;
VectorBuffer spectra_;
FftBuffer ffts_;
absl::optional<size_t> delay_;
absl::optional<int> internal_delay_;
RenderBuffer echo_remover_buffer_;
DownsampledRenderBuffer low_rate_;
Decimator render_decimator_;
const std::vector<std::vector<float>> zero_block_;
const Aec3Fft fft_;
std::vector<float> render_ds_;
const int buffer_headroom_;
@ -90,81 +83,25 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer {
bool render_activity_ = false;
size_t render_activity_counter_ = 0;
absl::optional<size_t> external_audio_buffer_delay_;
bool external_delay_verified_after_reset_ = false;
bool external_audio_buffer_delay_verified_after_reset_ = false;
size_t min_latency_blocks_ = 0;
size_t excess_render_detection_counter_ = 0;
size_t num_bands_;
int LowRateBufferOffset() const { return DelayEstimatorOffset(config_) >> 1; }
int MapExternalDelayToInternalDelay(size_t external_delay_blocks) const;
int MapInternalDelayToExternalDelay() const;
void ApplyDelay(int delay);
int MapDelayToTotalDelay(size_t delay) const;
int ComputeDelay() const;
void ApplyTotalDelay(int delay);
void InsertBlock(const std::vector<std::vector<float>>& block,
int previous_write);
bool DetectActiveRender(rtc::ArrayView<const float> x) const;
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayBufferImpl);
bool DetectExcessRenderBlocks();
void IncrementWriteIndices();
void IncrementLowRateReadIndices();
void IncrementReadIndices();
bool RenderOverrun();
bool RenderUnderrun();
};
// Increases the write indices for the render buffers.
void IncreaseWriteIndices(int sub_block_size,
MatrixBuffer* blocks,
VectorBuffer* spectra,
FftBuffer* ffts,
DownsampledRenderBuffer* low_rate) {
low_rate->UpdateWriteIndex(-sub_block_size);
blocks->IncWriteIndex();
spectra->DecWriteIndex();
ffts->DecWriteIndex();
}
// Increases the read indices for the render buffers.
void IncreaseReadIndices(const absl::optional<int>& delay,
int sub_block_size,
MatrixBuffer* blocks,
VectorBuffer* spectra,
FftBuffer* ffts,
DownsampledRenderBuffer* low_rate) {
RTC_DCHECK_NE(low_rate->read, low_rate->write);
low_rate->UpdateReadIndex(-sub_block_size);
if (blocks->read != blocks->write) {
blocks->IncReadIndex();
spectra->DecReadIndex();
ffts->DecReadIndex();
} else {
// Only allow underrun for blocks_ when the delay is not set.
RTC_DCHECK(!delay);
}
}
// Checks for a render buffer overrun.
bool RenderOverrun(const MatrixBuffer& b, const DownsampledRenderBuffer& l) {
return l.read == l.write || b.read == b.write;
}
// Checks for a render buffer underrun. If the delay is not specified, only the
// low rate buffer underrun is counted as the delay offset for the other buffers
// is unknown.
bool RenderUnderrun(const absl::optional<int>& delay,
const MatrixBuffer& b,
const DownsampledRenderBuffer& l) {
return l.read == l.write || (delay && b.read == b.write);
}
// Computes the latency in the buffer (the number of unread elements).
int BufferLatency(const DownsampledRenderBuffer& l) {
return (l.buffer.size() + l.read - l.write) % l.buffer.size();
}
// Computes the mismatch between the number of render and capture calls based on
// the known offset (achieved during reset) of the low rate buffer.
bool ApiCallSkew(const DownsampledRenderBuffer& low_rate_buffer,
int sub_block_size,
int low_rate_buffer_offset_sub_blocks) {
int latency = BufferLatency(low_rate_buffer);
int skew = abs(low_rate_buffer_offset_sub_blocks * sub_block_size - latency);
int skew_limit = low_rate_buffer_offset_sub_blocks * sub_block_size;
return skew >= skew_limit;
}
int RenderDelayBufferImpl::instance_count_ = 0;
RenderDelayBufferImpl::RenderDelayBufferImpl(const EchoCanceller3Config& config,
@ -174,7 +111,6 @@ RenderDelayBufferImpl::RenderDelayBufferImpl(const EchoCanceller3Config& config,
optimization_(DetectOptimization()),
config_(config),
down_sampling_factor_(config.delay.down_sampling_factor),
use_zero_external_delay_headroom_(EnableZeroExternalDelayHeadroom()),
sub_block_size_(static_cast<int>(down_sampling_factor_ > 0
? kBlockSize / down_sampling_factor_
: kBlockSize)),
@ -190,15 +126,13 @@ RenderDelayBufferImpl::RenderDelayBufferImpl(const EchoCanceller3Config& config,
low_rate_(GetDownSampledBufferSize(down_sampling_factor_,
config.delay.num_filters)),
render_decimator_(down_sampling_factor_),
zero_block_(num_bands, std::vector<float>(kBlockSize, 0.f)),
fft_(),
render_ds_(sub_block_size_, 0.f),
buffer_headroom_(config.filter.main.length_blocks) {
buffer_headroom_(config.filter.main.length_blocks),
num_bands_(num_bands) {
RTC_DCHECK_EQ(blocks_.buffer.size(), ffts_.buffer.size());
RTC_DCHECK_EQ(spectra_.buffer.size(), ffts_.buffer.size());
// Necessary condition to avoid unrecoverable echp due to noncausal alignment.
RTC_DCHECK_EQ(DelayEstimatorOffset(config_), LowRateBufferOffset() * 2);
Reset();
}
@ -208,39 +142,38 @@ RenderDelayBufferImpl::~RenderDelayBufferImpl() = default;
void RenderDelayBufferImpl::Reset() {
last_call_was_render_ = false;
num_api_calls_in_a_row_ = 1;
min_latency_blocks_ = 0;
excess_render_detection_counter_ = 0;
// Pre-fill the low rate buffer (which is used for delay estimation) to add
// headroom for the allowed api call jitter.
low_rate_.read = low_rate_.OffsetIndex(
low_rate_.write, LowRateBufferOffset() * sub_block_size_);
// Initialize the read index to one sub-block before the write index.
low_rate_.read = low_rate_.OffsetIndex(low_rate_.write, sub_block_size_);
// Check for any external audio buffer delay and whether it is feasible.
if (external_audio_buffer_delay_) {
const size_t headroom = use_zero_external_delay_headroom_ ? 0 : 2;
size_t external_delay_to_set = 0;
if (*external_audio_buffer_delay_ < headroom) {
external_delay_to_set = 0;
const size_t headroom = 2;
size_t audio_buffer_delay_to_set;
// Minimum delay is 1 (like the low-rate render buffer).
if (*external_audio_buffer_delay_ <= headroom) {
audio_buffer_delay_to_set = 1;
} else {
external_delay_to_set = *external_audio_buffer_delay_ - headroom;
audio_buffer_delay_to_set = *external_audio_buffer_delay_ - headroom;
}
external_delay_to_set = std::min(external_delay_to_set, MaxDelay());
audio_buffer_delay_to_set = std::min(audio_buffer_delay_to_set, MaxDelay());
// When an external delay estimate is available, use that delay as the
// initial render buffer delay.
internal_delay_ = external_delay_to_set;
ApplyDelay(*internal_delay_);
delay_ = MapInternalDelayToExternalDelay();
ApplyTotalDelay(audio_buffer_delay_to_set);
delay_ = ComputeDelay();
external_delay_verified_after_reset_ = false;
external_audio_buffer_delay_verified_after_reset_ = false;
} else {
// If an external delay estimate is not available, use that delay as the
// initial delay. Set the render buffer delays to the default delay.
ApplyDelay(config_.delay.default_delay);
ApplyTotalDelay(config_.delay.default_delay);
// Unset the delays which are set by SetDelay.
delay_ = absl::nullopt;
internal_delay_ = absl::nullopt;
}
}
@ -265,14 +198,12 @@ RenderDelayBuffer::BufferingEvent RenderDelayBufferImpl::Insert(
// Increase the write indices to where the new blocks should be written.
const int previous_write = blocks_.write;
IncreaseWriteIndices(sub_block_size_, &blocks_, &spectra_, &ffts_,
&low_rate_);
IncrementWriteIndices();
// Allow overrun and do a reset when render overrun occurrs due to more render
// data being inserted than capture data is received.
BufferingEvent event = RenderOverrun(blocks_, low_rate_)
? BufferingEvent::kRenderOverrun
: BufferingEvent::kNone;
BufferingEvent event =
RenderOverrun() ? BufferingEvent::kRenderOverrun : BufferingEvent::kNone;
// Detect and update render activity.
if (!render_activity_) {
@ -293,7 +224,7 @@ RenderDelayBuffer::BufferingEvent RenderDelayBufferImpl::Insert(
// Prepares the render buffers for processing another capture block.
RenderDelayBuffer::BufferingEvent
RenderDelayBufferImpl::PrepareCaptureProcessing() {
BufferingEvent event = BufferingEvent::kNone;
RenderDelayBuffer::BufferingEvent event = BufferingEvent::kNone;
++capture_call_counter_;
if (delay_) {
@ -311,26 +242,29 @@ RenderDelayBufferImpl::PrepareCaptureProcessing() {
}
}
if (RenderUnderrun(internal_delay_, blocks_, low_rate_)) {
// Don't increase the read indices if there is a render underrun.
if (DetectExcessRenderBlocks()) {
// Too many render blocks compared to capture blocks. Risk of delay ending
// up before the filter used by the delay estimator.
RTC_LOG(LS_WARNING) << "Excess render blocks detected at block "
<< capture_call_counter_;
Reset();
event = BufferingEvent::kRenderOverrun;
} else if (RenderUnderrun()) {
// Don't increment the read indices of the low rate buffer if there is a
// render underrun.
RTC_LOG(LS_WARNING) << "Render buffer underrun detected at block "
<< capture_call_counter_;
IncrementReadIndices();
// Incrementing the buffer index without increasing the low rate buffer
// index means that the delay is reduced by one.
if (delay_ && *delay_ > 0)
delay_ = *delay_ - 1;
event = BufferingEvent::kRenderUnderrun;
} else {
// Increase the read indices in the render buffers to point to the most
// Increment the read indices in the render buffers to point to the most
// recent block to use in the capture processing.
IncreaseReadIndices(internal_delay_, sub_block_size_, &blocks_, &spectra_,
&ffts_, &low_rate_);
// Check for skew in the API calls which, if too large, causes the delay
// estimation to be noncausal. Doing this check after the render indice
// increase saves one unit of allowed skew. Note that the skew check only
// should need to be one-sided as one of the skew directions results in an
// underrun.
bool skew = ApiCallSkew(low_rate_, sub_block_size_, LowRateBufferOffset());
event = skew ? BufferingEvent::kApiCallSkew : BufferingEvent::kNone;
}
if (event != BufferingEvent::kNone) {
Reset();
IncrementLowRateReadIndices();
IncrementReadIndices();
}
echo_remover_buffer_.SetRenderActivity(render_activity_);
@ -344,76 +278,60 @@ RenderDelayBufferImpl::PrepareCaptureProcessing() {
// Sets the delay and returns a bool indicating whether the delay was changed.
bool RenderDelayBufferImpl::SetDelay(size_t delay) {
if (!external_delay_verified_after_reset_ && external_audio_buffer_delay_ &&
delay_) {
if (!external_audio_buffer_delay_verified_after_reset_ &&
external_audio_buffer_delay_ && delay_) {
int difference = static_cast<int>(delay) - static_cast<int>(*delay_);
RTC_LOG(LS_WARNING) << "Mismatch between first estimated delay after reset "
"and external delay: "
"and externally reported audio buffer delay: "
<< difference << " blocks";
external_delay_verified_after_reset_ = true;
external_audio_buffer_delay_verified_after_reset_ = true;
}
if (delay_ && *delay_ == delay) {
return false;
}
delay_ = delay;
// Compute the internal delay and limit the delay to the allowed range.
int internal_delay = MapExternalDelayToInternalDelay(*delay_);
internal_delay_ =
std::min(MaxDelay(), static_cast<size_t>(std::max(internal_delay, 0)));
// Compute the total delay and limit the delay to the allowed range.
int total_delay = MapDelayToTotalDelay(*delay_);
total_delay =
std::min(MaxDelay(), static_cast<size_t>(std::max(total_delay, 0)));
// Apply the delay to the buffers.
ApplyDelay(*internal_delay_);
ApplyTotalDelay(total_delay);
return true;
}
// Returns whether the specified delay is causal.
bool RenderDelayBufferImpl::CausalDelay(size_t delay) const {
// Compute the internal delay and limit the delay to the allowed range.
int internal_delay = MapExternalDelayToInternalDelay(delay);
internal_delay =
std::min(MaxDelay(), static_cast<size_t>(std::max(internal_delay, 0)));
return internal_delay >=
static_cast<int>(config_.delay.min_echo_path_delay_blocks);
}
void RenderDelayBufferImpl::SetAudioBufferDelay(size_t delay_ms) {
if (!external_audio_buffer_delay_) {
RTC_LOG(LS_WARNING)
<< "Receiving a first reported externally buffer delay of " << delay_ms
<< " ms.";
<< "Receiving a first externally reported audio buffer delay of "
<< delay_ms << " ms.";
}
// Convert delay from milliseconds to blocks (rounded down).
external_audio_buffer_delay_ = delay_ms / 4;
external_audio_buffer_delay_ = delay_ms >> ((num_bands_ == 1) ? 1 : 2);
}
// Maps the externally computed delay to the delay used internally.
int RenderDelayBufferImpl::MapExternalDelayToInternalDelay(
int RenderDelayBufferImpl::MapDelayToTotalDelay(
size_t external_delay_blocks) const {
const int latency = BufferLatency(low_rate_);
RTC_DCHECK_LT(0, sub_block_size_);
RTC_DCHECK_EQ(0, latency % sub_block_size_);
int latency_blocks = latency / sub_block_size_;
return latency_blocks + static_cast<int>(external_delay_blocks) -
DelayEstimatorOffset(config_);
const int latency_blocks = BufferLatency();
return latency_blocks + static_cast<int>(external_delay_blocks);
}
// Maps the internally used delay to the delay used externally.
int RenderDelayBufferImpl::MapInternalDelayToExternalDelay() const {
const int latency = BufferLatency(low_rate_);
int latency_blocks = latency / sub_block_size_;
// Returns the delay (not including call jitter).
int RenderDelayBufferImpl::ComputeDelay() const {
const int latency_blocks = BufferLatency();
int internal_delay = spectra_.read >= spectra_.write
? spectra_.read - spectra_.write
: spectra_.size + spectra_.read - spectra_.write;
return internal_delay - latency_blocks + DelayEstimatorOffset(config_);
return internal_delay - latency_blocks;
}
// Set the read indices according to the delay.
void RenderDelayBufferImpl::ApplyDelay(int delay) {
RTC_LOG(LS_WARNING) << "Applying internal delay of " << delay << " blocks.";
void RenderDelayBufferImpl::ApplyTotalDelay(int delay) {
RTC_LOG(LS_WARNING) << "Applying total delay of " << delay << " blocks.";
blocks_.read = blocks_.OffsetIndex(blocks_.write, -delay);
spectra_.read = spectra_.OffsetIndex(spectra_.write, delay);
ffts_.read = ffts_.OffsetIndex(ffts_.write, delay);
@ -452,13 +370,72 @@ bool RenderDelayBufferImpl::DetectActiveRender(
kFftLengthBy2;
}
} // namespace
bool RenderDelayBufferImpl::DetectExcessRenderBlocks() {
bool excess_render_detected = false;
const size_t latency_blocks = static_cast<size_t>(BufferLatency());
// The recently seen minimum latency in blocks. Should be close to 0.
min_latency_blocks_ = std::min(min_latency_blocks_, latency_blocks);
// After processing a configurable number of blocks the minimum latency is
// checked.
if (++excess_render_detection_counter_ >=
config_.buffering.excess_render_detection_interval_blocks) {
// If the minimum latency is not lower than the threshold there have been
// more render than capture frames.
excess_render_detected = min_latency_blocks_ >
config_.buffering.max_allowed_excess_render_blocks;
// Reset the counter and let the minimum latency be the current latency.
min_latency_blocks_ = latency_blocks;
excess_render_detection_counter_ = 0;
}
int RenderDelayBuffer::RenderDelayBuffer::DelayEstimatorOffset(
const EchoCanceller3Config& config) {
return config.delay.api_call_jitter_blocks * 2;
data_dumper_->DumpRaw("aec3_latency_blocks", latency_blocks);
data_dumper_->DumpRaw("aec3_min_latency_blocks", min_latency_blocks_);
data_dumper_->DumpRaw("aec3_excess_render_detected", excess_render_detected);
return excess_render_detected;
}
// Computes the latency in the buffer (the number of unread sub-blocks).
int RenderDelayBufferImpl::BufferLatency() const {
const DownsampledRenderBuffer& l = low_rate_;
int latency_samples = (l.buffer.size() + l.read - l.write) % l.buffer.size();
int latency_blocks = latency_samples / sub_block_size_;
return latency_blocks;
}
// Increments the write indices for the render buffers.
void RenderDelayBufferImpl::IncrementWriteIndices() {
low_rate_.UpdateWriteIndex(-sub_block_size_);
blocks_.IncWriteIndex();
spectra_.DecWriteIndex();
ffts_.DecWriteIndex();
}
// Increments the read indices of the low rate render buffers.
void RenderDelayBufferImpl::IncrementLowRateReadIndices() {
low_rate_.UpdateReadIndex(-sub_block_size_);
}
// Increments the read indices for the render buffers.
void RenderDelayBufferImpl::IncrementReadIndices() {
if (blocks_.read != blocks_.write) {
blocks_.IncReadIndex();
spectra_.DecReadIndex();
ffts_.DecReadIndex();
}
}
// Checks for a render buffer overrun.
bool RenderDelayBufferImpl::RenderOverrun() {
return low_rate_.read == low_rate_.write || blocks_.read == blocks_.write;
}
// Checks for a render buffer underrun.
bool RenderDelayBufferImpl::RenderUnderrun() {
return low_rate_.read == low_rate_.write;
}
} // namespace
RenderDelayBuffer* RenderDelayBuffer::Create(const EchoCanceller3Config& config,
size_t num_bands) {
return new RenderDelayBufferImpl(config, num_bands);

5
modules/audio_processing/aec3/render_delay_buffer.h
View File

@ -33,8 +33,6 @@ class RenderDelayBuffer {
static RenderDelayBuffer* Create(const EchoCanceller3Config& config,
size_t num_bands);
static RenderDelayBuffer* Create2(const EchoCanceller3Config& config,
size_t num_bands);
virtual ~RenderDelayBuffer() = default;
// Resets the buffer alignment.
@ -64,9 +62,6 @@ class RenderDelayBuffer {
// Returns the downsampled render buffer.
virtual const DownsampledRenderBuffer& GetDownsampledRenderBuffer() const = 0;
// Returns whether the current delay is noncausal.
virtual bool CausalDelay(size_t delay) const = 0;
// Returns the maximum non calusal offset that can occur in the delay buffer.
static int DelayEstimatorOffset(const EchoCanceller3Config& config);

453
modules/audio_processing/aec3/render_delay_buffer2.cc
View File

@ -1,453 +0,0 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <string.h>
#include <algorithm>
#include <memory>
#include <numeric>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/aec3_fft.h"
#include "modules/audio_processing/aec3/decimator.h"
#include "modules/audio_processing/aec3/downsampled_render_buffer.h"
#include "modules/audio_processing/aec3/fft_buffer.h"
#include "modules/audio_processing/aec3/fft_data.h"
#include "modules/audio_processing/aec3/matrix_buffer.h"
#include "modules/audio_processing/aec3/render_buffer.h"
#include "modules/audio_processing/aec3/render_delay_buffer.h"
#include "modules/audio_processing/aec3/vector_buffer.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/atomic_ops.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
namespace webrtc {
namespace {
class RenderDelayBufferImpl2 final : public RenderDelayBuffer {
public:
RenderDelayBufferImpl2(const EchoCanceller3Config& config, size_t num_bands);
RenderDelayBufferImpl2() = delete;
~RenderDelayBufferImpl2() override;
void Reset() override;
BufferingEvent Insert(const std::vector<std::vector<float>>& block) override;
BufferingEvent PrepareCaptureProcessing() override;
bool SetDelay(size_t delay) override;
size_t Delay() const override { return ComputeDelay(); }
size_t MaxDelay() const override {
return blocks_.buffer.size() - 1 - buffer_headroom_;
}
RenderBuffer* GetRenderBuffer() override { return &echo_remover_buffer_; }
const DownsampledRenderBuffer& GetDownsampledRenderBuffer() const override {
return low_rate_;
}
int BufferLatency() const;
bool CausalDelay(size_t delay) const override;
void SetAudioBufferDelay(size_t delay_ms) override;
private:
static int instance_count_;
std::unique_ptr<ApmDataDumper> data_dumper_;
const Aec3Optimization optimization_;
const EchoCanceller3Config config_;
size_t down_sampling_factor_;
const int sub_block_size_;
MatrixBuffer blocks_;
VectorBuffer spectra_;
FftBuffer ffts_;
absl::optional<size_t> delay_;
RenderBuffer echo_remover_buffer_;
DownsampledRenderBuffer low_rate_;
Decimator render_decimator_;
const Aec3Fft fft_;
std::vector<float> render_ds_;
const int buffer_headroom_;
bool last_call_was_render_ = false;
int num_api_calls_in_a_row_ = 0;
int max_observed_jitter_ = 1;
size_t capture_call_counter_ = 0;
size_t render_call_counter_ = 0;
bool render_activity_ = false;
size_t render_activity_counter_ = 0;
absl::optional<size_t> external_audio_buffer_delay_;
bool external_audio_buffer_delay_verified_after_reset_ = false;
size_t min_latency_blocks_ = 0;
size_t excess_render_detection_counter_ = 0;
size_t num_bands_;
int MapDelayToTotalDelay(size_t delay) const;
int ComputeDelay() const;
void ApplyTotalDelay(int delay);
void InsertBlock(const std::vector<std::vector<float>>& block,
int previous_write);
bool DetectActiveRender(rtc::ArrayView<const float> x) const;
bool DetectExcessRenderBlocks();
void IncrementWriteIndices();
void IncrementLowRateReadIndices();
void IncrementReadIndices();
bool RenderOverrun();
bool RenderUnderrun();
};
int RenderDelayBufferImpl2::instance_count_ = 0;
RenderDelayBufferImpl2::RenderDelayBufferImpl2(
const EchoCanceller3Config& config,
size_t num_bands)
: data_dumper_(
new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
optimization_(DetectOptimization()),
config_(config),
down_sampling_factor_(config.delay.down_sampling_factor),
sub_block_size_(static_cast<int>(down_sampling_factor_ > 0
? kBlockSize / down_sampling_factor_
: kBlockSize)),
blocks_(GetRenderDelayBufferSize(down_sampling_factor_,
config.delay.num_filters,
config.filter.main.length_blocks),
num_bands,
kBlockSize),
spectra_(blocks_.buffer.size(), kFftLengthBy2Plus1),
ffts_(blocks_.buffer.size()),
delay_(config_.delay.default_delay),
echo_remover_buffer_(&blocks_, &spectra_, &ffts_),
low_rate_(GetDownSampledBufferSize(down_sampling_factor_,
config.delay.num_filters)),
render_decimator_(down_sampling_factor_),
fft_(),
render_ds_(sub_block_size_, 0.f),
buffer_headroom_(config.filter.main.length_blocks),
num_bands_(num_bands) {
RTC_DCHECK_EQ(blocks_.buffer.size(), ffts_.buffer.size());
RTC_DCHECK_EQ(spectra_.buffer.size(), ffts_.buffer.size());
Reset();
}
RenderDelayBufferImpl2::~RenderDelayBufferImpl2() = default;
// Resets the buffer delays and clears the reported delays.
void RenderDelayBufferImpl2::Reset() {
last_call_was_render_ = false;
num_api_calls_in_a_row_ = 1;
min_latency_blocks_ = 0;
excess_render_detection_counter_ = 0;
// Initialize the read index to one sub-block before the write index.
low_rate_.read = low_rate_.OffsetIndex(low_rate_.write, sub_block_size_);
// Check for any external audio buffer delay and whether it is feasible.
if (external_audio_buffer_delay_) {
const size_t headroom = 2;
size_t audio_buffer_delay_to_set;
// Minimum delay is 1 (like the low-rate render buffer).
if (*external_audio_buffer_delay_ <= headroom) {
audio_buffer_delay_to_set = 1;
} else {
audio_buffer_delay_to_set = *external_audio_buffer_delay_ - headroom;
}
audio_buffer_delay_to_set = std::min(audio_buffer_delay_to_set, MaxDelay());
// When an external delay estimate is available, use that delay as the
// initial render buffer delay.
ApplyTotalDelay(audio_buffer_delay_to_set);
delay_ = ComputeDelay();
external_audio_buffer_delay_verified_after_reset_ = false;
} else {
// If an external delay estimate is not available, use that delay as the
// initial delay. Set the render buffer delays to the default delay.
ApplyTotalDelay(config_.delay.default_delay);
// Unset the delays which are set by SetDelay.
delay_ = absl::nullopt;
}
}
// Inserts a new block into the render buffers.
RenderDelayBuffer::BufferingEvent RenderDelayBufferImpl2::Insert(
const std::vector<std::vector<float>>& block) {
++render_call_counter_;
if (delay_) {
if (!last_call_was_render_) {
last_call_was_render_ = true;
num_api_calls_in_a_row_ = 1;
} else {
if (++num_api_calls_in_a_row_ > max_observed_jitter_) {
max_observed_jitter_ = num_api_calls_in_a_row_;
RTC_LOG(LS_WARNING)
<< "New max number api jitter observed at render block "
<< render_call_counter_ << ": " << num_api_calls_in_a_row_
<< " blocks";
}
}
}
// Increase the write indices to where the new blocks should be written.
const int previous_write = blocks_.write;
IncrementWriteIndices();
// Allow overrun and do a reset when render overrun occurrs due to more render
// data being inserted than capture data is received.
BufferingEvent event =
RenderOverrun() ? BufferingEvent::kRenderOverrun : BufferingEvent::kNone;
// Detect and update render activity.
if (!render_activity_) {
render_activity_counter_ += DetectActiveRender(block[0]) ? 1 : 0;
render_activity_ = render_activity_counter_ >= 20;
}
// Insert the new render block into the specified position.
InsertBlock(block, previous_write);
if (event != BufferingEvent::kNone) {
Reset();
}
return event;
}
// Prepares the render buffers for processing another capture block.
RenderDelayBuffer::BufferingEvent
RenderDelayBufferImpl2::PrepareCaptureProcessing() {
RenderDelayBuffer::BufferingEvent event = BufferingEvent::kNone;
++capture_call_counter_;
if (delay_) {
if (last_call_was_render_) {
last_call_was_render_ = false;
num_api_calls_in_a_row_ = 1;
} else {
if (++num_api_calls_in_a_row_ > max_observed_jitter_) {
max_observed_jitter_ = num_api_calls_in_a_row_;
RTC_LOG(LS_WARNING)
<< "New max number api jitter observed at capture block "
<< capture_call_counter_ << ": " << num_api_calls_in_a_row_
<< " blocks";
}
}
}
if (DetectExcessRenderBlocks()) {
// Too many render blocks compared to capture blocks. Risk of delay ending
// up before the filter used by the delay estimator.
RTC_LOG(LS_WARNING) << "Excess render blocks detected at block "
<< capture_call_counter_;
Reset();
event = BufferingEvent::kRenderOverrun;
} else if (RenderUnderrun()) {
// Don't increment the read indices of the low rate buffer if there is a
// render underrun.
RTC_LOG(LS_WARNING) << "Render buffer underrun detected at block "
<< capture_call_counter_;
IncrementReadIndices();
// Incrementing the buffer index without increasing the low rate buffer
// index means that the delay is reduced by one.
if (delay_ && *delay_ > 0)
delay_ = *delay_ - 1;
event = BufferingEvent::kRenderUnderrun;
} else {
// Increment the read indices in the render buffers to point to the most
// recent block to use in the capture processing.
IncrementLowRateReadIndices();
IncrementReadIndices();
}
echo_remover_buffer_.SetRenderActivity(render_activity_);
if (render_activity_) {
render_activity_counter_ = 0;
render_activity_ = false;
}
return event;
}
// Sets the delay and returns a bool indicating whether the delay was changed.
bool RenderDelayBufferImpl2::SetDelay(size_t delay) {
if (!external_audio_buffer_delay_verified_after_reset_ &&
external_audio_buffer_delay_ && delay_) {
int difference = static_cast<int>(delay) - static_cast<int>(*delay_);
RTC_LOG(LS_WARNING) << "Mismatch between first estimated delay after reset "
"and externally reported audio buffer delay: "
<< difference << " blocks";
external_audio_buffer_delay_verified_after_reset_ = true;
}
if (delay_ && *delay_ == delay) {
return false;
}
delay_ = delay;
// Compute the total delay and limit the delay to the allowed range.
int total_delay = MapDelayToTotalDelay(*delay_);
total_delay =
std::min(MaxDelay(), static_cast<size_t>(std::max(total_delay, 0)));
// Apply the delay to the buffers.
ApplyTotalDelay(total_delay);
return true;
}
// Returns whether the specified delay is causal.
bool RenderDelayBufferImpl2::CausalDelay(size_t delay) const {
// TODO(gustaf): Remove this from RenderDelayBuffer.
return true;
}
void RenderDelayBufferImpl2::SetAudioBufferDelay(size_t delay_ms) {
if (!external_audio_buffer_delay_) {
RTC_LOG(LS_WARNING)
<< "Receiving a first externally reported audio buffer delay of "
<< delay_ms << " ms.";
}
// Convert delay from milliseconds to blocks (rounded down).
external_audio_buffer_delay_ = delay_ms >> ((num_bands_ == 1) ? 1 : 2);
}
// Maps the externally computed delay to the delay used internally.
int RenderDelayBufferImpl2::MapDelayToTotalDelay(
size_t external_delay_blocks) const {
const int latency_blocks = BufferLatency();
return latency_blocks + static_cast<int>(external_delay_blocks);
}
// Returns the delay (not including call jitter).
int RenderDelayBufferImpl2::ComputeDelay() const {
const int latency_blocks = BufferLatency();
int internal_delay = spectra_.read >= spectra_.write
? spectra_.read - spectra_.write
: spectra_.size + spectra_.read - spectra_.write;
return internal_delay - latency_blocks;
}
// Set the read indices according to the delay.
void RenderDelayBufferImpl2::ApplyTotalDelay(int delay) {
RTC_LOG(LS_WARNING) << "Applying total delay of " << delay << " blocks.";
blocks_.read = blocks_.OffsetIndex(blocks_.write, -delay);
spectra_.read = spectra_.OffsetIndex(spectra_.write, delay);
ffts_.read = ffts_.OffsetIndex(ffts_.write, delay);
}
// Inserts a block into the render buffers.
void RenderDelayBufferImpl2::InsertBlock(
const std::vector<std::vector<float>>& block,
int previous_write) {
auto& b = blocks_;
auto& lr = low_rate_;
auto& ds = render_ds_;
auto& f = ffts_;
auto& s = spectra_;
RTC_DCHECK_EQ(block.size(), b.buffer[b.write].size());
for (size_t k = 0; k < block.size(); ++k) {
RTC_DCHECK_EQ(block[k].size(), b.buffer[b.write][k].size());
std::copy(block[k].begin(), block[k].end(), b.buffer[b.write][k].begin());
}
data_dumper_->DumpWav("aec3_render_decimator_input", block[0].size(),
block[0].data(), 16000, 1);
render_decimator_.Decimate(block[0], ds);
data_dumper_->DumpWav("aec3_render_decimator_output", ds.size(), ds.data(),
16000 / down_sampling_factor_, 1);
std::copy(ds.rbegin(), ds.rend(), lr.buffer.begin() + lr.write);
fft_.PaddedFft(block[0], b.buffer[previous_write][0], &f.buffer[f.write]);
f.buffer[f.write].Spectrum(optimization_, s.buffer[s.write]);
}
bool RenderDelayBufferImpl2::DetectActiveRender(
rtc::ArrayView<const float> x) const {
const float x_energy = std::inner_product(x.begin(), x.end(), x.begin(), 0.f);
return x_energy > (config_.render_levels.active_render_limit *
config_.render_levels.active_render_limit) *
kFftLengthBy2;
}
bool RenderDelayBufferImpl2::DetectExcessRenderBlocks() {
bool excess_render_detected = false;
const size_t latency_blocks = static_cast<size_t>(BufferLatency());
// The recently seen minimum latency in blocks. Should be close to 0.
min_latency_blocks_ = std::min(min_latency_blocks_, latency_blocks);
// After processing a configurable number of blocks the minimum latency is
// checked.
if (++excess_render_detection_counter_ >=
config_.buffering.excess_render_detection_interval_blocks) {
// If the minimum latency is not lower than the threshold there have been
// more render than capture frames.
excess_render_detected = min_latency_blocks_ >
config_.buffering.max_allowed_excess_render_blocks;
// Reset the counter and let the minimum latency be the current latency.
min_latency_blocks_ = latency_blocks;
excess_render_detection_counter_ = 0;
}
data_dumper_->DumpRaw("aec3_latency_blocks", latency_blocks);
data_dumper_->DumpRaw("aec3_min_latency_blocks", min_latency_blocks_);
data_dumper_->DumpRaw("aec3_excess_render_detected", excess_render_detected);
return excess_render_detected;
}
// Computes the latency in the buffer (the number of unread sub-blocks).
int RenderDelayBufferImpl2::BufferLatency() const {
const DownsampledRenderBuffer& l = low_rate_;
int latency_samples = (l.buffer.size() + l.read - l.write) % l.buffer.size();
int latency_blocks = latency_samples / sub_block_size_;
return latency_blocks;
}
// Increments the write indices for the render buffers.
void RenderDelayBufferImpl2::IncrementWriteIndices() {
low_rate_.UpdateWriteIndex(-sub_block_size_);
blocks_.IncWriteIndex();
spectra_.DecWriteIndex();
ffts_.DecWriteIndex();
}
// Increments the read indices of the low rate render buffers.
void RenderDelayBufferImpl2::IncrementLowRateReadIndices() {
low_rate_.UpdateReadIndex(-sub_block_size_);
}
// Increments the read indices for the render buffers.
void RenderDelayBufferImpl2::IncrementReadIndices() {
if (blocks_.read != blocks_.write) {
blocks_.IncReadIndex();
spectra_.DecReadIndex();
ffts_.DecReadIndex();
}
}
// Checks for a render buffer overrun.
bool RenderDelayBufferImpl2::RenderOverrun() {
return low_rate_.read == low_rate_.write || blocks_.read == blocks_.write;
}
// Checks for a render buffer underrun.
bool RenderDelayBufferImpl2::RenderUnderrun() {
return low_rate_.read == low_rate_.write;
}
} // namespace
RenderDelayBuffer* RenderDelayBuffer::Create2(
const EchoCanceller3Config& config,
size_t num_bands) {
return new RenderDelayBufferImpl2(config, num_bands);
}
} // namespace webrtc

15
modules/audio_processing/aec3/render_delay_buffer_unittest.cc
View File

@ -38,7 +38,7 @@ TEST(RenderDelayBuffer, BufferOverflow) {
for (auto rate : {8000, 16000, 32000, 48000}) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<RenderDelayBuffer> delay_buffer(
RenderDelayBuffer::Create2(config, NumBandsForRate(rate)));
RenderDelayBuffer::Create(config, NumBandsForRate(rate)));
std::vector<std::vector<float>> block_to_insert(
NumBandsForRate(rate), std::vector<float>(kBlockSize, 0.f));
for (size_t k = 0; k < 10; ++k) {
@ -62,7 +62,7 @@ TEST(RenderDelayBuffer, BufferOverflow) {
TEST(RenderDelayBuffer, AvailableBlock) {
constexpr size_t kNumBands = 1;
std::unique_ptr<RenderDelayBuffer> delay_buffer(
RenderDelayBuffer::Create2(EchoCanceller3Config(), kNumBands));
RenderDelayBuffer::Create(EchoCanceller3Config(), kNumBands));
std::vector<std::vector<float>> input_block(
kNumBands, std::vector<float>(kBlockSize, 1.f));
EXPECT_EQ(RenderDelayBuffer::BufferingEvent::kNone,
@ -74,11 +74,10 @@ TEST(RenderDelayBuffer, AvailableBlock) {
TEST(RenderDelayBuffer, SetDelay) {
EchoCanceller3Config config;
std::unique_ptr<RenderDelayBuffer> delay_buffer(
RenderDelayBuffer::Create2(config, 1));
RenderDelayBuffer::Create(config, 1));
ASSERT_TRUE(delay_buffer->Delay());
delay_buffer->Reset();
size_t initial_internal_delay = config.delay.min_echo_path_delay_blocks +
config.delay.api_call_jitter_blocks;
size_t initial_internal_delay = 0;
for (size_t delay = initial_internal_delay;
delay < initial_internal_delay + 20; ++delay) {
ASSERT_TRUE(delay_buffer->SetDelay(delay));
@ -93,7 +92,7 @@ TEST(RenderDelayBuffer, SetDelay) {
// tests on test bots has been fixed.
TEST(RenderDelayBuffer, DISABLED_WrongDelay) {
std::unique_ptr<RenderDelayBuffer> delay_buffer(
RenderDelayBuffer::Create2(EchoCanceller3Config(), 3));
RenderDelayBuffer::Create(EchoCanceller3Config(), 3));
EXPECT_DEATH(delay_buffer->SetDelay(21), "");
}
@ -101,7 +100,7 @@ TEST(RenderDelayBuffer, DISABLED_WrongDelay) {
TEST(RenderDelayBuffer, WrongNumberOfBands) {
for (auto rate : {16000, 32000, 48000}) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<RenderDelayBuffer> delay_buffer(RenderDelayBuffer::Create2(
std::unique_ptr<RenderDelayBuffer> delay_buffer(RenderDelayBuffer::Create(
EchoCanceller3Config(), NumBandsForRate(rate)));
std::vector<std::vector<float>> block_to_insert(
NumBandsForRate(rate < 48000 ? rate + 16000 : 16000),
@ -115,7 +114,7 @@ TEST(RenderDelayBuffer, WrongBlockLength) {
for (auto rate : {8000, 16000, 32000, 48000}) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<RenderDelayBuffer> delay_buffer(
RenderDelayBuffer::Create2(EchoCanceller3Config(), 3));
RenderDelayBuffer::Create(EchoCanceller3Config(), 3));
std::vector<std::vector<float>> block_to_insert(
NumBandsForRate(rate), std::vector<float>(kBlockSize - 1, 0.f));
EXPECT_DEATH(delay_buffer->Insert(block_to_insert), "");

132
modules/audio_processing/aec3/render_delay_controller.cc
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
@ -7,54 +7,36 @@
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_processing/aec3/render_delay_controller.h"
#include <stdlib.h>
#include <stddef.h>
#include <algorithm>
#include <memory>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/delay_estimate.h"
#include "modules/audio_processing/aec3/downsampled_render_buffer.h"
#include "modules/audio_processing/aec3/echo_path_delay_estimator.h"
#include "modules/audio_processing/aec3/render_delay_controller.h"
#include "modules/audio_processing/aec3/render_delay_controller_metrics.h"
#include "modules/audio_processing/aec3/skew_estimator.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/atomic_ops.h"
#include "rtc_base/checks.h"
#include "rtc_base/constructor_magic.h"
#include "rtc_base/logging.h"
#include "system_wrappers/include/field_trial.h"
namespace webrtc {
namespace {
int GetSkewHysteresis(const EchoCanceller3Config& config) {
if (field_trial::IsEnabled("WebRTC-Aec3EnforceSkewHysteresis1")) {
return 1;
}
if (field_trial::IsEnabled("WebRTC-Aec3EnforceSkewHysteresis2")) {
return 2;
}
return static_cast<int>(config.delay.skew_hysteresis_blocks);
}
bool UseOffsetBlocks() {
return field_trial::IsEnabled("WebRTC-Aec3UseOffsetBlocks");
}
bool UseEarlyDelayDetection() {
return !field_trial::IsEnabled("WebRTC-Aec3EarlyDelayDetectionKillSwitch");
}
constexpr int kSkewHistorySizeLog2 = 8;
class RenderDelayControllerImpl final : public RenderDelayController {
public:
RenderDelayControllerImpl(const EchoCanceller3Config& config,
int non_causal_offset,
int sample_rate_hz);
~RenderDelayControllerImpl() override;
void Reset(bool reset_delay_confidence) override;
@ -73,21 +55,12 @@ class RenderDelayControllerImpl final : public RenderDelayController {
const int delay_headroom_blocks_;
const int hysteresis_limit_1_blocks_;
const int hysteresis_limit_2_blocks_;
const int skew_hysteresis_blocks_;
const bool use_offset_blocks_;
absl::optional<DelayEstimate> delay_;
EchoPathDelayEstimator delay_estimator_;
std::vector<float> delay_buf_;
int delay_buf_index_ = 0;
RenderDelayControllerMetrics metrics_;
SkewEstimator skew_estimator_;
absl::optional<DelayEstimate> delay_samples_;
absl::optional<int> skew_;
int previous_offset_blocks_ = 0;
int skew_shift_reporting_counter_ = 0;
size_t capture_call_counter_ = 0;
int delay_change_counter_ = 0;
size_t soft_reset_counter_ = 0;
DelayEstimate::Quality last_delay_estimate_quality_;
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayControllerImpl);
};
@ -97,14 +70,13 @@ DelayEstimate ComputeBufferDelay(
int delay_headroom_blocks,
int hysteresis_limit_1_blocks,
int hysteresis_limit_2_blocks,
int offset_blocks,
DelayEstimate estimated_delay) {
// The below division is not exact and the truncation is intended.
const int echo_path_delay_blocks = estimated_delay.delay >> kBlockSizeLog2;
// Compute the buffer delay increase required to achieve the desired latency.
size_t new_delay_blocks = std::max(
echo_path_delay_blocks + offset_blocks - delay_headroom_blocks, 0);
size_t new_delay_blocks =
std::max(echo_path_delay_blocks - delay_headroom_blocks, 0);
// Add hysteresis.
if (current_delay) {
@ -133,7 +105,6 @@ int RenderDelayControllerImpl::instance_count_ = 0;
RenderDelayControllerImpl::RenderDelayControllerImpl(
const EchoCanceller3Config& config,
int non_causal_offset,
int sample_rate_hz)
: data_dumper_(
new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
@ -144,15 +115,10 @@ RenderDelayControllerImpl::RenderDelayControllerImpl(
static_cast<int>(config.delay.hysteresis_limit_1_blocks)),
hysteresis_limit_2_blocks_(
static_cast<int>(config.delay.hysteresis_limit_2_blocks)),
skew_hysteresis_blocks_(GetSkewHysteresis(config)),
use_offset_blocks_(UseOffsetBlocks()),
delay_estimator_(data_dumper_.get(), config),
delay_buf_(kBlockSize * non_causal_offset, 0.f),
skew_estimator_(kSkewHistorySizeLog2),
last_delay_estimate_quality_(DelayEstimate::Quality::kCoarse) {
RTC_DCHECK(ValidFullBandRate(sample_rate_hz));
delay_estimator_.LogDelayEstimationProperties(sample_rate_hz,
delay_buf_.size());
delay_estimator_.LogDelayEstimationProperties(sample_rate_hz, 0);
}
RenderDelayControllerImpl::~RenderDelayControllerImpl() = default;
@ -160,21 +126,14 @@ RenderDelayControllerImpl::~RenderDelayControllerImpl() = default;
void RenderDelayControllerImpl::Reset(bool reset_delay_confidence) {
delay_ = absl::nullopt;
delay_samples_ = absl::nullopt;
skew_ = absl::nullopt;
previous_offset_blocks_ = 0;
std::fill(delay_buf_.begin(), delay_buf_.end(), 0.f);
delay_estimator_.Reset(reset_delay_confidence);
skew_estimator_.Reset();
delay_change_counter_ = 0;
soft_reset_counter_ = 0;
if (reset_delay_confidence) {
last_delay_estimate_quality_ = DelayEstimate::Quality::kCoarse;
}
}
void RenderDelayControllerImpl::LogRenderCall() {
skew_estimator_.LogRenderCall();
}
void RenderDelayControllerImpl::LogRenderCall() {}
absl::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
const DownsampledRenderBuffer& render_buffer,
@ -184,12 +143,7 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
RTC_DCHECK_EQ(kBlockSize, capture.size());
++capture_call_counter_;
// Estimate the delay with a delayed capture.
RTC_DCHECK_LT(delay_buf_index_ + kBlockSize - 1, delay_buf_.size());
rtc::ArrayView<const float> capture_delayed(&delay_buf_[delay_buf_index_],
kBlockSize);
auto delay_samples =
delay_estimator_.EstimateDelay(render_buffer, capture_delayed);
auto delay_samples = delay_estimator_.EstimateDelay(render_buffer, capture);
// Overrule the delay estimator delay if the echo remover reports a delay.
if (echo_remover_delay) {
@ -199,13 +153,6 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
total_echo_remover_delay_samples);
}
std::copy(capture.begin(), capture.end(),
delay_buf_.begin() + delay_buf_index_);
delay_buf_index_ = (delay_buf_index_ + kBlockSize) % delay_buf_.size();
// Compute the latest skew update.
absl::optional<int> skew = skew_estimator_.GetSkewFromCapture();
if (delay_samples) {
if (!delay_samples_ || delay_samples->delay != delay_samples_->delay) {
delay_change_counter_ = 0;
@ -230,46 +177,6 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
if (delay_change_counter_ < 2 * kNumBlocksPerSecond) {
++delay_change_counter_;
// If a new delay estimate is recently obtained, store the skew for that.
skew_ = skew;
} else {
// A reliable skew should have been obtained after 2 seconds.
RTC_DCHECK(skew_);
RTC_DCHECK(skew);
}
++soft_reset_counter_;
int offset_blocks = 0;
if (skew_ && skew && delay_samples_ &&
delay_samples_->quality == DelayEstimate::Quality::kRefined) {
// Compute the skew offset and add a margin.
offset_blocks = *skew_ - *skew;
if (abs(offset_blocks) <= skew_hysteresis_blocks_) {
offset_blocks = 0;
} else if (soft_reset_counter_ > 10 * kNumBlocksPerSecond) {
// Soft reset the delay estimator if there is a significant offset
// detected.
delay_estimator_.Reset(false);
soft_reset_counter_ = 0;
}
}
if (!use_offset_blocks_)
offset_blocks = 0;
// Log any changes in the skew.
skew_shift_reporting_counter_ =
std::max(0, skew_shift_reporting_counter_ - 1);
absl::optional<int> skew_shift =
skew_shift_reporting_counter_ == 0 &&
previous_offset_blocks_ != offset_blocks
? absl::optional<int>(offset_blocks - previous_offset_blocks_)
: absl::nullopt;
previous_offset_blocks_ = offset_blocks;
if (skew_shift) {
RTC_LOG(LS_WARNING) << "API call skew shift of " << *skew_shift
<< " blocks detected at capture block "
<< capture_call_counter_;
skew_shift_reporting_counter_ = 3 * kNumBlocksPerSecond;
}
if (delay_samples_) {
@ -280,26 +187,19 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
delay_ = ComputeBufferDelay(delay_, delay_headroom_blocks_,
use_hysteresis ? hysteresis_limit_1_blocks_ : 0,
use_hysteresis ? hysteresis_limit_2_blocks_ : 0,
offset_blocks, *delay_samples_);
*delay_samples_);
last_delay_estimate_quality_ = delay_samples_->quality;
}
metrics_.Update(delay_samples_ ? absl::optional<size_t>(delay_samples_->delay)
: absl::nullopt,
delay_ ? delay_->delay : 0, skew_shift,
delay_estimator_.Clockdrift());
delay_ ? delay_->delay : 0, 0, delay_estimator_.Clockdrift());
data_dumper_->DumpRaw("aec3_render_delay_controller_delay",
delay_samples ? delay_samples->delay : 0);
data_dumper_->DumpRaw("aec3_render_delay_controller_buffer_delay",
delay_ ? delay_->delay : 0);
data_dumper_->DumpRaw("aec3_render_delay_controller_new_skew",
skew ? *skew : 0);
data_dumper_->DumpRaw("aec3_render_delay_controller_old_skew",
skew_ ? *skew_ : 0);
data_dumper_->DumpRaw("aec3_render_delay_controller_offset", offset_blocks);
return delay_;
}
@ -311,10 +211,8 @@ bool RenderDelayControllerImpl::HasClockdrift() const {
RenderDelayController* RenderDelayController::Create(
const EchoCanceller3Config& config,
int non_causal_offset,
int sample_rate_hz) {
return new RenderDelayControllerImpl(config, non_causal_offset,
sample_rate_hz);
return new RenderDelayControllerImpl(config, sample_rate_hz);
}
} // namespace webrtc

3
modules/audio_processing/aec3/render_delay_controller.h
View File

@ -25,10 +25,7 @@ namespace webrtc {
class RenderDelayController {
public:
static RenderDelayController* Create(const EchoCanceller3Config& config,
int non_causal_offset,
int sample_rate_hz);
static RenderDelayController* Create2(const EchoCanceller3Config& config,
int sample_rate_hz);
virtual ~RenderDelayController() = default;
// Resets the delay controller. If the delay confidence is reset, the reset

218
modules/audio_processing/aec3/render_delay_controller2.cc
View File

@ -1,218 +0,0 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <stddef.h>
#include <algorithm>
#include <memory>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/delay_estimate.h"
#include "modules/audio_processing/aec3/downsampled_render_buffer.h"
#include "modules/audio_processing/aec3/echo_path_delay_estimator.h"
#include "modules/audio_processing/aec3/render_delay_controller.h"
#include "modules/audio_processing/aec3/render_delay_controller_metrics.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/atomic_ops.h"
#include "rtc_base/checks.h"
#include "rtc_base/constructor_magic.h"
#include "system_wrappers/include/field_trial.h"
namespace webrtc {
namespace {
bool UseEarlyDelayDetection() {
return !field_trial::IsEnabled("WebRTC-Aec3EarlyDelayDetectionKillSwitch");
}
class RenderDelayControllerImpl2 final : public RenderDelayController {
public:
RenderDelayControllerImpl2(const EchoCanceller3Config& config,
int sample_rate_hz);
~RenderDelayControllerImpl2() override;
void Reset(bool reset_delay_confidence) override;
void LogRenderCall() override;
absl::optional<DelayEstimate> GetDelay(
const DownsampledRenderBuffer& render_buffer,
size_t render_delay_buffer_delay,
const absl::optional<int>& echo_remover_delay,
rtc::ArrayView<const float> capture) override;
bool HasClockdrift() const override;
private:
static int instance_count_;
std::unique_ptr<ApmDataDumper> data_dumper_;
const bool use_early_delay_detection_;
const int delay_headroom_blocks_;
const int hysteresis_limit_1_blocks_;
const int hysteresis_limit_2_blocks_;
absl::optional<DelayEstimate> delay_;
EchoPathDelayEstimator delay_estimator_;
RenderDelayControllerMetrics metrics_;
absl::optional<DelayEstimate> delay_samples_;
size_t capture_call_counter_ = 0;
int delay_change_counter_ = 0;
DelayEstimate::Quality last_delay_estimate_quality_;
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayControllerImpl2);
};
DelayEstimate ComputeBufferDelay(
const absl::optional<DelayEstimate>& current_delay,
int delay_headroom_blocks,
int hysteresis_limit_1_blocks,
int hysteresis_limit_2_blocks,
DelayEstimate estimated_delay) {
// The below division is not exact and the truncation is intended.
const int echo_path_delay_blocks = estimated_delay.delay >> kBlockSizeLog2;
// Compute the buffer delay increase required to achieve the desired latency.
size_t new_delay_blocks =
std::max(echo_path_delay_blocks - delay_headroom_blocks, 0);
// Add hysteresis.
if (current_delay) {
size_t current_delay_blocks = current_delay->delay;
if (new_delay_blocks > current_delay_blocks) {
if (new_delay_blocks <=
current_delay_blocks + hysteresis_limit_1_blocks) {
new_delay_blocks = current_delay_blocks;
}
} else if (new_delay_blocks < current_delay_blocks) {
size_t hysteresis_limit = std::max(
static_cast<int>(current_delay_blocks) - hysteresis_limit_2_blocks,
0);
if (new_delay_blocks >= hysteresis_limit) {
new_delay_blocks = current_delay_blocks;
}
}
}
DelayEstimate new_delay = estimated_delay;
new_delay.delay = new_delay_blocks;
return new_delay;
}
int RenderDelayControllerImpl2::instance_count_ = 0;
RenderDelayControllerImpl2::RenderDelayControllerImpl2(
const EchoCanceller3Config& config,
int sample_rate_hz)
: data_dumper_(
new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
use_early_delay_detection_(UseEarlyDelayDetection()),
delay_headroom_blocks_(
static_cast<int>(config.delay.delay_headroom_blocks)),
hysteresis_limit_1_blocks_(
static_cast<int>(config.delay.hysteresis_limit_1_blocks)),
hysteresis_limit_2_blocks_(
static_cast<int>(config.delay.hysteresis_limit_2_blocks)),
delay_estimator_(data_dumper_.get(), config),
last_delay_estimate_quality_(DelayEstimate::Quality::kCoarse) {
RTC_DCHECK(ValidFullBandRate(sample_rate_hz));
delay_estimator_.LogDelayEstimationProperties(sample_rate_hz, 0);
}
RenderDelayControllerImpl2::~RenderDelayControllerImpl2() = default;
void RenderDelayControllerImpl2::Reset(bool reset_delay_confidence) {
delay_ = absl::nullopt;
delay_samples_ = absl::nullopt;
delay_estimator_.Reset(reset_delay_confidence);
delay_change_counter_ = 0;
if (reset_delay_confidence) {
last_delay_estimate_quality_ = DelayEstimate::Quality::kCoarse;
}
}
void RenderDelayControllerImpl2::LogRenderCall() {}
absl::optional<DelayEstimate> RenderDelayControllerImpl2::GetDelay(
const DownsampledRenderBuffer& render_buffer,
size_t render_delay_buffer_delay,
const absl::optional<int>& echo_remover_delay,
rtc::ArrayView<const float> capture) {
RTC_DCHECK_EQ(kBlockSize, capture.size());
++capture_call_counter_;
auto delay_samples = delay_estimator_.EstimateDelay(render_buffer, capture);
// Overrule the delay estimator delay if the echo remover reports a delay.
if (echo_remover_delay) {
int total_echo_remover_delay_samples =
(render_delay_buffer_delay + *echo_remover_delay) * kBlockSize;
delay_samples = DelayEstimate(DelayEstimate::Quality::kRefined,
total_echo_remover_delay_samples);
}
if (delay_samples) {
if (!delay_samples_ || delay_samples->delay != delay_samples_->delay) {
delay_change_counter_ = 0;
}
if (delay_samples_) {
delay_samples_->blocks_since_last_change =
delay_samples_->delay == delay_samples->delay
? delay_samples_->blocks_since_last_change + 1
: 0;
delay_samples_->blocks_since_last_update = 0;
delay_samples_->delay = delay_samples->delay;
delay_samples_->quality = delay_samples->quality;
} else {
delay_samples_ = delay_samples;
}
} else {
if (delay_samples_) {
++delay_samples_->blocks_since_last_change;
++delay_samples_->blocks_since_last_update;
}
}
if (delay_change_counter_ < 2 * kNumBlocksPerSecond) {
++delay_change_counter_;
}
if (delay_samples_) {
// Compute the render delay buffer delay.
const bool use_hysteresis =
last_delay_estimate_quality_ == DelayEstimate::Quality::kRefined &&
delay_samples_->quality == DelayEstimate::Quality::kRefined;
delay_ = ComputeBufferDelay(delay_, delay_headroom_blocks_,
use_hysteresis ? hysteresis_limit_1_blocks_ : 0,
use_hysteresis ? hysteresis_limit_2_blocks_ : 0,
*delay_samples_);
last_delay_estimate_quality_ = delay_samples_->quality;
}
metrics_.Update(delay_samples_ ? absl::optional<size_t>(delay_samples_->delay)
: absl::nullopt,
delay_ ? delay_->delay : 0, 0, delay_estimator_.Clockdrift());
data_dumper_->DumpRaw("aec3_render_delay_controller_delay",
delay_samples ? delay_samples->delay : 0);
data_dumper_->DumpRaw("aec3_render_delay_controller_buffer_delay",
delay_ ? delay_->delay : 0);
return delay_;
}
bool RenderDelayControllerImpl2::HasClockdrift() const {
return delay_estimator_.Clockdrift() != ClockdriftDetector::Level::kNone;
}
} // namespace
RenderDelayController* RenderDelayController::Create2(
const EchoCanceller3Config& config,
int sample_rate_hz) {
return new RenderDelayControllerImpl2(config, sample_rate_hz);
}
} // namespace webrtc

49
modules/audio_processing/aec3/render_delay_controller_unittest.cc
View File

@ -57,14 +57,14 @@ TEST(RenderDelayController, NoRenderSignal) {
for (auto rate : {8000, 16000, 32000, 48000}) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<RenderDelayBuffer> delay_buffer(
RenderDelayBuffer::Create2(config, NumBandsForRate(rate)));
RenderDelayBuffer::Create(config, NumBandsForRate(rate)));
std::unique_ptr<RenderDelayController> delay_controller(
RenderDelayController::Create2(config, rate));
RenderDelayController::Create(config, rate));
for (size_t k = 0; k < 100; ++k) {
auto delay = delay_controller->GetDelay(
delay_buffer->GetDownsampledRenderBuffer(), delay_buffer->Delay(),
echo_remover_delay_, block);
EXPECT_EQ(config.delay.min_echo_path_delay_blocks, delay->delay);
EXPECT_FALSE(delay->delay);
}
}
}
@ -86,9 +86,9 @@ TEST(RenderDelayController, BasicApiCalls) {
std::vector<std::vector<float>> render_block(
NumBandsForRate(rate), std::vector<float>(kBlockSize, 0.f));
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, NumBandsForRate(rate)));
RenderDelayBuffer::Create(config, NumBandsForRate(rate)));
std::unique_ptr<RenderDelayController> delay_controller(
RenderDelayController::Create2(EchoCanceller3Config(), rate));
RenderDelayController::Create(EchoCanceller3Config(), rate));
for (size_t k = 0; k < 10; ++k) {
render_delay_buffer->Insert(render_block);
render_delay_buffer->PrepareCaptureProcessing();
@ -98,7 +98,7 @@ TEST(RenderDelayController, BasicApiCalls) {
render_delay_buffer->Delay(), echo_remover_delay, capture_block);
}
EXPECT_TRUE(delay_blocks);
EXPECT_EQ(config.delay.min_echo_path_delay_blocks, delay_blocks->delay);
EXPECT_FALSE(delay_blocks->delay);
}
}
}
@ -125,9 +125,9 @@ TEST(RenderDelayController, Alignment) {
absl::optional<DelayEstimate> delay_blocks;
SCOPED_TRACE(ProduceDebugText(rate, delay_samples));
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, NumBandsForRate(rate)));
RenderDelayBuffer::Create(config, NumBandsForRate(rate)));
std::unique_ptr<RenderDelayController> delay_controller(
RenderDelayController::Create2(config, rate));
RenderDelayController::Create(config, rate));
DelayBuffer<float> signal_delay_buffer(delay_samples);
for (size_t k = 0; k < (400 + delay_samples / kBlockSize); ++k) {
RandomizeSampleVector(&random_generator, render_block[0]);
@ -174,9 +174,9 @@ TEST(RenderDelayController, NonCausalAlignment) {
absl::optional<DelayEstimate> delay_blocks;
SCOPED_TRACE(ProduceDebugText(rate, -delay_samples));
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, NumBandsForRate(rate)));
RenderDelayBuffer::Create(config, NumBandsForRate(rate)));
std::unique_ptr<RenderDelayController> delay_controller(
RenderDelayController::Create2(EchoCanceller3Config(), rate));
RenderDelayController::Create(EchoCanceller3Config(), rate));
DelayBuffer<float> signal_delay_buffer(-delay_samples);
for (int k = 0;
k < (400 - delay_samples / static_cast<int>(kBlockSize)); ++k) {
@ -216,24 +216,23 @@ TEST(RenderDelayController, AlignmentWithJitter) {
absl::optional<DelayEstimate> delay_blocks;
SCOPED_TRACE(ProduceDebugText(rate, delay_samples));
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, NumBandsForRate(rate)));
RenderDelayBuffer::Create(config, NumBandsForRate(rate)));
std::unique_ptr<RenderDelayController> delay_controller(
RenderDelayController::Create2(config, rate));
RenderDelayController::Create(config, rate));
DelayBuffer<float> signal_delay_buffer(delay_samples);
for (size_t j = 0; j < (1000 + delay_samples / kBlockSize) /
config.delay.api_call_jitter_blocks +
1;
constexpr size_t kMaxTestJitterBlocks = 26;
for (size_t j = 0;
j <
(1000 + delay_samples / kBlockSize) / kMaxTestJitterBlocks + 1;
++j) {
std::vector<std::vector<float>> capture_block_buffer;
for (size_t k = 0; k < (config.delay.api_call_jitter_blocks - 1);
++k) {
for (size_t k = 0; k < (kMaxTestJitterBlocks - 1); ++k) {
RandomizeSampleVector(&random_generator, render_block[0]);
signal_delay_buffer.Delay(render_block[0], capture_block);
capture_block_buffer.push_back(capture_block);
render_delay_buffer->Insert(render_block);
}
for (size_t k = 0; k < (config.delay.api_call_jitter_blocks - 1);
++k) {
for (size_t k = 0; k < (kMaxTestJitterBlocks - 1); ++k) {
render_delay_buffer->PrepareCaptureProcessing();
delay_blocks = delay_controller->GetDelay(
render_delay_buffer->GetDownsampledRenderBuffer(),
@ -271,10 +270,10 @@ TEST(RenderDelayController, InitialHeadroom) {
for (auto rate : {8000, 16000, 32000, 48000}) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, NumBandsForRate(rate)));
RenderDelayBuffer::Create(config, NumBandsForRate(rate)));
std::unique_ptr<RenderDelayController> delay_controller(
RenderDelayController::Create2(config, rate));
RenderDelayController::Create(config, rate));
}
}
}
@ -290,10 +289,10 @@ TEST(RenderDelayController, WrongCaptureSize) {
for (auto rate : {8000, 16000, 32000, 48000}) {
SCOPED_TRACE(ProduceDebugText(rate));
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, NumBandsForRate(rate)));
RenderDelayBuffer::Create(config, NumBandsForRate(rate)));
EXPECT_DEATH(
std::unique_ptr<RenderDelayController>(
RenderDelayController::Create2(EchoCanceller3Config(), rate))
RenderDelayController::Create(EchoCanceller3Config(), rate))
->GetDelay(render_delay_buffer->GetDownsampledRenderBuffer(),
render_delay_buffer->Delay(), echo_remover_delay, block),
"");
@ -308,10 +307,10 @@ TEST(RenderDelayController, DISABLED_WrongSampleRate) {
SCOPED_TRACE(ProduceDebugText(rate));
EchoCanceller3Config config;
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, NumBandsForRate(rate)));
RenderDelayBuffer::Create(config, NumBandsForRate(rate)));
EXPECT_DEATH(
std::unique_ptr<RenderDelayController>(
RenderDelayController::Create2(EchoCanceller3Config(), rate)),
RenderDelayController::Create(EchoCanceller3Config(), rate)),
"");
}
}

5
modules/audio_processing/aec3/render_signal_analyzer_unittest.cc
View File

@ -59,7 +59,7 @@ TEST(RenderSignalAnalyzer, NoFalseDetectionOfNarrowBands) {
std::vector<std::vector<float>> x(3, std::vector<float>(kBlockSize, 0.f));
std::array<float, kBlockSize> x_old;
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(EchoCanceller3Config(), 3));
RenderDelayBuffer::Create(EchoCanceller3Config(), 3));
std::array<float, kFftLengthBy2Plus1> mask;
x_old.fill(0.f);
@ -91,9 +91,8 @@ TEST(RenderSignalAnalyzer, NarrowBandDetection) {
std::array<float, kBlockSize> x_old;
Aec3Fft fft;
EchoCanceller3Config config;
config.delay.min_echo_path_delay_blocks = 0;
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
std::array<float, kFftLengthBy2Plus1> mask;
x_old.fill(0.f);

5
modules/audio_processing/aec3/residual_echo_estimator_unittest.cc
View File

@ -27,7 +27,7 @@ TEST(ResidualEchoEstimator, NullResidualEchoPowerOutput) {
EchoCanceller3Config config;
AecState aec_state(config);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
std::vector<std::array<float, kFftLengthBy2Plus1>> H2;
std::array<float, kFftLengthBy2Plus1> S2_linear;
std::array<float, kFftLengthBy2Plus1> Y2;
@ -44,11 +44,10 @@ TEST(ResidualEchoEstimator, NullResidualEchoPowerOutput) {
TEST(ResidualEchoEstimator, DISABLED_BasicTest) {
EchoCanceller3Config config;
config.ep_strength.default_len = 0.f;
config.delay.min_echo_path_delay_blocks = 0;
ResidualEchoEstimator estimator(config);
AecState aec_state(config);
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
std::array<float, kFftLengthBy2Plus1> E2_main;
std::array<float, kFftLengthBy2Plus1> E2_shadow;

3
modules/audio_processing/aec3/shadow_filter_update_gain_unittest.cc
View File

@ -50,10 +50,9 @@ void RunFilterUpdateTest(int num_blocks_to_process,
DetectOptimization(), &data_dumper);
Aec3Fft fft;
config.delay.min_echo_path_delay_blocks = 0;
config.delay.default_delay = 1;
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
std::array<float, kBlockSize> x_old;
x_old.fill(0.f);

2
modules/audio_processing/aec3/signal_dependent_erle_estimator_unittest.cc
View File

@ -62,7 +62,7 @@ class TestInputs {
};
TestInputs::TestInputs(const EchoCanceller3Config& cfg)
: render_delay_buffer_(RenderDelayBuffer::Create2(cfg, 1)),
: render_delay_buffer_(RenderDelayBuffer::Create(cfg, 1)),
H2_(cfg.filter.main.length_blocks),
x_(1, std::vector<float>(kBlockSize, 0.f)) {
render_delay_buffer_->SetDelay(4);

7
modules/audio_processing/aec3/subtractor_unittest.cc
View File

@ -41,10 +41,9 @@ float RunSubtractorTest(int num_blocks_to_process,
std::vector<float> y(kBlockSize, 0.f);
std::array<float, kBlockSize> x_old;
SubtractorOutput output;
config.delay.min_echo_path_delay_blocks = 0;
config.delay.default_delay = 1;
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
RenderSignalAnalyzer render_signal_analyzer(config);
Random random_generator(42U);
Aec3Fft fft;
@ -127,7 +126,7 @@ TEST(Subtractor, DISABLED_NullOutput) {
EchoCanceller3Config config;
Subtractor subtractor(config, &data_dumper, DetectOptimization());
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
RenderSignalAnalyzer render_signal_analyzer(config);
std::vector<float> y(kBlockSize, 0.f);
@ -143,7 +142,7 @@ TEST(Subtractor, WrongCaptureSize) {
EchoCanceller3Config config;
Subtractor subtractor(config, &data_dumper, DetectOptimization());
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
RenderSignalAnalyzer render_signal_analyzer(config);
std::vector<float> y(kBlockSize - 1, 0.f);
SubtractorOutput output;

2
modules/audio_processing/aec3/suppression_gain_unittest.cc
View File

@ -77,7 +77,7 @@ TEST(SuppressionGain, BasicGainComputation) {
ApmDataDumper data_dumper(42);
Subtractor subtractor(config, &data_dumper, DetectOptimization());
std::unique_ptr<RenderDelayBuffer> render_delay_buffer(
RenderDelayBuffer::Create2(config, 3));
RenderDelayBuffer::Create(config, 3));
absl::optional<DelayEstimate> delay_estimate;
// Ensure that a strong noise is detected to mask any echoes.

1
test/fuzzers/audio_processing_configs_fuzzer.cc
View File

@ -39,7 +39,6 @@ const std::string kFieldTrialNames[] = {
"WebRTC-Aec3FilterAnalyzerPreprocessorKillSwitch",
"WebRTC-Aec3MisadjustmentEstimatorKillSwitch",
"WebRTC-Aec3NewFilterParamsKillSwitch",
"WebRTC-Aec3NewRenderBufferingKillSwitch",
"WebRTC-Aec3OverrideEchoPathGainKillSwitch",
"WebRTC-Aec3RapidAgcGainRecoveryKillSwitch",
"WebRTC-Aec3ResetErleAtGainChangesKillSwitch",