admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allowing reduced computations in the AEC3 when the output is not used

Browse Source 
This CL adds functionality in AEC3 that allows the computational
complexity to be reduced when the output of APM is not used.

Bug: b/177830919
Change-Id: I08121364bf966f34311f54ffa5affbfd8b4db1e2
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/211341
Reviewed-by: Gustaf Ullberg <gustaf@webrtc.org>
Commit-Queue: Per Åhgren <peah@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#33476}
This commit is contained in:
Per Åhgren 2021-03-11 06:33:45 +00:00 committed by Commit Bot
parent 3e774f64b0
commit 8ee1ec82e4
8 changed files with 73 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
modules/audio_processing/aec3/block_processor.cc
View File

@ -63,6 +63,7 @@ class BlockProcessorImpl final : public BlockProcessor {
void GetMetrics(EchoControl::Metrics* metrics) const override;
void SetAudioBufferDelay(int delay_ms) override;
void SetCaptureOutputUsage(bool capture_output_used) override;
private:
static int instance_count_;
@ -237,6 +238,10 @@ void BlockProcessorImpl::SetAudioBufferDelay(int delay_ms) {
render_buffer_->SetAudioBufferDelay(delay_ms);
}
void BlockProcessorImpl::SetCaptureOutputUsage(bool capture_output_used) {
echo_remover_->SetCaptureOutputUsage(capture_output_used);
}
} // namespace
BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config,

6
modules/audio_processing/aec3/block_processor.h
View File

@ -69,6 +69,12 @@ class BlockProcessor {
// Reports whether echo leakage has been detected in the echo canceller
// output.
virtual void UpdateEchoLeakageStatus(bool leakage_detected) = 0;
// Specifies whether the capture output will be used. The purpose of this is
// to allow the block processor to deactivate some of the processing when the
// resulting output is anyway not used, for instance when the endpoint is
// muted.
virtual void SetCaptureOutputUsage(bool capture_output_used) = 0;
};
} // namespace webrtc

4
modules/audio_processing/aec3/echo_canceller3.cc
View File

@ -834,8 +834,8 @@ void EchoCanceller3::SetAudioBufferDelay(int delay_ms) {
}
void EchoCanceller3::SetCaptureOutputUsage(bool capture_output_used) {
// TODO(b/177830919): Add functionality for reducing the complexity when the
// echo canceller output is not used.
RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_);
block_processor_->SetCaptureOutputUsage(capture_output_used);
}
bool EchoCanceller3::ActiveProcessing() const {

4
modules/audio_processing/aec3/echo_canceller3_unittest.cc
View File

@ -131,6 +131,8 @@ class CaptureTransportVerificationProcessor : public BlockProcessor {
void GetMetrics(EchoControl::Metrics* metrics) const override {}
void SetAudioBufferDelay(int delay_ms) override {}
void SetCaptureOutputUsage(bool capture_output_used) {}
};
// Class for testing that the render data is properly received by the block
@ -169,6 +171,8 @@ class RenderTransportVerificationProcessor : public BlockProcessor {
void SetAudioBufferDelay(int delay_ms) override {}
void SetCaptureOutputUsage(bool capture_output_used) {}
private:
std::deque<std::vector<std::vector<std::vector<float>>>>
received_render_blocks_;

72
modules/audio_processing/aec3/echo_remover.cc
View File

@ -132,6 +132,10 @@ class EchoRemoverImpl final : public EchoRemover {
echo_leakage_detected_ = leakage_detected;
}
void SetCaptureOutputUsage(bool capture_output_used) override {
capture_output_used_ = capture_output_used;
}
private:
// Selects which of the coarse and refined linear filter outputs that is most
// appropriate to pass to the suppressor and forms the linear filter output by
@ -155,6 +159,7 @@ class EchoRemoverImpl final : public EchoRemover {
RenderSignalAnalyzer render_signal_analyzer_;
ResidualEchoEstimator residual_echo_estimator_;
bool echo_leakage_detected_ = false;
bool capture_output_used_ = true;
AecState aec_state_;
EchoRemoverMetrics metrics_;
std::vector<std::array<float, kFftLengthBy2>> e_old_;
@ -391,42 +396,49 @@ void EchoRemoverImpl::ProcessCapture(
1);
data_dumper_->DumpWav("aec3_output_linear2", kBlockSize, &e[0][0], 16000, 1);
// Estimate the residual echo power.
residual_echo_estimator_.Estimate(aec_state_, *render_buffer, S2_linear, Y2,
R2);
// Estimate the comfort noise.
cng_.Compute(aec_state_.SaturatedCapture(), Y2, comfort_noise,
high_band_comfort_noise);
// Suppressor nearend estimate.
if (aec_state_.UsableLinearEstimate()) {
// E2 is bound by Y2.
for (size_t ch = 0; ch < num_capture_channels_; ++ch) {
std::transform(E2[ch].begin(), E2[ch].end(), Y2[ch].begin(),
E2[ch].begin(),
[](float a, float b) { return std::min(a, b); });
}
}
const auto& nearend_spectrum = aec_state_.UsableLinearEstimate() ? E2 : Y2;
// Suppressor echo estimate.
const auto& echo_spectrum =
aec_state_.UsableLinearEstimate() ? S2_linear : R2;
// Determine if the suppressor should assume clock drift.
const bool clock_drift = config_.echo_removal_control.has_clock_drift ||
echo_path_variability.clock_drift;
// Compute preferred gains.
float high_bands_gain;
// Only do the below processing if the output of the audio processing module
// is used.
std::array<float, kFftLengthBy2Plus1> G;
suppression_gain_.GetGain(nearend_spectrum, echo_spectrum, R2,
cng_.NoiseSpectrum(), render_signal_analyzer_,
aec_state_, x, clock_drift, &high_bands_gain, &G);
if (capture_output_used_) {
// Estimate the residual echo power.
residual_echo_estimator_.Estimate(aec_state_, *render_buffer, S2_linear, Y2,
R2);
suppression_filter_.ApplyGain(comfort_noise, high_band_comfort_noise, G,
high_bands_gain, Y_fft, y);
// Suppressor nearend estimate.
if (aec_state_.UsableLinearEstimate()) {
// E2 is bound by Y2.
for (size_t ch = 0; ch < num_capture_channels_; ++ch) {
std::transform(E2[ch].begin(), E2[ch].end(), Y2[ch].begin(),
E2[ch].begin(),
[](float a, float b) { return std::min(a, b); });
}
}
const auto& nearend_spectrum = aec_state_.UsableLinearEstimate() ? E2 : Y2;
// Suppressor echo estimate.
const auto& echo_spectrum =
aec_state_.UsableLinearEstimate() ? S2_linear : R2;
// Determine if the suppressor should assume clock drift.
const bool clock_drift = config_.echo_removal_control.has_clock_drift ||
echo_path_variability.clock_drift;
// Compute preferred gains.
float high_bands_gain;
suppression_gain_.GetGain(nearend_spectrum, echo_spectrum, R2,
cng_.NoiseSpectrum(), render_signal_analyzer_,
aec_state_, x, clock_drift, &high_bands_gain, &G);
suppression_filter_.ApplyGain(comfort_noise, high_band_comfort_noise, G,
high_bands_gain, Y_fft, y);
} else {
G.fill(0.f);
}
// Update the metrics.
metrics_.Update(aec_state_, cng_.NoiseSpectrum()[0], G);

6
modules/audio_processing/aec3/echo_remover.h
View File

@ -48,6 +48,12 @@ class EchoRemover {
// Updates the status on whether echo leakage is detected in the output of the
// echo remover.
virtual void UpdateEchoLeakageStatus(bool leakage_detected) = 0;
// Specifies whether the capture output will be used. The purpose of this is
// to allow the echo remover to deactivate some of the processing when the
// resulting output is anyway not used, for instance when the endpoint is
// muted.
virtual void SetCaptureOutputUsage(bool capture_output_used) = 0;
};
} // namespace webrtc

4
modules/audio_processing/aec3/mock/mock_block_processor.h
View File

@ -44,6 +44,10 @@ class MockBlockProcessor : public BlockProcessor {
(EchoControl::Metrics * metrics),
(const, override));
MOCK_METHOD(void, SetAudioBufferDelay, (int delay_ms), (override));
MOCK_METHOD(void,
SetCaptureOutputUsage,
(bool capture_output_used),
(override));
};
} // namespace test

4
modules/audio_processing/aec3/mock/mock_echo_remover.h
View File

@ -44,6 +44,10 @@ class MockEchoRemover : public EchoRemover {
GetMetrics,
(EchoControl::Metrics * metrics),
(const, override));
MOCK_METHOD(void,
SetCaptureOutputUsage,
(bool capture_output_used),
(override));
};
} // namespace test