From b1786dbab00dd66a9e59a68414e85b2b2615a24f Mon Sep 17 00:00:00 2001 From: "bjornv@webrtc.org" Date: Tue, 3 Feb 2015 06:06:26 +0000 Subject: [PATCH] audio_processing: Added a new AEC delay metric value that gives the amount of poor delays To more easily determine if for example the AEC is not working properly one could monitor how often the estimated delay is out of bounds. With out of bounds we mean either being negative or too large, where both cases will break the AEC. A new delay metric is added telling the user how often poor delay values were estimated. This is measured in percentage since last time the metrics were calculated. All APIs have been updated with a third parameter with EchoCancellation::GetDelayMetrics() giving the option to exclude the new metric not to break existing code. The new metric has been added to audio_processing_unittests with an additional protobuf member, and reference files accordingly updated. voe_auto_test has not been updated to display the new metric. BUG=4246 TESTED=audioproc on files R=aluebs@webrtc.org, andrew@webrtc.org Review URL: https://webrtc-codereview.appspot.com/39739004 Cr-Commit-Position: refs/heads/master@{#8230} git-svn-id: http://webrtc.googlecode.com/svn/trunk@8230 4adac7df-926f-26a2-2b94-8c16560cd09d --- data/audio_processing/output_data_float.pb | Bin 1529 -> 1594 bytes data/audio_processing/output_data_mac.pb | Bin 1529 -> 1594 bytes .../modules/audio_processing/aec/aec_core.c | 21 +++++++++++++++--- .../modules/audio_processing/aec/aec_core.h | 11 ++++++--- .../audio_processing/aec/aec_core_internal.h | 1 + .../audio_processing/aec/echo_cancellation.c | 9 ++++++-- .../aec/include/echo_cancellation.h | 15 ++++++++----- .../echo_cancellation_impl.cc | 8 ++++++- .../audio_processing/echo_cancellation_impl.h | 2 ++ .../include/audio_processing.h | 12 ++++++++-- .../include/mock_audio_processing.h | 2 ++ .../test/audio_processing_unittest.cc | 19 ++++++++++++---- .../audio_processing/test/process_test.cc | 5 ++++- .../audio_processing/test/unittest.proto | 1 + 14 files changed, 85 insertions(+), 21 deletions(-) diff --git a/data/audio_processing/output_data_float.pb b/data/audio_processing/output_data_float.pb index 13805c05061a7612c78fdc1e55c35c096e081c57..03f757e8d64d6cecd44c34bbec425dd323fd3f06 100644 GIT binary patch delta 345 zcmey#y^DvnA}3|NbFX_y{ZX3FGZMu=e?5W}QPS!KY6rB7yM3PIK*Rl4D!9LThc z$(c-{s4}Ua!7@`OpJPIE+I6VYCMPhXxvd4}wzJG=ZsUi!&6)+pH0jcN(onaZWkGY> mJeb=IS<&343vpY<x6P0$wf+utH6sAnP)E`L delta 280 zcmdnR^OKwHD=Py7SN}w|CIJ=>1_6c)nbNOYr0pvhxUwhCjRlHv0L7$B*KU>viRDb@ zXN-djhUtL?`zJRsLUeNgRkF%}btg@hWD0?*lq%iuP!6O$d2$m|C|of0Ggz>H@*5_E z<=3H>Pi|&L*xmxO{VOxVc7B-c87wgE(xvyLp|<~LLD)VIW_t!J!ggJV?a7lbv4--2 Pyv~v#Rciem=p;q})-Xi% diff --git a/data/audio_processing/output_data_mac.pb b/data/audio_processing/output_data_mac.pb index a74778f222f3b1608062838c81daa072bb2fcc34..47c67f47ea54a2b9ea78242489a194b33609f965 100644 GIT binary patch delta 345 zcmey#y^DvnA}3|NbFX_y{ZX3FGZMu=e?5W}QPS!KY6rB7yM3PIK*Rl4D!9LThc z$(c-{s4}Ua!7@`OpJPIE+I6VYCMPhXxvd4}wzJG=ZsUi!&6)+pH0jcN(onaZWkGY> oJeb=IS<&343vpY<1_6c)nbNOYr0pvhxUwhCjRlHv0L7$B*KU>viRDb@ zXN-djhUtL?`zJRsLUeNgRkF%}btg@hWD0?*lq%iuP!6O$d2$m|C|of0Ggz>H@*5_E z<=3H>Pi|&L*xmxO{VOxVc7B-c87wgE(xvyLp|<~LLD)VIW_t!J!ggJV?a7lbv4--o RaPSDQWJs0T{{T9P5dhZRMFs!> diff --git a/webrtc/modules/audio_processing/aec/aec_core.c b/webrtc/modules/audio_processing/aec/aec_core.c index 1f864fcdc8..2cd7d852bd 100644 --- a/webrtc/modules/audio_processing/aec/aec_core.c +++ b/webrtc/modules/audio_processing/aec/aec_core.c @@ -774,6 +774,7 @@ static void UpdateDelayMetrics(AecCore* self) { int i = 0; int delay_values = 0; int median = 0; + int lookahead = WebRtc_lookahead(self->delay_estimator); const int kMsPerBlock = PART_LEN / (self->mult * 8); int64_t l1_norm = 0; @@ -785,6 +786,7 @@ static void UpdateDelayMetrics(AecCore* self) { // not able to estimate the delay. self->delay_median = -1; self->delay_std = -1; + self->fraction_poor_delays = -1; return; } @@ -799,8 +801,7 @@ static void UpdateDelayMetrics(AecCore* self) { } } // Account for lookahead. - self->delay_median = (median - WebRtc_lookahead(self->delay_estimator)) * - kMsPerBlock; + self->delay_median = (median - lookahead) * kMsPerBlock; // Calculate the L1 norm, with median value as central moment. for (i = 0; i < kHistorySizeBlocks; i++) { @@ -809,6 +810,17 @@ static void UpdateDelayMetrics(AecCore* self) { self->delay_std = (int)((l1_norm + self->num_delay_values / 2) / self->num_delay_values) * kMsPerBlock; + // Determine fraction of delays that are out of bounds, that is, either + // negative (anti-causal system) or larger than the AEC filter length. + { + int num_delays_out_of_bounds = self->num_delay_values; + for (i = lookahead; i < lookahead + self->num_partitions; ++i) { + num_delays_out_of_bounds -= self->delay_histogram[i]; + } + self->fraction_poor_delays = (float)num_delays_out_of_bounds / + self->num_delay_values; + } + // Reset histogram. memset(self->delay_histogram, 0, sizeof(self->delay_histogram)); self->num_delay_values = 0; @@ -1563,6 +1575,7 @@ int WebRtcAec_InitAec(AecCore* aec, int sampFreq) { aec->num_delay_values = 0; aec->delay_median = -1; aec->delay_std = -1; + aec->fraction_poor_delays = -1; aec->signal_delay_correction = 0; aec->previous_delay = -2; // (-2): Uninitialized. @@ -1833,7 +1846,8 @@ void WebRtcAec_ProcessFrames(AecCore* aec, } } -int WebRtcAec_GetDelayMetricsCore(AecCore* self, int* median, int* std) { +int WebRtcAec_GetDelayMetricsCore(AecCore* self, int* median, int* std, + float* fraction_poor_delays) { assert(self != NULL); assert(median != NULL); assert(std != NULL); @@ -1849,6 +1863,7 @@ int WebRtcAec_GetDelayMetricsCore(AecCore* self, int* median, int* std) { } *median = self->delay_median; *std = self->delay_std; + *fraction_poor_delays = self->fraction_poor_delays; return 0; } diff --git a/webrtc/modules/audio_processing/aec/aec_core.h b/webrtc/modules/audio_processing/aec/aec_core.h index 2bffeec6e4..2b614dd83f 100644 --- a/webrtc/modules/audio_processing/aec/aec_core.h +++ b/webrtc/modules/audio_processing/aec/aec_core.h @@ -75,9 +75,14 @@ void WebRtcAec_ProcessFrames(AecCore* aec, // corresponding amount in ms. int WebRtcAec_MoveFarReadPtr(AecCore* aec, int elements); -// Calculates the median and standard deviation among the delay estimates -// collected since the last call to this function. -int WebRtcAec_GetDelayMetricsCore(AecCore* self, int* median, int* std); +// Calculates the median, standard deviation and amount of poor values among the +// delay estimates aggregated up to the first call to the function. After that +// first call the metrics are aggregated and updated every second. With poor +// values we mean values that most likely will cause the AEC to perform poorly. +// TODO(bjornv): Consider changing tests and tools to handle constant +// constant aggregation window throughout the session instead. +int WebRtcAec_GetDelayMetricsCore(AecCore* self, int* median, int* std, + float* fraction_poor_delays); // Returns the echo state (1: echo, 0: no echo). int WebRtcAec_echo_state(AecCore* self); diff --git a/webrtc/modules/audio_processing/aec/aec_core_internal.h b/webrtc/modules/audio_processing/aec/aec_core_internal.h index b9950f6f38..bdb90413a7 100644 --- a/webrtc/modules/audio_processing/aec/aec_core_internal.h +++ b/webrtc/modules/audio_processing/aec/aec_core_internal.h @@ -131,6 +131,7 @@ struct AecCore { int num_delay_values; int delay_median; int delay_std; + float fraction_poor_delays; int delay_logging_enabled; void* delay_estimator_farend; void* delay_estimator; diff --git a/webrtc/modules/audio_processing/aec/echo_cancellation.c b/webrtc/modules/audio_processing/aec/echo_cancellation.c index 470fe0545d..016c45538f 100644 --- a/webrtc/modules/audio_processing/aec/echo_cancellation.c +++ b/webrtc/modules/audio_processing/aec/echo_cancellation.c @@ -556,7 +556,10 @@ int WebRtcAec_GetMetrics(void* handle, AecMetrics* metrics) { return 0; } -int WebRtcAec_GetDelayMetrics(void* handle, int* median, int* std) { +int WebRtcAec_GetDelayMetrics(void* handle, + int* median, + int* std, + float* fraction_poor_delays) { Aec* self = handle; if (median == NULL) { self->lastError = AEC_NULL_POINTER_ERROR; @@ -570,7 +573,9 @@ int WebRtcAec_GetDelayMetrics(void* handle, int* median, int* std) { self->lastError = AEC_UNINITIALIZED_ERROR; return -1; } - if (WebRtcAec_GetDelayMetricsCore(self->aec, median, std) == -1) { + if (WebRtcAec_GetDelayMetricsCore(self->aec, median, std, + fraction_poor_delays) == + -1) { // Logging disabled. self->lastError = AEC_UNSUPPORTED_FUNCTION_ERROR; return -1; diff --git a/webrtc/modules/audio_processing/aec/include/echo_cancellation.h b/webrtc/modules/audio_processing/aec/include/echo_cancellation.h index 51eb37a72d..df9b2a90cb 100644 --- a/webrtc/modules/audio_processing/aec/include/echo_cancellation.h +++ b/webrtc/modules/audio_processing/aec/include/echo_cancellation.h @@ -211,17 +211,22 @@ int WebRtcAec_GetMetrics(void* handle, AecMetrics* metrics); * * Inputs Description * ------------------------------------------------------------------- - * void* handle Pointer to the AEC instance + * void* handle Pointer to the AEC instance * * Outputs Description * ------------------------------------------------------------------- - * int* median Delay median value. - * int* std Delay standard deviation. + * int* median Delay median value. + * int* std Delay standard deviation. + * float* fraction_poor_delays Fraction of the delay estimates that may + * cause the AEC to perform poorly. * - * int return 0: OK + * int return 0: OK * -1: error */ -int WebRtcAec_GetDelayMetrics(void* handle, int* median, int* std); +int WebRtcAec_GetDelayMetrics(void* handle, + int* median, + int* std, + float* fraction_poor_delays); /* * Gets the last error code. diff --git a/webrtc/modules/audio_processing/echo_cancellation_impl.cc b/webrtc/modules/audio_processing/echo_cancellation_impl.cc index bf5ed72db6..dd3c7f1cc5 100644 --- a/webrtc/modules/audio_processing/echo_cancellation_impl.cc +++ b/webrtc/modules/audio_processing/echo_cancellation_impl.cc @@ -281,6 +281,12 @@ bool EchoCancellationImpl::is_delay_logging_enabled() const { // TODO(bjornv): How should we handle the multi-channel case? int EchoCancellationImpl::GetDelayMetrics(int* median, int* std) { + float fraction_poor_delays = 0; + return GetDelayMetrics(median, std, &fraction_poor_delays); +} + +int EchoCancellationImpl::GetDelayMetrics(int* median, int* std, + float* fraction_poor_delays) { CriticalSectionScoped crit_scoped(crit_); if (median == NULL) { return apm_->kNullPointerError; @@ -294,7 +300,7 @@ int EchoCancellationImpl::GetDelayMetrics(int* median, int* std) { } Handle* my_handle = static_cast(handle(0)); - if (WebRtcAec_GetDelayMetrics(my_handle, median, std) != + if (WebRtcAec_GetDelayMetrics(my_handle, median, std, fraction_poor_delays) != apm_->kNoError) { return GetHandleError(my_handle); } diff --git a/webrtc/modules/audio_processing/echo_cancellation_impl.h b/webrtc/modules/audio_processing/echo_cancellation_impl.h index b9c116a065..4545938957 100644 --- a/webrtc/modules/audio_processing/echo_cancellation_impl.h +++ b/webrtc/modules/audio_processing/echo_cancellation_impl.h @@ -52,6 +52,8 @@ class EchoCancellationImpl : public EchoCancellation, virtual int enable_delay_logging(bool enable) OVERRIDE; virtual bool is_delay_logging_enabled() const OVERRIDE; virtual int GetDelayMetrics(int* median, int* std) OVERRIDE; + virtual int GetDelayMetrics(int* median, int* std, + float* fraction_poor_delays) OVERRIDE; virtual struct AecCore* aec_core() const OVERRIDE; // ProcessingComponent implementation. diff --git a/webrtc/modules/audio_processing/include/audio_processing.h b/webrtc/modules/audio_processing/include/audio_processing.h index 6b761e14bf..3715b34b36 100644 --- a/webrtc/modules/audio_processing/include/audio_processing.h +++ b/webrtc/modules/audio_processing/include/audio_processing.h @@ -487,9 +487,17 @@ class EchoCancellation { virtual bool is_delay_logging_enabled() const = 0; // The delay metrics consists of the delay |median| and the delay standard - // deviation |std|. The values are averaged over the time period since the - // last call to |GetDelayMetrics()|. + // deviation |std|. It also consists of the fraction of delay estimates + // |fraction_poor_delays| that can make the echo cancellation perform poorly. + // The values are aggregated until the first call to |GetDelayMetrics()| and + // afterwards aggregated and updated every second. + // Note that if there are several clients pulling metrics from + // |GetDelayMetrics()| during a session the first call from any of them will + // change to one second aggregation window for all. + // TODO(bjornv): Deprecated, remove. virtual int GetDelayMetrics(int* median, int* std) = 0; + virtual int GetDelayMetrics(int* median, int* std, + float* fraction_poor_delays) = 0; // Returns a pointer to the low level AEC component. In case of multiple // channels, the pointer to the first one is returned. A NULL pointer is diff --git a/webrtc/modules/audio_processing/include/mock_audio_processing.h b/webrtc/modules/audio_processing/include/mock_audio_processing.h index 8258bb6cff..46a04aee2e 100644 --- a/webrtc/modules/audio_processing/include/mock_audio_processing.h +++ b/webrtc/modules/audio_processing/include/mock_audio_processing.h @@ -48,6 +48,8 @@ class MockEchoCancellation : public EchoCancellation { bool()); MOCK_METHOD2(GetDelayMetrics, int(int* median, int* std)); + MOCK_METHOD3(GetDelayMetrics, + int(int* median, int* std, float* fraction_poor_delays)); MOCK_CONST_METHOD0(aec_core, struct AecCore*()); }; diff --git a/webrtc/modules/audio_processing/test/audio_processing_unittest.cc b/webrtc/modules/audio_processing/test/audio_processing_unittest.cc index 7e36d67927..093c72ede5 100644 --- a/webrtc/modules/audio_processing/test/audio_processing_unittest.cc +++ b/webrtc/modules/audio_processing/test/audio_processing_unittest.cc @@ -635,9 +635,11 @@ void ApmTest::ProcessDelayVerificationTest(int delay_ms, int system_delay_ms, if (frame_count == 250) { int median; int std; + float poor_fraction; // Discard the first delay metrics to avoid convergence effects. EXPECT_EQ(apm_->kNoError, - apm_->echo_cancellation()->GetDelayMetrics(&median, &std)); + apm_->echo_cancellation()->GetDelayMetrics(&median, &std, + &poor_fraction)); } } @@ -659,8 +661,10 @@ void ApmTest::ProcessDelayVerificationTest(int delay_ms, int system_delay_ms, // Verify delay metrics. int median; int std; + float poor_fraction; EXPECT_EQ(apm_->kNoError, - apm_->echo_cancellation()->GetDelayMetrics(&median, &std)); + apm_->echo_cancellation()->GetDelayMetrics(&median, &std, + &poor_fraction)); EXPECT_GE(expected_median_high, median); EXPECT_LE(expected_median_low, median); } @@ -847,8 +851,10 @@ TEST_F(ApmTest, EchoCancellation) { int median = 0; int std = 0; + float poor_fraction = 0; EXPECT_EQ(apm_->kNotEnabledError, - apm_->echo_cancellation()->GetDelayMetrics(&median, &std)); + apm_->echo_cancellation()->GetDelayMetrics(&median, &std, + &poor_fraction)); EXPECT_EQ(apm_->kNoError, apm_->echo_cancellation()->enable_delay_logging(true)); @@ -2026,8 +2032,10 @@ TEST_F(ApmTest, Process) { apm_->echo_cancellation()->GetMetrics(&echo_metrics)); int median = 0; int std = 0; + float fraction_poor_delays = 0; EXPECT_EQ(apm_->kNoError, - apm_->echo_cancellation()->GetDelayMetrics(&median, &std)); + apm_->echo_cancellation()->GetDelayMetrics( + &median, &std, &fraction_poor_delays)); int rms_level = apm_->level_estimator()->RMS(); EXPECT_LE(0, rms_level); @@ -2079,6 +2087,8 @@ TEST_F(ApmTest, Process) { audioproc::Test::DelayMetrics reference_delay = test->delay_metrics(); EXPECT_NEAR(reference_delay.median(), median, kIntNear); EXPECT_NEAR(reference_delay.std(), std, kIntNear); + EXPECT_NEAR(reference_delay.fraction_poor_delays(), fraction_poor_delays, + kFloatNear); EXPECT_NEAR(test->rms_level(), rms_level, kIntNear); @@ -2109,6 +2119,7 @@ TEST_F(ApmTest, Process) { test->mutable_delay_metrics(); message_delay->set_median(median); message_delay->set_std(std); + message_delay->set_fraction_poor_delays(fraction_poor_delays); test->set_rms_level(rms_level); diff --git a/webrtc/modules/audio_processing/test/process_test.cc b/webrtc/modules/audio_processing/test/process_test.cc index 469ecf9760..3af495c19b 100644 --- a/webrtc/modules/audio_processing/test/process_test.cc +++ b/webrtc/modules/audio_processing/test/process_test.cc @@ -1081,10 +1081,13 @@ void void_main(int argc, char* argv[]) { if (apm->echo_cancellation()->is_delay_logging_enabled()) { int median = 0; int std = 0; - apm->echo_cancellation()->GetDelayMetrics(&median, &std); + float fraction_poor_delays = 0; + apm->echo_cancellation()->GetDelayMetrics(&median, &std, + &fraction_poor_delays); printf("\n--Delay metrics--\n"); printf("Median: %3d\n", median); printf("Standard deviation: %3d\n", std); + printf("Poor delay values: %3.1f%%\n", fraction_poor_delays * 100); } } diff --git a/webrtc/modules/audio_processing/test/unittest.proto b/webrtc/modules/audio_processing/test/unittest.proto index 47b962b997..ddce46b775 100644 --- a/webrtc/modules/audio_processing/test/unittest.proto +++ b/webrtc/modules/audio_processing/test/unittest.proto @@ -39,6 +39,7 @@ message Test { message DelayMetrics { optional int32 median = 1; optional int32 std = 2; + optional float fraction_poor_delays = 3; } optional DelayMetrics delay_metrics = 12;