From 1973ba6ef4909aad7c8bce307d8bf4180ce677fe Mon Sep 17 00:00:00 2001 From: ivoc Date: Mon, 20 Mar 2017 03:03:16 -0700 Subject: [PATCH] Improve stability of the echo detector complexity perf tests. The results of the echo detector complexity tests are currently notoriously spiky and unreliable. The following improvements are made in this CL: - Significantly longer warmup time before starting the test - More iterations and larger batches - Different number of iterations for slow and fast tests - Use the echo likelihood in the test so it cannot get optimized out BUG=webrtc:7353 Review-Url: https://codereview.webrtc.org/2750413002 Cr-Commit-Position: refs/heads/master@{#17303} --- ...idual_echo_detector_complexity_unittest.cc | 53 ++++++++++++------- .../test/performance_timer.cc | 34 ++++++++---- .../audio_processing/test/performance_timer.h | 5 ++ 3 files changed, 64 insertions(+), 28 deletions(-) diff --git a/webrtc/modules/audio_processing/residual_echo_detector_complexity_unittest.cc b/webrtc/modules/audio_processing/residual_echo_detector_complexity_unittest.cc index f1cf6fa3f8..a42b408b8b 100644 --- a/webrtc/modules/audio_processing/residual_echo_detector_complexity_unittest.cc +++ b/webrtc/modules/audio_processing/residual_echo_detector_complexity_unittest.cc @@ -26,16 +26,24 @@ namespace webrtc { namespace { -const size_t kNumFramesToProcess = 500; -const size_t kProcessingBatchSize = 20; -const size_t kWarmupBatchSize = 2 * kProcessingBatchSize; -const int kSampleRate = AudioProcessing::kSampleRate48kHz; -const int kNumberOfChannels = 1; +constexpr size_t kNumFramesToProcess = 20000; +constexpr size_t kNumFramesToProcessStandalone = 50 * kNumFramesToProcess; +constexpr size_t kProcessingBatchSize = 200; +constexpr size_t kProcessingBatchSizeStandalone = 50 * kProcessingBatchSize; +constexpr size_t kNumberOfWarmupMeasurements = + (kNumFramesToProcess / kProcessingBatchSize) / 2; +constexpr size_t kNumberOfWarmupMeasurementsStandalone = + (kNumFramesToProcessStandalone / kProcessingBatchSizeStandalone) / 2; +constexpr int kSampleRate = AudioProcessing::kSampleRate48kHz; +constexpr int kNumberOfChannels = 1; -std::string FormPerformanceMeasureString(const test::PerformanceTimer& timer) { - std::string s = std::to_string(timer.GetDurationAverage()); +std::string FormPerformanceMeasureString(const test::PerformanceTimer& timer, + int number_of_warmup_samples) { + std::string s = + std::to_string(timer.GetDurationAverage(number_of_warmup_samples)); s += ", "; - s += std::to_string(timer.GetDurationStandardDeviation()); + s += std::to_string( + timer.GetDurationStandardDeviation(number_of_warmup_samples)); return s; } @@ -43,16 +51,19 @@ void RunStandaloneSubmodule() { test::SimulatorBuffers buffers( kSampleRate, kSampleRate, kSampleRate, kSampleRate, kNumberOfChannels, kNumberOfChannels, kNumberOfChannels, kNumberOfChannels); - test::PerformanceTimer timer(kNumFramesToProcess); + test::PerformanceTimer timer(kNumFramesToProcessStandalone / + kProcessingBatchSizeStandalone); ResidualEchoDetector echo_detector; echo_detector.Initialize(); + float sum = 0.f; - for (size_t frame_no = 0; frame_no < kNumFramesToProcess; ++frame_no) { + for (size_t frame_no = 0; frame_no < kNumFramesToProcessStandalone; + ++frame_no) { // The first batch of frames are for warming up, and are not part of the // benchmark. After that the processing time is measured in chunks of // kProcessingBatchSize frames. - if (frame_no >= kWarmupBatchSize && frame_no % kProcessingBatchSize == 0) { + if (frame_no % kProcessingBatchSizeStandalone == 0) { timer.StartTimer(); } @@ -63,15 +74,19 @@ void RunStandaloneSubmodule() { echo_detector.AnalyzeCaptureAudio(rtc::ArrayView( buffers.capture_input_buffer->split_bands_const_f(0)[kBand0To8kHz], buffers.capture_input_buffer->num_frames_per_band())); + sum += echo_detector.echo_likelihood(); - if (frame_no >= kWarmupBatchSize && - frame_no % kProcessingBatchSize == kProcessingBatchSize - 1) { + if (frame_no % kProcessingBatchSizeStandalone == + kProcessingBatchSizeStandalone - 1) { timer.StopTimer(); } } + EXPECT_EQ(0.0f, sum); webrtc::test::PrintResultMeanAndError( "echo_detector_call_durations", "", "StandaloneEchoDetector", - FormPerformanceMeasureString(timer), "us", false); + FormPerformanceMeasureString(timer, + kNumberOfWarmupMeasurementsStandalone), + "us", false); } void RunTogetherWithApm(std::string test_description, @@ -80,7 +95,7 @@ void RunTogetherWithApm(std::string test_description, test::SimulatorBuffers buffers( kSampleRate, kSampleRate, kSampleRate, kSampleRate, kNumberOfChannels, kNumberOfChannels, kNumberOfChannels, kNumberOfChannels); - test::PerformanceTimer timer(kNumFramesToProcess); + test::PerformanceTimer timer(kNumFramesToProcess / kProcessingBatchSize); webrtc::Config config; AudioProcessing::Config apm_config; @@ -124,7 +139,7 @@ void RunTogetherWithApm(std::string test_description, // The first batch of frames are for warming up, and are not part of the // benchmark. After that the processing time is measured in chunks of // kProcessingBatchSize frames. - if (frame_no >= kWarmupBatchSize && frame_no % kProcessingBatchSize == 0) { + if (frame_no % kProcessingBatchSize == 0) { timer.StartTimer(); } @@ -146,15 +161,15 @@ void RunTogetherWithApm(std::string test_description, apm->ProcessStream(&buffers.capture_input[0], stream_config, stream_config, &buffers.capture_output[0])); - if (frame_no >= kWarmupBatchSize && - frame_no % kProcessingBatchSize == kProcessingBatchSize - 1) { + if (frame_no % kProcessingBatchSize == kProcessingBatchSize - 1) { timer.StopTimer(); } } webrtc::test::PrintResultMeanAndError( "echo_detector_call_durations", "_total", test_description, - FormPerformanceMeasureString(timer), "us", false); + FormPerformanceMeasureString(timer, kNumberOfWarmupMeasurements), "us", + false); } } // namespace diff --git a/webrtc/modules/audio_processing/test/performance_timer.cc b/webrtc/modules/audio_processing/test/performance_timer.cc index c4fcc0a712..a002fe35a0 100644 --- a/webrtc/modules/audio_processing/test/performance_timer.cc +++ b/webrtc/modules/audio_processing/test/performance_timer.cc @@ -36,23 +36,39 @@ void PerformanceTimer::StopTimer() { } double PerformanceTimer::GetDurationAverage() const { - RTC_DCHECK(!timestamps_us_.empty()); - return static_cast( - std::accumulate(timestamps_us_.begin(), timestamps_us_.end(), 0)) / - timestamps_us_.size(); + return GetDurationAverage(0); } double PerformanceTimer::GetDurationStandardDeviation() const { - RTC_DCHECK(!timestamps_us_.empty()); - double average_duration = GetDurationAverage(); + return GetDurationStandardDeviation(0); +} + +double PerformanceTimer::GetDurationAverage( + size_t number_of_warmup_samples) const { + RTC_DCHECK_GT(timestamps_us_.size(), number_of_warmup_samples); + const size_t number_of_samples = + timestamps_us_.size() - number_of_warmup_samples; + return static_cast( + std::accumulate(timestamps_us_.begin() + number_of_warmup_samples, + timestamps_us_.end(), static_cast(0))) / + number_of_samples; +} + +double PerformanceTimer::GetDurationStandardDeviation( + size_t number_of_warmup_samples) const { + RTC_DCHECK_GT(timestamps_us_.size(), number_of_warmup_samples); + const size_t number_of_samples = + timestamps_us_.size() - number_of_warmup_samples; + RTC_DCHECK_GT(number_of_samples, 0); + double average_duration = GetDurationAverage(number_of_warmup_samples); double variance = std::accumulate( - timestamps_us_.begin(), timestamps_us_.end(), 0.0, - [average_duration](const double& a, const int64_t& b) { + timestamps_us_.begin() + number_of_warmup_samples, timestamps_us_.end(), + 0.0, [average_duration](const double& a, const int64_t& b) { return a + (b - average_duration) * (b - average_duration); }); - return sqrt(variance / timestamps_us_.size()); + return sqrt(variance / number_of_samples); } } // namespace test diff --git a/webrtc/modules/audio_processing/test/performance_timer.h b/webrtc/modules/audio_processing/test/performance_timer.h index 5921fae070..92cda8bd5d 100644 --- a/webrtc/modules/audio_processing/test/performance_timer.h +++ b/webrtc/modules/audio_processing/test/performance_timer.h @@ -30,6 +30,11 @@ class PerformanceTimer { double GetDurationAverage() const; double GetDurationStandardDeviation() const; + // These methods are the same as those above, but they ignore the first + // |number_of_warmup_samples| measurements. + double GetDurationAverage(size_t number_of_warmup_samples) const; + double GetDurationStandardDeviation(size_t number_of_warmup_samples) const; + private: webrtc::Clock* clock_; rtc::Optional start_timestamp_us_;