admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve stability of the echo detector complexity perf tests.

Browse Source 
The results of the echo detector complexity tests are currently notoriously spiky and unreliable. The following improvements are made in this CL:
- Significantly longer warmup time before starting the test
- More iterations and larger batches
- Different number of iterations for slow and fast tests
- Use the echo likelihood in the test so it cannot get optimized out

BUG=webrtc:7353

Review-Url: https://codereview.webrtc.org/2750413002
Cr-Commit-Position: refs/heads/master@{#17303}
This commit is contained in:
ivoc 2017-03-20 03:03:16 -07:00 committed by Commit bot
parent 5e1ca78705
commit 1973ba6ef4
3 changed files with 64 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
webrtc/modules/audio_processing/residual_echo_detector_complexity_unittest.cc
View File

@ -26,16 +26,24 @@
namespace webrtc {
namespace {
const size_t kNumFramesToProcess = 500;
const size_t kProcessingBatchSize = 20;
const size_t kWarmupBatchSize = 2 * kProcessingBatchSize;
const int kSampleRate = AudioProcessing::kSampleRate48kHz;
const int kNumberOfChannels = 1;
constexpr size_t kNumFramesToProcess = 20000;
constexpr size_t kNumFramesToProcessStandalone = 50 * kNumFramesToProcess;
constexpr size_t kProcessingBatchSize = 200;
constexpr size_t kProcessingBatchSizeStandalone = 50 * kProcessingBatchSize;
constexpr size_t kNumberOfWarmupMeasurements =
(kNumFramesToProcess / kProcessingBatchSize) / 2;
constexpr size_t kNumberOfWarmupMeasurementsStandalone =
(kNumFramesToProcessStandalone / kProcessingBatchSizeStandalone) / 2;
constexpr int kSampleRate = AudioProcessing::kSampleRate48kHz;
constexpr int kNumberOfChannels = 1;
std::string FormPerformanceMeasureString(const test::PerformanceTimer& timer) {
std::string s = std::to_string(timer.GetDurationAverage());
std::string FormPerformanceMeasureString(const test::PerformanceTimer& timer,
int number_of_warmup_samples) {
std::string s =
std::to_string(timer.GetDurationAverage(number_of_warmup_samples));
s += ", ";
s += std::to_string(timer.GetDurationStandardDeviation());
s += std::to_string(
timer.GetDurationStandardDeviation(number_of_warmup_samples));
return s;
}
@ -43,16 +51,19 @@ void RunStandaloneSubmodule() {
test::SimulatorBuffers buffers(
kSampleRate, kSampleRate, kSampleRate, kSampleRate, kNumberOfChannels,
kNumberOfChannels, kNumberOfChannels, kNumberOfChannels);
test::PerformanceTimer timer(kNumFramesToProcess);
test::PerformanceTimer timer(kNumFramesToProcessStandalone /
kProcessingBatchSizeStandalone);
ResidualEchoDetector echo_detector;
echo_detector.Initialize();
float sum = 0.f;
for (size_t frame_no = 0; frame_no < kNumFramesToProcess; ++frame_no) {
for (size_t frame_no = 0; frame_no < kNumFramesToProcessStandalone;
++frame_no) {
// The first batch of frames are for warming up, and are not part of the
// benchmark. After that the processing time is measured in chunks of
// kProcessingBatchSize frames.
if (frame_no >= kWarmupBatchSize && frame_no % kProcessingBatchSize == 0) {
if (frame_no % kProcessingBatchSizeStandalone == 0) {
timer.StartTimer();
}
@ -63,15 +74,19 @@ void RunStandaloneSubmodule() {
echo_detector.AnalyzeCaptureAudio(rtc::ArrayView<const float>(
buffers.capture_input_buffer->split_bands_const_f(0)[kBand0To8kHz],
buffers.capture_input_buffer->num_frames_per_band()));
sum += echo_detector.echo_likelihood();
if (frame_no >= kWarmupBatchSize &&
frame_no % kProcessingBatchSize == kProcessingBatchSize - 1) {
if (frame_no % kProcessingBatchSizeStandalone ==
kProcessingBatchSizeStandalone - 1) {
timer.StopTimer();
}
}
EXPECT_EQ(0.0f, sum);
webrtc::test::PrintResultMeanAndError(
"echo_detector_call_durations", "", "StandaloneEchoDetector",
FormPerformanceMeasureString(timer), "us", false);
FormPerformanceMeasureString(timer,
kNumberOfWarmupMeasurementsStandalone),
"us", false);
}
void RunTogetherWithApm(std::string test_description,
@ -80,7 +95,7 @@ void RunTogetherWithApm(std::string test_description,
test::SimulatorBuffers buffers(
kSampleRate, kSampleRate, kSampleRate, kSampleRate, kNumberOfChannels,
kNumberOfChannels, kNumberOfChannels, kNumberOfChannels);
test::PerformanceTimer timer(kNumFramesToProcess);
test::PerformanceTimer timer(kNumFramesToProcess / kProcessingBatchSize);
webrtc::Config config;
AudioProcessing::Config apm_config;
@ -124,7 +139,7 @@ void RunTogetherWithApm(std::string test_description,
// The first batch of frames are for warming up, and are not part of the
// benchmark. After that the processing time is measured in chunks of
// kProcessingBatchSize frames.
if (frame_no >= kWarmupBatchSize && frame_no % kProcessingBatchSize == 0) {
if (frame_no % kProcessingBatchSize == 0) {
timer.StartTimer();
}
@ -146,15 +161,15 @@ void RunTogetherWithApm(std::string test_description,
apm->ProcessStream(&buffers.capture_input[0], stream_config,
stream_config, &buffers.capture_output[0]));
if (frame_no >= kWarmupBatchSize &&
frame_no % kProcessingBatchSize == kProcessingBatchSize - 1) {
if (frame_no % kProcessingBatchSize == kProcessingBatchSize - 1) {
timer.StopTimer();
}
}
webrtc::test::PrintResultMeanAndError(
"echo_detector_call_durations", "_total", test_description,
FormPerformanceMeasureString(timer), "us", false);
FormPerformanceMeasureString(timer, kNumberOfWarmupMeasurements), "us",
false);
}
} // namespace

34
webrtc/modules/audio_processing/test/performance_timer.cc
View File

@ -36,23 +36,39 @@ void PerformanceTimer::StopTimer() {
}
double PerformanceTimer::GetDurationAverage() const {
RTC_DCHECK(!timestamps_us_.empty());
return static_cast<double>(
std::accumulate(timestamps_us_.begin(), timestamps_us_.end(), 0)) /
timestamps_us_.size();
return GetDurationAverage(0);
}
double PerformanceTimer::GetDurationStandardDeviation() const {
RTC_DCHECK(!timestamps_us_.empty());
double average_duration = GetDurationAverage();
return GetDurationStandardDeviation(0);
}
double PerformanceTimer::GetDurationAverage(
size_t number_of_warmup_samples) const {
RTC_DCHECK_GT(timestamps_us_.size(), number_of_warmup_samples);
const size_t number_of_samples =
timestamps_us_.size() - number_of_warmup_samples;
return static_cast<double>(
std::accumulate(timestamps_us_.begin() + number_of_warmup_samples,
timestamps_us_.end(), static_cast<int64_t>(0))) /
number_of_samples;
}
double PerformanceTimer::GetDurationStandardDeviation(
size_t number_of_warmup_samples) const {
RTC_DCHECK_GT(timestamps_us_.size(), number_of_warmup_samples);
const size_t number_of_samples =
timestamps_us_.size() - number_of_warmup_samples;
RTC_DCHECK_GT(number_of_samples, 0);
double average_duration = GetDurationAverage(number_of_warmup_samples);
double variance = std::accumulate(
timestamps_us_.begin(), timestamps_us_.end(), 0.0,
[average_duration](const double& a, const int64_t& b) {
timestamps_us_.begin() + number_of_warmup_samples, timestamps_us_.end(),
0.0, [average_duration](const double& a, const int64_t& b) {
return a + (b - average_duration) * (b - average_duration);
});
return sqrt(variance / timestamps_us_.size());
return sqrt(variance / number_of_samples);
}
} // namespace test

5
webrtc/modules/audio_processing/test/performance_timer.h
View File

@ -30,6 +30,11 @@ class PerformanceTimer {
double GetDurationAverage() const;
double GetDurationStandardDeviation() const;
// These methods are the same as those above, but they ignore the first
// |number_of_warmup_samples| measurements.
double GetDurationAverage(size_t number_of_warmup_samples) const;
double GetDurationStandardDeviation(size_t number_of_warmup_samples) const;
private:
webrtc::Clock* clock_;
rtc::Optional<int64_t> start_timestamp_us_;