admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix collection of audio metrics from PC test framework for audio test

Browse Source 
Bug: webrtc:10138
Change-Id: I18a8509a0cdc4ed1db6894c7540d5c0a155d6233
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/144784
Reviewed-by: Oleh Prypin <oprypin@webrtc.org>
Reviewed-by: Oskar Sundbom <ossu@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#28514}
This commit is contained in:
Artem Titov 2019-07-09 13:50:18 +02:00 committed by Commit Bot
parent bc558cebdc
commit 2d0880b569
5 changed files with 94 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
audio/BUILD.gn
View File

@ -177,6 +177,7 @@ if (rtc_include_tests) {
sources = [
"test/low_bandwidth_audio_test.cc",
"test/low_bandwidth_audio_test_flags.cc",
"test/pc_low_bandwidth_audio_test.cc",
]

11
audio/test/low_bandwidth_audio_test.cc
View File

@ -14,15 +14,8 @@
#include "system_wrappers/include/sleep.h"
#include "test/testsupport/file_utils.h"
WEBRTC_DEFINE_int(sample_rate_hz,
16000,
"Sample rate (Hz) of the produced audio files.");
WEBRTC_DEFINE_bool(
quick,
false,
"Don't do the full audio recording. "
"Used to quickly check that the test runs without crashing.");
WEBRTC_DECLARE_int(sample_rate_hz);
WEBRTC_DECLARE_bool(quick);
namespace webrtc {
namespace test {

78
audio/test/low_bandwidth_audio_test.py
View File

@ -23,6 +23,7 @@ import re
import shutil
import subprocess
import sys
import tempfile
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
@ -206,7 +207,20 @@ def _AddChart(charts, metric, test_name, value, units):
}
Analyzer = collections.namedtuple('Analyzer', ['func', 'executable',
def _AddRunPerfResults(charts, run_perf_results_file):
with open(run_perf_results_file, 'rb') as f:
per_run_perf_results = json.load(f)
if 'charts' not in per_run_perf_results:
return
for metric, cases in per_run_perf_results['charts'].items():
chart = charts.setdefault(metric, {})
for case_name, case_value in cases.items():
if case_name in chart:
logging.error('Overriding results for %s/%s', metric, case_name)
chart[case_name] = case_value
Analyzer = collections.namedtuple('Analyzer', ['name', 'func', 'executable',
'sample_rate_hz'])
@ -228,47 +242,55 @@ def main():
else:
test_command = [os.path.join(args.build_dir, 'low_bandwidth_audio_test')]
analyzers = [Analyzer(_RunPesq, pesq_path, 16000)]
analyzers = [Analyzer('pesq', _RunPesq, pesq_path, 16000)]
# Check if POLQA can run at all, or skip the 48 kHz tests entirely.
example_path = os.path.join(SRC_DIR, 'resources',
'voice_engine', 'audio_tiny48.wav')
if polqa_path and _RunPolqa(polqa_path, example_path, example_path):
analyzers.append(Analyzer(_RunPolqa, polqa_path, 48000))
analyzers.append(Analyzer('polqa', _RunPolqa, polqa_path, 48000))
charts = {}
for analyzer in analyzers:
# Start the test executable that produces audio files.
test_process = subprocess.Popen(
_LogCommand(test_command + ['--sample_rate_hz=%d' %
analyzer.sample_rate_hz]),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
f, cur_perf_results = tempfile.mkstemp(prefix='audio_perf', suffix=".json")
try:
lines = iter(test_process.stdout.readline, '')
for result in ExtractTestRuns(lines, echo=True):
(android_device, test_name, reference_file, degraded_file) = result
# Start the test executable that produces audio files.
test_process = subprocess.Popen(
_LogCommand(test_command + [
'--sample_rate_hz=%d' % analyzer.sample_rate_hz,
'--test_case_prefix=%s' % analyzer.name,
'--isolated_script_test_perf_output=%s' % cur_perf_results,
]),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
try:
lines = iter(test_process.stdout.readline, '')
for result in ExtractTestRuns(lines, echo=True):
(android_device, test_name, reference_file, degraded_file) = result
adb_prefix = (args.adb_path,)
if android_device:
adb_prefix += ('-s', android_device)
adb_prefix = (args.adb_path,)
if android_device:
adb_prefix += ('-s', android_device)
reference_file = _GetFile(reference_file, out_dir,
android=args.android, adb_prefix=adb_prefix)
degraded_file = _GetFile(degraded_file, out_dir, move=True,
android=args.android, adb_prefix=adb_prefix)
reference_file = _GetFile(reference_file, out_dir,
android=args.android, adb_prefix=adb_prefix)
degraded_file = _GetFile(degraded_file, out_dir, move=True,
android=args.android, adb_prefix=adb_prefix)
analyzer_results = analyzer.func(analyzer.executable,
reference_file, degraded_file)
for metric, (value, units) in analyzer_results.items():
# Output a result for the perf dashboard.
print 'RESULT %s: %s= %s %s' % (metric, test_name, value, units)
_AddChart(charts, metric, test_name, value, units)
analyzer_results = analyzer.func(analyzer.executable,
reference_file, degraded_file)
for metric, (value, units) in analyzer_results.items():
# Output a result for the perf dashboard.
print 'RESULT %s: %s= %s %s' % (metric, test_name, value, units)
_AddChart(charts, metric, test_name, value, units)
if args.remove:
os.remove(reference_file)
os.remove(degraded_file)
if args.remove:
os.remove(reference_file)
os.remove(degraded_file)
finally:
test_process.terminate()
_AddRunPerfResults(charts, cur_perf_results)
finally:
test_process.terminate()
os.remove(cur_perf_results)
if args.isolated_script_test_perf_output:
with open(args.isolated_script_test_perf_output, 'w') as f:

27
audio/test/low_bandwidth_audio_test_flags.cc Normal file
View File

@ -0,0 +1,27 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
// #ifndef AUDIO_TEST_LOW_BANDWIDTH_AUDIO_TEST_FLAGS_H_
// #define AUDIO_TEST_LOW_BANDWIDTH_AUDIO_TEST_FLAGS_H_
#include "rtc_base/flags.h"
WEBRTC_DEFINE_int(sample_rate_hz,
16000,
"Sample rate (Hz) of the produced audio files.");
WEBRTC_DEFINE_bool(
quick,
false,
"Don't do the full audio recording. "
"Used to quickly check that the test runs without crashing.");
WEBRTC_DEFINE_string(test_case_prefix, "", "Test case prefix.");
// #endif // AUDIO_TEST_LOW_BANDWIDTH_AUDIO_TEST_FLAGS_H_

16
audio/test/pc_low_bandwidth_audio_test.cc
View File

@ -20,6 +20,8 @@
#include "test/pc/e2e/network_quality_metrics_reporter.h"
#include "test/testsupport/file_utils.h"
WEBRTC_DECLARE_string(test_case_prefix);
namespace webrtc {
namespace test {
@ -33,6 +35,16 @@ namespace {
constexpr int kTestDurationSec = 45;
std::string GetMetricTestCaseName() {
const ::testing::TestInfo* const test_info =
::testing::UnitTest::GetInstance()->current_test_info();
std::string test_case_prefix(FLAG_test_case_prefix);
if (test_case_prefix.empty()) {
return test_info->name();
}
return std::string(FLAG_test_case_prefix) + "_" + test_info->name();
}
EmulatedNetworkNode* CreateEmulatedNodeWithConfig(
NetworkEmulationManager* emulation,
const BuiltInNetworkBehaviorConfig& config) {
@ -104,7 +116,7 @@ TEST(PCLowBandwidthAudioTest, PCGoodNetworkHighBitrate) {
std::unique_ptr<NetworkEmulationManager> network_emulation_manager =
CreateNetworkEmulationManager();
auto fixture = CreateTestFixture(
"pc_good_network",
GetMetricTestCaseName(),
CreateTwoNetworkLinks(network_emulation_manager.get(),
BuiltInNetworkBehaviorConfig()),
[](PeerConfigurer* alice) {
@ -128,7 +140,7 @@ TEST(PCLowBandwidthAudioTest, PCMobile2GNetwork) {
config.queue_length_packets = 1500;
config.queue_delay_ms = 400;
auto fixture = CreateTestFixture(
"pc_mobile_2g_network",
GetMetricTestCaseName(),
CreateTwoNetworkLinks(network_emulation_manager.get(), config),
[](PeerConfigurer* alice) {
AudioConfig audio;