admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reland "Fix collection of audio metrics from PC test framework for audio test"

Browse Source 
This is a reland of 2d0880b56954f57548deea51dfa678b80dbf618f

To fix perf bot issue reading of perf results file was updated. Now perf
results file will be generated by each test and then returned via output to
the python script, which will get it and put into final file.

Original change's description:
> Fix collection of audio metrics from PC test framework for audio test
>
> Bug: webrtc:10138
> Change-Id: I18a8509a0cdc4ed1db6894c7540d5c0a155d6233
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/144784
> Reviewed-by: Oleh Prypin <oprypin@webrtc.org>
> Reviewed-by: Oskar Sundbom <ossu@webrtc.org>
> Commit-Queue: Artem Titov <titovartem@webrtc.org>
> Cr-Commit-Position: refs/heads/master@{#28514}

Bug: webrtc:10138
Change-Id: I1347f09427736362a2d550612b48e09c06cfb1d1
No-Presubmit: True
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/145201
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Oskar Sundbom <ossu@webrtc.org>
Reviewed-by: Oleh Prypin <oprypin@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#28527}
This commit is contained in:
Artem Titov 2019-07-10 14:40:58 +02:00 committed by Commit Bot
parent ca160215c9
commit b1f2d60456
6 changed files with 150 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
audio/BUILD.gn
View File

@ -177,6 +177,7 @@ if (rtc_include_tests) {
sources = [
"test/low_bandwidth_audio_test.cc",
"test/low_bandwidth_audio_test_flags.cc",
"test/pc_low_bandwidth_audio_test.cc",
]
@ -192,6 +193,7 @@ if (rtc_include_tests) {
"../rtc_base:rtc_base_approved",
"../system_wrappers",
"../test:fileutils",
"../test:perf_test",
"../test:test_common",
"../test:test_main",
"../test:test_support",

11
audio/test/low_bandwidth_audio_test.cc
View File

@ -14,15 +14,8 @@
#include "system_wrappers/include/sleep.h"
#include "test/testsupport/file_utils.h"
WEBRTC_DEFINE_int(sample_rate_hz,
16000,
"Sample rate (Hz) of the produced audio files.");
WEBRTC_DEFINE_bool(
quick,
false,
"Don't do the full audio recording. "
"Used to quickly check that the test runs without crashing.");
WEBRTC_DECLARE_int(sample_rate_hz);
WEBRTC_DECLARE_bool(quick);
namespace webrtc {
namespace test {

41
audio/test/low_bandwidth_audio_test.py
View File

@ -110,7 +110,8 @@ def _GetPathToTools():
def ExtractTestRuns(lines, echo=False):
"""Extracts information about tests from the output of a test runner.
Produces tuples (android_device, test_name, reference_file, degraded_file).
Produces tuples
(android_device, test_name, reference_file, degraded_file, cur_perf_results).
"""
for line in lines:
if echo:
@ -118,7 +119,8 @@ def ExtractTestRuns(lines, echo=False):
# Output from Android has a prefix with the device name.
android_prefix_re = r'(?:I\b.+\brun_tests_on_device\((.+?)\)\s*)?'
test_re = r'^' + android_prefix_re + r'TEST (\w+) ([^ ]+?) ([^ ]+?)\s*$'
test_re = r'^' + android_prefix_re + (r'TEST (\w+) ([^ ]+?) ([^\s]+)'
r' ?([^\s]+)?\s*$')
match = re.search(test_re, line)
if match:
@ -206,7 +208,20 @@ def _AddChart(charts, metric, test_name, value, units):
}
Analyzer = collections.namedtuple('Analyzer', ['func', 'executable',
def _AddRunPerfResults(charts, run_perf_results_file):
with open(run_perf_results_file, 'rb') as f:
per_run_perf_results = json.load(f)
if 'charts' not in per_run_perf_results:
return
for metric, cases in per_run_perf_results['charts'].items():
chart = charts.setdefault(metric, {})
for case_name, case_value in cases.items():
if case_name in chart:
logging.error('Overriding results for %s/%s', metric, case_name)
chart[case_name] = case_value
Analyzer = collections.namedtuple('Analyzer', ['name', 'func', 'executable',
'sample_rate_hz'])
@ -228,25 +243,29 @@ def main():
else:
test_command = [os.path.join(args.build_dir, 'low_bandwidth_audio_test')]
analyzers = [Analyzer(_RunPesq, pesq_path, 16000)]
analyzers = [Analyzer('pesq', _RunPesq, pesq_path, 16000)]
# Check if POLQA can run at all, or skip the 48 kHz tests entirely.
example_path = os.path.join(SRC_DIR, 'resources',
'voice_engine', 'audio_tiny48.wav')
if polqa_path and _RunPolqa(polqa_path, example_path, example_path):
analyzers.append(Analyzer(_RunPolqa, polqa_path, 48000))
analyzers.append(Analyzer('polqa', _RunPolqa, polqa_path, 48000))
charts = {}
for analyzer in analyzers:
# Start the test executable that produces audio files.
test_process = subprocess.Popen(
_LogCommand(test_command + ['--sample_rate_hz=%d' %
analyzer.sample_rate_hz]),
_LogCommand(test_command + [
'--sample_rate_hz=%d' % analyzer.sample_rate_hz,
'--test_case_prefix=%s' % analyzer.name
]),
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
perf_results_file = None
try:
lines = iter(test_process.stdout.readline, '')
for result in ExtractTestRuns(lines, echo=True):
(android_device, test_name, reference_file, degraded_file) = result
(android_device, test_name, reference_file, degraded_file,
perf_results_file) = result
adb_prefix = (args.adb_path,)
if android_device:
@ -269,6 +288,12 @@ def main():
os.remove(degraded_file)
finally:
test_process.terminate()
if perf_results_file:
perf_results_file = _GetFile(perf_results_file, out_dir, move=True,
android=args.android, adb_prefix=adb_prefix)
_AddRunPerfResults(charts, perf_results_file)
if args.remove:
os.remove(perf_results_file)
if args.isolated_script_test_perf_output:
with open(args.isolated_script_test_perf_output, 'w') as f:

27
audio/test/low_bandwidth_audio_test_flags.cc Normal file
View File

@ -0,0 +1,27 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
// #ifndef AUDIO_TEST_LOW_BANDWIDTH_AUDIO_TEST_FLAGS_H_
// #define AUDIO_TEST_LOW_BANDWIDTH_AUDIO_TEST_FLAGS_H_
#include "rtc_base/flags.h"
WEBRTC_DEFINE_int(sample_rate_hz,
16000,
"Sample rate (Hz) of the produced audio files.");
WEBRTC_DEFINE_bool(
quick,
false,
"Don't do the full audio recording. "
"Used to quickly check that the test runs without crashing.");
WEBRTC_DEFINE_string(test_case_prefix, "", "Test case prefix.");
// #endif // AUDIO_TEST_LOW_BANDWIDTH_AUDIO_TEST_FLAGS_H_

36
audio/test/pc_low_bandwidth_audio_test.cc
View File

@ -19,6 +19,9 @@
#include "test/gtest.h"
#include "test/pc/e2e/network_quality_metrics_reporter.h"
#include "test/testsupport/file_utils.h"
#include "test/testsupport/perf_test.h"
WEBRTC_DECLARE_string(test_case_prefix);
namespace webrtc {
namespace test {
@ -33,6 +36,16 @@ namespace {
constexpr int kTestDurationSec = 45;
std::string GetMetricTestCaseName() {
const ::testing::TestInfo* const test_info =
::testing::UnitTest::GetInstance()->current_test_info();
std::string test_case_prefix(FLAG_test_case_prefix);
if (test_case_prefix.empty()) {
return test_info->name();
}
return std::string(FLAG_test_case_prefix) + "_" + test_info->name();
}
EmulatedNetworkNode* CreateEmulatedNodeWithConfig(
NetworkEmulationManager* emulation,
const BuiltInNetworkBehaviorConfig& config) {
@ -84,18 +97,25 @@ std::string AudioInputFile() {
std::string AudioOutputFile() {
const ::testing::TestInfo* const test_info =
::testing::UnitTest::GetInstance()->current_test_info();
return webrtc::test::OutputPath() + "LowBandwidth_" + test_info->name() +
return webrtc::test::OutputPath() + "PCLowBandwidth_" + test_info->name() +
"_48.wav";
}
void PrintTestInfo() {
std::string PerfResultsOutputFile() {
return webrtc::test::OutputPath() + "PCLowBandwidth_perf_48.json";
}
void LogTestResults() {
std::string perf_results_output_file = PerfResultsOutputFile();
webrtc::test::WritePerfResults(perf_results_output_file);
const ::testing::TestInfo* const test_info =
::testing::UnitTest::GetInstance()->current_test_info();
// Output information about the input and output audio files so that further
// processing can be done by an external process.
printf("TEST %s %s %s\n", test_info->name(), AudioInputFile().c_str(),
AudioOutputFile().c_str());
printf("TEST %s %s %s %s\n", test_info->name(), AudioInputFile().c_str(),
AudioOutputFile().c_str(), perf_results_output_file.c_str());
}
} // namespace
@ -104,7 +124,7 @@ TEST(PCLowBandwidthAudioTest, PCGoodNetworkHighBitrate) {
std::unique_ptr<NetworkEmulationManager> network_emulation_manager =
CreateNetworkEmulationManager();
auto fixture = CreateTestFixture(
"pc_good_network",
GetMetricTestCaseName(),
CreateTwoNetworkLinks(network_emulation_manager.get(),
BuiltInNetworkBehaviorConfig()),
[](PeerConfigurer* alice) {
@ -117,7 +137,7 @@ TEST(PCLowBandwidthAudioTest, PCGoodNetworkHighBitrate) {
},
[](PeerConfigurer* bob) {});
fixture->Run(RunParams(TimeDelta::seconds(kTestDurationSec)));
PrintTestInfo();
LogTestResults();
}
TEST(PCLowBandwidthAudioTest, PCMobile2GNetwork) {
@ -128,7 +148,7 @@ TEST(PCLowBandwidthAudioTest, PCMobile2GNetwork) {
config.queue_length_packets = 1500;
config.queue_delay_ms = 400;
auto fixture = CreateTestFixture(
"pc_mobile_2g_network",
GetMetricTestCaseName(),
CreateTwoNetworkLinks(network_emulation_manager.get(), config),
[](PeerConfigurer* alice) {
AudioConfig audio;
@ -141,7 +161,7 @@ TEST(PCLowBandwidthAudioTest, PCMobile2GNetwork) {
[](PeerConfigurer* bob) {});
RunParams run_params(TimeDelta::seconds(kTestDurationSec));
fixture->Run(RunParams(TimeDelta::seconds(kTestDurationSec)));
PrintTestInfo();
LogTestResults();
}
} // namespace test

62
audio/test/unittests/low_bandwidth_audio_test_test.py
View File

@ -28,19 +28,37 @@ class TestExtractTestRuns(unittest.TestCase):
self._TestLog(LINUX_LOG,
(None, 'GoodNetworkHighBitrate',
'/webrtc/src/resources/voice_engine/audio_tiny16.wav',
'/webrtc/src/out/LowBandwidth_GoodNetworkHighBitrate.wav'),
'/webrtc/src/out/LowBandwidth_GoodNetworkHighBitrate.wav', None),
(None, 'Mobile2GNetwork',
'/webrtc/src/resources/voice_engine/audio_tiny16.wav',
'/webrtc/src/out/LowBandwidth_Mobile2GNetwork.wav'))
'/webrtc/src/out/LowBandwidth_Mobile2GNetwork.wav', None),
(None, 'PCGoodNetworkHighBitrate',
'/webrtc/src/resources/voice_engine/audio_tiny16.wav',
'/webrtc/src/out/PCLowBandwidth_PCGoodNetworkHighBitrate.wav',
'/webrtc/src/out/PCLowBandwidth_perf_48.json'),
(None, 'PCMobile2GNetwork',
'/webrtc/src/resources/voice_engine/audio_tiny16.wav',
'/webrtc/src/out/PCLowBandwidth_PCMobile2GNetwork.wav',
'/webrtc/src/out/PCLowBandwidth_perf_48.json'))
def testAndroid(self):
self._TestLog(ANDROID_LOG,
('ddfa6149', 'Mobile2GNetwork',
'/sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav',
'/sdcard/chromium_tests_root/LowBandwidth_Mobile2GNetwork.wav'),
'/sdcard/chromium_tests_root/LowBandwidth_Mobile2GNetwork.wav', None),
('TA99205CNO', 'GoodNetworkHighBitrate',
'/sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav',
'/sdcard/chromium_tests_root/LowBandwidth_GoodNetworkHighBitrate.wav'))
'/sdcard/chromium_tests_root/LowBandwidth_GoodNetworkHighBitrate.wav',
None),
('ddfa6149', 'PCMobile2GNetwork',
'/sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav',
'/sdcard/chromium_tests_root/PCLowBandwidth_PCMobile2GNetwork.wav',
'/sdcard/chromium_tests_root/PCLowBandwidth_perf_48.json'),
('TA99205CNO', 'PCGoodNetworkHighBitrate',
'/sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav',
('/sdcard/chromium_tests_root/'
'PCLowBandwidth_PCGoodNetworkHighBitrate.wav'),
'/sdcard/chromium_tests_root/PCLowBandwidth_perf_48.json'))
LINUX_LOG = r'''\
@ -54,6 +72,14 @@ TEST GoodNetworkHighBitrate /webrtc/src/resources/voice_engine/audio_tiny16.wav
TEST Mobile2GNetwork /webrtc/src/resources/voice_engine/audio_tiny16.wav /webrtc/src/out/LowBandwidth_Mobile2GNetwork.wav
[ OK ] LowBandwidthAudioTest.Mobile2GNetwork (6333 ms)
[----------] 2 tests from LowBandwidthAudioTest (12265 ms total)
[----------] 2 tests from PCLowBandwidthAudioTest
[ RUN ] PCLowBandwidthAudioTest.PCGoodNetworkHighBitrate
TEST PCGoodNetworkHighBitrate /webrtc/src/resources/voice_engine/audio_tiny16.wav /webrtc/src/out/PCLowBandwidth_PCGoodNetworkHighBitrate.wav /webrtc/src/out/PCLowBandwidth_perf_48.json
[ OK ] PCLowBandwidthAudioTest.PCGoodNetworkHighBitrate (5932 ms)
[ RUN ] PCLowBandwidthAudioTest.PCMobile2GNetwork
TEST PCMobile2GNetwork /webrtc/src/resources/voice_engine/audio_tiny16.wav /webrtc/src/out/PCLowBandwidth_PCMobile2GNetwork.wav /webrtc/src/out/PCLowBandwidth_perf_48.json
[ OK ] PCLowBandwidthAudioTest.PCMobile2GNetwork (6333 ms)
[----------] 2 tests from PCLowBandwidthAudioTest (12265 ms total)
[----------] Global test environment tear-down
[==========] 2 tests from 1 test case ran. (12266 ms total)
@ -165,6 +191,34 @@ I 16.576s run_tests_on_device(TA99205CNO) [----------] Global test environmen
I 16.576s run_tests_on_device(TA99205CNO) [==========] 1 test from 1 test case ran. (5968 ms total)
I 16.577s run_tests_on_device(TA99205CNO) [ PASSED ] 1 test.
I 16.577s run_tests_on_device(TA99205CNO) <<ScopedMainEntryLogger
I 14.078s run_tests_on_device(ddfa6149) >>ScopedMainEntryLogger
I 14.078s run_tests_on_device(ddfa6149) Note: Google Test filter = PCLowBandwidthAudioTest.PCMobile2GNetwork
I 14.078s run_tests_on_device(ddfa6149) [==========] Running 1 test from 1 test case.
I 14.078s run_tests_on_device(ddfa6149) [----------] Global test environment set-up.
I 14.078s run_tests_on_device(ddfa6149) [----------] 1 test from PCLowBandwidthAudioTest
I 14.078s run_tests_on_device(ddfa6149) [ RUN ] PCLowBandwidthAudioTest.PCMobile2GNetwork
I 14.078s run_tests_on_device(ddfa6149) TEST PCMobile2GNetwork /sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav /sdcard/chromium_tests_root/PCLowBandwidth_PCMobile2GNetwork.wav /sdcard/chromium_tests_root/PCLowBandwidth_perf_48.json
I 14.078s run_tests_on_device(ddfa6149) [ OK ] PCLowBandwidthAudioTest.PCMobile2GNetwork (6438 ms)
I 14.078s run_tests_on_device(ddfa6149) [----------] 1 test from PCLowBandwidthAudioTest (6438 ms total)
I 14.078s run_tests_on_device(ddfa6149)
I 14.078s run_tests_on_device(ddfa6149) [----------] Global test environment tear-down
I 14.079s run_tests_on_device(ddfa6149) [==========] 1 test from 1 test case ran. (6438 ms total)
I 14.079s run_tests_on_device(ddfa6149) [ PASSED ] 1 test.
I 14.079s run_tests_on_device(ddfa6149) <<ScopedMainEntryLogger
I 16.576s run_tests_on_device(TA99205CNO) >>ScopedMainEntryLogger
I 16.576s run_tests_on_device(TA99205CNO) Note: Google Test filter = PCLowBandwidthAudioTest.PCGoodNetworkHighBitrate
I 16.576s run_tests_on_device(TA99205CNO) [==========] Running 1 test from 1 test case.
I 16.576s run_tests_on_device(TA99205CNO) [----------] Global test environment set-up.
I 16.576s run_tests_on_device(TA99205CNO) [----------] 1 test from PCLowBandwidthAudioTest
I 16.576s run_tests_on_device(TA99205CNO) [ RUN ] PCLowBandwidthAudioTest.PCGoodNetworkHighBitrate
I 16.576s run_tests_on_device(TA99205CNO) TEST PCGoodNetworkHighBitrate /sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav /sdcard/chromium_tests_root/PCLowBandwidth_PCGoodNetworkHighBitrate.wav /sdcard/chromium_tests_root/PCLowBandwidth_perf_48.json
I 16.576s run_tests_on_device(TA99205CNO) [ OK ] PCLowBandwidthAudioTest.PCGoodNetworkHighBitrate (5968 ms)
I 16.576s run_tests_on_device(TA99205CNO) [----------] 1 test from PCLowBandwidthAudioTest (5968 ms total)
I 16.576s run_tests_on_device(TA99205CNO)
I 16.576s run_tests_on_device(TA99205CNO) [----------] Global test environment tear-down
I 16.576s run_tests_on_device(TA99205CNO) [==========] 1 test from 1 test case ran. (5968 ms total)
I 16.577s run_tests_on_device(TA99205CNO) [ PASSED ] 1 test.
I 16.577s run_tests_on_device(TA99205CNO) <<ScopedMainEntryLogger
I 16.577s run_tests_on_device(TA99205CNO) Finished running tests on this device.
I 16.577s run_tests_on_device(ddfa6149) Finished running tests on this device.
I 16.604s Main FINISHED TRY #1/3