From b1f2d604560f8e27552676fe9ec27ba946bd98d1 Mon Sep 17 00:00:00 2001 From: Artem Titov Date: Wed, 10 Jul 2019 14:40:58 +0200 Subject: [PATCH] Reland "Fix collection of audio metrics from PC test framework for audio test" This is a reland of 2d0880b56954f57548deea51dfa678b80dbf618f To fix perf bot issue reading of perf results file was updated. Now perf results file will be generated by each test and then returned via output to the python script, which will get it and put into final file. Original change's description: > Fix collection of audio metrics from PC test framework for audio test > > Bug: webrtc:10138 > Change-Id: I18a8509a0cdc4ed1db6894c7540d5c0a155d6233 > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/144784 > Reviewed-by: Oleh Prypin > Reviewed-by: Oskar Sundbom > Commit-Queue: Artem Titov > Cr-Commit-Position: refs/heads/master@{#28514} Bug: webrtc:10138 Change-Id: I1347f09427736362a2d550612b48e09c06cfb1d1 No-Presubmit: True Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/145201 Commit-Queue: Artem Titov Reviewed-by: Oskar Sundbom Reviewed-by: Oleh Prypin Cr-Commit-Position: refs/heads/master@{#28527} --- audio/BUILD.gn | 2 + audio/test/low_bandwidth_audio_test.cc | 11 +--- audio/test/low_bandwidth_audio_test.py | 41 +++++++++--- audio/test/low_bandwidth_audio_test_flags.cc | 27 ++++++++ audio/test/pc_low_bandwidth_audio_test.cc | 36 ++++++++--- .../low_bandwidth_audio_test_test.py | 62 +++++++++++++++++-- 6 files changed, 150 insertions(+), 29 deletions(-) create mode 100644 audio/test/low_bandwidth_audio_test_flags.cc diff --git a/audio/BUILD.gn b/audio/BUILD.gn index 4a3d8fd032..0e78a6895e 100644 --- a/audio/BUILD.gn +++ b/audio/BUILD.gn @@ -177,6 +177,7 @@ if (rtc_include_tests) { sources = [ "test/low_bandwidth_audio_test.cc", + "test/low_bandwidth_audio_test_flags.cc", "test/pc_low_bandwidth_audio_test.cc", ] @@ -192,6 +193,7 @@ if (rtc_include_tests) { "../rtc_base:rtc_base_approved", "../system_wrappers", "../test:fileutils", + "../test:perf_test", "../test:test_common", "../test:test_main", "../test:test_support", diff --git a/audio/test/low_bandwidth_audio_test.cc b/audio/test/low_bandwidth_audio_test.cc index 54191e85a6..db1ff2cbba 100644 --- a/audio/test/low_bandwidth_audio_test.cc +++ b/audio/test/low_bandwidth_audio_test.cc @@ -14,15 +14,8 @@ #include "system_wrappers/include/sleep.h" #include "test/testsupport/file_utils.h" -WEBRTC_DEFINE_int(sample_rate_hz, - 16000, - "Sample rate (Hz) of the produced audio files."); - -WEBRTC_DEFINE_bool( - quick, - false, - "Don't do the full audio recording. " - "Used to quickly check that the test runs without crashing."); +WEBRTC_DECLARE_int(sample_rate_hz); +WEBRTC_DECLARE_bool(quick); namespace webrtc { namespace test { diff --git a/audio/test/low_bandwidth_audio_test.py b/audio/test/low_bandwidth_audio_test.py index add4f2f72b..8ad820e932 100755 --- a/audio/test/low_bandwidth_audio_test.py +++ b/audio/test/low_bandwidth_audio_test.py @@ -110,7 +110,8 @@ def _GetPathToTools(): def ExtractTestRuns(lines, echo=False): """Extracts information about tests from the output of a test runner. - Produces tuples (android_device, test_name, reference_file, degraded_file). + Produces tuples + (android_device, test_name, reference_file, degraded_file, cur_perf_results). """ for line in lines: if echo: @@ -118,7 +119,8 @@ def ExtractTestRuns(lines, echo=False): # Output from Android has a prefix with the device name. android_prefix_re = r'(?:I\b.+\brun_tests_on_device\((.+?)\)\s*)?' - test_re = r'^' + android_prefix_re + r'TEST (\w+) ([^ ]+?) ([^ ]+?)\s*$' + test_re = r'^' + android_prefix_re + (r'TEST (\w+) ([^ ]+?) ([^\s]+)' + r' ?([^\s]+)?\s*$') match = re.search(test_re, line) if match: @@ -206,7 +208,20 @@ def _AddChart(charts, metric, test_name, value, units): } -Analyzer = collections.namedtuple('Analyzer', ['func', 'executable', +def _AddRunPerfResults(charts, run_perf_results_file): + with open(run_perf_results_file, 'rb') as f: + per_run_perf_results = json.load(f) + if 'charts' not in per_run_perf_results: + return + for metric, cases in per_run_perf_results['charts'].items(): + chart = charts.setdefault(metric, {}) + for case_name, case_value in cases.items(): + if case_name in chart: + logging.error('Overriding results for %s/%s', metric, case_name) + chart[case_name] = case_value + + +Analyzer = collections.namedtuple('Analyzer', ['name', 'func', 'executable', 'sample_rate_hz']) @@ -228,25 +243,29 @@ def main(): else: test_command = [os.path.join(args.build_dir, 'low_bandwidth_audio_test')] - analyzers = [Analyzer(_RunPesq, pesq_path, 16000)] + analyzers = [Analyzer('pesq', _RunPesq, pesq_path, 16000)] # Check if POLQA can run at all, or skip the 48 kHz tests entirely. example_path = os.path.join(SRC_DIR, 'resources', 'voice_engine', 'audio_tiny48.wav') if polqa_path and _RunPolqa(polqa_path, example_path, example_path): - analyzers.append(Analyzer(_RunPolqa, polqa_path, 48000)) + analyzers.append(Analyzer('polqa', _RunPolqa, polqa_path, 48000)) charts = {} for analyzer in analyzers: # Start the test executable that produces audio files. test_process = subprocess.Popen( - _LogCommand(test_command + ['--sample_rate_hz=%d' % - analyzer.sample_rate_hz]), + _LogCommand(test_command + [ + '--sample_rate_hz=%d' % analyzer.sample_rate_hz, + '--test_case_prefix=%s' % analyzer.name + ]), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + perf_results_file = None try: lines = iter(test_process.stdout.readline, '') for result in ExtractTestRuns(lines, echo=True): - (android_device, test_name, reference_file, degraded_file) = result + (android_device, test_name, reference_file, degraded_file, + perf_results_file) = result adb_prefix = (args.adb_path,) if android_device: @@ -269,6 +288,12 @@ def main(): os.remove(degraded_file) finally: test_process.terminate() + if perf_results_file: + perf_results_file = _GetFile(perf_results_file, out_dir, move=True, + android=args.android, adb_prefix=adb_prefix) + _AddRunPerfResults(charts, perf_results_file) + if args.remove: + os.remove(perf_results_file) if args.isolated_script_test_perf_output: with open(args.isolated_script_test_perf_output, 'w') as f: diff --git a/audio/test/low_bandwidth_audio_test_flags.cc b/audio/test/low_bandwidth_audio_test_flags.cc new file mode 100644 index 0000000000..a0f12c5bc9 --- /dev/null +++ b/audio/test/low_bandwidth_audio_test_flags.cc @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +// #ifndef AUDIO_TEST_LOW_BANDWIDTH_AUDIO_TEST_FLAGS_H_ +// #define AUDIO_TEST_LOW_BANDWIDTH_AUDIO_TEST_FLAGS_H_ + +#include "rtc_base/flags.h" + +WEBRTC_DEFINE_int(sample_rate_hz, + 16000, + "Sample rate (Hz) of the produced audio files."); + +WEBRTC_DEFINE_bool( + quick, + false, + "Don't do the full audio recording. " + "Used to quickly check that the test runs without crashing."); + +WEBRTC_DEFINE_string(test_case_prefix, "", "Test case prefix."); + +// #endif // AUDIO_TEST_LOW_BANDWIDTH_AUDIO_TEST_FLAGS_H_ diff --git a/audio/test/pc_low_bandwidth_audio_test.cc b/audio/test/pc_low_bandwidth_audio_test.cc index 4eec672b8b..d5550fe861 100644 --- a/audio/test/pc_low_bandwidth_audio_test.cc +++ b/audio/test/pc_low_bandwidth_audio_test.cc @@ -19,6 +19,9 @@ #include "test/gtest.h" #include "test/pc/e2e/network_quality_metrics_reporter.h" #include "test/testsupport/file_utils.h" +#include "test/testsupport/perf_test.h" + +WEBRTC_DECLARE_string(test_case_prefix); namespace webrtc { namespace test { @@ -33,6 +36,16 @@ namespace { constexpr int kTestDurationSec = 45; +std::string GetMetricTestCaseName() { + const ::testing::TestInfo* const test_info = + ::testing::UnitTest::GetInstance()->current_test_info(); + std::string test_case_prefix(FLAG_test_case_prefix); + if (test_case_prefix.empty()) { + return test_info->name(); + } + return std::string(FLAG_test_case_prefix) + "_" + test_info->name(); +} + EmulatedNetworkNode* CreateEmulatedNodeWithConfig( NetworkEmulationManager* emulation, const BuiltInNetworkBehaviorConfig& config) { @@ -84,18 +97,25 @@ std::string AudioInputFile() { std::string AudioOutputFile() { const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); - return webrtc::test::OutputPath() + "LowBandwidth_" + test_info->name() + + return webrtc::test::OutputPath() + "PCLowBandwidth_" + test_info->name() + "_48.wav"; } -void PrintTestInfo() { +std::string PerfResultsOutputFile() { + return webrtc::test::OutputPath() + "PCLowBandwidth_perf_48.json"; +} + +void LogTestResults() { + std::string perf_results_output_file = PerfResultsOutputFile(); + webrtc::test::WritePerfResults(perf_results_output_file); + const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); // Output information about the input and output audio files so that further // processing can be done by an external process. - printf("TEST %s %s %s\n", test_info->name(), AudioInputFile().c_str(), - AudioOutputFile().c_str()); + printf("TEST %s %s %s %s\n", test_info->name(), AudioInputFile().c_str(), + AudioOutputFile().c_str(), perf_results_output_file.c_str()); } } // namespace @@ -104,7 +124,7 @@ TEST(PCLowBandwidthAudioTest, PCGoodNetworkHighBitrate) { std::unique_ptr network_emulation_manager = CreateNetworkEmulationManager(); auto fixture = CreateTestFixture( - "pc_good_network", + GetMetricTestCaseName(), CreateTwoNetworkLinks(network_emulation_manager.get(), BuiltInNetworkBehaviorConfig()), [](PeerConfigurer* alice) { @@ -117,7 +137,7 @@ TEST(PCLowBandwidthAudioTest, PCGoodNetworkHighBitrate) { }, [](PeerConfigurer* bob) {}); fixture->Run(RunParams(TimeDelta::seconds(kTestDurationSec))); - PrintTestInfo(); + LogTestResults(); } TEST(PCLowBandwidthAudioTest, PCMobile2GNetwork) { @@ -128,7 +148,7 @@ TEST(PCLowBandwidthAudioTest, PCMobile2GNetwork) { config.queue_length_packets = 1500; config.queue_delay_ms = 400; auto fixture = CreateTestFixture( - "pc_mobile_2g_network", + GetMetricTestCaseName(), CreateTwoNetworkLinks(network_emulation_manager.get(), config), [](PeerConfigurer* alice) { AudioConfig audio; @@ -141,7 +161,7 @@ TEST(PCLowBandwidthAudioTest, PCMobile2GNetwork) { [](PeerConfigurer* bob) {}); RunParams run_params(TimeDelta::seconds(kTestDurationSec)); fixture->Run(RunParams(TimeDelta::seconds(kTestDurationSec))); - PrintTestInfo(); + LogTestResults(); } } // namespace test diff --git a/audio/test/unittests/low_bandwidth_audio_test_test.py b/audio/test/unittests/low_bandwidth_audio_test_test.py index ee59139478..7403663cd4 100755 --- a/audio/test/unittests/low_bandwidth_audio_test_test.py +++ b/audio/test/unittests/low_bandwidth_audio_test_test.py @@ -28,19 +28,37 @@ class TestExtractTestRuns(unittest.TestCase): self._TestLog(LINUX_LOG, (None, 'GoodNetworkHighBitrate', '/webrtc/src/resources/voice_engine/audio_tiny16.wav', - '/webrtc/src/out/LowBandwidth_GoodNetworkHighBitrate.wav'), + '/webrtc/src/out/LowBandwidth_GoodNetworkHighBitrate.wav', None), (None, 'Mobile2GNetwork', '/webrtc/src/resources/voice_engine/audio_tiny16.wav', - '/webrtc/src/out/LowBandwidth_Mobile2GNetwork.wav')) + '/webrtc/src/out/LowBandwidth_Mobile2GNetwork.wav', None), + (None, 'PCGoodNetworkHighBitrate', + '/webrtc/src/resources/voice_engine/audio_tiny16.wav', + '/webrtc/src/out/PCLowBandwidth_PCGoodNetworkHighBitrate.wav', + '/webrtc/src/out/PCLowBandwidth_perf_48.json'), + (None, 'PCMobile2GNetwork', + '/webrtc/src/resources/voice_engine/audio_tiny16.wav', + '/webrtc/src/out/PCLowBandwidth_PCMobile2GNetwork.wav', + '/webrtc/src/out/PCLowBandwidth_perf_48.json')) def testAndroid(self): self._TestLog(ANDROID_LOG, ('ddfa6149', 'Mobile2GNetwork', '/sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav', - '/sdcard/chromium_tests_root/LowBandwidth_Mobile2GNetwork.wav'), + '/sdcard/chromium_tests_root/LowBandwidth_Mobile2GNetwork.wav', None), ('TA99205CNO', 'GoodNetworkHighBitrate', '/sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav', - '/sdcard/chromium_tests_root/LowBandwidth_GoodNetworkHighBitrate.wav')) + '/sdcard/chromium_tests_root/LowBandwidth_GoodNetworkHighBitrate.wav', + None), + ('ddfa6149', 'PCMobile2GNetwork', + '/sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav', + '/sdcard/chromium_tests_root/PCLowBandwidth_PCMobile2GNetwork.wav', + '/sdcard/chromium_tests_root/PCLowBandwidth_perf_48.json'), + ('TA99205CNO', 'PCGoodNetworkHighBitrate', + '/sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav', + ('/sdcard/chromium_tests_root/' + 'PCLowBandwidth_PCGoodNetworkHighBitrate.wav'), + '/sdcard/chromium_tests_root/PCLowBandwidth_perf_48.json')) LINUX_LOG = r'''\ @@ -54,6 +72,14 @@ TEST GoodNetworkHighBitrate /webrtc/src/resources/voice_engine/audio_tiny16.wav TEST Mobile2GNetwork /webrtc/src/resources/voice_engine/audio_tiny16.wav /webrtc/src/out/LowBandwidth_Mobile2GNetwork.wav [ OK ] LowBandwidthAudioTest.Mobile2GNetwork (6333 ms) [----------] 2 tests from LowBandwidthAudioTest (12265 ms total) +[----------] 2 tests from PCLowBandwidthAudioTest +[ RUN ] PCLowBandwidthAudioTest.PCGoodNetworkHighBitrate +TEST PCGoodNetworkHighBitrate /webrtc/src/resources/voice_engine/audio_tiny16.wav /webrtc/src/out/PCLowBandwidth_PCGoodNetworkHighBitrate.wav /webrtc/src/out/PCLowBandwidth_perf_48.json +[ OK ] PCLowBandwidthAudioTest.PCGoodNetworkHighBitrate (5932 ms) +[ RUN ] PCLowBandwidthAudioTest.PCMobile2GNetwork +TEST PCMobile2GNetwork /webrtc/src/resources/voice_engine/audio_tiny16.wav /webrtc/src/out/PCLowBandwidth_PCMobile2GNetwork.wav /webrtc/src/out/PCLowBandwidth_perf_48.json +[ OK ] PCLowBandwidthAudioTest.PCMobile2GNetwork (6333 ms) +[----------] 2 tests from PCLowBandwidthAudioTest (12265 ms total) [----------] Global test environment tear-down [==========] 2 tests from 1 test case ran. (12266 ms total) @@ -165,6 +191,34 @@ I 16.576s run_tests_on_device(TA99205CNO) [----------] Global test environmen I 16.576s run_tests_on_device(TA99205CNO) [==========] 1 test from 1 test case ran. (5968 ms total) I 16.577s run_tests_on_device(TA99205CNO) [ PASSED ] 1 test. I 16.577s run_tests_on_device(TA99205CNO) <>ScopedMainEntryLogger +I 14.078s run_tests_on_device(ddfa6149) Note: Google Test filter = PCLowBandwidthAudioTest.PCMobile2GNetwork +I 14.078s run_tests_on_device(ddfa6149) [==========] Running 1 test from 1 test case. +I 14.078s run_tests_on_device(ddfa6149) [----------] Global test environment set-up. +I 14.078s run_tests_on_device(ddfa6149) [----------] 1 test from PCLowBandwidthAudioTest +I 14.078s run_tests_on_device(ddfa6149) [ RUN ] PCLowBandwidthAudioTest.PCMobile2GNetwork +I 14.078s run_tests_on_device(ddfa6149) TEST PCMobile2GNetwork /sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav /sdcard/chromium_tests_root/PCLowBandwidth_PCMobile2GNetwork.wav /sdcard/chromium_tests_root/PCLowBandwidth_perf_48.json +I 14.078s run_tests_on_device(ddfa6149) [ OK ] PCLowBandwidthAudioTest.PCMobile2GNetwork (6438 ms) +I 14.078s run_tests_on_device(ddfa6149) [----------] 1 test from PCLowBandwidthAudioTest (6438 ms total) +I 14.078s run_tests_on_device(ddfa6149) +I 14.078s run_tests_on_device(ddfa6149) [----------] Global test environment tear-down +I 14.079s run_tests_on_device(ddfa6149) [==========] 1 test from 1 test case ran. (6438 ms total) +I 14.079s run_tests_on_device(ddfa6149) [ PASSED ] 1 test. +I 14.079s run_tests_on_device(ddfa6149) <>ScopedMainEntryLogger +I 16.576s run_tests_on_device(TA99205CNO) Note: Google Test filter = PCLowBandwidthAudioTest.PCGoodNetworkHighBitrate +I 16.576s run_tests_on_device(TA99205CNO) [==========] Running 1 test from 1 test case. +I 16.576s run_tests_on_device(TA99205CNO) [----------] Global test environment set-up. +I 16.576s run_tests_on_device(TA99205CNO) [----------] 1 test from PCLowBandwidthAudioTest +I 16.576s run_tests_on_device(TA99205CNO) [ RUN ] PCLowBandwidthAudioTest.PCGoodNetworkHighBitrate +I 16.576s run_tests_on_device(TA99205CNO) TEST PCGoodNetworkHighBitrate /sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav /sdcard/chromium_tests_root/PCLowBandwidth_PCGoodNetworkHighBitrate.wav /sdcard/chromium_tests_root/PCLowBandwidth_perf_48.json +I 16.576s run_tests_on_device(TA99205CNO) [ OK ] PCLowBandwidthAudioTest.PCGoodNetworkHighBitrate (5968 ms) +I 16.576s run_tests_on_device(TA99205CNO) [----------] 1 test from PCLowBandwidthAudioTest (5968 ms total) +I 16.576s run_tests_on_device(TA99205CNO) +I 16.576s run_tests_on_device(TA99205CNO) [----------] Global test environment tear-down +I 16.576s run_tests_on_device(TA99205CNO) [==========] 1 test from 1 test case ran. (5968 ms total) +I 16.577s run_tests_on_device(TA99205CNO) [ PASSED ] 1 test. +I 16.577s run_tests_on_device(TA99205CNO) <