diff --git a/audio/BUILD.gn b/audio/BUILD.gn index 4a3d8fd032..0e78a6895e 100644 --- a/audio/BUILD.gn +++ b/audio/BUILD.gn @@ -177,6 +177,7 @@ if (rtc_include_tests) { sources = [ "test/low_bandwidth_audio_test.cc", + "test/low_bandwidth_audio_test_flags.cc", "test/pc_low_bandwidth_audio_test.cc", ] @@ -192,6 +193,7 @@ if (rtc_include_tests) { "../rtc_base:rtc_base_approved", "../system_wrappers", "../test:fileutils", + "../test:perf_test", "../test:test_common", "../test:test_main", "../test:test_support", diff --git a/audio/test/low_bandwidth_audio_test.cc b/audio/test/low_bandwidth_audio_test.cc index 54191e85a6..db1ff2cbba 100644 --- a/audio/test/low_bandwidth_audio_test.cc +++ b/audio/test/low_bandwidth_audio_test.cc @@ -14,15 +14,8 @@ #include "system_wrappers/include/sleep.h" #include "test/testsupport/file_utils.h" -WEBRTC_DEFINE_int(sample_rate_hz, - 16000, - "Sample rate (Hz) of the produced audio files."); - -WEBRTC_DEFINE_bool( - quick, - false, - "Don't do the full audio recording. " - "Used to quickly check that the test runs without crashing."); +WEBRTC_DECLARE_int(sample_rate_hz); +WEBRTC_DECLARE_bool(quick); namespace webrtc { namespace test { diff --git a/audio/test/low_bandwidth_audio_test.py b/audio/test/low_bandwidth_audio_test.py index add4f2f72b..8ad820e932 100755 --- a/audio/test/low_bandwidth_audio_test.py +++ b/audio/test/low_bandwidth_audio_test.py @@ -110,7 +110,8 @@ def _GetPathToTools(): def ExtractTestRuns(lines, echo=False): """Extracts information about tests from the output of a test runner. - Produces tuples (android_device, test_name, reference_file, degraded_file). + Produces tuples + (android_device, test_name, reference_file, degraded_file, cur_perf_results). """ for line in lines: if echo: @@ -118,7 +119,8 @@ def ExtractTestRuns(lines, echo=False): # Output from Android has a prefix with the device name. android_prefix_re = r'(?:I\b.+\brun_tests_on_device\((.+?)\)\s*)?' - test_re = r'^' + android_prefix_re + r'TEST (\w+) ([^ ]+?) ([^ ]+?)\s*$' + test_re = r'^' + android_prefix_re + (r'TEST (\w+) ([^ ]+?) ([^\s]+)' + r' ?([^\s]+)?\s*$') match = re.search(test_re, line) if match: @@ -206,7 +208,20 @@ def _AddChart(charts, metric, test_name, value, units): } -Analyzer = collections.namedtuple('Analyzer', ['func', 'executable', +def _AddRunPerfResults(charts, run_perf_results_file): + with open(run_perf_results_file, 'rb') as f: + per_run_perf_results = json.load(f) + if 'charts' not in per_run_perf_results: + return + for metric, cases in per_run_perf_results['charts'].items(): + chart = charts.setdefault(metric, {}) + for case_name, case_value in cases.items(): + if case_name in chart: + logging.error('Overriding results for %s/%s', metric, case_name) + chart[case_name] = case_value + + +Analyzer = collections.namedtuple('Analyzer', ['name', 'func', 'executable', 'sample_rate_hz']) @@ -228,25 +243,29 @@ def main(): else: test_command = [os.path.join(args.build_dir, 'low_bandwidth_audio_test')] - analyzers = [Analyzer(_RunPesq, pesq_path, 16000)] + analyzers = [Analyzer('pesq', _RunPesq, pesq_path, 16000)] # Check if POLQA can run at all, or skip the 48 kHz tests entirely. example_path = os.path.join(SRC_DIR, 'resources', 'voice_engine', 'audio_tiny48.wav') if polqa_path and _RunPolqa(polqa_path, example_path, example_path): - analyzers.append(Analyzer(_RunPolqa, polqa_path, 48000)) + analyzers.append(Analyzer('polqa', _RunPolqa, polqa_path, 48000)) charts = {} for analyzer in analyzers: # Start the test executable that produces audio files. test_process = subprocess.Popen( - _LogCommand(test_command + ['--sample_rate_hz=%d' % - analyzer.sample_rate_hz]), + _LogCommand(test_command + [ + '--sample_rate_hz=%d' % analyzer.sample_rate_hz, + '--test_case_prefix=%s' % analyzer.name + ]), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + perf_results_file = None try: lines = iter(test_process.stdout.readline, '') for result in ExtractTestRuns(lines, echo=True): - (android_device, test_name, reference_file, degraded_file) = result + (android_device, test_name, reference_file, degraded_file, + perf_results_file) = result adb_prefix = (args.adb_path,) if android_device: @@ -269,6 +288,12 @@ def main(): os.remove(degraded_file) finally: test_process.terminate() + if perf_results_file: + perf_results_file = _GetFile(perf_results_file, out_dir, move=True, + android=args.android, adb_prefix=adb_prefix) + _AddRunPerfResults(charts, perf_results_file) + if args.remove: + os.remove(perf_results_file) if args.isolated_script_test_perf_output: with open(args.isolated_script_test_perf_output, 'w') as f: diff --git a/audio/test/low_bandwidth_audio_test_flags.cc b/audio/test/low_bandwidth_audio_test_flags.cc new file mode 100644 index 0000000000..a0f12c5bc9 --- /dev/null +++ b/audio/test/low_bandwidth_audio_test_flags.cc @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +// #ifndef AUDIO_TEST_LOW_BANDWIDTH_AUDIO_TEST_FLAGS_H_ +// #define AUDIO_TEST_LOW_BANDWIDTH_AUDIO_TEST_FLAGS_H_ + +#include "rtc_base/flags.h" + +WEBRTC_DEFINE_int(sample_rate_hz, + 16000, + "Sample rate (Hz) of the produced audio files."); + +WEBRTC_DEFINE_bool( + quick, + false, + "Don't do the full audio recording. " + "Used to quickly check that the test runs without crashing."); + +WEBRTC_DEFINE_string(test_case_prefix, "", "Test case prefix."); + +// #endif // AUDIO_TEST_LOW_BANDWIDTH_AUDIO_TEST_FLAGS_H_ diff --git a/audio/test/pc_low_bandwidth_audio_test.cc b/audio/test/pc_low_bandwidth_audio_test.cc index 4eec672b8b..d5550fe861 100644 --- a/audio/test/pc_low_bandwidth_audio_test.cc +++ b/audio/test/pc_low_bandwidth_audio_test.cc @@ -19,6 +19,9 @@ #include "test/gtest.h" #include "test/pc/e2e/network_quality_metrics_reporter.h" #include "test/testsupport/file_utils.h" +#include "test/testsupport/perf_test.h" + +WEBRTC_DECLARE_string(test_case_prefix); namespace webrtc { namespace test { @@ -33,6 +36,16 @@ namespace { constexpr int kTestDurationSec = 45; +std::string GetMetricTestCaseName() { + const ::testing::TestInfo* const test_info = + ::testing::UnitTest::GetInstance()->current_test_info(); + std::string test_case_prefix(FLAG_test_case_prefix); + if (test_case_prefix.empty()) { + return test_info->name(); + } + return std::string(FLAG_test_case_prefix) + "_" + test_info->name(); +} + EmulatedNetworkNode* CreateEmulatedNodeWithConfig( NetworkEmulationManager* emulation, const BuiltInNetworkBehaviorConfig& config) { @@ -84,18 +97,25 @@ std::string AudioInputFile() { std::string AudioOutputFile() { const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); - return webrtc::test::OutputPath() + "LowBandwidth_" + test_info->name() + + return webrtc::test::OutputPath() + "PCLowBandwidth_" + test_info->name() + "_48.wav"; } -void PrintTestInfo() { +std::string PerfResultsOutputFile() { + return webrtc::test::OutputPath() + "PCLowBandwidth_perf_48.json"; +} + +void LogTestResults() { + std::string perf_results_output_file = PerfResultsOutputFile(); + webrtc::test::WritePerfResults(perf_results_output_file); + const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); // Output information about the input and output audio files so that further // processing can be done by an external process. - printf("TEST %s %s %s\n", test_info->name(), AudioInputFile().c_str(), - AudioOutputFile().c_str()); + printf("TEST %s %s %s %s\n", test_info->name(), AudioInputFile().c_str(), + AudioOutputFile().c_str(), perf_results_output_file.c_str()); } } // namespace @@ -104,7 +124,7 @@ TEST(PCLowBandwidthAudioTest, PCGoodNetworkHighBitrate) { std::unique_ptr network_emulation_manager = CreateNetworkEmulationManager(); auto fixture = CreateTestFixture( - "pc_good_network", + GetMetricTestCaseName(), CreateTwoNetworkLinks(network_emulation_manager.get(), BuiltInNetworkBehaviorConfig()), [](PeerConfigurer* alice) { @@ -117,7 +137,7 @@ TEST(PCLowBandwidthAudioTest, PCGoodNetworkHighBitrate) { }, [](PeerConfigurer* bob) {}); fixture->Run(RunParams(TimeDelta::seconds(kTestDurationSec))); - PrintTestInfo(); + LogTestResults(); } TEST(PCLowBandwidthAudioTest, PCMobile2GNetwork) { @@ -128,7 +148,7 @@ TEST(PCLowBandwidthAudioTest, PCMobile2GNetwork) { config.queue_length_packets = 1500; config.queue_delay_ms = 400; auto fixture = CreateTestFixture( - "pc_mobile_2g_network", + GetMetricTestCaseName(), CreateTwoNetworkLinks(network_emulation_manager.get(), config), [](PeerConfigurer* alice) { AudioConfig audio; @@ -141,7 +161,7 @@ TEST(PCLowBandwidthAudioTest, PCMobile2GNetwork) { [](PeerConfigurer* bob) {}); RunParams run_params(TimeDelta::seconds(kTestDurationSec)); fixture->Run(RunParams(TimeDelta::seconds(kTestDurationSec))); - PrintTestInfo(); + LogTestResults(); } } // namespace test diff --git a/audio/test/unittests/low_bandwidth_audio_test_test.py b/audio/test/unittests/low_bandwidth_audio_test_test.py index ee59139478..7403663cd4 100755 --- a/audio/test/unittests/low_bandwidth_audio_test_test.py +++ b/audio/test/unittests/low_bandwidth_audio_test_test.py @@ -28,19 +28,37 @@ class TestExtractTestRuns(unittest.TestCase): self._TestLog(LINUX_LOG, (None, 'GoodNetworkHighBitrate', '/webrtc/src/resources/voice_engine/audio_tiny16.wav', - '/webrtc/src/out/LowBandwidth_GoodNetworkHighBitrate.wav'), + '/webrtc/src/out/LowBandwidth_GoodNetworkHighBitrate.wav', None), (None, 'Mobile2GNetwork', '/webrtc/src/resources/voice_engine/audio_tiny16.wav', - '/webrtc/src/out/LowBandwidth_Mobile2GNetwork.wav')) + '/webrtc/src/out/LowBandwidth_Mobile2GNetwork.wav', None), + (None, 'PCGoodNetworkHighBitrate', + '/webrtc/src/resources/voice_engine/audio_tiny16.wav', + '/webrtc/src/out/PCLowBandwidth_PCGoodNetworkHighBitrate.wav', + '/webrtc/src/out/PCLowBandwidth_perf_48.json'), + (None, 'PCMobile2GNetwork', + '/webrtc/src/resources/voice_engine/audio_tiny16.wav', + '/webrtc/src/out/PCLowBandwidth_PCMobile2GNetwork.wav', + '/webrtc/src/out/PCLowBandwidth_perf_48.json')) def testAndroid(self): self._TestLog(ANDROID_LOG, ('ddfa6149', 'Mobile2GNetwork', '/sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav', - '/sdcard/chromium_tests_root/LowBandwidth_Mobile2GNetwork.wav'), + '/sdcard/chromium_tests_root/LowBandwidth_Mobile2GNetwork.wav', None), ('TA99205CNO', 'GoodNetworkHighBitrate', '/sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav', - '/sdcard/chromium_tests_root/LowBandwidth_GoodNetworkHighBitrate.wav')) + '/sdcard/chromium_tests_root/LowBandwidth_GoodNetworkHighBitrate.wav', + None), + ('ddfa6149', 'PCMobile2GNetwork', + '/sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav', + '/sdcard/chromium_tests_root/PCLowBandwidth_PCMobile2GNetwork.wav', + '/sdcard/chromium_tests_root/PCLowBandwidth_perf_48.json'), + ('TA99205CNO', 'PCGoodNetworkHighBitrate', + '/sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav', + ('/sdcard/chromium_tests_root/' + 'PCLowBandwidth_PCGoodNetworkHighBitrate.wav'), + '/sdcard/chromium_tests_root/PCLowBandwidth_perf_48.json')) LINUX_LOG = r'''\ @@ -54,6 +72,14 @@ TEST GoodNetworkHighBitrate /webrtc/src/resources/voice_engine/audio_tiny16.wav TEST Mobile2GNetwork /webrtc/src/resources/voice_engine/audio_tiny16.wav /webrtc/src/out/LowBandwidth_Mobile2GNetwork.wav [ OK ] LowBandwidthAudioTest.Mobile2GNetwork (6333 ms) [----------] 2 tests from LowBandwidthAudioTest (12265 ms total) +[----------] 2 tests from PCLowBandwidthAudioTest +[ RUN ] PCLowBandwidthAudioTest.PCGoodNetworkHighBitrate +TEST PCGoodNetworkHighBitrate /webrtc/src/resources/voice_engine/audio_tiny16.wav /webrtc/src/out/PCLowBandwidth_PCGoodNetworkHighBitrate.wav /webrtc/src/out/PCLowBandwidth_perf_48.json +[ OK ] PCLowBandwidthAudioTest.PCGoodNetworkHighBitrate (5932 ms) +[ RUN ] PCLowBandwidthAudioTest.PCMobile2GNetwork +TEST PCMobile2GNetwork /webrtc/src/resources/voice_engine/audio_tiny16.wav /webrtc/src/out/PCLowBandwidth_PCMobile2GNetwork.wav /webrtc/src/out/PCLowBandwidth_perf_48.json +[ OK ] PCLowBandwidthAudioTest.PCMobile2GNetwork (6333 ms) +[----------] 2 tests from PCLowBandwidthAudioTest (12265 ms total) [----------] Global test environment tear-down [==========] 2 tests from 1 test case ran. (12266 ms total) @@ -165,6 +191,34 @@ I 16.576s run_tests_on_device(TA99205CNO) [----------] Global test environmen I 16.576s run_tests_on_device(TA99205CNO) [==========] 1 test from 1 test case ran. (5968 ms total) I 16.577s run_tests_on_device(TA99205CNO) [ PASSED ] 1 test. I 16.577s run_tests_on_device(TA99205CNO) <>ScopedMainEntryLogger +I 14.078s run_tests_on_device(ddfa6149) Note: Google Test filter = PCLowBandwidthAudioTest.PCMobile2GNetwork +I 14.078s run_tests_on_device(ddfa6149) [==========] Running 1 test from 1 test case. +I 14.078s run_tests_on_device(ddfa6149) [----------] Global test environment set-up. +I 14.078s run_tests_on_device(ddfa6149) [----------] 1 test from PCLowBandwidthAudioTest +I 14.078s run_tests_on_device(ddfa6149) [ RUN ] PCLowBandwidthAudioTest.PCMobile2GNetwork +I 14.078s run_tests_on_device(ddfa6149) TEST PCMobile2GNetwork /sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav /sdcard/chromium_tests_root/PCLowBandwidth_PCMobile2GNetwork.wav /sdcard/chromium_tests_root/PCLowBandwidth_perf_48.json +I 14.078s run_tests_on_device(ddfa6149) [ OK ] PCLowBandwidthAudioTest.PCMobile2GNetwork (6438 ms) +I 14.078s run_tests_on_device(ddfa6149) [----------] 1 test from PCLowBandwidthAudioTest (6438 ms total) +I 14.078s run_tests_on_device(ddfa6149) +I 14.078s run_tests_on_device(ddfa6149) [----------] Global test environment tear-down +I 14.079s run_tests_on_device(ddfa6149) [==========] 1 test from 1 test case ran. (6438 ms total) +I 14.079s run_tests_on_device(ddfa6149) [ PASSED ] 1 test. +I 14.079s run_tests_on_device(ddfa6149) <>ScopedMainEntryLogger +I 16.576s run_tests_on_device(TA99205CNO) Note: Google Test filter = PCLowBandwidthAudioTest.PCGoodNetworkHighBitrate +I 16.576s run_tests_on_device(TA99205CNO) [==========] Running 1 test from 1 test case. +I 16.576s run_tests_on_device(TA99205CNO) [----------] Global test environment set-up. +I 16.576s run_tests_on_device(TA99205CNO) [----------] 1 test from PCLowBandwidthAudioTest +I 16.576s run_tests_on_device(TA99205CNO) [ RUN ] PCLowBandwidthAudioTest.PCGoodNetworkHighBitrate +I 16.576s run_tests_on_device(TA99205CNO) TEST PCGoodNetworkHighBitrate /sdcard/chromium_tests_root/resources/voice_engine/audio_tiny16.wav /sdcard/chromium_tests_root/PCLowBandwidth_PCGoodNetworkHighBitrate.wav /sdcard/chromium_tests_root/PCLowBandwidth_perf_48.json +I 16.576s run_tests_on_device(TA99205CNO) [ OK ] PCLowBandwidthAudioTest.PCGoodNetworkHighBitrate (5968 ms) +I 16.576s run_tests_on_device(TA99205CNO) [----------] 1 test from PCLowBandwidthAudioTest (5968 ms total) +I 16.576s run_tests_on_device(TA99205CNO) +I 16.576s run_tests_on_device(TA99205CNO) [----------] Global test environment tear-down +I 16.576s run_tests_on_device(TA99205CNO) [==========] 1 test from 1 test case ran. (5968 ms total) +I 16.577s run_tests_on_device(TA99205CNO) [ PASSED ] 1 test. +I 16.577s run_tests_on_device(TA99205CNO) <