From 7b84f061511a7936eaea1fb267ede804a0f26d20 Mon Sep 17 00:00:00 2001 From: Jeremy Leconte Date: Wed, 6 Apr 2022 12:22:08 +0200 Subject: [PATCH] Add the perf builders and 2 perf tests bots to waterfalls.pyl. Bug: webrtc:13899 Change-Id: I4a58f81afc62ac66b810ff3f501887bd09856238 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/258126 Reviewed-by: Mirko Bonadei Commit-Queue: Jeremy Leconte Cr-Commit-Position: refs/heads/main@{#36462} --- infra/specs/client.webrtc.json | 215 +++++++++++++++++++++++- infra/specs/client.webrtc.perf.json | 71 ++++++++ infra/specs/internal.client.webrtc.json | 53 ++++++ infra/specs/mixins.pyl | 48 ++++++ infra/specs/mixins_webrtc.pyl | 37 ++++ infra/specs/test_suites.pyl | 40 ++++- infra/specs/waterfalls.pyl | 53 ++++++ 7 files changed, 507 insertions(+), 10 deletions(-) create mode 100644 infra/specs/client.webrtc.perf.json diff --git a/infra/specs/client.webrtc.json b/infra/specs/client.webrtc.json index f9116b813d..21b30e6015 100644 --- a/infra/specs/client.webrtc.json +++ b/infra/specs/client.webrtc.json @@ -602,6 +602,125 @@ } ] }, + "Android32 Builder arm": { + "gtest_tests": [ + { + "args": [ + "--android", + "--adb-path", + "../../third_party/android_sdk/public/platform-tools/adb" + ], + "merge": { + "args": [ + "--test-suite", + "low_bandwidth_audio_perf_test" + ], + "script": "../../tools_webrtc/perf/process_perf_results_py2.py" + }, + "swarming": { + "can_use_on_swarming_builders": true + }, + "test": "low_bandwidth_audio_perf_test", + "test_id_prefix": "ninja://audio:low_bandwidth_audio_perf_test/" + }, + { + "args": [ + "--nologs" + ], + "merge": { + "args": [ + "--test-suite", + "webrtc_perf_tests" + ], + "script": "../../tools_webrtc/perf/process_perf_results_py2.py" + }, + "swarming": { + "can_use_on_swarming_builders": true + }, + "test": "webrtc_perf_tests", + "test_id_prefix": "ninja://:webrtc_perf_tests/" + } + ] + }, + "Android32 Builder arm64": { + "gtest_tests": [ + { + "args": [ + "--android", + "--adb-path", + "../../third_party/android_sdk/public/platform-tools/adb" + ], + "merge": { + "args": [ + "--test-suite", + "low_bandwidth_audio_perf_test" + ], + "script": "../../tools_webrtc/perf/process_perf_results_py2.py" + }, + "swarming": { + "can_use_on_swarming_builders": true + }, + "test": "low_bandwidth_audio_perf_test", + "test_id_prefix": "ninja://audio:low_bandwidth_audio_perf_test/" + }, + { + "args": [ + "--nologs" + ], + "merge": { + "args": [ + "--test-suite", + "webrtc_perf_tests" + ], + "script": "../../tools_webrtc/perf/process_perf_results_py2.py" + }, + "swarming": { + "can_use_on_swarming_builders": true + }, + "test": "webrtc_perf_tests", + "test_id_prefix": "ninja://:webrtc_perf_tests/" + } + ] + }, + "Linux64 Builder": { + "isolated_scripts": [ + { + "isolate_name": "low_bandwidth_audio_perf_test", + "merge": { + "args": [ + "--test-suite", + "low_bandwidth_audio_perf_test" + ], + "script": "../../tools_webrtc/perf/process_perf_results_py2.py" + }, + "name": "low_bandwidth_audio_perf_test", + "swarming": { + "can_use_on_swarming_builders": true + }, + "test_id_prefix": "ninja://audio:low_bandwidth_audio_perf_test/" + }, + { + "args": [ + "--test_artifacts_dir=${ISOLATED_OUTDIR}", + "--save_worst_frame", + "--nologs" + ], + "isolate_name": "webrtc_perf_tests", + "merge": { + "args": [ + "--test-suite", + "webrtc_perf_tests" + ], + "script": "../../tools_webrtc/perf/process_perf_results_py2.py" + }, + "name": "webrtc_perf_tests", + "swarming": { + "can_use_on_swarming_builders": true + }, + "test_id_prefix": "ninja://:webrtc_perf_tests/" + } + ] + }, "Linux64 Release": { "isolated_scripts": [ { @@ -1011,6 +1130,57 @@ } ] }, + "Mac64 Builder": { + "isolated_scripts": [ + { + "isolate_name": "low_bandwidth_audio_perf_test", + "merge": { + "args": [ + "--test-suite", + "low_bandwidth_audio_perf_test" + ], + "script": "../../tools_webrtc/perf/process_perf_results_py2.py" + }, + "name": "low_bandwidth_audio_perf_test", + "swarming": { + "can_use_on_swarming_builders": true, + "dimension_sets": [ + { + "cpu": "x86-64", + "os": "Mac" + } + ] + }, + "test_id_prefix": "ninja://audio:low_bandwidth_audio_perf_test/" + }, + { + "args": [ + "--test_artifacts_dir=${ISOLATED_OUTDIR}", + "--save_worst_frame", + "--nologs" + ], + "isolate_name": "webrtc_perf_tests", + "merge": { + "args": [ + "--test-suite", + "webrtc_perf_tests" + ], + "script": "../../tools_webrtc/perf/process_perf_results_py2.py" + }, + "name": "webrtc_perf_tests", + "swarming": { + "can_use_on_swarming_builders": true, + "dimension_sets": [ + { + "cpu": "x86-64", + "os": "Mac" + } + ] + }, + "test_id_prefix": "ninja://:webrtc_perf_tests/" + } + ] + }, "Mac64 Release": { "isolated_scripts": [ { @@ -1420,6 +1590,45 @@ } ] }, + "Win32 Builder (Clang)": { + "isolated_scripts": [ + { + "isolate_name": "low_bandwidth_audio_perf_test", + "merge": { + "args": [ + "--test-suite", + "low_bandwidth_audio_perf_test" + ], + "script": "../../tools_webrtc/perf/process_perf_results_py2.py" + }, + "name": "low_bandwidth_audio_perf_test", + "swarming": { + "can_use_on_swarming_builders": true + }, + "test_id_prefix": "ninja://audio:low_bandwidth_audio_perf_test/" + }, + { + "args": [ + "--test_artifacts_dir=${ISOLATED_OUTDIR}", + "--save_worst_frame", + "--nologs" + ], + "isolate_name": "webrtc_perf_tests", + "merge": { + "args": [ + "--test-suite", + "webrtc_perf_tests" + ], + "script": "../../tools_webrtc/perf/process_perf_results_py2.py" + }, + "name": "webrtc_perf_tests", + "swarming": { + "can_use_on_swarming_builders": true + }, + "test_id_prefix": "ninja://:webrtc_perf_tests/" + } + ] + }, "Win32 Release (Clang)": { "isolated_scripts": [ { @@ -2020,12 +2229,8 @@ "test_id_prefix": "ninja://net/dcsctp:dcsctp_unittests/" }, { - "$mixin_append": { - "args": [ - "--readline-timeout=360" - ] - }, "args": [ + "--readline-timeout=360", "--platform", "iPhone X", "--version", diff --git a/infra/specs/client.webrtc.perf.json b/infra/specs/client.webrtc.perf.json new file mode 100644 index 0000000000..8830b6c3ac --- /dev/null +++ b/infra/specs/client.webrtc.perf.json @@ -0,0 +1,71 @@ +{ + "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {}, + "AAAAA2 See generate_buildbot_json.py to make changes": {}, + "Perf Win7": { + "isolated_scripts": [ + { + "args": [ + "--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json" + ], + "isolate_name": "low_bandwidth_audio_perf_test", + "merge": { + "args": [ + "--test-suite", + "low_bandwidth_audio_perf_test" + ], + "script": "../../tools_webrtc/perf/process_perf_results_py2.py" + }, + "name": "low_bandwidth_audio_perf_test", + "resultdb": { + "result_file": "${ISOLATED_OUTDIR}/gtest_output.json", + "result_format": "gtest_json" + }, + "swarming": { + "can_use_on_swarming_builders": true, + "dimension_sets": [ + { + "os": "Windows", + "pool": "WebRTC-perf" + } + ], + "hard_timeout": 10800, + "io_timeout": 10800 + }, + "test_id_prefix": "ninja://audio:low_bandwidth_audio_perf_test/" + }, + { + "args": [ + "--test_artifacts_dir=${ISOLATED_OUTDIR}", + "--save_worst_frame", + "--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json", + "--nologs" + ], + "isolate_name": "webrtc_perf_tests", + "merge": { + "args": [ + "--test-suite", + "webrtc_perf_tests" + ], + "script": "../../tools_webrtc/perf/process_perf_results_py2.py" + }, + "name": "webrtc_perf_tests", + "resultdb": { + "result_file": "${ISOLATED_OUTDIR}/gtest_output.json", + "result_format": "gtest_json" + }, + "swarming": { + "can_use_on_swarming_builders": true, + "dimension_sets": [ + { + "os": "Windows", + "pool": "WebRTC-perf" + } + ], + "hard_timeout": 10800, + "io_timeout": 10800 + }, + "test_id_prefix": "ninja://:webrtc_perf_tests/" + } + ] + } +} diff --git a/infra/specs/internal.client.webrtc.json b/infra/specs/internal.client.webrtc.json index 933c604b8f..3c22633af9 100644 --- a/infra/specs/internal.client.webrtc.json +++ b/infra/specs/internal.client.webrtc.json @@ -1,6 +1,59 @@ { "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {}, "AAAAA2 See generate_buildbot_json.py to make changes": {}, + "iOS64 Perf": { + "isolated_scripts": [ + { + "args": [ + "--write_perf_output_on_ios", + "--nologs", + "--xcode-build-version", + "12d4e", + "--nologs" + ], + "isolate_name": "webrtc_perf_tests", + "merge": { + "args": [ + "--test-suite", + "webrtc_perf_tests" + ], + "script": "../../tools_webrtc/perf/process_perf_results_py2.py" + }, + "name": "webrtc_perf_tests", + "resultdb": { + "enable": true, + "has_native_resultdb_integration": true + }, + "swarming": { + "can_use_on_swarming_builders": true, + "cipd_packages": [ + { + "cipd_package": "infra/tools/mac_toolchain/${platform}", + "location": ".", + "revision": "git_revision:723fc1a6c8cdf2631a57851f5610e598db0c1de1" + } + ], + "dimension_sets": [ + { + "id": "build15-a7", + "os": "iOS-12.4.1", + "pool": "WebRTC" + } + ], + "hard_timeout": 10800, + "io_timeout": 10800, + "named_caches": [ + { + "name": "xcode_ios_12d4e", + "path": "Xcode.app" + } + ], + "service_account": "chrome-tester@chops-service-accounts.iam.gserviceaccount.com" + }, + "test_id_prefix": "ninja://:webrtc_perf_tests/" + } + ] + }, "iOS64 Release": { "isolated_scripts": [ { diff --git a/infra/specs/mixins.pyl b/infra/specs/mixins.pyl index 88fd5470ef..89f37db2b5 100644 --- a/infra/specs/mixins.pyl +++ b/infra/specs/mixins.pyl @@ -66,6 +66,15 @@ } } }, + 'ios-device-perf': { + 'swarming': { + 'dimensions': { + 'os': 'iOS-12.4.1', + 'pool': 'WebRTC', + 'id': 'build15-a7' + } + } + }, 'ios-simulator-14': { '$mixin_append': { 'args': ['--platform', 'iPhone X', '--version', '14.0'] @@ -140,6 +149,28 @@ } } }, + 'perf-low-bandwidth-audio-perf-test': { + 'merge': { + 'script': '../../tools_webrtc/perf/process_perf_results_py2.py', + 'args': ['--test-suite', 'low_bandwidth_audio_perf_test'] + } + }, + 'perf-pool': { + 'swarming': { + 'dimensions': { + 'pool': 'WebRTC-perf' + } + } + }, + 'perf-webrtc-perf-tests': { + 'merge': { + 'script': '../../tools_webrtc/perf/process_perf_results_py2.py', + 'args': ['--test-suite', 'webrtc_perf_tests'] + }, + '$mixin_append': { + 'args': ['--nologs'] + } + }, 'quick-perf-tests': { '$mixin_append': { 'args': @@ -160,6 +191,12 @@ 'result_format': 'json' } }, + 'timeout-3h': { + 'swarming': { + 'hard_timeout': 10800, + 'io_timeout': 10800 + } + }, 'webrtc-xctest': { '$mixin_append': { 'args': ['--xctest', '--undefok=enable-run-ios-unittests-with-xctest'] @@ -197,6 +234,17 @@ } } }, + 'xcode_12d4e': { + '$mixin_append': { + 'args': ['--xcode-build-version', '12d4e'], + 'swarming': { + 'named_caches': [{ + 'name': 'xcode_ios_12d4e', + 'path': 'Xcode.app' + }] + } + } + }, 'xcode_13_main': { '$mixin_append': { 'args': ['--xcode-build-version', '13c100'] diff --git a/infra/specs/mixins_webrtc.pyl b/infra/specs/mixins_webrtc.pyl index 753539d6eb..6ff637c7cd 100644 --- a/infra/specs/mixins_webrtc.pyl +++ b/infra/specs/mixins_webrtc.pyl @@ -36,6 +36,15 @@ }, }, }, + 'ios-device-perf': { + 'swarming': { + 'dimensions': { + 'os': 'iOS-12.4.1', + 'pool': 'WebRTC', + 'id': 'build15-a7', + }, + }, + }, 'ios_runtime_cache_14_0': { '$mixin_append': { 'swarming': { @@ -98,6 +107,28 @@ } } }, + 'perf-low-bandwidth-audio-perf-test': { + 'merge': { + 'script': '../../tools_webrtc/perf/process_perf_results_py2.py', + 'args': ['--test-suite', 'low_bandwidth_audio_perf_test'], + }, + }, + 'perf-pool': { + 'swarming': { + 'dimensions': { + 'pool': 'WebRTC-perf', + }, + }, + }, + 'perf-webrtc-perf-tests': { + 'merge': { + 'script': '../../tools_webrtc/perf/process_perf_results_py2.py', + 'args': ['--test-suite', 'webrtc_perf_tests'], + }, + '$mixin_append': { + 'args': ['--nologs'] + } + }, 'resultdb-gtest-json-format': { '$mixin_append': { 'args': [ @@ -114,6 +145,12 @@ 'result_format': 'json' } }, + 'timeout-3h': { + 'swarming': { + 'hard_timeout': 10800, + 'io_timeout': 10800, + }, + }, 'webrtc-xctest': { '$mixin_append': { 'args': [ diff --git a/infra/specs/test_suites.pyl b/infra/specs/test_suites.pyl index ba07a78f96..a80ce9002f 100644 --- a/infra/specs/test_suites.pyl +++ b/infra/specs/test_suites.pyl @@ -12,6 +12,19 @@ 'android_examples_junit_tests': {}, 'android_sdk_junit_tests': {}, }, + 'android_perf_tests': { + 'low_bandwidth_audio_perf_test': { + 'mixins': ['perf-low-bandwidth-audio-perf-test'], + 'args': [ + '--android', + '--adb-path', + '../../third_party/android_sdk/public/platform-tools/adb', + ] + }, + 'webrtc_perf_tests': { + 'mixins': ['perf-webrtc-perf-tests'], + }, + }, 'android_tests': { 'AppRTCMobile_test_apk': {}, 'android_instrumentation_test_apk': {}, @@ -58,6 +71,18 @@ 'mixins': ['quick-perf-tests'], } }, + 'desktop_perf_tests': { + 'low_bandwidth_audio_perf_test': { + 'mixins': ['perf-low-bandwidth-audio-perf-test'] + }, + 'webrtc_perf_tests': { + 'mixins': ['perf-webrtc-perf-tests'], + 'args': [ + '--test_artifacts_dir=${ISOLATED_OUTDIR}', + '--save_worst_frame', + ], + }, + }, 'desktop_tests': { 'audio_decoder_unittests': {}, 'common_audio_unittests': {}, @@ -116,6 +141,12 @@ 'video_capture_tests': {}, 'video_engine_tests': {}, }, + 'ios_perf_tests': { + 'webrtc_perf_tests': { + 'mixins': ['perf-webrtc-perf-tests'], + 'args': ['--write_perf_output_on_ios', '--nologs'] + }, + }, 'ios_simulator_tests': { # TODO(bugs.webrtc.org/12244): Some tests are skipped on iOS simulator # platforms because they fail or they are flaky. @@ -125,11 +156,10 @@ 'common_video_unittests': {}, 'dcsctp_unittests': {}, 'modules_tests': { - '$mixin_append': { - 'args': [ - '--readline-timeout=360', - ], - }, + 'args': [ + # Some tests exceed the default 180 seconds readline timeout. + '--readline-timeout=360', + ], }, 'modules_unittests': {}, 'rtc_media_unittests': {}, diff --git a/infra/specs/waterfalls.pyl b/infra/specs/waterfalls.pyl index 09a8199fb5..b23f13e13f 100644 --- a/infra/specs/waterfalls.pyl +++ b/infra/specs/waterfalls.pyl @@ -21,6 +21,22 @@ 'junit_tests': 'android_junit_tests', }, }, + 'Android32 Builder arm': { + 'test_suites': { + 'gtest_tests': 'android_perf_tests', + }, + }, + 'Android32 Builder arm64': { + 'test_suites': { + 'gtest_tests': 'android_perf_tests', + }, + }, + 'Linux64 Builder': { + 'os_type': 'linux', + 'test_suites': { + 'isolated_scripts': 'desktop_perf_tests', + }, + }, 'Linux64 Release': { 'os_type': 'linux', 'mixins': ['linux-bionic', 'x86-64', 'resultdb-json-format'], @@ -28,6 +44,13 @@ 'isolated_scripts': 'linux_tests', }, }, + 'Mac64 Builder': { + 'os_type': 'mac', + 'mixins': ['mac', 'x86-64'], + 'test_suites': { + 'isolated_scripts': 'desktop_perf_tests', + }, + }, 'Mac64 Release': { 'os_type': 'mac', 'mixins': ['mac11', 'x86-64', 'resultdb-json-format'], @@ -35,6 +58,12 @@ 'isolated_scripts': 'mac_tests', }, }, + 'Win32 Builder (Clang)': { + 'os_type': 'win', + 'test_suites': { + 'isolated_scripts': 'desktop_perf_tests', + }, + }, 'Win32 Release (Clang)': { 'os_type': 'win', 'mixins': ['win7', 'x86-64', 'resultdb-json-format'], @@ -54,10 +83,34 @@ }, }, }, + { + 'name': 'client.webrtc.perf', + 'mixins': [], + 'machines': { + 'Perf Win7': { + 'os_type': + 'win', + 'mixins': + ['win', 'perf-pool', 'timeout-3h', 'resultdb-gtest-json-format'], + 'test_suites': { + 'isolated_scripts': 'desktop_perf_tests', + }, + }, + }, + }, { 'name': 'internal.client.webrtc', 'mixins': [], 'machines': { + 'iOS64 Perf': { + 'mixins': [ + 'ios-device-perf', 'timeout-3h', 'chrome-tester-service-account', + 'xcode_12d4e', 'mac_toolchain', 'has_native_resultdb_integration' + ], + 'test_suites': { + 'isolated_scripts': 'ios_perf_tests', + }, + }, 'iOS64 Release': { 'mixins': [ 'ios-device-15.3', 'webrtc-xctest', 'chrome-tester-service-account',