admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add the perf builders and 2 perf tests bots to waterfalls.pyl.

Browse Source 
Bug: webrtc:13899
Change-Id: I4a58f81afc62ac66b810ff3f501887bd09856238
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/258126
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Jeremy Leconte <jleconte@google.com>
Cr-Commit-Position: refs/heads/main@{#36462}
This commit is contained in:
Jeremy Leconte 2022-04-06 12:22:08 +02:00 committed by WebRTC LUCI CQ
parent dbbc83c191
commit 7b84f06151
7 changed files with 507 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

215
infra/specs/client.webrtc.json
View File

@ -602,6 +602,125 @@
}
]
},
"Android32 Builder arm": {
"gtest_tests": [
{
"args": [
"--android",
"--adb-path",
"../../third_party/android_sdk/public/platform-tools/adb"
],
"merge": {
"args": [
"--test-suite",
"low_bandwidth_audio_perf_test"
],
"script": "../../tools_webrtc/perf/process_perf_results_py2.py"
},
"swarming": {
"can_use_on_swarming_builders": true
},
"test": "low_bandwidth_audio_perf_test",
"test_id_prefix": "ninja://audio:low_bandwidth_audio_perf_test/"
},
{
"args": [
"--nologs"
],
"merge": {
"args": [
"--test-suite",
"webrtc_perf_tests"
],
"script": "../../tools_webrtc/perf/process_perf_results_py2.py"
},
"swarming": {
"can_use_on_swarming_builders": true
},
"test": "webrtc_perf_tests",
"test_id_prefix": "ninja://:webrtc_perf_tests/"
}
]
},
"Android32 Builder arm64": {
"gtest_tests": [
{
"args": [
"--android",
"--adb-path",
"../../third_party/android_sdk/public/platform-tools/adb"
],
"merge": {
"args": [
"--test-suite",
"low_bandwidth_audio_perf_test"
],
"script": "../../tools_webrtc/perf/process_perf_results_py2.py"
},
"swarming": {
"can_use_on_swarming_builders": true
},
"test": "low_bandwidth_audio_perf_test",
"test_id_prefix": "ninja://audio:low_bandwidth_audio_perf_test/"
},
{
"args": [
"--nologs"
],
"merge": {
"args": [
"--test-suite",
"webrtc_perf_tests"
],
"script": "../../tools_webrtc/perf/process_perf_results_py2.py"
},
"swarming": {
"can_use_on_swarming_builders": true
},
"test": "webrtc_perf_tests",
"test_id_prefix": "ninja://:webrtc_perf_tests/"
}
]
},
"Linux64 Builder": {
"isolated_scripts": [
{
"isolate_name": "low_bandwidth_audio_perf_test",
"merge": {
"args": [
"--test-suite",
"low_bandwidth_audio_perf_test"
],
"script": "../../tools_webrtc/perf/process_perf_results_py2.py"
},
"name": "low_bandwidth_audio_perf_test",
"swarming": {
"can_use_on_swarming_builders": true
},
"test_id_prefix": "ninja://audio:low_bandwidth_audio_perf_test/"
},
{
"args": [
"--test_artifacts_dir=${ISOLATED_OUTDIR}",
"--save_worst_frame",
"--nologs"
],
"isolate_name": "webrtc_perf_tests",
"merge": {
"args": [
"--test-suite",
"webrtc_perf_tests"
],
"script": "../../tools_webrtc/perf/process_perf_results_py2.py"
},
"name": "webrtc_perf_tests",
"swarming": {
"can_use_on_swarming_builders": true
},
"test_id_prefix": "ninja://:webrtc_perf_tests/"
}
]
},
"Linux64 Release": {
"isolated_scripts": [
{
@ -1011,6 +1130,57 @@
}
]
},
"Mac64 Builder": {
"isolated_scripts": [
{
"isolate_name": "low_bandwidth_audio_perf_test",
"merge": {
"args": [
"--test-suite",
"low_bandwidth_audio_perf_test"
],
"script": "../../tools_webrtc/perf/process_perf_results_py2.py"
},
"name": "low_bandwidth_audio_perf_test",
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"cpu": "x86-64",
"os": "Mac"
}
]
},
"test_id_prefix": "ninja://audio:low_bandwidth_audio_perf_test/"
},
{
"args": [
"--test_artifacts_dir=${ISOLATED_OUTDIR}",
"--save_worst_frame",
"--nologs"
],
"isolate_name": "webrtc_perf_tests",
"merge": {
"args": [
"--test-suite",
"webrtc_perf_tests"
],
"script": "../../tools_webrtc/perf/process_perf_results_py2.py"
},
"name": "webrtc_perf_tests",
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"cpu": "x86-64",
"os": "Mac"
}
]
},
"test_id_prefix": "ninja://:webrtc_perf_tests/"
}
]
},
"Mac64 Release": {
"isolated_scripts": [
{
@ -1420,6 +1590,45 @@
}
]
},
"Win32 Builder (Clang)": {
"isolated_scripts": [
{
"isolate_name": "low_bandwidth_audio_perf_test",
"merge": {
"args": [
"--test-suite",
"low_bandwidth_audio_perf_test"
],
"script": "../../tools_webrtc/perf/process_perf_results_py2.py"
},
"name": "low_bandwidth_audio_perf_test",
"swarming": {
"can_use_on_swarming_builders": true
},
"test_id_prefix": "ninja://audio:low_bandwidth_audio_perf_test/"
},
{
"args": [
"--test_artifacts_dir=${ISOLATED_OUTDIR}",
"--save_worst_frame",
"--nologs"
],
"isolate_name": "webrtc_perf_tests",
"merge": {
"args": [
"--test-suite",
"webrtc_perf_tests"
],
"script": "../../tools_webrtc/perf/process_perf_results_py2.py"
},
"name": "webrtc_perf_tests",
"swarming": {
"can_use_on_swarming_builders": true
},
"test_id_prefix": "ninja://:webrtc_perf_tests/"
}
]
},
"Win32 Release (Clang)": {
"isolated_scripts": [
{
@ -2020,12 +2229,8 @@
"test_id_prefix": "ninja://net/dcsctp:dcsctp_unittests/"
},
{
"$mixin_append": {
"args": [
"--readline-timeout=360"
]
},
"args": [
"--readline-timeout=360",
"--platform",
"iPhone X",
"--version",

71
infra/specs/client.webrtc.perf.json Normal file
View File

@ -0,0 +1,71 @@
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"Perf Win7": {
"isolated_scripts": [
{
"args": [
"--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json"
],
"isolate_name": "low_bandwidth_audio_perf_test",
"merge": {
"args": [
"--test-suite",
"low_bandwidth_audio_perf_test"
],
"script": "../../tools_webrtc/perf/process_perf_results_py2.py"
},
"name": "low_bandwidth_audio_perf_test",
"resultdb": {
"result_file": "${ISOLATED_OUTDIR}/gtest_output.json",
"result_format": "gtest_json"
},
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"os": "Windows",
"pool": "WebRTC-perf"
}
],
"hard_timeout": 10800,
"io_timeout": 10800
},
"test_id_prefix": "ninja://audio:low_bandwidth_audio_perf_test/"
},
{
"args": [
"--test_artifacts_dir=${ISOLATED_OUTDIR}",
"--save_worst_frame",
"--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json",
"--nologs"
],
"isolate_name": "webrtc_perf_tests",
"merge": {
"args": [
"--test-suite",
"webrtc_perf_tests"
],
"script": "../../tools_webrtc/perf/process_perf_results_py2.py"
},
"name": "webrtc_perf_tests",
"resultdb": {
"result_file": "${ISOLATED_OUTDIR}/gtest_output.json",
"result_format": "gtest_json"
},
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"os": "Windows",
"pool": "WebRTC-perf"
}
],
"hard_timeout": 10800,
"io_timeout": 10800
},
"test_id_prefix": "ninja://:webrtc_perf_tests/"
}
]
}
}

53
infra/specs/internal.client.webrtc.json
View File

@ -1,6 +1,59 @@
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
"AAAAA2 See generate_buildbot_json.py to make changes": {},
"iOS64 Perf": {
"isolated_scripts": [
{
"args": [
"--write_perf_output_on_ios",
"--nologs",
"--xcode-build-version",
"12d4e",
"--nologs"
],
"isolate_name": "webrtc_perf_tests",
"merge": {
"args": [
"--test-suite",
"webrtc_perf_tests"
],
"script": "../../tools_webrtc/perf/process_perf_results_py2.py"
},
"name": "webrtc_perf_tests",
"resultdb": {
"enable": true,
"has_native_resultdb_integration": true
},
"swarming": {
"can_use_on_swarming_builders": true,
"cipd_packages": [
{
"cipd_package": "infra/tools/mac_toolchain/${platform}",
"location": ".",
"revision": "git_revision:723fc1a6c8cdf2631a57851f5610e598db0c1de1"
}
],
"dimension_sets": [
{
"id": "build15-a7",
"os": "iOS-12.4.1",
"pool": "WebRTC"
}
],
"hard_timeout": 10800,
"io_timeout": 10800,
"named_caches": [
{
"name": "xcode_ios_12d4e",
"path": "Xcode.app"
}
],
"service_account": "chrome-tester@chops-service-accounts.iam.gserviceaccount.com"
},
"test_id_prefix": "ninja://:webrtc_perf_tests/"
}
]
},
"iOS64 Release": {
"isolated_scripts": [
{

48
infra/specs/mixins.pyl
View File

@ -66,6 +66,15 @@
}
}
},
'ios-device-perf': {
'swarming': {
'dimensions': {
'os': 'iOS-12.4.1',
'pool': 'WebRTC',
'id': 'build15-a7'
}
}
},
'ios-simulator-14': {
'$mixin_append': {
'args': ['--platform', 'iPhone X', '--version', '14.0']
@ -140,6 +149,28 @@
}
}
},
'perf-low-bandwidth-audio-perf-test': {
'merge': {
'script': '../../tools_webrtc/perf/process_perf_results_py2.py',
'args': ['--test-suite', 'low_bandwidth_audio_perf_test']
}
},
'perf-pool': {
'swarming': {
'dimensions': {
'pool': 'WebRTC-perf'
}
}
},
'perf-webrtc-perf-tests': {
'merge': {
'script': '../../tools_webrtc/perf/process_perf_results_py2.py',
'args': ['--test-suite', 'webrtc_perf_tests']
},
'$mixin_append': {
'args': ['--nologs']
}
},
'quick-perf-tests': {
'$mixin_append': {
'args':
@ -160,6 +191,12 @@
'result_format': 'json'
}
},
'timeout-3h': {
'swarming': {
'hard_timeout': 10800,
'io_timeout': 10800
}
},
'webrtc-xctest': {
'$mixin_append': {
'args': ['--xctest', '--undefok=enable-run-ios-unittests-with-xctest']
@ -197,6 +234,17 @@
}
}
},
'xcode_12d4e': {
'$mixin_append': {
'args': ['--xcode-build-version', '12d4e'],
'swarming': {
'named_caches': [{
'name': 'xcode_ios_12d4e',
'path': 'Xcode.app'
}]
}
}
},
'xcode_13_main': {
'$mixin_append': {
'args': ['--xcode-build-version', '13c100']

37
infra/specs/mixins_webrtc.pyl
View File

@ -36,6 +36,15 @@
},
},
},
'ios-device-perf': {
'swarming': {
'dimensions': {
'os': 'iOS-12.4.1',
'pool': 'WebRTC',
'id': 'build15-a7',
},
},
},
'ios_runtime_cache_14_0': {
'$mixin_append': {
'swarming': {
@ -98,6 +107,28 @@
}
}
},
'perf-low-bandwidth-audio-perf-test': {
'merge': {
'script': '../../tools_webrtc/perf/process_perf_results_py2.py',
'args': ['--test-suite', 'low_bandwidth_audio_perf_test'],
},
},
'perf-pool': {
'swarming': {
'dimensions': {
'pool': 'WebRTC-perf',
},
},
},
'perf-webrtc-perf-tests': {
'merge': {
'script': '../../tools_webrtc/perf/process_perf_results_py2.py',
'args': ['--test-suite', 'webrtc_perf_tests'],
},
'$mixin_append': {
'args': ['--nologs']
}
},
'resultdb-gtest-json-format': {
'$mixin_append': {
'args': [
@ -114,6 +145,12 @@
'result_format': 'json'
}
},
'timeout-3h': {
'swarming': {
'hard_timeout': 10800,
'io_timeout': 10800,
},
},
'webrtc-xctest': {
'$mixin_append': {
'args': [

40
infra/specs/test_suites.pyl
View File

@ -12,6 +12,19 @@
'android_examples_junit_tests': {},
'android_sdk_junit_tests': {},
},
'android_perf_tests': {
'low_bandwidth_audio_perf_test': {
'mixins': ['perf-low-bandwidth-audio-perf-test'],
'args': [
'--android',
'--adb-path',
'../../third_party/android_sdk/public/platform-tools/adb',
]
},
'webrtc_perf_tests': {
'mixins': ['perf-webrtc-perf-tests'],
},
},
'android_tests': {
'AppRTCMobile_test_apk': {},
'android_instrumentation_test_apk': {},
@ -58,6 +71,18 @@
'mixins': ['quick-perf-tests'],
}
},
'desktop_perf_tests': {
'low_bandwidth_audio_perf_test': {
'mixins': ['perf-low-bandwidth-audio-perf-test']
},
'webrtc_perf_tests': {
'mixins': ['perf-webrtc-perf-tests'],
'args': [
'--test_artifacts_dir=${ISOLATED_OUTDIR}',
'--save_worst_frame',
],
},
},
'desktop_tests': {
'audio_decoder_unittests': {},
'common_audio_unittests': {},
@ -116,6 +141,12 @@
'video_capture_tests': {},
'video_engine_tests': {},
},
'ios_perf_tests': {
'webrtc_perf_tests': {
'mixins': ['perf-webrtc-perf-tests'],
'args': ['--write_perf_output_on_ios', '--nologs']
},
},
'ios_simulator_tests': {
# TODO(bugs.webrtc.org/12244): Some tests are skipped on iOS simulator
# platforms because they fail or they are flaky.
@ -125,11 +156,10 @@
'common_video_unittests': {},
'dcsctp_unittests': {},
'modules_tests': {
'$mixin_append': {
'args': [
'--readline-timeout=360',
],
},
'args': [
# Some tests exceed the default 180 seconds readline timeout.
'--readline-timeout=360',
],
},
'modules_unittests': {},
'rtc_media_unittests': {},

53
infra/specs/waterfalls.pyl
View File

@ -21,6 +21,22 @@
'junit_tests': 'android_junit_tests',
},
},
'Android32 Builder arm': {
'test_suites': {
'gtest_tests': 'android_perf_tests',
},
},
'Android32 Builder arm64': {
'test_suites': {
'gtest_tests': 'android_perf_tests',
},
},
'Linux64 Builder': {
'os_type': 'linux',
'test_suites': {
'isolated_scripts': 'desktop_perf_tests',
},
},
'Linux64 Release': {
'os_type': 'linux',
'mixins': ['linux-bionic', 'x86-64', 'resultdb-json-format'],
@ -28,6 +44,13 @@
'isolated_scripts': 'linux_tests',
},
},
'Mac64 Builder': {
'os_type': 'mac',
'mixins': ['mac', 'x86-64'],
'test_suites': {
'isolated_scripts': 'desktop_perf_tests',
},
},
'Mac64 Release': {
'os_type': 'mac',
'mixins': ['mac11', 'x86-64', 'resultdb-json-format'],
@ -35,6 +58,12 @@
'isolated_scripts': 'mac_tests',
},
},
'Win32 Builder (Clang)': {
'os_type': 'win',
'test_suites': {
'isolated_scripts': 'desktop_perf_tests',
},
},
'Win32 Release (Clang)': {
'os_type': 'win',
'mixins': ['win7', 'x86-64', 'resultdb-json-format'],
@ -54,10 +83,34 @@
},
},
},
{
'name': 'client.webrtc.perf',
'mixins': [],
'machines': {
'Perf Win7': {
'os_type':
'win',
'mixins':
['win', 'perf-pool', 'timeout-3h', 'resultdb-gtest-json-format'],
'test_suites': {
'isolated_scripts': 'desktop_perf_tests',
},
},
},
},
{
'name': 'internal.client.webrtc',
'mixins': [],
'machines': {
'iOS64 Perf': {
'mixins': [
'ios-device-perf', 'timeout-3h', 'chrome-tester-service-account',
'xcode_12d4e', 'mac_toolchain', 'has_native_resultdb_integration'
],
'test_suites': {
'isolated_scripts': 'ios_perf_tests',
},
},
'iOS64 Release': {
'mixins': [
'ios-device-15.3', 'webrtc-xctest', 'chrome-tester-service-account',