admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run the same perf tests on all platforms.

Browse Source 
Change-Id: I49ac9921d92e3ff81569b162c64c96559565ae62
Bug: b/284448060
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/307021
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Jeremy Leconte <jleconte@google.com>
Cr-Commit-Position: refs/heads/main@{#40204}
This commit is contained in:
Jeremy Leconte 2023-05-30 13:18:52 +02:00 committed by WebRTC LUCI CQ
parent 267040e29a
commit b29ee5bc8f
5 changed files with 232 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
infra/specs/client.webrtc.perf.json
View File

@ -40,6 +40,8 @@
},
{
"args": [
"--test_artifacts_dir=${ISOLATED_OUTDIR}",
"--save_worst_frame",
"--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb",
"--nologs"
],
@ -115,6 +117,8 @@
},
{
"args": [
"--test_artifacts_dir=${ISOLATED_OUTDIR}",
"--save_worst_frame",
"--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb",
"--nologs"
],
@ -190,6 +194,8 @@
},
{
"args": [
"--test_artifacts_dir=${ISOLATED_OUTDIR}",
"--save_worst_frame",
"--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb",
"--nologs"
],
@ -265,6 +271,8 @@
},
{
"args": [
"--test_artifacts_dir=${ISOLATED_OUTDIR}",
"--save_worst_frame",
"--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb",
"--nologs"
],
@ -305,6 +313,46 @@
"isolated_scripts": [
{
"args": [
"--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json",
"--test-launcher-summary-output=${ISOLATED_OUTDIR}/gtest_output.json",
"--test-arg=--gtest_output=json:/custom_artifacts/gtest_output.json",
"--test-arg=--undefok=test_launcher_summary_output",
"--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb"
],
"isolate_name": "video_codec_perf_tests",
"merge": {
"args": [
"--test-suite",
"video_codec_perf_tests"
],
"script": "//tools_webrtc/perf/process_perf_results.py"
},
"name": "video_codec_perf_tests",
"resultdb": {
"result_file": "${ISOLATED_OUTDIR}/gtest_output.json",
"result_format": "gtest_json"
},
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"cpu": "x86-64",
"gce": "1",
"os": "Ubuntu-18.04",
"pool": "WebRTC-perf"
}
],
"expiration": 10800,
"hard_timeout": 10800,
"idempotent": false,
"io_timeout": 10800
},
"test_id_prefix": "ninja://modules/video_coding:video_codec_perf_tests/"
},
{
"args": [
"--test_artifacts_dir=${ISOLATED_OUTDIR}",
"--save_worst_frame",
"--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json",
"--test-launcher-summary-output=${ISOLATED_OUTDIR}/gtest_output.json",
"--test-arg=--gtest_output=json:/custom_artifacts/gtest_output.json",
@ -496,6 +544,42 @@
"isolated_scripts": [
{
"args": [
"--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json"
],
"isolate_name": "video_codec_perf_tests",
"merge": {
"args": [
"--test-suite",
"video_codec_perf_tests"
],
"script": "//tools_webrtc/perf/process_perf_results.py"
},
"name": "video_codec_perf_tests",
"resultdb": {
"result_file": "${ISOLATED_OUTDIR}/gtest_output.json",
"result_format": "gtest_json"
},
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"cpu": "arm64-64-Apple_M1",
"gce": "0",
"os": "Mac-12",
"pool": "WebRTC-perf"
}
],
"expiration": 10800,
"hard_timeout": 10800,
"idempotent": false,
"io_timeout": 10800
},
"test_id_prefix": "ninja://modules/video_coding:video_codec_perf_tests/"
},
{
"args": [
"--test_artifacts_dir=${ISOLATED_OUTDIR}",
"--save_worst_frame",
"--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json",
"--nologs"
],

53
infra/specs/internal.client.webrtc.json
View File

@ -477,6 +477,59 @@
"isolated_scripts": [
{
"args": [
"--write_perf_output_on_ios",
"--xctest",
"--xcode-build-version",
"13c100",
"--out-dir",
"${ISOLATED_OUTDIR}"
],
"isolate_name": "video_codec_perf_tests",
"merge": {
"args": [
"--test-suite",
"video_codec_perf_tests"
],
"script": "//tools_webrtc/perf/process_perf_results.py"
},
"name": "video_codec_perf_tests",
"resultdb": {
"enable": true,
"has_native_resultdb_integration": true
},
"swarming": {
"can_use_on_swarming_builders": true,
"cipd_packages": [
{
"cipd_package": "infra/tools/mac_toolchain/${platform}",
"location": ".",
"revision": "git_revision:a598cd77abc01089647a79e3cfa3cc8e8ecb5118"
}
],
"dimension_sets": [
{
"id": "mac-254-e504",
"os": "iOS-12.4.1",
"pool": "WebRTC"
}
],
"hard_timeout": 10800,
"idempotent": false,
"io_timeout": 10800,
"named_caches": [
{
"name": "xcode_ios_13c100",
"path": "Xcode.app"
}
],
"service_account": "chrome-tester@chops-service-accounts.iam.gserviceaccount.com"
},
"test_id_prefix": "ninja://modules/video_coding:video_codec_perf_tests/"
},
{
"args": [
"--test_artifacts_dir=${ISOLATED_OUTDIR}",
"--save_worst_frame",
"--write_perf_output_on_ios",
"--xctest",
"--xcode-build-version",

86
infra/specs/test_suites.pyl
View File

@ -12,14 +12,6 @@
'android_examples_junit_tests': {},
'android_sdk_junit_tests': {},
},
'android_perf_tests': {
'video_codec_perf_tests': {
'mixins': ['perf-video-codec-perf-tests'],
},
'webrtc_perf_tests': {
'mixins': ['perf-webrtc-perf-tests'],
},
},
'android_tests': {
'AppRTCMobile_test_apk': {},
'android_instrumentation_test_apk': {},
@ -56,23 +48,11 @@
'voip_unittests': {},
'webrtc_nonparallel_tests': {},
},
'android_webrtc_perf_tests_tryserver': {
'android_tests_tryserver_specific': {
'webrtc_perf_tests': {
'mixins': ['quick-perf-tests'],
}
},
'desktop_perf_tests': {
'video_codec_perf_tests': {
'mixins': ['perf-video-codec-perf-tests'],
},
'webrtc_perf_tests': {
'mixins': ['perf-webrtc-perf-tests'],
'args': [
'--test_artifacts_dir=${ISOLATED_OUTDIR}',
'--save_worst_frame',
],
},
},
'desktop_tests': {
'audio_decoder_unittests': {},
'common_audio_unittests': {},
@ -107,6 +87,20 @@
'voip_unittests': {},
'webrtc_nonparallel_tests': {},
},
'desktop_tests_linux_specific': {
'shared_screencast_stream_test': {},
},
'desktop_tests_try_server_specific': {
'video_capture_tests': {
'mixins': ['baremetal-try-pool'],
},
'video_codec_perf_tests': {
'mixins': ['quick-perf-tests', 'resultdb-gtest-json-format'],
},
'webrtc_perf_tests': {
'mixins': ['quick-perf-tests', 'resultdb-gtest-json-format'],
}
},
'fuchsia_compatible_tests': {
'audio_decoder_unittests': {},
'common_audio_unittests': {},
@ -201,9 +195,6 @@
'voip_unittests': {},
'webrtc_nonparallel_tests': {},
},
'linux_desktop_specific_tests': {
'shared_screencast_stream_test': {},
},
'more_configs_tests': {
'peerconnection_unittests': {
'swarming': {
@ -211,31 +202,23 @@
},
},
},
'perf_tests': {
'video_codec_perf_tests': {
'mixins': ['perf-video-codec-perf-tests'],
},
'webrtc_perf_tests': {
'mixins': ['perf-webrtc-perf-tests'],
'args': [
'--test_artifacts_dir=${ISOLATED_OUTDIR}',
'--save_worst_frame',
],
},
},
'video_capture_tests': {
'video_capture_tests': {
'mixins': ['baremetal-pool'],
}
},
'video_capture_tests_tryserver': {
'video_capture_tests': {
'mixins': ['baremetal-try-pool'],
}
},
'video_codec_perf_tests_tryserver': {
'video_codec_perf_tests': {
'mixins': ['quick-perf-tests', 'resultdb-gtest-json-format'],
}
},
'webrtc_perf_tests': {
'webrtc_perf_tests': {
'mixins': ['perf-webrtc-perf-tests'],
},
},
'webrtc_perf_tests_tryserver': {
'webrtc_perf_tests': {
'mixins': ['quick-perf-tests', 'resultdb-gtest-json-format'],
}
},
},
##############################################################################
@ -244,12 +227,11 @@
'compound_suites': {
'android_tests_tryserver': [
'android_tests',
'android_webrtc_perf_tests_tryserver',
'android_tests_tryserver_specific',
],
'desktop_tests_tryserver': [
'desktop_tests',
'video_capture_tests_tryserver',
'webrtc_perf_tests_tryserver',
'desktop_tests_try_server_specific',
],
'desktop_tests_with_video_capture': [
'desktop_tests',
@ -257,19 +239,17 @@
],
'linux_desktop_tests_tryserver': [
'desktop_tests',
'linux_desktop_specific_tests',
'video_capture_tests_tryserver',
'video_codec_perf_tests_tryserver',
'webrtc_perf_tests_tryserver',
'desktop_tests_linux_specific',
'desktop_tests_try_server_specific',
],
'linux_desktop_tests_with_video_capture': [
'desktop_tests',
'linux_desktop_specific_tests',
'desktop_tests_linux_specific',
'video_capture_tests',
],
'linux_tests': [
'desktop_tests',
'linux_desktop_specific_tests',
'desktop_tests_linux_specific',
],
},

52
infra/specs/tryserver.webrtc.json
View File

@ -12248,6 +12248,32 @@
},
"test_id_prefix": "ninja://modules/video_capture:video_capture_tests/"
},
{
"args": [
"--force_fieldtrials=WebRTC-QuickPerfTest/Enabled/",
"--nologs",
"--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json"
],
"isolate_name": "video_codec_perf_tests",
"merge": {
"script": "//testing/merge_scripts/standard_isolated_script_merge.py"
},
"name": "video_codec_perf_tests",
"resultdb": {
"result_file": "${ISOLATED_OUTDIR}/gtest_output.json",
"result_format": "gtest_json"
},
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"cpu": "x86-64",
"os": "Mac-12"
}
]
},
"test_id_prefix": "ninja://modules/video_coding:video_codec_perf_tests/"
},
{
"isolate_name": "video_engine_tests",
"merge": {
@ -14759,6 +14785,32 @@
},
"test_id_prefix": "ninja://modules/video_capture:video_capture_tests/"
},
{
"args": [
"--force_fieldtrials=WebRTC-QuickPerfTest/Enabled/",
"--nologs",
"--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json"
],
"isolate_name": "video_codec_perf_tests",
"merge": {
"script": "//testing/merge_scripts/standard_isolated_script_merge.py"
},
"name": "video_codec_perf_tests",
"resultdb": {
"result_file": "${ISOLATED_OUTDIR}/gtest_output.json",
"result_format": "gtest_json"
},
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"cpu": "x86-64",
"os": "Windows-10-19045"
}
]
},
"test_id_prefix": "ninja://modules/video_coding:video_codec_perf_tests/"
},
{
"isolate_name": "video_engine_tests",
"merge": {

20
infra/specs/waterfalls.pyl
View File

@ -252,7 +252,7 @@
'chromium-tester-service-account'
],
'test_suites': {
'gtest_tests': 'android_perf_tests',
'gtest_tests': 'perf_tests',
},
},
'Perf Android32 (R Pixel5)': {
@ -261,7 +261,7 @@
'has_native_resultdb_integration', 'chromium-tester-service-account'
],
'test_suites': {
'gtest_tests': 'android_perf_tests',
'gtest_tests': 'perf_tests',
},
},
'Perf Android64 (O Pixel2)': {
@ -271,7 +271,7 @@
'chromium-tester-service-account'
],
'test_suites': {
'gtest_tests': 'android_perf_tests',
'gtest_tests': 'perf_tests',
},
},
'Perf Android64 (R Pixel5)': {
@ -280,7 +280,7 @@
'has_native_resultdb_integration', 'chromium-tester-service-account'
],
'test_suites': {
'gtest_tests': 'android_perf_tests',
'gtest_tests': 'perf_tests',
},
},
'Perf Fuchsia': {
@ -291,7 +291,7 @@
'resultdb-gtest-json-format', 'fuchsia-gtest-output', 'perf-output'
],
'test_suites': {
'isolated_scripts': 'webrtc_perf_tests',
'isolated_scripts': 'perf_tests',
}
},
'Perf Linux Bionic': {
@ -302,7 +302,7 @@
'resultdb-gtest-json-format'
],
'test_suites': {
'isolated_scripts': 'desktop_perf_tests',
'isolated_scripts': 'perf_tests',
},
},
'Perf Mac 11': {
@ -313,7 +313,7 @@
'resultdb-gtest-json-format'
],
'test_suites': {
'isolated_scripts': 'desktop_perf_tests',
'isolated_scripts': 'perf_tests',
},
},
'Perf Mac M1 Arm64 12': {
@ -324,7 +324,7 @@
'resultdb-gtest-json-format'
],
'test_suites': {
'isolated_scripts': 'webrtc_perf_tests',
'isolated_scripts': 'perf_tests',
},
},
'Perf Win 10': {
@ -333,7 +333,7 @@
'mixins':
['win10-any', 'perf-pool', 'timeout-3h', 'resultdb-gtest-json-format'],
'test_suites': {
'isolated_scripts': 'desktop_perf_tests',
'isolated_scripts': 'perf_tests',
},
},
},
@ -359,7 +359,7 @@
'has_native_resultdb_integration', 'out_dir_arg'
],
'test_suites': {
'isolated_scripts': 'webrtc_perf_tests',
'isolated_scripts': 'perf_tests',
},
'args': [
'--write_perf_output_on_ios',