diff --git a/infra/specs/client.webrtc.perf.json b/infra/specs/client.webrtc.perf.json index 32ec37fbd3..3f550204ca 100644 --- a/infra/specs/client.webrtc.perf.json +++ b/infra/specs/client.webrtc.perf.json @@ -40,6 +40,8 @@ }, { "args": [ + "--test_artifacts_dir=${ISOLATED_OUTDIR}", + "--save_worst_frame", "--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb", "--nologs" ], @@ -115,6 +117,8 @@ }, { "args": [ + "--test_artifacts_dir=${ISOLATED_OUTDIR}", + "--save_worst_frame", "--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb", "--nologs" ], @@ -190,6 +194,8 @@ }, { "args": [ + "--test_artifacts_dir=${ISOLATED_OUTDIR}", + "--save_worst_frame", "--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb", "--nologs" ], @@ -265,6 +271,8 @@ }, { "args": [ + "--test_artifacts_dir=${ISOLATED_OUTDIR}", + "--save_worst_frame", "--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb", "--nologs" ], @@ -305,6 +313,46 @@ "isolated_scripts": [ { "args": [ + "--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json", + "--test-launcher-summary-output=${ISOLATED_OUTDIR}/gtest_output.json", + "--test-arg=--gtest_output=json:/custom_artifacts/gtest_output.json", + "--test-arg=--undefok=test_launcher_summary_output", + "--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb" + ], + "isolate_name": "video_codec_perf_tests", + "merge": { + "args": [ + "--test-suite", + "video_codec_perf_tests" + ], + "script": "//tools_webrtc/perf/process_perf_results.py" + }, + "name": "video_codec_perf_tests", + "resultdb": { + "result_file": "${ISOLATED_OUTDIR}/gtest_output.json", + "result_format": "gtest_json" + }, + "swarming": { + "can_use_on_swarming_builders": true, + "dimension_sets": [ + { + "cpu": "x86-64", + "gce": "1", + "os": "Ubuntu-18.04", + "pool": "WebRTC-perf" + } + ], + "expiration": 10800, + "hard_timeout": 10800, + "idempotent": false, + "io_timeout": 10800 + }, + "test_id_prefix": "ninja://modules/video_coding:video_codec_perf_tests/" + }, + { + "args": [ + "--test_artifacts_dir=${ISOLATED_OUTDIR}", + "--save_worst_frame", "--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json", "--test-launcher-summary-output=${ISOLATED_OUTDIR}/gtest_output.json", "--test-arg=--gtest_output=json:/custom_artifacts/gtest_output.json", @@ -496,6 +544,42 @@ "isolated_scripts": [ { "args": [ + "--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json" + ], + "isolate_name": "video_codec_perf_tests", + "merge": { + "args": [ + "--test-suite", + "video_codec_perf_tests" + ], + "script": "//tools_webrtc/perf/process_perf_results.py" + }, + "name": "video_codec_perf_tests", + "resultdb": { + "result_file": "${ISOLATED_OUTDIR}/gtest_output.json", + "result_format": "gtest_json" + }, + "swarming": { + "can_use_on_swarming_builders": true, + "dimension_sets": [ + { + "cpu": "arm64-64-Apple_M1", + "gce": "0", + "os": "Mac-12", + "pool": "WebRTC-perf" + } + ], + "expiration": 10800, + "hard_timeout": 10800, + "idempotent": false, + "io_timeout": 10800 + }, + "test_id_prefix": "ninja://modules/video_coding:video_codec_perf_tests/" + }, + { + "args": [ + "--test_artifacts_dir=${ISOLATED_OUTDIR}", + "--save_worst_frame", "--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json", "--nologs" ], diff --git a/infra/specs/internal.client.webrtc.json b/infra/specs/internal.client.webrtc.json index 9fcb9225e7..0bb5b89028 100644 --- a/infra/specs/internal.client.webrtc.json +++ b/infra/specs/internal.client.webrtc.json @@ -477,6 +477,59 @@ "isolated_scripts": [ { "args": [ + "--write_perf_output_on_ios", + "--xctest", + "--xcode-build-version", + "13c100", + "--out-dir", + "${ISOLATED_OUTDIR}" + ], + "isolate_name": "video_codec_perf_tests", + "merge": { + "args": [ + "--test-suite", + "video_codec_perf_tests" + ], + "script": "//tools_webrtc/perf/process_perf_results.py" + }, + "name": "video_codec_perf_tests", + "resultdb": { + "enable": true, + "has_native_resultdb_integration": true + }, + "swarming": { + "can_use_on_swarming_builders": true, + "cipd_packages": [ + { + "cipd_package": "infra/tools/mac_toolchain/${platform}", + "location": ".", + "revision": "git_revision:a598cd77abc01089647a79e3cfa3cc8e8ecb5118" + } + ], + "dimension_sets": [ + { + "id": "mac-254-e504", + "os": "iOS-12.4.1", + "pool": "WebRTC" + } + ], + "hard_timeout": 10800, + "idempotent": false, + "io_timeout": 10800, + "named_caches": [ + { + "name": "xcode_ios_13c100", + "path": "Xcode.app" + } + ], + "service_account": "chrome-tester@chops-service-accounts.iam.gserviceaccount.com" + }, + "test_id_prefix": "ninja://modules/video_coding:video_codec_perf_tests/" + }, + { + "args": [ + "--test_artifacts_dir=${ISOLATED_OUTDIR}", + "--save_worst_frame", "--write_perf_output_on_ios", "--xctest", "--xcode-build-version", diff --git a/infra/specs/test_suites.pyl b/infra/specs/test_suites.pyl index 703ac34247..c0ee0a79e8 100644 --- a/infra/specs/test_suites.pyl +++ b/infra/specs/test_suites.pyl @@ -12,14 +12,6 @@ 'android_examples_junit_tests': {}, 'android_sdk_junit_tests': {}, }, - 'android_perf_tests': { - 'video_codec_perf_tests': { - 'mixins': ['perf-video-codec-perf-tests'], - }, - 'webrtc_perf_tests': { - 'mixins': ['perf-webrtc-perf-tests'], - }, - }, 'android_tests': { 'AppRTCMobile_test_apk': {}, 'android_instrumentation_test_apk': {}, @@ -56,23 +48,11 @@ 'voip_unittests': {}, 'webrtc_nonparallel_tests': {}, }, - 'android_webrtc_perf_tests_tryserver': { + 'android_tests_tryserver_specific': { 'webrtc_perf_tests': { 'mixins': ['quick-perf-tests'], } }, - 'desktop_perf_tests': { - 'video_codec_perf_tests': { - 'mixins': ['perf-video-codec-perf-tests'], - }, - 'webrtc_perf_tests': { - 'mixins': ['perf-webrtc-perf-tests'], - 'args': [ - '--test_artifacts_dir=${ISOLATED_OUTDIR}', - '--save_worst_frame', - ], - }, - }, 'desktop_tests': { 'audio_decoder_unittests': {}, 'common_audio_unittests': {}, @@ -107,6 +87,20 @@ 'voip_unittests': {}, 'webrtc_nonparallel_tests': {}, }, + 'desktop_tests_linux_specific': { + 'shared_screencast_stream_test': {}, + }, + 'desktop_tests_try_server_specific': { + 'video_capture_tests': { + 'mixins': ['baremetal-try-pool'], + }, + 'video_codec_perf_tests': { + 'mixins': ['quick-perf-tests', 'resultdb-gtest-json-format'], + }, + 'webrtc_perf_tests': { + 'mixins': ['quick-perf-tests', 'resultdb-gtest-json-format'], + } + }, 'fuchsia_compatible_tests': { 'audio_decoder_unittests': {}, 'common_audio_unittests': {}, @@ -201,9 +195,6 @@ 'voip_unittests': {}, 'webrtc_nonparallel_tests': {}, }, - 'linux_desktop_specific_tests': { - 'shared_screencast_stream_test': {}, - }, 'more_configs_tests': { 'peerconnection_unittests': { 'swarming': { @@ -211,31 +202,23 @@ }, }, }, + 'perf_tests': { + 'video_codec_perf_tests': { + 'mixins': ['perf-video-codec-perf-tests'], + }, + 'webrtc_perf_tests': { + 'mixins': ['perf-webrtc-perf-tests'], + 'args': [ + '--test_artifacts_dir=${ISOLATED_OUTDIR}', + '--save_worst_frame', + ], + }, + }, 'video_capture_tests': { 'video_capture_tests': { 'mixins': ['baremetal-pool'], } }, - 'video_capture_tests_tryserver': { - 'video_capture_tests': { - 'mixins': ['baremetal-try-pool'], - } - }, - 'video_codec_perf_tests_tryserver': { - 'video_codec_perf_tests': { - 'mixins': ['quick-perf-tests', 'resultdb-gtest-json-format'], - } - }, - 'webrtc_perf_tests': { - 'webrtc_perf_tests': { - 'mixins': ['perf-webrtc-perf-tests'], - }, - }, - 'webrtc_perf_tests_tryserver': { - 'webrtc_perf_tests': { - 'mixins': ['quick-perf-tests', 'resultdb-gtest-json-format'], - } - }, }, ############################################################################## @@ -244,12 +227,11 @@ 'compound_suites': { 'android_tests_tryserver': [ 'android_tests', - 'android_webrtc_perf_tests_tryserver', + 'android_tests_tryserver_specific', ], 'desktop_tests_tryserver': [ 'desktop_tests', - 'video_capture_tests_tryserver', - 'webrtc_perf_tests_tryserver', + 'desktop_tests_try_server_specific', ], 'desktop_tests_with_video_capture': [ 'desktop_tests', @@ -257,19 +239,17 @@ ], 'linux_desktop_tests_tryserver': [ 'desktop_tests', - 'linux_desktop_specific_tests', - 'video_capture_tests_tryserver', - 'video_codec_perf_tests_tryserver', - 'webrtc_perf_tests_tryserver', + 'desktop_tests_linux_specific', + 'desktop_tests_try_server_specific', ], 'linux_desktop_tests_with_video_capture': [ 'desktop_tests', - 'linux_desktop_specific_tests', + 'desktop_tests_linux_specific', 'video_capture_tests', ], 'linux_tests': [ 'desktop_tests', - 'linux_desktop_specific_tests', + 'desktop_tests_linux_specific', ], }, diff --git a/infra/specs/tryserver.webrtc.json b/infra/specs/tryserver.webrtc.json index 34de221afb..2e1ca16619 100644 --- a/infra/specs/tryserver.webrtc.json +++ b/infra/specs/tryserver.webrtc.json @@ -12248,6 +12248,32 @@ }, "test_id_prefix": "ninja://modules/video_capture:video_capture_tests/" }, + { + "args": [ + "--force_fieldtrials=WebRTC-QuickPerfTest/Enabled/", + "--nologs", + "--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json" + ], + "isolate_name": "video_codec_perf_tests", + "merge": { + "script": "//testing/merge_scripts/standard_isolated_script_merge.py" + }, + "name": "video_codec_perf_tests", + "resultdb": { + "result_file": "${ISOLATED_OUTDIR}/gtest_output.json", + "result_format": "gtest_json" + }, + "swarming": { + "can_use_on_swarming_builders": true, + "dimension_sets": [ + { + "cpu": "x86-64", + "os": "Mac-12" + } + ] + }, + "test_id_prefix": "ninja://modules/video_coding:video_codec_perf_tests/" + }, { "isolate_name": "video_engine_tests", "merge": { @@ -14759,6 +14785,32 @@ }, "test_id_prefix": "ninja://modules/video_capture:video_capture_tests/" }, + { + "args": [ + "--force_fieldtrials=WebRTC-QuickPerfTest/Enabled/", + "--nologs", + "--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json" + ], + "isolate_name": "video_codec_perf_tests", + "merge": { + "script": "//testing/merge_scripts/standard_isolated_script_merge.py" + }, + "name": "video_codec_perf_tests", + "resultdb": { + "result_file": "${ISOLATED_OUTDIR}/gtest_output.json", + "result_format": "gtest_json" + }, + "swarming": { + "can_use_on_swarming_builders": true, + "dimension_sets": [ + { + "cpu": "x86-64", + "os": "Windows-10-19045" + } + ] + }, + "test_id_prefix": "ninja://modules/video_coding:video_codec_perf_tests/" + }, { "isolate_name": "video_engine_tests", "merge": { diff --git a/infra/specs/waterfalls.pyl b/infra/specs/waterfalls.pyl index 046a5b4935..e14b79883b 100644 --- a/infra/specs/waterfalls.pyl +++ b/infra/specs/waterfalls.pyl @@ -252,7 +252,7 @@ 'chromium-tester-service-account' ], 'test_suites': { - 'gtest_tests': 'android_perf_tests', + 'gtest_tests': 'perf_tests', }, }, 'Perf Android32 (R Pixel5)': { @@ -261,7 +261,7 @@ 'has_native_resultdb_integration', 'chromium-tester-service-account' ], 'test_suites': { - 'gtest_tests': 'android_perf_tests', + 'gtest_tests': 'perf_tests', }, }, 'Perf Android64 (O Pixel2)': { @@ -271,7 +271,7 @@ 'chromium-tester-service-account' ], 'test_suites': { - 'gtest_tests': 'android_perf_tests', + 'gtest_tests': 'perf_tests', }, }, 'Perf Android64 (R Pixel5)': { @@ -280,7 +280,7 @@ 'has_native_resultdb_integration', 'chromium-tester-service-account' ], 'test_suites': { - 'gtest_tests': 'android_perf_tests', + 'gtest_tests': 'perf_tests', }, }, 'Perf Fuchsia': { @@ -291,7 +291,7 @@ 'resultdb-gtest-json-format', 'fuchsia-gtest-output', 'perf-output' ], 'test_suites': { - 'isolated_scripts': 'webrtc_perf_tests', + 'isolated_scripts': 'perf_tests', } }, 'Perf Linux Bionic': { @@ -302,7 +302,7 @@ 'resultdb-gtest-json-format' ], 'test_suites': { - 'isolated_scripts': 'desktop_perf_tests', + 'isolated_scripts': 'perf_tests', }, }, 'Perf Mac 11': { @@ -313,7 +313,7 @@ 'resultdb-gtest-json-format' ], 'test_suites': { - 'isolated_scripts': 'desktop_perf_tests', + 'isolated_scripts': 'perf_tests', }, }, 'Perf Mac M1 Arm64 12': { @@ -324,7 +324,7 @@ 'resultdb-gtest-json-format' ], 'test_suites': { - 'isolated_scripts': 'webrtc_perf_tests', + 'isolated_scripts': 'perf_tests', }, }, 'Perf Win 10': { @@ -333,7 +333,7 @@ 'mixins': ['win10-any', 'perf-pool', 'timeout-3h', 'resultdb-gtest-json-format'], 'test_suites': { - 'isolated_scripts': 'desktop_perf_tests', + 'isolated_scripts': 'perf_tests', }, }, }, @@ -359,7 +359,7 @@ 'has_native_resultdb_integration', 'out_dir_arg' ], 'test_suites': { - 'isolated_scripts': 'webrtc_perf_tests', + 'isolated_scripts': 'perf_tests', }, 'args': [ '--write_perf_output_on_ios',