From b15a9f01fc76d4798a5aaca8be94fcfe03188ce1 Mon Sep 17 00:00:00 2001 From: Jeremy Leconte Date: Fri, 2 Jun 2023 14:14:17 +0200 Subject: [PATCH] Fix perf tests. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turns out it's not possible to run the same perf tests on all os: --save_worst_frame doesn't work on Fuchsia bots: https://chromium-swarm.appspot.com/task?id=6294b51713605710 video_codec perf tests fail on iOS: https://chrome-swarming.appspot.com/task?id=629455defc12bc10 Change-Id: I5cd490f5728b6046615ba217c65fd2b125a9e4a7 Bug: None Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/307480 Reviewed-by: Björn Terelius Commit-Queue: Jeremy Leconte Cr-Commit-Position: refs/heads/main@{#40209} --- infra/specs/client.webrtc.perf.json | 10 ----- infra/specs/internal.client.webrtc.json | 53 ------------------------- infra/specs/test_suites.pyl | 13 ++++++ infra/specs/waterfalls.pyl | 10 ++--- 4 files changed, 18 insertions(+), 68 deletions(-) diff --git a/infra/specs/client.webrtc.perf.json b/infra/specs/client.webrtc.perf.json index 3f550204ca..9194fcf946 100644 --- a/infra/specs/client.webrtc.perf.json +++ b/infra/specs/client.webrtc.perf.json @@ -40,8 +40,6 @@ }, { "args": [ - "--test_artifacts_dir=${ISOLATED_OUTDIR}", - "--save_worst_frame", "--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb", "--nologs" ], @@ -117,8 +115,6 @@ }, { "args": [ - "--test_artifacts_dir=${ISOLATED_OUTDIR}", - "--save_worst_frame", "--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb", "--nologs" ], @@ -194,8 +190,6 @@ }, { "args": [ - "--test_artifacts_dir=${ISOLATED_OUTDIR}", - "--save_worst_frame", "--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb", "--nologs" ], @@ -271,8 +265,6 @@ }, { "args": [ - "--test_artifacts_dir=${ISOLATED_OUTDIR}", - "--save_worst_frame", "--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb", "--nologs" ], @@ -351,8 +343,6 @@ }, { "args": [ - "--test_artifacts_dir=${ISOLATED_OUTDIR}", - "--save_worst_frame", "--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json", "--test-launcher-summary-output=${ISOLATED_OUTDIR}/gtest_output.json", "--test-arg=--gtest_output=json:/custom_artifacts/gtest_output.json", diff --git a/infra/specs/internal.client.webrtc.json b/infra/specs/internal.client.webrtc.json index 0bb5b89028..9fcb9225e7 100644 --- a/infra/specs/internal.client.webrtc.json +++ b/infra/specs/internal.client.webrtc.json @@ -477,59 +477,6 @@ "isolated_scripts": [ { "args": [ - "--write_perf_output_on_ios", - "--xctest", - "--xcode-build-version", - "13c100", - "--out-dir", - "${ISOLATED_OUTDIR}" - ], - "isolate_name": "video_codec_perf_tests", - "merge": { - "args": [ - "--test-suite", - "video_codec_perf_tests" - ], - "script": "//tools_webrtc/perf/process_perf_results.py" - }, - "name": "video_codec_perf_tests", - "resultdb": { - "enable": true, - "has_native_resultdb_integration": true - }, - "swarming": { - "can_use_on_swarming_builders": true, - "cipd_packages": [ - { - "cipd_package": "infra/tools/mac_toolchain/${platform}", - "location": ".", - "revision": "git_revision:a598cd77abc01089647a79e3cfa3cc8e8ecb5118" - } - ], - "dimension_sets": [ - { - "id": "mac-254-e504", - "os": "iOS-12.4.1", - "pool": "WebRTC" - } - ], - "hard_timeout": 10800, - "idempotent": false, - "io_timeout": 10800, - "named_caches": [ - { - "name": "xcode_ios_13c100", - "path": "Xcode.app" - } - ], - "service_account": "chrome-tester@chops-service-accounts.iam.gserviceaccount.com" - }, - "test_id_prefix": "ninja://modules/video_coding:video_codec_perf_tests/" - }, - { - "args": [ - "--test_artifacts_dir=${ISOLATED_OUTDIR}", - "--save_worst_frame", "--write_perf_output_on_ios", "--xctest", "--xcode-build-version", diff --git a/infra/specs/test_suites.pyl b/infra/specs/test_suites.pyl index c0ee0a79e8..2e440a67d9 100644 --- a/infra/specs/test_suites.pyl +++ b/infra/specs/test_suites.pyl @@ -203,6 +203,19 @@ }, }, 'perf_tests': { + 'video_codec_perf_tests': { + 'mixins': ['perf-video-codec-perf-tests'], + }, + 'webrtc_perf_tests': { + 'mixins': ['perf-webrtc-perf-tests'], + }, + }, + 'perf_tests_no_video_codec': { + 'webrtc_perf_tests': { + 'mixins': ['perf-webrtc-perf-tests'], + }, + }, + 'perf_tests_save_worst_frame': { 'video_codec_perf_tests': { 'mixins': ['perf-video-codec-perf-tests'], }, diff --git a/infra/specs/waterfalls.pyl b/infra/specs/waterfalls.pyl index e14b79883b..d66da0fdfa 100644 --- a/infra/specs/waterfalls.pyl +++ b/infra/specs/waterfalls.pyl @@ -302,7 +302,7 @@ 'resultdb-gtest-json-format' ], 'test_suites': { - 'isolated_scripts': 'perf_tests', + 'isolated_scripts': 'perf_tests_save_worst_frame', }, }, 'Perf Mac 11': { @@ -313,7 +313,7 @@ 'resultdb-gtest-json-format' ], 'test_suites': { - 'isolated_scripts': 'perf_tests', + 'isolated_scripts': 'perf_tests_save_worst_frame', }, }, 'Perf Mac M1 Arm64 12': { @@ -324,7 +324,7 @@ 'resultdb-gtest-json-format' ], 'test_suites': { - 'isolated_scripts': 'perf_tests', + 'isolated_scripts': 'perf_tests_save_worst_frame', }, }, 'Perf Win 10': { @@ -333,7 +333,7 @@ 'mixins': ['win10-any', 'perf-pool', 'timeout-3h', 'resultdb-gtest-json-format'], 'test_suites': { - 'isolated_scripts': 'perf_tests', + 'isolated_scripts': 'perf_tests_save_worst_frame', }, }, }, @@ -359,7 +359,7 @@ 'has_native_resultdb_integration', 'out_dir_arg' ], 'test_suites': { - 'isolated_scripts': 'perf_tests', + 'isolated_scripts': 'perf_tests_no_video_codec', }, 'args': [ '--write_perf_output_on_ios',