Fix perf tests.
Turns out it's not possible to run the same perf tests on all os:
--save_worst_frame doesn't work on Fuchsia bots:
https://chromium-swarm.appspot.com/task?id=6294b51713605710
video_codec perf tests fail on iOS:
https://chrome-swarming.appspot.com/task?id=629455defc12bc10
Change-Id: I5cd490f5728b6046615ba217c65fd2b125a9e4a7
Bug: None
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/307480
Reviewed-by: Björn Terelius <terelius@webrtc.org>
Commit-Queue: Jeremy Leconte <jleconte@google.com>
Cr-Commit-Position: refs/heads/main@{#40209}
diff --git a/infra/specs/client.webrtc.perf.json b/infra/specs/client.webrtc.perf.json
index 3f55020..9194fcf 100644
--- a/infra/specs/client.webrtc.perf.json
+++ b/infra/specs/client.webrtc.perf.json
@@ -40,8 +40,6 @@
},
{
"args": [
- "--test_artifacts_dir=${ISOLATED_OUTDIR}",
- "--save_worst_frame",
"--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb",
"--nologs"
],
@@ -117,8 +115,6 @@
},
{
"args": [
- "--test_artifacts_dir=${ISOLATED_OUTDIR}",
- "--save_worst_frame",
"--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb",
"--nologs"
],
@@ -194,8 +190,6 @@
},
{
"args": [
- "--test_artifacts_dir=${ISOLATED_OUTDIR}",
- "--save_worst_frame",
"--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb",
"--nologs"
],
@@ -271,8 +265,6 @@
},
{
"args": [
- "--test_artifacts_dir=${ISOLATED_OUTDIR}",
- "--save_worst_frame",
"--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb",
"--nologs"
],
@@ -351,8 +343,6 @@
},
{
"args": [
- "--test_artifacts_dir=${ISOLATED_OUTDIR}",
- "--save_worst_frame",
"--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json",
"--test-launcher-summary-output=${ISOLATED_OUTDIR}/gtest_output.json",
"--test-arg=--gtest_output=json:/custom_artifacts/gtest_output.json",
diff --git a/infra/specs/internal.client.webrtc.json b/infra/specs/internal.client.webrtc.json
index 0bb5b89..9fcb922 100644
--- a/infra/specs/internal.client.webrtc.json
+++ b/infra/specs/internal.client.webrtc.json
@@ -482,59 +482,6 @@
"--xcode-build-version",
"13c100",
"--out-dir",
- "${ISOLATED_OUTDIR}"
- ],
- "isolate_name": "video_codec_perf_tests",
- "merge": {
- "args": [
- "--test-suite",
- "video_codec_perf_tests"
- ],
- "script": "//tools_webrtc/perf/process_perf_results.py"
- },
- "name": "video_codec_perf_tests",
- "resultdb": {
- "enable": true,
- "has_native_resultdb_integration": true
- },
- "swarming": {
- "can_use_on_swarming_builders": true,
- "cipd_packages": [
- {
- "cipd_package": "infra/tools/mac_toolchain/${platform}",
- "location": ".",
- "revision": "git_revision:a598cd77abc01089647a79e3cfa3cc8e8ecb5118"
- }
- ],
- "dimension_sets": [
- {
- "id": "mac-254-e504",
- "os": "iOS-12.4.1",
- "pool": "WebRTC"
- }
- ],
- "hard_timeout": 10800,
- "idempotent": false,
- "io_timeout": 10800,
- "named_caches": [
- {
- "name": "xcode_ios_13c100",
- "path": "Xcode.app"
- }
- ],
- "service_account": "chrome-tester@chops-service-accounts.iam.gserviceaccount.com"
- },
- "test_id_prefix": "ninja://modules/video_coding:video_codec_perf_tests/"
- },
- {
- "args": [
- "--test_artifacts_dir=${ISOLATED_OUTDIR}",
- "--save_worst_frame",
- "--write_perf_output_on_ios",
- "--xctest",
- "--xcode-build-version",
- "13c100",
- "--out-dir",
"${ISOLATED_OUTDIR}",
"--nologs"
],
diff --git a/infra/specs/test_suites.pyl b/infra/specs/test_suites.pyl
index c0ee0a7..2e440a6 100644
--- a/infra/specs/test_suites.pyl
+++ b/infra/specs/test_suites.pyl
@@ -208,6 +208,19 @@
},
'webrtc_perf_tests': {
'mixins': ['perf-webrtc-perf-tests'],
+ },
+ },
+ 'perf_tests_no_video_codec': {
+ 'webrtc_perf_tests': {
+ 'mixins': ['perf-webrtc-perf-tests'],
+ },
+ },
+ 'perf_tests_save_worst_frame': {
+ 'video_codec_perf_tests': {
+ 'mixins': ['perf-video-codec-perf-tests'],
+ },
+ 'webrtc_perf_tests': {
+ 'mixins': ['perf-webrtc-perf-tests'],
'args': [
'--test_artifacts_dir=${ISOLATED_OUTDIR}',
'--save_worst_frame',
diff --git a/infra/specs/waterfalls.pyl b/infra/specs/waterfalls.pyl
index e14b798..d66da0f 100644
--- a/infra/specs/waterfalls.pyl
+++ b/infra/specs/waterfalls.pyl
@@ -302,7 +302,7 @@
'resultdb-gtest-json-format'
],
'test_suites': {
- 'isolated_scripts': 'perf_tests',
+ 'isolated_scripts': 'perf_tests_save_worst_frame',
},
},
'Perf Mac 11': {
@@ -313,7 +313,7 @@
'resultdb-gtest-json-format'
],
'test_suites': {
- 'isolated_scripts': 'perf_tests',
+ 'isolated_scripts': 'perf_tests_save_worst_frame',
},
},
'Perf Mac M1 Arm64 12': {
@@ -324,7 +324,7 @@
'resultdb-gtest-json-format'
],
'test_suites': {
- 'isolated_scripts': 'perf_tests',
+ 'isolated_scripts': 'perf_tests_save_worst_frame',
},
},
'Perf Win 10': {
@@ -333,7 +333,7 @@
'mixins':
['win10-any', 'perf-pool', 'timeout-3h', 'resultdb-gtest-json-format'],
'test_suites': {
- 'isolated_scripts': 'perf_tests',
+ 'isolated_scripts': 'perf_tests_save_worst_frame',
},
},
},
@@ -359,7 +359,7 @@
'has_native_resultdb_integration', 'out_dir_arg'
],
'test_suites': {
- 'isolated_scripts': 'perf_tests',
+ 'isolated_scripts': 'perf_tests_no_video_codec',
},
'args': [
'--write_perf_output_on_ios',
