Run the same perf tests on all platforms.
Change-Id: I49ac9921d92e3ff81569b162c64c96559565ae62
Bug: b/284448060
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/307021
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Jeremy Leconte <jleconte@google.com>
Cr-Commit-Position: refs/heads/main@{#40204}
diff --git a/infra/specs/client.webrtc.perf.json b/infra/specs/client.webrtc.perf.json
index 32ec37f..3f55020 100644
--- a/infra/specs/client.webrtc.perf.json
+++ b/infra/specs/client.webrtc.perf.json
@@ -40,6 +40,8 @@
},
{
"args": [
+ "--test_artifacts_dir=${ISOLATED_OUTDIR}",
+ "--save_worst_frame",
"--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb",
"--nologs"
],
@@ -115,6 +117,8 @@
},
{
"args": [
+ "--test_artifacts_dir=${ISOLATED_OUTDIR}",
+ "--save_worst_frame",
"--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb",
"--nologs"
],
@@ -190,6 +194,8 @@
},
{
"args": [
+ "--test_artifacts_dir=${ISOLATED_OUTDIR}",
+ "--save_worst_frame",
"--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb",
"--nologs"
],
@@ -265,6 +271,8 @@
},
{
"args": [
+ "--test_artifacts_dir=${ISOLATED_OUTDIR}",
+ "--save_worst_frame",
"--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb",
"--nologs"
],
@@ -309,6 +317,46 @@
"--test-launcher-summary-output=${ISOLATED_OUTDIR}/gtest_output.json",
"--test-arg=--gtest_output=json:/custom_artifacts/gtest_output.json",
"--test-arg=--undefok=test_launcher_summary_output",
+ "--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb"
+ ],
+ "isolate_name": "video_codec_perf_tests",
+ "merge": {
+ "args": [
+ "--test-suite",
+ "video_codec_perf_tests"
+ ],
+ "script": "//tools_webrtc/perf/process_perf_results.py"
+ },
+ "name": "video_codec_perf_tests",
+ "resultdb": {
+ "result_file": "${ISOLATED_OUTDIR}/gtest_output.json",
+ "result_format": "gtest_json"
+ },
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "cpu": "x86-64",
+ "gce": "1",
+ "os": "Ubuntu-18.04",
+ "pool": "WebRTC-perf"
+ }
+ ],
+ "expiration": 10800,
+ "hard_timeout": 10800,
+ "idempotent": false,
+ "io_timeout": 10800
+ },
+ "test_id_prefix": "ninja://modules/video_coding:video_codec_perf_tests/"
+ },
+ {
+ "args": [
+ "--test_artifacts_dir=${ISOLATED_OUTDIR}",
+ "--save_worst_frame",
+ "--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json",
+ "--test-launcher-summary-output=${ISOLATED_OUTDIR}/gtest_output.json",
+ "--test-arg=--gtest_output=json:/custom_artifacts/gtest_output.json",
+ "--test-arg=--undefok=test_launcher_summary_output",
"--isolated-script-test-perf-output=${ISOLATED_OUTDIR}/perftest-output.pb",
"--nologs"
],
@@ -496,6 +544,42 @@
"isolated_scripts": [
{
"args": [
+ "--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json"
+ ],
+ "isolate_name": "video_codec_perf_tests",
+ "merge": {
+ "args": [
+ "--test-suite",
+ "video_codec_perf_tests"
+ ],
+ "script": "//tools_webrtc/perf/process_perf_results.py"
+ },
+ "name": "video_codec_perf_tests",
+ "resultdb": {
+ "result_file": "${ISOLATED_OUTDIR}/gtest_output.json",
+ "result_format": "gtest_json"
+ },
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "cpu": "arm64-64-Apple_M1",
+ "gce": "0",
+ "os": "Mac-12",
+ "pool": "WebRTC-perf"
+ }
+ ],
+ "expiration": 10800,
+ "hard_timeout": 10800,
+ "idempotent": false,
+ "io_timeout": 10800
+ },
+ "test_id_prefix": "ninja://modules/video_coding:video_codec_perf_tests/"
+ },
+ {
+ "args": [
+ "--test_artifacts_dir=${ISOLATED_OUTDIR}",
+ "--save_worst_frame",
"--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json",
"--nologs"
],
diff --git a/infra/specs/internal.client.webrtc.json b/infra/specs/internal.client.webrtc.json
index 9fcb922..0bb5b89 100644
--- a/infra/specs/internal.client.webrtc.json
+++ b/infra/specs/internal.client.webrtc.json
@@ -482,6 +482,59 @@
"--xcode-build-version",
"13c100",
"--out-dir",
+ "${ISOLATED_OUTDIR}"
+ ],
+ "isolate_name": "video_codec_perf_tests",
+ "merge": {
+ "args": [
+ "--test-suite",
+ "video_codec_perf_tests"
+ ],
+ "script": "//tools_webrtc/perf/process_perf_results.py"
+ },
+ "name": "video_codec_perf_tests",
+ "resultdb": {
+ "enable": true,
+ "has_native_resultdb_integration": true
+ },
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "cipd_packages": [
+ {
+ "cipd_package": "infra/tools/mac_toolchain/${platform}",
+ "location": ".",
+ "revision": "git_revision:a598cd77abc01089647a79e3cfa3cc8e8ecb5118"
+ }
+ ],
+ "dimension_sets": [
+ {
+ "id": "mac-254-e504",
+ "os": "iOS-12.4.1",
+ "pool": "WebRTC"
+ }
+ ],
+ "hard_timeout": 10800,
+ "idempotent": false,
+ "io_timeout": 10800,
+ "named_caches": [
+ {
+ "name": "xcode_ios_13c100",
+ "path": "Xcode.app"
+ }
+ ],
+ "service_account": "chrome-tester@chops-service-accounts.iam.gserviceaccount.com"
+ },
+ "test_id_prefix": "ninja://modules/video_coding:video_codec_perf_tests/"
+ },
+ {
+ "args": [
+ "--test_artifacts_dir=${ISOLATED_OUTDIR}",
+ "--save_worst_frame",
+ "--write_perf_output_on_ios",
+ "--xctest",
+ "--xcode-build-version",
+ "13c100",
+ "--out-dir",
"${ISOLATED_OUTDIR}",
"--nologs"
],
diff --git a/infra/specs/test_suites.pyl b/infra/specs/test_suites.pyl
index 703ac34..c0ee0a7 100644
--- a/infra/specs/test_suites.pyl
+++ b/infra/specs/test_suites.pyl
@@ -12,14 +12,6 @@
'android_examples_junit_tests': {},
'android_sdk_junit_tests': {},
},
- 'android_perf_tests': {
- 'video_codec_perf_tests': {
- 'mixins': ['perf-video-codec-perf-tests'],
- },
- 'webrtc_perf_tests': {
- 'mixins': ['perf-webrtc-perf-tests'],
- },
- },
'android_tests': {
'AppRTCMobile_test_apk': {},
'android_instrumentation_test_apk': {},
@@ -56,23 +48,11 @@
'voip_unittests': {},
'webrtc_nonparallel_tests': {},
},
- 'android_webrtc_perf_tests_tryserver': {
+ 'android_tests_tryserver_specific': {
'webrtc_perf_tests': {
'mixins': ['quick-perf-tests'],
}
},
- 'desktop_perf_tests': {
- 'video_codec_perf_tests': {
- 'mixins': ['perf-video-codec-perf-tests'],
- },
- 'webrtc_perf_tests': {
- 'mixins': ['perf-webrtc-perf-tests'],
- 'args': [
- '--test_artifacts_dir=${ISOLATED_OUTDIR}',
- '--save_worst_frame',
- ],
- },
- },
'desktop_tests': {
'audio_decoder_unittests': {},
'common_audio_unittests': {},
@@ -107,6 +87,20 @@
'voip_unittests': {},
'webrtc_nonparallel_tests': {},
},
+ 'desktop_tests_linux_specific': {
+ 'shared_screencast_stream_test': {},
+ },
+ 'desktop_tests_try_server_specific': {
+ 'video_capture_tests': {
+ 'mixins': ['baremetal-try-pool'],
+ },
+ 'video_codec_perf_tests': {
+ 'mixins': ['quick-perf-tests', 'resultdb-gtest-json-format'],
+ },
+ 'webrtc_perf_tests': {
+ 'mixins': ['quick-perf-tests', 'resultdb-gtest-json-format'],
+ }
+ },
'fuchsia_compatible_tests': {
'audio_decoder_unittests': {},
'common_audio_unittests': {},
@@ -201,9 +195,6 @@
'voip_unittests': {},
'webrtc_nonparallel_tests': {},
},
- 'linux_desktop_specific_tests': {
- 'shared_screencast_stream_test': {},
- },
'more_configs_tests': {
'peerconnection_unittests': {
'swarming': {
@@ -211,31 +202,23 @@
},
},
},
+ 'perf_tests': {
+ 'video_codec_perf_tests': {
+ 'mixins': ['perf-video-codec-perf-tests'],
+ },
+ 'webrtc_perf_tests': {
+ 'mixins': ['perf-webrtc-perf-tests'],
+ 'args': [
+ '--test_artifacts_dir=${ISOLATED_OUTDIR}',
+ '--save_worst_frame',
+ ],
+ },
+ },
'video_capture_tests': {
'video_capture_tests': {
'mixins': ['baremetal-pool'],
}
},
- 'video_capture_tests_tryserver': {
- 'video_capture_tests': {
- 'mixins': ['baremetal-try-pool'],
- }
- },
- 'video_codec_perf_tests_tryserver': {
- 'video_codec_perf_tests': {
- 'mixins': ['quick-perf-tests', 'resultdb-gtest-json-format'],
- }
- },
- 'webrtc_perf_tests': {
- 'webrtc_perf_tests': {
- 'mixins': ['perf-webrtc-perf-tests'],
- },
- },
- 'webrtc_perf_tests_tryserver': {
- 'webrtc_perf_tests': {
- 'mixins': ['quick-perf-tests', 'resultdb-gtest-json-format'],
- }
- },
},
##############################################################################
@@ -244,12 +227,11 @@
'compound_suites': {
'android_tests_tryserver': [
'android_tests',
- 'android_webrtc_perf_tests_tryserver',
+ 'android_tests_tryserver_specific',
],
'desktop_tests_tryserver': [
'desktop_tests',
- 'video_capture_tests_tryserver',
- 'webrtc_perf_tests_tryserver',
+ 'desktop_tests_try_server_specific',
],
'desktop_tests_with_video_capture': [
'desktop_tests',
@@ -257,19 +239,17 @@
],
'linux_desktop_tests_tryserver': [
'desktop_tests',
- 'linux_desktop_specific_tests',
- 'video_capture_tests_tryserver',
- 'video_codec_perf_tests_tryserver',
- 'webrtc_perf_tests_tryserver',
+ 'desktop_tests_linux_specific',
+ 'desktop_tests_try_server_specific',
],
'linux_desktop_tests_with_video_capture': [
'desktop_tests',
- 'linux_desktop_specific_tests',
+ 'desktop_tests_linux_specific',
'video_capture_tests',
],
'linux_tests': [
'desktop_tests',
- 'linux_desktop_specific_tests',
+ 'desktop_tests_linux_specific',
],
},
diff --git a/infra/specs/tryserver.webrtc.json b/infra/specs/tryserver.webrtc.json
index 34de221..2e1ca16 100644
--- a/infra/specs/tryserver.webrtc.json
+++ b/infra/specs/tryserver.webrtc.json
@@ -12249,6 +12249,32 @@
"test_id_prefix": "ninja://modules/video_capture:video_capture_tests/"
},
{
+ "args": [
+ "--force_fieldtrials=WebRTC-QuickPerfTest/Enabled/",
+ "--nologs",
+ "--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json"
+ ],
+ "isolate_name": "video_codec_perf_tests",
+ "merge": {
+ "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
+ },
+ "name": "video_codec_perf_tests",
+ "resultdb": {
+ "result_file": "${ISOLATED_OUTDIR}/gtest_output.json",
+ "result_format": "gtest_json"
+ },
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "cpu": "x86-64",
+ "os": "Mac-12"
+ }
+ ]
+ },
+ "test_id_prefix": "ninja://modules/video_coding:video_codec_perf_tests/"
+ },
+ {
"isolate_name": "video_engine_tests",
"merge": {
"script": "//testing/merge_scripts/standard_isolated_script_merge.py"
@@ -14760,6 +14786,32 @@
"test_id_prefix": "ninja://modules/video_capture:video_capture_tests/"
},
{
+ "args": [
+ "--force_fieldtrials=WebRTC-QuickPerfTest/Enabled/",
+ "--nologs",
+ "--gtest_output=json:${ISOLATED_OUTDIR}/gtest_output.json"
+ ],
+ "isolate_name": "video_codec_perf_tests",
+ "merge": {
+ "script": "//testing/merge_scripts/standard_isolated_script_merge.py"
+ },
+ "name": "video_codec_perf_tests",
+ "resultdb": {
+ "result_file": "${ISOLATED_OUTDIR}/gtest_output.json",
+ "result_format": "gtest_json"
+ },
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "cpu": "x86-64",
+ "os": "Windows-10-19045"
+ }
+ ]
+ },
+ "test_id_prefix": "ninja://modules/video_coding:video_codec_perf_tests/"
+ },
+ {
"isolate_name": "video_engine_tests",
"merge": {
"script": "//testing/merge_scripts/standard_isolated_script_merge.py"
diff --git a/infra/specs/waterfalls.pyl b/infra/specs/waterfalls.pyl
index 046a5b4..e14b798 100644
--- a/infra/specs/waterfalls.pyl
+++ b/infra/specs/waterfalls.pyl
@@ -252,7 +252,7 @@
'chromium-tester-service-account'
],
'test_suites': {
- 'gtest_tests': 'android_perf_tests',
+ 'gtest_tests': 'perf_tests',
},
},
'Perf Android32 (R Pixel5)': {
@@ -261,7 +261,7 @@
'has_native_resultdb_integration', 'chromium-tester-service-account'
],
'test_suites': {
- 'gtest_tests': 'android_perf_tests',
+ 'gtest_tests': 'perf_tests',
},
},
'Perf Android64 (O Pixel2)': {
@@ -271,7 +271,7 @@
'chromium-tester-service-account'
],
'test_suites': {
- 'gtest_tests': 'android_perf_tests',
+ 'gtest_tests': 'perf_tests',
},
},
'Perf Android64 (R Pixel5)': {
@@ -280,7 +280,7 @@
'has_native_resultdb_integration', 'chromium-tester-service-account'
],
'test_suites': {
- 'gtest_tests': 'android_perf_tests',
+ 'gtest_tests': 'perf_tests',
},
},
'Perf Fuchsia': {
@@ -291,7 +291,7 @@
'resultdb-gtest-json-format', 'fuchsia-gtest-output', 'perf-output'
],
'test_suites': {
- 'isolated_scripts': 'webrtc_perf_tests',
+ 'isolated_scripts': 'perf_tests',
}
},
'Perf Linux Bionic': {
@@ -302,7 +302,7 @@
'resultdb-gtest-json-format'
],
'test_suites': {
- 'isolated_scripts': 'desktop_perf_tests',
+ 'isolated_scripts': 'perf_tests',
},
},
'Perf Mac 11': {
@@ -313,7 +313,7 @@
'resultdb-gtest-json-format'
],
'test_suites': {
- 'isolated_scripts': 'desktop_perf_tests',
+ 'isolated_scripts': 'perf_tests',
},
},
'Perf Mac M1 Arm64 12': {
@@ -324,7 +324,7 @@
'resultdb-gtest-json-format'
],
'test_suites': {
- 'isolated_scripts': 'webrtc_perf_tests',
+ 'isolated_scripts': 'perf_tests',
},
},
'Perf Win 10': {
@@ -333,7 +333,7 @@
'mixins':
['win10-any', 'perf-pool', 'timeout-3h', 'resultdb-gtest-json-format'],
'test_suites': {
- 'isolated_scripts': 'desktop_perf_tests',
+ 'isolated_scripts': 'perf_tests',
},
},
},
@@ -359,7 +359,7 @@
'has_native_resultdb_integration', 'out_dir_arg'
],
'test_suites': {
- 'isolated_scripts': 'webrtc_perf_tests',
+ 'isolated_scripts': 'perf_tests',
},
'args': [
'--write_perf_output_on_ios',
