mandermo | ed582f7 | 2017-01-23 15:55:42 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. |
| 3 | # |
| 4 | # Use of this source code is governed by a BSD-style license |
| 5 | # that can be found in the LICENSE file in the root of the source |
| 6 | # tree. An additional intellectual property rights grant can be found |
| 7 | # in the file PATENTS. All contributing project authors may |
| 8 | # be found in the AUTHORS file in the root of the source tree. |
| 9 | |
| 10 | """ |
| 11 | This script is the wrapper that starts a loopback call with stubbed video in |
| 12 | and out. It then analyses the video quality of the output video against the |
| 13 | reference input video. |
| 14 | |
| 15 | It expect to be given the webrtc output build directory as the first argument |
| 16 | all other arguments are optional. |
| 17 | |
| 18 | It assumes you have a Android device plugged in. |
| 19 | """ |
| 20 | |
| 21 | import argparse |
oprypin | bed7a6b | 2017-06-19 08:16:45 | [diff] [blame] | 22 | import json |
mandermo | ed582f7 | 2017-01-23 15:55:42 | [diff] [blame] | 23 | import logging |
| 24 | import os |
mandermo | ed582f7 | 2017-01-23 15:55:42 | [diff] [blame] | 25 | import subprocess |
| 26 | import sys |
| 27 | import tempfile |
oprypin | 30cda5e | 2017-04-24 11:15:13 | [diff] [blame] | 28 | import time |
mandermo | ed582f7 | 2017-01-23 15:55:42 | [diff] [blame] | 29 | |
| 30 | |
| 31 | SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) |
Henrik Kjellander | cb3b1c1 | 2017-09-18 04:20:33 | [diff] [blame] | 32 | SRC_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, os.pardir, os.pardir)) |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 33 | RTC_TOOLS_DIR = os.path.join(SRC_DIR, 'rtc_tools', 'testing') |
| 34 | TOOLCHAIN_DIR = os.path.join(SRC_DIR, 'tools_webrtc', 'video_quality_toolchain', |
| 35 | 'linux') |
oprypin | bed7a6b | 2017-06-19 08:16:45 | [diff] [blame] | 36 | BAD_DEVICES_JSON = os.path.join(SRC_DIR, |
| 37 | os.environ.get('CHROMIUM_OUT_DIR', 'out'), |
| 38 | 'bad_devices.json') |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 39 | |
| 40 | sys.path.append(RTC_TOOLS_DIR) |
| 41 | import utils |
oprypin | 30cda5e | 2017-04-24 11:15:13 | [diff] [blame] | 42 | |
| 43 | |
| 44 | class Error(Exception): |
| 45 | pass |
| 46 | |
| 47 | |
| 48 | class VideoQualityTestError(Error): |
| 49 | pass |
mandermo | ed582f7 | 2017-01-23 15:55:42 | [diff] [blame] | 50 | |
| 51 | |
Edward Lemur | d5e17d6 | 2018-01-31 16:28:28 | [diff] [blame] | 52 | def _RunCommand(argv, **kwargs): |
mandermo | ed582f7 | 2017-01-23 15:55:42 | [diff] [blame] | 53 | logging.info('Running %r', argv) |
Edward Lemur | d5e17d6 | 2018-01-31 16:28:28 | [diff] [blame] | 54 | subprocess.check_call(argv, **kwargs) |
mandermo | ed582f7 | 2017-01-23 15:55:42 | [diff] [blame] | 55 | |
| 56 | |
Edward Lemur | d5e17d6 | 2018-01-31 16:28:28 | [diff] [blame] | 57 | def _RunCommandWithOutput(argv, **kwargs): |
oprypin | 30cda5e | 2017-04-24 11:15:13 | [diff] [blame] | 58 | logging.info('Running %r', argv) |
Edward Lemur | d5e17d6 | 2018-01-31 16:28:28 | [diff] [blame] | 59 | return subprocess.check_output(argv, **kwargs) |
oprypin | 30cda5e | 2017-04-24 11:15:13 | [diff] [blame] | 60 | |
| 61 | |
Edward Lemur | d5e17d6 | 2018-01-31 16:28:28 | [diff] [blame] | 62 | def _RunBackgroundCommand(argv): |
oprypin | 30cda5e | 2017-04-24 11:15:13 | [diff] [blame] | 63 | logging.info('Running %r', argv) |
Edward Lemur | d5e17d6 | 2018-01-31 16:28:28 | [diff] [blame] | 64 | process = subprocess.Popen(argv) |
oprypin | 30cda5e | 2017-04-24 11:15:13 | [diff] [blame] | 65 | time.sleep(0.5) |
| 66 | status = process.poll() |
| 67 | if status: # is not None or 0 |
| 68 | raise subprocess.CalledProcessError(status, argv) |
| 69 | return process |
| 70 | |
| 71 | |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 72 | def CreateEmptyDir(suggested_dir): |
| 73 | if not suggested_dir: |
| 74 | return tempfile.mkdtemp() |
| 75 | utils.RemoveDirectory(suggested_dir) |
| 76 | os.makedirs(suggested_dir) |
| 77 | return suggested_dir |
| 78 | |
| 79 | |
mandermo | ed582f7 | 2017-01-23 15:55:42 | [diff] [blame] | 80 | def _ParseArgs(): |
| 81 | parser = argparse.ArgumentParser(description='Start loopback video analysis.') |
mandermo | ed582f7 | 2017-01-23 15:55:42 | [diff] [blame] | 82 | parser.add_argument('build_dir_android', |
| 83 | help='The path to the build directory for Android.') |
mandermo | ed582f7 | 2017-01-23 15:55:42 | [diff] [blame] | 84 | parser.add_argument('--temp_dir', |
| 85 | help='A temporary directory to put the output.') |
oprypin | 30cda5e | 2017-04-24 11:15:13 | [diff] [blame] | 86 | parser.add_argument('--adb-path', help='Path to adb binary.', default='adb') |
Edward Lesmes | 9599fd4 | 2018-03-12 20:43:05 | [diff] [blame] | 87 | parser.add_argument('--num-retries', default='0', |
Edward Lesmes | 5b9c684 | 2018-03-09 18:07:22 | [diff] [blame] | 88 | help='Number of times to retry the test on Android.') |
Edward Lemur | 71d766e | 2018-02-05 20:59:36 | [diff] [blame] | 89 | parser.add_argument('--isolated-script-test-perf-output', |
Edward Lemur | 2e5966b | 2018-01-30 14:33:02 | [diff] [blame] | 90 | help='Where to store perf results in chartjson format.', default=None) |
Oleh Prypin | 637b0b5 | 2018-09-21 15:16:06 | [diff] [blame] | 91 | parser.add_argument('--isolated-script-test-output', default=None, |
| 92 | help='Path to output an empty JSON file which Chromium infra requires.') |
Edward Lemur | 21a35bc | 2018-01-31 12:46:50 | [diff] [blame] | 93 | args, unknown_args = parser.parse_known_args() |
| 94 | |
| 95 | # Ignore Chromium-specific flags |
| 96 | parser = argparse.ArgumentParser() |
Edward Lemur | 21a35bc | 2018-01-31 12:46:50 | [diff] [blame] | 97 | parser.add_argument('--test-launcher-summary-output', |
| 98 | type=str, default=None) |
| 99 | |
| 100 | parser.parse_args(unknown_args) |
| 101 | |
mandermo | ed582f7 | 2017-01-23 15:55:42 | [diff] [blame] | 102 | return args |
| 103 | |
| 104 | |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 105 | def SelectAndroidDevice(adb_path): |
| 106 | # Select an Android device in case multiple are connected. |
oprypin | bed7a6b | 2017-06-19 08:16:45 | [diff] [blame] | 107 | try: |
| 108 | with open(BAD_DEVICES_JSON) as bad_devices_file: |
| 109 | bad_devices = json.load(bad_devices_file) |
| 110 | except IOError: |
| 111 | if os.environ.get('CHROME_HEADLESS'): |
| 112 | logging.warning('Cannot read %r', BAD_DEVICES_JSON) |
| 113 | bad_devices = {} |
| 114 | |
oprypin | 30cda5e | 2017-04-24 11:15:13 | [diff] [blame] | 115 | for line in _RunCommandWithOutput([adb_path, 'devices']).splitlines(): |
| 116 | if line.endswith('\tdevice'): |
| 117 | android_device = line.split('\t')[0] |
oprypin | bed7a6b | 2017-06-19 08:16:45 | [diff] [blame] | 118 | if android_device not in bad_devices: |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 119 | return android_device |
| 120 | raise VideoQualityTestError('Cannot find any connected Android device.') |
| 121 | |
| 122 | |
| 123 | def SetUpTools(android_device, temp_dir, processes): |
| 124 | # Extract AppRTC. |
| 125 | apprtc_archive = os.path.join(RTC_TOOLS_DIR, 'prebuilt_apprtc.zip') |
| 126 | golang_archive = os.path.join(RTC_TOOLS_DIR, 'golang', 'linux', 'go.tar.gz') |
| 127 | |
| 128 | utils.UnpackArchiveTo(apprtc_archive, temp_dir) |
| 129 | utils.UnpackArchiveTo(golang_archive, temp_dir) |
| 130 | |
| 131 | # Build AppRTC. |
| 132 | build_apprtc_script = os.path.join(RTC_TOOLS_DIR, 'build_apprtc.py') |
Oleh Prypin | 1e90845 | 2018-04-04 12:31:58 | [diff] [blame] | 133 | apprtc_dir = os.path.join(temp_dir, 'apprtc') |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 134 | go_dir = os.path.join(temp_dir, 'go') |
| 135 | collider_dir = os.path.join(temp_dir, 'collider') |
| 136 | |
Oleh Prypin | 1e90845 | 2018-04-04 12:31:58 | [diff] [blame] | 137 | _RunCommand([sys.executable, build_apprtc_script, apprtc_dir, go_dir, |
| 138 | collider_dir]) |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 139 | |
| 140 | # Start AppRTC Server. |
| 141 | dev_appserver = os.path.join(temp_dir, 'apprtc', 'temp', 'google-cloud-sdk', |
| 142 | 'bin', 'dev_appserver.py') |
| 143 | appengine_dir = os.path.join(temp_dir, 'apprtc', 'out', 'app_engine') |
| 144 | processes.append(_RunBackgroundCommand([ |
| 145 | sys.executable, dev_appserver, appengine_dir, '--port=9999', |
| 146 | '--admin_port=9998', '--skip_sdk_update_check', '--clear_datastore=yes'])) |
| 147 | |
| 148 | # Start Collider. |
| 149 | collider_path = os.path.join(temp_dir, 'collider', 'collidermain') |
| 150 | processes.append(_RunBackgroundCommand([ |
| 151 | collider_path, '-tls=false', '-port=8089', |
| 152 | '-room-server=http://localhost:9999'])) |
| 153 | |
| 154 | # Start adb reverse forwarder. |
| 155 | reverseforwarder_path = os.path.join( |
| 156 | SRC_DIR, 'build', 'android', 'adb_reverse_forwarder.py') |
| 157 | processes.append(_RunBackgroundCommand([ |
| 158 | reverseforwarder_path, '--device', android_device, '9999', '9999', '8089', |
| 159 | '8089'])) |
| 160 | |
| 161 | |
Edward Lesmes | 5b9c684 | 2018-03-09 18:07:22 | [diff] [blame] | 162 | def RunTest(android_device, adb_path, build_dir, temp_dir, num_retries, |
Edward Lemur | 2e5966b | 2018-01-30 14:33:02 | [diff] [blame] | 163 | chartjson_result_file): |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 164 | ffmpeg_path = os.path.join(TOOLCHAIN_DIR, 'ffmpeg') |
Sami Kalliomäki | 0673bc9 | 2018-08-27 15:58:13 | [diff] [blame] | 165 | def ConvertVideo(input_video, output_video): |
| 166 | _RunCommand([ffmpeg_path, '-y', '-i', input_video, output_video]) |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 167 | |
| 168 | # Start loopback call and record video. |
| 169 | test_script = os.path.join( |
Sami Kalliomäki | d54f5f5 | 2018-08-03 11:23:05 | [diff] [blame] | 170 | build_dir, 'bin', 'run_AppRTCMobile_stubbed_video_io_test_apk') |
Edward Lesmes | 5b9c684 | 2018-03-09 18:07:22 | [diff] [blame] | 171 | _RunCommand([test_script, '--device', android_device, |
| 172 | '--num-retries', num_retries]) |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 173 | |
| 174 | # Pull the recorded video. |
| 175 | test_video = os.path.join(temp_dir, 'test_video.y4m') |
| 176 | _RunCommand([adb_path, '-s', android_device, |
| 177 | 'pull', '/sdcard/output.y4m', test_video]) |
| 178 | |
| 179 | # Convert the recorded and reference videos to YUV. |
| 180 | reference_video = os.path.join(SRC_DIR, |
| 181 | 'resources', 'reference_video_640x360_30fps.y4m') |
| 182 | |
Sami Kalliomäki | 0673bc9 | 2018-08-27 15:58:13 | [diff] [blame] | 183 | test_video_yuv = os.path.join(temp_dir, 'test_video.yuv') |
| 184 | reference_video_yuv = os.path.join( |
| 185 | temp_dir, 'reference_video_640x360_30fps.yuv') |
| 186 | |
| 187 | ConvertVideo(test_video, test_video_yuv) |
| 188 | ConvertVideo(reference_video, reference_video_yuv) |
| 189 | |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 190 | # Run comparison script. |
| 191 | compare_script = os.path.join(SRC_DIR, 'rtc_tools', 'compare_videos.py') |
Mirko Bonadei | b56706f | 2018-09-18 09:03:39 | [diff] [blame] | 192 | frame_analyzer = os.path.join(build_dir, 'frame_analyzer_host') |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 193 | |
Edward Lemur | 2e5966b | 2018-01-30 14:33:02 | [diff] [blame] | 194 | args = [ |
Sami Kalliomäki | 0673bc9 | 2018-08-27 15:58:13 | [diff] [blame] | 195 | '--ref_video', reference_video_yuv, |
| 196 | '--test_video', test_video_yuv, |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 197 | '--yuv_frame_width', '640', |
| 198 | '--yuv_frame_height', '360', |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 199 | '--frame_analyzer', frame_analyzer, |
Edward Lemur | 2e5966b | 2018-01-30 14:33:02 | [diff] [blame] | 200 | ] |
| 201 | if chartjson_result_file: |
| 202 | args.extend(['--chartjson_result_file', chartjson_result_file]) |
| 203 | |
| 204 | _RunCommand([sys.executable, compare_script] + args) |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 205 | |
| 206 | |
| 207 | def main(): |
| 208 | logging.basicConfig(level=logging.INFO) |
| 209 | |
| 210 | args = _ParseArgs() |
| 211 | |
| 212 | temp_dir = args.temp_dir |
| 213 | build_dir = args.build_dir_android |
| 214 | adb_path = args.adb_path |
oprypin | 30cda5e | 2017-04-24 11:15:13 | [diff] [blame] | 215 | |
oprypin | 1d7392a | 2017-05-16 12:36:15 | [diff] [blame] | 216 | processes = [] |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 217 | temp_dir = CreateEmptyDir(temp_dir) |
oprypin | 1d7392a | 2017-05-16 12:36:15 | [diff] [blame] | 218 | try: |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 219 | android_device = SelectAndroidDevice(adb_path) |
| 220 | SetUpTools(android_device, temp_dir, processes) |
Edward Lesmes | 5b9c684 | 2018-03-09 18:07:22 | [diff] [blame] | 221 | RunTest(android_device, adb_path, build_dir, temp_dir, args.num_retries, |
Edward Lemur | 71d766e | 2018-02-05 20:59:36 | [diff] [blame] | 222 | args.isolated_script_test_perf_output) |
oprypin | 1d7392a | 2017-05-16 12:36:15 | [diff] [blame] | 223 | finally: |
| 224 | for process in processes: |
| 225 | if process: |
| 226 | process.terminate() |
| 227 | process.wait() |
| 228 | |
Edward Lemur | 70ba9b4 | 2018-01-22 12:46:58 | [diff] [blame] | 229 | utils.RemoveDirectory(temp_dir) |
mandermo | ed582f7 | 2017-01-23 15:55:42 | [diff] [blame] | 230 | |
Oleh Prypin | 637b0b5 | 2018-09-21 15:16:06 | [diff] [blame] | 231 | if args.isolated_script_test_output: |
| 232 | with open(args.isolated_script_test_output, 'w') as f: |
| 233 | f.write('{"version": 3}') |
| 234 | |
mandermo | ed582f7 | 2017-01-23 15:55:42 | [diff] [blame] | 235 | |
| 236 | if __name__ == '__main__': |
| 237 | sys.exit(main()) |