blob: fc198a826f7962444b6485868b4b20a3d5d6e297 [file] [log] [blame]
mandermoed582f72017-01-23 15:55:421#!/usr/bin/env python
2# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3#
4# Use of this source code is governed by a BSD-style license
5# that can be found in the LICENSE file in the root of the source
6# tree. An additional intellectual property rights grant can be found
7# in the file PATENTS. All contributing project authors may
8# be found in the AUTHORS file in the root of the source tree.
9
10"""
11This script is the wrapper that starts a loopback call with stubbed video in
12and out. It then analyses the video quality of the output video against the
13reference input video.
14
15It expect to be given the webrtc output build directory as the first argument
16all other arguments are optional.
17
18It assumes you have a Android device plugged in.
19"""
20
21import argparse
oprypinbed7a6b2017-06-19 08:16:4522import json
mandermoed582f72017-01-23 15:55:4223import logging
24import os
mandermoed582f72017-01-23 15:55:4225import subprocess
26import sys
27import tempfile
oprypin30cda5e2017-04-24 11:15:1328import time
mandermoed582f72017-01-23 15:55:4229
30
31SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
Henrik Kjellandercb3b1c12017-09-18 04:20:3332SRC_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, os.pardir, os.pardir))
Edward Lemur70ba9b42018-01-22 12:46:5833RTC_TOOLS_DIR = os.path.join(SRC_DIR, 'rtc_tools', 'testing')
34TOOLCHAIN_DIR = os.path.join(SRC_DIR, 'tools_webrtc', 'video_quality_toolchain',
35 'linux')
oprypinbed7a6b2017-06-19 08:16:4536BAD_DEVICES_JSON = os.path.join(SRC_DIR,
37 os.environ.get('CHROMIUM_OUT_DIR', 'out'),
38 'bad_devices.json')
Edward Lemur70ba9b42018-01-22 12:46:5839
40sys.path.append(RTC_TOOLS_DIR)
41import utils
oprypin30cda5e2017-04-24 11:15:1342
43
44class Error(Exception):
45 pass
46
47
48class VideoQualityTestError(Error):
49 pass
mandermoed582f72017-01-23 15:55:4250
51
Edward Lemurd5e17d62018-01-31 16:28:2852def _RunCommand(argv, **kwargs):
mandermoed582f72017-01-23 15:55:4253 logging.info('Running %r', argv)
Edward Lemurd5e17d62018-01-31 16:28:2854 subprocess.check_call(argv, **kwargs)
mandermoed582f72017-01-23 15:55:4255
56
Edward Lemurd5e17d62018-01-31 16:28:2857def _RunCommandWithOutput(argv, **kwargs):
oprypin30cda5e2017-04-24 11:15:1358 logging.info('Running %r', argv)
Edward Lemurd5e17d62018-01-31 16:28:2859 return subprocess.check_output(argv, **kwargs)
oprypin30cda5e2017-04-24 11:15:1360
61
Edward Lemurd5e17d62018-01-31 16:28:2862def _RunBackgroundCommand(argv):
oprypin30cda5e2017-04-24 11:15:1363 logging.info('Running %r', argv)
Edward Lemurd5e17d62018-01-31 16:28:2864 process = subprocess.Popen(argv)
oprypin30cda5e2017-04-24 11:15:1365 time.sleep(0.5)
66 status = process.poll()
67 if status: # is not None or 0
68 raise subprocess.CalledProcessError(status, argv)
69 return process
70
71
Edward Lemur70ba9b42018-01-22 12:46:5872def CreateEmptyDir(suggested_dir):
73 if not suggested_dir:
74 return tempfile.mkdtemp()
75 utils.RemoveDirectory(suggested_dir)
76 os.makedirs(suggested_dir)
77 return suggested_dir
78
79
mandermoed582f72017-01-23 15:55:4280def _ParseArgs():
81 parser = argparse.ArgumentParser(description='Start loopback video analysis.')
mandermoed582f72017-01-23 15:55:4282 parser.add_argument('build_dir_android',
83 help='The path to the build directory for Android.')
mandermoed582f72017-01-23 15:55:4284 parser.add_argument('--temp_dir',
85 help='A temporary directory to put the output.')
oprypin30cda5e2017-04-24 11:15:1386 parser.add_argument('--adb-path', help='Path to adb binary.', default='adb')
Edward Lesmes9599fd42018-03-12 20:43:0587 parser.add_argument('--num-retries', default='0',
Edward Lesmes5b9c6842018-03-09 18:07:2288 help='Number of times to retry the test on Android.')
Edward Lemur71d766e2018-02-05 20:59:3689 parser.add_argument('--isolated-script-test-perf-output',
Edward Lemur2e5966b2018-01-30 14:33:0290 help='Where to store perf results in chartjson format.', default=None)
Oleh Prypin637b0b52018-09-21 15:16:0691 parser.add_argument('--isolated-script-test-output', default=None,
92 help='Path to output an empty JSON file which Chromium infra requires.')
Edward Lemur21a35bc2018-01-31 12:46:5093 args, unknown_args = parser.parse_known_args()
94
95 # Ignore Chromium-specific flags
96 parser = argparse.ArgumentParser()
Edward Lemur21a35bc2018-01-31 12:46:5097 parser.add_argument('--test-launcher-summary-output',
98 type=str, default=None)
99
100 parser.parse_args(unknown_args)
101
mandermoed582f72017-01-23 15:55:42102 return args
103
104
Edward Lemur70ba9b42018-01-22 12:46:58105def SelectAndroidDevice(adb_path):
106 # Select an Android device in case multiple are connected.
oprypinbed7a6b2017-06-19 08:16:45107 try:
108 with open(BAD_DEVICES_JSON) as bad_devices_file:
109 bad_devices = json.load(bad_devices_file)
110 except IOError:
111 if os.environ.get('CHROME_HEADLESS'):
112 logging.warning('Cannot read %r', BAD_DEVICES_JSON)
113 bad_devices = {}
114
oprypin30cda5e2017-04-24 11:15:13115 for line in _RunCommandWithOutput([adb_path, 'devices']).splitlines():
116 if line.endswith('\tdevice'):
117 android_device = line.split('\t')[0]
oprypinbed7a6b2017-06-19 08:16:45118 if android_device not in bad_devices:
Edward Lemur70ba9b42018-01-22 12:46:58119 return android_device
120 raise VideoQualityTestError('Cannot find any connected Android device.')
121
122
123def SetUpTools(android_device, temp_dir, processes):
124 # Extract AppRTC.
125 apprtc_archive = os.path.join(RTC_TOOLS_DIR, 'prebuilt_apprtc.zip')
126 golang_archive = os.path.join(RTC_TOOLS_DIR, 'golang', 'linux', 'go.tar.gz')
127
128 utils.UnpackArchiveTo(apprtc_archive, temp_dir)
129 utils.UnpackArchiveTo(golang_archive, temp_dir)
130
131 # Build AppRTC.
132 build_apprtc_script = os.path.join(RTC_TOOLS_DIR, 'build_apprtc.py')
Oleh Prypin1e908452018-04-04 12:31:58133 apprtc_dir = os.path.join(temp_dir, 'apprtc')
Edward Lemur70ba9b42018-01-22 12:46:58134 go_dir = os.path.join(temp_dir, 'go')
135 collider_dir = os.path.join(temp_dir, 'collider')
136
Oleh Prypin1e908452018-04-04 12:31:58137 _RunCommand([sys.executable, build_apprtc_script, apprtc_dir, go_dir,
138 collider_dir])
Edward Lemur70ba9b42018-01-22 12:46:58139
140 # Start AppRTC Server.
141 dev_appserver = os.path.join(temp_dir, 'apprtc', 'temp', 'google-cloud-sdk',
142 'bin', 'dev_appserver.py')
143 appengine_dir = os.path.join(temp_dir, 'apprtc', 'out', 'app_engine')
144 processes.append(_RunBackgroundCommand([
145 sys.executable, dev_appserver, appengine_dir, '--port=9999',
146 '--admin_port=9998', '--skip_sdk_update_check', '--clear_datastore=yes']))
147
148 # Start Collider.
149 collider_path = os.path.join(temp_dir, 'collider', 'collidermain')
150 processes.append(_RunBackgroundCommand([
151 collider_path, '-tls=false', '-port=8089',
152 '-room-server=http://localhost:9999']))
153
154 # Start adb reverse forwarder.
155 reverseforwarder_path = os.path.join(
156 SRC_DIR, 'build', 'android', 'adb_reverse_forwarder.py')
157 processes.append(_RunBackgroundCommand([
158 reverseforwarder_path, '--device', android_device, '9999', '9999', '8089',
159 '8089']))
160
161
Edward Lesmes5b9c6842018-03-09 18:07:22162def RunTest(android_device, adb_path, build_dir, temp_dir, num_retries,
Edward Lemur2e5966b2018-01-30 14:33:02163 chartjson_result_file):
Edward Lemur70ba9b42018-01-22 12:46:58164 ffmpeg_path = os.path.join(TOOLCHAIN_DIR, 'ffmpeg')
Sami Kalliomäki0673bc92018-08-27 15:58:13165 def ConvertVideo(input_video, output_video):
166 _RunCommand([ffmpeg_path, '-y', '-i', input_video, output_video])
Edward Lemur70ba9b42018-01-22 12:46:58167
168 # Start loopback call and record video.
169 test_script = os.path.join(
Sami Kalliomäkid54f5f52018-08-03 11:23:05170 build_dir, 'bin', 'run_AppRTCMobile_stubbed_video_io_test_apk')
Edward Lesmes5b9c6842018-03-09 18:07:22171 _RunCommand([test_script, '--device', android_device,
172 '--num-retries', num_retries])
Edward Lemur70ba9b42018-01-22 12:46:58173
174 # Pull the recorded video.
175 test_video = os.path.join(temp_dir, 'test_video.y4m')
176 _RunCommand([adb_path, '-s', android_device,
177 'pull', '/sdcard/output.y4m', test_video])
178
179 # Convert the recorded and reference videos to YUV.
180 reference_video = os.path.join(SRC_DIR,
181 'resources', 'reference_video_640x360_30fps.y4m')
182
Sami Kalliomäki0673bc92018-08-27 15:58:13183 test_video_yuv = os.path.join(temp_dir, 'test_video.yuv')
184 reference_video_yuv = os.path.join(
185 temp_dir, 'reference_video_640x360_30fps.yuv')
186
187 ConvertVideo(test_video, test_video_yuv)
188 ConvertVideo(reference_video, reference_video_yuv)
189
Edward Lemur70ba9b42018-01-22 12:46:58190 # Run comparison script.
191 compare_script = os.path.join(SRC_DIR, 'rtc_tools', 'compare_videos.py')
Mirko Bonadeib56706f2018-09-18 09:03:39192 frame_analyzer = os.path.join(build_dir, 'frame_analyzer_host')
Edward Lemur70ba9b42018-01-22 12:46:58193
Edward Lemur2e5966b2018-01-30 14:33:02194 args = [
Sami Kalliomäki0673bc92018-08-27 15:58:13195 '--ref_video', reference_video_yuv,
196 '--test_video', test_video_yuv,
Edward Lemur70ba9b42018-01-22 12:46:58197 '--yuv_frame_width', '640',
198 '--yuv_frame_height', '360',
Edward Lemur70ba9b42018-01-22 12:46:58199 '--frame_analyzer', frame_analyzer,
Edward Lemur2e5966b2018-01-30 14:33:02200 ]
201 if chartjson_result_file:
202 args.extend(['--chartjson_result_file', chartjson_result_file])
203
204 _RunCommand([sys.executable, compare_script] + args)
Edward Lemur70ba9b42018-01-22 12:46:58205
206
207def main():
208 logging.basicConfig(level=logging.INFO)
209
210 args = _ParseArgs()
211
212 temp_dir = args.temp_dir
213 build_dir = args.build_dir_android
214 adb_path = args.adb_path
oprypin30cda5e2017-04-24 11:15:13215
oprypin1d7392a2017-05-16 12:36:15216 processes = []
Edward Lemur70ba9b42018-01-22 12:46:58217 temp_dir = CreateEmptyDir(temp_dir)
oprypin1d7392a2017-05-16 12:36:15218 try:
Edward Lemur70ba9b42018-01-22 12:46:58219 android_device = SelectAndroidDevice(adb_path)
220 SetUpTools(android_device, temp_dir, processes)
Edward Lesmes5b9c6842018-03-09 18:07:22221 RunTest(android_device, adb_path, build_dir, temp_dir, args.num_retries,
Edward Lemur71d766e2018-02-05 20:59:36222 args.isolated_script_test_perf_output)
oprypin1d7392a2017-05-16 12:36:15223 finally:
224 for process in processes:
225 if process:
226 process.terminate()
227 process.wait()
228
Edward Lemur70ba9b42018-01-22 12:46:58229 utils.RemoveDirectory(temp_dir)
mandermoed582f72017-01-23 15:55:42230
Oleh Prypin637b0b52018-09-21 15:16:06231 if args.isolated_script_test_output:
232 with open(args.isolated_script_test_output, 'w') as f:
233 f.write('{"version": 3}')
234
mandermoed582f72017-01-23 15:55:42235
236if __name__ == '__main__':
237 sys.exit(main())