blob: c195b72a54f2fe7c937b277822b615c5942f6420 [file] [log] [blame]
ivica05cfcd32015-09-07 13:04:161#!/usr/bin/env python
2# Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3#
4# Use of this source code is governed by a BSD-style license
5# that can be found in the LICENSE file in the root of the source
6# tree. An additional intellectual property rights grant can be found
7# in the file PATENTS. All contributing project authors may
8# be found in the AUTHORS file in the root of the source tree.
ivica5d6a06c2015-09-17 12:30:249"""Generate graphs for data generated by loopback tests.
ivica05cfcd32015-09-07 13:04:1610
11Usage examples:
12 Show end to end time for a single full stack test.
brandtraa354c92016-12-01 08:20:0613 ./full_stack_tests_plot.py -df end_to_end -o 600 --frames 1000 vp9_data.txt
ivica05cfcd32015-09-07 13:04:1614
15 Show simultaneously PSNR and encoded frame size for two different runs of
16 full stack test. Averaged over a cycle of 200 frames. Used e.g. for
17 screenshare slide test.
brandtraa354c92016-12-01 08:20:0618 ./full_stack_tests_plot.py -c 200 -df psnr -drf encoded_frame_size \\
19 before.txt after.txt
ivica05cfcd32015-09-07 13:04:1620
21 Similar to the previous test, but multiple graphs.
brandtraa354c92016-12-01 08:20:0622 ./full_stack_tests_plot.py -c 200 -df psnr vp8.txt vp9.txt --next \\
23 -c 200 -df sender_time vp8.txt vp9.txt --next \\
24 -c 200 -df end_to_end vp8.txt vp9.txt
ivica05cfcd32015-09-07 13:04:1625"""
26
27import argparse
28from collections import defaultdict
29import itertools
30import sys
31import matplotlib.pyplot as plt
32import numpy
33
34# Fields
35DROPPED = 0
Mirko Bonadei8cc66952020-10-30 09:13:4536INPUT_TIME = 1 # ms (timestamp)
37SEND_TIME = 2 # ms (timestamp)
38RECV_TIME = 3 # ms (timestamp)
39RENDER_TIME = 4 # ms (timestamp)
40ENCODED_FRAME_SIZE = 5 # bytes
ivica8d15bd62015-10-07 09:43:1241PSNR = 6
42SSIM = 7
Mirko Bonadei8cc66952020-10-30 09:13:4543ENCODE_TIME = 8 # ms (time interval)
ivica05cfcd32015-09-07 13:04:1644
ivica8d15bd62015-10-07 09:43:1245TOTAL_RAW_FIELDS = 9
ivica05cfcd32015-09-07 13:04:1646
47SENDER_TIME = TOTAL_RAW_FIELDS + 0
48RECEIVER_TIME = TOTAL_RAW_FIELDS + 1
49END_TO_END = TOTAL_RAW_FIELDS + 2
50RENDERED_DELTA = TOTAL_RAW_FIELDS + 3
51
52FIELD_MASK = 255
53
54# Options
55HIDE_DROPPED = 256
56RIGHT_Y_AXIS = 512
57
58# internal field id, field name, title
kjellanderdd460e22017-04-12 19:06:1359_FIELDS = [
ivica05cfcd32015-09-07 13:04:1660 # Raw
61 (DROPPED, "dropped", "dropped"),
62 (INPUT_TIME, "input_time_ms", "input time"),
63 (SEND_TIME, "send_time_ms", "send time"),
64 (RECV_TIME, "recv_time_ms", "recv time"),
65 (ENCODED_FRAME_SIZE, "encoded_frame_size", "encoded frame size"),
66 (PSNR, "psnr", "PSNR"),
67 (SSIM, "ssim", "SSIM"),
68 (RENDER_TIME, "render_time_ms", "render time"),
ivica8d15bd62015-10-07 09:43:1269 (ENCODE_TIME, "encode_time_ms", "encode time"),
ivica05cfcd32015-09-07 13:04:1670 # Auto-generated
71 (SENDER_TIME, "sender_time", "sender time"),
72 (RECEIVER_TIME, "receiver_time", "receiver time"),
73 (END_TO_END, "end_to_end", "end to end"),
74 (RENDERED_DELTA, "rendered_delta", "rendered delta"),
75]
76
kjellanderdd460e22017-04-12 19:06:1377NAME_TO_ID = {field[1]: field[0] for field in _FIELDS}
78ID_TO_TITLE = {field[0]: field[2] for field in _FIELDS}
ivica05cfcd32015-09-07 13:04:1679
Mirko Bonadei8cc66952020-10-30 09:13:4580
kjellanderdd460e22017-04-12 19:06:1381def FieldArgToId(arg):
Mirko Bonadei8cc66952020-10-30 09:13:4582 if arg == "none":
83 return None
84 if arg in NAME_TO_ID:
85 return NAME_TO_ID[arg]
86 if arg + "_ms" in NAME_TO_ID:
87 return NAME_TO_ID[arg + "_ms"]
88 raise Exception("Unrecognized field name \"{}\"".format(arg))
ivica05cfcd32015-09-07 13:04:1689
90
91class PlotLine(object):
Mirko Bonadei8cc66952020-10-30 09:13:4592 """Data for a single graph line."""
ivica05cfcd32015-09-07 13:04:1693
Mirko Bonadei8cc66952020-10-30 09:13:4594 def __init__(self, label, values, flags):
95 self.label = label
96 self.values = values
97 self.flags = flags
ivica05cfcd32015-09-07 13:04:1698
99
100class Data(object):
Mirko Bonadei8cc66952020-10-30 09:13:45101 """Object representing one full stack test."""
ivica05cfcd32015-09-07 13:04:16102
Mirko Bonadei8cc66952020-10-30 09:13:45103 def __init__(self, filename):
104 self.title = ""
105 self.length = 0
106 self.samples = defaultdict(list)
ivica05cfcd32015-09-07 13:04:16107
Mirko Bonadei8cc66952020-10-30 09:13:45108 self._ReadSamples(filename)
ivica05cfcd32015-09-07 13:04:16109
Mirko Bonadei8cc66952020-10-30 09:13:45110 def _ReadSamples(self, filename):
111 """Reads graph data from the given file."""
112 f = open(filename)
113 it = iter(f)
ivica05cfcd32015-09-07 13:04:16114
Mirko Bonadei8cc66952020-10-30 09:13:45115 self.title = it.next().strip()
116 self.length = int(it.next())
117 field_names = [name.strip() for name in it.next().split()]
118 field_ids = [NAME_TO_ID[name] for name in field_names]
ivica05cfcd32015-09-07 13:04:16119
Mirko Bonadei8cc66952020-10-30 09:13:45120 for field_id in field_ids:
121 self.samples[field_id] = [0.0] * self.length
ivica05cfcd32015-09-07 13:04:16122
Mirko Bonadei8cc66952020-10-30 09:13:45123 for sample_id in xrange(self.length):
124 for col, value in enumerate(it.next().split()):
125 self.samples[field_ids[col]][sample_id] = float(value)
ivica05cfcd32015-09-07 13:04:16126
Mirko Bonadei8cc66952020-10-30 09:13:45127 self._SubtractFirstInputTime()
128 self._GenerateAdditionalData()
ivica05cfcd32015-09-07 13:04:16129
Mirko Bonadei8cc66952020-10-30 09:13:45130 f.close()
ivica05cfcd32015-09-07 13:04:16131
Mirko Bonadei8cc66952020-10-30 09:13:45132 def _SubtractFirstInputTime(self):
133 offset = self.samples[INPUT_TIME][0]
134 for field in [INPUT_TIME, SEND_TIME, RECV_TIME, RENDER_TIME]:
135 if field in self.samples:
136 self.samples[field] = [x - offset for x in self.samples[field]]
ivica05cfcd32015-09-07 13:04:16137
Mirko Bonadei8cc66952020-10-30 09:13:45138 def _GenerateAdditionalData(self):
139 """Calculates sender time, receiver time etc. from the raw data."""
140 s = self.samples
141 last_render_time = 0
142 for field_id in [
143 SENDER_TIME, RECEIVER_TIME, END_TO_END, RENDERED_DELTA
144 ]:
145 s[field_id] = [0] * self.length
ivica05cfcd32015-09-07 13:04:16146
Mirko Bonadei8cc66952020-10-30 09:13:45147 for k in range(self.length):
148 s[SENDER_TIME][k] = s[SEND_TIME][k] - s[INPUT_TIME][k]
ivica05cfcd32015-09-07 13:04:16149
Mirko Bonadei8cc66952020-10-30 09:13:45150 decoded_time = s[RENDER_TIME][k]
151 s[RECEIVER_TIME][k] = decoded_time - s[RECV_TIME][k]
152 s[END_TO_END][k] = decoded_time - s[INPUT_TIME][k]
153 if not s[DROPPED][k]:
154 if k > 0:
155 s[RENDERED_DELTA][k] = decoded_time - last_render_time
156 last_render_time = decoded_time
ivica05cfcd32015-09-07 13:04:16157
Mirko Bonadei8cc66952020-10-30 09:13:45158 def _Hide(self, values):
159 """
ivica05cfcd32015-09-07 13:04:16160 Replaces values for dropped frames with None.
kjellanderdd460e22017-04-12 19:06:13161 These values are then skipped by the Plot() method.
ivica05cfcd32015-09-07 13:04:16162 """
163
Mirko Bonadei8cc66952020-10-30 09:13:45164 return [
165 None if self.samples[DROPPED][k] else values[k]
166 for k in range(len(values))
167 ]
ivica05cfcd32015-09-07 13:04:16168
Mirko Bonadei8cc66952020-10-30 09:13:45169 def AddSamples(self, config, target_lines_list):
170 """Creates graph lines from the current data set with given config."""
171 for field in config.fields:
172 # field is None means the user wants just to skip the color.
173 if field is None:
174 target_lines_list.append(None)
175 continue
ivica05cfcd32015-09-07 13:04:16176
Mirko Bonadei8cc66952020-10-30 09:13:45177 field_id = field & FIELD_MASK
178 values = self.samples[field_id]
ivica05cfcd32015-09-07 13:04:16179
Mirko Bonadei8cc66952020-10-30 09:13:45180 if field & HIDE_DROPPED:
181 values = self._Hide(values)
ivica05cfcd32015-09-07 13:04:16182
Mirko Bonadei8cc66952020-10-30 09:13:45183 target_lines_list.append(
184 PlotLine(self.title + " " + ID_TO_TITLE[field_id], values,
185 field & ~FIELD_MASK))
ivica05cfcd32015-09-07 13:04:16186
187
kjellanderdd460e22017-04-12 19:06:13188def AverageOverCycle(values, length):
Mirko Bonadei8cc66952020-10-30 09:13:45189 """
ivica05cfcd32015-09-07 13:04:16190 Returns the list:
191 [
192 avg(values[0], values[length], ...),
193 avg(values[1], values[length + 1], ...),
194 ...
195 avg(values[length - 1], values[2 * length - 1], ...),
196 ]
197
198 Skips None values when calculating the average value.
199 """
200
Mirko Bonadei8cc66952020-10-30 09:13:45201 total = [0.0] * length
202 count = [0] * length
203 for k, val in enumerate(values):
204 if val is not None:
205 total[k % length] += val
206 count[k % length] += 1
ivica05cfcd32015-09-07 13:04:16207
Mirko Bonadei8cc66952020-10-30 09:13:45208 result = [0.0] * length
209 for k in range(length):
210 result[k] = total[k] / count[k] if count[k] else None
211 return result
ivica05cfcd32015-09-07 13:04:16212
213
214class PlotConfig(object):
Mirko Bonadei8cc66952020-10-30 09:13:45215 """Object representing a single graph."""
ivica05cfcd32015-09-07 13:04:16216
Mirko Bonadei8cc66952020-10-30 09:13:45217 def __init__(self,
218 fields,
219 data_list,
220 cycle_length=None,
221 frames=None,
222 offset=0,
223 output_filename=None,
224 title="Graph"):
225 self.fields = fields
226 self.data_list = data_list
227 self.cycle_length = cycle_length
228 self.frames = frames
229 self.offset = offset
230 self.output_filename = output_filename
231 self.title = title
ivica05cfcd32015-09-07 13:04:16232
Mirko Bonadei8cc66952020-10-30 09:13:45233 def Plot(self, ax1):
234 lines = []
235 for data in self.data_list:
236 if not data:
237 # Add None lines to skip the colors.
238 lines.extend([None] * len(self.fields))
239 else:
240 data.AddSamples(self, lines)
ivica05cfcd32015-09-07 13:04:16241
Mirko Bonadei8cc66952020-10-30 09:13:45242 def _SliceValues(values):
243 if self.offset:
244 values = values[self.offset:]
245 if self.frames:
246 values = values[:self.frames]
247 return values
ivica05cfcd32015-09-07 13:04:16248
Mirko Bonadei8cc66952020-10-30 09:13:45249 length = None
250 for line in lines:
251 if line is None:
252 continue
ivica05cfcd32015-09-07 13:04:16253
Mirko Bonadei8cc66952020-10-30 09:13:45254 line.values = _SliceValues(line.values)
255 if self.cycle_length:
256 line.values = AverageOverCycle(line.values, self.cycle_length)
ivica05cfcd32015-09-07 13:04:16257
Mirko Bonadei8cc66952020-10-30 09:13:45258 if length is None:
259 length = len(line.values)
260 elif length != len(line.values):
261 raise Exception("All arrays should have the same length!")
ivica05cfcd32015-09-07 13:04:16262
Mirko Bonadei8cc66952020-10-30 09:13:45263 ax1.set_xlabel("Frame", fontsize="large")
264 if any(line.flags & RIGHT_Y_AXIS for line in lines if line):
265 ax2 = ax1.twinx()
266 ax2.set_xlabel("Frame", fontsize="large")
267 else:
268 ax2 = None
ivica05cfcd32015-09-07 13:04:16269
Mirko Bonadei8cc66952020-10-30 09:13:45270 # Have to implement color_cycle manually, due to two scales in a graph.
271 color_cycle = ["b", "r", "g", "c", "m", "y", "k"]
272 color_iter = itertools.cycle(color_cycle)
ivica05cfcd32015-09-07 13:04:16273
Mirko Bonadei8cc66952020-10-30 09:13:45274 for line in lines:
275 if not line:
276 color_iter.next()
277 continue
ivica05cfcd32015-09-07 13:04:16278
Mirko Bonadei8cc66952020-10-30 09:13:45279 if self.cycle_length:
280 x = numpy.array(range(self.cycle_length))
281 else:
282 x = numpy.array(
283 range(self.offset, self.offset + len(line.values)))
284 y = numpy.array(line.values)
285 ax = ax2 if line.flags & RIGHT_Y_AXIS else ax1
286 ax.Plot(x,
287 y,
288 "o-",
289 label=line.label,
290 markersize=3.0,
291 linewidth=1.0,
292 color=color_iter.next())
ivica05cfcd32015-09-07 13:04:16293
Mirko Bonadei8cc66952020-10-30 09:13:45294 ax1.grid(True)
295 if ax2:
296 ax1.legend(loc="upper left", shadow=True, fontsize="large")
297 ax2.legend(loc="upper right", shadow=True, fontsize="large")
298 else:
299 ax1.legend(loc="best", shadow=True, fontsize="large")
ivica05cfcd32015-09-07 13:04:16300
301
kjellanderdd460e22017-04-12 19:06:13302def LoadFiles(filenames):
Mirko Bonadei8cc66952020-10-30 09:13:45303 result = []
304 for filename in filenames:
305 if filename in LoadFiles.cache:
306 result.append(LoadFiles.cache[filename])
307 else:
308 data = Data(filename)
309 LoadFiles.cache[filename] = data
310 result.append(data)
311 return result
312
313
kjellanderdd460e22017-04-12 19:06:13314LoadFiles.cache = {}
ivica05cfcd32015-09-07 13:04:16315
316
kjellanderdd460e22017-04-12 19:06:13317def GetParser():
Mirko Bonadei8cc66952020-10-30 09:13:45318 class CustomAction(argparse.Action):
319 def __call__(self, parser, namespace, values, option_string=None):
320 if "ordered_args" not in namespace:
321 namespace.ordered_args = []
322 namespace.ordered_args.append((self.dest, values))
ivica05cfcd32015-09-07 13:04:16323
Mirko Bonadei8cc66952020-10-30 09:13:45324 parser = argparse.ArgumentParser(
325 description=__doc__,
326 formatter_class=argparse.RawDescriptionHelpFormatter)
ivica05cfcd32015-09-07 13:04:16327
Mirko Bonadei8cc66952020-10-30 09:13:45328 parser.add_argument("-c",
329 "--cycle_length",
330 nargs=1,
331 action=CustomAction,
332 type=int,
333 help="Cycle length over which to average the values.")
334 parser.add_argument(
335 "-f",
336 "--field",
337 nargs=1,
338 action=CustomAction,
339 help="Name of the field to show. Use 'none' to skip a color.")
340 parser.add_argument("-r",
341 "--right",
342 nargs=0,
343 action=CustomAction,
344 help="Use right Y axis for given field.")
345 parser.add_argument("-d",
346 "--drop",
347 nargs=0,
348 action=CustomAction,
349 help="Hide values for dropped frames.")
350 parser.add_argument("-o",
351 "--offset",
352 nargs=1,
353 action=CustomAction,
354 type=int,
355 help="Frame offset.")
356 parser.add_argument("-n",
357 "--next",
358 nargs=0,
359 action=CustomAction,
360 help="Separator for multiple graphs.")
361 parser.add_argument(
362 "--frames",
363 nargs=1,
364 action=CustomAction,
365 type=int,
366 help="Frame count to show or take into account while averaging.")
367 parser.add_argument("-t",
368 "--title",
369 nargs=1,
370 action=CustomAction,
371 help="Title of the graph.")
372 parser.add_argument("-O",
373 "--output_filename",
374 nargs=1,
375 action=CustomAction,
376 help="Use to save the graph into a file. "
377 "Otherwise, a window will be shown.")
378 parser.add_argument(
379 "files",
380 nargs="+",
381 action=CustomAction,
382 help="List of text-based files generated by loopback tests.")
383 return parser
ivica05cfcd32015-09-07 13:04:16384
385
kjellanderdd460e22017-04-12 19:06:13386def _PlotConfigFromArgs(args, graph_num):
Mirko Bonadei8cc66952020-10-30 09:13:45387 # Pylint complains about using kwargs, so have to do it this way.
388 cycle_length = None
389 frames = None
390 offset = 0
391 output_filename = None
392 title = "Graph"
ivica05cfcd32015-09-07 13:04:16393
Mirko Bonadei8cc66952020-10-30 09:13:45394 fields = []
395 files = []
396 mask = 0
397 for key, values in args:
398 if key == "cycle_length":
399 cycle_length = values[0]
400 elif key == "frames":
401 frames = values[0]
402 elif key == "offset":
403 offset = values[0]
404 elif key == "output_filename":
405 output_filename = values[0]
406 elif key == "title":
407 title = values[0]
408 elif key == "drop":
409 mask |= HIDE_DROPPED
410 elif key == "right":
411 mask |= RIGHT_Y_AXIS
412 elif key == "field":
413 field_id = FieldArgToId(values[0])
414 fields.append(field_id | mask if field_id is not None else None)
415 mask = 0 # Reset mask after the field argument.
416 elif key == "files":
417 files.extend(values)
ivica05cfcd32015-09-07 13:04:16418
Mirko Bonadei8cc66952020-10-30 09:13:45419 if not files:
420 raise Exception(
421 "Missing file argument(s) for graph #{}".format(graph_num))
422 if not fields:
423 raise Exception(
424 "Missing field argument(s) for graph #{}".format(graph_num))
ivica05cfcd32015-09-07 13:04:16425
Mirko Bonadei8cc66952020-10-30 09:13:45426 return PlotConfig(fields,
427 LoadFiles(files),
428 cycle_length=cycle_length,
429 frames=frames,
430 offset=offset,
431 output_filename=output_filename,
432 title=title)
ivica05cfcd32015-09-07 13:04:16433
434
kjellanderdd460e22017-04-12 19:06:13435def PlotConfigsFromArgs(args):
Mirko Bonadei8cc66952020-10-30 09:13:45436 """Generates plot configs for given command line arguments."""
437 # The way it works:
438 # First we detect separators -n/--next and split arguments into groups, one
439 # for each plot. For each group, we partially parse it with
440 # argparse.ArgumentParser, modified to remember the order of arguments.
441 # Then we traverse the argument list and fill the PlotConfig.
442 args = itertools.groupby(args, lambda x: x in ["-n", "--next"])
443 prep_args = list(list(group) for match, group in args if not match)
ivica05cfcd32015-09-07 13:04:16444
Mirko Bonadei8cc66952020-10-30 09:13:45445 parser = GetParser()
446 plot_configs = []
447 for index, raw_args in enumerate(prep_args):
448 graph_args = parser.parse_args(raw_args).ordered_args
449 plot_configs.append(_PlotConfigFromArgs(graph_args, index))
450 return plot_configs
ivica05cfcd32015-09-07 13:04:16451
452
kjellanderdd460e22017-04-12 19:06:13453def ShowOrSavePlots(plot_configs):
Mirko Bonadei8cc66952020-10-30 09:13:45454 for config in plot_configs:
455 fig = plt.figure(figsize=(14.0, 10.0))
456 ax = fig.add_subPlot(1, 1, 1)
ivica05cfcd32015-09-07 13:04:16457
Mirko Bonadei8cc66952020-10-30 09:13:45458 plt.title(config.title)
459 config.Plot(ax)
460 if config.output_filename:
461 print "Saving to", config.output_filename
462 fig.savefig(config.output_filename)
463 plt.close(fig)
ivica05cfcd32015-09-07 13:04:16464
Mirko Bonadei8cc66952020-10-30 09:13:45465 plt.show()
466
ivica05cfcd32015-09-07 13:04:16467
468if __name__ == "__main__":
Mirko Bonadei8cc66952020-10-30 09:13:45469 ShowOrSavePlots(PlotConfigsFromArgs(sys.argv[1:]))