Rename full_stack.cc to full_stack_tests.cc.
Also rename the accompanying plot file.
BUG=None
Review-Url: https://codereview.webrtc.org/2529293006
Cr-Commit-Position: refs/heads/master@{#15349}
diff --git a/webrtc/video/full_stack_tests_plot.py b/webrtc/video/full_stack_tests_plot.py
new file mode 100755
index 0000000..b922289
--- /dev/null
+++ b/webrtc/video/full_stack_tests_plot.py
@@ -0,0 +1,414 @@
+#!/usr/bin/env python
+# Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+"""Generate graphs for data generated by loopback tests.
+
+Usage examples:
+ Show end to end time for a single full stack test.
+ ./full_stack_tests_plot.py -df end_to_end -o 600 --frames 1000 vp9_data.txt
+
+ Show simultaneously PSNR and encoded frame size for two different runs of
+ full stack test. Averaged over a cycle of 200 frames. Used e.g. for
+ screenshare slide test.
+ ./full_stack_tests_plot.py -c 200 -df psnr -drf encoded_frame_size \\
+ before.txt after.txt
+
+ Similar to the previous test, but multiple graphs.
+ ./full_stack_tests_plot.py -c 200 -df psnr vp8.txt vp9.txt --next \\
+ -c 200 -df sender_time vp8.txt vp9.txt --next \\
+ -c 200 -df end_to_end vp8.txt vp9.txt
+"""
+
+import argparse
+from collections import defaultdict
+import itertools
+import sys
+import matplotlib.pyplot as plt
+import numpy
+
+# Fields
+DROPPED = 0
+INPUT_TIME = 1 # ms (timestamp)
+SEND_TIME = 2 # ms (timestamp)
+RECV_TIME = 3 # ms (timestamp)
+RENDER_TIME = 4 # ms (timestamp)
+ENCODED_FRAME_SIZE = 5 # bytes
+PSNR = 6
+SSIM = 7
+ENCODE_TIME = 8 # ms (time interval)
+
+TOTAL_RAW_FIELDS = 9
+
+SENDER_TIME = TOTAL_RAW_FIELDS + 0
+RECEIVER_TIME = TOTAL_RAW_FIELDS + 1
+END_TO_END = TOTAL_RAW_FIELDS + 2
+RENDERED_DELTA = TOTAL_RAW_FIELDS + 3
+
+FIELD_MASK = 255
+
+# Options
+HIDE_DROPPED = 256
+RIGHT_Y_AXIS = 512
+
+# internal field id, field name, title
+_fields = [
+ # Raw
+ (DROPPED, "dropped", "dropped"),
+ (INPUT_TIME, "input_time_ms", "input time"),
+ (SEND_TIME, "send_time_ms", "send time"),
+ (RECV_TIME, "recv_time_ms", "recv time"),
+ (ENCODED_FRAME_SIZE, "encoded_frame_size", "encoded frame size"),
+ (PSNR, "psnr", "PSNR"),
+ (SSIM, "ssim", "SSIM"),
+ (RENDER_TIME, "render_time_ms", "render time"),
+ (ENCODE_TIME, "encode_time_ms", "encode time"),
+ # Auto-generated
+ (SENDER_TIME, "sender_time", "sender time"),
+ (RECEIVER_TIME, "receiver_time", "receiver time"),
+ (END_TO_END, "end_to_end", "end to end"),
+ (RENDERED_DELTA, "rendered_delta", "rendered delta"),
+]
+
+name_to_id = {field[1]: field[0] for field in _fields}
+id_to_title = {field[0]: field[2] for field in _fields}
+
+def field_arg_to_id(arg):
+ if arg == "none":
+ return None
+ if arg in name_to_id:
+ return name_to_id[arg]
+ if arg + "_ms" in name_to_id:
+ return name_to_id[arg + "_ms"]
+ raise Exception("Unrecognized field name \"{}\"".format(arg))
+
+
+class PlotLine(object):
+ """Data for a single graph line."""
+
+ def __init__(self, label, values, flags):
+ self.label = label
+ self.values = values
+ self.flags = flags
+
+
+class Data(object):
+ """Object representing one full stack test."""
+
+ def __init__(self, filename):
+ self.title = ""
+ self.length = 0
+ self.samples = defaultdict(list)
+
+ self._read_samples(filename)
+
+ def _read_samples(self, filename):
+ """Reads graph data from the given file."""
+ f = open(filename)
+ it = iter(f)
+
+ self.title = it.next().strip()
+ self.length = int(it.next())
+ field_names = [name.strip() for name in it.next().split()]
+ field_ids = [name_to_id[name] for name in field_names]
+
+ for field_id in field_ids:
+ self.samples[field_id] = [0.0] * self.length
+
+ for sample_id in xrange(self.length):
+ for col, value in enumerate(it.next().split()):
+ self.samples[field_ids[col]][sample_id] = float(value)
+
+ self._subtract_first_input_time()
+ self._generate_additional_data()
+
+ f.close()
+
+ def _subtract_first_input_time(self):
+ offset = self.samples[INPUT_TIME][0]
+ for field in [INPUT_TIME, SEND_TIME, RECV_TIME, RENDER_TIME]:
+ if field in self.samples:
+ self.samples[field] = [x - offset for x in self.samples[field]]
+
+ def _generate_additional_data(self):
+ """Calculates sender time, receiver time etc. from the raw data."""
+ s = self.samples
+ last_render_time = 0
+ for field_id in [SENDER_TIME, RECEIVER_TIME, END_TO_END, RENDERED_DELTA]:
+ s[field_id] = [0] * self.length
+
+ for k in range(self.length):
+ s[SENDER_TIME][k] = s[SEND_TIME][k] - s[INPUT_TIME][k]
+
+ decoded_time = s[RENDER_TIME][k]
+ s[RECEIVER_TIME][k] = decoded_time - s[RECV_TIME][k]
+ s[END_TO_END][k] = decoded_time - s[INPUT_TIME][k]
+ if not s[DROPPED][k]:
+ if k > 0:
+ s[RENDERED_DELTA][k] = decoded_time - last_render_time
+ last_render_time = decoded_time
+
+ def _hide(self, values):
+ """
+ Replaces values for dropped frames with None.
+ These values are then skipped by the plot() method.
+ """
+
+ return [None if self.samples[DROPPED][k] else values[k]
+ for k in range(len(values))]
+
+ def add_samples(self, config, target_lines_list):
+ """Creates graph lines from the current data set with given config."""
+ for field in config.fields:
+ # field is None means the user wants just to skip the color.
+ if field is None:
+ target_lines_list.append(None)
+ continue
+
+ field_id = field & FIELD_MASK
+ values = self.samples[field_id]
+
+ if field & HIDE_DROPPED:
+ values = self._hide(values)
+
+ target_lines_list.append(PlotLine(
+ self.title + " " + id_to_title[field_id],
+ values, field & ~FIELD_MASK))
+
+
+def average_over_cycle(values, length):
+ """
+ Returns the list:
+ [
+ avg(values[0], values[length], ...),
+ avg(values[1], values[length + 1], ...),
+ ...
+ avg(values[length - 1], values[2 * length - 1], ...),
+ ]
+
+ Skips None values when calculating the average value.
+ """
+
+ total = [0.0] * length
+ count = [0] * length
+ for k in range(len(values)):
+ if values[k] is not None:
+ total[k % length] += values[k]
+ count[k % length] += 1
+
+ result = [0.0] * length
+ for k in range(length):
+ result[k] = total[k] / count[k] if count[k] else None
+ return result
+
+
+class PlotConfig(object):
+ """Object representing a single graph."""
+
+ def __init__(self, fields, data_list, cycle_length=None, frames=None,
+ offset=0, output_filename=None, title="Graph"):
+ self.fields = fields
+ self.data_list = data_list
+ self.cycle_length = cycle_length
+ self.frames = frames
+ self.offset = offset
+ self.output_filename = output_filename
+ self.title = title
+
+ def plot(self, ax1):
+ lines = []
+ for data in self.data_list:
+ if not data:
+ # Add None lines to skip the colors.
+ lines.extend([None] * len(self.fields))
+ else:
+ data.add_samples(self, lines)
+
+ def _slice_values(values):
+ if self.offset:
+ values = values[self.offset:]
+ if self.frames:
+ values = values[:self.frames]
+ return values
+
+ length = None
+ for line in lines:
+ if line is None:
+ continue
+
+ line.values = _slice_values(line.values)
+ if self.cycle_length:
+ line.values = average_over_cycle(line.values, self.cycle_length)
+
+ if length is None:
+ length = len(line.values)
+ elif length != len(line.values):
+ raise Exception("All arrays should have the same length!")
+
+ ax1.set_xlabel("Frame", fontsize="large")
+ if any(line.flags & RIGHT_Y_AXIS for line in lines if line):
+ ax2 = ax1.twinx()
+ ax2.set_xlabel("Frame", fontsize="large")
+ else:
+ ax2 = None
+
+ # Have to implement color_cycle manually, due to two scales in a graph.
+ color_cycle = ["b", "r", "g", "c", "m", "y", "k"]
+ color_iter = itertools.cycle(color_cycle)
+
+ for line in lines:
+ if not line:
+ color_iter.next()
+ continue
+
+ if self.cycle_length:
+ x = numpy.array(range(self.cycle_length))
+ else:
+ x = numpy.array(range(self.offset, self.offset + len(line.values)))
+ y = numpy.array(line.values)
+ ax = ax2 if line.flags & RIGHT_Y_AXIS else ax1
+ ax.plot(x, y, "o-", label=line.label, markersize=3.0, linewidth=1.0,
+ color=color_iter.next())
+
+ ax1.grid(True)
+ if ax2:
+ ax1.legend(loc="upper left", shadow=True, fontsize="large")
+ ax2.legend(loc="upper right", shadow=True, fontsize="large")
+ else:
+ ax1.legend(loc="best", shadow=True, fontsize="large")
+
+
+def load_files(filenames):
+ result = []
+ for filename in filenames:
+ if filename in load_files.cache:
+ result.append(load_files.cache[filename])
+ else:
+ data = Data(filename)
+ load_files.cache[filename] = data
+ result.append(data)
+ return result
+load_files.cache = {}
+
+
+def get_parser():
+ class CustomAction(argparse.Action):
+ def __call__(self, parser, namespace, values, option_string=None):
+ if "ordered_args" not in namespace:
+ namespace.ordered_args = []
+ namespace.ordered_args.append((self.dest, values))
+
+ parser = argparse.ArgumentParser(
+ description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+
+ parser.add_argument(
+ "-c", "--cycle_length", nargs=1, action=CustomAction,
+ type=int, help="Cycle length over which to average the values.")
+ parser.add_argument(
+ "-f", "--field", nargs=1, action=CustomAction,
+ help="Name of the field to show. Use 'none' to skip a color.")
+ parser.add_argument("-r", "--right", nargs=0, action=CustomAction,
+ help="Use right Y axis for given field.")
+ parser.add_argument("-d", "--drop", nargs=0, action=CustomAction,
+ help="Hide values for dropped frames.")
+ parser.add_argument("-o", "--offset", nargs=1, action=CustomAction, type=int,
+ help="Frame offset.")
+ parser.add_argument("-n", "--next", nargs=0, action=CustomAction,
+ help="Separator for multiple graphs.")
+ parser.add_argument(
+ "--frames", nargs=1, action=CustomAction, type=int,
+ help="Frame count to show or take into account while averaging.")
+ parser.add_argument("-t", "--title", nargs=1, action=CustomAction,
+ help="Title of the graph.")
+ parser.add_argument(
+ "-O", "--output_filename", nargs=1, action=CustomAction,
+ help="Use to save the graph into a file. "
+ "Otherwise, a window will be shown.")
+ parser.add_argument(
+ "files", nargs="+", action=CustomAction,
+ help="List of text-based files generated by loopback tests.")
+ return parser
+
+
+def _plot_config_from_args(args, graph_num):
+ # Pylint complains about using kwargs, so have to do it this way.
+ cycle_length = None
+ frames = None
+ offset = 0
+ output_filename = None
+ title = "Graph"
+
+ fields = []
+ files = []
+ mask = 0
+ for key, values in args:
+ if key == "cycle_length":
+ cycle_length = values[0]
+ elif key == "frames":
+ frames = values[0]
+ elif key == "offset":
+ offset = values[0]
+ elif key == "output_filename":
+ output_filename = values[0]
+ elif key == "title":
+ title = values[0]
+ elif key == "drop":
+ mask |= HIDE_DROPPED
+ elif key == "right":
+ mask |= RIGHT_Y_AXIS
+ elif key == "field":
+ field_id = field_arg_to_id(values[0])
+ fields.append(field_id | mask if field_id is not None else None)
+ mask = 0 # Reset mask after the field argument.
+ elif key == "files":
+ files.extend(values)
+
+ if not files:
+ raise Exception("Missing file argument(s) for graph #{}".format(graph_num))
+ if not fields:
+ raise Exception("Missing field argument(s) for graph #{}".format(graph_num))
+
+ return PlotConfig(fields, load_files(files), cycle_length=cycle_length,
+ frames=frames, offset=offset, output_filename=output_filename,
+ title=title)
+
+
+def plot_configs_from_args(args):
+ """Generates plot configs for given command line arguments."""
+ # The way it works:
+ # First we detect separators -n/--next and split arguments into groups, one
+ # for each plot. For each group, we partially parse it with
+ # argparse.ArgumentParser, modified to remember the order of arguments.
+ # Then we traverse the argument list and fill the PlotConfig.
+ args = itertools.groupby(args, lambda x: x in ["-n", "--next"])
+ args = list(list(group) for match, group in args if not match)
+
+ parser = get_parser()
+ plot_configs = []
+ for index, raw_args in enumerate(args):
+ graph_args = parser.parse_args(raw_args).ordered_args
+ plot_configs.append(_plot_config_from_args(graph_args, index))
+ return plot_configs
+
+
+def show_or_save_plots(plot_configs):
+ for config in plot_configs:
+ fig = plt.figure(figsize=(14.0, 10.0))
+ ax = fig.add_subplot(1, 1, 1)
+
+ plt.title(config.title)
+ config.plot(ax)
+ if config.output_filename:
+ print "Saving to", config.output_filename
+ fig.savefig(config.output_filename)
+ plt.close(fig)
+
+ plt.show()
+
+if __name__ == "__main__":
+ show_or_save_plots(plot_configs_from_args(sys.argv[1:]))
