tools_webrtc/perf/histogram_util.py

#!/usr/bin/env python
# Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS.  All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.

"""Upload data to the chrome perf dashboard via add_histograms endpoint."""

import os
import sys
import logging

SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
CHECKOUT_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir, os.pardir))
sys.path.insert(0, os.path.join(CHECKOUT_ROOT, 'third_party', 'catapult',
                                'tracing'))

from tracing.value import histogram
from tracing.value import histogram_set
from tracing.value.diagnostics import generic_set
from tracing.value.diagnostics import reserved_infos

# Enums aren't supported in Chromium's python env, so do something similar:
class ImprovementDirection(object):
  DEFAULT = 1
  BIGGER_IS_BETTER = 2
  SMALLER_IS_BETTER = 3


def MakeWebRtcHistogramSet(stats, commit_pos, commit_hash, master, bot,
                           test_suite, build_url):
  """Converts a dict of stats into a list of points with additional info.

  Args:
    stats: A list of histograms to upload.
    piper_revision: Baseline piper revision that the test was run on.
    commit_hash: WebRTC commit hash that the test was run on.
    master:
    bot: Bot name as it will show up in the perf dashboard.
    test_suite: Top-level identifier of the test for Chrome perf dashboard.
    build_url: An URL pointing to the bot status page for this build.

  Returns:
    A histogram set in format that expect Chrome perf dashboard.
  """
  common_diagnostics = {
      reserved_infos.MASTERS: master,
      reserved_infos.BOTS: bot,
      reserved_infos.POINT_ID: commit_pos,
      reserved_infos.BENCHMARKS: test_suite,
      reserved_infos.WEBRTC_REVISIONS: str(commit_hash),
      reserved_infos.BUILD_URLS: build_url,
  }

  hs = histogram_set.HistogramSet()
  for h in stats:
    hs.AddHistogram(h)

  for k, v in common_diagnostics.items():
    hs.AddSharedDiagnosticToAllHistograms(k.name, generic_set.GenericSet([v]))

  return hs


def LoadHistograms(data):
  """Load histograms from Chart JSON format json file and fix them for API.

  Args:
    data: parsed json object of Chart JSON format.

  Raises:
    RuntimeError: input data contains standard deviation section.
  Returns:
    list of loaded histograms.
  """
  stats = []
  for metric, story in data['charts'].items():
    for story_name, story_desc in story.items():
      units = story_desc['units'].strip()
      if 'std' in story_desc:
        # TODO(bugs.webrtc.org/11084): This seems bad to throw away?
        logging.debug('std is not supported, specify list of values instead.')

      if 'value' in story_desc:
        values = [story_desc['value']]
      else:
        values = list(story_desc['values'])

      improvement_direction = ImprovementDirection.DEFAULT
      if 'improvement_direction' in story_desc:
        if story_desc['improvement_direction'] == 'bigger_is_better':
          improvement_direction = ImprovementDirection.BIGGER_IS_BETTER
        elif story_desc['improvement_direction'] == 'smaller_is_better':
          improvement_direction = ImprovementDirection.SMALLER_IS_BETTER
      if 'higher_is_better' in story_desc:
        if story_desc['higher_is_better']:
          improvement_direction = ImprovementDirection.BIGGER_IS_BETTER
        else:
          improvement_direction = ImprovementDirection.SMALLER_IS_BETTER

      new_metric, new_units, new_values = _FixUnits(metric, units, values)
      h = _BuildHistogram(new_metric, story_name, new_units, new_values,
                          improvement_direction)
      stats.append(h)
  return stats


def _FixUnits(metric_name, units, values):
  """Fix units and metric name with values if required.

  Args:
    metric_name: origin metric name
    units: raw trimmed units
    values: origin values

  Returns:
    (metric_name, units, values) triple with fixed content
  """
  if units == 'bps':
    return metric_name, 'bytesPerSecond', [v / 8.0 for v in values]
  elif units == 'dB':
    return metric_name + '_dB', 'unitless', values
  elif units == 'fps':
    return metric_name + '_fps', 'Hz', values
  elif units == 'frames':
    return metric_name, 'count', values
  elif units == 'ms':
    return metric_name, 'msBestFitFormat', values
  elif units == '%':
    return metric_name + '_%', 'unitless', values
  else:
    return metric_name, units, values


def _BuildHistogram(metric_name, story_name, units, values,
                    improvement_direction):
  """Build histogram. Uses unitless for unsupported units."""
  if units not in histogram.UNIT_NAMES:
    logging.debug(
        'Unsupported unit %s will be replaced by \'unitless\'', units)
    units = 'unitless'
  if improvement_direction is ImprovementDirection.BIGGER_IS_BETTER:
    units = units + '_biggerIsBetter'
  elif improvement_direction is ImprovementDirection.SMALLER_IS_BETTER:
    units = units + '_smallerIsBetter'
  h = histogram.Histogram(metric_name, units,
                          histogram.HistogramBinBoundaries.SINGULAR)
  h.diagnostics[reserved_infos.STORIES.name] = generic_set.GenericSet(
      [story_name])
  h.CustomizeSummaryOptions({
      'std': False,
      'avg': False,
      'count': False,
      'max': False,
      'min': False,
      'sum': False
  })
  for v in values:
    h.AddSample(v)
  return h