api/test/metrics/proto/metric.proto

/*
 *  Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

syntax = "proto3";

package webrtc.test_metrics;

// Root message of the proto file. Contains collection of all the metrics.
message MetricsSet {
  repeated Metric metrics = 1;
  // Metadata associated with the whole metrics set.
  map<string, string> metadata = 2;
}

enum Unit {
  // Default value that has to be defined.
  UNDEFINED_UNIT = 0;
  // General unitless value. Can be used either for dimensionless quantities
  // (ex ratio) or for units not presented in this enum and too specific to add
  // to this enum.
  UNITLESS = 1;
  MILLISECONDS = 2;
  PERCENT = 3;
  BYTES = 4;
  KILOBITS_PER_SECOND = 5;
  HERTZ = 6;
  COUNT = 7;
}

enum ImprovementDirection {
  // Default value that has to be defined.
  UNDEFINED_IMPROVEMENT_DIRECTION = 0;
  BIGGER_IS_BETTER = 1;
  NEITHER_IS_BETTER = 2;
  SMALLER_IS_BETTER = 3;
}

// Single performance metric with all related metadata.
message Metric {
  // Metric name, for example PSNR, SSIM, decode_time, etc.
  string name = 1;
  Unit unit = 2;
  ImprovementDirection improvement_direction = 3;
  // If the metric is generated by a test, this field can be used to specify
  // this information.
  string test_case = 4;
  // Metadata associated with the whole metric.
  map<string, string> metric_metadata = 5;

  message TimeSeries {
    message Sample {
      // Timestamp in microseconds associated with a sample. For example,
      // the timestamp when the sample was collected.
      int64 timestamp_us = 1;
      double value = 2;
      // Metadata associated with this particular sample.
      map<string, string> sample_metadata = 3;
    }
    // All samples collected for this metric. It can be empty if the Metric
    // object only contains `stats`.
    repeated Sample samples = 1;
  }
  // Contains all samples of the metric collected during test execution.
  // It can be empty if the user only stores precomputed statistics into
  // `stats`.
  TimeSeries time_series = 6;

  // Contains metric's precomputed statistics based on the `time_series` or if
  // `time_series` is omitted (has 0 samples) contains precomputed statistics
  // provided by the metric's calculator.
  message Stats {
    // Sample mean of the metric
    // (https://en.wikipedia.org/wiki/Sample_mean_and_covariance).
    optional double mean = 1;
    // Standard deviation (https://en.wikipedia.org/wiki/Standard_deviation).
    // Is undefined if `time_series` contains only a single sample.
    optional double stddev = 2;
    optional double min = 3;
    optional double max = 4;
  }
  Stats stats = 7;
}