test/testsupport/perf_test.cc

/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "test/testsupport/perf_test.h"

#include <algorithm>
#include <cmath>
#include <cstdio>
#include <ostream>
#include <set>
#include <span>
#include <sstream>
#include <string>
#include <vector>

#include "absl/strings/string_view.h"
#include "api/numerics/samples_stats_counter.h"
#include "rtc_base/checks.h"
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/thread_annotations.h"
#include "test/testsupport/file_utils.h"
#include "test/testsupport/perf_test_histogram_writer.h"
#include "test/testsupport/perf_test_result_writer.h"

namespace webrtc {
namespace test {

namespace {

std::string UnitWithDirection(absl::string_view units,
                              test::ImproveDirection improve_direction) {
  switch (improve_direction) {
    case test::ImproveDirection::kNone:
      return std::string(units);
    case test::ImproveDirection::kSmallerIsBetter:
      return std::string(units) + "_smallerIsBetter";
    case test::ImproveDirection::kBiggerIsBetter:
      return std::string(units) + "_biggerIsBetter";
  }
}

std::vector<SamplesStatsCounter::StatsSample> GetSortedSamples(
    const SamplesStatsCounter& counter) {
  std::span<const SamplesStatsCounter::StatsSample> view =
      counter.GetTimedSamples();
  std::vector<SamplesStatsCounter::StatsSample> out(view.begin(), view.end());
  std::stable_sort(out.begin(), out.end(),
                   [](const SamplesStatsCounter::StatsSample& a,
                      const SamplesStatsCounter::StatsSample& b) {
                     return a.time < b.time;
                   });
  return out;
}

template <typename Container>
void OutputListToStream(std::ostream* ostream, const Container& values) {
  const char* sep = "";
  for (const auto& v : values) {
    (*ostream) << sep << v;
    sep = ",";
  }
}

struct PlottableCounter {
  std::string graph_name;
  std::string trace_name;
  SamplesStatsCounter counter;
  std::string units;
};

class PlottableCounterPrinter {
 public:
  PlottableCounterPrinter() : output_(stdout) {}

  void SetOutput(FILE* output) {
    MutexLock lock(&mutex_);
    output_ = output;
  }

  void AddCounter(absl::string_view graph_name,
                  absl::string_view trace_name,
                  const SamplesStatsCounter& counter,
                  absl::string_view units) {
    MutexLock lock(&mutex_);
    plottable_counters_.push_back({.graph_name = std::string(graph_name),
                                   .trace_name = std::string(trace_name),
                                   .counter = counter,
                                   .units = std::string(units)});
  }

  void Print(const std::vector<std::string>& desired_graphs_raw) const {
    std::set<std::string> desired_graphs(desired_graphs_raw.begin(),
                                         desired_graphs_raw.end());
    MutexLock lock(&mutex_);
    for (auto& counter : plottable_counters_) {
      if (!desired_graphs.empty()) {
        auto it = desired_graphs.find(counter.graph_name);
        if (it == desired_graphs.end()) {
          continue;
        }
      }

      std::ostringstream value_stream;
      value_stream.precision(8);
      value_stream << R"({"graph_name":")" << counter.graph_name << R"(",)";
      value_stream << R"("trace_name":")" << counter.trace_name << R"(",)";
      value_stream << R"("units":")" << counter.units << R"(",)";
      if (!counter.counter.IsEmpty()) {
        value_stream << R"("mean":)" << counter.counter.GetAverage() << ',';
        value_stream << R"("std":)" << counter.counter.GetStandardDeviation()
                     << ',';
      }
      value_stream << R"("samples":[)";
      const char* sep = "";
      for (const auto& sample : counter.counter.GetTimedSamples()) {
        value_stream << sep << R"({"time":)" << sample.time.us() << ','
                     << R"("value":)" << sample.value << '}';
        sep = ",";
      }
      value_stream << "]}";

      fprintf(output_, "PLOTTABLE_DATA: %s\n", value_stream.str().c_str());
    }
  }

 private:
  mutable Mutex mutex_;
  std::vector<PlottableCounter> plottable_counters_ RTC_GUARDED_BY(&mutex_);
  FILE* output_ RTC_GUARDED_BY(&mutex_);
};

PlottableCounterPrinter& GetPlottableCounterPrinter() {
  static PlottableCounterPrinter* printer_ = new PlottableCounterPrinter();
  return *printer_;
}

class ResultsLinePrinter {
 public:
  ResultsLinePrinter() : output_(stdout) {}

  void SetOutput(FILE* output) {
    MutexLock lock(&mutex_);
    output_ = output;
  }

  void PrintResult(absl::string_view graph_name,
                   absl::string_view trace_name,
                   const double value,
                   absl::string_view units,
                   bool important,
                   ImproveDirection improve_direction) {
    std::ostringstream value_stream;
    value_stream.precision(8);
    value_stream << value;

    PrintResultImpl(graph_name, trace_name, value_stream.str(), std::string(),
                    std::string(), UnitWithDirection(units, improve_direction),
                    important);
  }

  void PrintResultMeanAndError(absl::string_view graph_name,
                               absl::string_view trace_name,
                               const double mean,
                               const double error,
                               absl::string_view units,
                               bool important,
                               ImproveDirection improve_direction) {
    std::ostringstream value_stream;
    value_stream.precision(8);
    value_stream << mean << ',' << error;
    PrintResultImpl(graph_name, trace_name, value_stream.str(), "{", "}",
                    UnitWithDirection(units, improve_direction), important);
  }

  void PrintResultList(absl::string_view graph_name,
                       absl::string_view trace_name,
                       const std::span<const double> values,
                       absl::string_view units,
                       const bool important,
                       test::ImproveDirection improve_direction) {
    std::ostringstream value_stream;
    value_stream.precision(8);
    OutputListToStream(&value_stream, values);
    PrintResultImpl(graph_name, trace_name, value_stream.str(), "[", "]", units,
                    important);
  }

 private:
  void PrintResultImpl(absl::string_view graph_name,
                       absl::string_view trace_name,
                       absl::string_view values,
                       absl::string_view prefix,
                       absl::string_view suffix,
                       absl::string_view units,
                       bool important) {
    MutexLock lock(&mutex_);
    StringBuilder message;
    message << (important ? "*" : "") << "RESULT " << graph_name << ": "
            << trace_name << "= " << prefix << values << suffix << " " << units;
    // <*>RESULT <graph_name>: <trace_name>= <value> <units>
    // <*>RESULT <graph_name>: <trace_name>= {<mean>, <std deviation>} <units>
    // <*>RESULT <graph_name>: <trace_name>= [<value>,value,value,...,] <units>
    fprintf(output_, "%s\n", message.str().c_str());
  }

  Mutex mutex_;
  FILE* output_ RTC_GUARDED_BY(&mutex_);
};

ResultsLinePrinter& GetResultsLinePrinter() {
  static ResultsLinePrinter* const printer_ = new ResultsLinePrinter();
  return *printer_;
}

PerfTestResultWriter& GetPerfWriter() {
  static PerfTestResultWriter* writer = CreateHistogramWriter();
  return *writer;
}

}  // namespace

void ClearPerfResults() {
  GetPerfWriter().ClearResults();
}

void SetPerfResultsOutput(FILE* output) {
  GetPlottableCounterPrinter().SetOutput(output);
  GetResultsLinePrinter().SetOutput(output);
}

std::string GetPerfResults() {
  return GetPerfWriter().Serialize();
}

void PrintPlottableResults(const std::vector<std::string>& desired_graphs) {
  GetPlottableCounterPrinter().Print(desired_graphs);
}

bool WritePerfResults(const std::string& output_path) {
  std::string results = GetPerfResults();
  CreateDir(DirName(output_path));
  FILE* output = fopen(output_path.c_str(), "wb");
  if (output == nullptr) {
    printf("Failed to write to %s.\n", output_path.c_str());
    return false;
  }
  size_t written =
      fwrite(results.c_str(), sizeof(char), results.size(), output);
  fclose(output);

  if (written != results.size()) {
    long expected = results.size();
    printf("Wrote %zu, tried to write %lu\n", written, expected);
    return false;
  }

  return true;
}

void PrintResult(absl::string_view measurement,
                 absl::string_view modifier,
                 absl::string_view trace,
                 const double value,
                 absl::string_view units,
                 bool important,
                 ImproveDirection improve_direction) {
  StringBuilder graph_name;
  graph_name << measurement << modifier;
  RTC_CHECK(std::isfinite(value))
      << "Expected finite value for graph " << graph_name.str()
      << ", trace name " << trace << ", units " << units << ", got " << value;
  GetPerfWriter().LogResult(graph_name.str(), trace, value, units, important,
                            improve_direction);
  GetResultsLinePrinter().PrintResult(graph_name.str(), trace, value, units,
                                      important, improve_direction);
}

void PrintResult(absl::string_view measurement,
                 absl::string_view modifier,
                 absl::string_view trace,
                 const SamplesStatsCounter& counter,
                 absl::string_view units,
                 const bool important,
                 ImproveDirection improve_direction) {
  StringBuilder graph_name;
  graph_name << measurement << modifier;
  GetPlottableCounterPrinter().AddCounter(graph_name.str(), trace, counter,
                                          units);

  double mean = counter.IsEmpty() ? 0 : counter.GetAverage();
  double error = counter.IsEmpty() ? 0 : counter.GetStandardDeviation();

  std::vector<SamplesStatsCounter::StatsSample> timed_samples =
      GetSortedSamples(counter);
  std::vector<double> samples(timed_samples.size());
  for (size_t i = 0; i < timed_samples.size(); ++i) {
    samples[i] = timed_samples[i].value;
  }
  // If we have an empty counter, default it to 0.
  if (samples.empty()) {
    samples.push_back(0);
  }

  GetPerfWriter().LogResultList(graph_name.str(), trace, samples, units,
                                important, improve_direction);
  GetResultsLinePrinter().PrintResultMeanAndError(graph_name.str(), trace, mean,
                                                  error, units, important,
                                                  improve_direction);
}

void PrintResultMeanAndError(absl::string_view measurement,
                             absl::string_view modifier,
                             absl::string_view trace,
                             const double mean,
                             const double error,
                             absl::string_view units,
                             bool important,
                             ImproveDirection improve_direction) {
  RTC_CHECK(std::isfinite(mean));
  RTC_CHECK(std::isfinite(error));

  StringBuilder graph_name;
  graph_name << measurement << modifier;
  GetPerfWriter().LogResultMeanAndError(graph_name.str(), trace, mean, error,
                                        units, important, improve_direction);
  GetResultsLinePrinter().PrintResultMeanAndError(graph_name.str(), trace, mean,
                                                  error, units, important,
                                                  improve_direction);
}

void PrintResultList(absl::string_view measurement,
                     absl::string_view modifier,
                     absl::string_view trace,
                     const std::span<const double> values,
                     absl::string_view units,
                     bool important,
                     ImproveDirection improve_direction) {
  for (double v : values) {
    RTC_CHECK(std::isfinite(v));
  }

  StringBuilder graph_name;
  graph_name << measurement << modifier;
  GetPerfWriter().LogResultList(graph_name.str(), trace, values, units,
                                important, improve_direction);
  GetResultsLinePrinter().PrintResultList(graph_name.str(), trace, values,
                                          units, important, improve_direction);
}

}  // namespace test
}  // namespace webrtc