Introduce PrintResultProxyMetricsExporter for migration from old to new API
Bug: b/246095034
Change-Id: I7597ddad84c4b2af1d59e38c558b1f0f56bd3f4e
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276047
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38141}
diff --git a/api/test/metrics/BUILD.gn b/api/test/metrics/BUILD.gn
index b2c0d7e..eb63ce0 100644
--- a/api/test/metrics/BUILD.gn
+++ b/api/test/metrics/BUILD.gn
@@ -26,6 +26,7 @@
deps = [
":metrics_logger_and_exporter_test",
+ ":print_result_proxy_metrics_exporter_test",
":stdout_metrics_exporter_test",
]
@@ -143,6 +144,21 @@
}
}
+rtc_library("print_result_proxy_metrics_exporter") {
+ visibility = [ "*" ]
+ testonly = true
+ sources = [
+ "print_result_proxy_metrics_exporter.cc",
+ "print_result_proxy_metrics_exporter.h",
+ ]
+ deps = [
+ ":metric",
+ ":metrics_exporter",
+ "../..:array_view",
+ "../../../test:perf_test",
+ ]
+}
+
if (rtc_include_tests) {
rtc_library("stdout_metrics_exporter_test") {
testonly = true
@@ -155,6 +171,17 @@
]
}
+ rtc_library("print_result_proxy_metrics_exporter_test") {
+ testonly = true
+ sources = [ "print_result_proxy_metrics_exporter_test.cc" ]
+ deps = [
+ ":metric",
+ ":print_result_proxy_metrics_exporter",
+ "../../../test:test_support",
+ "../../units:timestamp",
+ ]
+ }
+
rtc_library("metrics_logger_and_exporter_test") {
testonly = true
sources = [ "metrics_logger_and_exporter_test.cc" ]
diff --git a/api/test/metrics/print_result_proxy_metrics_exporter.cc b/api/test/metrics/print_result_proxy_metrics_exporter.cc
new file mode 100644
index 0000000..70cc9d5
--- /dev/null
+++ b/api/test/metrics/print_result_proxy_metrics_exporter.cc
@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "api/test/metrics/print_result_proxy_metrics_exporter.h"
+
+#include <string>
+
+#include "api/array_view.h"
+#include "api/test/metrics/metric.h"
+#include "test/testsupport/perf_test.h"
+
+namespace webrtc {
+namespace test {
+namespace {
+
+std::string ToPrintResultUnit(Unit unit) {
+ switch (unit) {
+ case Unit::kMilliseconds:
+ return "msBestFitFormat";
+ case Unit::kPercent:
+ return "n%";
+ case Unit::kBytes:
+ return "sizeInBytes";
+ case Unit::kKilobitsPerSecond:
+ // PrintResults prefer Chrome Perf Dashboard units, which doesn't have
+ // kpbs units, so we change the unit and value accordingly.
+ return "bytesPerSecond";
+ case Unit::kHertz:
+ return "Hz";
+ case Unit::kUnitless:
+ return "unitless";
+ case Unit::kCount:
+ return "count";
+ }
+}
+
+double ToPrintResultValue(double value, Unit unit) {
+ switch (unit) {
+ case Unit::kKilobitsPerSecond:
+ // PrintResults prefer Chrome Perf Dashboard units, which doesn't have
+ // kpbs units, so we change the unit and value accordingly.
+ return value * 1000 / 8;
+ default:
+ return value;
+ }
+}
+
+ImproveDirection ToPrintResultImproveDirection(ImprovementDirection direction) {
+ switch (direction) {
+ case ImprovementDirection::kBiggerIsBetter:
+ return ImproveDirection::kBiggerIsBetter;
+ case ImprovementDirection::kNeitherIsBetter:
+ return ImproveDirection::kNone;
+ case ImprovementDirection::kSmallerIsBetter:
+ return ImproveDirection::kSmallerIsBetter;
+ }
+}
+
+bool IsEmpty(const Metric::Stats& stats) {
+ return !stats.mean.has_value() && !stats.stddev.has_value() &&
+ !stats.min.has_value() && !stats.max.has_value();
+}
+
+} // namespace
+
+bool PrintResultProxyMetricsExporter::Export(
+ rtc::ArrayView<const Metric> metrics) {
+ for (const Metric& metric : metrics) {
+ if (metric.time_series.samples.empty() && IsEmpty(metric.stats)) {
+ // If there were no data collected for the metric it is expected that 0
+ // will be exported, so add 0 to the samples.
+ PrintResult(metric.name, /*modifier=*/"", metric.test_case,
+ ToPrintResultValue(0, metric.unit),
+ ToPrintResultUnit(metric.unit), /*important=*/false,
+ ToPrintResultImproveDirection(metric.improvement_direction));
+ continue;
+ }
+
+ if (metric.time_series.samples.empty()) {
+ PrintResultMeanAndError(
+ metric.name, /*modifier=*/"", metric.test_case,
+ ToPrintResultValue(*metric.stats.mean, metric.unit),
+ ToPrintResultValue(*metric.stats.stddev, metric.unit),
+ ToPrintResultUnit(metric.unit),
+ /*important=*/false,
+ ToPrintResultImproveDirection(metric.improvement_direction));
+ continue;
+ }
+
+ SamplesStatsCounter counter;
+ for (size_t i = 0; i < metric.time_series.samples.size(); ++i) {
+ counter.AddSample(SamplesStatsCounter::StatsSample{
+ .value = ToPrintResultValue(metric.time_series.samples[i].value,
+ metric.unit),
+ .time = metric.time_series.samples[i].timestamp,
+ .metadata = metric.time_series.samples[i].sample_metadata});
+ }
+
+ PrintResult(metric.name, /*modifier=*/"", metric.test_case, counter,
+ ToPrintResultUnit(metric.unit),
+ /*important=*/false,
+ ToPrintResultImproveDirection(metric.improvement_direction));
+ }
+ return true;
+}
+
+} // namespace test
+} // namespace webrtc
diff --git a/api/test/metrics/print_result_proxy_metrics_exporter.h b/api/test/metrics/print_result_proxy_metrics_exporter.h
new file mode 100644
index 0000000..bad0594
--- /dev/null
+++ b/api/test/metrics/print_result_proxy_metrics_exporter.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_TEST_METRICS_PRINT_RESULT_PROXY_METRICS_EXPORTER_H_
+#define API_TEST_METRICS_PRINT_RESULT_PROXY_METRICS_EXPORTER_H_
+
+#include "api/array_view.h"
+#include "api/test/metrics/metric.h"
+#include "api/test/metrics/metrics_exporter.h"
+
+namespace webrtc {
+namespace test {
+
+// Proxies all exported metrics to the `webrtc::test::PrintResult` API.
+class PrintResultProxyMetricsExporter : public MetricsExporter {
+ public:
+ ~PrintResultProxyMetricsExporter() override = default;
+
+ bool Export(rtc::ArrayView<const Metric> metrics) override;
+};
+
+} // namespace test
+} // namespace webrtc
+
+#endif // API_TEST_METRICS_PRINT_RESULT_PROXY_METRICS_EXPORTER_H_
diff --git a/api/test/metrics/print_result_proxy_metrics_exporter_test.cc b/api/test/metrics/print_result_proxy_metrics_exporter_test.cc
new file mode 100644
index 0000000..c783fba
--- /dev/null
+++ b/api/test/metrics/print_result_proxy_metrics_exporter_test.cc
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "api/test/metrics/print_result_proxy_metrics_exporter.h"
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "api/test/metrics/metric.h"
+#include "api/units/timestamp.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace test {
+namespace {
+
+using ::testing::TestWithParam;
+
+std::map<std::string, std::string> DefaultMetadata() {
+ return std::map<std::string, std::string>{{"key", "value"}};
+}
+
+Metric::TimeSeries::Sample Sample(double value) {
+ return Metric::TimeSeries::Sample{.timestamp = Timestamp::Seconds(1),
+ .value = value,
+ .sample_metadata = DefaultMetadata()};
+}
+
+TEST(PrintResultProxyMetricsExporterTest,
+ ExportMetricsWithTimeSeriesFormatCorrect) {
+ Metric metric1{
+ .name = "test_metric1",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .test_case = "test_case_name1",
+ .metric_metadata = DefaultMetadata(),
+ .time_series =
+ Metric::TimeSeries{.samples = std::vector{Sample(10), Sample(20)}},
+ .stats =
+ Metric::Stats{.mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}};
+ Metric metric2{
+ .name = "test_metric2",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .test_case = "test_case_name2",
+ .metric_metadata = DefaultMetadata(),
+ .time_series =
+ Metric::TimeSeries{.samples = std::vector{Sample(20), Sample(40)}},
+ .stats = Metric::Stats{
+ .mean = 30.0, .stddev = 10.0, .min = 20.0, .max = 40.0}};
+
+ testing::internal::CaptureStdout();
+ PrintResultProxyMetricsExporter exporter;
+
+ std::string expected =
+ "RESULT test_metric1: test_case_name1= {15,5} "
+ "msBestFitFormat_biggerIsBetter\n"
+ "RESULT test_metric2: test_case_name2= {3750,1250} "
+ "bytesPerSecond_smallerIsBetter\n";
+
+ EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric1, metric2}));
+ EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
+}
+
+TEST(PrintResultProxyMetricsExporterTest,
+ ExportMetricsWithStatsOnlyFormatCorrect) {
+ Metric metric1{.name = "test_metric1",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .test_case = "test_case_name1",
+ .metric_metadata = DefaultMetadata(),
+ .time_series = Metric::TimeSeries{.samples = {}},
+ .stats = Metric::Stats{
+ .mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}};
+ Metric metric2{
+ .name = "test_metric2",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .test_case = "test_case_name2",
+ .metric_metadata = DefaultMetadata(),
+ .time_series = Metric::TimeSeries{.samples = {}},
+ .stats = Metric::Stats{
+ .mean = 30.0, .stddev = 10.0, .min = 20.0, .max = 40.0}};
+
+ testing::internal::CaptureStdout();
+ PrintResultProxyMetricsExporter exporter;
+
+ std::string expected =
+ "RESULT test_metric1: test_case_name1= {15,5} "
+ "msBestFitFormat_biggerIsBetter\n"
+ "RESULT test_metric2: test_case_name2= {3750,1250} "
+ "bytesPerSecond_smallerIsBetter\n";
+
+ EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric1, metric2}));
+ EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
+}
+
+TEST(PrintResultProxyMetricsExporterTest, ExportEmptyMetricOnlyFormatCorrect) {
+ Metric metric{.name = "test_metric",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .test_case = "test_case_name",
+ .metric_metadata = DefaultMetadata(),
+ .time_series = Metric::TimeSeries{.samples = {}},
+ .stats = Metric::Stats{}};
+
+ testing::internal::CaptureStdout();
+ PrintResultProxyMetricsExporter exporter;
+
+ std::string expected =
+ "RESULT test_metric: test_case_name= 0 "
+ "msBestFitFormat_biggerIsBetter\n";
+
+ EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
+ EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
+}
+
+} // namespace
+} // namespace test
+} // namespace webrtc
