Introduce MetricsExporter API with stdout implementation
Bug: b/246095034
Change-Id: I9979fb03b9a02e76808145f43910420524fe633a
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/274880
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Tomas Gunnarsson <tommi@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38107}
diff --git a/BUILD.gn b/BUILD.gn
index fd5b799..3caa4c5 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -52,7 +52,6 @@
":voip_unittests",
":webrtc_nonparallel_tests",
":webrtc_perf_tests",
- "api/test/metrics:metrics_unittests",
"common_audio:common_audio_unittests",
"common_video:common_video_unittests",
"examples:examples_unittests",
@@ -565,6 +564,7 @@
"api/audio_codecs/test:audio_codecs_api_unittests",
"api/numerics:numerics_unittests",
"api/task_queue:pending_task_safety_flag_unittests",
+ "api/test/metrics:metrics_unittests",
"api/transport:stun_unittest",
"api/video/test:rtc_api_video_unittests",
"api/video_codecs/test:video_codecs_api_unittests",
diff --git a/api/test/metrics/BUILD.gn b/api/test/metrics/BUILD.gn
index 0d1ffff..099ee8a 100644
--- a/api/test/metrics/BUILD.gn
+++ b/api/test/metrics/BUILD.gn
@@ -9,14 +9,18 @@
import("../../../webrtc.gni")
group("metrics") {
- deps = [ ":metric" ]
+ deps = [
+ ":metric",
+ ":metrics_exporter",
+ ":stdout_metrics_exporter",
+ ]
}
if (rtc_include_tests) {
group("metrics_unittests") {
testonly = true
- deps = []
+ deps = [ ":stdout_metrics_exporter_test" ]
}
}
@@ -29,3 +33,40 @@
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
deps = [ "../../../api/units:timestamp" ]
}
+
+rtc_library("metrics_exporter") {
+ visibility = [ "*" ]
+ sources = [ "metrics_exporter.h" ]
+ deps = [
+ ":metric",
+ "../../../api:array_view",
+ ]
+}
+
+rtc_library("stdout_metrics_exporter") {
+ visibility = [ "*" ]
+ sources = [
+ "stdout_metrics_exporter.cc",
+ "stdout_metrics_exporter.h",
+ ]
+ deps = [
+ ":metric",
+ ":metrics_exporter",
+ "../../../api:array_view",
+ "../../../rtc_base:stringutils",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+}
+
+if (rtc_include_tests) {
+ rtc_library("stdout_metrics_exporter_test") {
+ testonly = true
+ sources = [ "stdout_metrics_exporter_test.cc" ]
+ deps = [
+ ":metric",
+ ":stdout_metrics_exporter",
+ "../../../api/units:timestamp",
+ "../../../test:test_support",
+ ]
+ }
+}
diff --git a/api/test/metrics/metrics_exporter.h b/api/test/metrics/metrics_exporter.h
new file mode 100644
index 0000000..23954b6
--- /dev/null
+++ b/api/test/metrics/metrics_exporter.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_TEST_METRICS_METRICS_EXPORTER_H_
+#define API_TEST_METRICS_METRICS_EXPORTER_H_
+
+#include "api/array_view.h"
+#include "api/test/metrics/metric.h"
+
+namespace webrtc {
+namespace test {
+
+// Exports metrics in the requested format.
+class MetricsExporter {
+ public:
+ virtual ~MetricsExporter() = default;
+
+ // Exports specified metrics in a format that depends on the implementation.
+ // Returns true if export succeeded, false otherwise.
+ virtual bool Export(rtc::ArrayView<const Metric> metrics) = 0;
+};
+
+} // namespace test
+} // namespace webrtc
+
+#endif // API_TEST_METRICS_METRICS_EXPORTER_H_
diff --git a/api/test/metrics/stdout_metrics_exporter.cc b/api/test/metrics/stdout_metrics_exporter.cc
new file mode 100644
index 0000000..f509591
--- /dev/null
+++ b/api/test/metrics/stdout_metrics_exporter.cc
@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "api/test/metrics/stdout_metrics_exporter.h"
+
+#include <stdio.h>
+
+#include <cmath>
+#include <string>
+
+#include "absl/types/optional.h"
+#include "api/array_view.h"
+#include "api/test/metrics/metric.h"
+#include "rtc_base/strings/string_builder.h"
+
+namespace webrtc {
+namespace test {
+namespace {
+
+// Returns positive integral part of the number.
+int64_t IntegralPart(double value) {
+ return std::lround(std::floor(std::abs(value)));
+}
+
+void AppendWithPrecision(double value,
+ int digits_after_comma,
+ rtc::StringBuilder& out) {
+ int64_t multiplier = std::lround(std::pow(10, digits_after_comma));
+ int64_t integral_part = IntegralPart(value);
+ double decimal_part = std::abs(value) - integral_part;
+
+ // If decimal part has leading zeros then when it will be multiplied on
+ // `multiplier`, leading zeros will be lost. To preserve them we add "1"
+ // so then leading digit will be greater than 0 and won't be removed.
+ //
+ // During conversion to the string leading digit has to be stripped.
+ //
+ // Also due to rounding it may happen that leading digit may be incremented,
+ // like with `digits_after_comma` 3 number 1.9995 will be rounded to 2. In
+ // such case this increment has to be propagated to the `integral_part`.
+ int64_t decimal_holder = std::lround((1 + decimal_part) * multiplier);
+ if (decimal_holder >= 2 * multiplier) {
+ // Rounding incremented added leading digit, so we need to transfer 1 to
+ // integral part.
+ integral_part++;
+ decimal_holder -= multiplier;
+ }
+ // Remove trailing zeros.
+ while (decimal_holder % 10 == 0) {
+ decimal_holder /= 10;
+ }
+
+ // Print serialized number to output.
+ if (value < 0) {
+ out << "-";
+ }
+ out << integral_part;
+ if (decimal_holder != 1) {
+ out << "." << std::to_string(decimal_holder).substr(1, digits_after_comma);
+ }
+}
+
+} // namespace
+
+StdoutMetricsExporter::StdoutMetricsExporter() : output_(stdout) {}
+
+bool StdoutMetricsExporter::Export(rtc::ArrayView<const Metric> metrics) {
+ for (const Metric& metric : metrics) {
+ PrintMetric(metric);
+ }
+ return true;
+}
+
+void StdoutMetricsExporter::PrintMetric(const Metric& metric) {
+ rtc::StringBuilder value_stream;
+ value_stream << metric.test_case << "/" << metric.name << "= {mean=";
+ if (metric.stats.mean.has_value()) {
+ AppendWithPrecision(*metric.stats.mean, 8, value_stream);
+ } else {
+ value_stream << "-";
+ }
+ value_stream << ", stddev=";
+ if (metric.stats.stddev.has_value()) {
+ AppendWithPrecision(*metric.stats.stddev, 8, value_stream);
+ } else {
+ value_stream << "-";
+ }
+ value_stream << "} " << ToString(metric.unit) << " ("
+ << ToString(metric.improvement_direction) << ")";
+
+ fprintf(output_, "RESULT: %s\n", value_stream.str().c_str());
+}
+
+} // namespace test
+} // namespace webrtc
diff --git a/api/test/metrics/stdout_metrics_exporter.h b/api/test/metrics/stdout_metrics_exporter.h
new file mode 100644
index 0000000..2c572cb
--- /dev/null
+++ b/api/test/metrics/stdout_metrics_exporter.h
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_TEST_METRICS_STDOUT_METRICS_EXPORTER_H_
+#define API_TEST_METRICS_STDOUT_METRICS_EXPORTER_H_
+
+#include "api/array_view.h"
+#include "api/test/metrics/metric.h"
+#include "api/test/metrics/metrics_exporter.h"
+
+namespace webrtc {
+namespace test {
+
+// Exports all collected metrics to stdout.
+class StdoutMetricsExporter : public MetricsExporter {
+ public:
+ StdoutMetricsExporter();
+ ~StdoutMetricsExporter() override = default;
+
+ StdoutMetricsExporter(const StdoutMetricsExporter&) = delete;
+ StdoutMetricsExporter& operator=(const StdoutMetricsExporter&) = delete;
+
+ bool Export(rtc::ArrayView<const Metric> metrics) override;
+
+ private:
+ void PrintMetric(const Metric& metric);
+
+ FILE* const output_;
+};
+
+} // namespace test
+} // namespace webrtc
+
+#endif // API_TEST_METRICS_STDOUT_METRICS_EXPORTER_H_
diff --git a/api/test/metrics/stdout_metrics_exporter_test.cc b/api/test/metrics/stdout_metrics_exporter_test.cc
new file mode 100644
index 0000000..b69adda
--- /dev/null
+++ b/api/test/metrics/stdout_metrics_exporter_test.cc
@@ -0,0 +1,211 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "api/test/metrics/stdout_metrics_exporter.h"
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "api/test/metrics/metric.h"
+#include "api/units/timestamp.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace test {
+namespace {
+
+using ::testing::TestWithParam;
+
+std::map<std::string, std::string> DefaultMetadata() {
+ return std::map<std::string, std::string>{{"key", "value"}};
+}
+
+Metric::TimeSeries::Sample Sample(double value) {
+ return Metric::TimeSeries::Sample{.timestamp = Timestamp::Seconds(1),
+ .value = value,
+ .sample_metadata = DefaultMetadata()};
+}
+
+Metric PsnrForTestFoo(double mean, double stddev) {
+ return Metric{.name = "psnr",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .test_case = "foo",
+ .time_series = Metric::TimeSeries{},
+ .stats = Metric::Stats{.mean = mean, .stddev = stddev}};
+}
+
+TEST(StdoutMetricsExporterTest, MAYBE_ExportMetricFormatCorrect) {
+ Metric metric1{
+ .name = "test_metric1",
+ .unit = Unit::kTimeMs,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .test_case = "test_case_name1",
+ .metric_metadata = DefaultMetadata(),
+ .time_series =
+ Metric::TimeSeries{.samples = std::vector{Sample(10), Sample(20)}},
+ .stats =
+ Metric::Stats{.mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}};
+ Metric metric2{
+ .name = "test_metric2",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .test_case = "test_case_name2",
+ .metric_metadata = DefaultMetadata(),
+ .time_series =
+ Metric::TimeSeries{.samples = std::vector{Sample(20), Sample(40)}},
+ .stats = Metric::Stats{
+ .mean = 30.0, .stddev = 10.0, .min = 20.0, .max = 40.0}};
+
+ testing::internal::CaptureStdout();
+ StdoutMetricsExporter exporter;
+
+ std::string expected =
+ "RESULT: test_case_name1/test_metric1= "
+ "{mean=15, stddev=5} TimeMs (BiggerIsBetter)\n"
+ "RESULT: test_case_name2/test_metric2= "
+ "{mean=30, stddev=10} KilobitsPerSecond (SmallerIsBetter)\n";
+
+ EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric1, metric2}));
+ EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
+}
+
+TEST(StdoutMetricsExporterNumberFormatTest, PositiveNumberMaxPrecision) {
+ testing::internal::CaptureStdout();
+ StdoutMetricsExporter exporter;
+
+ Metric metric = PsnrForTestFoo(15.00000001, 0.00000001);
+ std::string expected =
+ "RESULT: foo/psnr= "
+ "{mean=15.00000001, stddev=0.00000001} Unitless (BiggerIsBetter)\n";
+ EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
+ EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
+}
+
+TEST(StdoutMetricsExporterNumberFormatTest,
+ PositiveNumberTrailingZeroNotAdded) {
+ testing::internal::CaptureStdout();
+ StdoutMetricsExporter exporter;
+
+ Metric metric = PsnrForTestFoo(15.12345, 0.12);
+ std::string expected =
+ "RESULT: foo/psnr= "
+ "{mean=15.12345, stddev=0.12} Unitless (BiggerIsBetter)\n";
+ EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
+ EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
+}
+
+TEST(StdoutMetricsExporterNumberFormatTest,
+ PositiveNumberTrailingZeroAreRemoved) {
+ testing::internal::CaptureStdout();
+ StdoutMetricsExporter exporter;
+
+ Metric metric = PsnrForTestFoo(15.123450000, 0.120000000);
+ std::string expected =
+ "RESULT: foo/psnr= "
+ "{mean=15.12345, stddev=0.12} Unitless (BiggerIsBetter)\n";
+ EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
+ EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
+}
+
+TEST(StdoutMetricsExporterNumberFormatTest,
+ PositiveNumberRoundsUpOnPrecisionCorrectly) {
+ testing::internal::CaptureStdout();
+ StdoutMetricsExporter exporter;
+
+ Metric metric = PsnrForTestFoo(15.000000009, 0.999999999);
+ std::string expected =
+ "RESULT: foo/psnr= "
+ "{mean=15.00000001, stddev=1} Unitless (BiggerIsBetter)\n";
+ EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
+ EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
+}
+
+TEST(StdoutMetricsExporterNumberFormatTest,
+ PositiveNumberRoundsDownOnPrecisionCorrectly) {
+ testing::internal::CaptureStdout();
+ StdoutMetricsExporter exporter;
+
+ Metric metric = PsnrForTestFoo(15.0000000049, 0.9999999949);
+ std::string expected =
+ "RESULT: foo/psnr= "
+ "{mean=15, stddev=0.99999999} Unitless (BiggerIsBetter)\n";
+ EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
+ EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
+}
+
+TEST(StdoutMetricsExporterNumberFormatTest, NegativeNumberMaxPrecision) {
+ testing::internal::CaptureStdout();
+ StdoutMetricsExporter exporter;
+
+ Metric metric = PsnrForTestFoo(-15.00000001, -0.00000001);
+ std::string expected =
+ "RESULT: foo/psnr= "
+ "{mean=-15.00000001, stddev=-0.00000001} Unitless (BiggerIsBetter)\n";
+ EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
+ EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
+}
+
+TEST(StdoutMetricsExporterNumberFormatTest,
+ NegativeNumberTrailingZeroNotAdded) {
+ testing::internal::CaptureStdout();
+ StdoutMetricsExporter exporter;
+
+ Metric metric = PsnrForTestFoo(-15.12345, -0.12);
+ std::string expected =
+ "RESULT: foo/psnr= "
+ "{mean=-15.12345, stddev=-0.12} Unitless (BiggerIsBetter)\n";
+ EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
+ EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
+}
+
+TEST(StdoutMetricsExporterNumberFormatTest,
+ NegativeNumberTrailingZeroAreRemoved) {
+ testing::internal::CaptureStdout();
+ StdoutMetricsExporter exporter;
+
+ Metric metric = PsnrForTestFoo(-15.123450000, -0.120000000);
+ std::string expected =
+ "RESULT: foo/psnr= "
+ "{mean=-15.12345, stddev=-0.12} Unitless (BiggerIsBetter)\n";
+ EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
+ EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
+}
+
+TEST(StdoutMetricsExporterNumberFormatTest,
+ NegativeNumberRoundsUpOnPrecisionCorrectly) {
+ testing::internal::CaptureStdout();
+ StdoutMetricsExporter exporter;
+
+ Metric metric = PsnrForTestFoo(-15.000000009, -0.999999999);
+ std::string expected =
+ "RESULT: foo/psnr= "
+ "{mean=-15.00000001, stddev=-1} Unitless (BiggerIsBetter)\n";
+ EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
+ EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
+}
+
+TEST(StdoutMetricsExporterNumberFormatTest,
+ NegativeNumberRoundsDownOnPrecisionCorrectly) {
+ testing::internal::CaptureStdout();
+ StdoutMetricsExporter exporter;
+
+ Metric metric = PsnrForTestFoo(-15.0000000049, -0.9999999949);
+ std::string expected =
+ "RESULT: foo/psnr= "
+ "{mean=-15, stddev=-0.99999999} Unitless (BiggerIsBetter)\n";
+ EXPECT_TRUE(exporter.Export(std::vector<Metric>{metric}));
+ EXPECT_EQ(expected, testing::internal::GetCapturedStdout());
+}
+
+} // namespace
+} // namespace test
+} // namespace webrtc
