Add MetricsSetProtoFileExporter
Bug: b/246095034
Change-Id: I002d0d5b132e61887b4bc87542fbf70dd81e488b
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/275881
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38125}
diff --git a/api/test/metrics/BUILD.gn b/api/test/metrics/BUILD.gn
index 544808f..8bfcf39 100644
--- a/api/test/metrics/BUILD.gn
+++ b/api/test/metrics/BUILD.gn
@@ -7,6 +7,9 @@
# be found in the AUTHORS file in the root of the source tree.
import("../../../webrtc.gni")
+if (rtc_enable_protobuf) {
+ import("//third_party/protobuf/proto_library.gni")
+}
group("metrics") {
deps = [
@@ -25,6 +28,10 @@
":metrics_logger_and_exporter_test",
":stdout_metrics_exporter_test",
]
+
+ if (rtc_enable_protobuf) {
+ deps += [ ":metrics_set_proto_file_exporter_test" ]
+ }
}
}
@@ -84,6 +91,35 @@
]
}
+if (rtc_enable_protobuf) {
+ proto_library("metric_proto") {
+ visibility = [ "*" ]
+ sources = [ "proto/metric.proto" ]
+ proto_out_dir = "api/test/metrics/proto"
+ cc_generator_options = "lite"
+ }
+}
+
+rtc_library("metrics_set_proto_file_exporter") {
+ visibility = [ "*" ]
+ testonly = true
+ sources = [
+ "metrics_set_proto_file_exporter.cc",
+ "metrics_set_proto_file_exporter.h",
+ ]
+ deps = [
+ ":metric",
+ ":metrics_exporter",
+ "../..:array_view",
+ "../../../rtc_base:logging",
+ "../../../test:fileutils",
+ ]
+
+ if (rtc_enable_protobuf) {
+ deps += [ ":metric_proto" ]
+ }
+}
+
if (rtc_include_tests) {
rtc_library("stdout_metrics_exporter_test") {
testonly = true
@@ -109,4 +145,20 @@
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
+
+ if (rtc_enable_protobuf) {
+ rtc_library("metrics_set_proto_file_exporter_test") {
+ testonly = true
+ sources = [ "metrics_set_proto_file_exporter_test.cc" ]
+ deps = [
+ ":metric",
+ ":metric_proto",
+ ":metrics_set_proto_file_exporter",
+ "../../../rtc_base:protobuf_utils",
+ "../../../test:fileutils",
+ "../../../test:test_support",
+ "../../units:timestamp",
+ ]
+ }
+ }
}
diff --git a/api/test/metrics/metrics_set_proto_file_exporter.cc b/api/test/metrics/metrics_set_proto_file_exporter.cc
new file mode 100644
index 0000000..6a41098
--- /dev/null
+++ b/api/test/metrics/metrics_set_proto_file_exporter.cc
@@ -0,0 +1,157 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "api/test/metrics/metrics_set_proto_file_exporter.h"
+
+#include <stdio.h>
+
+#include <string>
+
+#include "api/test/metrics/metric.h"
+#include "rtc_base/logging.h"
+#include "test/testsupport/file_utils.h"
+
+#if WEBRTC_ENABLE_PROTOBUF
+#include "api/test/metrics/proto/metric.pb.h"
+#endif
+
+namespace webrtc {
+namespace test {
+namespace {
+
+#if WEBRTC_ENABLE_PROTOBUF
+webrtc::test_metrics::Unit ToProtoUnit(Unit unit) {
+ switch (unit) {
+ case Unit::kTimeMs:
+ return webrtc::test_metrics::Unit::MILLISECONDS;
+ case Unit::kPercent:
+ return webrtc::test_metrics::Unit::PERCENT;
+ case Unit::kSizeInBytes:
+ return webrtc::test_metrics::Unit::BYTES;
+ case Unit::kKilobitsPerSecond:
+ return webrtc::test_metrics::Unit::KILOBITS_PER_SECOND;
+ case Unit::kHertz:
+ return webrtc::test_metrics::Unit::HERTZ;
+ case Unit::kUnitless:
+ return webrtc::test_metrics::Unit::UNITLESS;
+ case Unit::kCount:
+ return webrtc::test_metrics::Unit::COUNT;
+ }
+}
+
+webrtc::test_metrics::ImprovementDirection ToProtoImprovementDirection(
+ ImprovementDirection direction) {
+ switch (direction) {
+ case ImprovementDirection::kBiggerIsBetter:
+ return webrtc::test_metrics::ImprovementDirection::BIGGER_IS_BETTER;
+ case ImprovementDirection::kNeitherIsBetter:
+ return webrtc::test_metrics::ImprovementDirection::NEITHER_IS_BETTER;
+ case ImprovementDirection::kSmallerIsBetter:
+ return webrtc::test_metrics::ImprovementDirection::SMALLER_IS_BETTER;
+ }
+}
+
+void SetTimeSeries(
+ const Metric::TimeSeries& time_series,
+ webrtc::test_metrics::Metric::TimeSeries* proto_time_series) {
+ for (const Metric::TimeSeries::Sample& sample : time_series.samples) {
+ webrtc::test_metrics::Metric::TimeSeries::Sample* proto_sample =
+ proto_time_series->add_samples();
+ proto_sample->set_value(sample.value);
+ proto_sample->set_timestamp_us(sample.timestamp.us());
+ for (const auto& [key, value] : sample.sample_metadata) {
+ proto_sample->mutable_sample_metadata()->insert({key, value});
+ }
+ }
+}
+
+void SetStats(const Metric::Stats& stats,
+ webrtc::test_metrics::Metric::Stats* proto_stats) {
+ if (stats.mean.has_value()) {
+ proto_stats->set_mean(*stats.mean);
+ }
+ if (stats.stddev.has_value()) {
+ proto_stats->set_stddev(*stats.stddev);
+ }
+ if (stats.min.has_value()) {
+ proto_stats->set_min(*stats.min);
+ }
+ if (stats.max.has_value()) {
+ proto_stats->set_max(*stats.max);
+ }
+}
+
+bool WriteMetricsToFile(const std::string& path,
+ const webrtc::test_metrics::MetricsSet& metrics_set) {
+ std::string data;
+ bool ok = metrics_set.SerializeToString(&data);
+ if (!ok) {
+ RTC_LOG(LS_ERROR) << "Failed to serialize histogram set to string";
+ return false;
+ }
+
+ CreateDir(DirName(path));
+ FILE* output = fopen(path.c_str(), "wb");
+ if (output == NULL) {
+ RTC_LOG(LS_ERROR) << "Failed to write to " << path;
+ return false;
+ }
+ size_t written = fwrite(data.c_str(), sizeof(char), data.size(), output);
+ fclose(output);
+
+ if (written != data.size()) {
+ size_t expected = data.size();
+ RTC_LOG(LS_ERROR) << "Wrote " << written << ", tried to write " << expected;
+ return false;
+ }
+ return true;
+}
+#endif // WEBRTC_ENABLE_PROTOBUF
+
+} // namespace
+
+MetricsSetProtoFileExporter::Options::Options(
+ absl::string_view export_file_path)
+ : export_file_path(export_file_path) {}
+MetricsSetProtoFileExporter::Options::Options(
+ absl::string_view export_file_path,
+ bool export_whole_time_series)
+ : export_file_path(export_file_path),
+ export_whole_time_series(export_whole_time_series) {}
+
+bool MetricsSetProtoFileExporter::Export(rtc::ArrayView<const Metric> metrics) {
+#if WEBRTC_ENABLE_PROTOBUF
+ webrtc::test_metrics::MetricsSet metrics_set;
+ for (const Metric& metric : metrics) {
+ webrtc::test_metrics::Metric* metric_proto = metrics_set.add_metrics();
+ metric_proto->set_name(metric.name);
+ metric_proto->set_unit(ToProtoUnit(metric.unit));
+ metric_proto->set_improvement_direction(
+ ToProtoImprovementDirection(metric.improvement_direction));
+ metric_proto->set_test_case(metric.test_case);
+ for (const auto& [key, value] : metric.metric_metadata) {
+ metric_proto->mutable_metric_metadata()->insert({key, value});
+ }
+
+ if (options_.export_whole_time_series) {
+ SetTimeSeries(metric.time_series, metric_proto->mutable_time_series());
+ }
+ SetStats(metric.stats, metric_proto->mutable_stats());
+ }
+
+ return WriteMetricsToFile(options_.export_file_path, metrics_set);
+#else
+ RTC_LOG(LS_ERROR)
+ << "Compile with protobuf support to properly use this class";
+ return false;
+#endif
+}
+
+} // namespace test
+} // namespace webrtc
diff --git a/api/test/metrics/metrics_set_proto_file_exporter.h b/api/test/metrics/metrics_set_proto_file_exporter.h
new file mode 100644
index 0000000..f996e9e7
--- /dev/null
+++ b/api/test/metrics/metrics_set_proto_file_exporter.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_TEST_METRICS_METRICS_SET_PROTO_FILE_EXPORTER_H_
+#define API_TEST_METRICS_METRICS_SET_PROTO_FILE_EXPORTER_H_
+
+#include <string>
+
+#include "api/array_view.h"
+#include "api/test/metrics/metric.h"
+#include "api/test/metrics/metrics_exporter.h"
+
+namespace webrtc {
+namespace test {
+
+// Exports all collected metrics to the proto file using
+// `webrtc::test_metrics::MetricsSet` format.
+class MetricsSetProtoFileExporter : public MetricsExporter {
+ public:
+ struct Options {
+ explicit Options(absl::string_view export_file_path);
+ Options(absl::string_view export_file_path, bool export_whole_time_series);
+
+ // File to export proto.
+ std::string export_file_path;
+ // If true will write all time series values to the output proto file,
+ // otherwise will write stats only.
+ bool export_whole_time_series = true;
+ };
+
+ explicit MetricsSetProtoFileExporter(const Options& options)
+ : options_(options) {}
+
+ MetricsSetProtoFileExporter(const MetricsSetProtoFileExporter&) = delete;
+ MetricsSetProtoFileExporter& operator=(const MetricsSetProtoFileExporter&) =
+ delete;
+
+ bool Export(rtc::ArrayView<const Metric> metrics) override;
+
+ private:
+ const Options options_;
+};
+
+} // namespace test
+} // namespace webrtc
+
+#endif // API_TEST_METRICS_METRICS_SET_PROTO_FILE_EXPORTER_H_
diff --git a/api/test/metrics/metrics_set_proto_file_exporter_test.cc b/api/test/metrics/metrics_set_proto_file_exporter_test.cc
new file mode 100644
index 0000000..97987d2
--- /dev/null
+++ b/api/test/metrics/metrics_set_proto_file_exporter_test.cc
@@ -0,0 +1,151 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "api/test/metrics/metrics_set_proto_file_exporter.h"
+
+#include <fstream>
+#include <map>
+#include <string>
+#include <vector>
+
+#include "api/test/metrics/metric.h"
+#include "api/test/metrics/proto/metric.pb.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/protobuf_utils.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/testsupport/file_utils.h"
+
+namespace webrtc {
+namespace test {
+namespace {
+
+using ::testing::Eq;
+using ::testing::Test;
+
+namespace proto = ::webrtc::test_metrics;
+
+std::string ReadFileAsString(const std::string& filename) {
+ std::ifstream infile(filename, std::ios_base::binary);
+ auto buffer = std::vector<char>(std::istreambuf_iterator<char>(infile),
+ std::istreambuf_iterator<char>());
+ return std::string(buffer.begin(), buffer.end());
+}
+
+std::map<std::string, std::string> DefaultMetadata() {
+ return std::map<std::string, std::string>{{"key", "value"}};
+}
+
+Metric::TimeSeries::Sample Sample(double value) {
+ return Metric::TimeSeries::Sample{.timestamp = Timestamp::Seconds(1),
+ .value = value,
+ .sample_metadata = DefaultMetadata()};
+}
+
+void AssertSamplesEqual(const proto::Metric::TimeSeries::Sample& actual_sample,
+ const Metric::TimeSeries::Sample& expected_sample) {
+ EXPECT_THAT(actual_sample.value(), Eq(expected_sample.value));
+ EXPECT_THAT(actual_sample.timestamp_us(), Eq(expected_sample.timestamp.us()));
+ EXPECT_THAT(actual_sample.sample_metadata().size(),
+ Eq(expected_sample.sample_metadata.size()));
+ for (const auto& [key, value] : expected_sample.sample_metadata) {
+ EXPECT_THAT(actual_sample.sample_metadata().at(key), Eq(value));
+ }
+}
+
+class MetricsSetProtoFileExporterTest : public Test {
+ protected:
+ ~MetricsSetProtoFileExporterTest() override = default;
+
+ void SetUp() override {
+ temp_filename_ = webrtc::test::TempFilename(
+ webrtc::test::OutputPath(), "metrics_set_proto_file_exporter_test");
+ }
+
+ void TearDown() override {
+ ASSERT_TRUE(webrtc::test::RemoveFile(temp_filename_));
+ }
+
+ std::string temp_filename_;
+};
+
+TEST_F(MetricsSetProtoFileExporterTest, MetricsAreExportedCorrectly) {
+ MetricsSetProtoFileExporter::Options options(temp_filename_);
+ MetricsSetProtoFileExporter exporter(options);
+
+ Metric metric1{
+ .name = "test_metric1",
+ .unit = Unit::kTimeMs,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .test_case = "test_case_name1",
+ .metric_metadata = DefaultMetadata(),
+ .time_series =
+ Metric::TimeSeries{.samples = std::vector{Sample(10), Sample(20)}},
+ .stats =
+ Metric::Stats{.mean = 15.0, .stddev = 5.0, .min = 10.0, .max = 20.0}};
+ Metric metric2{
+ .name = "test_metric2",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .test_case = "test_case_name2",
+ .metric_metadata = DefaultMetadata(),
+ .time_series =
+ Metric::TimeSeries{.samples = std::vector{Sample(20), Sample(40)}},
+ .stats = Metric::Stats{
+ .mean = 30.0, .stddev = 10.0, .min = 20.0, .max = 40.0}};
+
+ ASSERT_TRUE(exporter.Export(std::vector<Metric>{metric1, metric2}));
+ webrtc::test_metrics::MetricsSet actual_metrics_set;
+ actual_metrics_set.ParseFromString(ReadFileAsString(temp_filename_));
+ EXPECT_THAT(actual_metrics_set.metrics().size(), Eq(2));
+
+ EXPECT_THAT(actual_metrics_set.metrics(0).name(), Eq("test_metric1"));
+ EXPECT_THAT(actual_metrics_set.metrics(0).test_case(), Eq("test_case_name1"));
+ EXPECT_THAT(actual_metrics_set.metrics(0).unit(),
+ Eq(proto::Unit::MILLISECONDS));
+ EXPECT_THAT(actual_metrics_set.metrics(0).improvement_direction(),
+ Eq(proto::ImprovementDirection::BIGGER_IS_BETTER));
+ EXPECT_THAT(actual_metrics_set.metrics(0).metric_metadata().size(), Eq(1lu));
+ EXPECT_THAT(actual_metrics_set.metrics(0).metric_metadata().at("key"),
+ Eq("value"));
+ EXPECT_THAT(actual_metrics_set.metrics(0).time_series().samples().size(),
+ Eq(2));
+ AssertSamplesEqual(actual_metrics_set.metrics(0).time_series().samples(0),
+ Sample(10.0));
+ AssertSamplesEqual(actual_metrics_set.metrics(0).time_series().samples(1),
+ Sample(20.0));
+ EXPECT_THAT(actual_metrics_set.metrics(0).stats().mean(), Eq(15.0));
+ EXPECT_THAT(actual_metrics_set.metrics(0).stats().stddev(), Eq(5.0));
+ EXPECT_THAT(actual_metrics_set.metrics(0).stats().min(), Eq(10.0));
+ EXPECT_THAT(actual_metrics_set.metrics(0).stats().max(), Eq(20.0));
+
+ EXPECT_THAT(actual_metrics_set.metrics(1).name(), Eq("test_metric2"));
+ EXPECT_THAT(actual_metrics_set.metrics(1).test_case(), Eq("test_case_name2"));
+ EXPECT_THAT(actual_metrics_set.metrics(1).unit(),
+ Eq(proto::Unit::KILOBITS_PER_SECOND));
+ EXPECT_THAT(actual_metrics_set.metrics(1).improvement_direction(),
+ Eq(proto::ImprovementDirection::SMALLER_IS_BETTER));
+ EXPECT_THAT(actual_metrics_set.metrics(1).metric_metadata().size(), Eq(1lu));
+ EXPECT_THAT(actual_metrics_set.metrics(1).metric_metadata().at("key"),
+ Eq("value"));
+ EXPECT_THAT(actual_metrics_set.metrics(1).time_series().samples().size(),
+ Eq(2));
+ AssertSamplesEqual(actual_metrics_set.metrics(1).time_series().samples(0),
+ Sample(20.0));
+ AssertSamplesEqual(actual_metrics_set.metrics(1).time_series().samples(1),
+ Sample(40.0));
+ EXPECT_THAT(actual_metrics_set.metrics(1).stats().mean(), Eq(30.0));
+ EXPECT_THAT(actual_metrics_set.metrics(1).stats().stddev(), Eq(10.0));
+ EXPECT_THAT(actual_metrics_set.metrics(1).stats().min(), Eq(20.0));
+ EXPECT_THAT(actual_metrics_set.metrics(1).stats().max(), Eq(40.0));
+}
+
+} // namespace
+} // namespace test
+} // namespace webrtc
diff --git a/api/test/metrics/proto/metric.proto b/api/test/metrics/proto/metric.proto
new file mode 100644
index 0000000..cdd3125
--- /dev/null
+++ b/api/test/metrics/proto/metric.proto
@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+syntax = "proto3";
+
+package webrtc.test_metrics;
+
+// Root message of the proto file. Contains collection of all the metrics.
+message MetricsSet {
+ repeated Metric metrics = 1;
+}
+
+enum Unit {
+ // Default value that has to be defined.
+ UNDEFINED_UNIT = 0;
+ // General unitless value. Can be used either for dimensionless quantities
+ // (ex ratio) or for units not presented in this enum and too specific to add
+ // to this enum.
+ UNITLESS = 1;
+ MILLISECONDS = 2;
+ PERCENT = 3;
+ BYTES = 4;
+ KILOBITS_PER_SECOND = 5;
+ HERTZ = 6;
+ COUNT = 7;
+}
+
+enum ImprovementDirection {
+ // Default value that has to be defined.
+ UNDEFINED_IMPROVEMENT_DIRECTION = 0;
+ BIGGER_IS_BETTER = 1;
+ NEITHER_IS_BETTER = 2;
+ SMALLER_IS_BETTER = 3;
+}
+
+// Single performance metric with all related metadata.
+message Metric {
+ // Metric name, for example PSNR, SSIM, decode_time, etc.
+ string name = 1;
+ Unit unit = 2;
+ ImprovementDirection improvement_direction = 3;
+ // If the metric is generated by a test, this field can be used to specify
+ // this information.
+ string test_case = 4;
+ // Metadata associated with the whole metric.
+ map<string, string> metric_metadata = 5;
+
+ message TimeSeries {
+ message Sample {
+ // Timestamp in microseconds associated with a sample. For example,
+ // the timestamp when the sample was collected.
+ int64 timestamp_us = 1;
+ double value = 2;
+ // Metadata associated with this particular sample.
+ map<string, string> sample_metadata = 3;
+ }
+ // All samples collected for this metric. It can be empty if the Metric
+ // object only contains `stats`.
+ repeated Sample samples = 1;
+ }
+ // Contains all samples of the metric collected during test execution.
+ // It can be empty if the user only stores precomputed statistics into
+ // `stats`.
+ TimeSeries time_series = 6;
+
+ // Contains metric's precomputed statistics based on the `time_series` or if
+ // `time_series` is omitted (has 0 samples) contains precomputed statistics
+ // provided by the metric's calculator.
+ message Stats {
+ // Sample mean of the metric
+ // (https://en.wikipedia.org/wiki/Sample_mean_and_covariance).
+ optional double mean = 1;
+ // Standard deviation (https://en.wikipedia.org/wiki/Standard_deviation).
+ // Is undefined if `time_series` contains only a single sample.
+ optional double stddev = 2;
+ optional double min = 3;
+ optional double max = 4;
+ }
+ Stats stats = 7;
+}
