Introduce MetricsAccumulator
Bug: b/246095034
Change-Id: Ic267254245399238d3eece421e4e4e72134dd0e9
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276740
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38203}
diff --git a/api/test/metrics/BUILD.gn b/api/test/metrics/BUILD.gn
index 64e7c1c..b635cf2 100644
--- a/api/test/metrics/BUILD.gn
+++ b/api/test/metrics/BUILD.gn
@@ -15,6 +15,7 @@
deps = [
":global_metrics_logger_and_exporter",
":metric",
+ ":metrics_accumulator",
":metrics_exporter",
":metrics_logger",
":metrics_logger_and_exporter",
@@ -28,6 +29,7 @@
deps = [
":global_metrics_logger_and_exporter_test",
+ ":metrics_accumulator_test",
":metrics_logger_and_exporter_test",
":metrics_logger_test",
":print_result_proxy_metrics_exporter_test",
@@ -61,6 +63,7 @@
]
deps = [
":metric",
+ ":metrics_accumulator",
"../..:array_view",
"../../../rtc_base/synchronization:mutex",
"../../../system_wrappers",
@@ -69,6 +72,22 @@
absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
}
+rtc_library("metrics_accumulator") {
+ visibility = [ "*" ]
+ sources = [
+ "metrics_accumulator.cc",
+ "metrics_accumulator.h",
+ ]
+ deps = [
+ ":metric",
+ "../../../rtc_base:macromagic",
+ "../../../rtc_base/synchronization:mutex",
+ "../../numerics",
+ "../../units:timestamp",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
+}
+
rtc_library("metrics_exporter") {
visibility = [ "*" ]
sources = [ "metrics_exporter.h" ]
@@ -209,6 +228,18 @@
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
+ rtc_library("metrics_accumulator_test") {
+ testonly = true
+ sources = [ "metrics_accumulator_test.cc" ]
+ deps = [
+ ":metric",
+ ":metrics_accumulator",
+ "../../../test:test_support",
+ "../../units:timestamp",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+ }
+
rtc_library("stdout_metrics_exporter_test") {
testonly = true
sources = [ "stdout_metrics_exporter_test.cc" ]
diff --git a/api/test/metrics/DEPS b/api/test/metrics/DEPS
index a1c845b..74889c6 100644
--- a/api/test/metrics/DEPS
+++ b/api/test/metrics/DEPS
@@ -7,4 +7,8 @@
"+rtc_base/synchronization/mutex.h",
"+system_wrappers/include/clock.h",
],
+ "metrics_accumulator\.h": [
+ "+rtc_base/synchronization/mutex.h",
+ "+rtc_base/thread_annotations.h",
+ ],
}
diff --git a/api/test/metrics/global_metrics_logger_and_exporter.cc b/api/test/metrics/global_metrics_logger_and_exporter.cc
index 915dcac..9c3c897 100644
--- a/api/test/metrics/global_metrics_logger_and_exporter.cc
+++ b/api/test/metrics/global_metrics_logger_and_exporter.cc
@@ -22,8 +22,8 @@
namespace webrtc {
namespace test {
-MetricsLogger* GetGlobalMetricsLogger() {
- static MetricsLogger* logger_ =
+DefaultMetricsLogger* GetGlobalMetricsLogger() {
+ static DefaultMetricsLogger* logger_ =
new DefaultMetricsLogger(Clock::GetRealTimeClock());
return logger_;
}
diff --git a/api/test/metrics/global_metrics_logger_and_exporter.h b/api/test/metrics/global_metrics_logger_and_exporter.h
index 3f9bcec..42bdf93 100644
--- a/api/test/metrics/global_metrics_logger_and_exporter.h
+++ b/api/test/metrics/global_metrics_logger_and_exporter.h
@@ -21,7 +21,7 @@
namespace test {
// Returns non-null global `MetricsLogger` to log metrics.
-MetricsLogger* GetGlobalMetricsLogger();
+DefaultMetricsLogger* GetGlobalMetricsLogger();
bool ExportPerfMetric(MetricsLogger& logger,
std::vector<std::unique_ptr<MetricsExporter>> exporters);
diff --git a/api/test/metrics/metrics_accumulator.cc b/api/test/metrics/metrics_accumulator.cc
new file mode 100644
index 0000000..c34396b
--- /dev/null
+++ b/api/test/metrics/metrics_accumulator.cc
@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "api/test/metrics/metrics_accumulator.h"
+
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "api/numerics/samples_stats_counter.h"
+#include "api/test/metrics/metric.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/synchronization/mutex.h"
+
+namespace webrtc {
+namespace test {
+namespace {
+
+Metric::Stats ToStats(const SamplesStatsCounter& values) {
+ if (values.IsEmpty()) {
+ return Metric::Stats();
+ }
+ return Metric::Stats{.mean = values.GetAverage(),
+ .stddev = values.GetStandardDeviation(),
+ .min = values.GetMin(),
+ .max = values.GetMax()};
+}
+
+Metric SetTimeseries(const Metric& prototype,
+ const SamplesStatsCounter& counter) {
+ Metric output(prototype);
+ Metric::TimeSeries time_series;
+ for (const SamplesStatsCounter::StatsSample& sample :
+ counter.GetTimedSamples()) {
+ time_series.samples.push_back(
+ Metric::TimeSeries::Sample{.timestamp = sample.time,
+ .value = sample.value,
+ .sample_metadata = sample.metadata});
+ }
+ output.time_series = std::move(time_series);
+ output.stats = ToStats(counter);
+ return output;
+}
+
+} // namespace
+
+bool operator<(const MetricsAccumulator::MetricKey& a,
+ const MetricsAccumulator::MetricKey& b) {
+ if (a.test_case_name < b.test_case_name) {
+ return true;
+ } else if (a.test_case_name > b.test_case_name) {
+ return false;
+ } else {
+ return a.metric_name < b.metric_name;
+ }
+}
+
+bool MetricsAccumulator::AddSample(
+ absl::string_view metric_name,
+ absl::string_view test_case_name,
+ double value,
+ Timestamp timestamp,
+ std::map<std::string, std::string> point_metadata) {
+ MutexLock lock(&mutex_);
+ bool created;
+ MetricValue* metric_value =
+ GetOrCreateMetric(metric_name, test_case_name, &created);
+ metric_value->counter.AddSample(
+ SamplesStatsCounter::StatsSample{.value = value,
+ .time = timestamp,
+ .metadata = std::move(point_metadata)});
+ return created;
+}
+
+bool MetricsAccumulator::AddMetricMetadata(
+ absl::string_view metric_name,
+ absl::string_view test_case_name,
+ Unit unit,
+ ImprovementDirection improvement_direction,
+ std::map<std::string, std::string> metric_metadata) {
+ MutexLock lock(&mutex_);
+ bool created;
+ MetricValue* metric_value =
+ GetOrCreateMetric(metric_name, test_case_name, &created);
+ metric_value->metric.unit = unit;
+ metric_value->metric.improvement_direction = improvement_direction;
+ metric_value->metric.metric_metadata = std::move(metric_metadata);
+ return created;
+}
+
+std::vector<Metric> MetricsAccumulator::GetCollectedMetrics() const {
+ MutexLock lock(&mutex_);
+ std::vector<Metric> out;
+ out.reserve(metrics_.size());
+ for (const auto& [unused_key, metric_value] : metrics_) {
+ out.push_back(SetTimeseries(metric_value.metric, metric_value.counter));
+ }
+ return out;
+}
+
+MetricsAccumulator::MetricValue* MetricsAccumulator::GetOrCreateMetric(
+ absl::string_view metric_name,
+ absl::string_view test_case_name,
+ bool* created) {
+ MetricKey key(metric_name, test_case_name);
+ auto it = metrics_.find(key);
+ if (it != metrics_.end()) {
+ *created = false;
+ return &it->second;
+ }
+ *created = true;
+
+ Metric metric{
+ .name = key.metric_name,
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .test_case = key.test_case_name,
+ };
+ return &metrics_.emplace(key, MetricValue{.metric = std::move(metric)})
+ .first->second;
+}
+
+} // namespace test
+} // namespace webrtc
diff --git a/api/test/metrics/metrics_accumulator.h b/api/test/metrics/metrics_accumulator.h
new file mode 100644
index 0000000..c75bd94
--- /dev/null
+++ b/api/test/metrics/metrics_accumulator.h
@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_TEST_METRICS_METRICS_ACCUMULATOR_H_
+#define API_TEST_METRICS_METRICS_ACCUMULATOR_H_
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "api/numerics/samples_stats_counter.h"
+#include "api/test/metrics/metric.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "rtc_base/thread_annotations.h"
+
+namespace webrtc {
+namespace test {
+
+// Accumulates metrics' samples internally and provides API to get collected
+// ones.
+//
+// This object is thread safe.
+class MetricsAccumulator {
+ public:
+ MetricsAccumulator() = default;
+
+ // Adds sample for the specified `metric_name` within specified
+ // `test_case_name`. If it is the first time when this combination of
+ // `metric_name` and `test_case_name` is used, creates a new Metric to collect
+ // samples, otherwise adds a sample to the previously created Metric.
+ //
+ // By default metric will use `Unit::kUnitless` and
+ // `ImprovementDirection::kNeitherIsBetter`.
+ //
+ // `point_metadata` - the metadata to be added to the single data point that
+ // this method adds to the Metric (it is not a metric global metadata).
+ //
+ // Returns true if a new metric was created and false otherwise.
+ bool AddSample(absl::string_view metric_name,
+ absl::string_view test_case_name,
+ double value,
+ Timestamp timestamp,
+ std::map<std::string, std::string> point_metadata = {});
+
+ // Adds metadata to the metric specified by `metric_name` within specified
+ // `test_case_name`. If such a metric doesn't exist, creates a new one,
+ // otherwise overrides previously recorded values.
+ //
+ // Returns true if a new metric was created and false otherwise.
+ bool AddMetricMetadata(
+ absl::string_view metric_name,
+ absl::string_view test_case_name,
+ Unit unit,
+ ImprovementDirection improvement_direction,
+ std::map<std::string, std::string> metric_metadata = {});
+
+ // Returns all metrics collected by this accumulator. No order guarantees
+ // provided.
+ std::vector<Metric> GetCollectedMetrics() const;
+
+ private:
+ struct MetricKey {
+ MetricKey(absl::string_view metric_name, absl::string_view test_case_name)
+ : metric_name(metric_name), test_case_name(test_case_name) {}
+
+ std::string metric_name;
+ std::string test_case_name;
+ };
+ friend bool operator<(const MetricKey& a, const MetricKey& b);
+
+ struct MetricValue {
+ SamplesStatsCounter counter;
+ Metric metric;
+ };
+
+ // Gets existing metrics or creates a new one. If metric was created `created`
+ // will be set to true.
+ MetricValue* GetOrCreateMetric(absl::string_view metric_name,
+ absl::string_view test_case_name,
+ bool* created)
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+
+ mutable Mutex mutex_;
+ std::map<MetricKey, MetricValue> metrics_ RTC_GUARDED_BY(mutex_);
+};
+
+} // namespace test
+} // namespace webrtc
+
+#endif // API_TEST_METRICS_METRICS_ACCUMULATOR_H_
diff --git a/api/test/metrics/metrics_accumulator_test.cc b/api/test/metrics/metrics_accumulator_test.cc
new file mode 100644
index 0000000..677f523
--- /dev/null
+++ b/api/test/metrics/metrics_accumulator_test.cc
@@ -0,0 +1,315 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "api/test/metrics/metrics_accumulator.h"
+
+#include <map>
+#include <vector>
+
+#include "api/test/metrics/metric.h"
+#include "api/units/timestamp.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace test {
+namespace {
+
+using ::testing::Eq;
+using ::testing::IsEmpty;
+using ::testing::SizeIs;
+
+TEST(MetricsAccumulatorTest, AddSampleToTheNewMetricWillCreateOne) {
+ MetricsAccumulator accumulator;
+ ASSERT_TRUE(accumulator.AddSample(
+ "metric_name", "test_case_name",
+ /*value=*/10, Timestamp::Seconds(1),
+ /*point_metadata=*/std::map<std::string, std::string>{{"key", "value"}}));
+
+ std::vector<Metric> metrics = accumulator.GetCollectedMetrics();
+ ASSERT_THAT(metrics, SizeIs(1));
+ const Metric& metric = metrics[0];
+ EXPECT_THAT(metric.name, Eq("metric_name"));
+ EXPECT_THAT(metric.test_case, Eq("test_case_name"));
+ EXPECT_THAT(metric.unit, Eq(Unit::kUnitless));
+ EXPECT_THAT(metric.improvement_direction,
+ Eq(ImprovementDirection::kNeitherIsBetter));
+ EXPECT_THAT(metric.metric_metadata, IsEmpty());
+ ASSERT_THAT(metric.time_series.samples, SizeIs(1));
+ EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0));
+ EXPECT_THAT(metric.time_series.samples[0].timestamp,
+ Eq(Timestamp::Seconds(1)));
+ EXPECT_THAT(metric.time_series.samples[0].sample_metadata,
+ Eq(std::map<std::string, std::string>{{"key", "value"}}));
+ ASSERT_THAT(metric.stats.mean, absl::optional<double>(10.0));
+ ASSERT_THAT(metric.stats.stddev, absl::optional<double>(0.0));
+ ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0));
+ ASSERT_THAT(metric.stats.max, absl::optional<double>(10.0));
+}
+
+TEST(MetricsAccumulatorTest, AddSamplesToExistingMetricWontCreateNewOne) {
+ MetricsAccumulator accumulator;
+ ASSERT_TRUE(accumulator.AddSample(
+ "metric_name", "test_case_name",
+ /*value=*/10, Timestamp::Seconds(1),
+ /*point_metadata=*/
+ std::map<std::string, std::string>{{"key1", "value1"}}));
+ ASSERT_FALSE(accumulator.AddSample(
+ "metric_name", "test_case_name",
+ /*value=*/20, Timestamp::Seconds(2),
+ /*point_metadata=*/
+ std::map<std::string, std::string>{{"key2", "value2"}}));
+
+ std::vector<Metric> metrics = accumulator.GetCollectedMetrics();
+ ASSERT_THAT(metrics, SizeIs(1));
+ const Metric& metric = metrics[0];
+ EXPECT_THAT(metric.name, Eq("metric_name"));
+ EXPECT_THAT(metric.test_case, Eq("test_case_name"));
+ EXPECT_THAT(metric.unit, Eq(Unit::kUnitless));
+ EXPECT_THAT(metric.improvement_direction,
+ Eq(ImprovementDirection::kNeitherIsBetter));
+ EXPECT_THAT(metric.metric_metadata, IsEmpty());
+ ASSERT_THAT(metric.time_series.samples, SizeIs(2));
+ EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0));
+ EXPECT_THAT(metric.time_series.samples[0].timestamp,
+ Eq(Timestamp::Seconds(1)));
+ EXPECT_THAT(metric.time_series.samples[0].sample_metadata,
+ Eq(std::map<std::string, std::string>{{"key1", "value1"}}));
+ EXPECT_THAT(metric.time_series.samples[1].value, Eq(20.0));
+ EXPECT_THAT(metric.time_series.samples[1].timestamp,
+ Eq(Timestamp::Seconds(2)));
+ EXPECT_THAT(metric.time_series.samples[1].sample_metadata,
+ Eq(std::map<std::string, std::string>{{"key2", "value2"}}));
+ ASSERT_THAT(metric.stats.mean, absl::optional<double>(15.0));
+ ASSERT_THAT(metric.stats.stddev, absl::optional<double>(5.0));
+ ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0));
+ ASSERT_THAT(metric.stats.max, absl::optional<double>(20.0));
+}
+
+TEST(MetricsAccumulatorTest, AddSampleToDifferentMetricsWillCreateBoth) {
+ MetricsAccumulator accumulator;
+ ASSERT_TRUE(accumulator.AddSample(
+ "metric_name1", "test_case_name1",
+ /*value=*/10, Timestamp::Seconds(1),
+ /*point_metadata=*/
+ std::map<std::string, std::string>{{"key1", "value1"}}));
+ ASSERT_TRUE(accumulator.AddSample(
+ "metric_name2", "test_case_name2",
+ /*value=*/20, Timestamp::Seconds(2),
+ /*point_metadata=*/
+ std::map<std::string, std::string>{{"key2", "value2"}}));
+
+ std::vector<Metric> metrics = accumulator.GetCollectedMetrics();
+ ASSERT_THAT(metrics, SizeIs(2));
+ EXPECT_THAT(metrics[0].name, Eq("metric_name1"));
+ EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1"));
+ EXPECT_THAT(metrics[0].unit, Eq(Unit::kUnitless));
+ EXPECT_THAT(metrics[0].improvement_direction,
+ Eq(ImprovementDirection::kNeitherIsBetter));
+ EXPECT_THAT(metrics[0].metric_metadata, IsEmpty());
+ ASSERT_THAT(metrics[0].time_series.samples, SizeIs(1));
+ EXPECT_THAT(metrics[0].time_series.samples[0].value, Eq(10.0));
+ EXPECT_THAT(metrics[0].time_series.samples[0].timestamp,
+ Eq(Timestamp::Seconds(1)));
+ EXPECT_THAT(metrics[0].time_series.samples[0].sample_metadata,
+ Eq(std::map<std::string, std::string>{{"key1", "value1"}}));
+ ASSERT_THAT(metrics[0].stats.mean, absl::optional<double>(10.0));
+ ASSERT_THAT(metrics[0].stats.stddev, absl::optional<double>(0.0));
+ ASSERT_THAT(metrics[0].stats.min, absl::optional<double>(10.0));
+ ASSERT_THAT(metrics[0].stats.max, absl::optional<double>(10.0));
+ EXPECT_THAT(metrics[1].name, Eq("metric_name2"));
+ EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2"));
+ EXPECT_THAT(metrics[1].unit, Eq(Unit::kUnitless));
+ EXPECT_THAT(metrics[1].improvement_direction,
+ Eq(ImprovementDirection::kNeitherIsBetter));
+ EXPECT_THAT(metrics[1].metric_metadata, IsEmpty());
+ ASSERT_THAT(metrics[1].time_series.samples, SizeIs(1));
+ EXPECT_THAT(metrics[1].time_series.samples[0].value, Eq(20.0));
+ EXPECT_THAT(metrics[1].time_series.samples[0].timestamp,
+ Eq(Timestamp::Seconds(2)));
+ EXPECT_THAT(metrics[1].time_series.samples[0].sample_metadata,
+ Eq(std::map<std::string, std::string>{{"key2", "value2"}}));
+ ASSERT_THAT(metrics[1].stats.mean, absl::optional<double>(20.0));
+ ASSERT_THAT(metrics[1].stats.stddev, absl::optional<double>(0.0));
+ ASSERT_THAT(metrics[1].stats.min, absl::optional<double>(20.0));
+ ASSERT_THAT(metrics[1].stats.max, absl::optional<double>(20.0));
+}
+
+TEST(MetricsAccumulatorTest, AddMetadataToTheNewMetricWillCreateOne) {
+ MetricsAccumulator accumulator;
+ ASSERT_TRUE(accumulator.AddMetricMetadata(
+ "metric_name", "test_case_name", Unit::kMilliseconds,
+ ImprovementDirection::kBiggerIsBetter,
+ /*metric_metadata=*/
+ std::map<std::string, std::string>{{"key", "value"}}));
+
+ std::vector<Metric> metrics = accumulator.GetCollectedMetrics();
+ ASSERT_THAT(metrics, SizeIs(1));
+ const Metric& metric = metrics[0];
+ EXPECT_THAT(metric.name, Eq("metric_name"));
+ EXPECT_THAT(metric.test_case, Eq("test_case_name"));
+ EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds));
+ EXPECT_THAT(metric.improvement_direction,
+ Eq(ImprovementDirection::kBiggerIsBetter));
+ EXPECT_THAT(metric.metric_metadata,
+ Eq(std::map<std::string, std::string>{{"key", "value"}}));
+ ASSERT_THAT(metric.time_series.samples, IsEmpty());
+ ASSERT_THAT(metric.stats.mean, absl::nullopt);
+ ASSERT_THAT(metric.stats.stddev, absl::nullopt);
+ ASSERT_THAT(metric.stats.min, absl::nullopt);
+ ASSERT_THAT(metric.stats.max, absl::nullopt);
+}
+
+TEST(MetricsAccumulatorTest,
+ AddMetadataToTheExistingMetricWillOverwriteValues) {
+ MetricsAccumulator accumulator;
+ ASSERT_TRUE(accumulator.AddMetricMetadata(
+ "metric_name", "test_case_name", Unit::kMilliseconds,
+ ImprovementDirection::kBiggerIsBetter,
+ /*metric_metadata=*/
+ std::map<std::string, std::string>{{"key1", "value1"}}));
+
+ ASSERT_FALSE(accumulator.AddMetricMetadata(
+ "metric_name", "test_case_name", Unit::kBytes,
+ ImprovementDirection::kSmallerIsBetter,
+ /*metric_metadata=*/
+ std::map<std::string, std::string>{{"key2", "value2"}}));
+
+ std::vector<Metric> metrics = accumulator.GetCollectedMetrics();
+ ASSERT_THAT(metrics, SizeIs(1));
+ const Metric& metric = metrics[0];
+ EXPECT_THAT(metric.name, Eq("metric_name"));
+ EXPECT_THAT(metric.test_case, Eq("test_case_name"));
+ EXPECT_THAT(metric.unit, Eq(Unit::kBytes));
+ EXPECT_THAT(metric.improvement_direction,
+ Eq(ImprovementDirection::kSmallerIsBetter));
+ EXPECT_THAT(metric.metric_metadata,
+ Eq(std::map<std::string, std::string>{{"key2", "value2"}}));
+ ASSERT_THAT(metric.time_series.samples, IsEmpty());
+ ASSERT_THAT(metric.stats.mean, absl::nullopt);
+ ASSERT_THAT(metric.stats.stddev, absl::nullopt);
+ ASSERT_THAT(metric.stats.min, absl::nullopt);
+ ASSERT_THAT(metric.stats.max, absl::nullopt);
+}
+
+TEST(MetricsAccumulatorTest, AddMetadataToDifferentMetricsWillCreateBoth) {
+ MetricsAccumulator accumulator;
+ ASSERT_TRUE(accumulator.AddMetricMetadata(
+ "metric_name1", "test_case_name1", Unit::kMilliseconds,
+ ImprovementDirection::kBiggerIsBetter,
+ /*metric_metadata=*/
+ std::map<std::string, std::string>{{"key1", "value1"}}));
+
+ ASSERT_TRUE(accumulator.AddMetricMetadata(
+ "metric_name2", "test_case_name2", Unit::kBytes,
+ ImprovementDirection::kSmallerIsBetter,
+ /*metric_metadata=*/
+ std::map<std::string, std::string>{{"key2", "value2"}}));
+
+ std::vector<Metric> metrics = accumulator.GetCollectedMetrics();
+ ASSERT_THAT(metrics, SizeIs(2));
+ EXPECT_THAT(metrics[0].name, Eq("metric_name1"));
+ EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1"));
+ EXPECT_THAT(metrics[0].unit, Eq(Unit::kMilliseconds));
+ EXPECT_THAT(metrics[0].improvement_direction,
+ Eq(ImprovementDirection::kBiggerIsBetter));
+ EXPECT_THAT(metrics[0].metric_metadata,
+ Eq(std::map<std::string, std::string>{{"key1", "value1"}}));
+ ASSERT_THAT(metrics[0].time_series.samples, IsEmpty());
+ ASSERT_THAT(metrics[0].stats.mean, absl::nullopt);
+ ASSERT_THAT(metrics[0].stats.stddev, absl::nullopt);
+ ASSERT_THAT(metrics[0].stats.min, absl::nullopt);
+ ASSERT_THAT(metrics[0].stats.max, absl::nullopt);
+ EXPECT_THAT(metrics[1].name, Eq("metric_name2"));
+ EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2"));
+ EXPECT_THAT(metrics[1].unit, Eq(Unit::kBytes));
+ EXPECT_THAT(metrics[1].improvement_direction,
+ Eq(ImprovementDirection::kSmallerIsBetter));
+ EXPECT_THAT(metrics[1].metric_metadata,
+ Eq(std::map<std::string, std::string>{{"key2", "value2"}}));
+ ASSERT_THAT(metrics[1].time_series.samples, IsEmpty());
+ ASSERT_THAT(metrics[1].stats.mean, absl::nullopt);
+ ASSERT_THAT(metrics[1].stats.stddev, absl::nullopt);
+ ASSERT_THAT(metrics[1].stats.min, absl::nullopt);
+ ASSERT_THAT(metrics[1].stats.max, absl::nullopt);
+}
+
+TEST(MetricsAccumulatorTest, AddMetadataAfterAddingSampleWontCreateNewMetric) {
+ MetricsAccumulator accumulator;
+ ASSERT_TRUE(accumulator.AddSample(
+ "metric_name", "test_case_name",
+ /*value=*/10, Timestamp::Seconds(1),
+ /*point_metadata=*/
+ std::map<std::string, std::string>{{"key_s", "value_s"}}));
+ ASSERT_FALSE(accumulator.AddMetricMetadata(
+ "metric_name", "test_case_name", Unit::kMilliseconds,
+ ImprovementDirection::kBiggerIsBetter,
+ /*metric_metadata=*/
+ std::map<std::string, std::string>{{"key_m", "value_m"}}));
+
+ std::vector<Metric> metrics = accumulator.GetCollectedMetrics();
+ ASSERT_THAT(metrics, SizeIs(1));
+ const Metric& metric = metrics[0];
+ EXPECT_THAT(metric.name, Eq("metric_name"));
+ EXPECT_THAT(metric.test_case, Eq("test_case_name"));
+ EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds));
+ EXPECT_THAT(metric.improvement_direction,
+ Eq(ImprovementDirection::kBiggerIsBetter));
+ EXPECT_THAT(metric.metric_metadata,
+ Eq(std::map<std::string, std::string>{{"key_m", "value_m"}}));
+ ASSERT_THAT(metric.time_series.samples, SizeIs(1));
+ EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0));
+ EXPECT_THAT(metric.time_series.samples[0].timestamp,
+ Eq(Timestamp::Seconds(1)));
+ EXPECT_THAT(metric.time_series.samples[0].sample_metadata,
+ Eq(std::map<std::string, std::string>{{"key_s", "value_s"}}));
+ ASSERT_THAT(metric.stats.mean, absl::optional<double>(10.0));
+ ASSERT_THAT(metric.stats.stddev, absl::optional<double>(0.0));
+ ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0));
+ ASSERT_THAT(metric.stats.max, absl::optional<double>(10.0));
+}
+
+TEST(MetricsAccumulatorTest, AddSampleAfterAddingMetadataWontCreateNewMetric) {
+ MetricsAccumulator accumulator;
+ ASSERT_TRUE(accumulator.AddMetricMetadata(
+ "metric_name", "test_case_name", Unit::kMilliseconds,
+ ImprovementDirection::kBiggerIsBetter,
+ /*metric_metadata=*/
+ std::map<std::string, std::string>{{"key_m", "value_m"}}));
+ ASSERT_FALSE(accumulator.AddSample(
+ "metric_name", "test_case_name",
+ /*value=*/10, Timestamp::Seconds(1),
+ /*point_metadata=*/
+ std::map<std::string, std::string>{{"key_s", "value_s"}}));
+
+ std::vector<Metric> metrics = accumulator.GetCollectedMetrics();
+ ASSERT_THAT(metrics, SizeIs(1));
+ const Metric& metric = metrics[0];
+ EXPECT_THAT(metric.name, Eq("metric_name"));
+ EXPECT_THAT(metric.test_case, Eq("test_case_name"));
+ EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds));
+ EXPECT_THAT(metric.improvement_direction,
+ Eq(ImprovementDirection::kBiggerIsBetter));
+ EXPECT_THAT(metric.metric_metadata,
+ Eq(std::map<std::string, std::string>{{"key_m", "value_m"}}));
+ ASSERT_THAT(metric.time_series.samples, SizeIs(1));
+ EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0));
+ EXPECT_THAT(metric.time_series.samples[0].timestamp,
+ Eq(Timestamp::Seconds(1)));
+ EXPECT_THAT(metric.time_series.samples[0].sample_metadata,
+ Eq(std::map<std::string, std::string>{{"key_s", "value_s"}}));
+ ASSERT_THAT(metric.stats.mean, absl::optional<double>(10.0));
+ ASSERT_THAT(metric.stats.stddev, absl::optional<double>(0.0));
+ ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0));
+ ASSERT_THAT(metric.stats.max, absl::optional<double>(10.0));
+}
+
+} // namespace
+} // namespace test
+} // namespace webrtc
diff --git a/api/test/metrics/metrics_logger.cc b/api/test/metrics/metrics_logger.cc
index c207505..1e24400 100644
--- a/api/test/metrics/metrics_logger.cc
+++ b/api/test/metrics/metrics_logger.cc
@@ -99,6 +99,13 @@
.stats = std::move(metric_stats)});
}
+std::vector<Metric> DefaultMetricsLogger::GetCollectedMetrics() const {
+ std::vector<Metric> out = metrics_accumulator_.GetCollectedMetrics();
+ MutexLock lock(&mutex_);
+ out.insert(out.end(), metrics_.begin(), metrics_.end());
+ return out;
+}
+
Timestamp DefaultMetricsLogger::Now() {
return clock_->CurrentTime();
}
diff --git a/api/test/metrics/metrics_logger.h b/api/test/metrics/metrics_logger.h
index 6839bad..66f9e55 100644
--- a/api/test/metrics/metrics_logger.h
+++ b/api/test/metrics/metrics_logger.h
@@ -19,6 +19,7 @@
#include "absl/strings/string_view.h"
#include "api/numerics/samples_stats_counter.h"
#include "api/test/metrics/metric.h"
+#include "api/test/metrics/metrics_accumulator.h"
#include "rtc_base/synchronization/mutex.h"
#include "system_wrappers/include/clock.h"
@@ -90,16 +91,16 @@
ImprovementDirection improvement_direction,
std::map<std::string, std::string> metadata = {}) override;
- // Returns all metrics collected by this logger.
- std::vector<Metric> GetCollectedMetrics() const override {
- MutexLock lock(&mutex_);
- return metrics_;
- }
+ // Returns all metrics collected by this logger and its `MetricsAccumulator`.
+ std::vector<Metric> GetCollectedMetrics() const override;
+
+ MetricsAccumulator* GetMetricsAccumulator() { return &metrics_accumulator_; }
private:
webrtc::Timestamp Now();
webrtc::Clock* const clock_;
+ MetricsAccumulator metrics_accumulator_;
mutable Mutex mutex_;
std::vector<Metric> metrics_ RTC_GUARDED_BY(mutex_);
diff --git a/api/test/metrics/metrics_logger_test.cc b/api/test/metrics/metrics_logger_test.cc
index 9a59446..de4501c 100644
--- a/api/test/metrics/metrics_logger_test.cc
+++ b/api/test/metrics/metrics_logger_test.cc
@@ -27,6 +27,7 @@
using ::testing::Eq;
using ::testing::IsEmpty;
+using ::testing::SizeIs;
std::map<std::string, std::string> DefaultMetadata() {
return std::map<std::string, std::string>{{"key", "value"}};
@@ -40,7 +41,7 @@
std::map<std::string, std::string>{{"key", "value"}});
std::vector<Metric> metrics = logger.GetCollectedMetrics();
- ASSERT_THAT(metrics.size(), Eq(1lu));
+ ASSERT_THAT(metrics, SizeIs(1));
const Metric& metric = metrics[0];
EXPECT_THAT(metric.name, Eq("metric_name"));
EXPECT_THAT(metric.test_case, Eq("test_case_name"));
@@ -49,7 +50,7 @@
Eq(ImprovementDirection::kBiggerIsBetter));
EXPECT_THAT(metric.metric_metadata,
Eq(std::map<std::string, std::string>{{"key", "value"}}));
- ASSERT_THAT(metric.time_series.samples.size(), Eq(1lu));
+ ASSERT_THAT(metric.time_series.samples, SizeIs(1));
EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0));
EXPECT_THAT(metric.time_series.samples[0].sample_metadata,
Eq(std::map<std::string, std::string>{}));
@@ -78,7 +79,7 @@
std::map<std::string, std::string>{{"key", "value"}});
std::vector<Metric> metrics = logger.GetCollectedMetrics();
- ASSERT_THAT(metrics.size(), Eq(1lu));
+ ASSERT_THAT(metrics, SizeIs(1));
const Metric& metric = metrics[0];
EXPECT_THAT(metric.name, Eq("metric_name"));
EXPECT_THAT(metric.test_case, Eq("test_case_name"));
@@ -87,7 +88,7 @@
Eq(ImprovementDirection::kBiggerIsBetter));
EXPECT_THAT(metric.metric_metadata,
Eq(std::map<std::string, std::string>{{"key", "value"}}));
- ASSERT_THAT(metric.time_series.samples.size(), Eq(2lu));
+ ASSERT_THAT(metric.time_series.samples, SizeIs(2));
EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0));
EXPECT_THAT(metric.time_series.samples[0].sample_metadata,
Eq(std::map<std::string, std::string>{{"point_key1", "value1"}}));
@@ -108,7 +109,7 @@
ImprovementDirection::kBiggerIsBetter, DefaultMetadata());
std::vector<Metric> metrics = logger.GetCollectedMetrics();
- ASSERT_THAT(metrics.size(), Eq(1lu));
+ ASSERT_THAT(metrics, SizeIs(1));
EXPECT_THAT(metrics[0].name, Eq("metric_name"));
EXPECT_THAT(metrics[0].test_case, Eq("test_case_name"));
EXPECT_THAT(metrics[0].time_series.samples, IsEmpty());
@@ -126,7 +127,7 @@
std::map<std::string, std::string>{{"key", "value"}});
std::vector<Metric> metrics = logger.GetCollectedMetrics();
- ASSERT_THAT(metrics.size(), Eq(1lu));
+ ASSERT_THAT(metrics, SizeIs(1));
const Metric& metric = metrics[0];
EXPECT_THAT(metric.name, Eq("metric_name"));
EXPECT_THAT(metric.test_case, Eq("test_case_name"));
@@ -135,7 +136,7 @@
Eq(ImprovementDirection::kBiggerIsBetter));
EXPECT_THAT(metric.metric_metadata,
Eq(std::map<std::string, std::string>{{"key", "value"}}));
- ASSERT_THAT(metric.time_series.samples.size(), Eq(0lu));
+ ASSERT_THAT(metric.time_series.samples, IsEmpty());
ASSERT_THAT(metric.stats.mean, absl::optional<double>(15.0));
ASSERT_THAT(metric.stats.stddev, absl::optional<double>(5.0));
ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0));
@@ -155,7 +156,7 @@
DefaultMetadata());
std::vector<Metric> metrics = logger.GetCollectedMetrics();
- ASSERT_THAT(metrics.size(), Eq(2lu));
+ ASSERT_THAT(metrics, SizeIs(2));
EXPECT_THAT(metrics[0].name, Eq("metric_name1"));
EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1"));
EXPECT_THAT(metrics[1].name, Eq("metric_name2"));
@@ -183,7 +184,7 @@
DefaultMetadata());
std::vector<Metric> metrics = logger.GetCollectedMetrics();
- ASSERT_THAT(metrics.size(), Eq(2lu));
+ ASSERT_THAT(metrics, SizeIs(2));
EXPECT_THAT(metrics[0].name, Eq("metric_name1"));
EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1"));
EXPECT_THAT(metrics[1].name, Eq("metric_name2"));
@@ -202,7 +203,7 @@
DefaultMetadata());
std::vector<Metric> metrics = logger.GetCollectedMetrics();
- ASSERT_THAT(metrics.size(), Eq(2lu));
+ ASSERT_THAT(metrics, SizeIs(2));
EXPECT_THAT(metrics[0].name, Eq("metric_name1"));
EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1"));
EXPECT_THAT(metrics[1].name, Eq("metric_name2"));
@@ -244,6 +245,82 @@
EXPECT_THAT(metrics[2].test_case, Eq("test_case_name3"));
}
+TEST(DefaultMetricsLoggerTest, AccumulatedMetricsReturnedInCollectedMetrics) {
+ DefaultMetricsLogger logger(Clock::GetRealTimeClock());
+ logger.GetMetricsAccumulator()->AddSample(
+ "metric_name", "test_case_name",
+ /*value=*/10, Timestamp::Seconds(1),
+ /*point_metadata=*/std::map<std::string, std::string>{{"key", "value"}});
+
+ std::vector<Metric> metrics = logger.GetCollectedMetrics();
+ ASSERT_THAT(metrics, SizeIs(1));
+ const Metric& metric = metrics[0];
+ EXPECT_THAT(metric.name, Eq("metric_name"));
+ EXPECT_THAT(metric.test_case, Eq("test_case_name"));
+ EXPECT_THAT(metric.unit, Eq(Unit::kUnitless));
+ EXPECT_THAT(metric.improvement_direction,
+ Eq(ImprovementDirection::kNeitherIsBetter));
+ EXPECT_THAT(metric.metric_metadata, IsEmpty());
+ ASSERT_THAT(metric.time_series.samples, SizeIs(1));
+ EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0));
+ EXPECT_THAT(metric.time_series.samples[0].timestamp,
+ Eq(Timestamp::Seconds(1)));
+ EXPECT_THAT(metric.time_series.samples[0].sample_metadata,
+ Eq(std::map<std::string, std::string>{{"key", "value"}}));
+ ASSERT_THAT(metric.stats.mean, absl::optional<double>(10.0));
+ ASSERT_THAT(metric.stats.stddev, absl::optional<double>(0.0));
+ ASSERT_THAT(metric.stats.min, absl::optional<double>(10.0));
+ ASSERT_THAT(metric.stats.max, absl::optional<double>(10.0));
+}
+
+TEST(DefaultMetricsLoggerTest,
+ AccumulatedMetricsReturnedTogetherWithLoggedMetrics) {
+ DefaultMetricsLogger logger(Clock::GetRealTimeClock());
+ logger.LogSingleValueMetric(
+ "metric_name1", "test_case_name1",
+ /*value=*/10, Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter,
+ std::map<std::string, std::string>{{"key_m", "value_m"}});
+ logger.GetMetricsAccumulator()->AddSample(
+ "metric_name2", "test_case_name2",
+ /*value=*/10, Timestamp::Seconds(1),
+ /*point_metadata=*/
+ std::map<std::string, std::string>{{"key_s", "value_s"}});
+
+ std::vector<Metric> metrics = logger.GetCollectedMetrics();
+ ASSERT_THAT(metrics, SizeIs(2));
+ EXPECT_THAT(metrics[0].name, Eq("metric_name2"));
+ EXPECT_THAT(metrics[0].test_case, Eq("test_case_name2"));
+ EXPECT_THAT(metrics[0].unit, Eq(Unit::kUnitless));
+ EXPECT_THAT(metrics[0].improvement_direction,
+ Eq(ImprovementDirection::kNeitherIsBetter));
+ EXPECT_THAT(metrics[0].metric_metadata, IsEmpty());
+ ASSERT_THAT(metrics[0].time_series.samples, SizeIs(1));
+ EXPECT_THAT(metrics[0].time_series.samples[0].value, Eq(10.0));
+ EXPECT_THAT(metrics[0].time_series.samples[0].timestamp,
+ Eq(Timestamp::Seconds(1)));
+ EXPECT_THAT(metrics[0].time_series.samples[0].sample_metadata,
+ Eq(std::map<std::string, std::string>{{"key_s", "value_s"}}));
+ ASSERT_THAT(metrics[0].stats.mean, absl::optional<double>(10.0));
+ ASSERT_THAT(metrics[0].stats.stddev, absl::optional<double>(0.0));
+ ASSERT_THAT(metrics[0].stats.min, absl::optional<double>(10.0));
+ ASSERT_THAT(metrics[0].stats.max, absl::optional<double>(10.0));
+ EXPECT_THAT(metrics[1].name, Eq("metric_name1"));
+ EXPECT_THAT(metrics[1].test_case, Eq("test_case_name1"));
+ EXPECT_THAT(metrics[1].unit, Eq(Unit::kMilliseconds));
+ EXPECT_THAT(metrics[1].improvement_direction,
+ Eq(ImprovementDirection::kBiggerIsBetter));
+ EXPECT_THAT(metrics[1].metric_metadata,
+ Eq(std::map<std::string, std::string>{{"key_m", "value_m"}}));
+ ASSERT_THAT(metrics[1].time_series.samples, SizeIs(1));
+ EXPECT_THAT(metrics[1].time_series.samples[0].value, Eq(10.0));
+ EXPECT_THAT(metrics[1].time_series.samples[0].sample_metadata,
+ Eq(std::map<std::string, std::string>{}));
+ ASSERT_THAT(metrics[1].stats.mean, absl::optional<double>(10.0));
+ ASSERT_THAT(metrics[1].stats.stddev, absl::nullopt);
+ ASSERT_THAT(metrics[1].stats.min, absl::optional<double>(10.0));
+ ASSERT_THAT(metrics[1].stats.max, absl::optional<double>(10.0));
+}
+
} // namespace
} // namespace test
} // namespace webrtc