[DVQA] Add ability to export metrics with MetricsLoggerAndExporter
Bug: b/246095034
Change-Id: Ibbadd11ff27f65cc128efd891eace89df3c59316
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276101
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38145}
diff --git a/api/test/metrics/metrics_logger_and_exporter.cc b/api/test/metrics/metrics_logger_and_exporter.cc
index d76003b..9f91eac 100644
--- a/api/test/metrics/metrics_logger_and_exporter.cc
+++ b/api/test/metrics/metrics_logger_and_exporter.cc
@@ -24,6 +24,19 @@
namespace webrtc {
namespace test {
+namespace {
+
+Metric::Stats ToStats(const SamplesStatsCounter& values) {
+ if (values.IsEmpty()) {
+ return Metric::Stats();
+ }
+ return Metric::Stats{.mean = values.GetAverage(),
+ .stddev = values.GetStandardDeviation(),
+ .min = values.GetMin(),
+ .max = values.GetMax()};
+}
+
+} // namespace
MetricsLoggerAndExporter::~MetricsLoggerAndExporter() {
bool export_result = Export();
@@ -72,17 +85,13 @@
.sample_metadata = sample.metadata});
}
- metrics_.push_back(
- Metric{.name = std::string(name),
- .unit = unit,
- .improvement_direction = improvement_direction,
- .test_case = std::string(test_case_name),
- .metric_metadata = std::move(metadata),
- .time_series = std::move(time_series),
- .stats = Metric::Stats{.mean = values.GetAverage(),
- .stddev = values.GetStandardDeviation(),
- .min = values.GetMin(),
- .max = values.GetMax()}});
+ metrics_.push_back(Metric{.name = std::string(name),
+ .unit = unit,
+ .improvement_direction = improvement_direction,
+ .test_case = std::string(test_case_name),
+ .metric_metadata = std::move(metadata),
+ .time_series = std::move(time_series),
+ .stats = ToStats(values)});
}
void MetricsLoggerAndExporter::LogMetric(
diff --git a/api/test/metrics/metrics_logger_and_exporter_test.cc b/api/test/metrics/metrics_logger_and_exporter_test.cc
index a19a162..65b1d8f 100644
--- a/api/test/metrics/metrics_logger_and_exporter_test.cc
+++ b/api/test/metrics/metrics_logger_and_exporter_test.cc
@@ -145,6 +145,30 @@
ASSERT_THAT(metric.stats.max, absl::optional<double>(20.0));
}
+TEST(MetricsLoggerAndExporterTest,
+ LogMetricWithEmptySamplesStatsCounterRecordsEmptyMetric) {
+ TestMetricsExporterFactory exporter_factory;
+ {
+ std::vector<std::unique_ptr<MetricsExporter>> exporters;
+ exporters.push_back(exporter_factory.CreateExporter());
+ MetricsLoggerAndExporter logger(Clock::GetRealTimeClock(),
+ std::move(exporters));
+ SamplesStatsCounter values;
+ logger.LogMetric("metric_name", "test_case_name", values, Unit::kUnitless,
+ ImprovementDirection::kBiggerIsBetter, DefaultMetadata());
+ }
+
+ std::vector<Metric> metrics = exporter_factory.exported_metrics;
+ ASSERT_THAT(metrics.size(), Eq(1lu));
+ EXPECT_THAT(metrics[0].name, Eq("metric_name"));
+ EXPECT_THAT(metrics[0].test_case, Eq("test_case_name"));
+ EXPECT_THAT(metrics[0].time_series.samples, IsEmpty());
+ ASSERT_THAT(metrics[0].stats.mean, Eq(absl::nullopt));
+ ASSERT_THAT(metrics[0].stats.stddev, Eq(absl::nullopt));
+ ASSERT_THAT(metrics[0].stats.min, Eq(absl::nullopt));
+ ASSERT_THAT(metrics[0].stats.max, Eq(absl::nullopt));
+}
+
TEST(MetricsLoggerAndExporterTest, LogMetricWithStatsRecordsMetric) {
TestMetricsExporterFactory exporter_factory;
{
diff --git a/test/pc/e2e/BUILD.gn b/test/pc/e2e/BUILD.gn
index 49897a3..a44041c 100644
--- a/test/pc/e2e/BUILD.gn
+++ b/test/pc/e2e/BUILD.gn
@@ -35,6 +35,7 @@
deps = [
":default_video_quality_analyzer_frames_comparator_test",
+ ":default_video_quality_analyzer_metric_names_test",
":default_video_quality_analyzer_stream_state_test",
":default_video_quality_analyzer_test",
":multi_reader_queue_test",
@@ -564,6 +565,29 @@
]
}
+ rtc_library("default_video_quality_analyzer_metric_names_test") {
+ testonly = true
+ sources = [
+ "analyzer/video/default_video_quality_analyzer_metric_names_test.cc",
+ ]
+ deps = [
+ ":default_video_quality_analyzer",
+ ":default_video_quality_analyzer_shared",
+ "../..:test_support",
+ "../../../api:create_frame_generator",
+ "../../../api:rtp_packet_info",
+ "../../../api/test/metrics:metric",
+ "../../../api/test/metrics:metrics_logger_and_exporter",
+ "../../../api/test/metrics:stdout_metrics_exporter",
+ "../../../api/video:encoded_image",
+ "../../../api/video:video_frame",
+ "../../../common_video",
+ "../../../rtc_base:stringutils",
+ "../../../rtc_tools:video_quality_analysis",
+ "../../../system_wrappers",
+ ]
+ }
+
rtc_library("default_video_quality_analyzer_frames_comparator_test") {
testonly = true
sources = [ "analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc" ]
@@ -719,6 +743,8 @@
"../../../api:array_view",
"../../../api:video_quality_analyzer_api",
"../../../api/numerics",
+ "../../../api/test/metrics:metric",
+ "../../../api/test/metrics:metrics_logger_and_exporter",
"../../../api/units:data_size",
"../../../api/units:time_delta",
"../../../api/units:timestamp",
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
index b758c92..46d3270 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
@@ -18,6 +18,7 @@
#include "api/array_view.h"
#include "api/numerics/samples_stats_counter.h"
+#include "api/test/metrics/metric.h"
#include "api/units/time_delta.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_frame.h"
@@ -36,6 +37,9 @@
namespace webrtc {
namespace {
+using ::webrtc::test::ImprovementDirection;
+using ::webrtc::test::Unit;
+
constexpr int kBitsInByte = 8;
constexpr absl::string_view kSkipRenderedFrameReasonProcessed = "processed";
constexpr absl::string_view kSkipRenderedFrameReasonRendered = "rendered";
@@ -123,9 +127,19 @@
DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(
webrtc::Clock* clock,
DefaultVideoQualityAnalyzerOptions options)
+ : DefaultVideoQualityAnalyzer(clock,
+ /*metrics_logger=*/nullptr,
+ std::move(options)) {}
+
+DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(
+ webrtc::Clock* clock,
+ test::MetricsLoggerAndExporter* metrics_logger,
+ DefaultVideoQualityAnalyzerOptions options)
: options_(options),
clock_(clock),
+ metrics_logger_(metrics_logger),
frames_comparator_(clock, cpu_measurer_, options) {}
+
DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() {
Stop();
}
@@ -951,87 +965,186 @@
video_duration.seconds<double>() / sum_squared_interframe_delays_secs;
}
- ReportResult("psnr", test_case_name, stats.psnr, "dB",
- ImproveDirection::kBiggerIsBetter);
- ReportResult("ssim", test_case_name, stats.ssim, "unitless",
- ImproveDirection::kBiggerIsBetter);
- ReportResult("transport_time", test_case_name, stats.transport_time_ms, "ms",
- ImproveDirection::kSmallerIsBetter);
- ReportResult("total_delay_incl_transport", test_case_name,
- stats.total_delay_incl_transport_ms, "ms",
- ImproveDirection::kSmallerIsBetter);
- ReportResult("time_between_rendered_frames", test_case_name,
- stats.time_between_rendered_frames_ms, "ms",
- ImproveDirection::kSmallerIsBetter);
- test::PrintResult("harmonic_framerate", "", test_case_name,
- harmonic_framerate_fps, "Hz", /*important=*/false,
- ImproveDirection::kBiggerIsBetter);
- test::PrintResult("encode_frame_rate", "", test_case_name,
- stats.encode_frame_rate.IsEmpty()
- ? 0
- : stats.encode_frame_rate.GetEventsPerSecond(),
- "Hz", /*important=*/false,
- ImproveDirection::kBiggerIsBetter);
- ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms",
- ImproveDirection::kSmallerIsBetter);
- ReportResult("time_between_freezes", test_case_name,
- stats.time_between_freezes_ms, "ms",
- ImproveDirection::kBiggerIsBetter);
- ReportResult("freeze_time_ms", test_case_name, stats.freeze_time_ms, "ms",
- ImproveDirection::kSmallerIsBetter);
- ReportResult("pixels_per_frame", test_case_name,
- stats.resolution_of_rendered_frame, "count",
- ImproveDirection::kBiggerIsBetter);
- test::PrintResult("min_psnr", "", test_case_name,
- stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), "dB",
- /*important=*/false, ImproveDirection::kBiggerIsBetter);
- ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms",
- ImproveDirection::kSmallerIsBetter);
- ReportResult("receive_to_render_time", test_case_name,
- stats.receive_to_render_time_ms, "ms",
- ImproveDirection::kSmallerIsBetter);
- test::PrintResult("dropped_frames", "", test_case_name,
- frame_counters.dropped, "count",
- /*important=*/false, ImproveDirection::kSmallerIsBetter);
- test::PrintResult("frames_in_flight", "", test_case_name,
- frame_counters.captured - frame_counters.rendered -
- frame_counters.dropped,
- "count",
- /*important=*/false, ImproveDirection::kSmallerIsBetter);
- test::PrintResult("rendered_frames", "", test_case_name,
- frame_counters.rendered, "count", /*important=*/false,
- ImproveDirection::kBiggerIsBetter);
- ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered,
- "count", ImproveDirection::kSmallerIsBetter);
- ReportResult("target_encode_bitrate", test_case_name,
- stats.target_encode_bitrate / kBitsInByte, "bytesPerSecond",
- ImproveDirection::kNone);
- test::PrintResult("actual_encode_bitrate", "", test_case_name,
- static_cast<double>(stats.total_encoded_images_payload) /
- test_duration.seconds<double>(),
- "bytesPerSecond", /*important=*/false,
- ImproveDirection::kNone);
-
- if (options_.report_detailed_frame_stats) {
- test::PrintResult("num_encoded_frames", "", test_case_name,
- frame_counters.encoded, "count",
- /*important=*/false, ImproveDirection::kBiggerIsBetter);
- test::PrintResult("num_decoded_frames", "", test_case_name,
- frame_counters.decoded, "count",
- /*important=*/false, ImproveDirection::kBiggerIsBetter);
- test::PrintResult("num_send_key_frames", "", test_case_name,
- stats.num_send_key_frames, "count",
- /*important=*/false, ImproveDirection::kBiggerIsBetter);
- test::PrintResult("num_recv_key_frames", "", test_case_name,
- stats.num_recv_key_frames, "count",
- /*important=*/false, ImproveDirection::kBiggerIsBetter);
-
- ReportResult("recv_key_frame_size_bytes", test_case_name,
- stats.recv_key_frame_size_bytes, "count",
+ if (metrics_logger_ == nullptr) {
+ // Report results through old performance metrics API.
+ ReportResult("psnr", test_case_name, stats.psnr, "dB",
ImproveDirection::kBiggerIsBetter);
- ReportResult("recv_delta_frame_size_bytes", test_case_name,
- stats.recv_delta_frame_size_bytes, "count",
+ ReportResult("ssim", test_case_name, stats.ssim, "unitless",
ImproveDirection::kBiggerIsBetter);
+ ReportResult("transport_time", test_case_name, stats.transport_time_ms,
+ "ms", ImproveDirection::kSmallerIsBetter);
+ ReportResult("total_delay_incl_transport", test_case_name,
+ stats.total_delay_incl_transport_ms, "ms",
+ ImproveDirection::kSmallerIsBetter);
+ ReportResult("time_between_rendered_frames", test_case_name,
+ stats.time_between_rendered_frames_ms, "ms",
+ ImproveDirection::kSmallerIsBetter);
+ test::PrintResult("harmonic_framerate", "", test_case_name,
+ harmonic_framerate_fps, "Hz", /*important=*/false,
+ ImproveDirection::kBiggerIsBetter);
+ test::PrintResult("encode_frame_rate", "", test_case_name,
+ stats.encode_frame_rate.IsEmpty()
+ ? 0
+ : stats.encode_frame_rate.GetEventsPerSecond(),
+ "Hz", /*important=*/false,
+ ImproveDirection::kBiggerIsBetter);
+ ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms",
+ ImproveDirection::kSmallerIsBetter);
+ ReportResult("time_between_freezes", test_case_name,
+ stats.time_between_freezes_ms, "ms",
+ ImproveDirection::kBiggerIsBetter);
+ ReportResult("freeze_time_ms", test_case_name, stats.freeze_time_ms, "ms",
+ ImproveDirection::kSmallerIsBetter);
+ ReportResult("pixels_per_frame", test_case_name,
+ stats.resolution_of_rendered_frame, "count",
+ ImproveDirection::kBiggerIsBetter);
+ test::PrintResult("min_psnr", "", test_case_name,
+ stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), "dB",
+ /*important=*/false, ImproveDirection::kBiggerIsBetter);
+ ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms",
+ ImproveDirection::kSmallerIsBetter);
+ ReportResult("receive_to_render_time", test_case_name,
+ stats.receive_to_render_time_ms, "ms",
+ ImproveDirection::kSmallerIsBetter);
+ test::PrintResult("dropped_frames", "", test_case_name,
+ frame_counters.dropped, "count",
+ /*important=*/false, ImproveDirection::kSmallerIsBetter);
+ test::PrintResult("frames_in_flight", "", test_case_name,
+ frame_counters.captured - frame_counters.rendered -
+ frame_counters.dropped,
+ "count",
+ /*important=*/false, ImproveDirection::kSmallerIsBetter);
+ test::PrintResult("rendered_frames", "", test_case_name,
+ frame_counters.rendered, "count", /*important=*/false,
+ ImproveDirection::kBiggerIsBetter);
+ ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered,
+ "count", ImproveDirection::kSmallerIsBetter);
+ ReportResult("target_encode_bitrate", test_case_name,
+ stats.target_encode_bitrate / kBitsInByte, "bytesPerSecond",
+ ImproveDirection::kNone);
+ test::PrintResult("actual_encode_bitrate", "", test_case_name,
+ static_cast<double>(stats.total_encoded_images_payload) /
+ test_duration.seconds<double>(),
+ "bytesPerSecond", /*important=*/false,
+ ImproveDirection::kNone);
+
+ if (options_.report_detailed_frame_stats) {
+ test::PrintResult("num_encoded_frames", "", test_case_name,
+ frame_counters.encoded, "count",
+ /*important=*/false, ImproveDirection::kBiggerIsBetter);
+ test::PrintResult("num_decoded_frames", "", test_case_name,
+ frame_counters.decoded, "count",
+ /*important=*/false, ImproveDirection::kBiggerIsBetter);
+ test::PrintResult("num_send_key_frames", "", test_case_name,
+ stats.num_send_key_frames, "count",
+ /*important=*/false, ImproveDirection::kBiggerIsBetter);
+ test::PrintResult("num_recv_key_frames", "", test_case_name,
+ stats.num_recv_key_frames, "count",
+ /*important=*/false, ImproveDirection::kBiggerIsBetter);
+
+ ReportResult("recv_key_frame_size_bytes", test_case_name,
+ stats.recv_key_frame_size_bytes, "count",
+ ImproveDirection::kBiggerIsBetter);
+ ReportResult("recv_delta_frame_size_bytes", test_case_name,
+ stats.recv_delta_frame_size_bytes, "count",
+ ImproveDirection::kBiggerIsBetter);
+ }
+ } else {
+ metrics_logger_->LogMetric("psnr_dB", test_case_name, stats.psnr,
+ Unit::kUnitless,
+ ImprovementDirection::kBiggerIsBetter);
+ metrics_logger_->LogMetric("ssim", test_case_name, stats.ssim,
+ Unit::kUnitless,
+ ImprovementDirection::kBiggerIsBetter);
+ metrics_logger_->LogMetric("transport_time", test_case_name,
+ stats.transport_time_ms, Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogMetric("total_delay_incl_transport", test_case_name,
+ stats.total_delay_incl_transport_ms,
+ Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogMetric("time_between_rendered_frames", test_case_name,
+ stats.time_between_rendered_frames_ms,
+ Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "harmonic_framerate", test_case_name, harmonic_framerate_fps,
+ Unit::kHertz, ImprovementDirection::kBiggerIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "encode_frame_rate", test_case_name,
+ stats.encode_frame_rate.IsEmpty()
+ ? 0
+ : stats.encode_frame_rate.GetEventsPerSecond(),
+ Unit::kHertz, ImprovementDirection::kBiggerIsBetter);
+ metrics_logger_->LogMetric("encode_time", test_case_name,
+ stats.encode_time_ms, Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogMetric(
+ "time_between_freezes", test_case_name, stats.time_between_freezes_ms,
+ Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter);
+ metrics_logger_->LogMetric("freeze_time_ms", test_case_name,
+ stats.freeze_time_ms, Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogMetric("pixels_per_frame", test_case_name,
+ stats.resolution_of_rendered_frame, Unit::kCount,
+ ImprovementDirection::kBiggerIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "min_psnr_dB", test_case_name,
+ stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), Unit::kUnitless,
+ ImprovementDirection::kBiggerIsBetter);
+ metrics_logger_->LogMetric("decode_time", test_case_name,
+ stats.decode_time_ms, Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogMetric("receive_to_render_time", test_case_name,
+ stats.receive_to_render_time_ms,
+ Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "dropped_frames", test_case_name, frame_counters.dropped, Unit::kCount,
+ ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "frames_in_flight", test_case_name,
+ frame_counters.captured - frame_counters.rendered -
+ frame_counters.dropped,
+ Unit::kCount, ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "rendered_frames", test_case_name, frame_counters.rendered,
+ Unit::kCount, ImprovementDirection::kBiggerIsBetter);
+ metrics_logger_->LogMetric("max_skipped", test_case_name,
+ stats.skipped_between_rendered, Unit::kCount,
+ ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogMetric("target_encode_bitrate", test_case_name,
+ stats.target_encode_bitrate / 1000,
+ Unit::kKilobitsPerSecond,
+ ImprovementDirection::kNeitherIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "actual_encode_bitrate", test_case_name,
+ static_cast<double>(stats.total_encoded_images_payload) /
+ test_duration.seconds<double>() * kBitsInByte / 1000,
+ Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
+
+ if (options_.report_detailed_frame_stats) {
+ metrics_logger_->LogSingleValueMetric(
+ "num_encoded_frames", test_case_name, frame_counters.encoded,
+ Unit::kCount, ImprovementDirection::kBiggerIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "num_decoded_frames", test_case_name, frame_counters.decoded,
+ Unit::kCount, ImprovementDirection::kBiggerIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "num_send_key_frames", test_case_name, stats.num_send_key_frames,
+ Unit::kCount, ImprovementDirection::kBiggerIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "num_recv_key_frames", test_case_name, stats.num_recv_key_frames,
+ Unit::kCount, ImprovementDirection::kBiggerIsBetter);
+
+ metrics_logger_->LogMetric("recv_key_frame_size_bytes", test_case_name,
+ stats.recv_key_frame_size_bytes, Unit::kCount,
+ ImprovementDirection::kBiggerIsBetter);
+ metrics_logger_->LogMetric("recv_delta_frame_size_bytes", test_case_name,
+ stats.recv_delta_frame_size_bytes,
+ Unit::kCount,
+ ImprovementDirection::kBiggerIsBetter);
+ }
}
}
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
index 7b56ce9..c1b7fe8 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
@@ -22,6 +22,7 @@
#include "api/array_view.h"
#include "api/numerics/samples_stats_counter.h"
+#include "api/test/metrics/metrics_logger_and_exporter.h"
#include "api/test/video_quality_analyzer_interface.h"
#include "api/units/data_size.h"
#include "api/units/timestamp.h"
@@ -49,6 +50,9 @@
explicit DefaultVideoQualityAnalyzer(
webrtc::Clock* clock,
DefaultVideoQualityAnalyzerOptions options = {});
+ DefaultVideoQualityAnalyzer(webrtc::Clock* clock,
+ test::MetricsLoggerAndExporter* metrics_logger,
+ DefaultVideoQualityAnalyzerOptions options = {});
~DefaultVideoQualityAnalyzer() override;
void Start(std::string test_case_name,
@@ -148,6 +152,7 @@
const DefaultVideoQualityAnalyzerOptions options_;
webrtc::Clock* const clock_;
+ test::MetricsLoggerAndExporter* const metrics_logger_;
std::string test_label_;
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc
new file mode 100644
index 0000000..f2ca603
--- /dev/null
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc
@@ -0,0 +1,649 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "api/rtp_packet_info.h"
+#include "api/rtp_packet_infos.h"
+#include "api/test/create_frame_generator.h"
+#include "api/test/metrics/metric.h"
+#include "api/test/metrics/metrics_logger_and_exporter.h"
+#include "api/test/metrics/stdout_metrics_exporter.h"
+#include "api/video/encoded_image.h"
+#include "api/video/i420_buffer.h"
+#include "api/video/video_frame.h"
+#include "common_video/libyuv/include/webrtc_libyuv.h"
+#include "rtc_tools/frame_analyzer/video_geometry_aligner.h"
+#include "system_wrappers/include/sleep.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
+
+namespace webrtc {
+namespace {
+
+using ::testing::Contains;
+using ::testing::SizeIs;
+using ::testing::UnorderedElementsAre;
+
+using ::webrtc::test::ImprovementDirection;
+using ::webrtc::test::Metric;
+using ::webrtc::test::MetricsExporter;
+using ::webrtc::test::MetricsLoggerAndExporter;
+using ::webrtc::test::StdoutMetricsExporter;
+using ::webrtc::test::Unit;
+
+constexpr int kAnalyzerMaxThreadsCount = 1;
+constexpr int kMaxFramesInFlightPerStream = 10;
+constexpr int kFrameWidth = 320;
+constexpr int kFrameHeight = 240;
+
+DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() {
+ DefaultVideoQualityAnalyzerOptions options;
+ options.compute_psnr = true;
+ options.compute_ssim = true;
+ options.adjust_cropping_before_comparing_frames = false;
+ options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream;
+ options.report_detailed_frame_stats = true;
+ return options;
+}
+
+VideoFrame NextFrame(test::FrameGeneratorInterface* frame_generator,
+ int64_t timestamp_us) {
+ test::FrameGeneratorInterface::VideoFrameData frame_data =
+ frame_generator->NextFrame();
+ return VideoFrame::Builder()
+ .set_video_frame_buffer(frame_data.buffer)
+ .set_update_rect(frame_data.update_rect)
+ .set_timestamp_us(timestamp_us)
+ .build();
+}
+
+EncodedImage FakeEncode(const VideoFrame& frame) {
+ EncodedImage image;
+ std::vector<RtpPacketInfo> packet_infos;
+ packet_infos.push_back(RtpPacketInfo(
+ /*ssrc=*/1,
+ /*csrcs=*/{},
+ /*rtp_timestamp=*/frame.timestamp(),
+ /*receive_time=*/Timestamp::Micros(frame.timestamp_us() + 10000)));
+ image.SetPacketInfos(RtpPacketInfos(packet_infos));
+ return image;
+}
+
+VideoFrame DeepCopy(const VideoFrame& frame) {
+ VideoFrame copy = frame;
+ copy.set_video_frame_buffer(
+ I420Buffer::Copy(*frame.video_frame_buffer()->ToI420()));
+ return copy;
+}
+
+void PassFramesThroughAnalyzer(DefaultVideoQualityAnalyzer& analyzer,
+ absl::string_view sender,
+ absl::string_view stream_label,
+ std::vector<absl::string_view> receivers,
+ int frames_count,
+ test::FrameGeneratorInterface& frame_generator,
+ int interframe_delay_ms = 0) {
+ for (int i = 0; i < frames_count; ++i) {
+ VideoFrame frame = NextFrame(&frame_generator, /*timestamp_us=*/1);
+ uint16_t frame_id =
+ analyzer.OnFrameCaptured(sender, std::string(stream_label), frame);
+ frame.set_id(frame_id);
+ analyzer.OnFramePreEncode(sender, frame);
+ analyzer.OnFrameEncoded(sender, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats());
+ for (absl::string_view receiver : receivers) {
+ VideoFrame received_frame = DeepCopy(frame);
+ analyzer.OnFramePreDecode(receiver, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(receiver, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(receiver, received_frame);
+ }
+ if (i < frames_count - 1 && interframe_delay_ms > 0) {
+ SleepMs(interframe_delay_ms);
+ }
+ }
+}
+
+// Metric fields to assert on
+struct MetricValidationInfo {
+ std::string test_case;
+ std::string name;
+ Unit unit;
+ ImprovementDirection improvement_direction;
+};
+
+bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) {
+ return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit &&
+ a.improvement_direction == b.improvement_direction;
+}
+
+std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) {
+ os << "{ test_case=" << m.test_case << "; name=" << m.name
+ << "; unit=" << test::ToString(m.unit)
+ << "; improvement_direction=" << test::ToString(m.improvement_direction)
+ << " }";
+ return os;
+}
+
+std::vector<MetricValidationInfo> ToValidationInfo(
+ const std::vector<Metric>& metrics) {
+ std::vector<MetricValidationInfo> out;
+ for (const Metric& m : metrics) {
+ out.push_back(
+ MetricValidationInfo{.test_case = m.test_case,
+ .name = m.name,
+ .unit = m.unit,
+ .improvement_direction = m.improvement_direction});
+ }
+ return out;
+}
+
+std::vector<std::string> ToTestCases(const std::vector<Metric>& metrics) {
+ std::vector<std::string> out;
+ for (const Metric& m : metrics) {
+ out.push_back(m.test_case);
+ }
+ return out;
+}
+
+TEST(DefaultVideoQualityAnalyzerMetricNamesTest, MetricNamesForP2PAreCorrect) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ std::vector<std::unique_ptr<MetricsExporter>> exporters;
+ exporters.push_back(std::make_unique<StdoutMetricsExporter>());
+ MetricsLoggerAndExporter metrics_logger(Clock::GetRealTimeClock(),
+ std::move(exporters));
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ &metrics_logger, options);
+ analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
+ kAnalyzerMaxThreadsCount);
+
+ PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"},
+ /*frames_count=*/5, *frame_generator,
+ /*interframe_delay_ms=*/50);
+ analyzer.Stop();
+
+ std::vector<MetricValidationInfo> metrics =
+ ToValidationInfo(metrics_logger.GetCollectedMetrics());
+ EXPECT_THAT(
+ metrics,
+ UnorderedElementsAre(
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "ssim",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "transport_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "total_delay_incl_transport",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "time_between_rendered_frames",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "harmonic_framerate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "encode_frame_rate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "encode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "time_between_freezes",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "freeze_time_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "pixels_per_frame",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "min_psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "decode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "receive_to_render_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "dropped_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "frames_in_flight",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "rendered_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "max_skipped",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "target_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "actual_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "num_encoded_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "num_decoded_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "num_send_key_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "num_recv_key_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "recv_key_frame_size_bytes",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "recv_delta_frame_size_bytes",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter}));
+}
+
+TEST(DefaultVideoQualityAnalyzerMetricNamesTest,
+ MetricNamesFor3PeersAreCorrect) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ std::vector<std::unique_ptr<MetricsExporter>> exporters;
+ exporters.push_back(std::make_unique<StdoutMetricsExporter>());
+ MetricsLoggerAndExporter metrics_logger(Clock::GetRealTimeClock(),
+ std::move(exporters));
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ &metrics_logger, options);
+ analyzer.Start("test_case",
+ std::vector<std::string>{"alice", "bob", "charlie"},
+ kAnalyzerMaxThreadsCount);
+
+ PassFramesThroughAnalyzer(analyzer, "alice", "alice_video",
+ {"bob", "charlie"},
+ /*frames_count=*/5, *frame_generator,
+ /*interframe_delay_ms=*/50);
+ analyzer.Stop();
+
+ std::vector<MetricValidationInfo> metrics =
+ ToValidationInfo(metrics_logger.GetCollectedMetrics());
+ EXPECT_THAT(
+ metrics,
+ UnorderedElementsAre(
+ // Bob
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "ssim",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "transport_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "total_delay_incl_transport",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "time_between_rendered_frames",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "harmonic_framerate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "encode_frame_rate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "encode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "time_between_freezes",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "freeze_time_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "pixels_per_frame",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "min_psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "decode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "receive_to_render_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "dropped_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "frames_in_flight",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "rendered_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "max_skipped",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "target_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "actual_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "num_encoded_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "num_decoded_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "num_send_key_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "num_recv_key_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "recv_key_frame_size_bytes",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "recv_delta_frame_size_bytes",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+
+ // Charlie
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "ssim",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "transport_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "total_delay_incl_transport",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "time_between_rendered_frames",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "harmonic_framerate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "encode_frame_rate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "encode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "time_between_freezes",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "freeze_time_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "pixels_per_frame",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "min_psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "decode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "receive_to_render_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "dropped_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "frames_in_flight",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "rendered_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "max_skipped",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "target_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "actual_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "num_encoded_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "num_decoded_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "num_send_key_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "num_recv_key_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "recv_key_frame_size_bytes",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "recv_delta_frame_size_bytes",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter}));
+}
+
+TEST(DefaultVideoQualityAnalyzerMetricNamesTest,
+ TestCaseFor3PeerIsTheSameAfterAllPeersLeft) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ std::vector<std::unique_ptr<MetricsExporter>> exporters;
+ exporters.push_back(std::make_unique<StdoutMetricsExporter>());
+ MetricsLoggerAndExporter metrics_logger(Clock::GetRealTimeClock(),
+ std::move(exporters));
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ &metrics_logger, options);
+ analyzer.Start("test_case",
+ std::vector<std::string>{"alice", "bob", "charlie"},
+ kAnalyzerMaxThreadsCount);
+
+ PassFramesThroughAnalyzer(analyzer, "alice", "alice_video",
+ {"bob", "charlie"},
+ /*frames_count=*/5, *frame_generator,
+ /*interframe_delay_ms=*/50);
+ analyzer.UnregisterParticipantInCall("alice");
+ analyzer.UnregisterParticipantInCall("bob");
+ analyzer.UnregisterParticipantInCall("charlie");
+ analyzer.Stop();
+
+ std::vector<std::string> metrics =
+ ToTestCases(metrics_logger.GetCollectedMetrics());
+ EXPECT_THAT(metrics, SizeIs(52));
+ EXPECT_THAT(metrics, Contains("test_case/alice_video_alice_bob").Times(26));
+ EXPECT_THAT(metrics,
+ Contains("test_case/alice_video_alice_charlie").Times(26));
+}
+
+} // namespace
+} // namespace webrtc
