Migrate video_quality_analysis on new perf metrics logger API
Bug: b/246095034
Change-Id: Ibdaae04ccd874024ce8db5c1f015479c713264a0
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276624
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Tomas Gunnarsson <tommi@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38198}
diff --git a/rtc_tools/BUILD.gn b/rtc_tools/BUILD.gn
index bb4b8c0..21dc736 100644
--- a/rtc_tools/BUILD.gn
+++ b/rtc_tools/BUILD.gn
@@ -110,12 +110,14 @@
"../api:array_view",
"../api:make_ref_counted",
"../api:scoped_refptr",
+ "../api/numerics",
+ "../api/test/metrics:metric",
+ "../api/test/metrics:metrics_logger",
"../api/video:video_frame",
"../api/video:video_rtp_headers",
"../common_video",
"../rtc_base:checks",
"../rtc_base:logging",
- "../test:perf_test",
"//third_party/libyuv",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
@@ -138,6 +140,7 @@
":video_quality_analysis",
"../api:make_ref_counted",
"../api:scoped_refptr",
+ "../api/test/metrics:global_metrics_logger_and_exporter",
"../rtc_base:stringutils",
"../test:perf_test",
"//third_party/abseil-cpp/absl/flags:flag",
@@ -526,6 +529,8 @@
":video_file_writer",
":video_quality_analysis",
"../api:scoped_refptr",
+ "../api/test/metrics:metric",
+ "../api/test/metrics:metrics_logger",
"../api/video:video_frame",
"../api/video:video_rtp_headers",
"../common_video",
@@ -533,6 +538,7 @@
"../rtc_base:checks",
"../rtc_base:null_socket_server",
"../rtc_base:threading",
+ "../system_wrappers",
"../test:fileutils",
"../test:test_main",
"../test:test_support",
diff --git a/rtc_tools/frame_analyzer/frame_analyzer.cc b/rtc_tools/frame_analyzer/frame_analyzer.cc
index 70af305..2224f48 100644
--- a/rtc_tools/frame_analyzer/frame_analyzer.cc
+++ b/rtc_tools/frame_analyzer/frame_analyzer.cc
@@ -19,6 +19,7 @@
#include "absl/flags/parse.h"
#include "absl/strings/match.h"
#include "api/scoped_refptr.h"
+#include "api/test/metrics/global_metrics_logger_and_exporter.h"
#include "rtc_base/strings/string_builder.h"
#include "rtc_tools/frame_analyzer/video_color_aligner.h"
#include "rtc_tools/frame_analyzer/video_geometry_aligner.h"
@@ -159,7 +160,8 @@
results.decode_errors_ref = 0;
results.decode_errors_test = 0;
- webrtc::test::PrintAnalysisResults(absl::GetFlag(FLAGS_label), &results);
+ webrtc::test::PrintAnalysisResults(absl::GetFlag(FLAGS_label), results,
+ *webrtc::test::GetGlobalMetricsLogger());
std::string chartjson_result_file =
absl::GetFlag(FLAGS_chartjson_result_file);
diff --git a/rtc_tools/frame_analyzer/video_quality_analysis.cc b/rtc_tools/frame_analyzer/video_quality_analysis.cc
index 6989f3f..1832438 100644
--- a/rtc_tools/frame_analyzer/video_quality_analysis.cc
+++ b/rtc_tools/frame_analyzer/video_quality_analysis.cc
@@ -14,9 +14,10 @@
#include <array>
#include <cstddef>
+#include "api/numerics/samples_stats_counter.h"
+#include "api/test/metrics/metric.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
-#include "test/testsupport/perf_test.h"
#include "third_party/libyuv/include/libyuv/compare.h"
namespace webrtc {
@@ -117,39 +118,42 @@
return static_cast<int>(number_ref_frames - clusters.size());
}
-void PrintAnalysisResults(const std::string& label, ResultsContainer* results) {
- PrintAnalysisResults(stdout, label, results);
-}
+void PrintAnalysisResults(const std::string& label,
+ ResultsContainer& results,
+ MetricsLogger& logger) {
+ if (results.frames.size() > 0u) {
+ logger.LogSingleValueMetric("Unique_frames_count", label,
+ results.frames.size(), Unit::kUnitless,
+ ImprovementDirection::kNeitherIsBetter);
-void PrintAnalysisResults(FILE* output,
- const std::string& label,
- ResultsContainer* results) {
- SetPerfResultsOutput(output);
-
- if (results->frames.size() > 0u) {
- PrintResult("Unique_frames_count", "", label, results->frames.size(),
- "score", false);
-
- std::vector<double> psnr_values;
- std::vector<double> ssim_values;
- for (const auto& frame : results->frames) {
- psnr_values.push_back(frame.psnr_value);
- ssim_values.push_back(frame.ssim_value);
+ SamplesStatsCounter psnr_values;
+ SamplesStatsCounter ssim_values;
+ for (const auto& frame : results.frames) {
+ psnr_values.AddSample(frame.psnr_value);
+ ssim_values.AddSample(frame.ssim_value);
}
- PrintResultList("PSNR", "", label, psnr_values, "dB", false);
- PrintResultList("SSIM", "", label, ssim_values, "score", false);
+ logger.LogMetric("PSNR_dB", label, psnr_values, Unit::kUnitless,
+ ImprovementDirection::kNeitherIsBetter);
+ logger.LogMetric("SSIM", label, ssim_values, Unit::kUnitless,
+ ImprovementDirection::kNeitherIsBetter);
}
- PrintResult("Max_repeated", "", label, results->max_repeated_frames, "",
- false);
- PrintResult("Max_skipped", "", label, results->max_skipped_frames, "", false);
- PrintResult("Total_skipped", "", label, results->total_skipped_frames, "",
- false);
- PrintResult("Decode_errors_reference", "", label, results->decode_errors_ref,
- "", false);
- PrintResult("Decode_errors_test", "", label, results->decode_errors_test, "",
- false);
+ logger.LogSingleValueMetric("Max_repeated", label,
+ results.max_repeated_frames, Unit::kUnitless,
+ ImprovementDirection::kNeitherIsBetter);
+ logger.LogSingleValueMetric("Max_skipped", label, results.max_skipped_frames,
+ Unit::kUnitless,
+ ImprovementDirection::kNeitherIsBetter);
+ logger.LogSingleValueMetric("Total_skipped", label,
+ results.total_skipped_frames, Unit::kUnitless,
+ ImprovementDirection::kNeitherIsBetter);
+ logger.LogSingleValueMetric("Decode_errors_reference", label,
+ results.decode_errors_ref, Unit::kUnitless,
+ ImprovementDirection::kNeitherIsBetter);
+ logger.LogSingleValueMetric("Decode_errors_test", label,
+ results.decode_errors_test, Unit::kUnitless,
+ ImprovementDirection::kNeitherIsBetter);
}
} // namespace test
diff --git a/rtc_tools/frame_analyzer/video_quality_analysis.h b/rtc_tools/frame_analyzer/video_quality_analysis.h
index c5f3cb6..701b585 100644
--- a/rtc_tools/frame_analyzer/video_quality_analysis.h
+++ b/rtc_tools/frame_analyzer/video_quality_analysis.h
@@ -17,6 +17,7 @@
#include <vector>
#include "api/scoped_refptr.h"
+#include "api/test/metrics/metrics_logger.h"
#include "api/video/video_frame_buffer.h"
#include "rtc_tools/video_file_reader.h"
@@ -69,12 +70,9 @@
// Prints the result from the analysis in Chromium performance
// numbers compatible format to stdout. If the results object contains no frames
// no output will be written.
-void PrintAnalysisResults(const std::string& label, ResultsContainer* results);
-
-// Similar to the above, but will print to the specified file handle.
-void PrintAnalysisResults(FILE* output,
- const std::string& label,
- ResultsContainer* results);
+void PrintAnalysisResults(const std::string& label,
+ ResultsContainer& results,
+ MetricsLogger& logger);
struct Cluster {
// Corresponding reference frame index for this cluster.
diff --git a/rtc_tools/frame_analyzer/video_quality_analysis_unittest.cc b/rtc_tools/frame_analyzer/video_quality_analysis_unittest.cc
index 826cca0..d0227fb 100644
--- a/rtc_tools/frame_analyzer/video_quality_analysis_unittest.cc
+++ b/rtc_tools/frame_analyzer/video_quality_analysis_unittest.cc
@@ -7,102 +7,133 @@
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
-
-// This test doesn't actually verify the output since it's just printed
-// to stdout by void functions, but it's still useful as it executes the code.
-
#include "rtc_tools/frame_analyzer/video_quality_analysis.h"
-#include <stdio.h>
-
-#include <cstddef>
-#include <fstream>
#include <string>
+#include <vector>
+#include "api/test/metrics/metric.h"
+#include "api/test/metrics/metrics_logger.h"
+#include "system_wrappers/include/clock.h"
+#include "test/gmock.h"
#include "test/gtest.h"
#include "test/testsupport/file_utils.h"
namespace webrtc {
namespace test {
-
namespace {
-void VerifyLogOutput(const std::string& log_filename,
- const std::vector<std::string>& expected_out) {
- std::ifstream logf(log_filename);
- std::string line;
+using ::testing::IsSupersetOf;
- std::size_t i;
- for (i = 0; i < expected_out.size() && getline(logf, line); ++i) {
- ASSERT_EQ(expected_out.at(i), line);
- }
- ASSERT_TRUE(i == expected_out.size()) << "Not enough input data";
-}
-
-} // namespace
-
-// Setup a log file to write the output to instead of stdout because we don't
-// want those numbers to be picked up as perf numbers.
-class VideoQualityAnalysisTest : public ::testing::Test {
- protected:
- void SetUp() {
- std::string log_filename = TempFilename(webrtc::test::OutputPath(),
- "VideoQualityAnalysisTest.log");
- logfile_ = fopen(log_filename.c_str(), "w");
- ASSERT_TRUE(logfile_ != NULL);
- }
- void TearDown() { ASSERT_EQ(0, fclose(logfile_)); }
- FILE* logfile_;
+// Metric fields to assert on
+struct MetricValidationInfo {
+ std::string test_case;
+ std::string name;
+ Unit unit;
+ ImprovementDirection improvement_direction;
+ double mean;
};
-TEST_F(VideoQualityAnalysisTest, PrintAnalysisResultsEmpty) {
- ResultsContainer result;
- PrintAnalysisResults(logfile_, "Empty", &result);
+bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) {
+ return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit &&
+ a.improvement_direction == b.improvement_direction;
}
-TEST_F(VideoQualityAnalysisTest, PrintAnalysisResultsOneFrame) {
+std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) {
+ os << "{ test_case=" << m.test_case << "; name=" << m.name
+ << "; unit=" << test::ToString(m.unit)
+ << "; improvement_direction=" << test::ToString(m.improvement_direction)
+ << " }";
+ return os;
+}
+
+std::vector<MetricValidationInfo> ToValidationInfo(
+ const std::vector<Metric>& metrics) {
+ std::vector<MetricValidationInfo> out;
+ for (const Metric& m : metrics) {
+ out.push_back(
+ MetricValidationInfo{.test_case = m.test_case,
+ .name = m.name,
+ .unit = m.unit,
+ .improvement_direction = m.improvement_direction,
+ .mean = *m.stats.mean});
+ }
+ return out;
+}
+
+TEST(VideoQualityAnalysisTest, PrintAnalysisResultsEmpty) {
+ ResultsContainer result;
+ DefaultMetricsLogger logger(Clock::GetRealTimeClock());
+ PrintAnalysisResults("Empty", result, logger);
+}
+
+TEST(VideoQualityAnalysisTest, PrintAnalysisResultsOneFrame) {
ResultsContainer result;
result.frames.push_back(AnalysisResult(0, 35.0, 0.9));
- PrintAnalysisResults(logfile_, "OneFrame", &result);
+ DefaultMetricsLogger logger(Clock::GetRealTimeClock());
+ PrintAnalysisResults("OneFrame", result, logger);
}
-TEST_F(VideoQualityAnalysisTest, PrintAnalysisResultsThreeFrames) {
+TEST(VideoQualityAnalysisTest, PrintAnalysisResultsThreeFrames) {
ResultsContainer result;
result.frames.push_back(AnalysisResult(0, 35.0, 0.9));
result.frames.push_back(AnalysisResult(1, 34.0, 0.8));
result.frames.push_back(AnalysisResult(2, 33.0, 0.7));
- PrintAnalysisResults(logfile_, "ThreeFrames", &result);
+ DefaultMetricsLogger logger(Clock::GetRealTimeClock());
+ PrintAnalysisResults("ThreeFrames", result, logger);
}
-TEST_F(VideoQualityAnalysisTest,
- PrintMaxRepeatedAndSkippedFramesSkippedFrames) {
+TEST(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesSkippedFrames) {
ResultsContainer result;
- std::string log_filename =
- TempFilename(webrtc::test::OutputPath(), "log.log");
- FILE* logfile = fopen(log_filename.c_str(), "w");
- ASSERT_TRUE(logfile != NULL);
-
result.max_repeated_frames = 2;
result.max_skipped_frames = 2;
result.total_skipped_frames = 3;
result.decode_errors_ref = 0;
result.decode_errors_test = 0;
- PrintAnalysisResults(logfile, "NormalStatsFile", &result);
- ASSERT_EQ(0, fclose(logfile));
+ DefaultMetricsLogger logger(Clock::GetRealTimeClock());
+ PrintAnalysisResults("NormalStatsFile", result, logger);
- std::vector<std::string> expected_out = {
- "RESULT Max_repeated: NormalStatsFile= 2 ",
- "RESULT Max_skipped: NormalStatsFile= 2 ",
- "RESULT Total_skipped: NormalStatsFile= 3 ",
- "RESULT Decode_errors_reference: NormalStatsFile= 0 ",
- "RESULT Decode_errors_test: NormalStatsFile= 0 "};
- VerifyLogOutput(log_filename, expected_out);
+ std::vector<MetricValidationInfo> metrics =
+ ToValidationInfo(logger.GetCollectedMetrics());
+ EXPECT_THAT(
+ metrics,
+ IsSupersetOf(
+ {MetricValidationInfo{
+ .test_case = "NormalStatsFile",
+ .name = "Max_repeated",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .mean = 2},
+ MetricValidationInfo{
+ .test_case = "NormalStatsFile",
+ .name = "Max_skipped",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .mean = 2},
+ MetricValidationInfo{
+ .test_case = "NormalStatsFile",
+ .name = "Total_skipped",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .mean = 3},
+ MetricValidationInfo{
+ .test_case = "NormalStatsFile",
+ .name = "Decode_errors_reference",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .mean = 0},
+ MetricValidationInfo{
+ .test_case = "NormalStatsFile",
+ .name = "Decode_errors_test",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .mean = 0}}));
}
-TEST_F(VideoQualityAnalysisTest,
- PrintMaxRepeatedAndSkippedFramesDecodeErrorInTest) {
+TEST(VideoQualityAnalysisTest,
+ PrintMaxRepeatedAndSkippedFramesDecodeErrorInTest) {
ResultsContainer result;
std::string log_filename =
@@ -115,38 +146,67 @@
result.total_skipped_frames = 0;
result.decode_errors_ref = 0;
result.decode_errors_test = 3;
- PrintAnalysisResults(logfile, "NormalStatsFile", &result);
- ASSERT_EQ(0, fclose(logfile));
- std::vector<std::string> expected_out = {
- "RESULT Max_repeated: NormalStatsFile= 1 ",
- "RESULT Max_skipped: NormalStatsFile= 0 ",
- "RESULT Total_skipped: NormalStatsFile= 0 ",
- "RESULT Decode_errors_reference: NormalStatsFile= 0 ",
- "RESULT Decode_errors_test: NormalStatsFile= 3 "};
- VerifyLogOutput(log_filename, expected_out);
+ DefaultMetricsLogger logger(Clock::GetRealTimeClock());
+ PrintAnalysisResults("NormalStatsFile", result, logger);
+
+ std::vector<MetricValidationInfo> metrics =
+ ToValidationInfo(logger.GetCollectedMetrics());
+ EXPECT_THAT(
+ metrics,
+ IsSupersetOf(
+ {MetricValidationInfo{
+ .test_case = "NormalStatsFile",
+ .name = "Max_repeated",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .mean = 1},
+ MetricValidationInfo{
+ .test_case = "NormalStatsFile",
+ .name = "Max_skipped",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .mean = 0},
+ MetricValidationInfo{
+ .test_case = "NormalStatsFile",
+ .name = "Total_skipped",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .mean = 0},
+ MetricValidationInfo{
+ .test_case = "NormalStatsFile",
+ .name = "Decode_errors_reference",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .mean = 0},
+ MetricValidationInfo{
+ .test_case = "NormalStatsFile",
+ .name = "Decode_errors_test",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .mean = 3}}));
}
-TEST_F(VideoQualityAnalysisTest, CalculateFrameClustersOneValue) {
+TEST(VideoQualityAnalysisTest, CalculateFrameClustersOneValue) {
const std::vector<Cluster> result = CalculateFrameClusters({1});
EXPECT_EQ(1u, result.size());
EXPECT_EQ(1u, result[0].index);
EXPECT_EQ(1, result[0].number_of_repeated_frames);
}
-TEST_F(VideoQualityAnalysisTest, GetMaxRepeatedFramesOneValue) {
+TEST(VideoQualityAnalysisTest, GetMaxRepeatedFramesOneValue) {
EXPECT_EQ(1, GetMaxRepeatedFrames(CalculateFrameClusters({1})));
}
-TEST_F(VideoQualityAnalysisTest, GetMaxSkippedFramesOneValue) {
+TEST(VideoQualityAnalysisTest, GetMaxSkippedFramesOneValue) {
EXPECT_EQ(0, GetMaxSkippedFrames(CalculateFrameClusters({1})));
}
-TEST_F(VideoQualityAnalysisTest, GetTotalNumberOfSkippedFramesOneValue) {
+TEST(VideoQualityAnalysisTest, GetTotalNumberOfSkippedFramesOneValue) {
EXPECT_EQ(0, GetTotalNumberOfSkippedFrames(CalculateFrameClusters({1})));
}
-TEST_F(VideoQualityAnalysisTest, CalculateFrameClustersOneOneTwo) {
+TEST(VideoQualityAnalysisTest, CalculateFrameClustersOneOneTwo) {
const std::vector<Cluster> result = CalculateFrameClusters({1, 1, 2});
EXPECT_EQ(2u, result.size());
EXPECT_EQ(1u, result[0].index);
@@ -155,34 +215,35 @@
EXPECT_EQ(1, result[1].number_of_repeated_frames);
}
-TEST_F(VideoQualityAnalysisTest, GetMaxRepeatedFramesOneOneTwo) {
+TEST(VideoQualityAnalysisTest, GetMaxRepeatedFramesOneOneTwo) {
EXPECT_EQ(2, GetMaxRepeatedFrames(CalculateFrameClusters({1, 1, 2})));
}
-TEST_F(VideoQualityAnalysisTest, GetMaxSkippedFramesOneOneTwo) {
+TEST(VideoQualityAnalysisTest, GetMaxSkippedFramesOneOneTwo) {
EXPECT_EQ(0, GetMaxSkippedFrames(CalculateFrameClusters({1, 1, 2})));
}
-TEST_F(VideoQualityAnalysisTest, GetTotalNumberOfSkippedFramesOneOneTwo) {
+TEST(VideoQualityAnalysisTest, GetTotalNumberOfSkippedFramesOneOneTwo) {
EXPECT_EQ(0,
GetTotalNumberOfSkippedFrames(CalculateFrameClusters({1, 1, 2})));
}
-TEST_F(VideoQualityAnalysisTest, CalculateFrameClustersEmpty) {
+TEST(VideoQualityAnalysisTest, CalculateFrameClustersEmpty) {
EXPECT_TRUE(CalculateFrameClusters({}).empty());
}
-TEST_F(VideoQualityAnalysisTest, GetMaxRepeatedFramesEmpty) {
+TEST(VideoQualityAnalysisTest, GetMaxRepeatedFramesEmpty) {
EXPECT_EQ(0, GetMaxRepeatedFrames({}));
}
-TEST_F(VideoQualityAnalysisTest, GetMaxSkippedFramesEmpty) {
+TEST(VideoQualityAnalysisTest, GetMaxSkippedFramesEmpty) {
EXPECT_EQ(0, GetMaxSkippedFrames({}));
}
-TEST_F(VideoQualityAnalysisTest, GetTotalNumberOfSkippedFramesEmpty) {
+TEST(VideoQualityAnalysisTest, GetTotalNumberOfSkippedFramesEmpty) {
EXPECT_EQ(0, GetTotalNumberOfSkippedFrames({}));
}
+} // namespace
} // namespace test
} // namespace webrtc
