[DVQA] Make harmonic fps precomputed
Also add assertion on it for some DVQA tests
Bug: webrtc:14995, b/271542055
Change-Id: Ie35a85832b6d860885366fb613700bdef3db38f0
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/297820
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#39611}
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
index 96269a6..1b96423 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
@@ -1158,26 +1158,6 @@
{MetricMetadataKey::kReceiverMetadataKey, peers_->name(key.receiver)},
{MetricMetadataKey::kExperimentalTestNameMetadataKey, test_label_}};
- double sum_squared_interframe_delays_secs = 0;
- double video_duration_ms = 0;
- for (const SamplesStatsCounter::StatsSample& sample :
- stats.time_between_rendered_frames_ms.GetTimedSamples()) {
- double interframe_delay_ms = sample.value;
- const double interframe_delays_secs = interframe_delay_ms / 1000.0;
- // Sum of squared inter frame intervals is used to calculate the harmonic
- // frame rate metric. The metric aims to reflect overall experience related
- // to smoothness of video playback and includes both freezes and pauses.
- sum_squared_interframe_delays_secs +=
- interframe_delays_secs * interframe_delays_secs;
-
- video_duration_ms += sample.value;
- }
- double harmonic_framerate_fps = 0;
- if (sum_squared_interframe_delays_secs > 0.0) {
- harmonic_framerate_fps =
- video_duration_ms / 1000.0 / sum_squared_interframe_delays_secs;
- }
-
metrics_logger_->LogMetric(
"psnr_dB", test_case_name, stats.psnr, Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter, metric_metadata);
@@ -1197,7 +1177,7 @@
stats.time_between_rendered_frames_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
- "harmonic_framerate", test_case_name, harmonic_framerate_fps,
+ "harmonic_framerate", test_case_name, stats.harmonic_framerate_fps,
Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"encode_frame_rate", test_case_name,
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc
index fc91dd1..7ea2a2a 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc
@@ -37,6 +37,7 @@
constexpr TimeDelta kFreezeThreshold = TimeDelta::Millis(150);
constexpr int kMaxActiveComparisons = 10;
+constexpr int kMillisInSecond = 1000;
SamplesStatsCounter::StatsSample StatsSample(
double value,
@@ -235,13 +236,35 @@
Now(), /*metadata=*/{}));
}
- // Freeze Time:
- // If there were no freezes on a video stream, add only one sample with
- // value 0 (0ms freezes time).
for (auto& [key, stream_stats] : stream_stats_) {
+ // Freeze Time:
+ // If there were no freezes on a video stream, add only one sample with
+ // value 0 (0ms freezes time).
if (stream_stats.freeze_time_ms.IsEmpty()) {
stream_stats.freeze_time_ms.AddSample(0);
}
+
+ // Harmonic framerate (fps):
+ // sum of interframe delays / squared sum of interframe delays.
+ // The metric aims to reflect overall experience related to smoothness of
+ // video playback and includes both freezes and pauses.
+ double sum_squared_interframe_delays_secs = 0;
+ double sum_interframe_delays_ms = 0;
+ for (const SamplesStatsCounter::StatsSample& sample :
+ stream_stats.time_between_rendered_frames_ms.GetTimedSamples()) {
+ double interframe_delay_ms = sample.value;
+ const double interframe_delays_secs =
+ interframe_delay_ms / static_cast<double>(kMillisInSecond);
+ sum_squared_interframe_delays_secs +=
+ interframe_delays_secs * interframe_delays_secs;
+
+ sum_interframe_delays_ms += interframe_delay_ms;
+ }
+ if (sum_squared_interframe_delays_secs > 0.0) {
+ stream_stats.harmonic_framerate_fps =
+ sum_interframe_delays_ms / static_cast<double>(kMillisInSecond) /
+ sum_squared_interframe_delays_secs;
+ }
}
}
}
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h
index 17e7e0e..e8ded4f 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h
@@ -164,6 +164,8 @@
std::vector<StreamCodecInfo> encoders;
// Vectors of decoders used for this stream by receiving client.
std::vector<StreamCodecInfo> decoders;
+
+ double harmonic_framerate_fps = 0;
};
struct AnalyzerStats {
diff --git a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc
index 534e56c..4222fcc 100644
--- a/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc
+++ b/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc
@@ -2147,7 +2147,7 @@
ExpectRateIs(bob_stream_stats.capture_frame_rate, 13.559322);
// TODO(bugs.webrtc.org/14995): Fix encode_frame_rate (has to be ~20.0)
ExpectRateIs(bob_stream_stats.encode_frame_rate, 13.559322);
- // TODO(bugs.webrtc.org/14995): Assert on harmonic fps
+ EXPECT_DOUBLE_EQ(bob_stream_stats.harmonic_framerate_fps, 20);
StreamStats charlie_stream_stats =
streams_stats.at(StatsKey("alice_video", "charlie"));
@@ -2159,7 +2159,7 @@
ElementsAre(1050.0));
EXPECT_THAT(GetTimeSortedValues(charlie_stream_stats.time_between_freezes_ms),
ElementsAre(950.0, 950.0));
- // TODO(bugs.webrtc.org/14995): Assert on harmonic fps
+ EXPECT_NEAR(charlie_stream_stats.harmonic_framerate_fps, 2.463465, 1e-6);
}
TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest,
@@ -2217,7 +2217,7 @@
ElementsAre(950.0));
ExpectRateIs(bob_stream_stats.capture_frame_rate, 21.052631);
ExpectRateIs(bob_stream_stats.encode_frame_rate, 21.052631);
- // TODO(bugs.webrtc.org/14995): Assert on harmonic fps
+ EXPECT_DOUBLE_EQ(bob_stream_stats.harmonic_framerate_fps, 20);
StreamStats charlie_stream_stats =
streams_stats.at(StatsKey("alice_video", "charlie"));
@@ -2229,7 +2229,7 @@
ElementsAre(1050.0));
EXPECT_THAT(GetTimeSortedValues(charlie_stream_stats.time_between_freezes_ms),
ElementsAre(950.0, 950.0));
- // TODO(bugs.webrtc.org/14995): Assert on harmonic fps
+ EXPECT_NEAR(charlie_stream_stats.harmonic_framerate_fps, 2.463465, 1e-6);
}
TEST_F(DefaultVideoQualityAnalyzerSimulatedTimeTest,
