Reland "Report total and squared inter frame delays measured in OnRenderedFrame"
This is a reland of commit d49d49ad89e67d1a3c63fbc638af445af5648875
Fixed seconds to milliseconds conversion in VideoAnalyzer.
Original change's description:
> Report total and squared inter frame delays measured in OnRenderedFrame
>
> After https://webrtc-review.googlesource.com/c/src/+/160042 we ended up with two sets of metrics representing total and total squared inter frame delays: old is measured in OnDecodedFrame and new in OnRenderedFrame. Reporting of old metrics was unshipped in https://webrtc-review.googlesource.com/c/src/+/278100. The metrics are used for calculation of harmonic frame rate and are desired to be measured as close as possible to rendering. This CL removes calculation of inter frame delay metrics from OnDecodedFrame and reports the metrics calculated in OnRenderedFrame to the stats.
>
> Bug: webrtc:11108, b/261512902
> Change-Id: Ia21b321aab3a1ac0b6136dc0df7d95f2f0fd24c6
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/286842
> Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
> Reviewed-by: Erik Språng <sprang@webrtc.org>
> Reviewed-by: Harald Alvestrand <hta@webrtc.org>
> Reviewed-by: Henrik Boström <hbos@webrtc.org>
> Cr-Commit-Position: refs/heads/main@{#38909}
Bug: webrtc:11108, webrtc:14792, b/261512902
Change-Id: Ic5d0bc4622ee0cb46b6c225cdddccc217200e794
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/288641
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Reviewed-by: Johannes Kron <kron@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38929}
diff --git a/call/video_receive_stream.h b/call/video_receive_stream.h
index 2e2742a..0fa257e 100644
--- a/call/video_receive_stream.h
+++ b/call/video_receive_stream.h
@@ -132,8 +132,6 @@
uint32_t pause_count = 0;
uint32_t total_freezes_duration_ms = 0;
uint32_t total_pauses_duration_ms = 0;
- uint32_t total_frames_duration_ms = 0;
- double sum_squared_frame_durations = 0.0;
VideoContentType content_type = VideoContentType::UNSPECIFIED;
diff --git a/media/base/media_channel.h b/media/base/media_channel.h
index ee9a472..233f118 100644
--- a/media/base/media_channel.h
+++ b/media/base/media_channel.h
@@ -628,8 +628,6 @@
uint32_t pause_count = 0;
uint32_t total_freezes_duration_ms = 0;
uint32_t total_pauses_duration_ms = 0;
- uint32_t total_frames_duration_ms = 0;
- double sum_squared_frame_durations = 0.0;
uint32_t jitter_ms = 0;
webrtc::VideoContentType content_type = webrtc::VideoContentType::UNSPECIFIED;
diff --git a/video/g3doc/stats.md b/video/g3doc/stats.md
index 669651e..0a423e1 100644
--- a/video/g3doc/stats.md
+++ b/video/g3doc/stats.md
@@ -148,8 +148,6 @@
* `qp_sum` - sum of quantizer values of decoded frames [[rtcinboundrtpstreamstats-qpsum]].
* `content_type` - content type (UNSPECIFIED/SCREENSHARE).
* `interframe_delay_max_ms` - max inter-frame delay within a time window between decoded frames.
-* `total_inter_frame_delay` - sum of inter-frame delay in seconds between decoded frames [[rtcinboundrtpstreamstats-totalinterframedelay]].
-* `total_squared_inter_frame_delay` - sum of squared inter-frame delays in seconds between decoded frames [[rtcinboundrtpstreamstats-totalsquaredinterframedelay]].
Updated before a frame is sent to the renderer, `VideoReceiveStream2::OnFrame`.
* `frames_rendered` - total number of rendered frames.
@@ -162,8 +160,8 @@
* `pause_count` - total number of detected pauses.
* `total_freezes_duration_ms` - total duration of freezes in ms.
* `total_pauses_duration_ms` - total duration of pauses in ms.
-* `total_frames_duration_ms` - time in ms between the last rendered frame and the first rendered frame.
-* `sum_squared_frame_durations` - sum of squared inter-frame delays in seconds between rendered frames.
+* `total_inter_frame_delay` - sum of inter-frame delay in seconds between rendered frames [[rtcinboundrtpstreamstats-totalinterframedelay]].
+* `total_squared_inter_frame_delay` - sum of squared inter-frame delays in seconds between rendered frames [[rtcinboundrtpstreamstats-totalsquaredinterframedelay]].
`ReceiveStatisticsImpl::OnRtpPacket` is updated for received RTP packets. From `ReceiveStatistics`:
* `total_bitrate_bps` - incoming bitrate in bps.
diff --git a/video/receive_statistics_proxy2.cc b/video/receive_statistics_proxy2.cc
index 297f5d3..4f208a1 100644
--- a/video/receive_statistics_proxy2.cc
+++ b/video/receive_statistics_proxy2.cc
@@ -624,10 +624,12 @@
video_quality_observer_->TotalFreezesDurationMs();
stats_.total_pauses_duration_ms =
video_quality_observer_->TotalPausesDurationMs();
- stats_.total_frames_duration_ms =
- video_quality_observer_->TotalFramesDurationMs();
- stats_.sum_squared_frame_durations =
+ stats_.total_inter_frame_delay =
+ static_cast<double>(video_quality_observer_->TotalFramesDurationMs()) /
+ rtc::kNumMillisecsPerSec;
+ stats_.total_squared_inter_frame_delay =
video_quality_observer_->SumSquaredFrameDurationsSec();
+
stats_.content_type = last_content_type_;
stats_.timing_frame_info = timing_frame_info_counter_.Max(now_ms);
stats_.jitter_buffer_delay_seconds =
@@ -839,10 +841,6 @@
int64_t interframe_delay_ms =
frame_meta.decode_timestamp.ms() - *last_decoded_frame_time_ms_;
RTC_DCHECK_GE(interframe_delay_ms, 0);
- double interframe_delay = interframe_delay_ms / 1000.0;
- stats_.total_inter_frame_delay += interframe_delay;
- stats_.total_squared_inter_frame_delay +=
- interframe_delay * interframe_delay;
interframe_delay_max_moving_.Add(interframe_delay_ms,
frame_meta.decode_timestamp.ms());
content_specific_stats->interframe_delay_counter.Add(interframe_delay_ms);
diff --git a/video/receive_statistics_proxy2_unittest.cc b/video/receive_statistics_proxy2_unittest.cc
index f0869c4..0c628f7 100644
--- a/video/receive_statistics_proxy2_unittest.cc
+++ b/video/receive_statistics_proxy2_unittest.cc
@@ -328,62 +328,6 @@
videocontenttypehelpers::ToString(FlushAndGetStats().content_type));
}
-TEST_F(ReceiveStatisticsProxy2Test, ReportsMaxTotalInterFrameDelay) {
- webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
- const TimeDelta kInterFrameDelay1 = TimeDelta::Millis(100);
- const TimeDelta kInterFrameDelay2 = TimeDelta::Millis(200);
- const TimeDelta kInterFrameDelay3 = TimeDelta::Millis(300);
- double expected_total_inter_frame_delay = 0;
- double expected_total_squared_inter_frame_delay = 0;
- EXPECT_EQ(expected_total_inter_frame_delay,
- statistics_proxy_->GetStats().total_inter_frame_delay);
- EXPECT_EQ(expected_total_squared_inter_frame_delay,
- statistics_proxy_->GetStats().total_squared_inter_frame_delay);
-
- statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
- EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay,
- FlushAndGetStats().total_inter_frame_delay);
- EXPECT_DOUBLE_EQ(expected_total_squared_inter_frame_delay,
- FlushAndGetStats().total_squared_inter_frame_delay);
-
- time_controller_.AdvanceTime(kInterFrameDelay1);
- statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
- expected_total_inter_frame_delay += kInterFrameDelay1.seconds<double>();
- expected_total_squared_inter_frame_delay +=
- pow(kInterFrameDelay1.seconds<double>(), 2.0);
- EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay,
- FlushAndGetStats().total_inter_frame_delay);
- EXPECT_DOUBLE_EQ(
- expected_total_squared_inter_frame_delay,
- statistics_proxy_->GetStats().total_squared_inter_frame_delay);
-
- time_controller_.AdvanceTime(kInterFrameDelay2);
- statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
- expected_total_inter_frame_delay += kInterFrameDelay2.seconds<double>();
- expected_total_squared_inter_frame_delay +=
- pow(kInterFrameDelay2.seconds<double>(), 2.0);
- EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay,
- FlushAndGetStats().total_inter_frame_delay);
- EXPECT_DOUBLE_EQ(
- expected_total_squared_inter_frame_delay,
- statistics_proxy_->GetStats().total_squared_inter_frame_delay);
-
- time_controller_.AdvanceTime(kInterFrameDelay3);
- statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
- expected_total_inter_frame_delay += kInterFrameDelay3.seconds<double>();
- expected_total_squared_inter_frame_delay +=
- pow(kInterFrameDelay3.seconds<double>(), 2.0);
- EXPECT_DOUBLE_EQ(expected_total_inter_frame_delay,
- FlushAndGetStats().total_inter_frame_delay);
- EXPECT_DOUBLE_EQ(
- expected_total_squared_inter_frame_delay,
- statistics_proxy_->GetStats().total_squared_inter_frame_delay);
-}
-
TEST_F(ReceiveStatisticsProxy2Test, ReportsMaxInterframeDelay) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
const TimeDelta kInterframeDelay1 = TimeDelta::Millis(100);
@@ -503,9 +447,9 @@
EXPECT_EQ(0u, stats.total_pauses_duration_ms);
}
-TEST_F(ReceiveStatisticsProxy2Test, ReportsFramesDuration) {
+TEST_F(ReceiveStatisticsProxy2Test, ReportsTotalInterFrameDelay) {
VideoReceiveStreamInterface::Stats stats = statistics_proxy_->GetStats();
- ASSERT_EQ(0u, stats.total_frames_duration_ms);
+ ASSERT_EQ(0.0, stats.total_inter_frame_delay);
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
@@ -519,12 +463,12 @@
}
stats = statistics_proxy_->GetStats();
- EXPECT_EQ(10 * 30u, stats.total_frames_duration_ms);
+ EXPECT_EQ(10 * 30 / 1000.0, stats.total_inter_frame_delay);
}
-TEST_F(ReceiveStatisticsProxy2Test, ReportsSumSquaredFrameDurations) {
+TEST_F(ReceiveStatisticsProxy2Test, ReportsTotalSquaredInterFrameDelay) {
VideoReceiveStreamInterface::Stats stats = statistics_proxy_->GetStats();
- ASSERT_EQ(0u, stats.sum_squared_frame_durations);
+ ASSERT_EQ(0.0, stats.total_squared_inter_frame_delay);
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
for (int i = 0; i <= 10; ++i) {
@@ -533,10 +477,10 @@
}
stats = statistics_proxy_->GetStats();
- const double kExpectedSumSquaredFrameDurationsSecs =
+ const double kExpectedTotalSquaredInterFrameDelaySecs =
10 * (30 / 1000.0 * 30 / 1000.0);
- EXPECT_EQ(kExpectedSumSquaredFrameDurationsSecs,
- stats.sum_squared_frame_durations);
+ EXPECT_EQ(kExpectedTotalSquaredInterFrameDelaySecs,
+ stats.total_squared_inter_frame_delay);
}
TEST_F(ReceiveStatisticsProxy2Test, OnDecodedFrameWithoutQpQpSumWontExist) {
diff --git a/video/video_analyzer.cc b/video/video_analyzer.cc
index 3077a77..030c605 100644
--- a/video/video_analyzer.cc
+++ b/video/video_analyzer.cc
@@ -99,8 +99,8 @@
mean_decode_time_ms_(0.0),
freeze_count_(0),
total_freezes_duration_ms_(0),
- total_frames_duration_ms_(0),
- sum_squared_frame_durations_(0),
+ total_inter_frame_delay_(0),
+ total_squared_inter_frame_delay_(0),
decode_frame_rate_(0),
render_frame_rate_(0),
last_fec_bytes_(0),
@@ -502,6 +502,14 @@
if (receive_stream_ != nullptr) {
VideoReceiveStreamInterface::Stats receive_stats =
receive_stream_->GetStats();
+
+ // Freeze metrics.
+ freeze_count_ = receive_stats.freeze_count;
+ total_freezes_duration_ms_ = receive_stats.total_freezes_duration_ms;
+ total_inter_frame_delay_ = receive_stats.total_inter_frame_delay;
+ total_squared_inter_frame_delay_ =
+ receive_stats.total_squared_inter_frame_delay;
+
// `total_decode_time_ms` gives a good estimate of the mean decode time,
// `decode_ms` is used to keep track of the standard deviation.
if (receive_stats.frames_decoded > 0)
@@ -518,20 +526,12 @@
// `frames_decoded` and `frames_rendered` are used because they are more
// accurate than `decode_frame_rate` and `render_frame_rate`.
// The latter two are calculated on a momentary basis.
- const double total_frames_duration_sec_double =
- static_cast<double>(receive_stats.total_frames_duration_ms) / 1000.0;
- if (total_frames_duration_sec_double > 0) {
- decode_frame_rate_ = static_cast<double>(receive_stats.frames_decoded) /
- total_frames_duration_sec_double;
- render_frame_rate_ = static_cast<double>(receive_stats.frames_rendered) /
- total_frames_duration_sec_double;
+ if (total_inter_frame_delay_ > 0) {
+ decode_frame_rate_ =
+ receive_stats.frames_decoded / total_inter_frame_delay_;
+ render_frame_rate_ =
+ receive_stats.frames_rendered / total_inter_frame_delay_;
}
-
- // Freeze metrics.
- freeze_count_ = receive_stats.freeze_count;
- total_freezes_duration_ms_ = receive_stats.total_freezes_duration_ms;
- total_frames_duration_ms_ = receive_stats.total_frames_duration_ms;
- sum_squared_frame_durations_ = receive_stats.sum_squared_frame_durations;
}
if (audio_receive_stream_ != nullptr) {
@@ -673,7 +673,7 @@
const double total_freezes_duration_ms_double =
static_cast<double>(total_freezes_duration_ms_);
const double total_frames_duration_ms_double =
- static_cast<double>(total_frames_duration_ms_);
+ total_inter_frame_delay_ * rtc::kNumMillisecsPerSec;
if (total_frames_duration_ms_double > 0) {
GetGlobalMetricsLogger()->LogSingleValueMetric(
@@ -701,10 +701,11 @@
: 0,
Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter);
- if (1000 * sum_squared_frame_durations_ > 0) {
+ if (total_squared_inter_frame_delay_ > 0) {
GetGlobalMetricsLogger()->LogSingleValueMetric(
"harmonic_frame_rate_fps", test_label_,
- total_frames_duration_ms_double / (1000 * sum_squared_frame_durations_),
+ total_frames_duration_ms_double /
+ (1000 * total_squared_inter_frame_delay_),
Unit::kHertz, ImprovementDirection::kBiggerIsBetter);
}
diff --git a/video/video_analyzer.h b/video/video_analyzer.h
index 2cee5e1..9186d78 100644
--- a/video/video_analyzer.h
+++ b/video/video_analyzer.h
@@ -263,8 +263,8 @@
SamplesStatsCounter time_between_freezes_ RTC_GUARDED_BY(comparison_lock_);
uint32_t freeze_count_ RTC_GUARDED_BY(comparison_lock_);
uint32_t total_freezes_duration_ms_ RTC_GUARDED_BY(comparison_lock_);
- uint32_t total_frames_duration_ms_ RTC_GUARDED_BY(comparison_lock_);
- double sum_squared_frame_durations_ RTC_GUARDED_BY(comparison_lock_);
+ double total_inter_frame_delay_ RTC_GUARDED_BY(comparison_lock_);
+ double total_squared_inter_frame_delay_ RTC_GUARDED_BY(comparison_lock_);
double decode_frame_rate_ RTC_GUARDED_BY(comparison_lock_);
double render_frame_rate_ RTC_GUARDED_BY(comparison_lock_);
