Reland "Fix definition of keyframes decoded statistics"
This is a reland of commit 0e37f5ebd44183d9fe5318d844235aae28fda86a
with backward compability added to allow downstream tests to migrate to the new signature.
Original change's description:
> Fix definition of keyframes decoded statistics
>
> which are defined to be measured after decoding, not before:
> https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-keyframesdecoded
>
> BUG=webrtc:14728
>
> Change-Id: I0a83dde278e1ebe8acf787bdac729af369a1ecf8
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/315520
> Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
> Commit-Queue: Philipp Hancke <phancke@microsoft.com>
> Reviewed-by: Henrik Boström <hbos@webrtc.org>
> Cr-Commit-Position: refs/heads/main@{#40545}
BUG=webrtc:14728
Change-Id: I4cf52bb22ba8244155b4fa8c367b9c0306a77590
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/316120
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Philipp Hancke <phancke@microsoft.com>
Cr-Commit-Position: refs/heads/main@{#40553}
diff --git a/modules/video_coding/generic_decoder.cc b/modules/video_coding/generic_decoder.cc
index 46b8c95..4c0e381 100644
--- a/modules/video_coding/generic_decoder.cc
+++ b/modules/video_coding/generic_decoder.cc
@@ -201,7 +201,8 @@
decodedImage.set_timestamp_us(
frame_info->render_time ? frame_info->render_time->us() : -1);
_receiveCallback->FrameToRender(decodedImage, qp, decode_time,
- frame_info->content_type);
+ frame_info->content_type,
+ frame_info->frame_type);
}
void VCMDecodedFrameCallback::OnDecoderInfoChanged(
@@ -285,6 +286,7 @@
} else {
frame_info.content_type = _last_keyframe_content_type;
}
+ frame_info.frame_type = frame.FrameType();
_callback->Map(std::move(frame_info));
int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(),
diff --git a/modules/video_coding/generic_decoder.h b/modules/video_coding/generic_decoder.h
index 7dc6d34..5922b52 100644
--- a/modules/video_coding/generic_decoder.h
+++ b/modules/video_coding/generic_decoder.h
@@ -46,6 +46,7 @@
int64_t ntp_time_ms;
RtpPacketInfos packet_infos;
// ColorSpace is not stored here, as it might be modified by decoders.
+ VideoFrameType frame_type;
};
class VCMDecodedFrameCallback : public DecodedImageCallback {
diff --git a/modules/video_coding/generic_decoder_unittest.cc b/modules/video_coding/generic_decoder_unittest.cc
index 68bc307..529f13f 100644
--- a/modules/video_coding/generic_decoder_unittest.cc
+++ b/modules/video_coding/generic_decoder_unittest.cc
@@ -35,7 +35,8 @@
int32_t FrameToRender(VideoFrame& frame,
absl::optional<uint8_t> qp,
TimeDelta decode_time,
- VideoContentType content_type) override {
+ VideoContentType content_type,
+ VideoFrameType frame_type) override {
frames_.push_back(frame);
return 0;
}
diff --git a/modules/video_coding/include/video_coding_defines.h b/modules/video_coding/include/video_coding_defines.h
index fe7d0fb..bd71a8d 100644
--- a/modules/video_coding/include/video_coding_defines.h
+++ b/modules/video_coding/include/video_coding_defines.h
@@ -34,7 +34,7 @@
enum {
// Timing frames settings. Timing frames are sent every
// `kDefaultTimingFramesDelayMs`, or if the frame is at least
- // `kDefaultOutliserFrameSizePercent` in size of average frame.
+ // `kDefaultOutlierFrameSizePercent` in size of average frame.
kDefaultTimingFramesDelayMs = 200,
kDefaultOutlierFrameSizePercent = 500,
// Maximum number of frames for what we store encode start timing information.
@@ -50,10 +50,22 @@
// rendered.
class VCMReceiveCallback {
public:
+ // TODO(bugs.webrtc.org/14728): make pure virtual again.
virtual int32_t FrameToRender(VideoFrame& videoFrame, // NOLINT
absl::optional<uint8_t> qp,
TimeDelta decode_time,
- VideoContentType content_type) = 0;
+ VideoContentType content_type,
+ VideoFrameType frame_type) {
+ return FrameToRender(videoFrame, qp, decode_time, content_type);
+ }
+ // TODO(bugs.webrtc.org/14728): remove this signature.
+ virtual int32_t FrameToRender(VideoFrame& videoFrame, // NOLINT
+ absl::optional<uint8_t> qp,
+ TimeDelta decode_time,
+ VideoContentType content_type) {
+ return FrameToRender(videoFrame, qp, decode_time, content_type,
+ VideoFrameType::kEmptyFrame);
+ }
virtual void OnDroppedFrames(uint32_t frames_dropped);
diff --git a/modules/video_coding/video_receiver2_unittest.cc b/modules/video_coding/video_receiver2_unittest.cc
index 6edf123..b524316 100644
--- a/modules/video_coding/video_receiver2_unittest.cc
+++ b/modules/video_coding/video_receiver2_unittest.cc
@@ -35,11 +35,14 @@
public:
MockVCMReceiveCallback() = default;
- MOCK_METHOD(
- int32_t,
- FrameToRender,
- (VideoFrame&, absl::optional<uint8_t>, TimeDelta, VideoContentType),
- (override));
+ MOCK_METHOD(int32_t,
+ FrameToRender,
+ (VideoFrame&,
+ absl::optional<uint8_t>,
+ TimeDelta,
+ VideoContentType,
+ VideoFrameType),
+ (override));
MOCK_METHOD(void, OnIncomingPayloadType, (int), (override));
MOCK_METHOD(void,
OnDecoderInfoChanged,
diff --git a/modules/video_coding/video_receiver_unittest.cc b/modules/video_coding/video_receiver_unittest.cc
index fe9674e..8a70664 100644
--- a/modules/video_coding/video_receiver_unittest.cc
+++ b/modules/video_coding/video_receiver_unittest.cc
@@ -38,11 +38,14 @@
MockVCMReceiveCallback() {}
virtual ~MockVCMReceiveCallback() {}
- MOCK_METHOD(
- int32_t,
- FrameToRender,
- (VideoFrame&, absl::optional<uint8_t>, TimeDelta, VideoContentType),
- (override));
+ MOCK_METHOD(int32_t,
+ FrameToRender,
+ (VideoFrame&,
+ absl::optional<uint8_t>,
+ TimeDelta,
+ VideoContentType,
+ VideoFrameType),
+ (override));
MOCK_METHOD(void, OnIncomingPayloadType, (int), (override));
MOCK_METHOD(void,
OnDecoderInfoChanged,
diff --git a/video/receive_statistics_proxy.cc b/video/receive_statistics_proxy.cc
index 5bb3004..75512a2 100644
--- a/video/receive_statistics_proxy.cc
+++ b/video/receive_statistics_proxy.cc
@@ -592,7 +592,8 @@
void ReceiveStatisticsProxy::OnDecodedFrame(const VideoFrame& frame,
absl::optional<uint8_t> qp,
TimeDelta decode_time,
- VideoContentType content_type) {
+ VideoContentType content_type,
+ VideoFrameType frame_type) {
TimeDelta processing_delay = TimeDelta::Zero();
webrtc::Timestamp current_time = clock_->CurrentTime();
// TODO(bugs.webrtc.org/13984): some tests do not fill packet_infos().
@@ -615,11 +616,11 @@
// may be on. E.g. on iOS this gets called on
// "com.apple.coremedia.decompressionsession.clientcallback"
VideoFrameMetaData meta(frame, current_time);
- worker_thread_->PostTask(
- SafeTask(task_safety_.flag(), [meta, qp, decode_time, processing_delay,
- assembly_time, content_type, this]() {
+ worker_thread_->PostTask(SafeTask(
+ task_safety_.flag(), [meta, qp, decode_time, processing_delay,
+ assembly_time, content_type, frame_type, this]() {
OnDecodedFrame(meta, qp, decode_time, processing_delay, assembly_time,
- content_type);
+ content_type, frame_type);
}));
}
@@ -629,7 +630,8 @@
TimeDelta decode_time,
TimeDelta processing_delay,
TimeDelta assembly_time,
- VideoContentType content_type) {
+ VideoContentType content_type,
+ VideoFrameType frame_type) {
RTC_DCHECK_RUN_ON(&main_thread_);
const bool is_screenshare =
@@ -651,6 +653,11 @@
&content_specific_stats_[content_type];
++stats_.frames_decoded;
+ if (frame_type == VideoFrameType::kVideoFrameKey) {
+ ++stats_.frame_counts.key_frames;
+ } else {
+ ++stats_.frame_counts.delta_frames;
+ }
if (qp) {
if (!stats_.qp_sum) {
if (stats_.frames_decoded != 1) {
@@ -760,12 +767,6 @@
VideoContentType content_type) {
RTC_DCHECK_RUN_ON(&main_thread_);
- if (is_keyframe) {
- ++stats_.frame_counts.key_frames;
- } else {
- ++stats_.frame_counts.delta_frames;
- }
-
// Content type extension is set only for keyframes and should be propagated
// for all the following delta frames. Here we may receive frames out of order
// and miscategorise some delta frames near the layer switch.
diff --git a/video/receive_statistics_proxy.h b/video/receive_statistics_proxy.h
index d8da306..8e4941f 100644
--- a/video/receive_statistics_proxy.h
+++ b/video/receive_statistics_proxy.h
@@ -58,7 +58,8 @@
void OnDecodedFrame(const VideoFrame& frame,
absl::optional<uint8_t> qp,
TimeDelta decode_time,
- VideoContentType content_type);
+ VideoContentType content_type,
+ VideoFrameType frame_type);
// Called asyncronously on the worker thread as a result of a call to the
// above OnDecodedFrame method, which is called back on the thread where
@@ -68,7 +69,8 @@
TimeDelta decode_time,
TimeDelta processing_delay,
TimeDelta assembly_time,
- VideoContentType content_type);
+ VideoContentType content_type,
+ VideoFrameType frame_type);
void OnSyncOffsetUpdated(int64_t video_playout_ntp_ms,
int64_t sync_offset_ms,
diff --git a/video/receive_statistics_proxy_unittest.cc b/video/receive_statistics_proxy_unittest.cc
index bc11efe..a30a7e4 100644
--- a/video/receive_statistics_proxy_unittest.cc
+++ b/video/receive_statistics_proxy_unittest.cc
@@ -115,7 +115,8 @@
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
for (uint32_t i = 1; i <= 3; ++i) {
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
EXPECT_EQ(i, FlushAndGetStats().frames_decoded);
}
}
@@ -127,7 +128,8 @@
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
for (int i = 0; i < kRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(1 / kFps);
}
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);
@@ -144,7 +146,8 @@
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
for (int i = 0; i < kRequiredSamples - 1; ++i) {
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(1 / kFps);
}
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);
@@ -159,9 +162,9 @@
TimeDelta expected_total_decode_time = TimeDelta::Zero();
unsigned int expected_frames_decoded = 0;
for (uint32_t i = 1; i <= 3; ++i) {
- statistics_proxy_->OnDecodedFrame(frame, absl::nullopt,
- TimeDelta::Millis(1),
- VideoContentType::UNSPECIFIED);
+ statistics_proxy_->OnDecodedFrame(
+ frame, absl::nullopt, TimeDelta::Millis(1),
+ VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameKey);
expected_total_decode_time += TimeDelta::Millis(1);
++expected_frames_decoded;
time_controller_.AdvanceTime(TimeDelta::Zero());
@@ -171,7 +174,8 @@
statistics_proxy_->GetStats().total_decode_time);
}
statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
++expected_frames_decoded;
expected_total_decode_time += TimeDelta::Millis(3);
time_controller_.AdvanceTime(TimeDelta::Zero());
@@ -195,9 +199,9 @@
frame.set_packet_infos(RtpPacketInfos(packet_infos));
for (int i = 1; i <= 3; ++i) {
time_controller_.AdvanceTime(kProcessingDelay);
- statistics_proxy_->OnDecodedFrame(frame, absl::nullopt,
- TimeDelta::Millis(1),
- VideoContentType::UNSPECIFIED);
+ statistics_proxy_->OnDecodedFrame(
+ frame, absl::nullopt, TimeDelta::Millis(1),
+ VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameKey);
expected_total_processing_delay += i * kProcessingDelay;
++expected_frames_decoded;
time_controller_.AdvanceTime(TimeDelta::Zero());
@@ -208,7 +212,8 @@
}
time_controller_.AdvanceTime(kProcessingDelay);
statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
++expected_frames_decoded;
expected_total_processing_delay += 4 * kProcessingDelay;
time_controller_.AdvanceTime(TimeDelta::Zero());
@@ -232,7 +237,8 @@
/*ssrc=*/{}, /*csrcs=*/{}, /*rtp_timestamp=*/{}, /*receive_time=*/Now())};
frame.set_packet_infos(RtpPacketInfos(single_packet_frame));
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Millis(1),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
++expected_frames_decoded;
time_controller_.AdvanceTime(TimeDelta::Zero());
EXPECT_EQ(expected_total_assembly_time,
@@ -252,7 +258,8 @@
};
frame.set_packet_infos(RtpPacketInfos(ordered_frame));
statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
++expected_frames_decoded;
++expected_frames_assembled_from_multiple_packets;
expected_total_assembly_time += 2 * kAssemblyTime;
@@ -276,7 +283,8 @@
};
frame.set_packet_infos(RtpPacketInfos(unordered_frame));
statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
++expected_frames_decoded;
++expected_frames_assembled_from_multiple_packets;
expected_total_assembly_time += 2 * kAssemblyTime;
@@ -294,10 +302,12 @@
EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum);
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
EXPECT_EQ(3u, FlushAndGetStats().qp_sum);
statistics_proxy_->OnDecodedFrame(frame, 127u, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameDelta);
EXPECT_EQ(130u, FlushAndGetStats().qp_sum);
}
@@ -305,10 +315,12 @@
EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum);
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Millis(4),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
EXPECT_EQ(4u, FlushAndGetStats().total_decode_time.ms());
statistics_proxy_->OnDecodedFrame(frame, 127u, TimeDelta::Millis(7),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameDelta);
EXPECT_EQ(11u, FlushAndGetStats().total_decode_time.ms());
}
@@ -319,11 +331,13 @@
EXPECT_EQ(kRealtimeString, videocontenttypehelpers::ToString(
statistics_proxy_->GetStats().content_type));
statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(),
- VideoContentType::SCREENSHARE);
+ VideoContentType::SCREENSHARE,
+ VideoFrameType::kVideoFrameKey);
EXPECT_EQ(kScreenshareString,
videocontenttypehelpers::ToString(FlushAndGetStats().content_type));
statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameDelta);
EXPECT_EQ(kRealtimeString,
videocontenttypehelpers::ToString(FlushAndGetStats().content_type));
}
@@ -335,22 +349,26 @@
const TimeDelta kInterframeDelay3 = TimeDelta::Millis(100);
EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
EXPECT_EQ(-1, FlushAndGetStats().interframe_delay_max_ms);
time_controller_.AdvanceTime(kInterframeDelay1);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameDelta);
EXPECT_EQ(kInterframeDelay1.ms(), FlushAndGetStats().interframe_delay_max_ms);
time_controller_.AdvanceTime(kInterframeDelay2);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameDelta);
EXPECT_EQ(kInterframeDelay2.ms(), FlushAndGetStats().interframe_delay_max_ms);
time_controller_.AdvanceTime(kInterframeDelay3);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameDelta);
// kInterframeDelay3 is smaller than kInterframeDelay2.
EXPECT_EQ(kInterframeDelay2.ms(), FlushAndGetStats().interframe_delay_max_ms);
}
@@ -362,23 +380,27 @@
const TimeDelta kInterframeDelay3 = TimeDelta::Millis(700);
EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
EXPECT_EQ(-1, FlushAndGetStats().interframe_delay_max_ms);
time_controller_.AdvanceTime(kInterframeDelay1);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameDelta);
EXPECT_EQ(kInterframeDelay1.ms(), FlushAndGetStats().interframe_delay_max_ms);
time_controller_.AdvanceTime(kInterframeDelay2);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameDelta);
// Still first delay is the maximum
EXPECT_EQ(kInterframeDelay1.ms(), FlushAndGetStats().interframe_delay_max_ms);
time_controller_.AdvanceTime(kInterframeDelay3);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameDelta);
// Now the first sample is out of the window, so the second is the maximum.
EXPECT_EQ(kInterframeDelay2.ms(), FlushAndGetStats().interframe_delay_max_ms);
}
@@ -487,7 +509,8 @@
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
EXPECT_EQ(absl::nullopt, FlushAndGetStats().qp_sum);
}
@@ -495,10 +518,12 @@
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum);
statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
EXPECT_EQ(3u, FlushAndGetStats().qp_sum);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameDelta);
EXPECT_EQ(absl::nullopt, FlushAndGetStats().qp_sum);
}
@@ -543,7 +568,7 @@
VideoContentType::UNSPECIFIED);
VideoReceiveStreamInterface::Stats stats = statistics_proxy_->GetStats();
EXPECT_EQ(1, stats.network_frame_rate);
- EXPECT_EQ(1, stats.frame_counts.key_frames);
+ EXPECT_EQ(0, stats.frame_counts.key_frames);
EXPECT_EQ(0, stats.frame_counts.delta_frames);
}
@@ -628,16 +653,21 @@
TEST_F(ReceiveStatisticsProxyTest, GetStatsReportsFrameCounts) {
const int kKeyFrames = 3;
const int kDeltaFrames = 22;
+ webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
for (int i = 0; i < kKeyFrames; i++) {
- statistics_proxy_->OnCompleteFrame(true, 0, VideoContentType::UNSPECIFIED);
+ statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
}
for (int i = 0; i < kDeltaFrames; i++) {
- statistics_proxy_->OnCompleteFrame(false, 0, VideoContentType::UNSPECIFIED);
+ statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameDelta);
}
VideoReceiveStreamInterface::Stats stats = statistics_proxy_->GetStats();
- EXPECT_EQ(kKeyFrames, stats.frame_counts.key_frames);
- EXPECT_EQ(kDeltaFrames, stats.frame_counts.delta_frames);
+ EXPECT_EQ(0, stats.frame_counts.key_frames);
+ EXPECT_EQ(0, stats.frame_counts.delta_frames);
}
TEST_F(ReceiveStatisticsProxyTest, GetStatsReportsCName) {
@@ -700,8 +730,15 @@
TEST_F(ReceiveStatisticsProxyTest, LifetimeHistogramIsUpdated) {
const TimeDelta kLifetime = TimeDelta::Seconds(3);
time_controller_.AdvanceTime(kLifetime);
- // Need at least one frame to report stream lifetime.
+ // Need at least one decoded frame to report stream lifetime.
+
+ webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
statistics_proxy_->OnCompleteFrame(true, 1000, VideoContentType::UNSPECIFIED);
+ statistics_proxy_->OnDecodedFrame(
+ frame, absl::nullopt, TimeDelta::Millis(1000),
+ VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameKey);
+ FlushAndGetStats();
+
statistics_proxy_->UpdateHistograms(absl::nullopt, StreamDataCounters(),
nullptr);
EXPECT_METRIC_EQ(
@@ -841,10 +878,16 @@
KeyFrameHistogramNotUpdatedForTooFewSamples) {
const bool kIsKeyFrame = false;
const int kFrameSizeBytes = 1000;
+ webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
- for (int i = 0; i < kMinRequiredSamples - 1; ++i)
+ for (int i = 0; i < kMinRequiredSamples - 1; ++i) {
statistics_proxy_->OnCompleteFrame(kIsKeyFrame, kFrameSizeBytes,
VideoContentType::UNSPECIFIED);
+ statistics_proxy_->OnDecodedFrame(
+ frame, absl::nullopt, TimeDelta::Millis(1000),
+ VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameDelta);
+ }
+ FlushAndGetStats();
EXPECT_EQ(0, statistics_proxy_->GetStats().frame_counts.key_frames);
EXPECT_EQ(kMinRequiredSamples - 1,
@@ -860,10 +903,16 @@
KeyFrameHistogramUpdatedForMinRequiredSamples) {
const bool kIsKeyFrame = false;
const int kFrameSizeBytes = 1000;
+ webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
- for (int i = 0; i < kMinRequiredSamples; ++i)
+ for (int i = 0; i < kMinRequiredSamples; ++i) {
statistics_proxy_->OnCompleteFrame(kIsKeyFrame, kFrameSizeBytes,
VideoContentType::UNSPECIFIED);
+ statistics_proxy_->OnDecodedFrame(
+ frame, absl::nullopt, TimeDelta::Millis(1000),
+ VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameDelta);
+ }
+ FlushAndGetStats();
EXPECT_EQ(0, statistics_proxy_->GetStats().frame_counts.key_frames);
EXPECT_EQ(kMinRequiredSamples,
@@ -879,14 +928,23 @@
TEST_F(ReceiveStatisticsProxyTest, KeyFrameHistogramIsUpdated) {
const int kFrameSizeBytes = 1000;
+ webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
- for (int i = 0; i < kMinRequiredSamples; ++i)
+ for (int i = 0; i < kMinRequiredSamples; ++i) {
statistics_proxy_->OnCompleteFrame(true, kFrameSizeBytes,
VideoContentType::UNSPECIFIED);
-
- for (int i = 0; i < kMinRequiredSamples; ++i)
+ statistics_proxy_->OnDecodedFrame(
+ frame, absl::nullopt, TimeDelta::Millis(1000),
+ VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameKey);
+ }
+ for (int i = 0; i < kMinRequiredSamples; ++i) {
statistics_proxy_->OnCompleteFrame(false, kFrameSizeBytes,
VideoContentType::UNSPECIFIED);
+ statistics_proxy_->OnDecodedFrame(
+ frame, absl::nullopt, TimeDelta::Millis(1000),
+ VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameDelta);
+ }
+ FlushAndGetStats();
EXPECT_EQ(kMinRequiredSamples,
statistics_proxy_->GetStats().frame_counts.key_frames);
@@ -966,7 +1024,8 @@
frame.set_ntp_time_ms(
time_controller_.GetClock()->CurrentNtpInMilliseconds());
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(MetaData(frame));
time_controller_.AdvanceTime(1 / kDefaultFps);
}
@@ -1038,7 +1097,8 @@
TEST_F(ReceiveStatisticsProxyTest, ZeroDelayReportedIfFrameNotDelayed) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
// Frame not delayed, delayed frames to render: 0%.
statistics_proxy_->OnRenderedFrame(
@@ -1060,7 +1120,8 @@
DelayedFrameHistogramsAreNotUpdatedIfMinRuntimeHasNotPassed) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
// Frame not delayed, delayed frames to render: 0%.
statistics_proxy_->OnRenderedFrame(
@@ -1081,7 +1142,8 @@
DelayedFramesHistogramsAreNotUpdatedIfNoRenderedFrames) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
// Min run time has passed. No rendered frames.
time_controller_.AdvanceTime(
@@ -1097,7 +1159,8 @@
TEST_F(ReceiveStatisticsProxyTest, DelayReportedIfFrameIsDelayed) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
// Frame delayed 1 ms, delayed frames to render: 100%.
statistics_proxy_->OnRenderedFrame(
@@ -1121,7 +1184,8 @@
TEST_F(ReceiveStatisticsProxyTest, AverageDelayOfDelayedFramesIsReported) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- VideoContentType::UNSPECIFIED);
+ VideoContentType::UNSPECIFIED,
+ VideoFrameType::kVideoFrameKey);
// Two frames delayed (6 ms, 10 ms), delayed frames to render: 50%.
const int64_t kNowMs = Now().ms();
@@ -1274,13 +1338,15 @@
for (int i = 0; i < kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- content_type_);
+ content_type_,
+ VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(kInterFrameDelay);
}
// One extra with double the interval.
time_controller_.AdvanceTime(kInterFrameDelay);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- content_type_);
+ content_type_,
+ VideoFrameType::kVideoFrameDelta);
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);
const TimeDelta kExpectedInterFrame =
@@ -1310,18 +1376,21 @@
for (int i = 0; i <= kMinRequiredSamples - kLastFivePercentsSamples; ++i) {
time_controller_.AdvanceTime(kInterFrameDelay);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- content_type_);
+ content_type_,
+ VideoFrameType::kVideoFrameKey);
}
// Last 5% of intervals are double in size.
for (int i = 0; i < kLastFivePercentsSamples; ++i) {
time_controller_.AdvanceTime(2 * kInterFrameDelay);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- content_type_);
+ content_type_,
+ VideoFrameType::kVideoFrameKey);
}
// Final sample is outlier and 10 times as big.
time_controller_.AdvanceTime(10 * kInterFrameDelay);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- content_type_);
+ content_type_,
+ VideoFrameType::kVideoFrameKey);
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);
const TimeDelta kExpectedInterFrame = kInterFrameDelay * 2;
@@ -1344,7 +1413,8 @@
for (int i = 0; i < kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- content_type_);
+ content_type_,
+ VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(kInterFrameDelay);
}
@@ -1367,7 +1437,8 @@
for (int i = 0; i <= kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- content_type_);
+ content_type_,
+ VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(kInterFrameDelay);
}
@@ -1378,10 +1449,12 @@
// Insert two more frames. The interval during the pause should be
// disregarded in the stats.
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- content_type_);
+ content_type_,
+ VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(kInterFrameDelay);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- content_type_);
+ content_type_,
+ VideoFrameType::kVideoFrameDelta);
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);
if (videocontenttypehelpers::IsScreenshare(content_type_)) {
@@ -1416,18 +1489,18 @@
for (int i = 0; i < kMinRequiredSamples; ++i) {
VideoFrameMetaData meta = MetaData(frame);
- statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(),
- TimeDelta::Zero(), TimeDelta::Zero(),
- content_type_);
+ statistics_proxy_->OnDecodedFrame(
+ meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(),
+ TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(meta);
time_controller_.AdvanceTime(kInterFrameDelay);
}
// Add extra freeze.
time_controller_.AdvanceTime(kFreezeDelay);
VideoFrameMetaData meta = MetaData(frame);
- statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(),
- TimeDelta::Zero(), TimeDelta::Zero(),
- content_type_);
+ statistics_proxy_->OnDecodedFrame(
+ meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(),
+ TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameDelta);
statistics_proxy_->OnRenderedFrame(meta);
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);
@@ -1466,7 +1539,8 @@
for (int i = 0; i < kMinRequiredSamples; ++i) {
time_controller_.AdvanceTime(kFrameDuration);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- content_type_);
+ content_type_,
+ VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(MetaData(frame));
}
@@ -1474,14 +1548,16 @@
// Add freeze.
time_controller_.AdvanceTime(kFreezeDuration);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- content_type_);
+ content_type_,
+ VideoFrameType::kVideoFrameDelta);
statistics_proxy_->OnRenderedFrame(MetaData(frame));
// Add pause.
time_controller_.AdvanceTime(kPauseDuration);
statistics_proxy_->OnStreamInactive();
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- content_type_);
+ content_type_,
+ VideoFrameType::kVideoFrameDelta);
statistics_proxy_->OnRenderedFrame(MetaData(frame));
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);
@@ -1511,9 +1587,9 @@
for (int i = 0; i <= kMinRequiredSamples; ++i) {
VideoFrameMetaData meta = MetaData(frame);
- statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(),
- TimeDelta::Zero(), TimeDelta::Zero(),
- content_type_);
+ statistics_proxy_->OnDecodedFrame(
+ meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(),
+ TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(meta);
time_controller_.AdvanceTime(kInterFrameDelay);
}
@@ -1523,9 +1599,9 @@
// Second playback interval with triple the length.
for (int i = 0; i <= kMinRequiredSamples * 3; ++i) {
VideoFrameMetaData meta = MetaData(frame);
- statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(),
- TimeDelta::Zero(), TimeDelta::Zero(),
- content_type_);
+ statistics_proxy_->OnDecodedFrame(
+ meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(),
+ TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameDelta);
statistics_proxy_->OnRenderedFrame(meta);
time_controller_.AdvanceTime(kInterFrameDelay);
}
@@ -1556,12 +1632,14 @@
for (int i = 0; i <= kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- content_type_);
+ content_type_,
+ VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(kInterFrameDelay);
statistics_proxy_->OnStreamInactive();
time_controller_.AdvanceTime(kPauseDuration);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
- content_type_);
+ content_type_,
+ VideoFrameType::kVideoFrameDelta);
time_controller_.AdvanceTime(kInterFrameDelay);
}
@@ -1586,18 +1664,18 @@
// HD frames.
for (int i = 0; i < kMinRequiredSamples; ++i) {
VideoFrameMetaData meta = MetaData(frame_hd);
- statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(),
- TimeDelta::Zero(), TimeDelta::Zero(),
- content_type_);
+ statistics_proxy_->OnDecodedFrame(
+ meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(),
+ TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(meta);
time_controller_.AdvanceTime(kInterFrameDelay);
}
// SD frames.
for (int i = 0; i < 2 * kMinRequiredSamples; ++i) {
VideoFrameMetaData meta = MetaData(frame_sd);
- statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(),
- TimeDelta::Zero(), TimeDelta::Zero(),
- content_type_);
+ statistics_proxy_->OnDecodedFrame(
+ meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(),
+ TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(meta);
time_controller_.AdvanceTime(kInterFrameDelay);
}
@@ -1626,24 +1704,25 @@
// High quality frames.
for (int i = 0; i < kMinRequiredSamples; ++i) {
VideoFrameMetaData meta = MetaData(frame);
- statistics_proxy_->OnDecodedFrame(meta, kLowQp, TimeDelta::Zero(),
- TimeDelta::Zero(), TimeDelta::Zero(),
- content_type_);
+ statistics_proxy_->OnDecodedFrame(
+ meta, kLowQp, TimeDelta::Zero(), TimeDelta::Zero(), TimeDelta::Zero(),
+ content_type_, VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(meta);
time_controller_.AdvanceTime(kInterFrameDelay);
}
// Blocky frames.
for (int i = 0; i < 2 * kMinRequiredSamples; ++i) {
VideoFrameMetaData meta = MetaData(frame);
- statistics_proxy_->OnDecodedFrame(meta, kHighQp, TimeDelta::Zero(),
- TimeDelta::Zero(), TimeDelta::Zero(),
- content_type_);
+ statistics_proxy_->OnDecodedFrame(
+ meta, kHighQp, TimeDelta::Zero(), TimeDelta::Zero(), TimeDelta::Zero(),
+ content_type_, VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(meta);
time_controller_.AdvanceTime(kInterFrameDelay);
}
// Extra last frame.
statistics_proxy_->OnDecodedFrame(frame, kHighQp, TimeDelta::Zero(),
- content_type_);
+ content_type_,
+ VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(MetaData(frame));
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);
@@ -1670,7 +1749,8 @@
// Call once to pass content type.
statistics_proxy_->OnDecodedFrame(frame_hd, absl::nullopt, TimeDelta::Zero(),
- content_type_);
+ content_type_,
+ VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(TimeDelta::Zero());
statistics_proxy_->OnRenderedFrame(MetaData(frame_hd));
@@ -1699,8 +1779,8 @@
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
for (int i = 0; i < kMinRequiredSamples; ++i) {
- statistics_proxy_->OnDecodedFrame(frame, kLowQp, kDecodeTime,
- content_type_);
+ statistics_proxy_->OnDecodedFrame(frame, kLowQp, kDecodeTime, content_type_,
+ VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(kInterFrameDelay);
}
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);
diff --git a/video/video_stream_decoder2.cc b/video/video_stream_decoder2.cc
index ba36d12..5640835 100644
--- a/video/video_stream_decoder2.cc
+++ b/video/video_stream_decoder2.cc
@@ -46,9 +46,10 @@
int32_t VideoStreamDecoder::FrameToRender(VideoFrame& video_frame,
absl::optional<uint8_t> qp,
TimeDelta decode_time,
- VideoContentType content_type) {
+ VideoContentType content_type,
+ VideoFrameType frame_type) {
receive_stats_callback_->OnDecodedFrame(video_frame, qp, decode_time,
- content_type);
+ content_type, frame_type);
incoming_video_stream_->OnFrame(video_frame);
return 0;
}
diff --git a/video/video_stream_decoder2.h b/video/video_stream_decoder2.h
index 473d463..19db810 100644
--- a/video/video_stream_decoder2.h
+++ b/video/video_stream_decoder2.h
@@ -43,7 +43,8 @@
int32_t FrameToRender(VideoFrame& video_frame,
absl::optional<uint8_t> qp,
TimeDelta decode_time,
- VideoContentType content_type) override;
+ VideoContentType content_type,
+ VideoFrameType frame_type) override;
void OnDroppedFrames(uint32_t frames_dropped) override;
void OnIncomingPayloadType(int payload_type) override;
void OnDecoderInfoChanged(
