Add is_repeat_frame flag to VideoFrame
This flag indicates if a VideoFrame is a copy of a previous frame.
It is set to true by the FrameCadenceAdapter when generating
repeated frames in zero-hertz mode.
Bug: webrtc:443906251
Change-Id: I9447c08a5b09c8d758462284546f08e54bc1b748
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/408782
Reviewed-by: Markus Handell <handellm@webrtc.org>
Commit-Queue: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#45604}
diff --git a/api/video/video_frame.cc b/api/video/video_frame.cc
index 8a99177..85816ed 100644
--- a/api/video/video_frame.cc
+++ b/api/video/video_frame.cc
@@ -174,7 +174,7 @@
return VideoFrame(id_, video_frame_buffer_, timestamp_us_,
presentation_timestamp_, reference_time_, timestamp_rtp_,
ntp_time_ms_, rotation_, color_space_, render_parameters_,
- update_rect_, packet_infos_);
+ update_rect_, packet_infos_, is_repeat_frame_);
}
VideoFrame::Builder& VideoFrame::Builder::set_video_frame_buffer(
@@ -264,6 +264,12 @@
return *this;
}
+VideoFrame::Builder& VideoFrame::Builder::set_is_repeat_frame(
+ bool is_repeat_frame) {
+ is_repeat_frame_ = is_repeat_frame;
+ return *this;
+}
+
VideoFrame::VideoFrame(const scoped_refptr<VideoFrameBuffer>& buffer,
VideoRotation rotation,
int64_t timestamp_us)
@@ -271,7 +277,8 @@
timestamp_rtp_(0),
ntp_time_ms_(0),
timestamp_us_(timestamp_us),
- rotation_(rotation) {}
+ rotation_(rotation),
+ is_repeat_frame_(false) {}
VideoFrame::VideoFrame(const scoped_refptr<VideoFrameBuffer>& buffer,
uint32_t timestamp_rtp,
@@ -281,40 +288,9 @@
timestamp_rtp_(timestamp_rtp),
ntp_time_ms_(0),
timestamp_us_(render_time_ms * kNumMicrosecsPerMillisec),
- rotation_(rotation) {
- RTC_DCHECK(buffer);
-}
-
-VideoFrame::VideoFrame(uint16_t id,
- const scoped_refptr<VideoFrameBuffer>& buffer,
- int64_t timestamp_us,
- const std::optional<Timestamp>& presentation_timestamp,
- const std::optional<Timestamp>& reference_time,
- uint32_t timestamp_rtp,
- int64_t ntp_time_ms,
- VideoRotation rotation,
- const std::optional<ColorSpace>& color_space,
- const RenderParameters& render_parameters,
- const std::optional<UpdateRect>& update_rect,
- RtpPacketInfos packet_infos)
- : id_(id),
- video_frame_buffer_(buffer),
- timestamp_rtp_(timestamp_rtp),
- ntp_time_ms_(ntp_time_ms),
- timestamp_us_(timestamp_us),
- presentation_timestamp_(presentation_timestamp),
- reference_time_(reference_time),
rotation_(rotation),
- color_space_(color_space),
- render_parameters_(render_parameters),
- update_rect_(update_rect),
- packet_infos_(std::move(packet_infos)) {
- if (update_rect_) {
- RTC_DCHECK_GE(update_rect_->offset_x, 0);
- RTC_DCHECK_GE(update_rect_->offset_y, 0);
- RTC_DCHECK_LE(update_rect_->offset_x + update_rect_->width, width());
- RTC_DCHECK_LE(update_rect_->offset_y + update_rect_->height, height());
- }
+ is_repeat_frame_(false) {
+ RTC_DCHECK(buffer);
}
VideoFrame::~VideoFrame() = default;
diff --git a/api/video/video_frame.h b/api/video/video_frame.h
index e39d06f..395a68d 100644
--- a/api/video/video_frame.h
+++ b/api/video/video_frame.h
@@ -124,6 +124,7 @@
Builder& set_id(uint16_t id);
Builder& set_update_rect(const std::optional<UpdateRect>& update_rect);
Builder& set_packet_infos(RtpPacketInfos packet_infos);
+ Builder& set_is_repeat_frame(bool is_repeat_frame);
private:
uint16_t id_ = kNotSetId;
@@ -138,6 +139,7 @@
RenderParameters render_parameters_;
std::optional<UpdateRect> update_rect_;
RtpPacketInfos packet_infos_;
+ bool is_repeat_frame_ = false;
};
// To be deprecated. Migrate all use to Builder.
@@ -283,6 +285,11 @@
processing_time_ = processing_time;
}
+ bool is_repeat_frame() const { return is_repeat_frame_; }
+ void set_is_repeat_frame(bool is_repeat_frame) {
+ is_repeat_frame_ = is_repeat_frame;
+ }
+
private:
VideoFrame(uint16_t id,
const scoped_refptr<VideoFrameBuffer>& buffer,
@@ -295,7 +302,21 @@
const std::optional<ColorSpace>& color_space,
const RenderParameters& render_parameters,
const std::optional<UpdateRect>& update_rect,
- RtpPacketInfos packet_infos);
+ RtpPacketInfos packet_infos,
+ bool is_repeat_frame)
+ : id_(id),
+ video_frame_buffer_(buffer),
+ timestamp_rtp_(timestamp_rtp),
+ ntp_time_ms_(ntp_time_ms),
+ timestamp_us_(timestamp_us),
+ presentation_timestamp_(presentation_timestamp),
+ reference_time_(reference_time),
+ rotation_(rotation),
+ color_space_(color_space),
+ render_parameters_(render_parameters),
+ update_rect_(update_rect),
+ packet_infos_(std::move(packet_infos)),
+ is_repeat_frame_(is_repeat_frame) {}
uint16_t id_;
// An opaque reference counted handle that stores the pixel data.
@@ -328,6 +349,11 @@
// returned from the decoder.
// Currently, not set for locally captured video frames.
std::optional<ProcessingTime> processing_time_;
+ // Indicates if this is a "repeat frame" - i.e. a copy a previous frame,
+ // inserted in order to make a video codec converge towards a stable quality
+ // in cases where a capturer is using a variable frame rate and stops
+ // producing frames when nothing has changed.
+ bool is_repeat_frame_;
};
} // namespace webrtc
diff --git a/common_video/video_frame_unittest.cc b/common_video/video_frame_unittest.cc
index bc7aa79..de96e6d 100644
--- a/common_video/video_frame_unittest.cc
+++ b/common_video/video_frame_unittest.cc
@@ -643,4 +643,32 @@
EXPECT_EQ(scaled, VideoFrame::UpdateRect({42, 22, 56, 106}));
}
+TEST(TestVideoFrame, IsRepeatFrame) {
+ // Defaults to false.
+ VideoFrame frame1 = VideoFrame::Builder()
+ .set_video_frame_buffer(I420Buffer::Create(10, 10))
+ .set_rotation(kVideoRotation_0)
+ .set_timestamp_ms(123)
+ .build();
+ EXPECT_FALSE(frame1.is_repeat_frame());
+
+ // Set to true.
+ VideoFrame frame2 = VideoFrame::Builder()
+ .set_video_frame_buffer(I420Buffer::Create(10, 10))
+ .set_rotation(kVideoRotation_0)
+ .set_timestamp_ms(456)
+ .set_is_repeat_frame(true)
+ .build();
+ EXPECT_TRUE(frame2.is_repeat_frame());
+
+ // Set to false.
+ VideoFrame frame3 = VideoFrame::Builder()
+ .set_video_frame_buffer(I420Buffer::Create(10, 10))
+ .set_rotation(kVideoRotation_0)
+ .set_timestamp_ms(789)
+ .set_is_repeat_frame(false)
+ .build();
+ EXPECT_FALSE(frame3.is_repeat_frame());
+}
+
} // namespace webrtc
diff --git a/video/frame_cadence_adapter.cc b/video/frame_cadence_adapter.cc
index 63feaee..a519c10 100644
--- a/video/frame_cadence_adapter.cc
+++ b/video/frame_cadence_adapter.cc
@@ -691,6 +691,7 @@
// Schedule another repeat before sending the frame off which could take time.
ScheduleRepeat(frame_id, HasQualityConverged());
+ frame.set_is_repeat_frame(true);
SendFrameNow(std::nullopt, frame);
}
diff --git a/video/frame_cadence_adapter_unittest.cc b/video/frame_cadence_adapter_unittest.cc
index a0e1ddf..9f36d10 100644
--- a/video/frame_cadence_adapter_unittest.cc
+++ b/video/frame_cadence_adapter_unittest.cc
@@ -338,6 +338,45 @@
time_controller.AdvanceTime(TimeDelta::Seconds(1));
}
+TEST(FrameCadenceAdapterTest, SetsIsRepeatFrameFlag) {
+ MockCallback callback;
+ GlobalSimulatedTimeController time_controller(Timestamp::Millis(47892223));
+ FieldTrials no_field_trials = CreateTestFieldTrials();
+ auto adapter = CreateAdapter(no_field_trials, time_controller.GetClock());
+ adapter->Initialize(&callback);
+ adapter->SetZeroHertzModeEnabled(
+ FrameCadenceAdapterInterface::ZeroHertzModeParams{});
+ adapter->OnConstraintsChanged(
+ VideoTrackSourceConstraints{.min_fps = 0, .max_fps = 1});
+
+ // Send one frame, expect is_repeat_frame to be false.
+ auto frame = CreateFrameWithTimestamps(&time_controller);
+ adapter->OnFrame(frame);
+ EXPECT_CALL(callback, OnFrame)
+ .WillOnce(Invoke([&](Timestamp, bool, const VideoFrame& frame) {
+ EXPECT_FALSE(frame.is_repeat_frame());
+ }));
+ time_controller.AdvanceTime(TimeDelta::Seconds(1));
+ Mock::VerifyAndClearExpectations(&callback);
+
+ // Expect the repeated frame to have is_repeat_frame set to true.
+ EXPECT_CALL(callback, OnFrame)
+ .WillOnce(Invoke([&](Timestamp, bool, const VideoFrame& frame) {
+ EXPECT_TRUE(frame.is_repeat_frame());
+ }));
+ time_controller.AdvanceTime(TimeDelta::Seconds(1));
+ Mock::VerifyAndClearExpectations(&callback);
+
+ // Send a new frame, expect is_repeat_frame to be false again.
+ auto new_frame = CreateFrameWithTimestamps(&time_controller);
+ adapter->OnFrame(new_frame);
+ EXPECT_CALL(callback, OnFrame)
+ .WillOnce(Invoke([&](Timestamp, bool, const VideoFrame& frame) {
+ EXPECT_FALSE(frame.is_repeat_frame());
+ }));
+ time_controller.AdvanceTime(TimeDelta::Seconds(1));
+}
+
TEST(FrameCadenceAdapterTest,
RepeatsFramesWithoutTimestampsWithUnsetTimestamps) {
// Logic in the frame cadence adapter avoids modifying frame NTP and render
