Send VideoLayersAllocation with valid frame rate when frame rate change
Sends a VideoLayersAllocation header extension if frame rate change more than 5fps since the last time it was sent with valid frame rate and resolution.
Bug: webrtc:12000
Change-Id: I2572c966025cc2c22743bbe2187cec7cceb86d01
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/234752
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Commit-Queue: Per Kjellander <perkj@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35180}
diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc
index 906f2bc..9e909c9 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_video.cc
@@ -292,12 +292,26 @@
void RTPSenderVideo::SetVideoLayersAllocationInternal(
VideoLayersAllocation allocation) {
RTC_DCHECK_RUNS_SERIALIZED(&send_checker_);
- if (!allocation_ || allocation.active_spatial_layers.size() >
+ if (!allocation_ || allocation.active_spatial_layers.size() !=
allocation_->active_spatial_layers.size()) {
send_allocation_ = SendVideoLayersAllocation::kSendWithResolution;
} else if (send_allocation_ == SendVideoLayersAllocation::kDontSend) {
send_allocation_ = SendVideoLayersAllocation::kSendWithoutResolution;
}
+ if (send_allocation_ == SendVideoLayersAllocation::kSendWithoutResolution) {
+ // Check if frame rate changed more than 5fps since the last time the
+ // extension was sent with frame rate and resolution.
+ for (size_t i = 0; i < allocation.active_spatial_layers.size(); ++i) {
+ if (abs(static_cast<int>(
+ allocation.active_spatial_layers[i].frame_rate_fps) -
+ static_cast<int>(
+ last_full_sent_allocation_->active_spatial_layers[i]
+ .frame_rate_fps)) > 5) {
+ send_allocation_ = SendVideoLayersAllocation::kSendWithResolution;
+ break;
+ }
+ }
+ }
allocation_ = std::move(allocation);
}
@@ -726,6 +740,9 @@
// This frame will likely be delivered, no need to populate playout
// delay extensions until it changes again.
playout_delay_pending_ = false;
+ if (send_allocation_ == SendVideoLayersAllocation::kSendWithResolution) {
+ last_full_sent_allocation_ = allocation_;
+ }
send_allocation_ = SendVideoLayersAllocation::kDontSend;
}
diff --git a/modules/rtp_rtcp/source/rtp_sender_video.h b/modules/rtp_rtcp/source/rtp_sender_video.h
index 226c406..f4f15f3 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video.h
+++ b/modules/rtp_rtcp/source/rtp_sender_video.h
@@ -210,6 +210,8 @@
RTC_GUARDED_BY(send_checker_);
// Flag indicating if we should send `allocation_`.
SendVideoLayersAllocation send_allocation_ RTC_GUARDED_BY(send_checker_);
+ absl::optional<VideoLayersAllocation> last_full_sent_allocation_
+ RTC_GUARDED_BY(send_checker_);
// Current target playout delay.
VideoPlayoutDelay current_playout_delay_ RTC_GUARDED_BY(send_checker_);
diff --git a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
index 78e842d..d3782be 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
@@ -1008,6 +1008,89 @@
EXPECT_TRUE(sent_allocation.resolution_and_frame_rate_is_valid);
}
+TEST_P(RtpSenderVideoTest,
+ VideoLayersAllocationWithResolutionSentOnLargeFrameRateChange) {
+ const size_t kFrameSize = 100;
+ uint8_t kFrame[kFrameSize];
+ rtp_module_->RegisterRtpHeaderExtension(
+ RtpVideoLayersAllocationExtension::Uri(),
+ kVideoLayersAllocationExtensionId);
+
+ VideoLayersAllocation allocation;
+ allocation.resolution_and_frame_rate_is_valid = true;
+ VideoLayersAllocation::SpatialLayer layer;
+ layer.width = 360;
+ layer.height = 180;
+ layer.spatial_id = 0;
+ layer.frame_rate_fps = 10;
+ layer.target_bitrate_per_temporal_layer.push_back(
+ DataRate::KilobitsPerSec(50));
+ allocation.active_spatial_layers.push_back(layer);
+ rtp_sender_video_->SetVideoLayersAllocation(allocation);
+
+ RTPVideoHeader hdr;
+ hdr.frame_type = VideoFrameType::kVideoFrameKey;
+ rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
+ kDefaultExpectedRetransmissionTimeMs);
+ ASSERT_TRUE(transport_.last_sent_packet()
+ .HasExtension<RtpVideoLayersAllocationExtension>());
+
+ // Update frame rate only.
+ allocation.active_spatial_layers[0].frame_rate_fps = 20;
+ rtp_sender_video_->SetVideoLayersAllocation(allocation);
+ hdr.frame_type = VideoFrameType::kVideoFrameDelta;
+ rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
+ kDefaultExpectedRetransmissionTimeMs);
+
+ VideoLayersAllocation sent_allocation;
+ EXPECT_TRUE(
+ transport_.last_sent_packet()
+ .GetExtension<RtpVideoLayersAllocationExtension>(&sent_allocation));
+ ASSERT_TRUE(sent_allocation.resolution_and_frame_rate_is_valid);
+ EXPECT_EQ(sent_allocation.active_spatial_layers[0].frame_rate_fps, 20);
+}
+
+TEST_P(RtpSenderVideoTest,
+ VideoLayersAllocationWithoutResolutionSentOnSmallFrameRateChange) {
+ const size_t kFrameSize = 100;
+ uint8_t kFrame[kFrameSize];
+ rtp_module_->RegisterRtpHeaderExtension(
+ RtpVideoLayersAllocationExtension::Uri(),
+ kVideoLayersAllocationExtensionId);
+
+ VideoLayersAllocation allocation;
+ allocation.resolution_and_frame_rate_is_valid = true;
+ VideoLayersAllocation::SpatialLayer layer;
+ layer.width = 360;
+ layer.height = 180;
+ layer.spatial_id = 0;
+ layer.frame_rate_fps = 10;
+ layer.target_bitrate_per_temporal_layer.push_back(
+ DataRate::KilobitsPerSec(50));
+ allocation.active_spatial_layers.push_back(layer);
+ rtp_sender_video_->SetVideoLayersAllocation(allocation);
+
+ RTPVideoHeader hdr;
+ hdr.frame_type = VideoFrameType::kVideoFrameKey;
+ rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
+ kDefaultExpectedRetransmissionTimeMs);
+ ASSERT_TRUE(transport_.last_sent_packet()
+ .HasExtension<RtpVideoLayersAllocationExtension>());
+
+ // Update frame rate slightly.
+ allocation.active_spatial_layers[0].frame_rate_fps = 9;
+ rtp_sender_video_->SetVideoLayersAllocation(allocation);
+ hdr.frame_type = VideoFrameType::kVideoFrameDelta;
+ rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
+ kDefaultExpectedRetransmissionTimeMs);
+
+ VideoLayersAllocation sent_allocation;
+ EXPECT_TRUE(
+ transport_.last_sent_packet()
+ .GetExtension<RtpVideoLayersAllocationExtension>(&sent_allocation));
+ EXPECT_FALSE(sent_allocation.resolution_and_frame_rate_is_valid);
+}
+
TEST_P(RtpSenderVideoTest, VideoLayersAllocationSentOnDeltaFramesOnlyOnUpdate) {
const size_t kFrameSize = 100;
uint8_t kFrame[kFrameSize];
