Send VideoLayersAllocation with resolution if number of spatial layers
increase.
VP9 and other codecs can in theory add spatial layers without a key
frame.
Bug: webrtc:12000
Change-Id: I27461af2e34c855203a130e400a6aa01144d3cf7
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/198781
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Commit-Queue: Per Kjellander <perkj@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32883}
diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc
index 2499d35..934be82 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_video.cc
@@ -153,7 +153,7 @@
: (kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers)),
last_rotation_(kVideoRotation_0),
transmit_color_space_next_frame_(false),
- send_allocation_(false),
+ send_allocation_(SendVideoLayersAllocation::kDontSend),
current_playout_delay_{-1, -1},
playout_delay_pending_(false),
forced_playout_delay_(LoadVideoPlayoutDelayOverride(config.field_trials)),
@@ -293,8 +293,13 @@
void RTPSenderVideo::SetVideoLayersAllocationInternal(
VideoLayersAllocation allocation) {
RTC_DCHECK_RUNS_SERIALIZED(&send_checker_);
+ if (!allocation_ || allocation.active_spatial_layers.size() >
+ allocation_->active_spatial_layers.size()) {
+ send_allocation_ = SendVideoLayersAllocation::kSendWithResolution;
+ } else if (send_allocation_ == SendVideoLayersAllocation::kDontSend) {
+ send_allocation_ = SendVideoLayersAllocation::kSendWithoutResolution;
+ }
allocation_ = std::move(allocation);
- send_allocation_ = true;
}
void RTPSenderVideo::AddRtpHeaderExtensions(
@@ -433,16 +438,14 @@
}
}
- if (first_packet && send_allocation_) {
- if (video_header.frame_type == VideoFrameType::kVideoFrameKey) {
- packet->SetExtension<RtpVideoLayersAllocationExtension>(
- allocation_.value());
- } else if (PacketWillLikelyBeRequestedForRestransmitionIfLost(
- video_header)) {
- VideoLayersAllocation allocation = allocation_.value();
- allocation.resolution_and_frame_rate_is_valid = false;
- packet->SetExtension<RtpVideoLayersAllocationExtension>(allocation);
- }
+ if (first_packet &&
+ send_allocation_ != SendVideoLayersAllocation::kDontSend &&
+ (video_header.frame_type == VideoFrameType::kVideoFrameKey ||
+ PacketWillLikelyBeRequestedForRestransmitionIfLost(video_header))) {
+ VideoLayersAllocation allocation = allocation_.value();
+ allocation.resolution_and_frame_rate_is_valid =
+ send_allocation_ == SendVideoLayersAllocation::kSendWithResolution;
+ packet->SetExtension<RtpVideoLayersAllocationExtension>(allocation);
}
}
@@ -481,7 +484,7 @@
}
if (allocation_) {
// Send the bitrate allocation on every key frame.
- send_allocation_ = true;
+ send_allocation_ = SendVideoLayersAllocation::kSendWithResolution;
}
}
@@ -706,7 +709,7 @@
// This frame will likely be delivered, no need to populate playout
// delay extensions until it changes again.
playout_delay_pending_ = false;
- send_allocation_ = false;
+ send_allocation_ = SendVideoLayersAllocation::kDontSend;
}
TRACE_EVENT_ASYNC_END1("webrtc", "Video", capture_time_ms, "timestamp",
diff --git a/modules/rtp_rtcp/source/rtp_sender_video.h b/modules/rtp_rtcp/source/rtp_sender_video.h
index 3f431df..6e46990 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video.h
+++ b/modules/rtp_rtcp/source/rtp_sender_video.h
@@ -159,6 +159,12 @@
int64_t last_frame_time_ms;
};
+ enum class SendVideoLayersAllocation {
+ kSendWithResolution,
+ kSendWithoutResolution,
+ kDontSend
+ };
+
void SetVideoStructureInternal(
const FrameDependencyStructure* video_structure);
void SetVideoLayersAllocationInternal(VideoLayersAllocation allocation);
@@ -202,7 +208,7 @@
absl::optional<VideoLayersAllocation> allocation_
RTC_GUARDED_BY(send_checker_);
// Flag indicating if we should send |allocation_|.
- bool send_allocation_ RTC_GUARDED_BY(send_checker_);
+ SendVideoLayersAllocation send_allocation_ RTC_GUARDED_BY(send_checker_);
// Current target playout delay.
VideoPlayoutDelay current_playout_delay_ RTC_GUARDED_BY(send_checker_);
diff --git a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
index e415bad..55bafdc 100644
--- a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
+++ b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc
@@ -862,7 +862,7 @@
}
TEST_P(RtpSenderVideoTest,
- VideoLayersAllocationWithoutResolutionSentOnDeltaFrames) {
+ VideoLayersAllocationWithoutResolutionSentOnDeltaWhenUpdated) {
const size_t kFrameSize = 100;
uint8_t kFrame[kFrameSize];
rtp_module_->RegisterRtpHeaderExtension(
@@ -876,14 +876,28 @@
allocation.resolution_and_frame_rate_is_valid = true;
layer.target_bitrate_per_temporal_layer.push_back(
DataRate::KilobitsPerSec(50));
-
allocation.active_spatial_layers.push_back(layer);
rtp_sender_video_->SetVideoLayersAllocation(allocation);
RTPVideoHeader hdr;
+ hdr.frame_type = VideoFrameType::kVideoFrameKey;
+ rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
+ kDefaultExpectedRetransmissionTimeMs);
+ EXPECT_TRUE(transport_.last_sent_packet()
+ .HasExtension<RtpVideoLayersAllocationExtension>());
+
+ // No allocation sent on delta frame unless it has been updated.
hdr.frame_type = VideoFrameType::kVideoFrameDelta;
rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
kDefaultExpectedRetransmissionTimeMs);
+ EXPECT_FALSE(transport_.last_sent_packet()
+ .HasExtension<RtpVideoLayersAllocationExtension>());
+
+ // Update the allocation.
+ rtp_sender_video_->SetVideoLayersAllocation(allocation);
+ rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
+ kDefaultExpectedRetransmissionTimeMs);
+
VideoLayersAllocation sent_allocation;
EXPECT_TRUE(
transport_.last_sent_packet()
@@ -895,6 +909,52 @@
SizeIs(1));
}
+TEST_P(RtpSenderVideoTest,
+ VideoLayersAllocationWithResolutionSentOnDeltaWhenSpatialLayerAdded) {
+ const size_t kFrameSize = 100;
+ uint8_t kFrame[kFrameSize];
+ rtp_module_->RegisterRtpHeaderExtension(
+ RtpVideoLayersAllocationExtension::kUri,
+ kVideoLayersAllocationExtensionId);
+
+ VideoLayersAllocation allocation;
+ allocation.resolution_and_frame_rate_is_valid = true;
+ VideoLayersAllocation::SpatialLayer layer;
+ layer.width = 360;
+ layer.height = 180;
+ layer.spatial_id = 0;
+ layer.target_bitrate_per_temporal_layer.push_back(
+ DataRate::KilobitsPerSec(50));
+ allocation.active_spatial_layers.push_back(layer);
+ rtp_sender_video_->SetVideoLayersAllocation(allocation);
+
+ RTPVideoHeader hdr;
+ hdr.frame_type = VideoFrameType::kVideoFrameKey;
+ rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
+ kDefaultExpectedRetransmissionTimeMs);
+ ASSERT_TRUE(transport_.last_sent_packet()
+ .HasExtension<RtpVideoLayersAllocationExtension>());
+
+ // Update the allocation.
+ layer.width = 640;
+ layer.height = 320;
+ layer.spatial_id = 1;
+ layer.target_bitrate_per_temporal_layer.push_back(
+ DataRate::KilobitsPerSec(100));
+ allocation.active_spatial_layers.push_back(layer);
+ rtp_sender_video_->SetVideoLayersAllocation(allocation);
+ hdr.frame_type = VideoFrameType::kVideoFrameDelta;
+ rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
+ kDefaultExpectedRetransmissionTimeMs);
+
+ VideoLayersAllocation sent_allocation;
+ EXPECT_TRUE(
+ transport_.last_sent_packet()
+ .GetExtension<RtpVideoLayersAllocationExtension>(&sent_allocation));
+ EXPECT_THAT(sent_allocation.active_spatial_layers, SizeIs(2));
+ EXPECT_TRUE(sent_allocation.resolution_and_frame_rate_is_valid);
+}
+
TEST_P(RtpSenderVideoTest, VideoLayersAllocationSentOnDeltaFramesOnlyOnUpdate) {
const size_t kFrameSize = 100;
uint8_t kFrame[kFrameSize];
@@ -904,6 +964,8 @@
VideoLayersAllocation allocation;
VideoLayersAllocation::SpatialLayer layer;
+ layer.width = 360;
+ layer.height = 180;
layer.target_bitrate_per_temporal_layer.push_back(
DataRate::KilobitsPerSec(50));
allocation.active_spatial_layers.push_back(layer);
@@ -943,7 +1005,10 @@
kVideoLayersAllocationExtensionId);
VideoLayersAllocation allocation;
+ allocation.resolution_and_frame_rate_is_valid = true;
VideoLayersAllocation::SpatialLayer layer;
+ layer.width = 360;
+ layer.height = 180;
layer.target_bitrate_per_temporal_layer.push_back(
DataRate::KilobitsPerSec(50));
allocation.active_spatial_layers.push_back(layer);
@@ -957,18 +1022,15 @@
rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
kDefaultExpectedRetransmissionTimeMs);
- VideoLayersAllocation sent_allocation;
- EXPECT_FALSE(
- transport_.last_sent_packet()
- .GetExtension<RtpVideoLayersAllocationExtension>(&sent_allocation));
+ EXPECT_FALSE(transport_.last_sent_packet()
+ .HasExtension<RtpVideoLayersAllocationExtension>());
// Send a delta frame on tl0.
vp8_header.temporalIdx = 0;
rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, 0, kFrame, hdr,
kDefaultExpectedRetransmissionTimeMs);
- EXPECT_TRUE(
- transport_.last_sent_packet()
- .GetExtension<RtpVideoLayersAllocationExtension>(&sent_allocation));
+ EXPECT_TRUE(transport_.last_sent_packet()
+ .HasExtension<RtpVideoLayersAllocationExtension>());
}
TEST_P(RtpSenderVideoTest, AbsoluteCaptureTime) {
