Revert "VP9 encoder: handle disabled layers correctly"
This reverts commit 88fe84b7fbcb8dffe07b98d21d8a11572259c0d0.
Reason for revert: Downstream project isn't updated to the latest libvpx roll yet, thus some tests are broken.
Original change's description:
> VP9 encoder: handle disabled layers correctly
>
> Now vp9 screenshare would enable new layers as soon as requested and will
> force all spatial layers present on the next frame, even if they should be
> dropped because of frame-rate limiting.
>
> This might cause frame-rate liming to be exceeded if layer is toggling on
> and off very often, but this situation is bad itself. E.g. in realtime video
> it will cause too many key-frames.
>
> Now SvcRateAllocator and VP9EncoderImpl are aware that there may be some skipped
> layers before the first enabled. Key-frames and ss_info triggering logic is also
> updated.
>
> Bug: webrtc:10977
> Change-Id: Ie2555210c0368a1d3c51ddf6670d0052e6d679de
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/153483
> Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org>
> Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
> Cr-Commit-Position: refs/heads/master@{#29296}
TBR=ilnik@webrtc.org,ssilkin@webrtc.org
Change-Id: If33886a5f8a0c3b33168dcadfe45c11a6f4387c1
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: webrtc:10977
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/154354
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29299}
diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc
index a02e69a..8513b43 100644
--- a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc
+++ b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc
@@ -25,47 +25,34 @@
const float kSpatialLayeringRateScalingFactor = 0.55f;
const float kTemporalLayeringRateScalingFactor = 0.55f;
-// Returns numberOfSpatialLayers if no layers are active.
-size_t GetFirstActiveLayer(const VideoCodec& codec) {
- RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9);
- RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u);
- size_t layer = 0;
- for (; layer < codec.VP9().numberOfSpatialLayers; ++layer) {
- if (codec.spatialLayers[layer].active) {
- break;
- }
- }
- return layer;
-}
-
static size_t GetNumActiveSpatialLayers(const VideoCodec& codec) {
RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9);
RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u);
- const size_t first_active_layer = GetFirstActiveLayer(codec);
- size_t last_active_layer = first_active_layer;
- for (; last_active_layer < codec.VP9().numberOfSpatialLayers;
- ++last_active_layer) {
- if (!codec.spatialLayers[last_active_layer].active) {
+ size_t num_spatial_layers = 0;
+ for (; num_spatial_layers < codec.VP9().numberOfSpatialLayers;
+ ++num_spatial_layers) {
+ if (!codec.spatialLayers[num_spatial_layers].active) {
+ // TODO(bugs.webrtc.org/9350): Deactivation of middle layer is not
+ // implemented. For now deactivation of a VP9 layer deactivates all
+ // layers above the deactivated one.
break;
}
}
- return last_active_layer - first_active_layer;
+
+ return num_spatial_layers;
}
std::vector<DataRate> AdjustAndVerify(
const VideoCodec& codec,
- size_t first_active_layer,
const std::vector<DataRate>& spatial_layer_rates) {
std::vector<DataRate> adjusted_spatial_layer_rates;
// Keep track of rate that couldn't be applied to the previous layer due to
// max bitrate constraint, try to pass it forward to the next one.
DataRate excess_rate = DataRate::Zero();
for (size_t sl_idx = 0; sl_idx < spatial_layer_rates.size(); ++sl_idx) {
- DataRate min_rate = DataRate::kbps(
- codec.spatialLayers[first_active_layer + sl_idx].minBitrate);
- DataRate max_rate = DataRate::kbps(
- codec.spatialLayers[first_active_layer + sl_idx].maxBitrate);
+ DataRate min_rate = DataRate::kbps(codec.spatialLayers[sl_idx].minBitrate);
+ DataRate max_rate = DataRate::kbps(codec.spatialLayers[sl_idx].maxBitrate);
DataRate layer_rate = spatial_layer_rates[sl_idx] + excess_rate;
if (layer_rate < min_rate) {
@@ -122,7 +109,6 @@
// Returns the minimum bitrate needed for |num_active_layers| spatial layers to
// become active using the configuration specified by |codec|.
DataRate FindLayerTogglingThreshold(const VideoCodec& codec,
- size_t first_active_layer,
size_t num_active_layers) {
if (num_active_layers == 1) {
return DataRate::kbps(codec.spatialLayers[0].minBitrate);
@@ -133,10 +119,8 @@
DataRate upper_bound = DataRate::Zero();
if (num_active_layers > 1) {
for (size_t i = 0; i < num_active_layers - 1; ++i) {
- lower_bound += DataRate::kbps(
- codec.spatialLayers[first_active_layer + i].minBitrate);
- upper_bound += DataRate::kbps(
- codec.spatialLayers[first_active_layer + i].maxBitrate);
+ lower_bound += DataRate::kbps(codec.spatialLayers[i].minBitrate);
+ upper_bound += DataRate::kbps(codec.spatialLayers[i].maxBitrate);
}
}
upper_bound +=
@@ -147,7 +131,7 @@
// layers respectively.
while (upper_bound - lower_bound > DataRate::bps(1)) {
DataRate try_rate = (lower_bound + upper_bound) / 2;
- if (AdjustAndVerify(codec, first_active_layer,
+ if (AdjustAndVerify(codec,
SplitBitrate(num_active_layers, try_rate,
kSpatialLayeringRateScalingFactor))
.size() == num_active_layers) {
@@ -160,12 +144,10 @@
} else {
DataRate toggling_rate = DataRate::Zero();
for (size_t i = 0; i < num_active_layers - 1; ++i) {
- toggling_rate += DataRate::kbps(
- codec.spatialLayers[first_active_layer + i].targetBitrate);
+ toggling_rate += DataRate::kbps(codec.spatialLayers[i].targetBitrate);
}
- toggling_rate += DataRate::kbps(
- codec.spatialLayers[first_active_layer + num_active_layers - 1]
- .minBitrate);
+ toggling_rate +=
+ DataRate::kbps(codec.spatialLayers[num_active_layers - 1].minBitrate);
return toggling_rate;
}
}
@@ -210,9 +192,7 @@
return bitrate_allocation;
}
- const size_t first_active_layer = GetFirstActiveLayer(codec_);
size_t num_spatial_layers = GetNumActiveSpatialLayers(codec_);
-
if (num_spatial_layers == 0) {
return VideoBitrateAllocation(); // All layers are deactivated.
}
@@ -245,17 +225,14 @@
last_active_layer_count_ = num_spatial_layers;
if (codec_.mode == VideoCodecMode::kRealtimeVideo) {
- return GetAllocationNormalVideo(total_bitrate, first_active_layer,
- num_spatial_layers);
+ return GetAllocationNormalVideo(total_bitrate, num_spatial_layers);
} else {
- return GetAllocationScreenSharing(total_bitrate, first_active_layer,
- num_spatial_layers);
+ return GetAllocationScreenSharing(total_bitrate, num_spatial_layers);
}
}
VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo(
DataRate total_bitrate,
- size_t first_active_layer,
size_t num_spatial_layers) const {
std::vector<DataRate> spatial_layer_rates;
if (num_spatial_layers == 0) {
@@ -264,10 +241,9 @@
num_spatial_layers = 1;
spatial_layer_rates.push_back(total_bitrate);
} else {
- spatial_layer_rates =
- AdjustAndVerify(codec_, first_active_layer,
- SplitBitrate(num_spatial_layers, total_bitrate,
- kSpatialLayeringRateScalingFactor));
+ spatial_layer_rates = AdjustAndVerify(
+ codec_, SplitBitrate(num_spatial_layers, total_bitrate,
+ kSpatialLayeringRateScalingFactor));
RTC_DCHECK_EQ(spatial_layer_rates.size(), num_spatial_layers);
}
@@ -283,13 +259,10 @@
// layers since they are used for prediction of higher layers and their
// references are far apart.
if (num_temporal_layers == 1) {
- bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
- temporal_layer_rates[0].bps());
+ bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[0].bps());
} else if (num_temporal_layers == 2) {
- bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
- temporal_layer_rates[1].bps());
- bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1,
- temporal_layer_rates[0].bps());
+ bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[1].bps());
+ bitrate_allocation.SetBitrate(sl_idx, 1, temporal_layer_rates[0].bps());
} else {
RTC_CHECK_EQ(num_temporal_layers, 3);
// In case of three temporal layers the high layer has two frames and the
@@ -297,12 +270,9 @@
// layer frames). Thus high layer requires more bits (comparing pure
// bitrate of layer, excluding bitrate of base layers) to keep quality on
// par with lower layers.
- bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
- temporal_layer_rates[2].bps());
- bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1,
- temporal_layer_rates[0].bps());
- bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 2,
- temporal_layer_rates[1].bps());
+ bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[2].bps());
+ bitrate_allocation.SetBitrate(sl_idx, 1, temporal_layer_rates[0].bps());
+ bitrate_allocation.SetBitrate(sl_idx, 2, temporal_layer_rates[1].bps());
}
}
@@ -314,11 +284,9 @@
// bit-rate allocated.
VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing(
DataRate total_bitrate,
- size_t first_active_layer,
size_t num_spatial_layers) const {
if (num_spatial_layers == 0 ||
- total_bitrate <
- DataRate::kbps(codec_.spatialLayers[first_active_layer].minBitrate)) {
+ total_bitrate < DataRate::kbps(codec_.spatialLayers[0].minBitrate)) {
return VideoBitrateAllocation();
}
VideoBitrateAllocation bitrate_allocation;
@@ -326,8 +294,7 @@
DataRate allocated_rate = DataRate::Zero();
DataRate top_layer_rate = DataRate::Zero();
size_t sl_idx;
- for (sl_idx = first_active_layer;
- sl_idx < first_active_layer + num_spatial_layers; ++sl_idx) {
+ for (sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
const DataRate min_rate =
DataRate::kbps(codec_.spatialLayers[sl_idx].minBitrate);
const DataRate target_rate =
@@ -373,13 +340,11 @@
}
DataRate SvcRateAllocator::GetMaxBitrate(const VideoCodec& codec) {
- const size_t first_active_layer = GetFirstActiveLayer(codec);
const size_t num_spatial_layers = GetNumActiveSpatialLayers(codec);
DataRate max_bitrate = DataRate::Zero();
for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
- max_bitrate += DataRate::kbps(
- codec.spatialLayers[first_active_layer + sl_idx].maxBitrate);
+ max_bitrate += DataRate::kbps(codec.spatialLayers[sl_idx].maxBitrate);
}
if (codec.maxBitrate != 0) {
@@ -401,12 +366,10 @@
absl::InlinedVector<DataRate, kMaxSpatialLayers>
SvcRateAllocator::GetLayerStartBitrates(const VideoCodec& codec) {
absl::InlinedVector<DataRate, kMaxSpatialLayers> start_bitrates;
- const size_t first_active_layer = GetFirstActiveLayer(codec);
- const size_t num_layers = GetNumActiveSpatialLayers(codec);
+ size_t num_layers = GetNumActiveSpatialLayers(codec);
DataRate last_rate = DataRate::Zero();
for (size_t i = 1; i <= num_layers; ++i) {
- DataRate layer_toggling_rate =
- FindLayerTogglingThreshold(codec, first_active_layer, i);
+ DataRate layer_toggling_rate = FindLayerTogglingThreshold(codec, i);
start_bitrates.push_back(layer_toggling_rate);
RTC_DCHECK_LE(last_rate, layer_toggling_rate);
last_rate = layer_toggling_rate;
diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.h b/modules/video_coding/codecs/vp9/svc_rate_allocator.h
index a4e0c28..1b14dd6 100644
--- a/modules/video_coding/codecs/vp9/svc_rate_allocator.h
+++ b/modules/video_coding/codecs/vp9/svc_rate_allocator.h
@@ -38,12 +38,10 @@
private:
VideoBitrateAllocation GetAllocationNormalVideo(
DataRate total_bitrate,
- size_t first_active_layer,
size_t num_spatial_layers) const;
VideoBitrateAllocation GetAllocationScreenSharing(
DataRate total_bitrate,
- size_t first_active_layer,
size_t num_spatial_layers) const;
// Returns the number of layers that are active and have enough bitrate to
diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc
index 06240a3..f4d0924 100644
--- a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc
+++ b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc
@@ -173,13 +173,12 @@
EXPECT_EQ(allocation.GetSpatialLayerSum(1) / 1000, layers[1].minBitrate);
}
-TEST(SvcRateAllocatorTest, DeactivateHigherLayers) {
+TEST(SvcRateAllocatorTest, DeativateLayers) {
for (int deactivated_idx = 2; deactivated_idx >= 0; --deactivated_idx) {
VideoCodec codec = Configure(1280, 720, 3, 1, false);
EXPECT_LE(codec.VP9()->numberOfSpatialLayers, 3U);
- for (int i = deactivated_idx; i < 3; ++i)
- codec.spatialLayers[i].active = false;
+ codec.spatialLayers[deactivated_idx].active = false;
SvcRateAllocator allocator = SvcRateAllocator(codec);
@@ -198,39 +197,11 @@
}
}
-TEST(SvcRateAllocatorTest, DeactivateLowerLayers) {
- for (int deactivated_idx = 0; deactivated_idx < 3; ++deactivated_idx) {
- VideoCodec codec = Configure(1280, 720, 3, 1, false);
- EXPECT_LE(codec.VP9()->numberOfSpatialLayers, 3U);
-
- for (int i = deactivated_idx; i >= 0; --i)
- codec.spatialLayers[i].active = false;
-
- SvcRateAllocator allocator = SvcRateAllocator(codec);
-
- VideoBitrateAllocation allocation = allocator.Allocate(
- VideoBitrateAllocationParameters(10 * 1000 * 1000, 30));
-
- // Ensure layers spatial_idx <= deactivated_idx are deactivated.
- for (int spatial_idx = 0; spatial_idx <= deactivated_idx; ++spatial_idx) {
- EXPECT_EQ(allocation.GetSpatialLayerSum(spatial_idx), 0UL);
- }
-
- // Ensure layers spatial_idx > deactivated_idx are activated.
- for (int spatial_idx = deactivated_idx + 1; spatial_idx < 3;
- ++spatial_idx) {
- EXPECT_GT(allocation.GetSpatialLayerSum(spatial_idx), 0UL);
- }
- }
-}
-
TEST(SvcRateAllocatorTest, NoPaddingIfAllLayersAreDeactivated) {
VideoCodec codec = Configure(1280, 720, 3, 1, false);
EXPECT_EQ(codec.VP9()->numberOfSpatialLayers, 3U);
// Deactivation of base layer deactivates all layers.
codec.spatialLayers[0].active = false;
- codec.spatialLayers[1].active = false;
- codec.spatialLayers[2].active = false;
DataRate padding_rate = SvcRateAllocator::GetPaddingBitrate(codec);
EXPECT_EQ(padding_rate, DataRate::Zero());
}
@@ -309,15 +280,6 @@
EXPECT_GT(allocation.GetSpatialLayerSum(0), 0UL);
EXPECT_EQ(allocation.GetSpatialLayerSum(1), 0UL);
EXPECT_EQ(allocation.GetSpatialLayerSum(2), 0UL);
-
- // Deactivate all layers.
- codec.spatialLayers[0].active = false;
- codec.spatialLayers[1].active = false;
- codec.spatialLayers[2].active = false;
-
- padding_bitrate = SvcRateAllocator::GetPaddingBitrate(codec);
- // No padding expected.
- EXPECT_EQ(DataRate::Zero(), padding_bitrate);
}
TEST_P(SvcRateAllocatorTestParametrizedContentType, StableBitrate) {
diff --git a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
index e54ac34..648bf64 100644
--- a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
+++ b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
@@ -352,120 +352,6 @@
}
}
-TEST_F(TestVp9Impl, DisableEnableBaseLayerTriggersKeyFrame) {
- // Configure encoder to produce N spatial layers. Encode frames for all
- // layers. Then disable all but the last layer. Then reenable all back again.
- const size_t num_spatial_layers = 3;
- const size_t num_frames_to_encode = 5;
-
- ConfigureSvc(num_spatial_layers);
- codec_settings_.VP9()->frameDroppingOn = false;
-
- EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
- encoder_->InitEncode(&codec_settings_, kSettings));
-
- VideoBitrateAllocation bitrate_allocation;
- for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
- // Allocate high bit rate to avoid frame dropping due to rate control.
- bitrate_allocation.SetBitrate(
- sl_idx, 0,
- codec_settings_.spatialLayers[sl_idx].targetBitrate * 1000 * 2);
- }
- encoder_->SetRates(VideoEncoder::RateControlParameters(
- bitrate_allocation, codec_settings_.maxFramerate));
-
- for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
- SetWaitForEncodedFramesThreshold(num_spatial_layers);
- EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
- encoder_->Encode(*NextInputFrame(), nullptr));
- std::vector<EncodedImage> encoded_frame;
- std::vector<CodecSpecificInfo> codec_specific_info;
- ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
- EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
- frame_num == 0);
- }
-
- // Disable all but top layer.
- for (size_t sl_idx = 0; sl_idx < num_spatial_layers - 1; ++sl_idx) {
- bitrate_allocation.SetBitrate(sl_idx, 0, 0);
- }
- encoder_->SetRates(VideoEncoder::RateControlParameters(
- bitrate_allocation, codec_settings_.maxFramerate));
-
- for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
- SetWaitForEncodedFramesThreshold(1);
- EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
- encoder_->Encode(*NextInputFrame(), nullptr));
- std::vector<EncodedImage> encoded_frame;
- std::vector<CodecSpecificInfo> codec_specific_info;
- ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
- // SS available immediatly after switching off.
- EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
- frame_num == 0);
- // No key-frames generated for disabling layers.
- EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameDelta);
- }
-
- // Force key-frame.
- std::vector<VideoFrameType> frame_types = {VideoFrameType::kVideoFrameKey};
- SetWaitForEncodedFramesThreshold(1);
- EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
- encoder_->Encode(*NextInputFrame(), &frame_types));
- std::vector<EncodedImage> encoded_frame;
- std::vector<CodecSpecificInfo> codec_specific_info;
- ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
- // Key-frame should be produced.
- EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameKey);
-
- // Enable the second layer back.
- // Allocate high bit rate to avoid frame dropping due to rate control.
- bitrate_allocation.SetBitrate(
- 1, 0, codec_settings_.spatialLayers[0].targetBitrate * 1000 * 2);
- encoder_->SetRates(VideoEncoder::RateControlParameters(
- bitrate_allocation, codec_settings_.maxFramerate));
-
- for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
- SetWaitForEncodedFramesThreshold(2);
- EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
- encoder_->Encode(*NextInputFrame(), nullptr));
- std::vector<EncodedImage> encoded_frame;
- std::vector<CodecSpecificInfo> codec_specific_info;
- ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
- // SS available immediatly after switching on.
- EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
- frame_num == 0);
- // Keyframe should be generated when enabling lower layers.
- const VideoFrameType expected_type = frame_num == 0
- ? VideoFrameType::kVideoFrameKey
- : VideoFrameType::kVideoFrameDelta;
- EXPECT_EQ(encoded_frame[0]._frameType, expected_type);
- }
-
- // Enable the first layer back.
- // Allocate high bit rate to avoid frame dropping due to rate control.
- bitrate_allocation.SetBitrate(
- 0, 0, codec_settings_.spatialLayers[1].targetBitrate * 1000 * 2);
- encoder_->SetRates(VideoEncoder::RateControlParameters(
- bitrate_allocation, codec_settings_.maxFramerate));
-
- for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
- SetWaitForEncodedFramesThreshold(num_spatial_layers);
- EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
- encoder_->Encode(*NextInputFrame(), nullptr));
- std::vector<EncodedImage> encoded_frame;
- std::vector<CodecSpecificInfo> codec_specific_info;
- ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
- // SS available immediatly after switching on.
- EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
- frame_num == 0);
- // Keyframe should be generated when enabling lower layers.
- const VideoFrameType expected_type = frame_num == 0
- ? VideoFrameType::kVideoFrameKey
- : VideoFrameType::kVideoFrameDelta;
- EXPECT_EQ(encoded_frame[0]._frameType, expected_type);
- }
-}
-
TEST_F(TestVp9Impl, EndOfPicture) {
const size_t num_spatial_layers = 2;
ConfigureSvc(num_spatial_layers);
@@ -846,11 +732,14 @@
false);
}
-TEST_F(TestVp9Impl, EnablingNewLayerInScreenshareForcesAllLayersWithSS) {
+TEST_F(TestVp9Impl, EnablingNewLayerIsDelayedInScreenshareAndAddsSsInfo) {
const size_t num_spatial_layers = 3;
// Chosen by hand, the 2nd frame is dropped with configured per-layer max
// framerate.
const size_t num_frames_to_encode_before_drop = 1;
+ // Chosen by hand, exactly 5 frames are dropped for input fps=30 and max
+ // framerate = 5.
+ const size_t num_dropped_frames = 5;
codec_settings_.maxFramerate = 30;
ConfigureSvc(num_spatial_layers);
@@ -895,8 +784,18 @@
encoder_->SetRates(VideoEncoder::RateControlParameters(
bitrate_allocation, codec_settings_.maxFramerate));
- // All layers are encoded, even though frame dropping should happen.
- SetWaitForEncodedFramesThreshold(num_spatial_layers);
+ for (size_t frame_num = 0; frame_num < num_dropped_frames; ++frame_num) {
+ SetWaitForEncodedFramesThreshold(1);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->Encode(*NextInputFrame(), nullptr));
+ // First layer is dropped due to frame rate cap. The last layer should not
+ // be enabled yet.
+ std::vector<EncodedImage> encoded_frames;
+ std::vector<CodecSpecificInfo> codec_specific_info;
+ ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific_info));
+ }
+
+ SetWaitForEncodedFramesThreshold(2);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->Encode(*NextInputFrame(), nullptr));
// Now all 3 layers should be encoded.
diff --git a/modules/video_coding/codecs/vp9/vp9_impl.cc b/modules/video_coding/codecs/vp9/vp9_impl.cc
index b379e79..42ab4f7 100644
--- a/modules/video_coding/codecs/vp9/vp9_impl.cc
+++ b/modules/video_coding/codecs/vp9/vp9_impl.cc
@@ -15,7 +15,6 @@
#include <algorithm>
#include <limits>
-#include <utility>
#include <vector>
#include "absl/memory/memory.h"
@@ -138,19 +137,15 @@
return ColorSpace(primaries, transfer, matrix, range);
}
-std::pair<size_t, size_t> GetActiveLayers(
- const VideoBitrateAllocation& allocation) {
+bool MoreLayersEnabled(const VideoBitrateAllocation& first,
+ const VideoBitrateAllocation& second) {
for (size_t sl_idx = 0; sl_idx < kMaxSpatialLayers; ++sl_idx) {
- if (allocation.GetSpatialLayerSum(sl_idx) > 0) {
- size_t last_layer = sl_idx + 1;
- while (last_layer < kMaxSpatialLayers &&
- allocation.GetSpatialLayerSum(last_layer) > 0) {
- ++last_layer;
- }
- return std::make_pair(sl_idx, last_layer);
+ if (first.GetSpatialLayerSum(sl_idx) > 0 &&
+ second.GetSpatialLayerSum(sl_idx) == 0) {
+ return true;
}
}
- return {0, 0};
+ return false;
}
uint32_t Interpolate(uint32_t low,
@@ -229,7 +224,6 @@
num_temporal_layers_(0),
num_spatial_layers_(0),
num_active_spatial_layers_(0),
- first_active_layer_(0),
layer_deactivation_requires_key_frame_(
field_trial::IsEnabled("WebRTC-Vp9IssueKeyFrameOnLayerDeactivation")),
is_svc_(false),
@@ -243,7 +237,6 @@
full_superframe_drop_(true),
first_frame_in_picture_(true),
ss_info_needed_(false),
- force_all_active_layers_(false),
is_flexible_mode_(false),
variable_framerate_experiment_(ParseVariableFramerateConfig(
"WebRTC-VP9VariableFramerateScreenshare")),
@@ -296,31 +289,13 @@
bool VP9EncoderImpl::SetSvcRates(
const VideoBitrateAllocation& bitrate_allocation) {
- std::pair<size_t, size_t> current_layers =
- GetActiveLayers(current_bitrate_allocation_);
- std::pair<size_t, size_t> new_layers = GetActiveLayers(bitrate_allocation);
-
- const bool layer_activation_requires_key_frame =
- inter_layer_pred_ == InterLayerPredMode::kOff ||
- inter_layer_pred_ == InterLayerPredMode::kOnKeyPic;
- const bool lower_layers_enabled = new_layers.first < current_layers.first;
- const bool higher_layers_enabled = new_layers.second > current_layers.second;
- const bool disabled_layers = new_layers.first > current_layers.first ||
- new_layers.second < current_layers.second;
-
- if (lower_layers_enabled ||
- (higher_layers_enabled && layer_activation_requires_key_frame) ||
- (disabled_layers && layer_deactivation_requires_key_frame_)) {
- force_key_frame_ = true;
- }
-
- if (current_layers != new_layers) {
- ss_info_needed_ = true;
- }
-
config_->rc_target_bitrate = bitrate_allocation.get_sum_kbps();
if (ExplicitlyConfiguredSpatialLayers()) {
+ const bool layer_activation_requires_key_frame =
+ inter_layer_pred_ == InterLayerPredMode::kOff ||
+ inter_layer_pred_ == InterLayerPredMode::kOnKeyPic;
+
for (size_t sl_idx = 0; sl_idx < num_spatial_layers_; ++sl_idx) {
const bool was_layer_active = (config_->ss_target_bitrate[sl_idx] > 0);
config_->ss_target_bitrate[sl_idx] =
@@ -331,6 +306,15 @@
bitrate_allocation.GetTemporalLayerSum(sl_idx, tl_idx) / 1000;
}
+ const bool is_active_layer = (config_->ss_target_bitrate[sl_idx] > 0);
+ if (!was_layer_active && is_active_layer &&
+ layer_activation_requires_key_frame) {
+ force_key_frame_ = true;
+ } else if (was_layer_active && !is_active_layer &&
+ layer_deactivation_requires_key_frame_) {
+ force_key_frame_ = true;
+ }
+
if (!was_layer_active) {
// Reset frame rate controller if layer is resumed after pause.
framerate_controller_[sl_idx].Reset();
@@ -383,34 +367,13 @@
}
num_active_spatial_layers_ = 0;
- first_active_layer_ = 0;
- bool seen_active_layer = false;
- bool expect_no_more_active_layers = false;
for (int i = 0; i < num_spatial_layers_; ++i) {
if (config_->ss_target_bitrate[i] > 0) {
- RTC_DCHECK(!expect_no_more_active_layers) << "Only middle layer is "
- "deactivated.";
- if (!seen_active_layer) {
- first_active_layer_ = i;
- }
- num_active_spatial_layers_ = i + 1;
- seen_active_layer = true;
- } else {
- expect_no_more_active_layers = seen_active_layer;
+ ++num_active_spatial_layers_;
}
}
RTC_DCHECK_GT(num_active_spatial_layers_, 0);
- if (higher_layers_enabled && !force_key_frame_) {
- // Prohibit drop of all layers for the next frame, so newly enabled
- // layer would have a valid spatial reference.
- for (size_t i = 0; i < num_spatial_layers_; ++i) {
- svc_drop_frame_.framedrop_thresh[i] = 0;
- }
- force_all_active_layers_ = true;
- }
-
- current_bitrate_allocation_ = bitrate_allocation;
return true;
}
@@ -430,16 +393,7 @@
}
codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
-
- if (dynamic_rate_settings_) {
- // Tweak rate control settings based on available network headroom.
- UpdateRateSettings(
- config_, GetRateSettings(parameters.bandwidth_allocation.bps<double>() /
- parameters.bitrate.get_sum_bps()));
- }
-
- bool res = SetSvcRates(parameters.bitrate);
- RTC_DCHECK(res) << "Failed to set new bitrate allocation";
+ requested_rate_settings_ = parameters;
}
// TODO(eladalon): s/inst/codec_settings/g.
@@ -876,10 +830,6 @@
num_steady_state_frames_ >=
variable_framerate_experiment_.frames_before_steady_state;
- // Need to check all frame limiters, even if lower layers are disabled,
- // because variable frame-rate limiter should be checked after the first
- // layer. It's easier to overwrite active layers after, then check all
- // cases.
for (uint8_t sl_idx = 0; sl_idx < num_active_spatial_layers_; ++sl_idx) {
const float layer_fps =
framerate_controller_[layer_id.spatial_layer_id].GetTargetRate();
@@ -906,11 +856,6 @@
}
}
- if (force_all_active_layers_) {
- layer_id.spatial_layer_id = first_active_layer_;
- force_all_active_layers_ = false;
- }
-
RTC_DCHECK_LE(layer_id.spatial_layer_id, num_active_spatial_layers_);
if (layer_id.spatial_layer_id >= num_active_spatial_layers_) {
// Drop entire picture.
@@ -922,12 +867,50 @@
layer_id.temporal_layer_id_per_spatial[sl_idx] = layer_id.temporal_layer_id;
}
- if (layer_id.spatial_layer_id < first_active_layer_) {
- layer_id.spatial_layer_id = first_active_layer_;
- }
-
vpx_codec_control(encoder_, VP9E_SET_SVC_LAYER_ID, &layer_id);
+ if (requested_rate_settings_) {
+ if (dynamic_rate_settings_) {
+ // Tweak rate control settings based on available network headroom.
+ UpdateRateSettings(
+ config_,
+ GetRateSettings(
+ requested_rate_settings_->bandwidth_allocation.bps<double>() /
+ requested_rate_settings_->bitrate.get_sum_bps()));
+ }
+
+ bool more_layers_requested = MoreLayersEnabled(
+ requested_rate_settings_->bitrate, current_bitrate_allocation_);
+ bool less_layers_requested = MoreLayersEnabled(
+ current_bitrate_allocation_, requested_rate_settings_->bitrate);
+ // In SVC can enable new layers only if all lower layers are encoded and at
+ // the base temporal layer.
+ // This will delay rate allocation change until the next frame on the base
+ // spatial layer.
+ // In KSVC or simulcast modes KF will be generated for a new layer, so can
+ // update allocation any time.
+ bool can_upswitch =
+ inter_layer_pred_ != InterLayerPredMode::kOn ||
+ (layer_id.spatial_layer_id == 0 && layer_id.temporal_layer_id == 0);
+ if (!more_layers_requested || can_upswitch) {
+ current_bitrate_allocation_ = requested_rate_settings_->bitrate;
+ requested_rate_settings_ = absl::nullopt;
+ if (!SetSvcRates(current_bitrate_allocation_)) {
+ return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+ }
+ if (less_layers_requested || more_layers_requested) {
+ ss_info_needed_ = true;
+ }
+ if (more_layers_requested && !force_key_frame_) {
+ // Prohibit drop of all layers for the next frame, so newly enabled
+ // layer would have a valid spatial reference.
+ for (size_t i = 0; i < num_spatial_layers_; ++i) {
+ svc_drop_frame_.framedrop_thresh[i] = 0;
+ }
+ }
+ }
+ }
+
if (num_spatial_layers_ > 1) {
// Update frame dropping settings as they may change on per-frame basis.
vpx_codec_control(encoder_, VP9E_SET_SVC_FRAME_DROP_LAYER,
@@ -1134,15 +1117,10 @@
// of key picture (inter-layer prediction is enabled).
const bool is_key_frame = is_key_pic && !vp9_info->inter_layer_predicted;
if (is_key_frame || (ss_info_needed_ && layer_id.temporal_layer_id == 0 &&
- layer_id.spatial_layer_id == first_active_layer_)) {
+ layer_id.spatial_layer_id == 0)) {
vp9_info->ss_data_available = true;
vp9_info->spatial_layer_resolution_present = true;
- // Signal disabled layers.
- for (size_t i = 0; i < first_active_layer_; ++i) {
- vp9_info->width[i] = 0;
- vp9_info->height[i] = 0;
- }
- for (size_t i = first_active_layer_; i < num_active_spatial_layers_; ++i) {
+ for (size_t i = 0; i < num_active_spatial_layers_; ++i) {
vp9_info->width[i] = codec_.width * svc_params_.scaling_factor_num[i] /
svc_params_.scaling_factor_den[i];
vp9_info->height[i] = codec_.height * svc_params_.scaling_factor_num[i] /
diff --git a/modules/video_coding/codecs/vp9/vp9_impl.h b/modules/video_coding/codecs/vp9/vp9_impl.h
index a5f2f35..19c77b6 100644
--- a/modules/video_coding/codecs/vp9/vp9_impl.h
+++ b/modules/video_coding/codecs/vp9/vp9_impl.h
@@ -119,7 +119,6 @@
uint8_t num_temporal_layers_;
uint8_t num_spatial_layers_; // Number of configured SLs
uint8_t num_active_spatial_layers_; // Number of actively encoded SLs
- uint8_t first_active_layer_;
bool layer_deactivation_requires_key_frame_;
bool is_svc_;
InterLayerPredMode inter_layer_pred_;
@@ -131,8 +130,8 @@
vpx_svc_frame_drop_t svc_drop_frame_;
bool first_frame_in_picture_;
VideoBitrateAllocation current_bitrate_allocation_;
+ absl::optional<RateControlParameters> requested_rate_settings_;
bool ss_info_needed_;
- bool force_all_active_layers_;
std::vector<FramerateController> framerate_controller_;
