Reland "VP9 encoder: handle disabled layers correctly"
Now vp9 screenshare would enable new layers as soon as requested and will force all spatial layers present on the next frame, even if they should be dropped because of frame-rate limiting.
This might cause frame-rate liming to be exceeded if layer is toggling on and off very often, but this situation is bad itself. E.g. in realtime video it will cause too many key-frames.
Now SvcRateAllocator and VP9EncoderImpl are aware that there may be some skipped layers before the first enabled. Key-frames and ss_info triggering logic is also updated.
(This is a reland without changes after updates to downstream projects)
Original-Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/153483
Bug: webrtc:10977
Change-Id: I02459c5982da2e0542a837514f5753c5f96401c6
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/154355
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29330}
diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc
index 8513b43..a02e69a 100644
--- a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc
+++ b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc
@@ -25,34 +25,47 @@
const float kSpatialLayeringRateScalingFactor = 0.55f;
const float kTemporalLayeringRateScalingFactor = 0.55f;
+// Returns numberOfSpatialLayers if no layers are active.
+size_t GetFirstActiveLayer(const VideoCodec& codec) {
+ RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9);
+ RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u);
+ size_t layer = 0;
+ for (; layer < codec.VP9().numberOfSpatialLayers; ++layer) {
+ if (codec.spatialLayers[layer].active) {
+ break;
+ }
+ }
+ return layer;
+}
+
static size_t GetNumActiveSpatialLayers(const VideoCodec& codec) {
RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9);
RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u);
- size_t num_spatial_layers = 0;
- for (; num_spatial_layers < codec.VP9().numberOfSpatialLayers;
- ++num_spatial_layers) {
- if (!codec.spatialLayers[num_spatial_layers].active) {
- // TODO(bugs.webrtc.org/9350): Deactivation of middle layer is not
- // implemented. For now deactivation of a VP9 layer deactivates all
- // layers above the deactivated one.
+ const size_t first_active_layer = GetFirstActiveLayer(codec);
+ size_t last_active_layer = first_active_layer;
+ for (; last_active_layer < codec.VP9().numberOfSpatialLayers;
+ ++last_active_layer) {
+ if (!codec.spatialLayers[last_active_layer].active) {
break;
}
}
-
- return num_spatial_layers;
+ return last_active_layer - first_active_layer;
}
std::vector<DataRate> AdjustAndVerify(
const VideoCodec& codec,
+ size_t first_active_layer,
const std::vector<DataRate>& spatial_layer_rates) {
std::vector<DataRate> adjusted_spatial_layer_rates;
// Keep track of rate that couldn't be applied to the previous layer due to
// max bitrate constraint, try to pass it forward to the next one.
DataRate excess_rate = DataRate::Zero();
for (size_t sl_idx = 0; sl_idx < spatial_layer_rates.size(); ++sl_idx) {
- DataRate min_rate = DataRate::kbps(codec.spatialLayers[sl_idx].minBitrate);
- DataRate max_rate = DataRate::kbps(codec.spatialLayers[sl_idx].maxBitrate);
+ DataRate min_rate = DataRate::kbps(
+ codec.spatialLayers[first_active_layer + sl_idx].minBitrate);
+ DataRate max_rate = DataRate::kbps(
+ codec.spatialLayers[first_active_layer + sl_idx].maxBitrate);
DataRate layer_rate = spatial_layer_rates[sl_idx] + excess_rate;
if (layer_rate < min_rate) {
@@ -109,6 +122,7 @@
// Returns the minimum bitrate needed for |num_active_layers| spatial layers to
// become active using the configuration specified by |codec|.
DataRate FindLayerTogglingThreshold(const VideoCodec& codec,
+ size_t first_active_layer,
size_t num_active_layers) {
if (num_active_layers == 1) {
return DataRate::kbps(codec.spatialLayers[0].minBitrate);
@@ -119,8 +133,10 @@
DataRate upper_bound = DataRate::Zero();
if (num_active_layers > 1) {
for (size_t i = 0; i < num_active_layers - 1; ++i) {
- lower_bound += DataRate::kbps(codec.spatialLayers[i].minBitrate);
- upper_bound += DataRate::kbps(codec.spatialLayers[i].maxBitrate);
+ lower_bound += DataRate::kbps(
+ codec.spatialLayers[first_active_layer + i].minBitrate);
+ upper_bound += DataRate::kbps(
+ codec.spatialLayers[first_active_layer + i].maxBitrate);
}
}
upper_bound +=
@@ -131,7 +147,7 @@
// layers respectively.
while (upper_bound - lower_bound > DataRate::bps(1)) {
DataRate try_rate = (lower_bound + upper_bound) / 2;
- if (AdjustAndVerify(codec,
+ if (AdjustAndVerify(codec, first_active_layer,
SplitBitrate(num_active_layers, try_rate,
kSpatialLayeringRateScalingFactor))
.size() == num_active_layers) {
@@ -144,10 +160,12 @@
} else {
DataRate toggling_rate = DataRate::Zero();
for (size_t i = 0; i < num_active_layers - 1; ++i) {
- toggling_rate += DataRate::kbps(codec.spatialLayers[i].targetBitrate);
+ toggling_rate += DataRate::kbps(
+ codec.spatialLayers[first_active_layer + i].targetBitrate);
}
- toggling_rate +=
- DataRate::kbps(codec.spatialLayers[num_active_layers - 1].minBitrate);
+ toggling_rate += DataRate::kbps(
+ codec.spatialLayers[first_active_layer + num_active_layers - 1]
+ .minBitrate);
return toggling_rate;
}
}
@@ -192,7 +210,9 @@
return bitrate_allocation;
}
+ const size_t first_active_layer = GetFirstActiveLayer(codec_);
size_t num_spatial_layers = GetNumActiveSpatialLayers(codec_);
+
if (num_spatial_layers == 0) {
return VideoBitrateAllocation(); // All layers are deactivated.
}
@@ -225,14 +245,17 @@
last_active_layer_count_ = num_spatial_layers;
if (codec_.mode == VideoCodecMode::kRealtimeVideo) {
- return GetAllocationNormalVideo(total_bitrate, num_spatial_layers);
+ return GetAllocationNormalVideo(total_bitrate, first_active_layer,
+ num_spatial_layers);
} else {
- return GetAllocationScreenSharing(total_bitrate, num_spatial_layers);
+ return GetAllocationScreenSharing(total_bitrate, first_active_layer,
+ num_spatial_layers);
}
}
VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo(
DataRate total_bitrate,
+ size_t first_active_layer,
size_t num_spatial_layers) const {
std::vector<DataRate> spatial_layer_rates;
if (num_spatial_layers == 0) {
@@ -241,9 +264,10 @@
num_spatial_layers = 1;
spatial_layer_rates.push_back(total_bitrate);
} else {
- spatial_layer_rates = AdjustAndVerify(
- codec_, SplitBitrate(num_spatial_layers, total_bitrate,
- kSpatialLayeringRateScalingFactor));
+ spatial_layer_rates =
+ AdjustAndVerify(codec_, first_active_layer,
+ SplitBitrate(num_spatial_layers, total_bitrate,
+ kSpatialLayeringRateScalingFactor));
RTC_DCHECK_EQ(spatial_layer_rates.size(), num_spatial_layers);
}
@@ -259,10 +283,13 @@
// layers since they are used for prediction of higher layers and their
// references are far apart.
if (num_temporal_layers == 1) {
- bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[0].bps());
+ bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
+ temporal_layer_rates[0].bps());
} else if (num_temporal_layers == 2) {
- bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[1].bps());
- bitrate_allocation.SetBitrate(sl_idx, 1, temporal_layer_rates[0].bps());
+ bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
+ temporal_layer_rates[1].bps());
+ bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1,
+ temporal_layer_rates[0].bps());
} else {
RTC_CHECK_EQ(num_temporal_layers, 3);
// In case of three temporal layers the high layer has two frames and the
@@ -270,9 +297,12 @@
// layer frames). Thus high layer requires more bits (comparing pure
// bitrate of layer, excluding bitrate of base layers) to keep quality on
// par with lower layers.
- bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[2].bps());
- bitrate_allocation.SetBitrate(sl_idx, 1, temporal_layer_rates[0].bps());
- bitrate_allocation.SetBitrate(sl_idx, 2, temporal_layer_rates[1].bps());
+ bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
+ temporal_layer_rates[2].bps());
+ bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1,
+ temporal_layer_rates[0].bps());
+ bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 2,
+ temporal_layer_rates[1].bps());
}
}
@@ -284,9 +314,11 @@
// bit-rate allocated.
VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing(
DataRate total_bitrate,
+ size_t first_active_layer,
size_t num_spatial_layers) const {
if (num_spatial_layers == 0 ||
- total_bitrate < DataRate::kbps(codec_.spatialLayers[0].minBitrate)) {
+ total_bitrate <
+ DataRate::kbps(codec_.spatialLayers[first_active_layer].minBitrate)) {
return VideoBitrateAllocation();
}
VideoBitrateAllocation bitrate_allocation;
@@ -294,7 +326,8 @@
DataRate allocated_rate = DataRate::Zero();
DataRate top_layer_rate = DataRate::Zero();
size_t sl_idx;
- for (sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
+ for (sl_idx = first_active_layer;
+ sl_idx < first_active_layer + num_spatial_layers; ++sl_idx) {
const DataRate min_rate =
DataRate::kbps(codec_.spatialLayers[sl_idx].minBitrate);
const DataRate target_rate =
@@ -340,11 +373,13 @@
}
DataRate SvcRateAllocator::GetMaxBitrate(const VideoCodec& codec) {
+ const size_t first_active_layer = GetFirstActiveLayer(codec);
const size_t num_spatial_layers = GetNumActiveSpatialLayers(codec);
DataRate max_bitrate = DataRate::Zero();
for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
- max_bitrate += DataRate::kbps(codec.spatialLayers[sl_idx].maxBitrate);
+ max_bitrate += DataRate::kbps(
+ codec.spatialLayers[first_active_layer + sl_idx].maxBitrate);
}
if (codec.maxBitrate != 0) {
@@ -366,10 +401,12 @@
absl::InlinedVector<DataRate, kMaxSpatialLayers>
SvcRateAllocator::GetLayerStartBitrates(const VideoCodec& codec) {
absl::InlinedVector<DataRate, kMaxSpatialLayers> start_bitrates;
- size_t num_layers = GetNumActiveSpatialLayers(codec);
+ const size_t first_active_layer = GetFirstActiveLayer(codec);
+ const size_t num_layers = GetNumActiveSpatialLayers(codec);
DataRate last_rate = DataRate::Zero();
for (size_t i = 1; i <= num_layers; ++i) {
- DataRate layer_toggling_rate = FindLayerTogglingThreshold(codec, i);
+ DataRate layer_toggling_rate =
+ FindLayerTogglingThreshold(codec, first_active_layer, i);
start_bitrates.push_back(layer_toggling_rate);
RTC_DCHECK_LE(last_rate, layer_toggling_rate);
last_rate = layer_toggling_rate;
diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.h b/modules/video_coding/codecs/vp9/svc_rate_allocator.h
index 1b14dd6..a4e0c28 100644
--- a/modules/video_coding/codecs/vp9/svc_rate_allocator.h
+++ b/modules/video_coding/codecs/vp9/svc_rate_allocator.h
@@ -38,10 +38,12 @@
private:
VideoBitrateAllocation GetAllocationNormalVideo(
DataRate total_bitrate,
+ size_t first_active_layer,
size_t num_spatial_layers) const;
VideoBitrateAllocation GetAllocationScreenSharing(
DataRate total_bitrate,
+ size_t first_active_layer,
size_t num_spatial_layers) const;
// Returns the number of layers that are active and have enough bitrate to
diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc
index f4d0924..06240a3 100644
--- a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc
+++ b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc
@@ -173,12 +173,13 @@
EXPECT_EQ(allocation.GetSpatialLayerSum(1) / 1000, layers[1].minBitrate);
}
-TEST(SvcRateAllocatorTest, DeativateLayers) {
+TEST(SvcRateAllocatorTest, DeactivateHigherLayers) {
for (int deactivated_idx = 2; deactivated_idx >= 0; --deactivated_idx) {
VideoCodec codec = Configure(1280, 720, 3, 1, false);
EXPECT_LE(codec.VP9()->numberOfSpatialLayers, 3U);
- codec.spatialLayers[deactivated_idx].active = false;
+ for (int i = deactivated_idx; i < 3; ++i)
+ codec.spatialLayers[i].active = false;
SvcRateAllocator allocator = SvcRateAllocator(codec);
@@ -197,11 +198,39 @@
}
}
+TEST(SvcRateAllocatorTest, DeactivateLowerLayers) {
+ for (int deactivated_idx = 0; deactivated_idx < 3; ++deactivated_idx) {
+ VideoCodec codec = Configure(1280, 720, 3, 1, false);
+ EXPECT_LE(codec.VP9()->numberOfSpatialLayers, 3U);
+
+ for (int i = deactivated_idx; i >= 0; --i)
+ codec.spatialLayers[i].active = false;
+
+ SvcRateAllocator allocator = SvcRateAllocator(codec);
+
+ VideoBitrateAllocation allocation = allocator.Allocate(
+ VideoBitrateAllocationParameters(10 * 1000 * 1000, 30));
+
+ // Ensure layers spatial_idx <= deactivated_idx are deactivated.
+ for (int spatial_idx = 0; spatial_idx <= deactivated_idx; ++spatial_idx) {
+ EXPECT_EQ(allocation.GetSpatialLayerSum(spatial_idx), 0UL);
+ }
+
+ // Ensure layers spatial_idx > deactivated_idx are activated.
+ for (int spatial_idx = deactivated_idx + 1; spatial_idx < 3;
+ ++spatial_idx) {
+ EXPECT_GT(allocation.GetSpatialLayerSum(spatial_idx), 0UL);
+ }
+ }
+}
+
TEST(SvcRateAllocatorTest, NoPaddingIfAllLayersAreDeactivated) {
VideoCodec codec = Configure(1280, 720, 3, 1, false);
EXPECT_EQ(codec.VP9()->numberOfSpatialLayers, 3U);
// Deactivation of base layer deactivates all layers.
codec.spatialLayers[0].active = false;
+ codec.spatialLayers[1].active = false;
+ codec.spatialLayers[2].active = false;
DataRate padding_rate = SvcRateAllocator::GetPaddingBitrate(codec);
EXPECT_EQ(padding_rate, DataRate::Zero());
}
@@ -280,6 +309,15 @@
EXPECT_GT(allocation.GetSpatialLayerSum(0), 0UL);
EXPECT_EQ(allocation.GetSpatialLayerSum(1), 0UL);
EXPECT_EQ(allocation.GetSpatialLayerSum(2), 0UL);
+
+ // Deactivate all layers.
+ codec.spatialLayers[0].active = false;
+ codec.spatialLayers[1].active = false;
+ codec.spatialLayers[2].active = false;
+
+ padding_bitrate = SvcRateAllocator::GetPaddingBitrate(codec);
+ // No padding expected.
+ EXPECT_EQ(DataRate::Zero(), padding_bitrate);
}
TEST_P(SvcRateAllocatorTestParametrizedContentType, StableBitrate) {
diff --git a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
index 648bf64..e54ac34 100644
--- a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
+++ b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
@@ -352,6 +352,120 @@
}
}
+TEST_F(TestVp9Impl, DisableEnableBaseLayerTriggersKeyFrame) {
+ // Configure encoder to produce N spatial layers. Encode frames for all
+ // layers. Then disable all but the last layer. Then reenable all back again.
+ const size_t num_spatial_layers = 3;
+ const size_t num_frames_to_encode = 5;
+
+ ConfigureSvc(num_spatial_layers);
+ codec_settings_.VP9()->frameDroppingOn = false;
+
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+
+ VideoBitrateAllocation bitrate_allocation;
+ for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
+ // Allocate high bit rate to avoid frame dropping due to rate control.
+ bitrate_allocation.SetBitrate(
+ sl_idx, 0,
+ codec_settings_.spatialLayers[sl_idx].targetBitrate * 1000 * 2);
+ }
+ encoder_->SetRates(VideoEncoder::RateControlParameters(
+ bitrate_allocation, codec_settings_.maxFramerate));
+
+ for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
+ SetWaitForEncodedFramesThreshold(num_spatial_layers);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->Encode(*NextInputFrame(), nullptr));
+ std::vector<EncodedImage> encoded_frame;
+ std::vector<CodecSpecificInfo> codec_specific_info;
+ ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
+ EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
+ frame_num == 0);
+ }
+
+ // Disable all but top layer.
+ for (size_t sl_idx = 0; sl_idx < num_spatial_layers - 1; ++sl_idx) {
+ bitrate_allocation.SetBitrate(sl_idx, 0, 0);
+ }
+ encoder_->SetRates(VideoEncoder::RateControlParameters(
+ bitrate_allocation, codec_settings_.maxFramerate));
+
+ for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
+ SetWaitForEncodedFramesThreshold(1);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->Encode(*NextInputFrame(), nullptr));
+ std::vector<EncodedImage> encoded_frame;
+ std::vector<CodecSpecificInfo> codec_specific_info;
+ ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
+ // SS available immediatly after switching off.
+ EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
+ frame_num == 0);
+ // No key-frames generated for disabling layers.
+ EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameDelta);
+ }
+
+ // Force key-frame.
+ std::vector<VideoFrameType> frame_types = {VideoFrameType::kVideoFrameKey};
+ SetWaitForEncodedFramesThreshold(1);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->Encode(*NextInputFrame(), &frame_types));
+ std::vector<EncodedImage> encoded_frame;
+ std::vector<CodecSpecificInfo> codec_specific_info;
+ ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
+ // Key-frame should be produced.
+ EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameKey);
+
+ // Enable the second layer back.
+ // Allocate high bit rate to avoid frame dropping due to rate control.
+ bitrate_allocation.SetBitrate(
+ 1, 0, codec_settings_.spatialLayers[0].targetBitrate * 1000 * 2);
+ encoder_->SetRates(VideoEncoder::RateControlParameters(
+ bitrate_allocation, codec_settings_.maxFramerate));
+
+ for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
+ SetWaitForEncodedFramesThreshold(2);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->Encode(*NextInputFrame(), nullptr));
+ std::vector<EncodedImage> encoded_frame;
+ std::vector<CodecSpecificInfo> codec_specific_info;
+ ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
+ // SS available immediatly after switching on.
+ EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
+ frame_num == 0);
+ // Keyframe should be generated when enabling lower layers.
+ const VideoFrameType expected_type = frame_num == 0
+ ? VideoFrameType::kVideoFrameKey
+ : VideoFrameType::kVideoFrameDelta;
+ EXPECT_EQ(encoded_frame[0]._frameType, expected_type);
+ }
+
+ // Enable the first layer back.
+ // Allocate high bit rate to avoid frame dropping due to rate control.
+ bitrate_allocation.SetBitrate(
+ 0, 0, codec_settings_.spatialLayers[1].targetBitrate * 1000 * 2);
+ encoder_->SetRates(VideoEncoder::RateControlParameters(
+ bitrate_allocation, codec_settings_.maxFramerate));
+
+ for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
+ SetWaitForEncodedFramesThreshold(num_spatial_layers);
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->Encode(*NextInputFrame(), nullptr));
+ std::vector<EncodedImage> encoded_frame;
+ std::vector<CodecSpecificInfo> codec_specific_info;
+ ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
+ // SS available immediatly after switching on.
+ EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
+ frame_num == 0);
+ // Keyframe should be generated when enabling lower layers.
+ const VideoFrameType expected_type = frame_num == 0
+ ? VideoFrameType::kVideoFrameKey
+ : VideoFrameType::kVideoFrameDelta;
+ EXPECT_EQ(encoded_frame[0]._frameType, expected_type);
+ }
+}
+
TEST_F(TestVp9Impl, EndOfPicture) {
const size_t num_spatial_layers = 2;
ConfigureSvc(num_spatial_layers);
@@ -732,14 +846,11 @@
false);
}
-TEST_F(TestVp9Impl, EnablingNewLayerIsDelayedInScreenshareAndAddsSsInfo) {
+TEST_F(TestVp9Impl, EnablingNewLayerInScreenshareForcesAllLayersWithSS) {
const size_t num_spatial_layers = 3;
// Chosen by hand, the 2nd frame is dropped with configured per-layer max
// framerate.
const size_t num_frames_to_encode_before_drop = 1;
- // Chosen by hand, exactly 5 frames are dropped for input fps=30 and max
- // framerate = 5.
- const size_t num_dropped_frames = 5;
codec_settings_.maxFramerate = 30;
ConfigureSvc(num_spatial_layers);
@@ -784,18 +895,8 @@
encoder_->SetRates(VideoEncoder::RateControlParameters(
bitrate_allocation, codec_settings_.maxFramerate));
- for (size_t frame_num = 0; frame_num < num_dropped_frames; ++frame_num) {
- SetWaitForEncodedFramesThreshold(1);
- EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
- encoder_->Encode(*NextInputFrame(), nullptr));
- // First layer is dropped due to frame rate cap. The last layer should not
- // be enabled yet.
- std::vector<EncodedImage> encoded_frames;
- std::vector<CodecSpecificInfo> codec_specific_info;
- ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific_info));
- }
-
- SetWaitForEncodedFramesThreshold(2);
+ // All layers are encoded, even though frame dropping should happen.
+ SetWaitForEncodedFramesThreshold(num_spatial_layers);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->Encode(*NextInputFrame(), nullptr));
// Now all 3 layers should be encoded.
diff --git a/modules/video_coding/codecs/vp9/vp9_impl.cc b/modules/video_coding/codecs/vp9/vp9_impl.cc
index 42ab4f7..b379e79 100644
--- a/modules/video_coding/codecs/vp9/vp9_impl.cc
+++ b/modules/video_coding/codecs/vp9/vp9_impl.cc
@@ -15,6 +15,7 @@
#include <algorithm>
#include <limits>
+#include <utility>
#include <vector>
#include "absl/memory/memory.h"
@@ -137,15 +138,19 @@
return ColorSpace(primaries, transfer, matrix, range);
}
-bool MoreLayersEnabled(const VideoBitrateAllocation& first,
- const VideoBitrateAllocation& second) {
+std::pair<size_t, size_t> GetActiveLayers(
+ const VideoBitrateAllocation& allocation) {
for (size_t sl_idx = 0; sl_idx < kMaxSpatialLayers; ++sl_idx) {
- if (first.GetSpatialLayerSum(sl_idx) > 0 &&
- second.GetSpatialLayerSum(sl_idx) == 0) {
- return true;
+ if (allocation.GetSpatialLayerSum(sl_idx) > 0) {
+ size_t last_layer = sl_idx + 1;
+ while (last_layer < kMaxSpatialLayers &&
+ allocation.GetSpatialLayerSum(last_layer) > 0) {
+ ++last_layer;
+ }
+ return std::make_pair(sl_idx, last_layer);
}
}
- return false;
+ return {0, 0};
}
uint32_t Interpolate(uint32_t low,
@@ -224,6 +229,7 @@
num_temporal_layers_(0),
num_spatial_layers_(0),
num_active_spatial_layers_(0),
+ first_active_layer_(0),
layer_deactivation_requires_key_frame_(
field_trial::IsEnabled("WebRTC-Vp9IssueKeyFrameOnLayerDeactivation")),
is_svc_(false),
@@ -237,6 +243,7 @@
full_superframe_drop_(true),
first_frame_in_picture_(true),
ss_info_needed_(false),
+ force_all_active_layers_(false),
is_flexible_mode_(false),
variable_framerate_experiment_(ParseVariableFramerateConfig(
"WebRTC-VP9VariableFramerateScreenshare")),
@@ -289,13 +296,31 @@
bool VP9EncoderImpl::SetSvcRates(
const VideoBitrateAllocation& bitrate_allocation) {
+ std::pair<size_t, size_t> current_layers =
+ GetActiveLayers(current_bitrate_allocation_);
+ std::pair<size_t, size_t> new_layers = GetActiveLayers(bitrate_allocation);
+
+ const bool layer_activation_requires_key_frame =
+ inter_layer_pred_ == InterLayerPredMode::kOff ||
+ inter_layer_pred_ == InterLayerPredMode::kOnKeyPic;
+ const bool lower_layers_enabled = new_layers.first < current_layers.first;
+ const bool higher_layers_enabled = new_layers.second > current_layers.second;
+ const bool disabled_layers = new_layers.first > current_layers.first ||
+ new_layers.second < current_layers.second;
+
+ if (lower_layers_enabled ||
+ (higher_layers_enabled && layer_activation_requires_key_frame) ||
+ (disabled_layers && layer_deactivation_requires_key_frame_)) {
+ force_key_frame_ = true;
+ }
+
+ if (current_layers != new_layers) {
+ ss_info_needed_ = true;
+ }
+
config_->rc_target_bitrate = bitrate_allocation.get_sum_kbps();
if (ExplicitlyConfiguredSpatialLayers()) {
- const bool layer_activation_requires_key_frame =
- inter_layer_pred_ == InterLayerPredMode::kOff ||
- inter_layer_pred_ == InterLayerPredMode::kOnKeyPic;
-
for (size_t sl_idx = 0; sl_idx < num_spatial_layers_; ++sl_idx) {
const bool was_layer_active = (config_->ss_target_bitrate[sl_idx] > 0);
config_->ss_target_bitrate[sl_idx] =
@@ -306,15 +331,6 @@
bitrate_allocation.GetTemporalLayerSum(sl_idx, tl_idx) / 1000;
}
- const bool is_active_layer = (config_->ss_target_bitrate[sl_idx] > 0);
- if (!was_layer_active && is_active_layer &&
- layer_activation_requires_key_frame) {
- force_key_frame_ = true;
- } else if (was_layer_active && !is_active_layer &&
- layer_deactivation_requires_key_frame_) {
- force_key_frame_ = true;
- }
-
if (!was_layer_active) {
// Reset frame rate controller if layer is resumed after pause.
framerate_controller_[sl_idx].Reset();
@@ -367,13 +383,34 @@
}
num_active_spatial_layers_ = 0;
+ first_active_layer_ = 0;
+ bool seen_active_layer = false;
+ bool expect_no_more_active_layers = false;
for (int i = 0; i < num_spatial_layers_; ++i) {
if (config_->ss_target_bitrate[i] > 0) {
- ++num_active_spatial_layers_;
+ RTC_DCHECK(!expect_no_more_active_layers) << "Only middle layer is "
+ "deactivated.";
+ if (!seen_active_layer) {
+ first_active_layer_ = i;
+ }
+ num_active_spatial_layers_ = i + 1;
+ seen_active_layer = true;
+ } else {
+ expect_no_more_active_layers = seen_active_layer;
}
}
RTC_DCHECK_GT(num_active_spatial_layers_, 0);
+ if (higher_layers_enabled && !force_key_frame_) {
+ // Prohibit drop of all layers for the next frame, so newly enabled
+ // layer would have a valid spatial reference.
+ for (size_t i = 0; i < num_spatial_layers_; ++i) {
+ svc_drop_frame_.framedrop_thresh[i] = 0;
+ }
+ force_all_active_layers_ = true;
+ }
+
+ current_bitrate_allocation_ = bitrate_allocation;
return true;
}
@@ -393,7 +430,16 @@
}
codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
- requested_rate_settings_ = parameters;
+
+ if (dynamic_rate_settings_) {
+ // Tweak rate control settings based on available network headroom.
+ UpdateRateSettings(
+ config_, GetRateSettings(parameters.bandwidth_allocation.bps<double>() /
+ parameters.bitrate.get_sum_bps()));
+ }
+
+ bool res = SetSvcRates(parameters.bitrate);
+ RTC_DCHECK(res) << "Failed to set new bitrate allocation";
}
// TODO(eladalon): s/inst/codec_settings/g.
@@ -830,6 +876,10 @@
num_steady_state_frames_ >=
variable_framerate_experiment_.frames_before_steady_state;
+ // Need to check all frame limiters, even if lower layers are disabled,
+ // because variable frame-rate limiter should be checked after the first
+ // layer. It's easier to overwrite active layers after, then check all
+ // cases.
for (uint8_t sl_idx = 0; sl_idx < num_active_spatial_layers_; ++sl_idx) {
const float layer_fps =
framerate_controller_[layer_id.spatial_layer_id].GetTargetRate();
@@ -856,6 +906,11 @@
}
}
+ if (force_all_active_layers_) {
+ layer_id.spatial_layer_id = first_active_layer_;
+ force_all_active_layers_ = false;
+ }
+
RTC_DCHECK_LE(layer_id.spatial_layer_id, num_active_spatial_layers_);
if (layer_id.spatial_layer_id >= num_active_spatial_layers_) {
// Drop entire picture.
@@ -867,50 +922,12 @@
layer_id.temporal_layer_id_per_spatial[sl_idx] = layer_id.temporal_layer_id;
}
- vpx_codec_control(encoder_, VP9E_SET_SVC_LAYER_ID, &layer_id);
-
- if (requested_rate_settings_) {
- if (dynamic_rate_settings_) {
- // Tweak rate control settings based on available network headroom.
- UpdateRateSettings(
- config_,
- GetRateSettings(
- requested_rate_settings_->bandwidth_allocation.bps<double>() /
- requested_rate_settings_->bitrate.get_sum_bps()));
- }
-
- bool more_layers_requested = MoreLayersEnabled(
- requested_rate_settings_->bitrate, current_bitrate_allocation_);
- bool less_layers_requested = MoreLayersEnabled(
- current_bitrate_allocation_, requested_rate_settings_->bitrate);
- // In SVC can enable new layers only if all lower layers are encoded and at
- // the base temporal layer.
- // This will delay rate allocation change until the next frame on the base
- // spatial layer.
- // In KSVC or simulcast modes KF will be generated for a new layer, so can
- // update allocation any time.
- bool can_upswitch =
- inter_layer_pred_ != InterLayerPredMode::kOn ||
- (layer_id.spatial_layer_id == 0 && layer_id.temporal_layer_id == 0);
- if (!more_layers_requested || can_upswitch) {
- current_bitrate_allocation_ = requested_rate_settings_->bitrate;
- requested_rate_settings_ = absl::nullopt;
- if (!SetSvcRates(current_bitrate_allocation_)) {
- return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
- }
- if (less_layers_requested || more_layers_requested) {
- ss_info_needed_ = true;
- }
- if (more_layers_requested && !force_key_frame_) {
- // Prohibit drop of all layers for the next frame, so newly enabled
- // layer would have a valid spatial reference.
- for (size_t i = 0; i < num_spatial_layers_; ++i) {
- svc_drop_frame_.framedrop_thresh[i] = 0;
- }
- }
- }
+ if (layer_id.spatial_layer_id < first_active_layer_) {
+ layer_id.spatial_layer_id = first_active_layer_;
}
+ vpx_codec_control(encoder_, VP9E_SET_SVC_LAYER_ID, &layer_id);
+
if (num_spatial_layers_ > 1) {
// Update frame dropping settings as they may change on per-frame basis.
vpx_codec_control(encoder_, VP9E_SET_SVC_FRAME_DROP_LAYER,
@@ -1117,10 +1134,15 @@
// of key picture (inter-layer prediction is enabled).
const bool is_key_frame = is_key_pic && !vp9_info->inter_layer_predicted;
if (is_key_frame || (ss_info_needed_ && layer_id.temporal_layer_id == 0 &&
- layer_id.spatial_layer_id == 0)) {
+ layer_id.spatial_layer_id == first_active_layer_)) {
vp9_info->ss_data_available = true;
vp9_info->spatial_layer_resolution_present = true;
- for (size_t i = 0; i < num_active_spatial_layers_; ++i) {
+ // Signal disabled layers.
+ for (size_t i = 0; i < first_active_layer_; ++i) {
+ vp9_info->width[i] = 0;
+ vp9_info->height[i] = 0;
+ }
+ for (size_t i = first_active_layer_; i < num_active_spatial_layers_; ++i) {
vp9_info->width[i] = codec_.width * svc_params_.scaling_factor_num[i] /
svc_params_.scaling_factor_den[i];
vp9_info->height[i] = codec_.height * svc_params_.scaling_factor_num[i] /
diff --git a/modules/video_coding/codecs/vp9/vp9_impl.h b/modules/video_coding/codecs/vp9/vp9_impl.h
index 19c77b6..a5f2f35 100644
--- a/modules/video_coding/codecs/vp9/vp9_impl.h
+++ b/modules/video_coding/codecs/vp9/vp9_impl.h
@@ -119,6 +119,7 @@
uint8_t num_temporal_layers_;
uint8_t num_spatial_layers_; // Number of configured SLs
uint8_t num_active_spatial_layers_; // Number of actively encoded SLs
+ uint8_t first_active_layer_;
bool layer_deactivation_requires_key_frame_;
bool is_svc_;
InterLayerPredMode inter_layer_pred_;
@@ -130,8 +131,8 @@
vpx_svc_frame_drop_t svc_drop_frame_;
bool first_frame_in_picture_;
VideoBitrateAllocation current_bitrate_allocation_;
- absl::optional<RateControlParameters> requested_rate_settings_;
bool ss_info_needed_;
+ bool force_all_active_layers_;
std::vector<FramerateController> framerate_controller_;
