Add frame rate parameter to SpatialLayer struct.
This will allow us to configure VP9 encoder to produce spatial layers
with different frame rates.
Bug: webrtc:9650
Change-Id: I3a9c58072003b8a8da681d5291d8f7ede7f52fa4
Reviewed-on: https://webrtc-review.googlesource.com/95427
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24435}diff --git a/common_types.h b/common_types.h
index c1c5b4e..beb6bca 100644
--- a/common_types.h
+++ b/common_types.h
@@ -361,6 +361,7 @@
unsigned short width;
unsigned short height;
+ float maxFramerate; // fps.
unsigned char numberOfTemporalLayers;
unsigned int maxBitrate; // kilobits/sec.
unsigned int targetBitrate; // kilobits/sec.
diff --git a/media/engine/vp8_encoder_simulcast_proxy_unittest.cc b/media/engine/vp8_encoder_simulcast_proxy_unittest.cc
index dfa35cb..47e57aa 100644
--- a/media/engine/vp8_encoder_simulcast_proxy_unittest.cc
+++ b/media/engine/vp8_encoder_simulcast_proxy_unittest.cc
@@ -63,12 +63,30 @@
"SimulcastEncoderAdapter (Fake, Fake, Fake)";
VideoCodec codec_settings;
webrtc::test::CodecSettings(kVideoCodecVP8, &codec_settings);
- codec_settings.simulcastStream[0] = {
- test::kTestWidth, test::kTestHeight, 2, 2000, 1000, 1000, 56};
- codec_settings.simulcastStream[1] = {
- test::kTestWidth, test::kTestHeight, 2, 3000, 1000, 1000, 56};
- codec_settings.simulcastStream[2] = {
- test::kTestWidth, test::kTestHeight, 2, 5000, 1000, 1000, 56};
+ codec_settings.simulcastStream[0] = {test::kTestWidth,
+ test::kTestHeight,
+ test::kTestFrameRate,
+ 2,
+ 2000,
+ 1000,
+ 1000,
+ 56};
+ codec_settings.simulcastStream[1] = {test::kTestWidth,
+ test::kTestHeight,
+ test::kTestFrameRate,
+ 2,
+ 3000,
+ 1000,
+ 1000,
+ 56};
+ codec_settings.simulcastStream[2] = {test::kTestWidth,
+ test::kTestHeight,
+ test::kTestFrameRate,
+ 2,
+ 5000,
+ 1000,
+ 1000,
+ 56};
codec_settings.numberOfSimulcastStreams = 3;
NiceMock<MockEncoder>* mock_encoder = new NiceMock<MockEncoder>();
diff --git a/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc
index a5c76ad..2958a9a 100644
--- a/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc
+++ b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc
@@ -70,11 +70,11 @@
void ConfigureSvc(VideoCodec* codec_settings) {
RTC_CHECK_EQ(kVideoCodecVP9, codec_settings->codecType);
- const std::vector<SpatialLayer> layers =
- GetSvcConfig(codec_settings->width, codec_settings->height,
- codec_settings->VP9()->numberOfSpatialLayers,
- codec_settings->VP9()->numberOfTemporalLayers,
- /* is_screen_sharing = */ false);
+ const std::vector<SpatialLayer> layers = GetSvcConfig(
+ codec_settings->width, codec_settings->height, kMaxFramerateFps,
+ codec_settings->VP9()->numberOfSpatialLayers,
+ codec_settings->VP9()->numberOfTemporalLayers,
+ /* is_screen_sharing = */ false);
ASSERT_EQ(codec_settings->VP9()->numberOfSpatialLayers, layers.size())
<< "GetSvcConfig returned fewer spatial layers than configured.";
diff --git a/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc b/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc
index be92b34..aaa10dd 100644
--- a/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc
+++ b/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc
@@ -31,6 +31,7 @@
constexpr int kDefaultMinPixelsPerFrame = 320 * 180;
constexpr int kWidth = 172;
constexpr int kHeight = 144;
+constexpr float kFramerateFps = 30;
} // namespace
class TestVp8Impl : public VideoCodecUnitTest {
@@ -182,57 +183,57 @@
TEST_F(TestVp8Impl, ChecksSimulcastSettings) {
codec_settings_.numberOfSimulcastStreams = 2;
// Reslutions are not scaled by 2, temporal layers do not match.
- codec_settings_.simulcastStream[0] = {kWidth, kHeight, 2, 4000,
- 3000, 2000, 80};
- codec_settings_.simulcastStream[1] = {kWidth, kHeight, 3, 4000,
- 3000, 2000, 80};
+ codec_settings_.simulcastStream[0] = {kWidth, kHeight, kFramerateFps, 2,
+ 4000, 3000, 2000, 80};
+ codec_settings_.simulcastStream[1] = {kWidth, kHeight, 30, 3,
+ 4000, 3000, 2000, 80};
EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED,
encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize));
codec_settings_.numberOfSimulcastStreams = 3;
// Reslutions are not scaled by 2.
- codec_settings_.simulcastStream[0] = {kWidth / 2, kHeight / 2, 1, 4000,
- 3000, 2000, 80};
- codec_settings_.simulcastStream[1] = {kWidth / 2, kHeight / 2, 1, 4000,
- 3000, 2000, 80};
- codec_settings_.simulcastStream[2] = {kWidth, kHeight, 1, 4000,
- 3000, 2000, 80};
+ codec_settings_.simulcastStream[0] = {
+ kWidth / 2, kHeight / 2, kFramerateFps, 1, 4000, 3000, 2000, 80};
+ codec_settings_.simulcastStream[1] = {
+ kWidth / 2, kHeight / 2, kFramerateFps, 1, 4000, 3000, 2000, 80};
+ codec_settings_.simulcastStream[2] = {kWidth, kHeight, 30, 1,
+ 4000, 3000, 2000, 80};
EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED,
encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize));
// Reslutions are not scaled by 2.
- codec_settings_.simulcastStream[0] = {kWidth, kHeight, 1, 4000,
- 3000, 2000, 80};
- codec_settings_.simulcastStream[1] = {kWidth, kHeight, 1, 4000,
- 3000, 2000, 80};
- codec_settings_.simulcastStream[2] = {kWidth, kHeight, 1, 4000,
- 3000, 2000, 80};
+ codec_settings_.simulcastStream[0] = {kWidth, kHeight, kFramerateFps, 1,
+ 4000, 3000, 2000, 80};
+ codec_settings_.simulcastStream[1] = {kWidth, kHeight, kFramerateFps, 1,
+ 4000, 3000, 2000, 80};
+ codec_settings_.simulcastStream[2] = {kWidth, kHeight, kFramerateFps, 1,
+ 4000, 3000, 2000, 80};
EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED,
encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize));
// Temporal layers do not match.
- codec_settings_.simulcastStream[0] = {kWidth / 4, kHeight / 4, 1, 4000,
- 3000, 2000, 80};
- codec_settings_.simulcastStream[1] = {kWidth / 2, kHeight / 2, 2, 4000,
- 3000, 2000, 80};
- codec_settings_.simulcastStream[2] = {kWidth, kHeight, 3, 4000,
- 3000, 2000, 80};
+ codec_settings_.simulcastStream[0] = {
+ kWidth / 4, kHeight / 4, kFramerateFps, 1, 4000, 3000, 2000, 80};
+ codec_settings_.simulcastStream[1] = {
+ kWidth / 2, kHeight / 2, kFramerateFps, 2, 4000, 3000, 2000, 80};
+ codec_settings_.simulcastStream[2] = {kWidth, kHeight, kFramerateFps, 3,
+ 4000, 3000, 2000, 80};
EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED,
encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize));
// Resolutions do not match codec config.
codec_settings_.simulcastStream[0] = {
- kWidth / 4 + 1, kHeight / 4 + 1, 1, 4000, 3000, 2000, 80};
+ kWidth / 4 + 1, kHeight / 4 + 1, kFramerateFps, 1, 4000, 3000, 2000, 80};
codec_settings_.simulcastStream[1] = {
- kWidth / 2 + 2, kHeight / 2 + 2, 1, 4000, 3000, 2000, 80};
- codec_settings_.simulcastStream[2] = {kWidth + 4, kHeight + 4, 1, 4000,
- 3000, 2000, 80};
+ kWidth / 2 + 2, kHeight / 2 + 2, kFramerateFps, 1, 4000, 3000, 2000, 80};
+ codec_settings_.simulcastStream[2] = {
+ kWidth + 4, kHeight + 4, kFramerateFps, 1, 4000, 3000, 2000, 80};
EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED,
encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize));
// Everything fine: scaling by 2, top resolution matches video, temporal
// settings are the same for all layers.
- codec_settings_.simulcastStream[0] = {kWidth / 4, kHeight / 4, 1, 4000,
- 3000, 2000, 80};
- codec_settings_.simulcastStream[1] = {kWidth / 2, kHeight / 2, 1, 4000,
- 3000, 2000, 80};
- codec_settings_.simulcastStream[2] = {kWidth, kHeight, 1, 4000,
- 3000, 2000, 80};
+ codec_settings_.simulcastStream[0] = {
+ kWidth / 4, kHeight / 4, kFramerateFps, 1, 4000, 3000, 2000, 80};
+ codec_settings_.simulcastStream[1] = {
+ kWidth / 2, kHeight / 2, kFramerateFps, 1, 4000, 3000, 2000, 80};
+ codec_settings_.simulcastStream[2] = {kWidth, kHeight, kFramerateFps, 1,
+ 4000, 3000, 2000, 80};
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize));
}
diff --git a/modules/video_coding/codecs/vp9/svc_config.cc b/modules/video_coding/codecs/vp9/svc_config.cc
index 22902fe..6807698 100644
--- a/modules/video_coding/codecs/vp9/svc_config.cc
+++ b/modules/video_coding/codecs/vp9/svc_config.cc
@@ -23,11 +23,13 @@
const size_t kMinVp9SvcBitrateKbps = 30;
const size_t kMaxNumLayersForScreenSharing = 2;
+const float kMaxScreenSharingLayerFramerateFps[] = {5.0, 5.0};
const size_t kMaxScreenSharingLayerBitrateKbps[] = {200, 500};
} // namespace
std::vector<SpatialLayer> ConfigureSvcScreenSharing(size_t input_width,
size_t input_height,
+ float max_framerate_fps,
size_t num_spatial_layers) {
num_spatial_layers =
std::min(num_spatial_layers, kMaxNumLayersForScreenSharing);
@@ -37,6 +39,8 @@
SpatialLayer spatial_layer = {0};
spatial_layer.width = input_width;
spatial_layer.height = input_height;
+ spatial_layer.maxFramerate =
+ std::min(kMaxScreenSharingLayerFramerateFps[sl_idx], max_framerate_fps);
spatial_layer.numberOfTemporalLayers = 1;
spatial_layer.minBitrate = static_cast<int>(kMinVp9SvcBitrateKbps);
spatial_layer.maxBitrate =
@@ -50,6 +54,7 @@
std::vector<SpatialLayer> ConfigureSvcNormalVideo(size_t input_width,
size_t input_height,
+ float max_framerate_fps,
size_t num_spatial_layers,
size_t num_temporal_layers) {
std::vector<SpatialLayer> spatial_layers;
@@ -69,6 +74,7 @@
SpatialLayer spatial_layer = {0};
spatial_layer.width = input_width >> (num_spatial_layers - sl_idx - 1);
spatial_layer.height = input_height >> (num_spatial_layers - sl_idx - 1);
+ spatial_layer.maxFramerate = max_framerate_fps;
spatial_layer.numberOfTemporalLayers = num_temporal_layers;
// minBitrate and maxBitrate formulas were derived from
@@ -102,6 +108,7 @@
std::vector<SpatialLayer> GetSvcConfig(size_t input_width,
size_t input_height,
+ float max_framerate_fps,
size_t num_spatial_layers,
size_t num_temporal_layers,
bool is_screen_sharing) {
@@ -112,9 +119,9 @@
if (is_screen_sharing) {
return ConfigureSvcScreenSharing(input_width, input_height,
- num_spatial_layers);
+ max_framerate_fps, num_spatial_layers);
} else {
- return ConfigureSvcNormalVideo(input_width, input_height,
+ return ConfigureSvcNormalVideo(input_width, input_height, max_framerate_fps,
num_spatial_layers, num_temporal_layers);
}
}
diff --git a/modules/video_coding/codecs/vp9/svc_config.h b/modules/video_coding/codecs/vp9/svc_config.h
index c8561a4..f671ced 100644
--- a/modules/video_coding/codecs/vp9/svc_config.h
+++ b/modules/video_coding/codecs/vp9/svc_config.h
@@ -18,6 +18,7 @@
std::vector<SpatialLayer> GetSvcConfig(size_t input_width,
size_t input_height,
+ float max_framerate_fps,
size_t num_spatial_layers,
size_t num_temporal_layers,
bool is_screen_sharing);
diff --git a/modules/video_coding/codecs/vp9/svc_config_unittest.cc b/modules/video_coding/codecs/vp9/svc_config_unittest.cc
index ebefdda..257c5df 100644
--- a/modules/video_coding/codecs/vp9/svc_config_unittest.cc
+++ b/modules/video_coding/codecs/vp9/svc_config_unittest.cc
@@ -23,7 +23,7 @@
std::vector<SpatialLayer> spatial_layers =
GetSvcConfig(kMinVp9SpatialLayerWidth << (num_spatial_layers - 1),
- kMinVp9SpatialLayerHeight << (num_spatial_layers - 1),
+ kMinVp9SpatialLayerHeight << (num_spatial_layers - 1), 30,
max_num_spatial_layers, 1, false);
EXPECT_EQ(spatial_layers.size(), num_spatial_layers);
@@ -33,7 +33,7 @@
const size_t num_spatial_layers = 3;
std::vector<SpatialLayer> spatial_layers =
GetSvcConfig(kMinVp9SpatialLayerWidth << (num_spatial_layers - 1),
- kMinVp9SpatialLayerHeight << (num_spatial_layers - 1),
+ kMinVp9SpatialLayerHeight << (num_spatial_layers - 1), 30,
num_spatial_layers, 1, false);
EXPECT_EQ(spatial_layers.size(), num_spatial_layers);
@@ -47,13 +47,14 @@
TEST(SvcConfig, ScreenSharing) {
std::vector<SpatialLayer> spatial_layers =
- GetSvcConfig(1920, 1080, 3, 3, true);
+ GetSvcConfig(1920, 1080, 30, 3, 3, true);
EXPECT_EQ(spatial_layers.size(), 2UL);
for (const SpatialLayer& layer : spatial_layers) {
EXPECT_EQ(layer.width, 1920);
EXPECT_EQ(layer.height, 1080);
+ EXPECT_EQ(layer.maxFramerate, 5);
EXPECT_EQ(layer.numberOfTemporalLayers, 1);
EXPECT_LE(layer.minBitrate, layer.maxBitrate);
EXPECT_LE(layer.minBitrate, layer.targetBitrate);
diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc
index 1b6aa10..048bf7d 100644
--- a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc
+++ b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc
@@ -29,7 +29,7 @@
: VideoCodecMode::kRealtimeVideo;
std::vector<SpatialLayer> spatial_layers =
- GetSvcConfig(width, height, num_spatial_layers, num_temporal_layers,
+ GetSvcConfig(width, height, 30, num_spatial_layers, num_temporal_layers,
is_screen_sharing);
RTC_CHECK_LE(spatial_layers.size(), kMaxSpatialLayers);
diff --git a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
index a6774a4..5b5db88 100644
--- a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
+++ b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
@@ -82,9 +82,9 @@
codec_settings_.VP9()->numberOfTemporalLayers = 1;
codec_settings_.VP9()->frameDroppingOn = false;
- std::vector<SpatialLayer> layers =
- GetSvcConfig(codec_settings_.width, codec_settings_.height,
- num_spatial_layers, 1, false);
+ std::vector<SpatialLayer> layers = GetSvcConfig(
+ codec_settings_.width, codec_settings_.height,
+ codec_settings_.maxFramerate, num_spatial_layers, 1, false);
for (size_t i = 0; i < layers.size(); ++i) {
codec_settings_.spatialLayers[i] = layers[i];
}
diff --git a/modules/video_coding/video_codec_initializer.cc b/modules/video_coding/video_codec_initializer.cc
index fe59afc..e7125ce 100644
--- a/modules/video_coding/video_codec_initializer.cc
+++ b/modules/video_coding/video_codec_initializer.cc
@@ -126,6 +126,7 @@
sim_stream->width = static_cast<uint16_t>(streams[i].width);
sim_stream->height = static_cast<uint16_t>(streams[i].height);
+ sim_stream->maxFramerate = streams[i].max_framerate;
sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000;
sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000;
sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000;
@@ -198,11 +199,11 @@
// Layering is set explicitly.
spatial_layers = config.spatial_layers;
} else {
- spatial_layers =
- GetSvcConfig(video_codec.width, video_codec.height,
- video_codec.VP9()->numberOfSpatialLayers,
- video_codec.VP9()->numberOfTemporalLayers,
- video_codec.mode == VideoCodecMode::kScreensharing);
+ spatial_layers = GetSvcConfig(
+ video_codec.width, video_codec.height, video_codec.maxFramerate,
+ video_codec.VP9()->numberOfSpatialLayers,
+ video_codec.VP9()->numberOfTemporalLayers,
+ video_codec.mode == VideoCodecMode::kScreensharing);
const bool no_spatial_layering = (spatial_layers.size() == 1);
if (no_spatial_layering) {
