Fix resolution normalization in EncoderStreamFactory Before this change the normalization ignored scale_resolution_down_by and assumed 1/2 scaling. Now it takes scale_resolution_down_by into account and normalizes resolution to be multiple of the maximum scale factor (if all specified scale factors are power of two numbers). Bug: webrtc:448376656, b/445664659 Change-Id: I846d3c98749efab5d9a754c0b7cf98dfdbe077ba Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/413500 Commit-Queue: Sergey Silkin <ssilkin@webrtc.org> Reviewed-by: Åsa Persson <asapersson@webrtc.org> Cr-Commit-Position: refs/heads/main@{#45810}
diff --git a/video/config/encoder_stream_factory.cc b/video/config/encoder_stream_factory.cc index 21eef9a..dc961d3 100644 --- a/video/config/encoder_stream_factory.cc +++ b/video/config/encoder_stream_factory.cc
@@ -41,20 +41,26 @@ namespace webrtc { namespace { +constexpr int kMinLayerSize = 16; - -const int kMinLayerSize = 16; - -int ScaleDownResolution(int resolution, - double scale_down_by, - int min_resolution) { - // Resolution is never scalied down to smaller than min_resolution. - // If the input resolution is already smaller than min_resolution, - // no scaling should be done at all. - if (resolution <= min_resolution) +Resolution ScaleResolutionDownBy(const Resolution& resolution, + double scale_down_by) { + if (resolution.width <= kMinLayerSize || resolution.height <= kMinLayerSize) { return resolution; - return std::max(static_cast<int>(resolution / scale_down_by + 0.5), - min_resolution); + } + int short_side = std::min(resolution.width / scale_down_by, + resolution.height / scale_down_by); + if (short_side < kMinLayerSize) { + // Adjust scale factor to make the short side equal to kMinLayerSize. + scale_down_by = + std::min(resolution.width / static_cast<double>(kMinLayerSize), + resolution.height / static_cast<double>(kMinLayerSize)); + } + + return { + .width = static_cast<int>(std::round(resolution.width / scale_down_by)), + .height = + static_cast<int>(std::round(resolution.height / scale_down_by))}; } bool PowerOfTwo(int value) { @@ -120,20 +126,57 @@ } } -// Round size to nearest simulcast-friendly size. -// Simulcast stream width and height must both be dividable by -// |2 ^ (simulcast_layers - 1)|. -int NormalizeSimulcastSize(const FieldTrialsView& field_trials, - int size, - size_t simulcast_layers) { - int base2_exponent = static_cast<int>(simulcast_layers) - 1; - const std::optional<int> experimental_base2_exponent = - NormalizeSimulcastSizeExperiment::GetBase2Exponent(field_trials); - if (experimental_base2_exponent && - (size > (1 << *experimental_base2_exponent))) { - base2_exponent = *experimental_base2_exponent; +Resolution NormalizeResolution(const Resolution& resolution, + const VideoEncoderConfig& encoder_config, + int max_num_layers, + const FieldTrialsView& trials) { + if (resolution.width < kMinLayerSize || resolution.height < kMinLayerSize) { + // Resolution is already too low. Avoid further adjustments. + return resolution; } - return ((size >> base2_exponent) << base2_exponent); + + std::optional<int> log2_scale_factor = + NormalizeSimulcastSizeExperiment::GetBase2Exponent(trials); + if (log2_scale_factor && (resolution.width < (1 << *log2_scale_factor) || + resolution.height < (1 << *log2_scale_factor))) { + // The experimental scale factor is too large for the given input + // resolution. Fall back to the default logic. + log2_scale_factor = std::nullopt; + } + + if (!log2_scale_factor) { + // If `scale_resolution_down_by` is not set, default to 1/2 scaling. If + // `scale_resolution_down_by` is set for one or more simulcast layers, + // normalize the resolution to be a multiple of the maximum scale factor if + // both of the following conditions are true: + // 1. All configured scale factors are power of two. + // 2. The maximum scale factor is greater than one. + // Otherwise, keep the resolution unchanged. + if (!encoder_config.HasScaleResolutionDownBy()) { + log2_scale_factor = max_num_layers - 1; + } else if (IsScaleFactorsPowerOfTwo(encoder_config)) { + auto smallest_layer = absl::c_max_element( + encoder_config.simulcast_layers, [](const auto& a, const auto& b) { + return a.scale_resolution_down_by < b.scale_resolution_down_by; + }); + + double max_scale_factor = smallest_layer->scale_resolution_down_by; + if (max_scale_factor > 1.0) { + log2_scale_factor = static_cast<int>(std::log2(max_scale_factor)); + } + } + } + + if (!log2_scale_factor || (resolution.width < (1 << *log2_scale_factor) || + resolution.height < (1 << *log2_scale_factor))) { + // Too large scale factor. + return resolution; + } + + return {.width = (resolution.width >> *log2_scale_factor) + << *log2_scale_factor, + .height = (resolution.height >> *log2_scale_factor) + << *log2_scale_factor}; } // Override bitrate limits and other stream settings with values from @@ -160,6 +203,7 @@ layer.active = overrides.active; layer.scalability_mode = overrides.scalability_mode; layer.scale_resolution_down_to = overrides.scale_resolution_down_to; + layer.scale_resolution_down_by = overrides.scale_resolution_down_by; // Update with configured num temporal layers if supported by codec. if (overrides.num_temporal_layers > 0 && temporal_layers_supported) { layer.num_temporal_layers = *overrides.num_temporal_layers; @@ -357,9 +401,9 @@ : kDefaultVideoMaxFramerate; VideoStream layer; - layer.width = width; - layer.height = height; layer.max_framerate = max_framerate; + layer.scale_resolution_down_by = + encoder_config.simulcast_layers[0].scale_resolution_down_by; layer.scale_resolution_down_to = encoder_config.simulcast_layers[0].scale_resolution_down_to; // Note: VP9 seems to have be sending if any layer is active, @@ -375,14 +419,14 @@ layer.width = res.width; layer.height = res.height; } else if (encoder_config.simulcast_layers[0].scale_resolution_down_by > 1.) { - layer.width = ScaleDownResolution( - layer.width, - encoder_config.simulcast_layers[0].scale_resolution_down_by, - kMinLayerSize); - layer.height = ScaleDownResolution( - layer.height, - encoder_config.simulcast_layers[0].scale_resolution_down_by, - kMinLayerSize); + Resolution res = ScaleResolutionDownBy( + {.width = width, .height = height}, + encoder_config.simulcast_layers[0].scale_resolution_down_by); + layer.width = res.width; + layer.height = res.height; + } else { + layer.width = width; + layer.height = height; } if (encoder_config.codec_type == VideoCodecType::kVideoCodecVP9) { @@ -534,9 +578,11 @@ resolutions.push_back({.width = width, .height = height}); } } else { + const bool has_scale_resolution_down_to = + encoder_config.HasScaleResolutionDownTo(); size_t min_num_layers = FindRequiredActiveLayers(encoder_config); size_t max_num_layers = - !encoder_config.HasScaleResolutionDownTo() + !has_scale_resolution_down_to ? LimitSimulcastLayerCount( min_num_layers, encoder_config.number_of_streams, width, height, trials, encoder_config.codec_type) @@ -550,8 +596,7 @@ // adaptation (bitrate allocator disabling layers rather than downscaling) // and means we don't have to break power of two optimization paths (i.e. // S-modes based simulcast). Note that the lowest layer is never disabled. - if (encoder_config.HasScaleResolutionDownTo() && - restrictions_.has_value() && + if (has_scale_resolution_down_to && restrictions_.has_value() && restrictions_->max_pixels_per_frame().has_value()) { int max_pixels = dchecked_cast<int>(restrictions_->max_pixels_per_frame().value()); @@ -578,47 +623,31 @@ max_num_layers = restricted_num_layers.value_or(max_num_layers); } - const bool has_scale_resolution_down_by = absl::c_any_of( - encoder_config.simulcast_layers, [](const VideoStream& layer) { - return layer.scale_resolution_down_by != -1.; - }); + Resolution norm_resolution = + NormalizeResolution({.width = width, .height = height}, encoder_config, + max_num_layers, trials); - bool default_scale_factors_used = true; - if (has_scale_resolution_down_by) { - default_scale_factors_used = IsScaleFactorsPowerOfTwo(encoder_config); - } - - const bool norm_size_configured = - NormalizeSimulcastSizeExperiment::GetBase2Exponent(trials).has_value(); - const int normalized_width = - (default_scale_factors_used || norm_size_configured) && - (width >= kMinLayerSize) - ? NormalizeSimulcastSize(trials, width, max_num_layers) - : width; - const int normalized_height = - (default_scale_factors_used || norm_size_configured) && - (height >= kMinLayerSize) - ? NormalizeSimulcastSize(trials, height, max_num_layers) - : height; + const bool has_scale_resolution_down_by = + encoder_config.HasScaleResolutionDownBy(); resolutions.resize(max_num_layers); for (size_t i = 0; i < max_num_layers; i++) { if (encoder_config.simulcast_layers[i] .scale_resolution_down_to.has_value()) { resolutions[i] = GetLayerResolutionFromScaleResolutionDownTo( - normalized_width, normalized_height, + norm_resolution.width, norm_resolution.height, *encoder_config.simulcast_layers[i].scale_resolution_down_to); } else if (has_scale_resolution_down_by) { const double scale_resolution_down_by = std::max( encoder_config.simulcast_layers[i].scale_resolution_down_by, 1.0); - resolutions[i].width = ScaleDownResolution( - normalized_width, scale_resolution_down_by, kMinLayerSize); - resolutions[i].height = ScaleDownResolution( - normalized_height, scale_resolution_down_by, kMinLayerSize); + resolutions[i] = + ScaleResolutionDownBy(norm_resolution, scale_resolution_down_by); } else { // Resolutions with default 1/2 scale factor, from low to high. - resolutions[i].width = normalized_width >> (max_num_layers - i - 1); - resolutions[i].height = normalized_height >> (max_num_layers - i - 1); + resolutions[i].width = + norm_resolution.width >> (max_num_layers - i - 1); + resolutions[i].height = + norm_resolution.height >> (max_num_layers - i - 1); } } }
diff --git a/video/config/encoder_stream_factory_unittest.cc b/video/config/encoder_stream_factory_unittest.cc index 1ca111e..527480a 100644 --- a/video/config/encoder_stream_factory_unittest.cc +++ b/video/config/encoder_stream_factory_unittest.cc
@@ -102,9 +102,8 @@ field_trials, {.width = 1280, .height = 720}, encoder_config); EXPECT_EQ(streams[0].scale_resolution_down_to, (Resolution{.width = 640, .height = 360})); - EXPECT_EQ(GetStreamResolutions(streams), (std::vector<Resolution>{ - {.width = 640, .height = 360}, - })); + EXPECT_THAT(GetStreamResolutions(streams), + ElementsAre(Resolution{.width = 640, .height = 360})); } TEST(EncoderStreamFactory, SinglecastScaleResolutionDownToWithAdaptation) { @@ -123,9 +122,8 @@ encoder_config, restrictions); EXPECT_EQ(streams[0].scale_resolution_down_to, (Resolution{.width = 640, .height = 360})); - EXPECT_EQ(GetStreamResolutions(streams), (std::vector<Resolution>{ - {.width = 320, .height = 180}, - })); + EXPECT_THAT(GetStreamResolutions(streams), + ElementsAre(Resolution{.width = 320, .height = 180})); } TEST(EncoderStreamFactory, SimulcastScaleResolutionDownToUnrestricted) { @@ -141,11 +139,10 @@ .height = 720}; auto streams = CreateEncoderStreams( field_trials, {.width = 1280, .height = 720}, encoder_config); - std::vector<Resolution> stream_resolutions = GetStreamResolutions(streams); - ASSERT_THAT(stream_resolutions, SizeIs(3)); - EXPECT_EQ(stream_resolutions[0], (Resolution{.width = 320, .height = 180})); - EXPECT_EQ(stream_resolutions[1], (Resolution{.width = 640, .height = 360})); - EXPECT_EQ(stream_resolutions[2], (Resolution{.width = 1280, .height = 720})); + EXPECT_THAT(GetStreamResolutions(streams), + ElementsAre(Resolution{.width = 320, .height = 180}, + Resolution{.width = 640, .height = 360}, + Resolution{.width = 1280, .height = 720})); } TEST(EncoderStreamFactory, SimulcastScaleResolutionDownToWith360pRestriction) { @@ -166,11 +163,10 @@ auto streams = CreateEncoderStreams(field_trials, {.width = 1280, .height = 720}, encoder_config, restrictions); - std::vector<Resolution> stream_resolutions = GetStreamResolutions(streams); // 720p layer is dropped due to 360p restrictions. - ASSERT_THAT(stream_resolutions, SizeIs(2)); - EXPECT_EQ(stream_resolutions[0], (Resolution{.width = 320, .height = 180})); - EXPECT_EQ(stream_resolutions[1], (Resolution{.width = 640, .height = 360})); + EXPECT_THAT(GetStreamResolutions(streams), + ElementsAre(Resolution{.width = 320, .height = 180}, + Resolution{.width = 640, .height = 360})); } TEST(EncoderStreamFactory, SimulcastScaleResolutionDownToWith90pRestriction) { @@ -191,11 +187,10 @@ auto streams = CreateEncoderStreams(field_trials, {.width = 1280, .height = 720}, encoder_config, restrictions); - std::vector<Resolution> stream_resolutions = GetStreamResolutions(streams); - ASSERT_THAT(stream_resolutions, SizeIs(1)); // 90p restriction means all but the first layer (180p) is dropped. The one // and only layer is downsized to 90p. - EXPECT_EQ(stream_resolutions[0], (Resolution{.width = 160, .height = 90})); + EXPECT_THAT(GetStreamResolutions(streams), + ElementsAre(Resolution{.width = 160, .height = 90})); } TEST(EncoderStreamFactory, @@ -218,14 +213,156 @@ auto streams = CreateEncoderStreams(field_trials, {.width = 1280, .height = 720}, encoder_config, restrictions); - std::vector<Resolution> stream_resolutions = GetStreamResolutions(streams); // The layer dropping that is performed for lower-to-higher ordered simulcast // streams is not applicable when higher-to-lower order is used. In this case // the 360p restriction is applied to all layers. - ASSERT_THAT(stream_resolutions, SizeIs(3)); - EXPECT_EQ(stream_resolutions[0], (Resolution{.width = 640, .height = 360})); - EXPECT_EQ(stream_resolutions[1], (Resolution{.width = 640, .height = 360})); - EXPECT_EQ(stream_resolutions[2], (Resolution{.width = 320, .height = 180})); + EXPECT_THAT(GetStreamResolutions(streams), + ElementsAre(Resolution{.width = 640, .height = 360}, + Resolution{.width = 640, .height = 360}, + Resolution{.width = 320, .height = 180})); +} + +TEST(EncoderStreamFactory, + SinglecastScaleResolutionDownToTakesPrecedenceOverBy) { + FieldTrials field_trials = CreateTestFieldTrials(); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 1; + encoder_config.simulcast_layers.resize(1); + encoder_config.simulcast_layers[0].scale_resolution_down_to = {.width = 960, + .height = 650}; + encoder_config.simulcast_layers[0].scale_resolution_down_by = 2; + auto streams = CreateEncoderStreams( + field_trials, {.width = 1280, .height = 720}, encoder_config); + EXPECT_THAT(GetStreamResolutions(streams), + ElementsAre(Resolution{.width = 960, .height = 540})); +} + +TEST(EncoderStreamFactory, ScaleResolutionDownBy) { + FieldTrials field_trials = CreateTestFieldTrials(); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 1; + encoder_config.simulcast_layers.resize(1); + encoder_config.simulcast_layers[0].scale_resolution_down_by = 2; + auto streams = CreateEncoderStreams( + field_trials, {.width = 1280, .height = 720}, encoder_config); + EXPECT_EQ(streams[0].scale_resolution_down_by, 2); + EXPECT_EQ(GetStreamResolutions(streams), (std::vector<Resolution>{ + {.width = 640, .height = 360}, + })); +} + +TEST(EncoderStreamFactory, ScaleResolutionDownByWithZeroScaleFactor) { + FieldTrials field_trials = CreateTestFieldTrials(); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 1; + encoder_config.simulcast_layers.resize(1); + encoder_config.simulcast_layers[0].scale_resolution_down_by = 0; + auto streams = CreateEncoderStreams( + field_trials, {.width = 1280, .height = 720}, encoder_config); + EXPECT_THAT(GetStreamResolutions(streams), + ElementsAre(Resolution{.width = 1280, .height = 720})); +} + +TEST(EncoderStreamFactory, ScaleResolutionDownByWithNegativeScaleFactor) { + FieldTrials field_trials = CreateTestFieldTrials(); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 1; + encoder_config.simulcast_layers.resize(1); + encoder_config.simulcast_layers[0].scale_resolution_down_by = -2; + auto streams = CreateEncoderStreams( + field_trials, {.width = 1280, .height = 720}, encoder_config); + EXPECT_THAT(GetStreamResolutions(streams), + ElementsAre(Resolution{.width = 1280, .height = 720})); +} + +TEST(EncoderStreamFactory, ScaleResolutionDownByWithPartiallySetScaleFactors) { + FieldTrials field_trials = CreateTestFieldTrials(); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 2; + encoder_config.simulcast_layers.resize(2); + encoder_config.simulcast_layers[0].scale_resolution_down_by = 2; + encoder_config.simulcast_layers[1].scale_resolution_down_by = -1; + auto streams = CreateEncoderStreams( + field_trials, {.width = 1280, .height = 720}, encoder_config); + EXPECT_THAT(GetStreamResolutions(streams), + ElementsAre(Resolution{.width = 640, .height = 360}, + Resolution{.width = 1280, .height = 720})); +} + +TEST(EncoderStreamFactory, ScaleDownByWithUnalignResolutionDyadicScaling) { + FieldTrials field_trials = CreateTestFieldTrials(); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 2; + encoder_config.simulcast_layers.resize(2); + encoder_config.simulcast_layers[0].scale_resolution_down_by = 1; + encoder_config.simulcast_layers[1].scale_resolution_down_by = 2; + auto streams = CreateEncoderStreams( + field_trials, {.width = 513, .height = 1025}, encoder_config); + EXPECT_THAT(GetStreamResolutions(streams), + ElementsAre(Resolution{.width = 512, .height = 1024}, + Resolution{.width = 256, .height = 512})); +} + +TEST(EncoderStreamFactory, ScaleDownByWithUnalignResolutionNonDyadicScaling) { + // Keeps resolution unchanged if any of `scale_resolution_down_by` is not a + // power of two number. + FieldTrials field_trials = CreateTestFieldTrials(); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 2; + encoder_config.simulcast_layers.resize(2); + encoder_config.simulcast_layers[0].scale_resolution_down_by = 1; + encoder_config.simulcast_layers[1].scale_resolution_down_by = 3; + auto streams = CreateEncoderStreams( + field_trials, {.width = 513, .height = 1025}, encoder_config); + EXPECT_THAT(GetStreamResolutions(streams), + ElementsAre(Resolution{.width = 513, .height = 1025}, + Resolution{.width = 171, .height = 342})); +} + +TEST(EncoderStreamFactory, ScaleDownByWithUnalignedResolutionLargeScaleFactor) { + FieldTrials field_trials = CreateTestFieldTrials(); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 1; + encoder_config.simulcast_layers.resize(1); + encoder_config.simulcast_layers[0].scale_resolution_down_by = 1024; + auto streams = CreateEncoderStreams( + field_trials, {.width = 513, .height = 1025}, encoder_config); + EXPECT_THAT(GetStreamResolutions(streams), + ElementsAre(Resolution{.width = 16, .height = 32})); +} + +TEST(EncoderStreamFactory, ScaleDownByWithUnalignedResolutionWithoutScaling) { + FieldTrials field_trials = CreateTestFieldTrials(); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 2; + encoder_config.simulcast_layers.resize(2); + encoder_config.simulcast_layers[0].scale_resolution_down_by = 1; + encoder_config.simulcast_layers[1].scale_resolution_down_by = 1; + auto streams = CreateEncoderStreams( + field_trials, {.width = 513, .height = 1025}, encoder_config); + EXPECT_THAT(GetStreamResolutions(streams), + ElementsAre(Resolution{.width = 513, .height = 1025}, + Resolution{.width = 513, .height = 1025})); +} + +TEST(EncoderStreamFactory, ScaleDownByNormalizesResolutionUpToMaxScaleFactor) { + // Without `scale_resolution_down_by` resolution normalization assumes 1/2 + // scaling. When three streams are requested, resolution is normalized to be + // multiple of 8. With `scale_resolution_down_by` resolution is normalized + // to be multiple of the maximum scale factor. + FieldTrials field_trials = CreateTestFieldTrials(); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 3; + encoder_config.simulcast_layers.resize(3); + encoder_config.simulcast_layers[0].scale_resolution_down_by = 1; + encoder_config.simulcast_layers[1].scale_resolution_down_by = 1; + encoder_config.simulcast_layers[2].scale_resolution_down_by = 2; + auto streams = CreateEncoderStreams( + field_trials, {.width = 511, .height = 1027}, encoder_config); + EXPECT_THAT(GetStreamResolutions(streams), + ElementsAre(Resolution{.width = 510, .height = 1026}, + Resolution{.width = 510, .height = 1026}, + Resolution{.width = 255, .height = 513})); } TEST(EncoderStreamFactory, BitratePriority) {
diff --git a/video/config/video_encoder_config.cc b/video/config/video_encoder_config.cc index b08afcf..dc0c581 100644 --- a/video/config/video_encoder_config.cc +++ b/video/config/video_encoder_config.cc
@@ -108,6 +108,15 @@ return false; } +bool VideoEncoderConfig::HasScaleResolutionDownBy() const { + for (const VideoStream& simulcast_layer : simulcast_layers) { + if (simulcast_layer.scale_resolution_down_by >= 1.0) { + return true; + } + } + return false; +} + VideoEncoderConfig::VideoEncoderConfig(const VideoEncoderConfig&) = default; SdpVideoFormat VideoEncoderConfig::GetSimulcastVideoFormat(
diff --git a/video/config/video_encoder_config.h b/video/config/video_encoder_config.h index e647cd6..cbb3bf1 100644 --- a/video/config/video_encoder_config.h +++ b/video/config/video_encoder_config.h
@@ -175,6 +175,8 @@ bool HasScaleResolutionDownTo() const; + bool HasScaleResolutionDownBy() const; + SdpVideoFormat GetSimulcastVideoFormat(size_t stream_index) const; // TODO(bugs.webrtc.org/6883): Consolidate on one of these.