Keep SVC spatial layer count and scaling factors across partial TUs Previously, GetSvcParams derived number_spatial_layers and scaling factors solely from the current temporal unit's frame_settings. When a temporal unit omitted the top spatial layer(s), number_spatial_layers shrunk for that TU and grew again on the next full TU, causing libaom to reset SVC state and force a keyframe. Track the high-water mark of spatial layers and scaling factors across Encode() calls since InitEncode() so partial TUs preserve the full SVC configuration. Bug: webrtc:496266459 Change-Id: I86982578de18c3915e01bb70e8753a2f57810ba6 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/507842 Commit-Queue: Erik Språng <sprang@webrtc.org> Reviewed-by: Sergey Silkin <ssilkin@webrtc.org> Cr-Commit-Position: refs/heads/main@{#48820}
diff --git a/api/video_codecs/test/video_encoder_functional_unittest.cc b/api/video_codecs/test/video_encoder_functional_unittest.cc index 491baf7..8978051 100644 --- a/api/video_codecs/test/video_encoder_functional_unittest.cc +++ b/api/video_codecs/test/video_encoder_functional_unittest.cc
@@ -1861,6 +1861,89 @@ EXPECT_THAT(Psnr(tu2_frame, f_tu2_s2), Gt(40.0)); } +// Verifies that omitting the top spatial layer in a temporal unit does not +// force a keyframe on the remaining or subsequent layers. +TEST_P(VideoEncoderFunctionalTest, SkipTopLayer) { + Capabilities capabilities = factory_->GetEncoderCapabilities(); + int max_spatial_layers = + capabilities.prediction_constraints().max_spatial_layers(); + if (max_spatial_layers < 2) { + GTEST_SKIP() << "Encoder doesn't support at least 2 spatial layers."; + } + + std::vector<Rational> factors = + FindSpatialLayerScalingFactors(capabilities, 2); + if (factors.empty()) { + GTEST_SKIP() << "Could not find 2 valid scaling factors."; + } + + int alignment = capabilities.input_constraints().pixel_alignment(); + std::vector<Resolution> resolutions = + GetSpatialLayerResolutions(kDefaultResolution, factors, alignment); + + TestConfig config = CreateTestConfig(capabilities); + std::unique_ptr<VideoEncoderInterface> enc = + factory_->CreateEncoder(config.static_settings, {}); + std::unique_ptr<test::FrameReader> frame_reader = CreateFrameReader(); + + TestDecoder dec(env_, decoder_factory_.get(), factory_->CodecName()); + if (!dec.IsSupported()) { + GTEST_SKIP() << "No matching decoder found."; + } + + EncOut tu0_s0, tu0_s1; + enc->Encode( + frame_reader->PullFrame(), TemporalUnitSettings(Timestamp::Millis(0)), + ToVec({BuildSettings( + std::move( + Fb().Res(resolutions[0]).S(0).Upd(0).Key().Out(tu0_s0)), + config.rate_options), + BuildSettings( + std::move( + Fb().Res(resolutions[1]).S(1).Ref({0}).Upd(1).Out(tu0_s1)), + config.rate_options)})); + + EncOut tu1_s0; + enc->Encode( + frame_reader->PullFrame(), TemporalUnitSettings(Timestamp::Millis(100)), + ToVec({BuildSettings( + std::move(Fb().Res(resolutions[0]).S(0).Ref({0}).Upd(0).Out(tu1_s0)), + config.rate_options)})); + ASSERT_THAT(tu1_s0, HasBitstreamAndMetaData()); + EXPECT_EQ(std::get<EncodedData>(tu1_s0.res).frame_type, + FrameType::kDeltaFrame); + + EncOut tu2_s0, tu2_s1; + scoped_refptr<I420Buffer> tu2_frame = frame_reader->PullFrame(); + enc->Encode( + tu2_frame, TemporalUnitSettings(Timestamp::Millis(200)), + ToVec({BuildSettings( + std::move( + Fb().Res(resolutions[0]).S(0).Ref({0}).Upd(0).Out(tu2_s0)), + config.rate_options), + BuildSettings(std::move(Fb().Res(resolutions[1]) + .S(1) + .Ref({0, 1}) + .Upd(1) + .Out(tu2_s1)), + config.rate_options)})); + ASSERT_THAT(tu2_s0, HasBitstreamAndMetaData()); + ASSERT_THAT(tu2_s1, HasBitstreamAndMetaData()); + EXPECT_EQ(std::get<EncodedData>(tu2_s0.res).frame_type, + FrameType::kDeltaFrame); + EXPECT_EQ(std::get<EncodedData>(tu2_s1.res).frame_type, + FrameType::kDeltaFrame); + + EXPECT_EQ(GetResolution(dec.Decode(tu0_s0.bitstream)), resolutions[0]); + EXPECT_EQ(GetResolution(dec.Decode(tu0_s1.bitstream)), resolutions[1]); + EXPECT_EQ(GetResolution(dec.Decode(tu1_s0.bitstream)), resolutions[0]); + EXPECT_EQ(GetResolution(dec.Decode(tu2_s0.bitstream)), resolutions[0]); + + VideoFrame f_tu2_s1 = dec.Decode(tu2_s1.bitstream); + EXPECT_EQ(GetResolution(f_tu2_s1), resolutions[1]); + EXPECT_THAT(Psnr(tu2_frame, f_tu2_s1), Gt(40.0)); +} + // Verifies encoding and decoding of a StartFrame (independent frame without // clearing all references). TEST_P(VideoEncoderFunctionalTest, EncodesAndDecodesStartFrame) {
diff --git a/modules/video_coding/codecs/av1/libaom_av1_encoder_v2.cc b/modules/video_coding/codecs/av1/libaom_av1_encoder_v2.cc index 1dad1a4..36f60d2 100644 --- a/modules/video_coding/codecs/av1/libaom_av1_encoder_v2.cc +++ b/modules/video_coding/codecs/av1/libaom_av1_encoder_v2.cc
@@ -546,7 +546,7 @@ const VideoFrameBuffer& frame_buffer, const std::vector<FrameEncodeSettings>& frame_settings) const { aom_svc_params_t svc_params = {}; - svc_params.number_spatial_layers = frame_settings.back().spatial_id() + 1; + svc_params.number_spatial_layers = num_spatial_layers_; // Unlike the spatial layers, the temporal layers are always declared in // full: changing the count would force a keyframe, see // `kMaxAdvertisedTemporalLayers`. @@ -587,10 +587,11 @@ }; // If the scaling factor is left at zero for unused layers a division by zero - // will happen inside libaom, default all layers to one. + // will happen inside libaom, default all layers to their last scaling factor. for (int sid = 0; sid < svc_params.number_spatial_layers; ++sid) { - scaling_factor_num_view[sid] = 1; - scaling_factor_den_view[sid] = 1; + scaling_factor_num_view[sid] = scaling_factor_by_spatial_id_[sid].numerator; + scaling_factor_den_view[sid] = + scaling_factor_by_spatial_id_[sid].denominator; } for (const FrameEncodeSettings& settings : frame_settings) { @@ -734,6 +735,8 @@ effort_level_by_spatial_id_.fill(std::nullopt); applied_cfg_.reset(); applied_svc_params_.reset(); + num_spatial_layers_ = 0; + scaling_factor_by_spatial_id_.fill({.numerator = 1, .denominator = 1}); if (aom_codec_err_t ret = aom_codec_enc_config_default( aom_codec_av1_cx(), &cfg_, AOM_USAGE_REALTIME); @@ -923,6 +926,15 @@ } applied_cfg_ = cfg_; } + num_spatial_layers_ = + std::max(num_spatial_layers_, frame_settings.back().spatial_id() + 1); + for (const FrameEncodeSettings& settings : frame_settings) { + const int gcd = + std::gcd(settings.resolution().width, frame_buffer->width()); + scaling_factor_by_spatial_id_[settings.spatial_id()] = { + .numerator = settings.resolution().width / gcd, + .denominator = frame_buffer->width() / gcd}; + } aom_svc_params_t svc_params = GetSvcParams(*frame_buffer, frame_settings); if (NeedsApply(applied_svc_params_, svc_params)) { applied_svc_params_.reset();
diff --git a/modules/video_coding/codecs/av1/libaom_av1_encoder_v2.h b/modules/video_coding/codecs/av1/libaom_av1_encoder_v2.h index e44d878..d2a1f6b 100644 --- a/modules/video_coding/codecs/av1/libaom_av1_encoder_v2.h +++ b/modules/video_coding/codecs/av1/libaom_av1_encoder_v2.h
@@ -26,6 +26,7 @@ #include "api/video_codecs/video_encoder_interface.h" #include "modules/video_coding/utility/cbr_layer_rate_tracker.h" #include "modules/video_coding/utility/reference_buffer_tracker.h" +#include "rtc_base/numerics/rational.h" #include "third_party/libaom/source/libaom/aom/aom_codec.h" #include "third_party/libaom/source/libaom/aom/aom_encoder.h" #include "third_party/libaom/source/libaom/aom/aom_image.h" @@ -68,6 +69,18 @@ std::optional<ContentHint> content_type_; std::array<std::optional<int>, kMaxSpatialLayers> effort_level_by_spatial_id_; + // Spatial layers declared to libaom: the most seen in a temporal unit since + // `InitEncode`. Changing `number_spatial_layers` updates the sequence header + // operating points and makes libaom force a keyframe. We keep the state for + // layers we have seen so far in order to minimize resets. + // TODO(bugs.webrtc.org/496266459): The RTP profile for AV1 specified that an + // implementation may specify a single operating point with a value of 0xFFF, + // indicating there are no operating points signalled in the bitstream. + // Consider implementing that behavior in libaom to avoid this workaround. + int num_spatial_layers_ = 0; + // Last scaling factor per spatial layer, kept for the temporal units without + // a frame in the layer. + std::array<Rational, kMaxSpatialLayers> scaling_factor_by_spatial_id_; int max_number_of_threads_ = 0; std::array<std::optional<Resolution>, kNumBuffers> last_resolution_in_buffer_; ReferenceBufferTracker reference_buffer_tracker_{kNumBuffers};