Reland "Add option to disable quality scaling for AV1."
This reverts commit 83102d39077f82f2d4539c160c659dcf789a5fdb.
Reason for revert: reland with fix
Original change's description:
> Revert "Add option to disable quality scaling for AV1."
>
> This reverts commit 446dbc66fde7e9d5e684d3f71e357c2076a91740.
>
> Reason for revert: downstream break
>
> Original change's description:
> > Add option to disable quality scaling for AV1.
> >
> > The main goal of this change is to disable the quality scaler when multiple spatial layers are used.
> >
> > Bug: b/295129711
> > Change-Id: I25e0b7440a8c2adee3e97720a1e0ee5e0a914334
> > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/319181
> > Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
> > Reviewed-by: Erik Språng <sprang@webrtc.org>
> > Commit-Queue: Philip Eliasson <philipel@webrtc.org>
> > Cr-Commit-Position: refs/heads/main@{#40709}
>
> Bug: b/295129711
> Change-Id: Iaeb13951d1b839bc0426120436035843bb3ee98f
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/320081
> Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
> Bot-Commit: rubber-stamper@appspot.gserviceaccount.com <rubber-stamper@appspot.gserviceaccount.com>
> Commit-Queue: Philip Eliasson <philipel@webrtc.org>
> Owners-Override: Philip Eliasson <philipel@webrtc.org>
> Cr-Commit-Position: refs/heads/main@{#40742}
Bug: b/295129711
Change-Id: Iab4846c2cd6074f50a3ebe9551432d449243b5d7
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/320120
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#40743}
diff --git a/api/video_codecs/video_codec.cc b/api/video_codecs/video_codec.cc
index c6122d3..dfd2e21 100644
--- a/api/video_codecs/video_codec.cc
+++ b/api/video_codecs/video_codec.cc
@@ -102,6 +102,16 @@
return codec_specific_.H264;
}
+VideoCodecAV1* VideoCodec::AV1() {
+ RTC_DCHECK_EQ(codecType, kVideoCodecAV1);
+ return &codec_specific_.AV1;
+}
+
+const VideoCodecAV1& VideoCodec::AV1() const {
+ RTC_DCHECK_EQ(codecType, kVideoCodecAV1);
+ return codec_specific_.AV1;
+}
+
const char* CodecTypeToPayloadString(VideoCodecType type) {
switch (type) {
case kVideoCodecVP8:
diff --git a/api/video_codecs/video_codec.h b/api/video_codecs/video_codec.h
index 496cfb5..a596af1 100644
--- a/api/video_codecs/video_codec.h
+++ b/api/video_codecs/video_codec.h
@@ -97,6 +97,16 @@
uint8_t numberOfTemporalLayers;
};
+struct VideoCodecAV1 {
+ bool operator==(const VideoCodecAV1& other) const {
+ return automatic_resize_on == other.automatic_resize_on;
+ }
+ bool operator!=(const VideoCodecAV1& other) const {
+ return !(*this == other);
+ }
+ bool automatic_resize_on;
+};
+
// Translates from name of codec to codec type and vice versa.
RTC_EXPORT const char* CodecTypeToPayloadString(VideoCodecType type);
RTC_EXPORT VideoCodecType PayloadStringToCodecType(const std::string& name);
@@ -105,6 +115,7 @@
VideoCodecVP8 VP8;
VideoCodecVP9 VP9;
VideoCodecH264 H264;
+ VideoCodecAV1 AV1;
};
enum class VideoCodecMode { kRealtimeVideo, kScreensharing };
@@ -193,6 +204,8 @@
const VideoCodecVP9& VP9() const;
VideoCodecH264* H264();
const VideoCodecH264& H264() const;
+ VideoCodecAV1* AV1();
+ const VideoCodecAV1& AV1() const;
private:
// TODO(hta): Consider replacing the union with a pointer type.
diff --git a/media/engine/fake_webrtc_call.cc b/media/engine/fake_webrtc_call.cc
index 846be4d..16e7169 100644
--- a/media/engine/fake_webrtc_call.cc
+++ b/media/engine/fake_webrtc_call.cc
@@ -211,6 +211,16 @@
return true;
}
+bool FakeVideoSendStream::GetAv1Settings(
+ webrtc::VideoCodecAV1* settings) const {
+ if (!codec_settings_set_) {
+ return false;
+ }
+
+ *settings = codec_specific_settings_.av1;
+ return true;
+}
+
int FakeVideoSendStream::GetNumberOfSwappedFrames() const {
return num_swapped_frames_;
}
@@ -315,6 +325,9 @@
} else if (config_.rtp.payload_name == "H264") {
codec_specific_settings_.h264.numberOfTemporalLayers =
num_temporal_layers;
+ } else if (config_.rtp.payload_name == "AV1") {
+ config.encoder_specific_settings->FillVideoCodecAv1(
+ &codec_specific_settings_.av1);
} else {
ADD_FAILURE() << "Unsupported encoder payload: "
<< config_.rtp.payload_name;
diff --git a/media/engine/fake_webrtc_call.h b/media/engine/fake_webrtc_call.h
index 8d9ecb6..3dd6bdf 100644
--- a/media/engine/fake_webrtc_call.h
+++ b/media/engine/fake_webrtc_call.h
@@ -169,6 +169,7 @@
bool GetVp8Settings(webrtc::VideoCodecVP8* settings) const;
bool GetVp9Settings(webrtc::VideoCodecVP9* settings) const;
bool GetH264Settings(webrtc::VideoCodecH264* settings) const;
+ bool GetAv1Settings(webrtc::VideoCodecAV1* settings) const;
int GetNumberOfSwappedFrames() const;
int GetLastWidth() const;
@@ -226,6 +227,7 @@
webrtc::VideoCodecVP8 vp8;
webrtc::VideoCodecVP9 vp9;
webrtc::VideoCodecH264 h264;
+ webrtc::VideoCodecAV1 av1;
} codec_specific_settings_;
bool resolution_scaling_enabled_;
bool framerate_scaling_enabled_;
diff --git a/media/engine/fake_webrtc_video_engine.cc b/media/engine/fake_webrtc_video_engine.cc
index adbaf6c..cf40247 100644
--- a/media/engine/fake_webrtc_video_engine.cc
+++ b/media/engine/fake_webrtc_video_engine.cc
@@ -281,11 +281,13 @@
}
void FakeWebRtcVideoEncoderFactory::AddSupportedVideoCodecType(
- const std::string& name) {
+ const std::string& name,
+ const std::vector<webrtc::ScalabilityMode>& scalability_modes) {
// This is to match the default H264 params of cricket::VideoCodec.
cricket::VideoCodec video_codec = cricket::CreateVideoCodec(name);
- formats_.push_back(
- webrtc::SdpVideoFormat(video_codec.name, video_codec.params));
+ formats_.push_back(webrtc::SdpVideoFormat(
+ video_codec.name, video_codec.params,
+ {scalability_modes.begin(), scalability_modes.end()}));
}
int FakeWebRtcVideoEncoderFactory::GetNumCreatedEncoders() {
diff --git a/media/engine/fake_webrtc_video_engine.h b/media/engine/fake_webrtc_video_engine.h
index 87d107a..3db4225 100644
--- a/media/engine/fake_webrtc_video_engine.h
+++ b/media/engine/fake_webrtc_video_engine.h
@@ -124,7 +124,9 @@
void EncoderDestroyed(FakeWebRtcVideoEncoder* encoder);
void set_encoders_have_internal_sources(bool internal_source);
void AddSupportedVideoCodec(const webrtc::SdpVideoFormat& format);
- void AddSupportedVideoCodecType(const std::string& name);
+ void AddSupportedVideoCodecType(
+ const std::string& name,
+ const std::vector<webrtc::ScalabilityMode>& scalability_modes = {});
int GetNumCreatedEncoders();
const std::vector<FakeWebRtcVideoEncoder*> encoders();
diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc
index e8bffc2..70bcb07 100644
--- a/media/engine/webrtc_video_engine.cc
+++ b/media/engine/webrtc_video_engine.cc
@@ -963,6 +963,15 @@
return rtc::make_ref_counted<
webrtc::VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
}
+ if (absl::EqualsIgnoreCase(codec.name, kAv1CodecName)) {
+ webrtc::VideoCodecAV1 av1_settings = {.automatic_resize_on =
+ automatic_resize};
+ if (NumSpatialLayersFromEncoding(rtp_parameters_, /*idx=*/0) > 1) {
+ av1_settings.automatic_resize_on = false;
+ }
+ return rtc::make_ref_counted<
+ webrtc::VideoEncoderConfig::Av1EncoderSpecificSettings>(av1_settings);
+ }
return nullptr;
}
std::vector<VideoCodecSettings> WebRtcVideoSendChannel::SelectSendVideoCodecs(
diff --git a/media/engine/webrtc_video_engine_unittest.cc b/media/engine/webrtc_video_engine_unittest.cc
index 4262aa0..2c44c47 100644
--- a/media/engine/webrtc_video_engine_unittest.cc
+++ b/media/engine/webrtc_video_engine_unittest.cc
@@ -382,7 +382,9 @@
// Find the codec in the engine with the given name. The codec must be
// present.
cricket::VideoCodec GetEngineCodec(const std::string& name) const;
- void AddSupportedVideoCodecType(const std::string& name);
+ void AddSupportedVideoCodecType(
+ const std::string& name,
+ const std::vector<webrtc::ScalabilityMode>& scalability_modes = {});
std::unique_ptr<VideoMediaSendChannelInterface>
SetSendParamsWithAllSupportedCodecs();
@@ -855,8 +857,9 @@
}
void WebRtcVideoEngineTest::AddSupportedVideoCodecType(
- const std::string& name) {
- encoder_factory_->AddSupportedVideoCodecType(name);
+ const std::string& name,
+ const std::vector<webrtc::ScalabilityMode>& scalability_modes) {
+ encoder_factory_->AddSupportedVideoCodecType(name, scalability_modes);
decoder_factory_->AddSupportedVideoCodecType(name);
}
@@ -2645,6 +2648,8 @@
void SetUp() override {
AddSupportedVideoCodecType("VP8");
AddSupportedVideoCodecType("VP9");
+ AddSupportedVideoCodecType(
+ "AV1", {ScalabilityMode::kL1T3, ScalabilityMode::kL2T3});
#if defined(WEBRTC_USE_H264)
AddSupportedVideoCodecType("H264");
#endif
@@ -3664,6 +3669,42 @@
EXPECT_TRUE(send_channel_->SetVideoSend(last_ssrc_, nullptr, nullptr));
}
+TEST_F(WebRtcVideoChannelTest, VerifyAv1SpecificSettings) {
+ cricket::VideoSenderParameters parameters;
+ parameters.codecs.push_back(GetEngineCodec("AV1"));
+ ASSERT_TRUE(send_channel_->SetSenderParameters(parameters));
+ webrtc::test::FrameForwarder frame_forwarder;
+ webrtc::VideoCodecAV1 settings;
+
+ // Single-stream settings should apply with RTX as well (verifies that we
+ // check number of regular SSRCs and not StreamParams::ssrcs which contains
+ // both RTX and regular SSRCs).
+ FakeVideoSendStream* stream = SetUpSimulcast(false, /*with_rtx=*/true);
+ EXPECT_TRUE(
+ send_channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder));
+ send_channel_->SetSend(true);
+ frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame());
+
+ ASSERT_TRUE(stream->GetAv1Settings(&settings)) << "No AV1 config set.";
+ EXPECT_TRUE(settings.automatic_resize_on);
+
+ webrtc::RtpParameters rtp_parameters =
+ send_channel_->GetRtpSendParameters(last_ssrc_);
+ EXPECT_THAT(
+ rtp_parameters.encodings,
+ ElementsAre(Field(&webrtc::RtpEncodingParameters::scalability_mode,
+ absl::nullopt)));
+ rtp_parameters.encodings[0].scalability_mode = "L2T3";
+ EXPECT_TRUE(
+ send_channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters).ok());
+ frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame());
+
+ ASSERT_TRUE(stream->GetAv1Settings(&settings)) << "No AV1 config set.";
+ EXPECT_FALSE(settings.automatic_resize_on);
+
+ EXPECT_TRUE(send_channel_->SetVideoSend(last_ssrc_, nullptr, nullptr));
+}
+
// Test that setting the same options doesn't result in the encoder being
// reconfigured.
TEST_F(WebRtcVideoChannelTest, SetIdenticalOptionsDoesntReconfigureEncoder) {
diff --git a/modules/video_coding/codecs/av1/libaom_av1_encoder.cc b/modules/video_coding/codecs/av1/libaom_av1_encoder.cc
index 28a8e5f..1d333e7 100644
--- a/modules/video_coding/codecs/av1/libaom_av1_encoder.cc
+++ b/modules/video_coding/codecs/av1/libaom_av1_encoder.cc
@@ -824,7 +824,10 @@
info.implementation_name = "libaom";
info.has_trusted_rate_controller = true;
info.is_hardware_accelerated = false;
- info.scaling_settings = VideoEncoder::ScalingSettings(kMinQindex, kMaxQindex);
+ info.scaling_settings =
+ (inited_ && !encoder_settings_.AV1().automatic_resize_on)
+ ? VideoEncoder::ScalingSettings::kOff
+ : VideoEncoder::ScalingSettings(kMinQindex, kMaxQindex);
info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420,
VideoFrameBuffer::Type::kNV12};
if (SvcEnabled()) {
diff --git a/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc b/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc
index 09bf1bf..04ee916 100644
--- a/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc
+++ b/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc
@@ -36,6 +36,7 @@
VideoCodec DefaultCodecSettings() {
VideoCodec codec_settings;
+ codec_settings.codecType = kVideoCodecAV1;
codec_settings.width = 320;
codec_settings.height = 180;
codec_settings.maxFramerate = 30;
@@ -398,5 +399,16 @@
kTargetBitrateBps, kTargetBitrateBps / 10);
}
+TEST(LibaomAv1EncoderTest, DisableAutomaticResize) {
+ std::unique_ptr<VideoEncoder> encoder = CreateLibaomAv1Encoder();
+ ASSERT_TRUE(encoder);
+ VideoCodec codec_settings = DefaultCodecSettings();
+ codec_settings.AV1()->automatic_resize_on = false;
+ EXPECT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()),
+ WEBRTC_VIDEO_CODEC_OK);
+ EXPECT_EQ(encoder->GetEncoderInfo().scaling_settings.thresholds,
+ absl::nullopt);
+}
+
} // namespace
} // namespace webrtc
diff --git a/video/config/video_encoder_config.cc b/video/config/video_encoder_config.cc
index 6ea2052..5eec7b5 100644
--- a/video/config/video_encoder_config.cc
+++ b/video/config/video_encoder_config.cc
@@ -96,6 +96,8 @@
FillVideoCodecVp8(codec->VP8());
} else if (codec->codecType == kVideoCodecVP9) {
FillVideoCodecVp9(codec->VP9());
+ } else if (codec->codecType == kVideoCodecAV1) {
+ FillVideoCodecAv1(codec->AV1());
} else {
RTC_DCHECK_NOTREACHED()
<< "Encoder specifics set/used for unknown codec type.";
@@ -112,6 +114,11 @@
RTC_DCHECK_NOTREACHED();
}
+void VideoEncoderConfig::EncoderSpecificSettings::FillVideoCodecAv1(
+ VideoCodecAV1* av1_settings) const {
+ RTC_DCHECK_NOTREACHED();
+}
+
VideoEncoderConfig::Vp8EncoderSpecificSettings::Vp8EncoderSpecificSettings(
const VideoCodecVP8& specifics)
: specifics_(specifics) {}
@@ -130,4 +137,13 @@
*vp9_settings = specifics_;
}
+VideoEncoderConfig::Av1EncoderSpecificSettings::Av1EncoderSpecificSettings(
+ const VideoCodecAV1& specifics)
+ : specifics_(specifics) {}
+
+void VideoEncoderConfig::Av1EncoderSpecificSettings::FillVideoCodecAv1(
+ VideoCodecAV1* av1_settings) const {
+ *av1_settings = specifics_;
+}
+
} // namespace webrtc
diff --git a/video/config/video_encoder_config.h b/video/config/video_encoder_config.h
index 59c9a39..cb0644a 100644
--- a/video/config/video_encoder_config.h
+++ b/video/config/video_encoder_config.h
@@ -99,6 +99,7 @@
virtual void FillVideoCodecVp8(VideoCodecVP8* vp8_settings) const;
virtual void FillVideoCodecVp9(VideoCodecVP9* vp9_settings) const;
+ virtual void FillVideoCodecAv1(VideoCodecAV1* av1_settings) const;
private:
~EncoderSpecificSettings() override {}
@@ -123,6 +124,15 @@
VideoCodecVP9 specifics_;
};
+ class Av1EncoderSpecificSettings : public EncoderSpecificSettings {
+ public:
+ explicit Av1EncoderSpecificSettings(const VideoCodecAV1& specifics);
+ void FillVideoCodecAv1(VideoCodecAV1* av1_settings) const override;
+
+ private:
+ VideoCodecAV1 specifics_;
+ };
+
enum class ContentType {
kRealtimeVideo,
kScreen,
