Apply resolution-bitrate limits collected from field trial (cl/294600) for AV1.
Bug: webrtc:14931
Change-Id: I1e8471a499bc884cb9479609a2b093de90f638d8
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/296120
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Commit-Queue: Michael Horowitz <mhoro@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#39582}
diff --git a/call/adaptation/BUILD.gn b/call/adaptation/BUILD.gn
index 58fadc4..b69196f 100644
--- a/call/adaptation/BUILD.gn
+++ b/call/adaptation/BUILD.gn
@@ -44,6 +44,7 @@
"../../api/video:video_stream_encoder",
"../../api/video_codecs:video_codecs_api",
"../../modules/video_coding:video_coding_utility",
+ "../../modules/video_coding/svc:scalability_mode_util",
"../../rtc_base:checks",
"../../rtc_base:logging",
"../../rtc_base:macromagic",
diff --git a/call/adaptation/video_stream_adapter.cc b/call/adaptation/video_stream_adapter.cc
index f30a4d7..5a970fb 100644
--- a/call/adaptation/video_stream_adapter.cc
+++ b/call/adaptation/video_stream_adapter.cc
@@ -22,6 +22,7 @@
#include "api/video_codecs/video_encoder.h"
#include "call/adaptation/video_source_restrictions.h"
#include "call/adaptation/video_stream_input_state.h"
+#include "modules/video_coding/svc/scalability_mode_util.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/safe_conversions.h"
@@ -720,7 +721,17 @@
const VideoCodec& codec) {
int num_active = 0;
absl::optional<uint32_t> pixels;
- if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
+ if (codec.codecType == VideoCodecType::kVideoCodecAV1 &&
+ codec.GetScalabilityMode().has_value()) {
+ for (int i = 0;
+ i < ScalabilityModeToNumSpatialLayers(*(codec.GetScalabilityMode()));
+ ++i) {
+ if (codec.spatialLayers[i].active) {
+ ++num_active;
+ pixels = codec.spatialLayers[i].width * codec.spatialLayers[i].height;
+ }
+ }
+ } else if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) {
if (codec.spatialLayers[i].active) {
++num_active;
diff --git a/rtc_base/experiments/encoder_info_settings.cc b/rtc_base/experiments/encoder_info_settings.cc
index 062ba02..2e90a21 100644
--- a/rtc_base/experiments/encoder_info_settings.cc
+++ b/rtc_base/experiments/encoder_info_settings.cc
@@ -38,7 +38,8 @@
std::vector<VideoEncoder::ResolutionBitrateLimits>
EncoderInfoSettings::GetDefaultSinglecastBitrateLimits(
VideoCodecType codec_type) {
- // Specific limits for VP9. Other codecs use VP8 limits.
+ // Specific limits for VP9. Determining specific limits for AV1 via
+ // field trial experiment is a work in progress. Other codecs use VP8 limits.
if (codec_type == kVideoCodecVP9) {
return {{320 * 180, 0, 30000, 150000},
{480 * 270, 120000, 30000, 300000},
@@ -46,6 +47,10 @@
{960 * 540, 350000, 30000, 1000000},
{1280 * 720, 480000, 30000, 1500000}};
}
+ // Don't override existing AV1 limits with default values.
+ if (codec_type == kVideoCodecAV1) {
+ return {};
+ }
return {{320 * 180, 0, 30000, 300000},
{480 * 270, 200000, 30000, 500000},
diff --git a/video/BUILD.gn b/video/BUILD.gn
index 9acb0a5..3161435 100644
--- a/video/BUILD.gn
+++ b/video/BUILD.gn
@@ -403,6 +403,7 @@
"../modules/video_coding:video_codec_interface",
"../modules/video_coding:video_coding_utility",
"../modules/video_coding:webrtc_vp9_helpers",
+ "../modules/video_coding/svc:scalability_mode_util",
"../modules/video_coding/svc:scalability_structures",
"../modules/video_coding/svc:svc_rate_allocator",
"../rtc_base:checks",
diff --git a/video/end_to_end_tests/resolution_bitrate_limits_tests.cc b/video/end_to_end_tests/resolution_bitrate_limits_tests.cc
index 8455832..d550339 100644
--- a/video/end_to_end_tests/resolution_bitrate_limits_tests.cc
+++ b/video/end_to_end_tests/resolution_bitrate_limits_tests.cc
@@ -41,7 +41,18 @@
};
BitrateLimits GetLayerBitrateLimits(int pixels, const VideoCodec& codec) {
- if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
+ if (codec.codecType == VideoCodecType::kVideoCodecAV1) {
+ EXPECT_TRUE(codec.GetScalabilityMode().has_value());
+ for (int i = 0;
+ i < ScalabilityModeToNumSpatialLayers(*(codec.GetScalabilityMode()));
+ ++i) {
+ if (codec.spatialLayers[i].width * codec.spatialLayers[i].height ==
+ pixels) {
+ return {DataRate::KilobitsPerSec(codec.spatialLayers[i].minBitrate),
+ DataRate::KilobitsPerSec(codec.spatialLayers[i].maxBitrate)};
+ }
+ }
+ } else if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
for (size_t i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) {
if (codec.spatialLayers[i].width * codec.spatialLayers[i].height ==
pixels) {
@@ -62,6 +73,10 @@
return BitrateLimits();
}
+bool SupportsSpatialLayers(const std::string& payload_name) {
+ return payload_name == "VP9" || payload_name == "AV1";
+}
+
} // namespace
class ResolutionBitrateLimitsWithScalabilityModeTest : public test::CallTest {};
@@ -122,7 +137,7 @@
const rtc::VideoSinkWants& wants) override {}
size_t GetNumVideoStreams() const override {
- return (payload_name_ == "VP9") ? 1 : configs_.size();
+ return SupportsSpatialLayers(payload_name_) ? 1 : configs_.size();
}
void ModifyVideoConfigs(
@@ -142,7 +157,7 @@
encoder_config->max_bitrate_bps = -1;
if (configs_.size() == 1 && configs_[0].bitrate.max)
encoder_config->max_bitrate_bps = configs_[0].bitrate.max->bps();
- if (payload_name_ == "VP9") {
+ if (SupportsSpatialLayers(payload_name_)) {
// Simulcast layers indicates which spatial layers are active.
encoder_config->simulcast_layers.resize(configs_.size());
}
@@ -156,7 +171,7 @@
if (configs_[i].bitrate.max)
stream.max_bitrate_bps = configs_[i].bitrate.max->bps();
stream.scale_resolution_down_by = scale_factor;
- scale_factor *= (payload_name_ == "VP9") ? 1.0 : 2.0;
+ scale_factor *= SupportsSpatialLayers(payload_name_) ? 1.0 : 2.0;
}
SetEncoderSpecific(encoder_config, codec_type, configs_.size());
}
@@ -458,6 +473,67 @@
RunBaseTest(&test);
}
+TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest,
+ OneStreamLimitsAppliedForAv1OneSpatialLayer) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-GetEncoderInfoOverride/"
+ "frame_size_pixels:921600,"
+ "min_start_bitrate_bps:0,"
+ "min_bitrate_bps:32000,"
+ "max_bitrate_bps:133000/");
+
+ InitEncodeTest test(
+ "AV1", {{.active = true, .scalability_mode = ScalabilityMode::kL1T1}},
+ // Expectations:
+ {{.pixels = 1280 * 720,
+ .eq_bitrate = {DataRate::KilobitsPerSec(32),
+ DataRate::KilobitsPerSec(133)}}});
+ RunBaseTest(&test);
+}
+
+TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest,
+ LimitsAppliedForAv1Simulcast) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-GetEncoderInfoOverride/"
+ "frame_size_pixels:230400|921600,"
+ "min_start_bitrate_bps:0|0,"
+ "min_bitrate_bps:25000|80000,"
+ "max_bitrate_bps:400000|1200000/");
+
+ InitEncodeTest test(
+ "AV1",
+ {{.active = true, .scalability_mode = ScalabilityMode::kL1T1},
+ {.active = false}},
+ // Expectations:
+ {{.pixels = 1280 * 720,
+ .eq_bitrate = {DataRate::KilobitsPerSec(80),
+ DataRate::KilobitsPerSec(1200)}}});
+ RunBaseTest(&test);
+}
+
+TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest,
+ LimitsNotAppliedForAv1MultipleSpatialLayers) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-GetEncoderInfoOverride/"
+ "frame_size_pixels:230400|921600,"
+ "min_start_bitrate_bps:0|0,"
+ "min_bitrate_bps:20000|25000,"
+ "max_bitrate_bps:900000|1333000/");
+
+ InitEncodeTest test(
+ "AV1",
+ {{.active = true, .scalability_mode = ScalabilityMode::kL2T1},
+ {.active = false}},
+ // Expectations:
+ {{.pixels = 640 * 360,
+ .ne_bitrate = {DataRate::KilobitsPerSec(20),
+ DataRate::KilobitsPerSec(900)}},
+ {.pixels = 1280 * 720,
+ .ne_bitrate = {DataRate::KilobitsPerSec(25),
+ DataRate::KilobitsPerSec(1333)}}});
+ RunBaseTest(&test);
+}
+
TEST_P(ResolutionBitrateLimitsTest, LimitsNotAppliedSimulcast) {
webrtc::test::ScopedFieldTrials field_trials(
"WebRTC-GetEncoderInfoOverride/"
diff --git a/video/video_send_stream_impl.cc b/video/video_send_stream_impl.cc
index f34388e..5fa2af3 100644
--- a/video/video_send_stream_impl.cc
+++ b/video/video_send_stream_impl.cc
@@ -52,6 +52,9 @@
constexpr double kVideoHysteresis = 1.2;
constexpr double kScreenshareHysteresis = 1.35;
+constexpr int kMinDefaultAv1BitrateBps =
+ 15000; // This value acts as an absolute minimum AV1 bitrate limit.
+
// When send-side BWE is used a stricter 1.1x pacing factor is used, rather than
// the 2.5x which is used with receive-side BWE. Provides a more careful
// bandwidth rampup with less risk of overshoots causing adverse effects like
@@ -194,6 +197,13 @@
return kFallbackMaxBitrateBps;
}
+int GetDefaultMinVideoBitrateBps(VideoCodecType codec_type) {
+ if (codec_type == VideoCodecType::kVideoCodecAV1) {
+ return kMinDefaultAv1BitrateBps;
+ }
+ return kDefaultMinVideoBitrateBps;
+}
+
} // namespace
PacingConfig::PacingConfig(const FieldTrialsView& field_trials)
@@ -485,7 +495,8 @@
encoder_min_bitrate_bps_ =
experimental_min_bitrate
? experimental_min_bitrate->bps()
- : std::max(streams[0].min_bitrate_bps, kDefaultMinVideoBitrateBps);
+ : std::max(streams[0].min_bitrate_bps,
+ GetDefaultMinVideoBitrateBps(codec_type));
encoder_max_bitrate_bps_ = 0;
double stream_bitrate_priority_sum = 0;
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index 34fa362..39f46b9 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -37,6 +37,7 @@
#include "call/adaptation/video_stream_adapter.h"
#include "media/base/media_channel.h"
#include "modules/video_coding/include/video_codec_initializer.h"
+#include "modules/video_coding/svc/scalability_mode_util.h"
#include "modules/video_coding/svc/svc_rate_allocator.h"
#include "modules/video_coding/utility/vp8_constants.h"
#include "rtc_base/arraysize.h"
@@ -82,6 +83,17 @@
constexpr int kDefaultMinScreenSharebps = 1200000;
+int GetNumSpatialLayers(const VideoCodec& codec) {
+ if (codec.codecType == kVideoCodecVP9) {
+ return codec.VP9().numberOfSpatialLayers;
+ } else if (codec.codecType == kVideoCodecAV1 &&
+ codec.GetScalabilityMode().has_value()) {
+ return ScalabilityModeToNumSpatialLayers(*(codec.GetScalabilityMode()));
+ } else {
+ return 0;
+ }
+}
+
bool RequiresEncoderReset(const VideoCodec& prev_send_codec,
const VideoCodec& new_send_codec,
bool was_encode_called_since_last_initialization) {
@@ -383,13 +395,18 @@
return num_active;
}
-void ApplyVp9BitrateLimits(const VideoEncoder::EncoderInfo& encoder_info,
- const VideoEncoderConfig& encoder_config,
- VideoCodec* codec) {
- if (codec->codecType != VideoCodecType::kVideoCodecVP9 ||
- encoder_config.simulcast_layers.size() <= 1 ||
- VideoStreamEncoderResourceManager::IsSimulcastOrMultipleSpatialLayers(
- encoder_config)) {
+void ApplySpatialLayerBitrateLimits(
+ const VideoEncoder::EncoderInfo& encoder_info,
+ const VideoEncoderConfig& encoder_config,
+ VideoCodec* codec) {
+ if (!(GetNumSpatialLayers(*codec) > 0)) {
+ // ApplySpatialLayerBitrateLimits() supports VP9 and AV1 (the latter with
+ // scalability mode set) only.
+ return;
+ }
+ if (VideoStreamEncoderResourceManager::IsSimulcastOrMultipleSpatialLayers(
+ encoder_config) ||
+ encoder_config.simulcast_layers.size() <= 1) {
// Resolution bitrate limits usage is restricted to singlecast.
return;
}
@@ -405,7 +422,6 @@
if (!bitrate_limits.has_value()) {
return;
}
-
// Index for the active stream.
absl::optional<size_t> index;
for (size_t i = 0; i < encoder_config.simulcast_layers.size(); ++i) {
@@ -415,7 +431,6 @@
if (!index.has_value()) {
return;
}
-
int min_bitrate_bps;
if (encoder_config.simulcast_layers[*index].min_bitrate_bps <= 0) {
min_bitrate_bps = bitrate_limits->min_bitrate_bps;
@@ -439,7 +454,7 @@
return;
}
- for (int i = 0; i < codec->VP9()->numberOfSpatialLayers; ++i) {
+ for (int i = 0; i < GetNumSpatialLayers(*codec); ++i) {
if (codec->spatialLayers[i].active) {
codec->spatialLayers[i].minBitrate = min_bitrate_bps / 1000;
codec->spatialLayers[i].maxBitrate = max_bitrate_bps / 1000;
@@ -1141,15 +1156,17 @@
RTC_LOG(LS_ERROR) << "Failed to create encoder configuration.";
}
- if (encoder_config_.codec_type == kVideoCodecVP9) {
+ if (encoder_config_.codec_type == kVideoCodecVP9 ||
+ encoder_config_.codec_type == kVideoCodecAV1) {
// Spatial layers configuration might impose some parity restrictions,
// thus some cropping might be needed.
crop_width_ = last_frame_info_->width - codec.width;
crop_height_ = last_frame_info_->height - codec.height;
- ApplyVp9BitrateLimits(GetEncoderInfoWithBitrateLimitUpdate(
- encoder_->GetEncoderInfo(), encoder_config_,
- default_limits_allowed_),
- encoder_config_, &codec);
+ ApplySpatialLayerBitrateLimits(
+ GetEncoderInfoWithBitrateLimitUpdate(encoder_->GetEncoderInfo(),
+ encoder_config_,
+ default_limits_allowed_),
+ encoder_config_, &codec);
}
char log_stream_buf[4 * 1024];
@@ -1168,10 +1185,10 @@
<< " active: "
<< (codec.simulcastStream[i].active ? "true" : "false") << "\n";
}
- if (encoder_config_.codec_type == kVideoCodecVP9) {
- size_t num_spatial_layers = codec.VP9()->numberOfSpatialLayers;
+ if (encoder_config_.codec_type == kVideoCodecVP9 ||
+ encoder_config_.codec_type == kVideoCodecAV1) {
log_stream << "Spatial layers:\n";
- for (size_t i = 0; i < num_spatial_layers; ++i) {
+ for (int i = 0; i < GetNumSpatialLayers(codec); ++i) {
log_stream << i << ": " << codec.spatialLayers[i].width << "x"
<< codec.spatialLayers[i].height
<< " min_kbps: " << codec.spatialLayers[i].minBitrate
@@ -1331,6 +1348,10 @@
num_layers = codec.VP8()->numberOfTemporalLayers;
} else if (codec.codecType == kVideoCodecVP9) {
num_layers = codec.VP9()->numberOfTemporalLayers;
+ } else if (codec.codecType == kVideoCodecAV1 &&
+ codec.GetScalabilityMode().has_value()) {
+ num_layers =
+ ScalabilityModeToNumTemporalLayers(*(codec.GetScalabilityMode()));
} else if (codec.codecType == kVideoCodecH264) {
num_layers = codec.H264()->numberOfTemporalLayers;
} else if (codec.codecType == kVideoCodecGeneric &&
@@ -1375,8 +1396,9 @@
bool is_svc = false;
// Set min_bitrate_bps, max_bitrate_bps, and max padding bit rate for VP9
- // and leave only one stream containing all necessary information.
- if (encoder_config_.codec_type == kVideoCodecVP9 &&
+ // and AV1 and leave only one stream containing all necessary information.
+ if ((encoder_config_.codec_type == kVideoCodecVP9 ||
+ encoder_config_.codec_type == kVideoCodecAV1) &&
encoder_config_.number_of_streams == 1) {
// Lower max bitrate to the level codec actually can produce.
streams[0].max_bitrate_bps =
@@ -1388,7 +1410,7 @@
SvcRateAllocator::GetPaddingBitrate(codec).bps<int>();
streams[0].width = streams.back().width;
streams[0].height = streams.back().height;
- is_svc = codec.VP9()->numberOfSpatialLayers > 1;
+ is_svc = GetNumSpatialLayers(codec) > 1;
streams.resize(1);
}
