BalancedDegradationSettings: Add option to configure QP thresholds.
Add possibility to configure low/high QP thresholds based on resolution.
Bug: none
Change-Id: Iaa3168b77678bd74feb67295d7658c0140721231
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/141867
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Commit-Queue: Åsa Persson <asapersson@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#28348}
diff --git a/modules/video_coding/utility/quality_scaler.cc b/modules/video_coding/utility/quality_scaler.cc
index 1f1f192..42c40c5 100644
--- a/modules/video_coding/utility/quality_scaler.cc
+++ b/modules/video_coding/utility/quality_scaler.cc
@@ -134,6 +134,11 @@
return sampling_period_ms_ * initial_scale_factor_;
}
+void QualityScaler::SetQpThresholds(VideoEncoder::QpThresholds thresholds) {
+ RTC_DCHECK_RUN_ON(&task_checker_);
+ thresholds_ = thresholds;
+}
+
void QualityScaler::ReportDroppedFrameByMediaOpt() {
RTC_DCHECK_RUN_ON(&task_checker_);
framedrop_percent_media_opt_.AddSample(100);
diff --git a/modules/video_coding/utility/quality_scaler.h b/modules/video_coding/utility/quality_scaler.h
index 7886fc0..367db0e 100644
--- a/modules/video_coding/utility/quality_scaler.h
+++ b/modules/video_coding/utility/quality_scaler.h
@@ -60,6 +60,8 @@
// Inform the QualityScaler of the last seen QP.
void ReportQp(int qp, int64_t time_sent_us);
+ void SetQpThresholds(VideoEncoder::QpThresholds thresholds);
+
// The following members declared protected for testing purposes.
protected:
QualityScaler(rtc::TaskQueue* task_queue,
@@ -80,7 +82,7 @@
AdaptationObserverInterface* const observer_ RTC_GUARDED_BY(&task_checker_);
SequenceChecker task_checker_;
- const VideoEncoder::QpThresholds thresholds_;
+ VideoEncoder::QpThresholds thresholds_ RTC_GUARDED_BY(&task_checker_);
const int64_t sampling_period_ms_;
bool fast_rampup_ RTC_GUARDED_BY(&task_checker_);
rtc::MovingAverage average_qp_ RTC_GUARDED_BY(&task_checker_);
diff --git a/rtc_base/experiments/BUILD.gn b/rtc_base/experiments/BUILD.gn
index 3a78afe..849e740 100644
--- a/rtc_base/experiments/BUILD.gn
+++ b/rtc_base/experiments/BUILD.gn
@@ -107,7 +107,9 @@
deps = [
":field_trial_parser",
"../:rtc_base_approved",
+ "../../api/video_codecs:video_codecs_api",
"../../system_wrappers:field_trial",
+ "//third_party/abseil-cpp/absl/types:optional",
]
}
diff --git a/rtc_base/experiments/balanced_degradation_settings.cc b/rtc_base/experiments/balanced_degradation_settings.cc
index d6a33e1..a8d7d4d 100644
--- a/rtc_base/experiments/balanced_degradation_settings.cc
+++ b/rtc_base/experiments/balanced_degradation_settings.cc
@@ -23,7 +23,23 @@
constexpr int kMaxFps = 100;
std::vector<BalancedDegradationSettings::Config> DefaultConfigs() {
- return {{320 * 240, 7}, {480 * 270, 10}, {640 * 480, 15}};
+ return {{320 * 240, 7, {0, 0}, {0, 0}, {0, 0}, {0, 0}},
+ {480 * 270, 10, {0, 0}, {0, 0}, {0, 0}, {0, 0}},
+ {640 * 480, 15, {0, 0}, {0, 0}, {0, 0}, {0, 0}}};
+}
+
+bool IsValidThreshold(
+ const BalancedDegradationSettings::QpThreshold& threshold) {
+ if (threshold.GetLow().has_value() != threshold.GetHigh().has_value()) {
+ RTC_LOG(LS_WARNING) << "Neither or both values should be set.";
+ return false;
+ }
+ if (threshold.GetLow().has_value() && threshold.GetHigh().has_value() &&
+ threshold.GetLow().value() >= threshold.GetHigh().value()) {
+ RTC_LOG(LS_WARNING) << "Invalid threshold value, low >= high threshold.";
+ return false;
+ }
+ return true;
}
bool IsValid(const std::vector<BalancedDegradationSettings::Config>& configs) {
@@ -40,7 +56,24 @@
for (size_t i = 1; i < configs.size(); ++i) {
if (configs[i].pixels < configs[i - 1].pixels ||
configs[i].fps < configs[i - 1].fps) {
- RTC_LOG(LS_WARNING) << "Invalid parameter value provided.";
+ RTC_LOG(LS_WARNING) << "Invalid fps/pixel value provided.";
+ return false;
+ }
+ if (((configs[i].vp8.low > 0) != (configs[i - 1].vp8.low > 0)) ||
+ ((configs[i].vp9.low > 0) != (configs[i - 1].vp9.low > 0)) ||
+ ((configs[i].h264.low > 0) != (configs[i - 1].h264.low > 0)) ||
+ ((configs[i].generic.low > 0) != (configs[i - 1].generic.low > 0)) ||
+ ((configs[i].vp8.high > 0) != (configs[i - 1].vp8.high > 0)) ||
+ ((configs[i].vp9.high > 0) != (configs[i - 1].vp9.high > 0)) ||
+ ((configs[i].h264.high > 0) != (configs[i - 1].h264.high > 0)) ||
+ ((configs[i].generic.high > 0) != (configs[i - 1].generic.high > 0))) {
+ RTC_LOG(LS_WARNING) << "Invalid threshold value, all/none should be set.";
+ return false;
+ }
+ }
+ for (const auto& config : configs) {
+ if (!IsValidThreshold(config.vp8) || !IsValidThreshold(config.vp9) ||
+ !IsValidThreshold(config.h264) || !IsValidThreshold(config.generic)) {
return false;
}
}
@@ -54,17 +87,86 @@
}
return DefaultConfigs();
}
+
+absl::optional<VideoEncoder::QpThresholds> GetThresholds(
+ VideoCodecType type,
+ const BalancedDegradationSettings::Config& config) {
+ absl::optional<int> low;
+ absl::optional<int> high;
+
+ switch (type) {
+ case kVideoCodecVP8:
+ low = config.vp8.GetLow();
+ high = config.vp8.GetHigh();
+ break;
+ case kVideoCodecVP9:
+ low = config.vp9.GetLow();
+ high = config.vp9.GetHigh();
+ break;
+ case kVideoCodecH264:
+ low = config.h264.GetLow();
+ high = config.h264.GetHigh();
+ break;
+ case kVideoCodecGeneric:
+ low = config.generic.GetLow();
+ high = config.generic.GetHigh();
+ break;
+ default:
+ break;
+ }
+
+ if (low && high) {
+ RTC_LOG(LS_INFO) << "QP thresholds: low: " << *low << ", high: " << *high;
+ return absl::optional<VideoEncoder::QpThresholds>(
+ VideoEncoder::QpThresholds(*low, *high));
+ }
+ return absl::nullopt;
+}
} // namespace
+absl::optional<int> BalancedDegradationSettings::QpThreshold::GetLow() const {
+ return (low > 0) ? absl::optional<int>(low) : absl::nullopt;
+}
+
+absl::optional<int> BalancedDegradationSettings::QpThreshold::GetHigh() const {
+ return (high > 0) ? absl::optional<int>(high) : absl::nullopt;
+}
+
BalancedDegradationSettings::Config::Config() = default;
-BalancedDegradationSettings::Config::Config(int pixels, int fps)
- : pixels(pixels), fps(fps) {}
+BalancedDegradationSettings::Config::Config(int pixels,
+ int fps,
+ QpThreshold vp8,
+ QpThreshold vp9,
+ QpThreshold h264,
+ QpThreshold generic)
+ : pixels(pixels),
+ fps(fps),
+ vp8(vp8),
+ vp9(vp9),
+ h264(h264),
+ generic(generic) {}
BalancedDegradationSettings::BalancedDegradationSettings() {
FieldTrialStructList<Config> configs(
{FieldTrialStructMember("pixels", [](Config* c) { return &c->pixels; }),
- FieldTrialStructMember("fps", [](Config* c) { return &c->fps; })},
+ FieldTrialStructMember("fps", [](Config* c) { return &c->fps; }),
+ FieldTrialStructMember("vp8_qp_low",
+ [](Config* c) { return &c->vp8.low; }),
+ FieldTrialStructMember("vp8_qp_high",
+ [](Config* c) { return &c->vp8.high; }),
+ FieldTrialStructMember("vp9_qp_low",
+ [](Config* c) { return &c->vp9.low; }),
+ FieldTrialStructMember("vp9_qp_high",
+ [](Config* c) { return &c->vp9.high; }),
+ FieldTrialStructMember("h264_qp_low",
+ [](Config* c) { return &c->h264.low; }),
+ FieldTrialStructMember("h264_qp_high",
+ [](Config* c) { return &c->h264.high; }),
+ FieldTrialStructMember("generic_qp_low",
+ [](Config* c) { return &c->generic.low; }),
+ FieldTrialStructMember("generic_qp_high",
+ [](Config* c) { return &c->generic.high; })},
{});
ParseFieldTrial({&configs}, field_trial::FindFullName(kFieldTrial));
@@ -96,4 +198,19 @@
return std::numeric_limits<int>::max();
}
+absl::optional<VideoEncoder::QpThresholds>
+BalancedDegradationSettings::GetQpThresholds(VideoCodecType type,
+ int pixels) const {
+ return GetThresholds(type, GetConfig(pixels));
+}
+
+BalancedDegradationSettings::Config BalancedDegradationSettings::GetConfig(
+ int pixels) const {
+ for (const auto& config : configs_) {
+ if (pixels <= config.pixels)
+ return config;
+ }
+ return configs_.back(); // Use last above highest pixels.
+}
+
} // namespace webrtc
diff --git a/rtc_base/experiments/balanced_degradation_settings.h b/rtc_base/experiments/balanced_degradation_settings.h
index f006493..ef4b587 100644
--- a/rtc_base/experiments/balanced_degradation_settings.h
+++ b/rtc_base/experiments/balanced_degradation_settings.h
@@ -13,6 +13,9 @@
#include <vector>
+#include "absl/types/optional.h"
+#include "api/video_codecs/video_encoder.h"
+
namespace webrtc {
class BalancedDegradationSettings {
@@ -20,17 +23,40 @@
BalancedDegradationSettings();
~BalancedDegradationSettings();
- struct Config {
- Config();
- Config(int pixels, int fps);
+ struct QpThreshold {
+ QpThreshold() {}
+ QpThreshold(int low, int high) : low(low), high(high) {}
- bool operator==(const Config& o) const {
- return pixels == o.pixels && fps == o.fps;
+ bool operator==(const QpThreshold& o) const {
+ return low == o.low && high == o.high;
}
- int pixels = 0; // The video frame size.
- int fps = 0; // The framerate to be used if the frame size is less than
- // or equal to |pixels|.
+ absl::optional<int> GetLow() const;
+ absl::optional<int> GetHigh() const;
+ int low = 0;
+ int high = 0;
+ };
+
+ struct Config {
+ Config();
+ Config(int pixels,
+ int fps,
+ QpThreshold vp8,
+ QpThreshold vp9,
+ QpThreshold h264,
+ QpThreshold generic);
+
+ bool operator==(const Config& o) const {
+ return pixels == o.pixels && fps == o.fps && vp8 == o.vp8 &&
+ vp9 == o.vp9 && h264 == o.h264 && generic == o.generic;
+ }
+
+ int pixels = 0; // The video frame size.
+ int fps = 0; // The framerate and thresholds to be used if the frame
+ QpThreshold vp8; // size is less than or equal to |pixels|.
+ QpThreshold vp9;
+ QpThreshold h264;
+ QpThreshold generic;
};
// Returns configurations from field trial on success (default on failure).
@@ -40,7 +66,14 @@
int MinFps(int pixels) const;
int MaxFps(int pixels) const;
+ // Gets QpThresholds for the codec |type| based on |pixels|.
+ absl::optional<VideoEncoder::QpThresholds> GetQpThresholds(
+ VideoCodecType type,
+ int pixels) const;
+
private:
+ Config GetConfig(int pixels) const;
+
std::vector<Config> configs_;
};
diff --git a/rtc_base/experiments/balanced_degradation_settings_unittest.cc b/rtc_base/experiments/balanced_degradation_settings_unittest.cc
index 7a6eb65..8cbadac 100644
--- a/rtc_base/experiments/balanced_degradation_settings_unittest.cc
+++ b/rtc_base/experiments/balanced_degradation_settings_unittest.cc
@@ -22,9 +22,12 @@
void VerifyIsDefault(
const std::vector<BalancedDegradationSettings::Config>& config) {
EXPECT_THAT(config, ::testing::ElementsAre(
- BalancedDegradationSettings::Config{320 * 240, 7},
- BalancedDegradationSettings::Config{480 * 270, 10},
- BalancedDegradationSettings::Config{640 * 480, 15}));
+ BalancedDegradationSettings::Config{
+ 320 * 240, 7, {0, 0}, {0, 0}, {0, 0}, {0, 0}},
+ BalancedDegradationSettings::Config{
+ 480 * 270, 10, {0, 0}, {0, 0}, {0, 0}, {0, 0}},
+ BalancedDegradationSettings::Config{
+ 640 * 480, 15, {0, 0}, {0, 0}, {0, 0}, {0, 0}}));
}
} // namespace
@@ -32,18 +35,26 @@
webrtc::test::ScopedFieldTrials field_trials("");
BalancedDegradationSettings settings;
VerifyIsDefault(settings.GetConfigs());
+ EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecVP8, 1));
+ EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecVP9, 1));
+ EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecH264, 1));
+ EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecGeneric, 1));
+ EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecMultiplex, 1));
}
TEST(BalancedDegradationSettings, GetsConfig) {
webrtc::test::ScopedFieldTrials field_trials(
"WebRTC-Video-BalancedDegradationSettings/"
- "pixels:1000|2000|3000,fps:5|15|25/");
+ "pixels:11|22|33,fps:5|15|25,other:4|5|6/");
BalancedDegradationSettings settings;
- EXPECT_THAT(
- settings.GetConfigs(),
- ::testing::ElementsAre(BalancedDegradationSettings::Config{1000, 5},
- BalancedDegradationSettings::Config{2000, 15},
- BalancedDegradationSettings::Config{3000, 25}));
+ EXPECT_THAT(settings.GetConfigs(),
+ ::testing::ElementsAre(
+ BalancedDegradationSettings::Config{
+ 11, 5, {0, 0}, {0, 0}, {0, 0}, {0, 0}},
+ BalancedDegradationSettings::Config{
+ 22, 15, {0, 0}, {0, 0}, {0, 0}, {0, 0}},
+ BalancedDegradationSettings::Config{
+ 33, 25, {0, 0}, {0, 0}, {0, 0}, {0, 0}}));
}
TEST(BalancedDegradationSettings, GetsDefaultConfigForZeroFpsValue) {
@@ -96,4 +107,128 @@
EXPECT_EQ(std::numeric_limits<int>::max(), settings.MaxFps(2000 + 1));
}
+TEST(BalancedDegradationSettings, QpThresholdsNotSetByDefault) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:1000|2000|3000,fps:5|15|25/");
+ BalancedDegradationSettings settings;
+ EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecVP8, 1));
+ EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecVP9, 1));
+ EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecH264, 1));
+ EXPECT_FALSE(settings.GetQpThresholds(kVideoCodecGeneric, 1));
+}
+
+TEST(BalancedDegradationSettings, GetsConfigWithQpThresholds) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:1000|2000|3000,fps:5|15|25,vp8_qp_low:89|90|88,"
+ "vp8_qp_high:90|91|92,vp9_qp_low:27|28|29,vp9_qp_high:120|130|140,"
+ "h264_qp_low:12|13|14,h264_qp_high:20|30|40,generic_qp_low:7|6|5,"
+ "generic_qp_high:22|23|24/");
+ BalancedDegradationSettings settings;
+ EXPECT_THAT(settings.GetConfigs(),
+ ::testing::ElementsAre(
+ BalancedDegradationSettings::Config{
+ 1000, 5, {89, 90}, {27, 120}, {12, 20}, {7, 22}},
+ BalancedDegradationSettings::Config{
+ 2000, 15, {90, 91}, {28, 130}, {13, 30}, {6, 23}},
+ BalancedDegradationSettings::Config{
+ 3000, 25, {88, 92}, {29, 140}, {14, 40}, {5, 24}}));
+}
+
+TEST(BalancedDegradationSettings, GetsDefaultConfigIfOnlyHasLowThreshold) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:1000|2000|3000,fps:5|15|25,vp8_qp_low:89|90|88/");
+ BalancedDegradationSettings settings;
+ VerifyIsDefault(settings.GetConfigs());
+}
+
+TEST(BalancedDegradationSettings, GetsDefaultConfigIfOnlyHasHighThreshold) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:1000|2000|3000,fps:5|15|25,vp8_qp_high:90|91|92/");
+ BalancedDegradationSettings settings;
+ VerifyIsDefault(settings.GetConfigs());
+}
+
+TEST(BalancedDegradationSettings, GetsDefaultConfigIfLowEqualsHigh) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:1000|2000|3000,fps:5|15|25,"
+ "vp8_qp_low:89|90|88,vp8_qp_high:90|91|88/");
+ BalancedDegradationSettings settings;
+ VerifyIsDefault(settings.GetConfigs());
+}
+
+TEST(BalancedDegradationSettings, GetsDefaultConfigIfLowGreaterThanHigh) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:1000|2000|3000,fps:5|15|25,"
+ "vp8_qp_low:89|90|88,vp8_qp_high:90|91|87/");
+ BalancedDegradationSettings settings;
+ VerifyIsDefault(settings.GetConfigs());
+}
+
+TEST(BalancedDegradationSettings, GetsDefaultConfigForZeroQpValue) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:1000|2000|3000,fps:5|15|25,"
+ "vp8_qp_low:89|0|88,vp8_qp_high:90|91|92/");
+ BalancedDegradationSettings settings;
+ VerifyIsDefault(settings.GetConfigs());
+}
+
+TEST(BalancedDegradationSettings, GetsVp8QpThresholds) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:1000|2000|3000,fps:5|15|25,"
+ "vp8_qp_low:89|90|88,vp8_qp_high:90|91|92/");
+ BalancedDegradationSettings settings;
+ EXPECT_EQ(89, settings.GetQpThresholds(kVideoCodecVP8, 1)->low);
+ EXPECT_EQ(90, settings.GetQpThresholds(kVideoCodecVP8, 1)->high);
+ EXPECT_EQ(90, settings.GetQpThresholds(kVideoCodecVP8, 1000)->high);
+ EXPECT_EQ(91, settings.GetQpThresholds(kVideoCodecVP8, 1001)->high);
+ EXPECT_EQ(91, settings.GetQpThresholds(kVideoCodecVP8, 2000)->high);
+ EXPECT_EQ(92, settings.GetQpThresholds(kVideoCodecVP8, 2001)->high);
+ EXPECT_EQ(92, settings.GetQpThresholds(kVideoCodecVP8, 3000)->high);
+ EXPECT_EQ(92, settings.GetQpThresholds(kVideoCodecVP8, 3001)->high);
+}
+
+TEST(BalancedDegradationSettings, GetsVp9QpThresholds) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:1000|2000|3000,fps:5|15|25,"
+ "vp9_qp_low:55|56|57,vp9_qp_high:155|156|157/");
+ BalancedDegradationSettings settings;
+ const auto thresholds = settings.GetQpThresholds(kVideoCodecVP9, 1000);
+ EXPECT_TRUE(thresholds);
+ EXPECT_EQ(55, thresholds->low);
+ EXPECT_EQ(155, thresholds->high);
+}
+
+TEST(BalancedDegradationSettings, GetsH264QpThresholds) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:1000|2000|3000,fps:5|15|25,"
+ "h264_qp_low:21|22|23,h264_qp_high:41|43|42/");
+ BalancedDegradationSettings settings;
+ const auto thresholds = settings.GetQpThresholds(kVideoCodecH264, 2000);
+ EXPECT_TRUE(thresholds);
+ EXPECT_EQ(22, thresholds->low);
+ EXPECT_EQ(43, thresholds->high);
+}
+
+TEST(BalancedDegradationSettings, GetsGenericQpThresholds) {
+ webrtc::test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-BalancedDegradationSettings/"
+ "pixels:1000|2000|3000,fps:5|15|25,"
+ "generic_qp_low:2|3|4,generic_qp_high:22|23|24/");
+ BalancedDegradationSettings settings;
+ const auto thresholds = settings.GetQpThresholds(kVideoCodecGeneric, 3000);
+ EXPECT_TRUE(thresholds);
+ EXPECT_EQ(4, thresholds->low);
+ EXPECT_EQ(24, thresholds->high);
+}
+
} // namespace webrtc
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index ebaaa44..a2bf3ef 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -873,6 +873,16 @@
initial_framedrop_ = kMaxInitialFramedrop;
}
+ if (degradation_preference_ == DegradationPreference::BALANCED &&
+ quality_scaler_ && last_frame_info_) {
+ absl::optional<VideoEncoder::QpThresholds> thresholds =
+ balanced_settings_.GetQpThresholds(encoder_config_.codec_type,
+ last_frame_info_->pixel_count());
+ if (thresholds) {
+ quality_scaler_->SetQpThresholds(*thresholds);
+ }
+ }
+
encoder_stats_observer_->OnAdaptationChanged(
VideoStreamEncoderObserver::AdaptationReason::kNone,
GetActiveCounts(kCpu), GetActiveCounts(kQuality));
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h
index 8bc3dc4..3756e80 100644
--- a/video/video_stream_encoder.h
+++ b/video/video_stream_encoder.h
@@ -282,7 +282,7 @@
// Set depending on degradation preferences.
DegradationPreference degradation_preference_ RTC_GUARDED_BY(&encoder_queue_);
- BalancedDegradationSettings balanced_settings_;
+ const BalancedDegradationSettings balanced_settings_;
struct AdaptationRequest {
// The pixel count produced by the source at the time of the adaptation.
