video: Implement bandwidth based scaler
The |slice_qp_detla| reported by the hardware is not credible, which
causing the quality scaler cannot work properly,the resolution cannot
be adjusted correctly.
To fix this issue, this CL implements a bandwidth scaler which is used
for adjust resolution, this scaler will be used when QP based quality
scaler is not working due to untrusted QP reported by HW AVC encoder.
Bug: webrtc:12942
Change-Id: I2fc5f07a5400ec7e5ead2c2c502faee84d7f2a76
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/228860
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Commit-Queue: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35120}
diff --git a/AUTHORS b/AUTHORS
index 11da0f4..4ab41e0 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -97,6 +97,7 @@
Satender Saroha <ssaroha@yahoo.com>
Saul Kravitz <Saul.Kravitz@celera.com>
Sergio Garcia Murillo <sergio.garcia.murillo@gmail.com>
+Shuhai Peng <shuhai.peng@intel.com>
Silviu Caragea <silviu.cpp@gmail.com>
Stefan Gula <steweg@gmail.com>
Stephan Hartmann <stha09@googlemail.com>
diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index 129d4ea..f92dc63 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -364,6 +364,8 @@
rtc_library("video_coding_utility") {
visibility = [ "*" ]
sources = [
+ "utility/bandwidth_quality_scaler.cc",
+ "utility/bandwidth_quality_scaler.h",
"utility/decoded_frames_history.cc",
"utility/decoded_frames_history.h",
"utility/frame_dropper.cc",
@@ -410,6 +412,8 @@
"../../rtc_base:rtc_numerics",
"../../rtc_base:rtc_task_queue",
"../../rtc_base:weak_ptr",
+ "../../rtc_base/experiments:bandwidth_quality_scaler_settings",
+ "../../rtc_base/experiments:encoder_info_settings",
"../../rtc_base/experiments:quality_scaler_settings",
"../../rtc_base/experiments:quality_scaling_experiment",
"../../rtc_base/experiments:rate_control_settings",
@@ -1031,6 +1035,7 @@
"timestamp_map_unittest.cc",
"timing_unittest.cc",
"unique_timestamp_counter_unittest.cc",
+ "utility/bandwidth_quality_scaler_unittest.cc",
"utility/decoded_frames_history_unittest.cc",
"utility/frame_dropper_unittest.cc",
"utility/framerate_controller_deprecated_unittest.cc",
@@ -1106,6 +1111,7 @@
"../../rtc_base:rtc_numerics",
"../../rtc_base:rtc_task_queue",
"../../rtc_base:task_queue_for_test",
+ "../../rtc_base/experiments:encoder_info_settings",
"../../rtc_base/experiments:jitter_upper_bound_experiment",
"../../rtc_base/synchronization:mutex",
"../../rtc_base/system:unused",
diff --git a/modules/video_coding/utility/bandwidth_quality_scaler.cc b/modules/video_coding/utility/bandwidth_quality_scaler.cc
new file mode 100644
index 0000000..0d6501a
--- /dev/null
+++ b/modules/video_coding/utility/bandwidth_quality_scaler.cc
@@ -0,0 +1,150 @@
+/*
+ * Copyright 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/utility/bandwidth_quality_scaler.h"
+
+#include <algorithm>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "api/video/video_adaptation_reason.h"
+#include "api/video_codecs/video_encoder.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/experiments/bandwidth_quality_scaler_settings.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/numerics/exp_filter.h"
+#include "rtc_base/task_queue.h"
+#include "rtc_base/task_utils/to_queued_task.h"
+#include "rtc_base/time_utils.h"
+#include "rtc_base/weak_ptr.h"
+
+namespace webrtc {
+
+namespace {
+
+constexpr int kDefaultMaxWindowSizeMs = 5000;
+constexpr float kHigherMaxBitrateTolerationFactor = 0.95;
+constexpr float kLowerMinBitrateTolerationFactor = 0.8;
+constexpr int kDefaultBitrateStateUpdateIntervalSeconds = 5;
+} // namespace
+
+BandwidthQualityScaler::BandwidthQualityScaler(
+ BandwidthQualityScalerUsageHandlerInterface* handler)
+ : kBitrateStateUpdateInterval(TimeDelta::Seconds(
+ BandwidthQualityScalerSettings::ParseFromFieldTrials()
+ .BitrateStateUpdateInterval()
+ .value_or(kDefaultBitrateStateUpdateIntervalSeconds))),
+ handler_(handler),
+ encoded_bitrate_(kDefaultMaxWindowSizeMs, RateStatistics::kBpsScale),
+ weak_ptr_factory_(this) {
+ RTC_DCHECK_RUN_ON(&task_checker_);
+ RTC_DCHECK(handler_ != nullptr);
+
+ StartCheckForBitrate();
+}
+
+BandwidthQualityScaler::~BandwidthQualityScaler() {
+ RTC_DCHECK_RUN_ON(&task_checker_);
+}
+
+void BandwidthQualityScaler::StartCheckForBitrate() {
+ RTC_DCHECK_RUN_ON(&task_checker_);
+ TaskQueueBase::Current()->PostDelayedTask(
+ ToQueuedTask([this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] {
+ if (!this_weak_ptr) {
+ // The caller BandwidthQualityScaler has been deleted.
+ return;
+ }
+ RTC_DCHECK_RUN_ON(&task_checker_);
+ switch (CheckBitrate()) {
+ case BandwidthQualityScaler::CheckBitrateResult::kHighBitRate: {
+ handler_->OnReportUsageBandwidthHigh();
+ last_frame_size_pixels_.reset();
+ break;
+ }
+ case BandwidthQualityScaler::CheckBitrateResult::kLowBitRate: {
+ handler_->OnReportUsageBandwidthLow();
+ last_frame_size_pixels_.reset();
+ break;
+ }
+ case BandwidthQualityScaler::CheckBitrateResult::kNormalBitrate: {
+ break;
+ }
+ case BandwidthQualityScaler::CheckBitrateResult::
+ kInsufficientSamples: {
+ break;
+ }
+ }
+ StartCheckForBitrate();
+ }),
+ kBitrateStateUpdateInterval.ms());
+}
+
+void BandwidthQualityScaler::ReportEncodeInfo(int frame_size_bytes,
+ int64_t time_sent_in_ms,
+ uint32_t encoded_width,
+ uint32_t encoded_height) {
+ RTC_DCHECK_RUN_ON(&task_checker_);
+ last_time_sent_in_ms_ = time_sent_in_ms;
+ last_frame_size_pixels_ = encoded_width * encoded_height;
+ encoded_bitrate_.Update(frame_size_bytes, time_sent_in_ms);
+}
+
+void BandwidthQualityScaler::SetResolutionBitrateLimits(
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>&
+ resolution_bitrate_limits) {
+ if (resolution_bitrate_limits.empty()) {
+ resolution_bitrate_limits_ = EncoderInfoSettings::
+ GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted();
+ } else {
+ resolution_bitrate_limits_ = resolution_bitrate_limits;
+ }
+}
+
+BandwidthQualityScaler::CheckBitrateResult
+BandwidthQualityScaler::CheckBitrate() {
+ RTC_DCHECK_RUN_ON(&task_checker_);
+ if (!last_frame_size_pixels_.has_value() ||
+ !last_time_sent_in_ms_.has_value()) {
+ return BandwidthQualityScaler::CheckBitrateResult::kInsufficientSamples;
+ }
+
+ absl::optional<int64_t> current_bitrate_bps =
+ encoded_bitrate_.Rate(last_time_sent_in_ms_.value());
+ if (!current_bitrate_bps.has_value()) {
+ // We can't get a valid bitrate due to not enough data points.
+ return BandwidthQualityScaler::CheckBitrateResult::kInsufficientSamples;
+ }
+ absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
+ EncoderInfoSettings::
+ GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
+ last_frame_size_pixels_, resolution_bitrate_limits_);
+
+ if (!suitable_bitrate_limit.has_value()) {
+ return BandwidthQualityScaler::CheckBitrateResult::kInsufficientSamples;
+ }
+
+ // Multiply by toleration factor to solve the frequent adaptation due to
+ // critical value.
+ if (current_bitrate_bps > suitable_bitrate_limit->max_bitrate_bps *
+ kHigherMaxBitrateTolerationFactor) {
+ return BandwidthQualityScaler::CheckBitrateResult::kLowBitRate;
+ } else if (current_bitrate_bps <
+ suitable_bitrate_limit->min_start_bitrate_bps *
+ kLowerMinBitrateTolerationFactor) {
+ return BandwidthQualityScaler::CheckBitrateResult::kHighBitRate;
+ }
+ return BandwidthQualityScaler::CheckBitrateResult::kNormalBitrate;
+}
+
+BandwidthQualityScalerUsageHandlerInterface::
+ ~BandwidthQualityScalerUsageHandlerInterface() {}
+} // namespace webrtc
diff --git a/modules/video_coding/utility/bandwidth_quality_scaler.h b/modules/video_coding/utility/bandwidth_quality_scaler.h
new file mode 100644
index 0000000..bb356dd
--- /dev/null
+++ b/modules/video_coding/utility/bandwidth_quality_scaler.h
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_UTILITY_BANDWIDTH_QUALITY_SCALER_H_
+#define MODULES_VIDEO_CODING_UTILITY_BANDWIDTH_QUALITY_SCALER_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <memory>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/scoped_refptr.h"
+#include "api/sequence_checker.h"
+#include "api/video_codecs/video_encoder.h"
+#include "rtc_base/experiments/encoder_info_settings.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/numerics/exp_filter.h"
+#include "rtc_base/rate_statistics.h"
+#include "rtc_base/ref_count.h"
+#include "rtc_base/ref_counted_object.h"
+#include "rtc_base/system/no_unique_address.h"
+#include "rtc_base/task_queue.h"
+#include "rtc_base/weak_ptr.h"
+
+namespace webrtc {
+
+class BandwidthQualityScalerUsageHandlerInterface {
+ public:
+ virtual ~BandwidthQualityScalerUsageHandlerInterface();
+
+ virtual void OnReportUsageBandwidthHigh() = 0;
+ virtual void OnReportUsageBandwidthLow() = 0;
+};
+
+// BandwidthQualityScaler runs asynchronously and monitors bandwidth values of
+// encoded frames. It holds a reference to a
+// BandwidthQualityScalerUsageHandlerInterface implementation to signal an
+// overuse or underuse of bandwidth (which indicate a desire to scale the video
+// stream down or up).
+class BandwidthQualityScaler {
+ public:
+ explicit BandwidthQualityScaler(
+ BandwidthQualityScalerUsageHandlerInterface* handler);
+ virtual ~BandwidthQualityScaler();
+
+ void ReportEncodeInfo(int frame_size_bytes,
+ int64_t time_sent_in_ms,
+ uint32_t encoded_width,
+ uint32_t encoded_height);
+
+ // We prioritise to using the |resolution_bitrate_limits| provided by the
+ // current decoder. If not provided, we will use the default data by
+ // GetDefaultResolutionBitrateLimits().
+ void SetResolutionBitrateLimits(
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>&
+ resolution_bitrate_limits);
+
+ const TimeDelta kBitrateStateUpdateInterval;
+
+ private:
+ enum class CheckBitrateResult {
+ kInsufficientSamples,
+ kNormalBitrate,
+ kHighBitRate,
+ kLowBitRate,
+ };
+
+ // We will periodically check encode bitrate, this function will make
+ // resolution up or down decisions and report the decision to the adapter.
+ void StartCheckForBitrate();
+ CheckBitrateResult CheckBitrate();
+
+ RTC_NO_UNIQUE_ADDRESS SequenceChecker task_checker_;
+ BandwidthQualityScalerUsageHandlerInterface* const handler_
+ RTC_GUARDED_BY(&task_checker_);
+
+ absl::optional<int64_t> last_time_sent_in_ms_ RTC_GUARDED_BY(&task_checker_);
+ RateStatistics encoded_bitrate_ RTC_GUARDED_BY(&task_checker_);
+ absl::optional<int> last_frame_size_pixels_ RTC_GUARDED_BY(&task_checker_);
+ rtc::WeakPtrFactory<BandwidthQualityScaler> weak_ptr_factory_;
+
+ std::vector<VideoEncoder::ResolutionBitrateLimits> resolution_bitrate_limits_;
+};
+
+} // namespace webrtc
+#endif // MODULES_VIDEO_CODING_UTILITY_BANDWIDTH_QUALITY_SCALER_H_
diff --git a/modules/video_coding/utility/bandwidth_quality_scaler_unittest.cc b/modules/video_coding/utility/bandwidth_quality_scaler_unittest.cc
new file mode 100644
index 0000000..30af1f9
--- /dev/null
+++ b/modules/video_coding/utility/bandwidth_quality_scaler_unittest.cc
@@ -0,0 +1,279 @@
+/*
+ * Copyright 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/utility/bandwidth_quality_scaler.h"
+
+#include <memory>
+#include <string>
+
+#include "rtc_base/checks.h"
+#include "rtc_base/event.h"
+#include "rtc_base/experiments/encoder_info_settings.h"
+#include "rtc_base/task_queue_for_test.h"
+#include "rtc_base/time_utils.h"
+#include "test/field_trial.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+namespace {
+constexpr int kFramerateFps = 30;
+constexpr int kDefaultBitrateStateUpdateIntervalSeconds = 5;
+constexpr int kDefaultEncodeDeltaTimeMs = 33; // 1/30(s) => 33(ms)
+
+} // namespace
+
+class FakeBandwidthQualityScalerHandler
+ : public BandwidthQualityScalerUsageHandlerInterface {
+ public:
+ ~FakeBandwidthQualityScalerHandler() override = default;
+ void OnReportUsageBandwidthHigh() override {
+ adapt_down_event_count_++;
+ event_.Set();
+ }
+
+ void OnReportUsageBandwidthLow() override {
+ adapt_up_event_count_++;
+ event_.Set();
+ }
+
+ rtc::Event event_;
+ int adapt_up_event_count_ = 0;
+ int adapt_down_event_count_ = 0;
+};
+
+class BandwidthQualityScalerUnderTest : public BandwidthQualityScaler {
+ public:
+ explicit BandwidthQualityScalerUnderTest(
+ BandwidthQualityScalerUsageHandlerInterface* handler)
+ : BandwidthQualityScaler(handler) {}
+
+ int GetBitrateStateUpdateIntervalMs() {
+ return this->kBitrateStateUpdateInterval.ms() + 200;
+ }
+};
+
+class BandwidthQualityScalerTest
+ : public ::testing::Test,
+ public ::testing::WithParamInterface<std::string> {
+ protected:
+ enum ScaleDirection {
+ kKeepScaleNormalBandwidth,
+ kKeepScaleAboveMaxBandwidth,
+ kKeepScaleUnderMinBandwidth,
+ };
+
+ enum FrameType {
+ kKeyFrame,
+ kNormalFrame,
+ kNormalFrame_Overuse,
+ kNormalFrame_Underuse,
+ };
+ struct FrameConfig {
+ FrameConfig(int frame_num,
+ FrameType frame_type,
+ int actual_width,
+ int actual_height)
+ : frame_num(frame_num),
+ frame_type(frame_type),
+ actual_width(actual_width),
+ actual_height(actual_height) {}
+
+ int frame_num;
+ FrameType frame_type;
+ int actual_width;
+ int actual_height;
+ };
+
+ BandwidthQualityScalerTest()
+ : scoped_field_trial_(GetParam()),
+ task_queue_("BandwidthQualityScalerTestQueue"),
+ handler_(std::make_unique<FakeBandwidthQualityScalerHandler>()) {
+ task_queue_.SendTask(
+ [this] {
+ bandwidth_quality_scaler_ =
+ std::unique_ptr<BandwidthQualityScalerUnderTest>(
+ new BandwidthQualityScalerUnderTest(handler_.get()));
+ bandwidth_quality_scaler_->SetResolutionBitrateLimits(
+ EncoderInfoSettings::
+ GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
+ // Only for testing. Set first_timestamp_ in RateStatistics to 0.
+ bandwidth_quality_scaler_->ReportEncodeInfo(0, 0, 0, 0);
+ },
+ RTC_FROM_HERE);
+ }
+
+ ~BandwidthQualityScalerTest() {
+ task_queue_.SendTask([this] { bandwidth_quality_scaler_ = nullptr; },
+ RTC_FROM_HERE);
+ }
+
+ int GetFrameSizeBytes(
+ const FrameConfig& config,
+ const VideoEncoder::ResolutionBitrateLimits& bitrate_limits) {
+ int scale = 8 * kFramerateFps;
+ switch (config.frame_type) {
+ case FrameType::kKeyFrame: {
+ // 4 is experimental value. Based on the test, the number of bytes of
+ // the key frame is about four times of the normal frame
+ return bitrate_limits.max_bitrate_bps * 4 / scale;
+ }
+ case FrameType::kNormalFrame_Overuse: {
+ return bitrate_limits.max_bitrate_bps * 3 / 2 / scale;
+ }
+ case FrameType::kNormalFrame_Underuse: {
+ return bitrate_limits.min_start_bitrate_bps * 3 / 4 / scale;
+ }
+ case FrameType::kNormalFrame: {
+ return (bitrate_limits.max_bitrate_bps +
+ bitrate_limits.min_start_bitrate_bps) /
+ 2 / scale;
+ }
+ }
+ return -1;
+ }
+
+ absl::optional<VideoEncoder::ResolutionBitrateLimits>
+ GetDefaultSuitableBitrateLimit(int frame_size_pixels) {
+ return EncoderInfoSettings::
+ GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
+ frame_size_pixels,
+ EncoderInfoSettings::
+ GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
+ }
+
+ void TriggerBandwidthQualityScalerTest(
+ const std::vector<FrameConfig>& frame_configs) {
+ task_queue_.SendTask(
+ [frame_configs, this] {
+ RTC_CHECK(!frame_configs.empty());
+
+ int total_frame_nums = 0;
+ for (const FrameConfig& frame_config : frame_configs) {
+ total_frame_nums += frame_config.frame_num;
+ }
+
+ EXPECT_EQ(kFramerateFps * kDefaultBitrateStateUpdateIntervalSeconds,
+ total_frame_nums);
+
+ uint32_t time_send_to_scaler_ms_ = rtc::TimeMillis();
+ for (size_t i = 0; i < frame_configs.size(); ++i) {
+ const FrameConfig& config = frame_configs[i];
+ absl::optional<VideoEncoder::ResolutionBitrateLimits>
+ suitable_bitrate = GetDefaultSuitableBitrateLimit(
+ config.actual_width * config.actual_height);
+ EXPECT_TRUE(suitable_bitrate);
+ for (int j = 0; j <= config.frame_num; ++j) {
+ time_send_to_scaler_ms_ += kDefaultEncodeDeltaTimeMs;
+ int frame_size_bytes =
+ GetFrameSizeBytes(config, suitable_bitrate.value());
+ RTC_CHECK(frame_size_bytes > 0);
+ bandwidth_quality_scaler_->ReportEncodeInfo(
+ frame_size_bytes, time_send_to_scaler_ms_,
+ config.actual_width, config.actual_height);
+ }
+ }
+ },
+ RTC_FROM_HERE);
+ }
+
+ test::ScopedFieldTrials scoped_field_trial_;
+ TaskQueueForTest task_queue_;
+ std::unique_ptr<BandwidthQualityScalerUnderTest> bandwidth_quality_scaler_;
+ std::unique_ptr<FakeBandwidthQualityScalerHandler> handler_;
+};
+
+INSTANTIATE_TEST_SUITE_P(
+ FieldTrials,
+ BandwidthQualityScalerTest,
+ ::testing::Values("WebRTC-Video-BandwidthQualityScalerSettings/"
+ "bitrate_state_update_interval_s_:1/",
+ "WebRTC-Video-BandwidthQualityScalerSettings/"
+ "bitrate_state_update_interval_s_:2/"));
+
+TEST_P(BandwidthQualityScalerTest, AllNormalFrame_640x360) {
+ const std::vector<FrameConfig> frame_configs{
+ FrameConfig(150, FrameType::kNormalFrame, 640, 360)};
+ TriggerBandwidthQualityScalerTest(frame_configs);
+
+ // When resolution is 640*360, experimental working bitrate range is
+ // [500000,800000] bps. Encoded bitrate is 654253, so it falls in the range
+ // without any operation(up/down).
+ EXPECT_FALSE(handler_->event_.Wait(
+ bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs()));
+ EXPECT_EQ(0, handler_->adapt_down_event_count_);
+ EXPECT_EQ(0, handler_->adapt_up_event_count_);
+}
+
+TEST_P(BandwidthQualityScalerTest, AllNoramlFrame_AboveMaxBandwidth_640x360) {
+ const std::vector<FrameConfig> frame_configs{
+ FrameConfig(150, FrameType::kNormalFrame_Overuse, 640, 360)};
+ TriggerBandwidthQualityScalerTest(frame_configs);
+
+ // When resolution is 640*360, experimental working bitrate range is
+ // [500000,800000] bps. Encoded bitrate is 1208000 > 800000 * 0.95, so it
+ // triggers adapt_up_event_count_.
+ EXPECT_TRUE(handler_->event_.Wait(
+ bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs()));
+ EXPECT_EQ(0, handler_->adapt_down_event_count_);
+ EXPECT_EQ(1, handler_->adapt_up_event_count_);
+}
+
+TEST_P(BandwidthQualityScalerTest, AllNormalFrame_Underuse_640x360) {
+ const std::vector<FrameConfig> frame_configs{
+ FrameConfig(150, FrameType::kNormalFrame_Underuse, 640, 360)};
+ TriggerBandwidthQualityScalerTest(frame_configs);
+
+ // When resolution is 640*360, experimental working bitrate range is
+ // [500000,800000] bps. Encoded bitrate is 377379 < 500000 * 0.8, so it
+ // triggers adapt_down_event_count_.
+ EXPECT_TRUE(handler_->event_.Wait(
+ bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs()));
+ EXPECT_EQ(1, handler_->adapt_down_event_count_);
+ EXPECT_EQ(0, handler_->adapt_up_event_count_);
+}
+
+TEST_P(BandwidthQualityScalerTest, FixedFrameTypeTest1_640x360) {
+ const std::vector<FrameConfig> frame_configs{
+ FrameConfig(5, FrameType::kNormalFrame_Underuse, 640, 360),
+ FrameConfig(110, FrameType::kNormalFrame, 640, 360),
+ FrameConfig(20, FrameType::kNormalFrame_Overuse, 640, 360),
+ FrameConfig(15, FrameType::kKeyFrame, 640, 360),
+ };
+ TriggerBandwidthQualityScalerTest(frame_configs);
+
+ // When resolution is 640*360, experimental working bitrate range is
+ // [500000,800000] bps. Encoded bitrate is 1059462 > 800000 * 0.95, so it
+ // triggers adapt_up_event_count_.
+ EXPECT_TRUE(handler_->event_.Wait(
+ bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs()));
+ EXPECT_EQ(0, handler_->adapt_down_event_count_);
+ EXPECT_EQ(1, handler_->adapt_up_event_count_);
+}
+
+TEST_P(BandwidthQualityScalerTest, FixedFrameTypeTest2_640x360) {
+ const std::vector<FrameConfig> frame_configs{
+ FrameConfig(10, FrameType::kNormalFrame_Underuse, 640, 360),
+ FrameConfig(50, FrameType::kNormalFrame, 640, 360),
+ FrameConfig(5, FrameType::kKeyFrame, 640, 360),
+ FrameConfig(85, FrameType::kNormalFrame_Overuse, 640, 360),
+ };
+ TriggerBandwidthQualityScalerTest(frame_configs);
+
+ // When resolution is 640*360, experimental working bitrate range is
+ // [500000,800000] bps. Encoded bitrate is 1059462 > 800000 * 0.95, so it
+ // triggers adapt_up_event_count_.
+ EXPECT_TRUE(handler_->event_.Wait(
+ bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs()));
+ EXPECT_EQ(0, handler_->adapt_down_event_count_);
+ EXPECT_EQ(1, handler_->adapt_up_event_count_);
+}
+
+} // namespace webrtc
diff --git a/rtc_base/experiments/BUILD.gn b/rtc_base/experiments/BUILD.gn
index b0a729a..56d5000 100644
--- a/rtc_base/experiments/BUILD.gn
+++ b/rtc_base/experiments/BUILD.gn
@@ -78,6 +78,21 @@
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
+rtc_library("bandwidth_quality_scaler_settings") {
+ sources = [
+ "bandwidth_quality_scaler_settings.cc",
+ "bandwidth_quality_scaler_settings.h",
+ ]
+ deps = [
+ ":field_trial_parser",
+ "../:rtc_base_approved",
+ "../../api/transport:field_trial_based_config",
+ "../../api/transport:webrtc_key_value_config",
+ "../../system_wrappers:field_trial",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+}
+
rtc_library("quality_scaling_experiment") {
sources = [
"quality_scaling_experiment.cc",
@@ -237,6 +252,7 @@
sources = [
"balanced_degradation_settings_unittest.cc",
+ "bandwidth_quality_scaler_settings_unittest.cc",
"cpu_speed_experiment_unittest.cc",
"encoder_info_settings_unittest.cc",
"field_trial_list_unittest.cc",
@@ -255,6 +271,7 @@
]
deps = [
":balanced_degradation_settings",
+ ":bandwidth_quality_scaler_settings",
":cpu_speed_experiment",
":encoder_info_settings",
":field_trial_parser",
diff --git a/rtc_base/experiments/bandwidth_quality_scaler_settings.cc b/rtc_base/experiments/bandwidth_quality_scaler_settings.cc
new file mode 100644
index 0000000..332ab6b
--- /dev/null
+++ b/rtc_base/experiments/bandwidth_quality_scaler_settings.cc
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "rtc_base/experiments/bandwidth_quality_scaler_settings.h"
+
+#include "api/transport/field_trial_based_config.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+
+BandwidthQualityScalerSettings::BandwidthQualityScalerSettings(
+ const WebRtcKeyValueConfig* const key_value_config)
+ : bitrate_state_update_interval_s_("bitrate_state_update_interval_s_") {
+ ParseFieldTrial(
+ {&bitrate_state_update_interval_s_},
+ key_value_config->Lookup("WebRTC-Video-BandwidthQualityScalerSettings"));
+}
+
+BandwidthQualityScalerSettings
+BandwidthQualityScalerSettings::ParseFromFieldTrials() {
+ FieldTrialBasedConfig field_trial_config;
+ return BandwidthQualityScalerSettings(&field_trial_config);
+}
+
+absl::optional<uint32_t>
+BandwidthQualityScalerSettings::BitrateStateUpdateInterval() const {
+ if (bitrate_state_update_interval_s_ &&
+ bitrate_state_update_interval_s_.Value() <= 0) {
+ RTC_LOG(LS_WARNING)
+ << "Unsupported bitrate_state_update_interval_s_ value, ignored.";
+ return absl::nullopt;
+ }
+ return bitrate_state_update_interval_s_.GetOptional();
+}
+
+} // namespace webrtc
diff --git a/rtc_base/experiments/bandwidth_quality_scaler_settings.h b/rtc_base/experiments/bandwidth_quality_scaler_settings.h
new file mode 100644
index 0000000..959aea5
--- /dev/null
+++ b/rtc_base/experiments/bandwidth_quality_scaler_settings.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef RTC_BASE_EXPERIMENTS_BANDWIDTH_QUALITY_SCALER_SETTINGS_H_
+#define RTC_BASE_EXPERIMENTS_BANDWIDTH_QUALITY_SCALER_SETTINGS_H_
+
+#include "absl/types/optional.h"
+#include "api/transport/webrtc_key_value_config.h"
+#include "rtc_base/experiments/field_trial_parser.h"
+
+namespace webrtc {
+
+class BandwidthQualityScalerSettings final {
+ public:
+ static BandwidthQualityScalerSettings ParseFromFieldTrials();
+
+ absl::optional<uint32_t> BitrateStateUpdateInterval() const;
+
+ private:
+ explicit BandwidthQualityScalerSettings(
+ const WebRtcKeyValueConfig* const key_value_config);
+
+ FieldTrialOptional<uint32_t> bitrate_state_update_interval_s_;
+};
+
+} // namespace webrtc
+
+#endif // RTC_BASE_EXPERIMENTS_BANDWIDTH_QUALITY_SCALER_SETTINGS_H_
diff --git a/rtc_base/experiments/bandwidth_quality_scaler_settings_unittest.cc b/rtc_base/experiments/bandwidth_quality_scaler_settings_unittest.cc
new file mode 100644
index 0000000..fab22ce
--- /dev/null
+++ b/rtc_base/experiments/bandwidth_quality_scaler_settings_unittest.cc
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "rtc_base/experiments/bandwidth_quality_scaler_settings.h"
+
+#include "test/field_trial.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace {
+
+TEST(BandwidthQualityScalerSettingsTest, ValuesNotSetByDefault) {
+ const auto settings = BandwidthQualityScalerSettings::ParseFromFieldTrials();
+ EXPECT_FALSE(settings.BitrateStateUpdateInterval());
+}
+
+TEST(BandwidthQualityScalerSettingsTest, ParseBitrateStateUpdateInterval) {
+ test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-BandwidthQualityScalerSettings/"
+ "bitrate_state_update_interval_s_:100/");
+ EXPECT_EQ(100u, BandwidthQualityScalerSettings::ParseFromFieldTrials()
+ .BitrateStateUpdateInterval());
+}
+
+TEST(BandwidthQualityScalerSettingsTest, ParseAll) {
+ test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-BandwidthQualityScalerSettings/"
+ "bitrate_state_update_interval_s_:100/");
+ EXPECT_EQ(100u, BandwidthQualityScalerSettings::ParseFromFieldTrials()
+ .BitrateStateUpdateInterval());
+}
+
+TEST(BandwidthQualityScalerSettingsTest, DoesNotParseIncorrectValue) {
+ test::ScopedFieldTrials field_trials(
+ "WebRTC-Video-BandwidthQualityScalerSettings/"
+ "bitrate_state_update_interval_s_:??/");
+ const auto settings = BandwidthQualityScalerSettings::ParseFromFieldTrials();
+ EXPECT_FALSE(settings.BitrateStateUpdateInterval());
+}
+
+} // namespace
+} // namespace webrtc
diff --git a/rtc_base/experiments/encoder_info_settings.cc b/rtc_base/experiments/encoder_info_settings.cc
index 9e1a519..b39c684 100644
--- a/rtc_base/experiments/encoder_info_settings.cc
+++ b/rtc_base/experiments/encoder_info_settings.cc
@@ -29,7 +29,7 @@
}
return result;
}
-
+constexpr float kDefaultMinBitratebps = 30000;
} // namespace
// Default bitrate limits for simulcast with one active stream:
@@ -63,6 +63,98 @@
return info.GetEncoderBitrateLimitsForResolution(frame_size_pixels);
}
+// Return the suitable bitrate limits for specified resolution when qp is
+// untrusted, they are experimental values.
+// TODO(bugs.webrtc.org/12942): Maybe we need to add other codecs(VP8/VP9)
+// experimental values.
+std::vector<VideoEncoder::ResolutionBitrateLimits>
+EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted() {
+ // Specific limits for H264/AVC
+ return {{0 * 0, 0, 0, 0},
+ {320 * 180, 0, 30000, 300000},
+ {480 * 270, 300000, 30000, 500000},
+ {640 * 360, 500000, 30000, 800000},
+ {960 * 540, 800000, 30000, 1500000},
+ {1280 * 720, 1500000, 30000, 2500000},
+ {1920 * 1080, 2500000, 30000, 4000000}};
+}
+
+// Through linear interpolation, return the bitrate limit corresponding to the
+// specified |frame_size_pixels|.
+absl::optional<VideoEncoder::ResolutionBitrateLimits>
+EncoderInfoSettings::GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
+ absl::optional<int> frame_size_pixels,
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>&
+ resolution_bitrate_limits) {
+ if (!frame_size_pixels.has_value() || frame_size_pixels.value() <= 0) {
+ return absl::nullopt;
+ }
+
+ std::vector<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
+ resolution_bitrate_limits;
+
+ // Sort the list of bitrate limits by resolution.
+ sort(bitrate_limits.begin(), bitrate_limits.end(),
+ [](const VideoEncoder::ResolutionBitrateLimits& lhs,
+ const VideoEncoder::ResolutionBitrateLimits& rhs) {
+ return lhs.frame_size_pixels < rhs.frame_size_pixels;
+ });
+
+ if (bitrate_limits.empty()) {
+ return absl::nullopt;
+ }
+
+ int interpolation_index = -1;
+ for (size_t i = 0; i < bitrate_limits.size(); ++i) {
+ if (bitrate_limits[i].frame_size_pixels >= frame_size_pixels.value()) {
+ interpolation_index = i;
+ break;
+ }
+ }
+
+ // -1 means that the maximum resolution is exceeded, we will select the
+ // largest data as the return result.
+ if (interpolation_index == -1) {
+ return *bitrate_limits.rbegin();
+ }
+
+ // If we have a matching resolution, return directly without interpolation.
+ if (bitrate_limits[interpolation_index].frame_size_pixels ==
+ frame_size_pixels.value()) {
+ return bitrate_limits[interpolation_index];
+ }
+
+ // No matching resolution, do a linear interpolate.
+ int lower_pixel_count =
+ bitrate_limits[interpolation_index - 1].frame_size_pixels;
+ int upper_pixel_count = bitrate_limits[interpolation_index].frame_size_pixels;
+ float alpha = (frame_size_pixels.value() - lower_pixel_count) * 1.0 /
+ (upper_pixel_count - lower_pixel_count);
+ int min_start_bitrate_bps = static_cast<int>(
+ bitrate_limits[interpolation_index].min_start_bitrate_bps * alpha +
+ bitrate_limits[interpolation_index - 1].min_start_bitrate_bps *
+ (1.0 - alpha));
+ int max_bitrate_bps = static_cast<int>(
+ bitrate_limits[interpolation_index].max_bitrate_bps * alpha +
+ bitrate_limits[interpolation_index - 1].max_bitrate_bps * (1.0 - alpha));
+
+ if (max_bitrate_bps >= min_start_bitrate_bps) {
+ return VideoEncoder::ResolutionBitrateLimits(
+ frame_size_pixels.value(), min_start_bitrate_bps, kDefaultMinBitratebps,
+ max_bitrate_bps);
+ } else {
+ RTC_LOG(LS_WARNING)
+ << "BitRate interpolation calculating result is abnormal. "
+ << " lower_pixel_count = " << lower_pixel_count
+ << " upper_pixel_count = " << upper_pixel_count
+ << " frame_size_pixels = " << frame_size_pixels.value()
+ << " min_start_bitrate_bps = " << min_start_bitrate_bps
+ << " min_bitrate_bps = " << kDefaultMinBitratebps
+ << " max_bitrate_bps = " << max_bitrate_bps;
+ return absl::nullopt;
+ }
+}
+
EncoderInfoSettings::EncoderInfoSettings(std::string name)
: requested_resolution_alignment_("requested_resolution_alignment"),
apply_alignment_to_all_simulcast_layers_(
diff --git a/rtc_base/experiments/encoder_info_settings.h b/rtc_base/experiments/encoder_info_settings.h
index 9cbb587..e4dc459 100644
--- a/rtc_base/experiments/encoder_info_settings.h
+++ b/rtc_base/experiments/encoder_info_settings.h
@@ -48,6 +48,15 @@
GetDefaultSinglecastBitrateLimitsForResolution(VideoCodecType codec_type,
int frame_size_pixels);
+ static std::vector<VideoEncoder::ResolutionBitrateLimits>
+ GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted();
+
+ static absl::optional<VideoEncoder::ResolutionBitrateLimits>
+ GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
+ absl::optional<int> frame_size_pixels,
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>&
+ resolution_bitrate_limits);
+
protected:
explicit EncoderInfoSettings(std::string name);
diff --git a/video/adaptation/BUILD.gn b/video/adaptation/BUILD.gn
index 20a2370..11962c0 100644
--- a/video/adaptation/BUILD.gn
+++ b/video/adaptation/BUILD.gn
@@ -12,6 +12,8 @@
sources = [
"balanced_constraint.cc",
"balanced_constraint.h",
+ "bandwidth_quality_scaler_resource.cc",
+ "bandwidth_quality_scaler_resource.h",
"bitrate_constraint.cc",
"bitrate_constraint.h",
"encode_usage_resource.cc",
diff --git a/video/adaptation/bandwidth_quality_scaler_resource.cc b/video/adaptation/bandwidth_quality_scaler_resource.cc
new file mode 100644
index 0000000..6f49135
--- /dev/null
+++ b/video/adaptation/bandwidth_quality_scaler_resource.cc
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/adaptation/bandwidth_quality_scaler_resource.h"
+
+#include <utility>
+
+#include "rtc_base/checks.h"
+#include "rtc_base/experiments/balanced_degradation_settings.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/ref_counted_object.h"
+#include "rtc_base/task_utils/to_queued_task.h"
+#include "rtc_base/time_utils.h"
+
+namespace webrtc {
+
+// static
+rtc::scoped_refptr<BandwidthQualityScalerResource>
+BandwidthQualityScalerResource::Create() {
+ return rtc::make_ref_counted<BandwidthQualityScalerResource>();
+}
+
+BandwidthQualityScalerResource::BandwidthQualityScalerResource()
+ : VideoStreamEncoderResource("BandwidthQualityScalerResource"),
+ bandwidth_quality_scaler_(nullptr) {}
+
+BandwidthQualityScalerResource::~BandwidthQualityScalerResource() {
+ RTC_DCHECK(!bandwidth_quality_scaler_);
+}
+
+bool BandwidthQualityScalerResource::is_started() const {
+ RTC_DCHECK_RUN_ON(encoder_queue());
+ return bandwidth_quality_scaler_.get();
+}
+
+void BandwidthQualityScalerResource::StartCheckForOveruse(
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>&
+ resolution_bitrate_limits) {
+ RTC_DCHECK_RUN_ON(encoder_queue());
+ RTC_DCHECK(!is_started());
+ bandwidth_quality_scaler_ = std::make_unique<BandwidthQualityScaler>(this);
+
+ // If the configuration parameters more than one, we should define and
+ // declare the function BandwidthQualityScaler::Initialize() and call it.
+ bandwidth_quality_scaler_->SetResolutionBitrateLimits(
+ resolution_bitrate_limits);
+}
+
+void BandwidthQualityScalerResource::StopCheckForOveruse() {
+ RTC_DCHECK_RUN_ON(encoder_queue());
+ RTC_DCHECK(is_started());
+ // Ensure we have no pending callbacks. This makes it safe to destroy the
+ // BandwidthQualityScaler and even task queues with tasks in-flight.
+ bandwidth_quality_scaler_.reset();
+}
+
+void BandwidthQualityScalerResource::OnReportUsageBandwidthHigh() {
+ OnResourceUsageStateMeasured(ResourceUsageState::kOveruse);
+}
+
+void BandwidthQualityScalerResource::OnReportUsageBandwidthLow() {
+ OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse);
+}
+
+void BandwidthQualityScalerResource::OnEncodeCompleted(
+ const EncodedImage& encoded_image,
+ int64_t time_sent_in_us,
+ int64_t encoded_image_size_bytes) {
+ RTC_DCHECK_RUN_ON(encoder_queue());
+
+ if (bandwidth_quality_scaler_) {
+ bandwidth_quality_scaler_->ReportEncodeInfo(
+ encoded_image_size_bytes, time_sent_in_us / 1000,
+ encoded_image._encodedWidth, encoded_image._encodedHeight);
+ }
+}
+
+} // namespace webrtc
diff --git a/video/adaptation/bandwidth_quality_scaler_resource.h b/video/adaptation/bandwidth_quality_scaler_resource.h
new file mode 100644
index 0000000..f33ce97
--- /dev/null
+++ b/video/adaptation/bandwidth_quality_scaler_resource.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_
+#define VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_
+
+#include <memory>
+#include <queue>
+#include <string>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/scoped_refptr.h"
+#include "api/video/video_adaptation_reason.h"
+#include "api/video_codecs/video_encoder.h"
+#include "call/adaptation/degradation_preference_provider.h"
+#include "call/adaptation/resource_adaptation_processor_interface.h"
+#include "modules/video_coding/utility/bandwidth_quality_scaler.h"
+#include "rtc_base/ref_counted_object.h"
+#include "rtc_base/task_queue.h"
+#include "video/adaptation/video_stream_encoder_resource.h"
+
+namespace webrtc {
+
+// Handles interaction with the BandwidthQualityScaler.
+class BandwidthQualityScalerResource
+ : public VideoStreamEncoderResource,
+ public BandwidthQualityScalerUsageHandlerInterface {
+ public:
+ static rtc::scoped_refptr<BandwidthQualityScalerResource> Create();
+
+ BandwidthQualityScalerResource();
+ ~BandwidthQualityScalerResource() override;
+
+ bool is_started() const;
+
+ void OnEncodeCompleted(const EncodedImage& encoded_image,
+ int64_t time_sent_in_us,
+ int64_t encoded_image_size_bytes);
+
+ void StartCheckForOveruse(
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>&
+ resolution_bitrate_limits);
+ void StopCheckForOveruse();
+
+ // BandwidthScalerQpUsageHandlerInterface implementation.
+ void OnReportUsageBandwidthHigh() override;
+ void OnReportUsageBandwidthLow() override;
+
+ private:
+ std::unique_ptr<BandwidthQualityScaler> bandwidth_quality_scaler_
+ RTC_GUARDED_BY(encoder_queue());
+};
+
+} // namespace webrtc
+
+#endif // VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_
diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc
index 0b2fa89..6a1e921 100644
--- a/video/adaptation/video_stream_encoder_resource_manager.cc
+++ b/video/adaptation/video_stream_encoder_resource_manager.cc
@@ -275,6 +275,8 @@
EncodeUsageResource::Create(std::move(overuse_detector))),
quality_scaler_resource_(QualityScalerResource::Create()),
pixel_limit_resource_(nullptr),
+ bandwidth_quality_scaler_resource_(
+ BandwidthQualityScalerResource::Create()),
encoder_queue_(nullptr),
input_state_provider_(input_state_provider),
adaptation_processor_(nullptr),
@@ -307,6 +309,8 @@
encoder_queue_ = encoder_queue;
encode_usage_resource_->RegisterEncoderTaskQueue(encoder_queue_->Get());
quality_scaler_resource_->RegisterEncoderTaskQueue(encoder_queue_->Get());
+ bandwidth_quality_scaler_resource_->RegisterEncoderTaskQueue(
+ encoder_queue_->Get());
}
void VideoStreamEncoderResourceManager::SetAdaptationProcessor(
@@ -385,6 +389,10 @@
RemoveResource(pixel_limit_resource_);
pixel_limit_resource_ = nullptr;
}
+ if (bandwidth_quality_scaler_resource_->is_started()) {
+ bandwidth_quality_scaler_resource_->StopCheckForOveruse();
+ RemoveResource(bandwidth_quality_scaler_resource_);
+ }
}
void VideoStreamEncoderResourceManager::AddResource(
@@ -491,7 +499,8 @@
void VideoStreamEncoderResourceManager::OnEncodeCompleted(
const EncodedImage& encoded_image,
int64_t time_sent_in_us,
- absl::optional<int> encode_duration_us) {
+ absl::optional<int> encode_duration_us,
+ DataSize frame_size) {
RTC_DCHECK_RUN_ON(encoder_queue_);
// Inform `encode_usage_resource_` of the encode completed event.
uint32_t timestamp = encoded_image.Timestamp();
@@ -500,6 +509,8 @@
encode_usage_resource_->OnEncodeCompleted(
timestamp, time_sent_in_us, capture_time_us, encode_duration_us);
quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us);
+ bandwidth_quality_scaler_resource_->OnEncodeCompleted(
+ encoded_image, time_sent_in_us, frame_size.bytes());
}
void VideoStreamEncoderResourceManager::OnFrameDropped(
@@ -556,6 +567,29 @@
initial_frame_dropper_->OnQualityScalerSettingsUpdated();
}
+void VideoStreamEncoderResourceManager::UpdateBandwidthQualityScalerSettings(
+ bool bandwidth_quality_scaling_allowed,
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>&
+ resolution_bitrate_limits) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+
+ if (!bandwidth_quality_scaling_allowed) {
+ if (bandwidth_quality_scaler_resource_->is_started()) {
+ bandwidth_quality_scaler_resource_->StopCheckForOveruse();
+ RemoveResource(bandwidth_quality_scaler_resource_);
+ }
+ } else {
+ if (!bandwidth_quality_scaler_resource_->is_started()) {
+ // Before executing "StartCheckForOveruse",we must execute "AddResource"
+ // firstly,because it can make the listener valid.
+ AddResource(bandwidth_quality_scaler_resource_,
+ webrtc::VideoAdaptationReason::kQuality);
+ bandwidth_quality_scaler_resource_->StartCheckForOveruse(
+ resolution_bitrate_limits);
+ }
+ }
+}
+
void VideoStreamEncoderResourceManager::ConfigureQualityScaler(
const VideoEncoder::EncoderInfo& encoder_info) {
RTC_DCHECK_RUN_ON(encoder_queue_);
@@ -601,6 +635,20 @@
UpdateStatsAdaptationSettings();
}
+void VideoStreamEncoderResourceManager::ConfigureBandwidthQualityScaler(
+ const VideoEncoder::EncoderInfo& encoder_info) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ const bool bandwidth_quality_scaling_allowed =
+ IsResolutionScalingEnabled(degradation_preference_) &&
+ (encoder_settings_.has_value() &&
+ encoder_settings_->encoder_config().is_quality_scaling_allowed) &&
+ !encoder_info.is_qp_trusted.value_or(true);
+
+ UpdateBandwidthQualityScalerSettings(bandwidth_quality_scaling_allowed,
+ encoder_info.resolution_bitrate_limits);
+ UpdateStatsAdaptationSettings();
+}
+
VideoAdaptationReason VideoStreamEncoderResourceManager::GetReasonFromResource(
rtc::scoped_refptr<Resource> resource) const {
RTC_DCHECK_RUN_ON(encoder_queue_);
@@ -727,7 +775,8 @@
IsFramerateScalingEnabled(degradation_preference_));
VideoStreamEncoderObserver::AdaptationSettings quality_settings =
- quality_scaler_resource_->is_started()
+ (quality_scaler_resource_->is_started() ||
+ bandwidth_quality_scaler_resource_->is_started())
? cpu_settings
: VideoStreamEncoderObserver::AdaptationSettings();
encoder_stats_observer_->UpdateAdaptationSettings(cpu_settings,
diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h
index 2f5dfcd..f1bc885 100644
--- a/video/adaptation/video_stream_encoder_resource_manager.h
+++ b/video/adaptation/video_stream_encoder_resource_manager.h
@@ -43,6 +43,7 @@
#include "rtc_base/thread_annotations.h"
#include "system_wrappers/include/clock.h"
#include "video/adaptation/balanced_constraint.h"
+#include "video/adaptation/bandwidth_quality_scaler_resource.h"
#include "video/adaptation/bitrate_constraint.h"
#include "video/adaptation/encode_usage_resource.h"
#include "video/adaptation/overuse_frame_detector.h"
@@ -109,6 +110,8 @@
// TODO(https://crbug.com/webrtc/11338): This can be made private if we
// configure on SetDegredationPreference and SetEncoderSettings.
void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info);
+ void ConfigureBandwidthQualityScaler(
+ const VideoEncoder::EncoderInfo& encoder_info);
// Methods corresponding to different points in the encoding pipeline.
void OnFrameDroppedDueToSize();
@@ -117,7 +120,8 @@
int64_t time_when_first_seen_us);
void OnEncodeCompleted(const EncodedImage& encoded_image,
int64_t time_sent_in_us,
- absl::optional<int> encode_duration_us);
+ absl::optional<int> encode_duration_us,
+ DataSize frame_size);
void OnFrameDropped(EncodedImageCallback::DropReason reason);
// Resources need to be mapped to an AdaptReason (kCpu or kQuality) in order
@@ -166,6 +170,11 @@
void UpdateQualityScalerSettings(
absl::optional<VideoEncoder::QpThresholds> qp_thresholds);
+ void UpdateBandwidthQualityScalerSettings(
+ bool bandwidth_quality_scaling_allowed,
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>&
+ resolution_bitrate_limits);
+
void UpdateStatsAdaptationSettings() const;
static std::string ActiveCountsToString(
@@ -180,6 +189,8 @@
const rtc::scoped_refptr<EncodeUsageResource> encode_usage_resource_;
const rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_;
rtc::scoped_refptr<PixelLimitResource> pixel_limit_resource_;
+ const rtc::scoped_refptr<BandwidthQualityScalerResource>
+ bandwidth_quality_scaler_resource_;
rtc::TaskQueue* encoder_queue_;
VideoStreamInputStateProvider* const input_state_provider_
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index 8ea6750..8774ff7 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -73,6 +73,8 @@
// Animation is capped to 720p.
constexpr int kMaxAnimationPixels = 1280 * 720;
+constexpr int kDefaultMinScreenSharebps = 1200000;
+
bool RequiresEncoderReset(const VideoCodec& prev_send_codec,
const VideoCodec& new_send_codec,
bool was_encode_called_since_last_initialization) {
@@ -783,6 +785,8 @@
if (encoder_) {
stream_resource_manager_.ConfigureQualityScaler(
encoder_->GetEncoderInfo());
+ stream_resource_manager_.ConfigureBandwidthQualityScaler(
+ encoder_->GetEncoderInfo());
}
});
}
@@ -914,50 +918,97 @@
crop_width_ = last_frame_info_->width - highest_stream_width;
crop_height_ = last_frame_info_->height - highest_stream_height;
- absl::optional<VideoEncoder::ResolutionBitrateLimits> encoder_bitrate_limits =
- encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution(
- last_frame_info_->width * last_frame_info_->height);
+ if (!encoder_->GetEncoderInfo().is_qp_trusted.value_or(true)) {
+ // when qp is not trusted, we priorities to using the
+ // |resolution_bitrate_limits| provided by the decoder.
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>& bitrate_limits =
+ encoder_->GetEncoderInfo().resolution_bitrate_limits.empty()
+ ? EncoderInfoSettings::
+ GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted()
+ : encoder_->GetEncoderInfo().resolution_bitrate_limits;
- if (encoder_bitrate_limits) {
- if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) {
- // Bitrate limits can be set by app (in SDP or RtpEncodingParameters)
- // or/and can be provided by encoder. In presence of both set of limits,
- // the final set is derived as their intersection.
- int min_bitrate_bps;
- if (encoder_config_.simulcast_layers.empty() ||
- encoder_config_.simulcast_layers[0].min_bitrate_bps <= 0) {
- min_bitrate_bps = encoder_bitrate_limits->min_bitrate_bps;
- } else {
- min_bitrate_bps = std::max(encoder_bitrate_limits->min_bitrate_bps,
- streams.back().min_bitrate_bps);
- }
+ // For BandwidthQualityScaler, its implement based on a certain pixel_count
+ // correspond a certain bps interval. In fact, WebRTC default max_bps is
+ // 2500Kbps when width * height > 960 * 540. For example, we assume:
+ // 1.the camera support 1080p.
+ // 2.ResolutionBitrateLimits set 720p bps interval is [1500Kbps,2000Kbps].
+ // 3.ResolutionBitrateLimits set 1080p bps interval is [2000Kbps,2500Kbps].
+ // We will never be stable at 720p due to actual encoding bps of 720p and
+ // 1080p are both 2500Kbps. So it is necessary to do a linear interpolation
+ // to get a certain bitrate for certain pixel_count. It also doesn't work
+ // for 960*540 and 640*520, we will nerver be stable at 640*520 due to their
+ // |target_bitrate_bps| are both 2000Kbps.
+ absl::optional<VideoEncoder::ResolutionBitrateLimits>
+ qp_untrusted_bitrate_limit = EncoderInfoSettings::
+ GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
+ last_frame_info_->width * last_frame_info_->height,
+ bitrate_limits);
- int max_bitrate_bps;
- // We don't check encoder_config_.simulcast_layers[0].max_bitrate_bps
- // here since encoder_config_.max_bitrate_bps is derived from it (as
- // well as from other inputs).
- if (encoder_config_.max_bitrate_bps <= 0) {
- max_bitrate_bps = encoder_bitrate_limits->max_bitrate_bps;
- } else {
- max_bitrate_bps = std::min(encoder_bitrate_limits->max_bitrate_bps,
- streams.back().max_bitrate_bps);
- }
-
- if (min_bitrate_bps < max_bitrate_bps) {
- streams.back().min_bitrate_bps = min_bitrate_bps;
- streams.back().max_bitrate_bps = max_bitrate_bps;
+ if (qp_untrusted_bitrate_limit) {
+ // bandwidth_quality_scaler is only used for singlecast.
+ if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) {
+ streams.back().min_bitrate_bps =
+ qp_untrusted_bitrate_limit->min_bitrate_bps;
+ streams.back().max_bitrate_bps =
+ qp_untrusted_bitrate_limit->max_bitrate_bps;
+ // If it is screen share mode, the minimum value of max_bitrate should
+ // be greater than/equal to 1200kbps.
+ if (encoder_config_.content_type ==
+ VideoEncoderConfig::ContentType::kScreen) {
+ streams.back().max_bitrate_bps = std::max(
+ streams.back().max_bitrate_bps, kDefaultMinScreenSharebps);
+ }
streams.back().target_bitrate_bps =
- std::min(streams.back().target_bitrate_bps,
- encoder_bitrate_limits->max_bitrate_bps);
- } else {
- RTC_LOG(LS_WARNING)
- << "Bitrate limits provided by encoder"
- << " (min=" << encoder_bitrate_limits->min_bitrate_bps
- << ", max=" << encoder_bitrate_limits->max_bitrate_bps
- << ") do not intersect with limits set by app"
- << " (min=" << streams.back().min_bitrate_bps
- << ", max=" << encoder_config_.max_bitrate_bps
- << "). The app bitrate limits will be used.";
+ qp_untrusted_bitrate_limit->max_bitrate_bps;
+ }
+ }
+ } else {
+ absl::optional<VideoEncoder::ResolutionBitrateLimits>
+ encoder_bitrate_limits =
+ encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution(
+ last_frame_info_->width * last_frame_info_->height);
+
+ if (encoder_bitrate_limits) {
+ if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) {
+ // Bitrate limits can be set by app (in SDP or RtpEncodingParameters)
+ // or/and can be provided by encoder. In presence of both set of
+ // limits, the final set is derived as their intersection.
+ int min_bitrate_bps;
+ if (encoder_config_.simulcast_layers.empty() ||
+ encoder_config_.simulcast_layers[0].min_bitrate_bps <= 0) {
+ min_bitrate_bps = encoder_bitrate_limits->min_bitrate_bps;
+ } else {
+ min_bitrate_bps = std::max(encoder_bitrate_limits->min_bitrate_bps,
+ streams.back().min_bitrate_bps);
+ }
+
+ int max_bitrate_bps;
+ // We don't check encoder_config_.simulcast_layers[0].max_bitrate_bps
+ // here since encoder_config_.max_bitrate_bps is derived from it (as
+ // well as from other inputs).
+ if (encoder_config_.max_bitrate_bps <= 0) {
+ max_bitrate_bps = encoder_bitrate_limits->max_bitrate_bps;
+ } else {
+ max_bitrate_bps = std::min(encoder_bitrate_limits->max_bitrate_bps,
+ streams.back().max_bitrate_bps);
+ }
+
+ if (min_bitrate_bps < max_bitrate_bps) {
+ streams.back().min_bitrate_bps = min_bitrate_bps;
+ streams.back().max_bitrate_bps = max_bitrate_bps;
+ streams.back().target_bitrate_bps =
+ std::min(streams.back().target_bitrate_bps,
+ encoder_bitrate_limits->max_bitrate_bps);
+ } else {
+ RTC_LOG(LS_WARNING)
+ << "Bitrate limits provided by encoder"
+ << " (min=" << encoder_bitrate_limits->min_bitrate_bps
+ << ", max=" << encoder_bitrate_limits->max_bitrate_bps
+ << ") do not intersect with limits set by app"
+ << " (min=" << streams.back().min_bitrate_bps
+ << ", max=" << encoder_config_.max_bitrate_bps
+ << "). The app bitrate limits will be used.";
+ }
}
}
}
@@ -1195,6 +1246,7 @@
encoder_config_.min_transmit_bitrate_bps);
stream_resource_manager_.ConfigureQualityScaler(info);
+ stream_resource_manager_.ConfigureBandwidthQualityScaler(info);
}
void VideoStreamEncoder::OnEncoderSettingsChanged() {
@@ -2140,7 +2192,7 @@
}
stream_resource_manager_.OnEncodeCompleted(encoded_image, time_sent_us,
- encode_duration_us);
+ encode_duration_us, frame_size);
if (bitrate_adjuster_) {
bitrate_adjuster_->OnEncodedFrame(
frame_size, encoded_image.SpatialIndex().value_or(0), temporal_index);
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index d643419..d9f1136 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -8210,36 +8210,6 @@
video_stream_encoder_->Stop();
}
-TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
- VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
-
- // Disable scaling settings in encoder info.
- fake_encoder_.SetQualityScaling(false);
- // Set QP not trusted in encoder info.
- fake_encoder_.SetIsQpTrusted(false);
- // Enable quality scaling in encoder config.
- video_encoder_config.is_quality_scaling_allowed = true;
- ConfigureEncoder(std::move(video_encoder_config));
-
- video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
- DataRate::BitsPerSec(kTargetBitrateBps),
- DataRate::BitsPerSec(kTargetBitrateBps),
- DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
-
- test::FrameForwarder source;
- video_stream_encoder_->SetSource(
- &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
- EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
- EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
-
- source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
- WaitForEncodedFrame(1);
- video_stream_encoder_->TriggerQualityLow();
- EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
-
- video_stream_encoder_->Stop();
-}
-
TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
@@ -8270,6 +8240,147 @@
video_stream_encoder_->Stop();
}
+TEST_F(VideoStreamEncoderTest,
+ QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(true);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = false;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(false);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = false;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(false);
+
+ const int MinEncBitrateKbps = 30;
+ const int MaxEncBitrateKbps = 100;
+ const int MinStartBitrateKbp = 50;
+ const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
+ /*frame_size_pixels=*/codec_width_ * codec_height_,
+ /*min_start_bitrate_bps=*/MinStartBitrateKbp,
+ /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
+ /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
+
+ fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
+ video_encoder_config.simulcast_layers[0].min_bitrate_bps =
+ MinEncBitrateKbps * 1000;
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(
+ MaxEncBitrateKbps,
+ static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
+ EXPECT_EQ(
+ MinEncBitrateKbps,
+ static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(false);
+
+ absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
+ EncoderInfoSettings::
+ GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
+ codec_width_ * codec_height_,
+ EncoderInfoSettings::
+ GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
+ EXPECT_TRUE(suitable_bitrate_limit.has_value());
+
+ const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
+ const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
+ const int TargetEncBitrate = MaxEncBitrate;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(TargetEncBitrate),
+ DataRate::BitsPerSec(TargetEncBitrate),
+ DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = MaxEncBitrate;
+ video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(
+ MaxEncBitrate / 1000,
+ static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
+ EXPECT_EQ(
+ MinEncBitrate / 1000,
+ static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
+
+ video_stream_encoder_->Stop();
+}
+
#if !defined(WEBRTC_IOS)
// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
// disabled by default on iOS.
@@ -8330,6 +8441,102 @@
video_stream_encoder_->Stop();
}
+
+TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Set QP not trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(false);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = true;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ // When quality_scaler doesn't work and is_quality_scaling_allowed is
+ // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(true);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = true;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ // bandwidth_quality_scaler isn't working, but quality_scaler is working.
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(false);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = true;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
#endif
// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
