video: Implement bandwidth based scaler
The |slice_qp_detla| reported by the hardware is not credible, which
causing the quality scaler cannot work properly,the resolution cannot
be adjusted correctly.
To fix this issue, this CL implements a bandwidth scaler which is used
for adjust resolution, this scaler will be used when QP based quality
scaler is not working due to untrusted QP reported by HW AVC encoder.
Bug: webrtc:12942
Change-Id: I2fc5f07a5400ec7e5ead2c2c502faee84d7f2a76
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/228860
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Commit-Queue: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35120}
diff --git a/video/adaptation/BUILD.gn b/video/adaptation/BUILD.gn
index 20a2370..11962c0 100644
--- a/video/adaptation/BUILD.gn
+++ b/video/adaptation/BUILD.gn
@@ -12,6 +12,8 @@
sources = [
"balanced_constraint.cc",
"balanced_constraint.h",
+ "bandwidth_quality_scaler_resource.cc",
+ "bandwidth_quality_scaler_resource.h",
"bitrate_constraint.cc",
"bitrate_constraint.h",
"encode_usage_resource.cc",
diff --git a/video/adaptation/bandwidth_quality_scaler_resource.cc b/video/adaptation/bandwidth_quality_scaler_resource.cc
new file mode 100644
index 0000000..6f49135
--- /dev/null
+++ b/video/adaptation/bandwidth_quality_scaler_resource.cc
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/adaptation/bandwidth_quality_scaler_resource.h"
+
+#include <utility>
+
+#include "rtc_base/checks.h"
+#include "rtc_base/experiments/balanced_degradation_settings.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/ref_counted_object.h"
+#include "rtc_base/task_utils/to_queued_task.h"
+#include "rtc_base/time_utils.h"
+
+namespace webrtc {
+
+// static
+rtc::scoped_refptr<BandwidthQualityScalerResource>
+BandwidthQualityScalerResource::Create() {
+ return rtc::make_ref_counted<BandwidthQualityScalerResource>();
+}
+
+BandwidthQualityScalerResource::BandwidthQualityScalerResource()
+ : VideoStreamEncoderResource("BandwidthQualityScalerResource"),
+ bandwidth_quality_scaler_(nullptr) {}
+
+BandwidthQualityScalerResource::~BandwidthQualityScalerResource() {
+ RTC_DCHECK(!bandwidth_quality_scaler_);
+}
+
+bool BandwidthQualityScalerResource::is_started() const {
+ RTC_DCHECK_RUN_ON(encoder_queue());
+ return bandwidth_quality_scaler_.get();
+}
+
+void BandwidthQualityScalerResource::StartCheckForOveruse(
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>&
+ resolution_bitrate_limits) {
+ RTC_DCHECK_RUN_ON(encoder_queue());
+ RTC_DCHECK(!is_started());
+ bandwidth_quality_scaler_ = std::make_unique<BandwidthQualityScaler>(this);
+
+ // If the configuration parameters more than one, we should define and
+ // declare the function BandwidthQualityScaler::Initialize() and call it.
+ bandwidth_quality_scaler_->SetResolutionBitrateLimits(
+ resolution_bitrate_limits);
+}
+
+void BandwidthQualityScalerResource::StopCheckForOveruse() {
+ RTC_DCHECK_RUN_ON(encoder_queue());
+ RTC_DCHECK(is_started());
+ // Ensure we have no pending callbacks. This makes it safe to destroy the
+ // BandwidthQualityScaler and even task queues with tasks in-flight.
+ bandwidth_quality_scaler_.reset();
+}
+
+void BandwidthQualityScalerResource::OnReportUsageBandwidthHigh() {
+ OnResourceUsageStateMeasured(ResourceUsageState::kOveruse);
+}
+
+void BandwidthQualityScalerResource::OnReportUsageBandwidthLow() {
+ OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse);
+}
+
+void BandwidthQualityScalerResource::OnEncodeCompleted(
+ const EncodedImage& encoded_image,
+ int64_t time_sent_in_us,
+ int64_t encoded_image_size_bytes) {
+ RTC_DCHECK_RUN_ON(encoder_queue());
+
+ if (bandwidth_quality_scaler_) {
+ bandwidth_quality_scaler_->ReportEncodeInfo(
+ encoded_image_size_bytes, time_sent_in_us / 1000,
+ encoded_image._encodedWidth, encoded_image._encodedHeight);
+ }
+}
+
+} // namespace webrtc
diff --git a/video/adaptation/bandwidth_quality_scaler_resource.h b/video/adaptation/bandwidth_quality_scaler_resource.h
new file mode 100644
index 0000000..f33ce97
--- /dev/null
+++ b/video/adaptation/bandwidth_quality_scaler_resource.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_
+#define VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_
+
+#include <memory>
+#include <queue>
+#include <string>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/scoped_refptr.h"
+#include "api/video/video_adaptation_reason.h"
+#include "api/video_codecs/video_encoder.h"
+#include "call/adaptation/degradation_preference_provider.h"
+#include "call/adaptation/resource_adaptation_processor_interface.h"
+#include "modules/video_coding/utility/bandwidth_quality_scaler.h"
+#include "rtc_base/ref_counted_object.h"
+#include "rtc_base/task_queue.h"
+#include "video/adaptation/video_stream_encoder_resource.h"
+
+namespace webrtc {
+
+// Handles interaction with the BandwidthQualityScaler.
+class BandwidthQualityScalerResource
+ : public VideoStreamEncoderResource,
+ public BandwidthQualityScalerUsageHandlerInterface {
+ public:
+ static rtc::scoped_refptr<BandwidthQualityScalerResource> Create();
+
+ BandwidthQualityScalerResource();
+ ~BandwidthQualityScalerResource() override;
+
+ bool is_started() const;
+
+ void OnEncodeCompleted(const EncodedImage& encoded_image,
+ int64_t time_sent_in_us,
+ int64_t encoded_image_size_bytes);
+
+ void StartCheckForOveruse(
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>&
+ resolution_bitrate_limits);
+ void StopCheckForOveruse();
+
+ // BandwidthScalerQpUsageHandlerInterface implementation.
+ void OnReportUsageBandwidthHigh() override;
+ void OnReportUsageBandwidthLow() override;
+
+ private:
+ std::unique_ptr<BandwidthQualityScaler> bandwidth_quality_scaler_
+ RTC_GUARDED_BY(encoder_queue());
+};
+
+} // namespace webrtc
+
+#endif // VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_
diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc
index 0b2fa89..6a1e921 100644
--- a/video/adaptation/video_stream_encoder_resource_manager.cc
+++ b/video/adaptation/video_stream_encoder_resource_manager.cc
@@ -275,6 +275,8 @@
EncodeUsageResource::Create(std::move(overuse_detector))),
quality_scaler_resource_(QualityScalerResource::Create()),
pixel_limit_resource_(nullptr),
+ bandwidth_quality_scaler_resource_(
+ BandwidthQualityScalerResource::Create()),
encoder_queue_(nullptr),
input_state_provider_(input_state_provider),
adaptation_processor_(nullptr),
@@ -307,6 +309,8 @@
encoder_queue_ = encoder_queue;
encode_usage_resource_->RegisterEncoderTaskQueue(encoder_queue_->Get());
quality_scaler_resource_->RegisterEncoderTaskQueue(encoder_queue_->Get());
+ bandwidth_quality_scaler_resource_->RegisterEncoderTaskQueue(
+ encoder_queue_->Get());
}
void VideoStreamEncoderResourceManager::SetAdaptationProcessor(
@@ -385,6 +389,10 @@
RemoveResource(pixel_limit_resource_);
pixel_limit_resource_ = nullptr;
}
+ if (bandwidth_quality_scaler_resource_->is_started()) {
+ bandwidth_quality_scaler_resource_->StopCheckForOveruse();
+ RemoveResource(bandwidth_quality_scaler_resource_);
+ }
}
void VideoStreamEncoderResourceManager::AddResource(
@@ -491,7 +499,8 @@
void VideoStreamEncoderResourceManager::OnEncodeCompleted(
const EncodedImage& encoded_image,
int64_t time_sent_in_us,
- absl::optional<int> encode_duration_us) {
+ absl::optional<int> encode_duration_us,
+ DataSize frame_size) {
RTC_DCHECK_RUN_ON(encoder_queue_);
// Inform `encode_usage_resource_` of the encode completed event.
uint32_t timestamp = encoded_image.Timestamp();
@@ -500,6 +509,8 @@
encode_usage_resource_->OnEncodeCompleted(
timestamp, time_sent_in_us, capture_time_us, encode_duration_us);
quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us);
+ bandwidth_quality_scaler_resource_->OnEncodeCompleted(
+ encoded_image, time_sent_in_us, frame_size.bytes());
}
void VideoStreamEncoderResourceManager::OnFrameDropped(
@@ -556,6 +567,29 @@
initial_frame_dropper_->OnQualityScalerSettingsUpdated();
}
+void VideoStreamEncoderResourceManager::UpdateBandwidthQualityScalerSettings(
+ bool bandwidth_quality_scaling_allowed,
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>&
+ resolution_bitrate_limits) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+
+ if (!bandwidth_quality_scaling_allowed) {
+ if (bandwidth_quality_scaler_resource_->is_started()) {
+ bandwidth_quality_scaler_resource_->StopCheckForOveruse();
+ RemoveResource(bandwidth_quality_scaler_resource_);
+ }
+ } else {
+ if (!bandwidth_quality_scaler_resource_->is_started()) {
+ // Before executing "StartCheckForOveruse",we must execute "AddResource"
+ // firstly,because it can make the listener valid.
+ AddResource(bandwidth_quality_scaler_resource_,
+ webrtc::VideoAdaptationReason::kQuality);
+ bandwidth_quality_scaler_resource_->StartCheckForOveruse(
+ resolution_bitrate_limits);
+ }
+ }
+}
+
void VideoStreamEncoderResourceManager::ConfigureQualityScaler(
const VideoEncoder::EncoderInfo& encoder_info) {
RTC_DCHECK_RUN_ON(encoder_queue_);
@@ -601,6 +635,20 @@
UpdateStatsAdaptationSettings();
}
+void VideoStreamEncoderResourceManager::ConfigureBandwidthQualityScaler(
+ const VideoEncoder::EncoderInfo& encoder_info) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ const bool bandwidth_quality_scaling_allowed =
+ IsResolutionScalingEnabled(degradation_preference_) &&
+ (encoder_settings_.has_value() &&
+ encoder_settings_->encoder_config().is_quality_scaling_allowed) &&
+ !encoder_info.is_qp_trusted.value_or(true);
+
+ UpdateBandwidthQualityScalerSettings(bandwidth_quality_scaling_allowed,
+ encoder_info.resolution_bitrate_limits);
+ UpdateStatsAdaptationSettings();
+}
+
VideoAdaptationReason VideoStreamEncoderResourceManager::GetReasonFromResource(
rtc::scoped_refptr<Resource> resource) const {
RTC_DCHECK_RUN_ON(encoder_queue_);
@@ -727,7 +775,8 @@
IsFramerateScalingEnabled(degradation_preference_));
VideoStreamEncoderObserver::AdaptationSettings quality_settings =
- quality_scaler_resource_->is_started()
+ (quality_scaler_resource_->is_started() ||
+ bandwidth_quality_scaler_resource_->is_started())
? cpu_settings
: VideoStreamEncoderObserver::AdaptationSettings();
encoder_stats_observer_->UpdateAdaptationSettings(cpu_settings,
diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h
index 2f5dfcd..f1bc885 100644
--- a/video/adaptation/video_stream_encoder_resource_manager.h
+++ b/video/adaptation/video_stream_encoder_resource_manager.h
@@ -43,6 +43,7 @@
#include "rtc_base/thread_annotations.h"
#include "system_wrappers/include/clock.h"
#include "video/adaptation/balanced_constraint.h"
+#include "video/adaptation/bandwidth_quality_scaler_resource.h"
#include "video/adaptation/bitrate_constraint.h"
#include "video/adaptation/encode_usage_resource.h"
#include "video/adaptation/overuse_frame_detector.h"
@@ -109,6 +110,8 @@
// TODO(https://crbug.com/webrtc/11338): This can be made private if we
// configure on SetDegredationPreference and SetEncoderSettings.
void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info);
+ void ConfigureBandwidthQualityScaler(
+ const VideoEncoder::EncoderInfo& encoder_info);
// Methods corresponding to different points in the encoding pipeline.
void OnFrameDroppedDueToSize();
@@ -117,7 +120,8 @@
int64_t time_when_first_seen_us);
void OnEncodeCompleted(const EncodedImage& encoded_image,
int64_t time_sent_in_us,
- absl::optional<int> encode_duration_us);
+ absl::optional<int> encode_duration_us,
+ DataSize frame_size);
void OnFrameDropped(EncodedImageCallback::DropReason reason);
// Resources need to be mapped to an AdaptReason (kCpu or kQuality) in order
@@ -166,6 +170,11 @@
void UpdateQualityScalerSettings(
absl::optional<VideoEncoder::QpThresholds> qp_thresholds);
+ void UpdateBandwidthQualityScalerSettings(
+ bool bandwidth_quality_scaling_allowed,
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>&
+ resolution_bitrate_limits);
+
void UpdateStatsAdaptationSettings() const;
static std::string ActiveCountsToString(
@@ -180,6 +189,8 @@
const rtc::scoped_refptr<EncodeUsageResource> encode_usage_resource_;
const rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_;
rtc::scoped_refptr<PixelLimitResource> pixel_limit_resource_;
+ const rtc::scoped_refptr<BandwidthQualityScalerResource>
+ bandwidth_quality_scaler_resource_;
rtc::TaskQueue* encoder_queue_;
VideoStreamInputStateProvider* const input_state_provider_
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index 8ea6750..8774ff7 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -73,6 +73,8 @@
// Animation is capped to 720p.
constexpr int kMaxAnimationPixels = 1280 * 720;
+constexpr int kDefaultMinScreenSharebps = 1200000;
+
bool RequiresEncoderReset(const VideoCodec& prev_send_codec,
const VideoCodec& new_send_codec,
bool was_encode_called_since_last_initialization) {
@@ -783,6 +785,8 @@
if (encoder_) {
stream_resource_manager_.ConfigureQualityScaler(
encoder_->GetEncoderInfo());
+ stream_resource_manager_.ConfigureBandwidthQualityScaler(
+ encoder_->GetEncoderInfo());
}
});
}
@@ -914,50 +918,97 @@
crop_width_ = last_frame_info_->width - highest_stream_width;
crop_height_ = last_frame_info_->height - highest_stream_height;
- absl::optional<VideoEncoder::ResolutionBitrateLimits> encoder_bitrate_limits =
- encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution(
- last_frame_info_->width * last_frame_info_->height);
+ if (!encoder_->GetEncoderInfo().is_qp_trusted.value_or(true)) {
+ // when qp is not trusted, we priorities to using the
+ // |resolution_bitrate_limits| provided by the decoder.
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>& bitrate_limits =
+ encoder_->GetEncoderInfo().resolution_bitrate_limits.empty()
+ ? EncoderInfoSettings::
+ GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted()
+ : encoder_->GetEncoderInfo().resolution_bitrate_limits;
- if (encoder_bitrate_limits) {
- if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) {
- // Bitrate limits can be set by app (in SDP or RtpEncodingParameters)
- // or/and can be provided by encoder. In presence of both set of limits,
- // the final set is derived as their intersection.
- int min_bitrate_bps;
- if (encoder_config_.simulcast_layers.empty() ||
- encoder_config_.simulcast_layers[0].min_bitrate_bps <= 0) {
- min_bitrate_bps = encoder_bitrate_limits->min_bitrate_bps;
- } else {
- min_bitrate_bps = std::max(encoder_bitrate_limits->min_bitrate_bps,
- streams.back().min_bitrate_bps);
- }
+ // For BandwidthQualityScaler, its implement based on a certain pixel_count
+ // correspond a certain bps interval. In fact, WebRTC default max_bps is
+ // 2500Kbps when width * height > 960 * 540. For example, we assume:
+ // 1.the camera support 1080p.
+ // 2.ResolutionBitrateLimits set 720p bps interval is [1500Kbps,2000Kbps].
+ // 3.ResolutionBitrateLimits set 1080p bps interval is [2000Kbps,2500Kbps].
+ // We will never be stable at 720p due to actual encoding bps of 720p and
+ // 1080p are both 2500Kbps. So it is necessary to do a linear interpolation
+ // to get a certain bitrate for certain pixel_count. It also doesn't work
+ // for 960*540 and 640*520, we will nerver be stable at 640*520 due to their
+ // |target_bitrate_bps| are both 2000Kbps.
+ absl::optional<VideoEncoder::ResolutionBitrateLimits>
+ qp_untrusted_bitrate_limit = EncoderInfoSettings::
+ GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
+ last_frame_info_->width * last_frame_info_->height,
+ bitrate_limits);
- int max_bitrate_bps;
- // We don't check encoder_config_.simulcast_layers[0].max_bitrate_bps
- // here since encoder_config_.max_bitrate_bps is derived from it (as
- // well as from other inputs).
- if (encoder_config_.max_bitrate_bps <= 0) {
- max_bitrate_bps = encoder_bitrate_limits->max_bitrate_bps;
- } else {
- max_bitrate_bps = std::min(encoder_bitrate_limits->max_bitrate_bps,
- streams.back().max_bitrate_bps);
- }
-
- if (min_bitrate_bps < max_bitrate_bps) {
- streams.back().min_bitrate_bps = min_bitrate_bps;
- streams.back().max_bitrate_bps = max_bitrate_bps;
+ if (qp_untrusted_bitrate_limit) {
+ // bandwidth_quality_scaler is only used for singlecast.
+ if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) {
+ streams.back().min_bitrate_bps =
+ qp_untrusted_bitrate_limit->min_bitrate_bps;
+ streams.back().max_bitrate_bps =
+ qp_untrusted_bitrate_limit->max_bitrate_bps;
+ // If it is screen share mode, the minimum value of max_bitrate should
+ // be greater than/equal to 1200kbps.
+ if (encoder_config_.content_type ==
+ VideoEncoderConfig::ContentType::kScreen) {
+ streams.back().max_bitrate_bps = std::max(
+ streams.back().max_bitrate_bps, kDefaultMinScreenSharebps);
+ }
streams.back().target_bitrate_bps =
- std::min(streams.back().target_bitrate_bps,
- encoder_bitrate_limits->max_bitrate_bps);
- } else {
- RTC_LOG(LS_WARNING)
- << "Bitrate limits provided by encoder"
- << " (min=" << encoder_bitrate_limits->min_bitrate_bps
- << ", max=" << encoder_bitrate_limits->max_bitrate_bps
- << ") do not intersect with limits set by app"
- << " (min=" << streams.back().min_bitrate_bps
- << ", max=" << encoder_config_.max_bitrate_bps
- << "). The app bitrate limits will be used.";
+ qp_untrusted_bitrate_limit->max_bitrate_bps;
+ }
+ }
+ } else {
+ absl::optional<VideoEncoder::ResolutionBitrateLimits>
+ encoder_bitrate_limits =
+ encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution(
+ last_frame_info_->width * last_frame_info_->height);
+
+ if (encoder_bitrate_limits) {
+ if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) {
+ // Bitrate limits can be set by app (in SDP or RtpEncodingParameters)
+ // or/and can be provided by encoder. In presence of both set of
+ // limits, the final set is derived as their intersection.
+ int min_bitrate_bps;
+ if (encoder_config_.simulcast_layers.empty() ||
+ encoder_config_.simulcast_layers[0].min_bitrate_bps <= 0) {
+ min_bitrate_bps = encoder_bitrate_limits->min_bitrate_bps;
+ } else {
+ min_bitrate_bps = std::max(encoder_bitrate_limits->min_bitrate_bps,
+ streams.back().min_bitrate_bps);
+ }
+
+ int max_bitrate_bps;
+ // We don't check encoder_config_.simulcast_layers[0].max_bitrate_bps
+ // here since encoder_config_.max_bitrate_bps is derived from it (as
+ // well as from other inputs).
+ if (encoder_config_.max_bitrate_bps <= 0) {
+ max_bitrate_bps = encoder_bitrate_limits->max_bitrate_bps;
+ } else {
+ max_bitrate_bps = std::min(encoder_bitrate_limits->max_bitrate_bps,
+ streams.back().max_bitrate_bps);
+ }
+
+ if (min_bitrate_bps < max_bitrate_bps) {
+ streams.back().min_bitrate_bps = min_bitrate_bps;
+ streams.back().max_bitrate_bps = max_bitrate_bps;
+ streams.back().target_bitrate_bps =
+ std::min(streams.back().target_bitrate_bps,
+ encoder_bitrate_limits->max_bitrate_bps);
+ } else {
+ RTC_LOG(LS_WARNING)
+ << "Bitrate limits provided by encoder"
+ << " (min=" << encoder_bitrate_limits->min_bitrate_bps
+ << ", max=" << encoder_bitrate_limits->max_bitrate_bps
+ << ") do not intersect with limits set by app"
+ << " (min=" << streams.back().min_bitrate_bps
+ << ", max=" << encoder_config_.max_bitrate_bps
+ << "). The app bitrate limits will be used.";
+ }
}
}
}
@@ -1195,6 +1246,7 @@
encoder_config_.min_transmit_bitrate_bps);
stream_resource_manager_.ConfigureQualityScaler(info);
+ stream_resource_manager_.ConfigureBandwidthQualityScaler(info);
}
void VideoStreamEncoder::OnEncoderSettingsChanged() {
@@ -2140,7 +2192,7 @@
}
stream_resource_manager_.OnEncodeCompleted(encoded_image, time_sent_us,
- encode_duration_us);
+ encode_duration_us, frame_size);
if (bitrate_adjuster_) {
bitrate_adjuster_->OnEncodedFrame(
frame_size, encoded_image.SpatialIndex().value_or(0), temporal_index);
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index d643419..d9f1136 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -8210,36 +8210,6 @@
video_stream_encoder_->Stop();
}
-TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
- VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
-
- // Disable scaling settings in encoder info.
- fake_encoder_.SetQualityScaling(false);
- // Set QP not trusted in encoder info.
- fake_encoder_.SetIsQpTrusted(false);
- // Enable quality scaling in encoder config.
- video_encoder_config.is_quality_scaling_allowed = true;
- ConfigureEncoder(std::move(video_encoder_config));
-
- video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
- DataRate::BitsPerSec(kTargetBitrateBps),
- DataRate::BitsPerSec(kTargetBitrateBps),
- DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
-
- test::FrameForwarder source;
- video_stream_encoder_->SetSource(
- &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
- EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
- EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
-
- source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
- WaitForEncodedFrame(1);
- video_stream_encoder_->TriggerQualityLow();
- EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
-
- video_stream_encoder_->Stop();
-}
-
TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
@@ -8270,6 +8240,147 @@
video_stream_encoder_->Stop();
}
+TEST_F(VideoStreamEncoderTest,
+ QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(true);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = false;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(false);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = false;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(false);
+
+ const int MinEncBitrateKbps = 30;
+ const int MaxEncBitrateKbps = 100;
+ const int MinStartBitrateKbp = 50;
+ const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
+ /*frame_size_pixels=*/codec_width_ * codec_height_,
+ /*min_start_bitrate_bps=*/MinStartBitrateKbp,
+ /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
+ /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
+
+ fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
+ video_encoder_config.simulcast_layers[0].min_bitrate_bps =
+ MinEncBitrateKbps * 1000;
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(
+ MaxEncBitrateKbps,
+ static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
+ EXPECT_EQ(
+ MinEncBitrateKbps,
+ static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(false);
+
+ absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
+ EncoderInfoSettings::
+ GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
+ codec_width_ * codec_height_,
+ EncoderInfoSettings::
+ GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
+ EXPECT_TRUE(suitable_bitrate_limit.has_value());
+
+ const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
+ const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
+ const int TargetEncBitrate = MaxEncBitrate;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(TargetEncBitrate),
+ DataRate::BitsPerSec(TargetEncBitrate),
+ DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
+
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
+ video_encoder_config.max_bitrate_bps = MaxEncBitrate;
+ video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+
+ video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
+ WaitForEncodedFrame(1);
+ EXPECT_EQ(
+ MaxEncBitrate / 1000,
+ static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
+ EXPECT_EQ(
+ MinEncBitrate / 1000,
+ static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
+
+ video_stream_encoder_->Stop();
+}
+
#if !defined(WEBRTC_IOS)
// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
// disabled by default on iOS.
@@ -8330,6 +8441,102 @@
video_stream_encoder_->Stop();
}
+
+TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Set QP not trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(false);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = true;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ // When quality_scaler doesn't work and is_quality_scaling_allowed is
+ // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(true);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = true;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ // bandwidth_quality_scaler isn't working, but quality_scaler is working.
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
+ VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
+
+ // Disable scaling settings in encoder info.
+ fake_encoder_.SetQualityScaling(false);
+ // Set QP trusted in encoder info.
+ fake_encoder_.SetIsQpTrusted(false);
+ // Enable quality scaling in encoder config.
+ video_encoder_config.is_quality_scaling_allowed = true;
+ ConfigureEncoder(std::move(video_encoder_config));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
+
+ test::FrameForwarder source;
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+ EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
+ EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
+ WaitForEncodedFrame(1);
+ video_stream_encoder_->TriggerQualityLow();
+ EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
+
+ video_stream_encoder_->Stop();
+}
+
#endif
// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
