video: Implement bandwidth based scaler
The |slice_qp_detla| reported by the hardware is not credible, which
causing the quality scaler cannot work properly,the resolution cannot
be adjusted correctly.
To fix this issue, this CL implements a bandwidth scaler which is used
for adjust resolution, this scaler will be used when QP based quality
scaler is not working due to untrusted QP reported by HW AVC encoder.
Bug: webrtc:12942
Change-Id: I2fc5f07a5400ec7e5ead2c2c502faee84d7f2a76
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/228860
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Evan Shrubsole <eshr@google.com>
Commit-Queue: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35120}
diff --git a/video/adaptation/BUILD.gn b/video/adaptation/BUILD.gn
index 20a2370..11962c0 100644
--- a/video/adaptation/BUILD.gn
+++ b/video/adaptation/BUILD.gn
@@ -12,6 +12,8 @@
sources = [
"balanced_constraint.cc",
"balanced_constraint.h",
+ "bandwidth_quality_scaler_resource.cc",
+ "bandwidth_quality_scaler_resource.h",
"bitrate_constraint.cc",
"bitrate_constraint.h",
"encode_usage_resource.cc",
diff --git a/video/adaptation/bandwidth_quality_scaler_resource.cc b/video/adaptation/bandwidth_quality_scaler_resource.cc
new file mode 100644
index 0000000..6f49135
--- /dev/null
+++ b/video/adaptation/bandwidth_quality_scaler_resource.cc
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/adaptation/bandwidth_quality_scaler_resource.h"
+
+#include <utility>
+
+#include "rtc_base/checks.h"
+#include "rtc_base/experiments/balanced_degradation_settings.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/ref_counted_object.h"
+#include "rtc_base/task_utils/to_queued_task.h"
+#include "rtc_base/time_utils.h"
+
+namespace webrtc {
+
+// static
+rtc::scoped_refptr<BandwidthQualityScalerResource>
+BandwidthQualityScalerResource::Create() {
+ return rtc::make_ref_counted<BandwidthQualityScalerResource>();
+}
+
+BandwidthQualityScalerResource::BandwidthQualityScalerResource()
+ : VideoStreamEncoderResource("BandwidthQualityScalerResource"),
+ bandwidth_quality_scaler_(nullptr) {}
+
+BandwidthQualityScalerResource::~BandwidthQualityScalerResource() {
+ RTC_DCHECK(!bandwidth_quality_scaler_);
+}
+
+bool BandwidthQualityScalerResource::is_started() const {
+ RTC_DCHECK_RUN_ON(encoder_queue());
+ return bandwidth_quality_scaler_.get();
+}
+
+void BandwidthQualityScalerResource::StartCheckForOveruse(
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>&
+ resolution_bitrate_limits) {
+ RTC_DCHECK_RUN_ON(encoder_queue());
+ RTC_DCHECK(!is_started());
+ bandwidth_quality_scaler_ = std::make_unique<BandwidthQualityScaler>(this);
+
+ // If the configuration parameters more than one, we should define and
+ // declare the function BandwidthQualityScaler::Initialize() and call it.
+ bandwidth_quality_scaler_->SetResolutionBitrateLimits(
+ resolution_bitrate_limits);
+}
+
+void BandwidthQualityScalerResource::StopCheckForOveruse() {
+ RTC_DCHECK_RUN_ON(encoder_queue());
+ RTC_DCHECK(is_started());
+ // Ensure we have no pending callbacks. This makes it safe to destroy the
+ // BandwidthQualityScaler and even task queues with tasks in-flight.
+ bandwidth_quality_scaler_.reset();
+}
+
+void BandwidthQualityScalerResource::OnReportUsageBandwidthHigh() {
+ OnResourceUsageStateMeasured(ResourceUsageState::kOveruse);
+}
+
+void BandwidthQualityScalerResource::OnReportUsageBandwidthLow() {
+ OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse);
+}
+
+void BandwidthQualityScalerResource::OnEncodeCompleted(
+ const EncodedImage& encoded_image,
+ int64_t time_sent_in_us,
+ int64_t encoded_image_size_bytes) {
+ RTC_DCHECK_RUN_ON(encoder_queue());
+
+ if (bandwidth_quality_scaler_) {
+ bandwidth_quality_scaler_->ReportEncodeInfo(
+ encoded_image_size_bytes, time_sent_in_us / 1000,
+ encoded_image._encodedWidth, encoded_image._encodedHeight);
+ }
+}
+
+} // namespace webrtc
diff --git a/video/adaptation/bandwidth_quality_scaler_resource.h b/video/adaptation/bandwidth_quality_scaler_resource.h
new file mode 100644
index 0000000..f33ce97
--- /dev/null
+++ b/video/adaptation/bandwidth_quality_scaler_resource.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_
+#define VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_
+
+#include <memory>
+#include <queue>
+#include <string>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/scoped_refptr.h"
+#include "api/video/video_adaptation_reason.h"
+#include "api/video_codecs/video_encoder.h"
+#include "call/adaptation/degradation_preference_provider.h"
+#include "call/adaptation/resource_adaptation_processor_interface.h"
+#include "modules/video_coding/utility/bandwidth_quality_scaler.h"
+#include "rtc_base/ref_counted_object.h"
+#include "rtc_base/task_queue.h"
+#include "video/adaptation/video_stream_encoder_resource.h"
+
+namespace webrtc {
+
+// Handles interaction with the BandwidthQualityScaler.
+class BandwidthQualityScalerResource
+ : public VideoStreamEncoderResource,
+ public BandwidthQualityScalerUsageHandlerInterface {
+ public:
+ static rtc::scoped_refptr<BandwidthQualityScalerResource> Create();
+
+ BandwidthQualityScalerResource();
+ ~BandwidthQualityScalerResource() override;
+
+ bool is_started() const;
+
+ void OnEncodeCompleted(const EncodedImage& encoded_image,
+ int64_t time_sent_in_us,
+ int64_t encoded_image_size_bytes);
+
+ void StartCheckForOveruse(
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>&
+ resolution_bitrate_limits);
+ void StopCheckForOveruse();
+
+ // BandwidthScalerQpUsageHandlerInterface implementation.
+ void OnReportUsageBandwidthHigh() override;
+ void OnReportUsageBandwidthLow() override;
+
+ private:
+ std::unique_ptr<BandwidthQualityScaler> bandwidth_quality_scaler_
+ RTC_GUARDED_BY(encoder_queue());
+};
+
+} // namespace webrtc
+
+#endif // VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_
diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc
index 0b2fa89..6a1e921 100644
--- a/video/adaptation/video_stream_encoder_resource_manager.cc
+++ b/video/adaptation/video_stream_encoder_resource_manager.cc
@@ -275,6 +275,8 @@
EncodeUsageResource::Create(std::move(overuse_detector))),
quality_scaler_resource_(QualityScalerResource::Create()),
pixel_limit_resource_(nullptr),
+ bandwidth_quality_scaler_resource_(
+ BandwidthQualityScalerResource::Create()),
encoder_queue_(nullptr),
input_state_provider_(input_state_provider),
adaptation_processor_(nullptr),
@@ -307,6 +309,8 @@
encoder_queue_ = encoder_queue;
encode_usage_resource_->RegisterEncoderTaskQueue(encoder_queue_->Get());
quality_scaler_resource_->RegisterEncoderTaskQueue(encoder_queue_->Get());
+ bandwidth_quality_scaler_resource_->RegisterEncoderTaskQueue(
+ encoder_queue_->Get());
}
void VideoStreamEncoderResourceManager::SetAdaptationProcessor(
@@ -385,6 +389,10 @@
RemoveResource(pixel_limit_resource_);
pixel_limit_resource_ = nullptr;
}
+ if (bandwidth_quality_scaler_resource_->is_started()) {
+ bandwidth_quality_scaler_resource_->StopCheckForOveruse();
+ RemoveResource(bandwidth_quality_scaler_resource_);
+ }
}
void VideoStreamEncoderResourceManager::AddResource(
@@ -491,7 +499,8 @@
void VideoStreamEncoderResourceManager::OnEncodeCompleted(
const EncodedImage& encoded_image,
int64_t time_sent_in_us,
- absl::optional<int> encode_duration_us) {
+ absl::optional<int> encode_duration_us,
+ DataSize frame_size) {
RTC_DCHECK_RUN_ON(encoder_queue_);
// Inform `encode_usage_resource_` of the encode completed event.
uint32_t timestamp = encoded_image.Timestamp();
@@ -500,6 +509,8 @@
encode_usage_resource_->OnEncodeCompleted(
timestamp, time_sent_in_us, capture_time_us, encode_duration_us);
quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us);
+ bandwidth_quality_scaler_resource_->OnEncodeCompleted(
+ encoded_image, time_sent_in_us, frame_size.bytes());
}
void VideoStreamEncoderResourceManager::OnFrameDropped(
@@ -556,6 +567,29 @@
initial_frame_dropper_->OnQualityScalerSettingsUpdated();
}
+void VideoStreamEncoderResourceManager::UpdateBandwidthQualityScalerSettings(
+ bool bandwidth_quality_scaling_allowed,
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>&
+ resolution_bitrate_limits) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+
+ if (!bandwidth_quality_scaling_allowed) {
+ if (bandwidth_quality_scaler_resource_->is_started()) {
+ bandwidth_quality_scaler_resource_->StopCheckForOveruse();
+ RemoveResource(bandwidth_quality_scaler_resource_);
+ }
+ } else {
+ if (!bandwidth_quality_scaler_resource_->is_started()) {
+ // Before executing "StartCheckForOveruse",we must execute "AddResource"
+ // firstly,because it can make the listener valid.
+ AddResource(bandwidth_quality_scaler_resource_,
+ webrtc::VideoAdaptationReason::kQuality);
+ bandwidth_quality_scaler_resource_->StartCheckForOveruse(
+ resolution_bitrate_limits);
+ }
+ }
+}
+
void VideoStreamEncoderResourceManager::ConfigureQualityScaler(
const VideoEncoder::EncoderInfo& encoder_info) {
RTC_DCHECK_RUN_ON(encoder_queue_);
@@ -601,6 +635,20 @@
UpdateStatsAdaptationSettings();
}
+void VideoStreamEncoderResourceManager::ConfigureBandwidthQualityScaler(
+ const VideoEncoder::EncoderInfo& encoder_info) {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ const bool bandwidth_quality_scaling_allowed =
+ IsResolutionScalingEnabled(degradation_preference_) &&
+ (encoder_settings_.has_value() &&
+ encoder_settings_->encoder_config().is_quality_scaling_allowed) &&
+ !encoder_info.is_qp_trusted.value_or(true);
+
+ UpdateBandwidthQualityScalerSettings(bandwidth_quality_scaling_allowed,
+ encoder_info.resolution_bitrate_limits);
+ UpdateStatsAdaptationSettings();
+}
+
VideoAdaptationReason VideoStreamEncoderResourceManager::GetReasonFromResource(
rtc::scoped_refptr<Resource> resource) const {
RTC_DCHECK_RUN_ON(encoder_queue_);
@@ -727,7 +775,8 @@
IsFramerateScalingEnabled(degradation_preference_));
VideoStreamEncoderObserver::AdaptationSettings quality_settings =
- quality_scaler_resource_->is_started()
+ (quality_scaler_resource_->is_started() ||
+ bandwidth_quality_scaler_resource_->is_started())
? cpu_settings
: VideoStreamEncoderObserver::AdaptationSettings();
encoder_stats_observer_->UpdateAdaptationSettings(cpu_settings,
diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h
index 2f5dfcd..f1bc885 100644
--- a/video/adaptation/video_stream_encoder_resource_manager.h
+++ b/video/adaptation/video_stream_encoder_resource_manager.h
@@ -43,6 +43,7 @@
#include "rtc_base/thread_annotations.h"
#include "system_wrappers/include/clock.h"
#include "video/adaptation/balanced_constraint.h"
+#include "video/adaptation/bandwidth_quality_scaler_resource.h"
#include "video/adaptation/bitrate_constraint.h"
#include "video/adaptation/encode_usage_resource.h"
#include "video/adaptation/overuse_frame_detector.h"
@@ -109,6 +110,8 @@
// TODO(https://crbug.com/webrtc/11338): This can be made private if we
// configure on SetDegredationPreference and SetEncoderSettings.
void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info);
+ void ConfigureBandwidthQualityScaler(
+ const VideoEncoder::EncoderInfo& encoder_info);
// Methods corresponding to different points in the encoding pipeline.
void OnFrameDroppedDueToSize();
@@ -117,7 +120,8 @@
int64_t time_when_first_seen_us);
void OnEncodeCompleted(const EncodedImage& encoded_image,
int64_t time_sent_in_us,
- absl::optional<int> encode_duration_us);
+ absl::optional<int> encode_duration_us,
+ DataSize frame_size);
void OnFrameDropped(EncodedImageCallback::DropReason reason);
// Resources need to be mapped to an AdaptReason (kCpu or kQuality) in order
@@ -166,6 +170,11 @@
void UpdateQualityScalerSettings(
absl::optional<VideoEncoder::QpThresholds> qp_thresholds);
+ void UpdateBandwidthQualityScalerSettings(
+ bool bandwidth_quality_scaling_allowed,
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>&
+ resolution_bitrate_limits);
+
void UpdateStatsAdaptationSettings() const;
static std::string ActiveCountsToString(
@@ -180,6 +189,8 @@
const rtc::scoped_refptr<EncodeUsageResource> encode_usage_resource_;
const rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_;
rtc::scoped_refptr<PixelLimitResource> pixel_limit_resource_;
+ const rtc::scoped_refptr<BandwidthQualityScalerResource>
+ bandwidth_quality_scaler_resource_;
rtc::TaskQueue* encoder_queue_;
VideoStreamInputStateProvider* const input_state_provider_
