Add QualityConvergenceController to VideoStreamEncoder
QualityConvergenceController is a layer between VideoStreamEncoder
and QualityConvergenceMonitor that takes care of the simulcast
logic.
Bug: chromium:328598314
Change-Id: Iad8a9d9138e69a60fd508a7ef038220947888f0a
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/356420
Commit-Queue: Johannes Kron <kron@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42642}
diff --git a/video/BUILD.gn b/video/BUILD.gn
index 7e85a55..a23d4bb 100644
--- a/video/BUILD.gn
+++ b/video/BUILD.gn
@@ -388,6 +388,8 @@
"encoder_overshoot_detector.h",
"frame_encode_metadata_writer.cc",
"frame_encode_metadata_writer.h",
+ "quality_convergence_controller.cc",
+ "quality_convergence_controller.h",
"quality_convergence_monitor.cc",
"quality_convergence_monitor.h",
"rate_utilization_tracker.cc",
@@ -778,6 +780,7 @@
"frame_decode_timing_unittest.cc",
"frame_encode_metadata_writer_unittest.cc",
"picture_id_tests.cc",
+ "quality_convergence_controller_unittest.cc",
"quality_convergence_monitor_unittest.cc",
"quality_limitation_reason_tracker_unittest.cc",
"quality_scaling_tests.cc",
diff --git a/video/quality_convergence_controller.cc b/video/quality_convergence_controller.cc
new file mode 100644
index 0000000..7a46e11
--- /dev/null
+++ b/video/quality_convergence_controller.cc
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/quality_convergence_controller.h"
+
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+namespace {
+// TODO(https://crbug.com/328598314): Remove default values once HW encoders
+// correctly report the minimum QP value. These thresholds correspond to the
+// default configurations used for the software encoders.
+constexpr int kVp8DefaultStaticQpThreshold = 15;
+constexpr int kVp9DefaultStaticQpThreshold = 32;
+constexpr int kAv1DefaultStaticQpThreshold = 40;
+
+int GetDefaultStaticQpThreshold(VideoCodecType codec) {
+ switch (codec) {
+ case kVideoCodecVP8:
+ return kVp8DefaultStaticQpThreshold;
+ case kVideoCodecVP9:
+ return kVp9DefaultStaticQpThreshold;
+ case kVideoCodecAV1:
+ return kAv1DefaultStaticQpThreshold;
+ case kVideoCodecGeneric:
+ case kVideoCodecH264:
+ case kVideoCodecH265:
+ // -1 will effectively disable the static QP threshold since QP values are
+ // always >= 0.
+ return -1;
+ }
+}
+} // namespace
+
+void QualityConvergenceController::Initialize(
+ int number_of_layers,
+ absl::optional<int> static_qp_threshold,
+ VideoCodecType codec,
+ const FieldTrialsView& trials) {
+ RTC_CHECK(number_of_layers > 0);
+ number_of_layers_ = number_of_layers;
+ convergence_monitors_.clear();
+
+ int qp_threshold =
+ static_qp_threshold.value_or(GetDefaultStaticQpThreshold(codec));
+ for (int i = 0; i < number_of_layers_; ++i) {
+ convergence_monitors_.push_back(
+ QualityConvergenceMonitor::Create(qp_threshold, codec, trials));
+ }
+ initialized_ = true;
+}
+
+bool QualityConvergenceController::AddSampleAndCheckTargetQuality(
+ int layer_index,
+ int qp,
+ bool is_refresh_frame) {
+ RTC_CHECK(initialized_);
+ if (layer_index < 0 || layer_index >= number_of_layers_) {
+ return false;
+ }
+
+ convergence_monitors_[layer_index]->AddSample(qp, is_refresh_frame);
+ return convergence_monitors_[layer_index]->AtTargetQuality();
+}
+
+} // namespace webrtc
diff --git a/video/quality_convergence_controller.h b/video/quality_convergence_controller.h
new file mode 100644
index 0000000..46f8419
--- /dev/null
+++ b/video/quality_convergence_controller.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VIDEO_QUALITY_CONVERGENCE_CONTROLLER_H_
+#define VIDEO_QUALITY_CONVERGENCE_CONTROLLER_H_
+
+#include <memory>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/field_trials_view.h"
+#include "api/video/video_codec_type.h"
+#include "video/quality_convergence_monitor.h"
+
+namespace webrtc {
+
+class QualityConvergenceController {
+ public:
+ void Initialize(int number_of_layers,
+ absl::optional<int> static_qp_threshold,
+ VideoCodecType codec,
+ const FieldTrialsView& trials);
+
+ // Add the supplied `qp` value to the detection window for specified layer.
+ // `is_refresh_frame` must only be `true` if the corresponding
+ // video frame is a refresh frame that is used to improve the visual quality.
+ // Returns `true` if the algorithm has determined that the supplied QP values
+ // have converged and reached the target quality for this layer.
+ bool AddSampleAndCheckTargetQuality(int layer_index,
+ int qp,
+ bool is_refresh_frame);
+
+ private:
+ bool initialized_ = false;
+ int number_of_layers_ = 0;
+ std::vector<std::unique_ptr<QualityConvergenceMonitor>> convergence_monitors_;
+};
+
+} // namespace webrtc
+
+#endif // VIDEO_QUALITY_CONVERGENCE_CONTROLLER_H_
diff --git a/video/quality_convergence_controller_unittest.cc b/video/quality_convergence_controller_unittest.cc
new file mode 100644
index 0000000..c1378e0
--- /dev/null
+++ b/video/quality_convergence_controller_unittest.cc
@@ -0,0 +1,66 @@
+
+/*
+ * Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "video/quality_convergence_controller.h"
+
+#include "test/gtest.h"
+#include "test/scoped_key_value_config.h"
+
+namespace webrtc {
+namespace {
+constexpr int kStaticQpThreshold = 15;
+
+TEST(QualityConvergenceController, Singlecast) {
+ test::ScopedKeyValueConfig field_trials;
+ QualityConvergenceController controller;
+ controller.Initialize(1, kStaticQpThreshold, kVideoCodecVP8, field_trials);
+
+ EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality(
+ /*layer_index=*/0, kStaticQpThreshold + 1, /*is_refresh_frame=*/false));
+ EXPECT_TRUE(controller.AddSampleAndCheckTargetQuality(
+ /*layer_index=*/0, kStaticQpThreshold, /*is_refresh_frame=*/false));
+}
+
+TEST(QualityConvergenceController, Simulcast) {
+ test::ScopedKeyValueConfig field_trials;
+ QualityConvergenceController controller;
+ controller.Initialize(2, kStaticQpThreshold, kVideoCodecVP8, field_trials);
+
+ EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality(
+ /*layer_index=*/0, kStaticQpThreshold + 1, /*is_refresh_frame=*/false));
+ EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality(
+ /*layer_index=*/1, kStaticQpThreshold + 1, /*is_refresh_frame=*/false));
+
+ // Layer 0 reaches target quality.
+ EXPECT_TRUE(controller.AddSampleAndCheckTargetQuality(
+ /*layer_index=*/0, kStaticQpThreshold, /*is_refresh_frame=*/false));
+ EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality(
+ /*layer_index=*/1, kStaticQpThreshold + 1, /*is_refresh_frame=*/false));
+
+ // Frames are repeated for both layers. Layer 0 still at target quality.
+ EXPECT_TRUE(controller.AddSampleAndCheckTargetQuality(
+ /*layer_index=*/0, kStaticQpThreshold, /*is_refresh_frame=*/true));
+ EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality(
+ /*layer_index=*/1, kStaticQpThreshold + 1, /*is_refresh_frame=*/true));
+}
+
+TEST(QualityConvergenceController, InvalidLayerIndex) {
+ test::ScopedKeyValueConfig field_trials;
+ QualityConvergenceController controller;
+ controller.Initialize(2, kStaticQpThreshold, kVideoCodecVP8, field_trials);
+
+ EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality(
+ /*layer_index=*/-1, kStaticQpThreshold, /*is_refresh_frame=*/false));
+ EXPECT_FALSE(controller.AddSampleAndCheckTargetQuality(
+ /*layer_index=*/3, kStaticQpThreshold, /*is_refresh_frame=*/false));
+}
+
+} // namespace
+} // namespace webrtc
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index 92f7308..af26db8 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -1254,6 +1254,10 @@
codec.SetVideoEncoderComplexity(VideoCodecComplexity::kComplexityLow);
}
+ quality_convergence_controller_.Initialize(
+ codec.numberOfSimulcastStreams, encoder_->GetEncoderInfo().min_qp,
+ codec.codecType, env_.field_trials());
+
send_codec_ = codec;
// Keep the same encoder, as long as the video_format is unchanged.
@@ -2085,14 +2089,23 @@
.Parse(codec_type, stream_idx, image_copy.data(), image_copy.size())
.value_or(-1);
}
+
+ // Check if the encoded image has reached target quality.
+ const size_t simulcast_index = encoded_image.SimulcastIndex().value_or(0);
+ bool at_target_quality =
+ quality_convergence_controller_.AddSampleAndCheckTargetQuality(
+ simulcast_index, image_copy.qp_,
+ image_copy.IsSteadyStateRefreshFrame());
+ image_copy.SetAtTargetQuality(at_target_quality);
TRACE_EVENT2("webrtc", "VideoStreamEncoder::AugmentEncodedImage",
"stream_idx", stream_idx, "qp", image_copy.qp_);
+ TRACE_EVENT_INSTANT2("webrtc", "VideoStreamEncoder::AugmentEncodedImage",
+ TRACE_EVENT_SCOPE_GLOBAL, "simulcast_idx",
+ simulcast_index, "at_target_quality", at_target_quality);
RTC_LOG(LS_VERBOSE) << __func__ << " ntp time " << encoded_image.NtpTimeMs()
<< " stream_idx " << stream_idx << " qp "
- << image_copy.qp_;
- image_copy.SetAtTargetQuality(codec_type == kVideoCodecVP8 &&
- image_copy.qp_ <= kVp8SteadyStateQpThreshold);
-
+ << image_copy.qp_ << " at target quality "
+ << at_target_quality;
return image_copy;
}
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h
index fe04fbf..579cd32 100644
--- a/video/video_stream_encoder.h
+++ b/video/video_stream_encoder.h
@@ -47,6 +47,7 @@
#include "video/encoder_bitrate_adjuster.h"
#include "video/frame_cadence_adapter.h"
#include "video/frame_encode_metadata_writer.h"
+#include "video/quality_convergence_controller.h"
#include "video/video_source_sink_controller.h"
#include "video/video_stream_encoder_interface.h"
#include "video/video_stream_encoder_observer.h"
@@ -418,6 +419,11 @@
QpParser qp_parser_;
const bool qp_parsing_allowed_;
+ // The quality convergence controller is used to determine if a codec has
+ // reached its target quality. This is used for screenshare to determine when
+ // there's no need to continue encoding the same repeated frame.
+ QualityConvergenceController quality_convergence_controller_;
+
// Enables encoder switching on initialization failures.
bool switch_encoder_on_init_failures_;
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index aab791f..ac4aa3e 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -701,6 +701,10 @@
(EncodedImage & encoded_image,
rtc::scoped_refptr<EncodedImageBuffer> buffer),
(override));
+ MOCK_METHOD(VideoEncoder::EncoderInfo,
+ GetEncoderInfo,
+ (),
+ (const, override));
};
SimpleVideoStreamEncoderFactory() {
@@ -9475,6 +9479,12 @@
auto video_stream_encoder =
factory.Create(std::move(adapter), &encoder_queue);
+ // Set minimum QP.
+ VideoEncoder::EncoderInfo info;
+ info.min_qp = kVp8SteadyStateQpThreshold;
+ EXPECT_CALL(factory.GetMockFakeEncoder(), GetEncoderInfo)
+ .WillRepeatedly(Return(info));
+
// Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
VideoEncoderConfig video_encoder_config;
test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
