Use bandwidth allocation in DropDueToSize when incoming resolution increases.
Use bandwidth allocation instead of encoder target bitrate in DropDueToSize when incoming resolution increases to avoid downgrades due to target bitrate being limited by the max bitrate at low resolutions.
Bug: none
Change-Id: Ic41b31c1a86911d4e97b61b0cbc41ce0da739bd4
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/205622
Commit-Queue: Åsa Persson <asapersson@webrtc.org>
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#33168}
diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc
index 22f0b56..379df7b 100644
--- a/video/adaptation/video_stream_encoder_resource_manager.cc
+++ b/video/adaptation/video_stream_encoder_resource_manager.cc
@@ -98,6 +98,8 @@
set_start_bitrate_(DataRate::Zero()),
set_start_bitrate_time_ms_(0),
initial_framedrop_(0),
+ use_bandwidth_allocation_(false),
+ bandwidth_allocation_(DataRate::Zero()),
last_input_width_(0),
last_input_height_(0) {
RTC_DCHECK(quality_scaler_resource_);
@@ -112,12 +114,23 @@
return single_active_stream_pixels_;
}
+ absl::optional<uint32_t> UseBandwidthAllocationBps() const {
+ return (use_bandwidth_allocation_ &&
+ bandwidth_allocation_ > DataRate::Zero())
+ ? absl::optional<uint32_t>(bandwidth_allocation_.bps())
+ : absl::nullopt;
+ }
+
// Input signals.
void SetStartBitrate(DataRate start_bitrate, int64_t now_ms) {
set_start_bitrate_ = start_bitrate;
set_start_bitrate_time_ms_ = now_ms;
}
+ void SetBandwidthAllocation(DataRate bandwidth_allocation) {
+ bandwidth_allocation_ = bandwidth_allocation;
+ }
+
void SetTargetBitrate(DataRate target_bitrate, int64_t now_ms) {
if (set_start_bitrate_ > DataRate::Zero() && !has_seen_first_bwe_drop_ &&
quality_scaler_resource_->is_started() &&
@@ -158,6 +171,11 @@
RTC_LOG(LS_INFO) << "Resetting initial_framedrop_ due to changed "
"stream parameters";
initial_framedrop_ = 0;
+ if (single_active_stream_pixels_ &&
+ GetSingleActiveLayerPixels(codec) > *single_active_stream_pixels_) {
+ // Resolution increased.
+ use_bandwidth_allocation_ = true;
+ }
}
}
last_adaptation_counters_ = adaptation_counters;
@@ -169,7 +187,10 @@
void OnFrameDroppedDueToSize() { ++initial_framedrop_; }
- void Disable() { initial_framedrop_ = kMaxInitialFramedrop; }
+ void Disable() {
+ initial_framedrop_ = kMaxInitialFramedrop;
+ use_bandwidth_allocation_ = false;
+ }
void OnQualityScalerSettingsUpdated() {
if (quality_scaler_resource_->is_started()) {
@@ -177,7 +198,7 @@
initial_framedrop_ = 0;
} else {
// Quality scaling disabled so we shouldn't drop initial frames.
- initial_framedrop_ = kMaxInitialFramedrop;
+ Disable();
}
}
@@ -194,6 +215,8 @@
// Counts how many frames we've dropped in the initial framedrop phase.
int initial_framedrop_;
absl::optional<uint32_t> single_active_stream_pixels_;
+ bool use_bandwidth_allocation_;
+ DataRate bandwidth_allocation_;
std::vector<bool> last_active_flags_;
VideoAdaptationCounters last_adaptation_counters_;
@@ -398,6 +421,8 @@
const VideoEncoder::RateControlParameters& encoder_rates) {
RTC_DCHECK_RUN_ON(encoder_queue_);
encoder_rates_ = encoder_rates;
+ initial_frame_dropper_->SetBandwidthAllocation(
+ encoder_rates.bandwidth_allocation);
}
void VideoStreamEncoderResourceManager::OnFrameDroppedDueToSize() {
@@ -449,6 +474,12 @@
return initial_frame_dropper_->single_active_stream_pixels();
}
+absl::optional<uint32_t>
+VideoStreamEncoderResourceManager::UseBandwidthAllocationBps() const {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ return initial_frame_dropper_->UseBandwidthAllocationBps();
+}
+
void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() {
RTC_DCHECK_RUN_ON(encoder_queue_);
initial_frame_dropper_->Disable();
diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h
index e5cbeb0..8eb5229 100644
--- a/video/adaptation/video_stream_encoder_resource_manager.h
+++ b/video/adaptation/video_stream_encoder_resource_manager.h
@@ -130,6 +130,7 @@
// frames based on size and bitrate.
bool DropInitialFrames() const;
absl::optional<uint32_t> SingleActiveStreamPixels() const;
+ absl::optional<uint32_t> UseBandwidthAllocationBps() const;
// VideoSourceRestrictionsListener implementation.
// Updates |video_source_restrictions_|.
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index 025481a..1a2c20e 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -2055,19 +2055,23 @@
}
}
+ uint32_t bitrate_bps =
+ stream_resource_manager_.UseBandwidthAllocationBps().value_or(
+ encoder_target_bitrate_bps_.value());
+
absl::optional<VideoEncoder::ResolutionBitrateLimits> encoder_bitrate_limits =
encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution(
pixel_count);
if (encoder_bitrate_limits.has_value()) {
// Use bitrate limits provided by encoder.
- return encoder_target_bitrate_bps_.value() <
+ return bitrate_bps <
static_cast<uint32_t>(encoder_bitrate_limits->min_start_bitrate_bps);
}
- if (encoder_target_bitrate_bps_.value() < 300000 /* qvga */) {
+ if (bitrate_bps < 300000 /* qvga */) {
return pixel_count > 320 * 240;
- } else if (encoder_target_bitrate_bps_.value() < 500000 /* vga */) {
+ } else if (bitrate_bps < 500000 /* vga */) {
return pixel_count > 640 * 480;
}
return false;
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index 38e5111..cf46413 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -5252,6 +5252,10 @@
VideoEncoderConfig video_encoder_config;
test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
&video_encoder_config);
+ video_encoder_config.video_stream_factory =
+ new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
+ "VP8", /*max qp*/ 56, /*screencast*/ false,
+ /*screenshare enabled*/ false);
for (auto& layer : video_encoder_config.simulcast_layers) {
layer.num_temporal_layers = 1;
layer.max_framerate = kDefaultFramerate;
@@ -5565,6 +5569,59 @@
video_stream_encoder_->Stop();
}
+TEST_F(VideoStreamEncoderTest,
+ FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
+ const int kMinStartBps360p = 222000;
+ fake_encoder_.SetResolutionBitrateLimits(
+ {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
+ VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
+ 800000)});
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
+ DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
+ DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
+ 0, 0, 0);
+ // Frame should not be dropped, bitrate not too low for frame.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
+ WaitForEncodedFrame(1);
+
+ // Incoming resolution increases, initial frame drop activates.
+ // Frame should be dropped, link allocation too low for frame.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
+ ExpectDroppedFrame();
+
+ // Expect sink_wants to specify a scaled frame.
+ EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
+ 5000);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
+ const int kMinStartBps360p = 222000;
+ fake_encoder_.SetResolutionBitrateLimits(
+ {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
+ VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
+ 800000)});
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
+ DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
+ DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
+ 0, 0, 0);
+ // Frame should not be dropped, bitrate not too low for frame.
+ video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
+ WaitForEncodedFrame(1);
+
+ // Incoming resolution increases, initial frame drop activates.
+ // Frame should be dropped, link allocation not too low for frame.
+ video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
+ WaitForEncodedFrame(2);
+
+ video_stream_encoder_->Stop();
+}
+
TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
webrtc::test::ScopedFieldTrials field_trials(
"WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
