Reset initial frame dropper if the stream changes for external reasons
External reasons here are simulcast configuration and
source resolution change.
Initial frame dropper should be enabled in these cases because the
client can request way too big resolution for available bitrate and
usual quality scaling would take too long.
Bug: none
Change-Id: I02fbbd3c15b53b39672c083c2a1f9a780256c507
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/195004
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32707}
diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc
index 697ebbe..7247850 100644
--- a/video/adaptation/video_stream_encoder_resource_manager.cc
+++ b/video/adaptation/video_stream_encoder_resource_manager.cc
@@ -10,6 +10,7 @@
#include "video/adaptation/video_stream_encoder_resource_manager.h"
+#include <algorithm>
#include <cmath>
#include <limits>
#include <memory>
@@ -72,6 +73,21 @@
return pixels;
}
+std::vector<bool> GetActiveLayersFlags(const VideoCodec& codec) {
+ const int num_streams = codec.numberOfSimulcastStreams;
+ std::vector<bool> flags(num_streams);
+ for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
+ flags[i] = codec.simulcastStream[i].active;
+ }
+ return flags;
+}
+
+bool EqualFlags(const std::vector<bool>& a, const std::vector<bool>& b) {
+ if (a.size() != b.size())
+ return false;
+ return std::equal(a.begin(), a.end(), b.begin());
+}
+
} // namespace
class VideoStreamEncoderResourceManager::InitialFrameDropper {
@@ -83,7 +99,9 @@
has_seen_first_bwe_drop_(false),
set_start_bitrate_(DataRate::Zero()),
set_start_bitrate_time_ms_(0),
- initial_framedrop_(0) {
+ initial_framedrop_(0),
+ last_input_width_(0),
+ last_input_height_(0) {
RTC_DCHECK(quality_scaler_resource_);
}
@@ -122,13 +140,38 @@
}
}
- void OnEncoderSettingsUpdated(const VideoCodec& codec) {
+ void OnEncoderSettingsUpdated(
+ const VideoCodec& codec,
+ const VideoAdaptationCounters& adaptation_counters) {
+ std::vector<bool> active_flags = GetActiveLayersFlags(codec);
+ // Check if the source resolution has changed for the external reasons,
+ // i.e. without any adaptation from WebRTC.
+ const bool source_resolution_changed =
+ (last_input_width_ != codec.width ||
+ last_input_height_ != codec.height) &&
+ adaptation_counters.resolution_adaptations ==
+ last_adaptation_counters_.resolution_adaptations;
+ if (!EqualFlags(active_flags, last_active_flags_) ||
+ source_resolution_changed) {
+ // Streams configuration has changed.
+ // Initial frame drop must be enabled because BWE might be way too low
+ // for the selected resolution.
+ if (quality_scaler_resource_->is_started()) {
+ RTC_LOG(LS_INFO) << "Resetting initial_framedrop_ due to changed "
+ "stream parameters";
+ initial_framedrop_ = 0;
+ }
+ }
+ last_adaptation_counters_ = adaptation_counters;
+ last_active_flags_ = active_flags;
+ last_input_width_ = codec.width;
+ last_input_height_ = codec.height;
single_active_stream_pixels_ = GetSingleActiveStreamPixels(codec);
}
void OnFrameDroppedDueToSize() { ++initial_framedrop_; }
- void OnMaybeEncodeFrame() { initial_framedrop_ = kMaxInitialFramedrop; }
+ void Disable() { initial_framedrop_ = kMaxInitialFramedrop; }
void OnQualityScalerSettingsUpdated() {
if (quality_scaler_resource_->is_started()) {
@@ -153,6 +196,11 @@
// Counts how many frames we've dropped in the initial framedrop phase.
int initial_framedrop_;
absl::optional<uint32_t> single_active_stream_pixels_;
+
+ std::vector<bool> last_active_flags_;
+ VideoAdaptationCounters last_adaptation_counters_;
+ int last_input_width_;
+ int last_input_height_;
};
VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager(
@@ -283,7 +331,7 @@
encoder_settings_ = std::move(encoder_settings);
bitrate_constraint_->OnEncoderSettingsUpdated(encoder_settings_);
initial_frame_dropper_->OnEncoderSettingsUpdated(
- encoder_settings_->video_codec());
+ encoder_settings_->video_codec(), current_adaptation_counters_);
MaybeUpdateTargetFrameRate();
}
@@ -372,7 +420,7 @@
void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() {
RTC_DCHECK_RUN_ON(encoder_queue_);
- initial_frame_dropper_->OnMaybeEncodeFrame();
+ initial_frame_dropper_->Disable();
if (quality_rampup_experiment_ && quality_scaler_resource_->is_started()) {
DataRate bandwidth = encoder_rates_.has_value()
? encoder_rates_->bandwidth_allocation
@@ -488,6 +536,8 @@
rtc::scoped_refptr<Resource> reason,
const VideoSourceRestrictions& unfiltered_restrictions) {
RTC_DCHECK_RUN_ON(encoder_queue_);
+ current_adaptation_counters_ = adaptation_counters;
+
// TODO(bugs.webrtc.org/11553) Remove reason parameter and add reset callback.
if (!reason && adaptation_counters.Total() == 0) {
// Adaptation was manually reset - clear the per-reason counters too.
diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h
index 5190373..623d17a 100644
--- a/video/adaptation/video_stream_encoder_resource_manager.h
+++ b/video/adaptation/video_stream_encoder_resource_manager.h
@@ -184,6 +184,9 @@
VideoSourceRestrictions video_source_restrictions_
RTC_GUARDED_BY(encoder_queue_);
+ VideoAdaptationCounters current_adaptation_counters_
+ RTC_GUARDED_BY(encoder_queue_);
+
const BalancedDegradationSettings balanced_settings_;
Clock* clock_ RTC_GUARDED_BY(encoder_queue_);
const bool experiment_cpu_load_estimator_ RTC_GUARDED_BY(encoder_queue_);
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index 1e7dabb..4501e9f 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -4560,6 +4560,157 @@
video_stream_encoder_->Stop();
}
+TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
+ const int kLowTargetBitrateBps = 400000;
+ // Set simulcast.
+ ResetEncoder("VP8", 3, 1, 1, false);
+ fake_encoder_.SetQualityScaling(true);
+ const int kWidth = 1280;
+ const int kHeight = 720;
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kLowTargetBitrateBps),
+ DataRate::BitsPerSec(kLowTargetBitrateBps),
+ DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
+ // Frame should not be dropped.
+ WaitForEncodedFrame(1);
+
+ // Trigger QVGA "singlecast"
+ // Update the config.
+ VideoEncoderConfig video_encoder_config;
+ test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
+ &video_encoder_config);
+ for (auto& layer : video_encoder_config.simulcast_layers) {
+ layer.num_temporal_layers = 1;
+ layer.max_framerate = kDefaultFramerate;
+ }
+ video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
+ video_encoder_config.content_type =
+ VideoEncoderConfig::ContentType::kRealtimeVideo;
+
+ video_encoder_config.simulcast_layers[0].active = true;
+ video_encoder_config.simulcast_layers[1].active = false;
+ video_encoder_config.simulcast_layers[2].active = false;
+
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
+ // Frame should not be dropped.
+ WaitForEncodedFrame(2);
+
+ // Trigger HD "singlecast"
+ video_encoder_config.simulcast_layers[0].active = false;
+ video_encoder_config.simulcast_layers[1].active = false;
+ video_encoder_config.simulcast_layers[2].active = true;
+
+ video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
+ kMaxPayloadLength);
+ video_stream_encoder_->WaitUntilTaskQueueIsIdle();
+
+ video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
+ // Frame should be dropped because of initial frame drop.
+ ExpectDroppedFrame();
+
+ // Expect the sink_wants to specify a scaled frame.
+ EXPECT_TRUE_WAIT(
+ video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+ InitialFrameDropActivatesWhenResolutionIncreases) {
+ const int kWidth = 640;
+ const int kHeight = 360;
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
+ video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
+ // Frame should not be dropped.
+ WaitForEncodedFrame(1);
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kLowTargetBitrateBps),
+ DataRate::BitsPerSec(kLowTargetBitrateBps),
+ DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
+ video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
+ // Frame should not be dropped, bitrate not too low for frame.
+ WaitForEncodedFrame(2);
+
+ // Incoming resolution increases.
+ video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
+ // Expect to drop this frame, bitrate too low for frame.
+ ExpectDroppedFrame();
+
+ // Expect the sink_wants to specify a scaled frame.
+ EXPECT_TRUE_WAIT(
+ video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
+ video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
+ const int kWidth = 640;
+ const int kHeight = 360;
+ // So that quality scaling doesn't happen by itself.
+ fake_encoder_.SetQp(kQpHigh);
+
+ AdaptingFrameForwarder source(&time_controller_);
+ source.set_adaptation_enabled(true);
+ video_stream_encoder_->SetSource(
+ &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
+
+ int timestamp = 1;
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps),
+ DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
+ source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp);
+ timestamp += 9000;
+ // Long pause to disable all first BWE drop logic.
+ AdvanceTime(TimeDelta::Millis(1000));
+
+ video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+ DataRate::BitsPerSec(kLowTargetBitrateBps),
+ DataRate::BitsPerSec(kLowTargetBitrateBps),
+ DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
+ source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
+ // Not dropped frame, as initial frame drop is disabled by now.
+ WaitForEncodedFrame(timestamp);
+ timestamp += 9000;
+ AdvanceTime(TimeDelta::Millis(100));
+
+ // Quality adaptation down.
+ video_stream_encoder_->TriggerQualityLow();
+
+ // Adaptation has an effect.
+ EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
+ 5000);
+
+ // Frame isn't dropped as initial frame dropper is disabled.
+ source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
+ WaitForEncodedFrame(timestamp);
+ timestamp += 9000;
+ AdvanceTime(TimeDelta::Millis(100));
+
+ // Quality adaptation up.
+ video_stream_encoder_->TriggerQualityHigh();
+
+ // Adaptation has an effect.
+ EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
+ 5000);
+
+ source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
+ // Frame should not be dropped, as initial framedropper is off.
+ WaitForEncodedFrame(timestamp);
+
+ video_stream_encoder_->Stop();
+}
+
TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
webrtc::test::ScopedFieldTrials field_trials(
"WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
