Enable initial frame drop for one active simulcast layer.
Bug: webrtc:12216
Change-Id: Ib2ac2fab45e560ba3eae30a926ce72667a257b07
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/193840
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Åsa Persson <asapersson@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32652}
diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc
index c7ca4bc..697ebbe 100644
--- a/video/adaptation/video_stream_encoder_resource_manager.cc
+++ b/video/adaptation/video_stream_encoder_resource_manager.cc
@@ -58,6 +58,20 @@
RTC_CHECK_NOTREACHED();
}
+absl::optional<uint32_t> GetSingleActiveStreamPixels(const VideoCodec& codec) {
+ int num_active = 0;
+ absl::optional<uint32_t> pixels;
+ for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
+ if (codec.simulcastStream[i].active) {
+ ++num_active;
+ pixels = codec.simulcastStream[i].width * codec.simulcastStream[i].height;
+ }
+ if (num_active > 1)
+ return absl::nullopt;
+ }
+ return pixels;
+}
+
} // namespace
class VideoStreamEncoderResourceManager::InitialFrameDropper {
@@ -78,6 +92,10 @@
return initial_framedrop_ < kMaxInitialFramedrop;
}
+ absl::optional<uint32_t> single_active_stream_pixels() const {
+ return single_active_stream_pixels_;
+ }
+
// Input signals.
void SetStartBitrate(DataRate start_bitrate, int64_t now_ms) {
set_start_bitrate_ = start_bitrate;
@@ -104,6 +122,10 @@
}
}
+ void OnEncoderSettingsUpdated(const VideoCodec& codec) {
+ single_active_stream_pixels_ = GetSingleActiveStreamPixels(codec);
+ }
+
void OnFrameDroppedDueToSize() { ++initial_framedrop_; }
void OnMaybeEncodeFrame() { initial_framedrop_ = kMaxInitialFramedrop; }
@@ -130,6 +152,7 @@
int64_t set_start_bitrate_time_ms_;
// Counts how many frames we've dropped in the initial framedrop phase.
int initial_framedrop_;
+ absl::optional<uint32_t> single_active_stream_pixels_;
};
VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager(
@@ -230,7 +253,7 @@
RTC_DCHECK(resource);
bool inserted;
std::tie(std::ignore, inserted) = resources_.emplace(resource, reason);
- RTC_DCHECK(inserted) << "Resurce " << resource->Name()
+ RTC_DCHECK(inserted) << "Resource " << resource->Name()
<< " already was inserted";
adaptation_processor_->AddResource(resource);
}
@@ -259,6 +282,8 @@
RTC_DCHECK_RUN_ON(encoder_queue_);
encoder_settings_ = std::move(encoder_settings);
bitrate_constraint_->OnEncoderSettingsUpdated(encoder_settings_);
+ initial_frame_dropper_->OnEncoderSettingsUpdated(
+ encoder_settings_->video_codec());
MaybeUpdateTargetFrameRate();
}
@@ -339,6 +364,12 @@
return initial_frame_dropper_->DropInitialFrames();
}
+absl::optional<uint32_t>
+VideoStreamEncoderResourceManager::SingleActiveStreamPixels() const {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ return initial_frame_dropper_->single_active_stream_pixels();
+}
+
void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() {
RTC_DCHECK_RUN_ON(encoder_queue_);
initial_frame_dropper_->OnMaybeEncodeFrame();
diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h
index 932d90c..5190373 100644
--- a/video/adaptation/video_stream_encoder_resource_manager.h
+++ b/video/adaptation/video_stream_encoder_resource_manager.h
@@ -121,9 +121,10 @@
VideoAdaptationReason reason);
void RemoveResource(rtc::scoped_refptr<Resource> resource);
std::vector<AdaptationConstraint*> AdaptationConstraints() const;
- // If true, the VideoStreamEncoder should eexecute its logic to maybe drop
- // frames baseed on size and bitrate.
+ // If true, the VideoStreamEncoder should execute its logic to maybe drop
+ // frames based on size and bitrate.
bool DropInitialFrames() const;
+ absl::optional<uint32_t> SingleActiveStreamPixels() const;
// VideoSourceRestrictionsListener implementation.
// Updates |video_source_restrictions_|.
diff --git a/video/quality_scaling_tests.cc b/video/quality_scaling_tests.cc
index 65a23db..b72b25b 100644
--- a/video/quality_scaling_tests.cc
+++ b/video/quality_scaling_tests.cc
@@ -56,6 +56,7 @@
protected:
void RunTest(VideoEncoderFactory* encoder_factory,
const std::string& payload_name,
+ const std::vector<bool>& streams_active,
int start_bps,
bool automatic_resize,
bool frame_dropping,
@@ -67,6 +68,7 @@
void QualityScalingTest::RunTest(VideoEncoderFactory* encoder_factory,
const std::string& payload_name,
+ const std::vector<bool>& streams_active,
int start_bps,
bool automatic_resize,
bool frame_dropping,
@@ -77,6 +79,7 @@
public:
ScalingObserver(VideoEncoderFactory* encoder_factory,
const std::string& payload_name,
+ const std::vector<bool>& streams_active,
int start_bps,
bool automatic_resize,
bool frame_dropping,
@@ -84,6 +87,7 @@
: SendTest(expect_adaptation ? kDefaultTimeoutMs : kTimeoutMs),
encoder_factory_(encoder_factory),
payload_name_(payload_name),
+ streams_active_(streams_active),
start_bps_(start_bps),
automatic_resize_(automatic_resize),
frame_dropping_(frame_dropping),
@@ -108,6 +112,10 @@
bitrate_config->start_bitrate_bps = start_bps_;
}
+ size_t GetNumVideoStreams() const override {
+ return streams_active_.size();
+ }
+
void ModifyVideoConfigs(
VideoSendStream::Config* send_config,
std::vector<VideoReceiveStream::Config>* receive_configs,
@@ -117,7 +125,15 @@
send_config->rtp.payload_type = kVideoSendPayloadType;
const VideoCodecType codec_type = PayloadStringToCodecType(payload_name_);
encoder_config->codec_type = codec_type;
- encoder_config->max_bitrate_bps = start_bps_;
+ encoder_config->max_bitrate_bps =
+ std::max(start_bps_, encoder_config->max_bitrate_bps);
+ double scale_factor = 1.0;
+ for (int i = streams_active_.size() - 1; i >= 0; --i) {
+ VideoStream& stream = encoder_config->simulcast_layers[i];
+ stream.active = streams_active_[i];
+ stream.scale_resolution_down_by = scale_factor;
+ scale_factor *= 2.0;
+ }
SetEncoderSpecific(encoder_config, codec_type, automatic_resize_,
frame_dropping_);
}
@@ -129,12 +145,13 @@
VideoEncoderFactory* const encoder_factory_;
const std::string payload_name_;
+ const std::vector<bool> streams_active_;
const int start_bps_;
const bool automatic_resize_;
const bool frame_dropping_;
const bool expect_adaptation_;
- } test(encoder_factory, payload_name, start_bps, automatic_resize,
- frame_dropping, expect_adaptation);
+ } test(encoder_factory, payload_name, streams_active, start_bps,
+ automatic_resize, frame_dropping, expect_adaptation);
RunBaseTest(&test);
}
@@ -150,7 +167,7 @@
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return VP8Encoder::Create(); });
- RunTest(&encoder_factory, "VP8", kHighStartBps, kAutomaticResize,
+ RunTest(&encoder_factory, "VP8", {true}, kHighStartBps, kAutomaticResize,
kFrameDropping, kExpectAdapt);
}
@@ -165,7 +182,7 @@
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return VP8Encoder::Create(); });
- RunTest(&encoder_factory, "VP8", kHighStartBps, kAutomaticResize,
+ RunTest(&encoder_factory, "VP8", {true}, kHighStartBps, kAutomaticResize,
kFrameDropping, kExpectAdapt);
}
@@ -182,7 +199,7 @@
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return VP8Encoder::Create(); });
- RunTest(&encoder_factory, "VP8", kHighStartBps, kAutomaticResize,
+ RunTest(&encoder_factory, "VP8", {true}, kHighStartBps, kAutomaticResize,
kFrameDropping, kExpectAdapt);
}
@@ -197,7 +214,7 @@
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return VP8Encoder::Create(); });
- RunTest(&encoder_factory, "VP8", kHighStartBps, kAutomaticResize,
+ RunTest(&encoder_factory, "VP8", {true}, kHighStartBps, kAutomaticResize,
kFrameDropping, kExpectAdapt);
}
@@ -212,10 +229,57 @@
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return VP8Encoder::Create(); });
- RunTest(&encoder_factory, "VP8", kLowStartBps, kAutomaticResize,
+ RunTest(&encoder_factory, "VP8", {true}, kLowStartBps, kAutomaticResize,
kFrameDropping, kExpectAdapt);
}
+TEST_F(QualityScalingTest, NoAdaptDownForLowStartBitrate_Simulcast) {
+ // VP8 QP thresholds, low:1, high:127 -> normal QP.
+ test::ScopedFieldTrials field_trials(kPrefix + "1,127,0,0,0,0" + kEnd);
+
+ // QualityScaler disabled.
+ const bool kAutomaticResize = false;
+ const bool kFrameDropping = true;
+ const bool kExpectAdapt = false;
+
+ test::FunctionVideoEncoderFactory encoder_factory(
+ []() { return VP8Encoder::Create(); });
+ RunTest(&encoder_factory, "VP8", {true, true}, kLowStartBps, kAutomaticResize,
+ kFrameDropping, kExpectAdapt);
+}
+
+TEST_F(QualityScalingTest,
+ AdaptsDownForLowStartBitrate_SimulcastOneActiveHighRes) {
+ // VP8 QP thresholds, low:1, high:127 -> normal QP.
+ test::ScopedFieldTrials field_trials(kPrefix + "1,127,0,0,0,0" + kEnd);
+
+ // QualityScaler enabled.
+ const bool kAutomaticResize = true;
+ const bool kFrameDropping = true;
+ const bool kExpectAdapt = true;
+
+ test::FunctionVideoEncoderFactory encoder_factory(
+ []() { return VP8Encoder::Create(); });
+ RunTest(&encoder_factory, "VP8", {false, false, true}, kLowStartBps,
+ kAutomaticResize, kFrameDropping, kExpectAdapt);
+}
+
+TEST_F(QualityScalingTest,
+ NoAdaptDownForLowStartBitrate_SimulcastOneActiveLowRes) {
+ // VP8 QP thresholds, low:1, high:127 -> normal QP.
+ test::ScopedFieldTrials field_trials(kPrefix + "1,127,0,0,0,0" + kEnd);
+
+ // QualityScaler enabled.
+ const bool kAutomaticResize = true;
+ const bool kFrameDropping = true;
+ const bool kExpectAdapt = false;
+
+ test::FunctionVideoEncoderFactory encoder_factory(
+ []() { return VP8Encoder::Create(); });
+ RunTest(&encoder_factory, "VP8", {true, false, false}, kLowStartBps,
+ kAutomaticResize, kFrameDropping, kExpectAdapt);
+}
+
TEST_F(QualityScalingTest, NoAdaptDownForLowStartBitrateWithScalingOff) {
// VP8 QP thresholds, low:1, high:127 -> normal QP.
test::ScopedFieldTrials field_trials(kPrefix + "1,127,0,0,0,0" + kEnd);
@@ -227,7 +291,7 @@
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return VP8Encoder::Create(); });
- RunTest(&encoder_factory, "VP8", kLowStartBps, kAutomaticResize,
+ RunTest(&encoder_factory, "VP8", {true}, kLowStartBps, kAutomaticResize,
kFrameDropping, kExpectAdapt);
}
@@ -243,7 +307,7 @@
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return VP9Encoder::Create(); });
- RunTest(&encoder_factory, "VP9", kHighStartBps, kAutomaticResize,
+ RunTest(&encoder_factory, "VP9", {true}, kHighStartBps, kAutomaticResize,
kFrameDropping, kExpectAdapt);
}
@@ -259,7 +323,7 @@
test::FunctionVideoEncoderFactory encoder_factory(
[]() { return H264Encoder::Create(cricket::VideoCodec("H264")); });
- RunTest(&encoder_factory, "H264", kHighStartBps, kAutomaticResize,
+ RunTest(&encoder_factory, "H264", {true}, kHighStartBps, kAutomaticResize,
kFrameDropping, kExpectAdapt);
}
#endif // defined(WEBRTC_USE_H264)
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index 836b20c..c197d23 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -565,6 +565,7 @@
void VideoStreamEncoder::SetStartBitrate(int start_bitrate_bps) {
encoder_queue_.PostTask([this, start_bitrate_bps] {
RTC_DCHECK_RUN_ON(&encoder_queue_);
+ RTC_LOG(LS_INFO) << "SetStartBitrate " << start_bitrate_bps;
encoder_target_bitrate_bps_ =
start_bitrate_bps != 0 ? absl::optional<uint32_t>(start_bitrate_bps)
: absl::nullopt;
@@ -1848,14 +1849,20 @@
bool simulcast_or_svc =
(send_codec_.codecType == VideoCodecType::kVideoCodecVP9 &&
send_codec_.VP9().numberOfSpatialLayers > 1) ||
- send_codec_.numberOfSimulcastStreams > 1 ||
- encoder_config_.simulcast_layers.size() > 1;
+ ((send_codec_.numberOfSimulcastStreams > 1 ||
+ encoder_config_.simulcast_layers.size() > 1) &&
+ !stream_resource_manager_.SingleActiveStreamPixels());
if (simulcast_or_svc || !stream_resource_manager_.DropInitialFrames() ||
!encoder_target_bitrate_bps_.has_value()) {
return false;
}
+ if (send_codec_.numberOfSimulcastStreams > 1 &&
+ stream_resource_manager_.SingleActiveStreamPixels()) {
+ pixel_count = stream_resource_manager_.SingleActiveStreamPixels().value();
+ }
+
absl::optional<VideoEncoder::ResolutionBitrateLimits> encoder_bitrate_limits =
encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution(
pixel_count);
