RtpEncodingParameters::request_resolution patch 3
This cl/ adds resource adapation to the requested_resolution
feature. The restrictions that are sent to the video source
are also saved inside video_stream_encoder and used when
determining layer resolution.
Anticipated further patches
4) Let VideoSource do adaption if possible
Bug: webrtc:14451
Change-Id: Ia9b990a6b92b76af7ff6665a562f84585f79c35b
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/277580
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Jonas Oreland <jonaso@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38306}
diff --git a/call/adaptation/video_source_restrictions.cc b/call/adaptation/video_source_restrictions.cc
index e9d6c26..719bc53 100644
--- a/call/adaptation/video_source_restrictions.cc
+++ b/call/adaptation/video_source_restrictions.cc
@@ -10,6 +10,7 @@
#include "call/adaptation/video_source_restrictions.h"
+#include <algorithm>
#include <limits>
#include "rtc_base/checks.h"
@@ -79,6 +80,30 @@
max_frame_rate_ = std::move(max_frame_rate);
}
+void VideoSourceRestrictions::UpdateMin(const VideoSourceRestrictions& other) {
+ if (max_pixels_per_frame_.has_value()) {
+ max_pixels_per_frame_ = std::min(*max_pixels_per_frame_,
+ other.max_pixels_per_frame().value_or(
+ std::numeric_limits<size_t>::max()));
+ } else {
+ max_pixels_per_frame_ = other.max_pixels_per_frame();
+ }
+ if (target_pixels_per_frame_.has_value()) {
+ target_pixels_per_frame_ = std::min(
+ *target_pixels_per_frame_, other.target_pixels_per_frame().value_or(
+ std::numeric_limits<size_t>::max()));
+ } else {
+ target_pixels_per_frame_ = other.target_pixels_per_frame();
+ }
+ if (max_frame_rate_.has_value()) {
+ max_frame_rate_ = std::min(
+ *max_frame_rate_,
+ other.max_frame_rate().value_or(std::numeric_limits<double>::max()));
+ } else {
+ max_frame_rate_ = other.max_frame_rate();
+ }
+}
+
bool DidRestrictionsIncrease(VideoSourceRestrictions before,
VideoSourceRestrictions after) {
bool decreased_resolution = DidDecreaseResolution(before, after);
diff --git a/call/adaptation/video_source_restrictions.h b/call/adaptation/video_source_restrictions.h
index 004cc09..be8520a 100644
--- a/call/adaptation/video_source_restrictions.h
+++ b/call/adaptation/video_source_restrictions.h
@@ -60,6 +60,9 @@
absl::optional<size_t> target_pixels_per_frame);
void set_max_frame_rate(absl::optional<double> max_frame_rate);
+ // Update `this` with min(`this`, `other`).
+ void UpdateMin(const VideoSourceRestrictions& other);
+
private:
// These map to rtc::VideoSinkWants's `max_pixel_count` and
// `target_pixel_count`.
diff --git a/call/adaptation/video_source_restrictions_unittest.cc b/call/adaptation/video_source_restrictions_unittest.cc
index 92e34f9..8c1ae4c 100644
--- a/call/adaptation/video_source_restrictions_unittest.cc
+++ b/call/adaptation/video_source_restrictions_unittest.cc
@@ -126,4 +126,21 @@
EXPECT_FALSE(DidRestrictionsDecrease(kHd, k15fps));
}
+TEST(VideoSourceRestrictions, UpdateMin) {
+ VideoSourceRestrictions one(kHdPixels / 2, kHdPixels, 7.0);
+ VideoSourceRestrictions two(kHdPixels, kHdPixels / 3, 15.0);
+
+ one.UpdateMin(two);
+
+ EXPECT_EQ(one.max_pixels_per_frame(), kHdPixels / 2);
+ EXPECT_EQ(one.target_pixels_per_frame(), kHdPixels / 3);
+ EXPECT_EQ(one.max_frame_rate(), 7.0);
+
+ two.UpdateMin(one);
+
+ EXPECT_EQ(two.max_pixels_per_frame(), kHdPixels / 2);
+ EXPECT_EQ(two.target_pixels_per_frame(), kHdPixels / 3);
+ EXPECT_EQ(two.max_frame_rate(), 7.0);
+}
+
} // namespace webrtc
diff --git a/video/BUILD.gn b/video/BUILD.gn
index 425e754..6b5eb2a 100644
--- a/video/BUILD.gn
+++ b/video/BUILD.gn
@@ -938,6 +938,7 @@
"adaptation:video_adaptation",
"config:encoder_config",
"config:streams_config",
+ "config:video_config_tests",
]
absl_deps = [
"//third_party/abseil-cpp/absl/algorithm:container",
diff --git a/video/config/BUILD.gn b/video/config/BUILD.gn
index 11871b8..8a5f03b 100644
--- a/video/config/BUILD.gn
+++ b/video/config/BUILD.gn
@@ -23,6 +23,7 @@
"../../api/units:data_rate",
"../../api/video:video_codec_constants",
"../../api/video_codecs:video_codecs_api",
+ "../../call/adaptation:resource_adaptation",
"../../media:rtc_media_base",
"../../modules/video_coding:video_coding_utility",
"../../modules/video_coding:webrtc_vp9_helpers",
@@ -69,10 +70,14 @@
testonly = true
defines = []
- sources = [ "simulcast_unittest.cc" ]
+ sources = [
+ "encoder_stream_factory_unittest.cc",
+ "simulcast_unittest.cc",
+ ]
deps = [
":streams_config",
"../../api/transport:field_trial_based_config",
+ "../../call/adaptation:resource_adaptation",
"../../test:field_trial",
"../../test:test_support",
]
diff --git a/video/config/encoder_stream_factory.cc b/video/config/encoder_stream_factory.cc
index c43922d..a413e4a 100644
--- a/video/config/encoder_stream_factory.cc
+++ b/video/config/encoder_stream_factory.cc
@@ -10,6 +10,7 @@
#include "video/config/encoder_stream_factory.h"
#include <algorithm>
+#include <limits>
#include <set>
#include <string>
#include <utility>
@@ -115,6 +116,7 @@
bool is_screenshare,
bool conference_mode,
const webrtc::VideoEncoder::EncoderInfo& encoder_info,
+ absl::optional<webrtc::VideoSourceRestrictions> restrictions,
const webrtc::FieldTrialsView* trials)
: codec_name_(codec_name),
max_qp_(max_qp),
@@ -122,7 +124,8 @@
conference_mode_(conference_mode),
trials_(trials ? *trials : fallback_trials_),
encoder_info_requested_resolution_alignment_(
- encoder_info.requested_resolution_alignment) {}
+ encoder_info.requested_resolution_alignment),
+ restrictions_(restrictions) {}
std::vector<webrtc::VideoStream> EncoderStreamFactory::CreateEncoderStreams(
int frame_width,
@@ -434,6 +437,16 @@
adapter.OnOutputFormatRequest(requested_resolution.ToPair(),
requested_resolution.PixelCount(),
absl::nullopt);
+ if (restrictions_) {
+ rtc::VideoSinkWants wants;
+ wants.is_active = true;
+ wants.target_pixel_count = restrictions_->target_pixels_per_frame();
+ wants.max_pixel_count =
+ rtc::dchecked_cast<int>(restrictions_->max_pixels_per_frame().value_or(
+ std::numeric_limits<int>::max()));
+ wants.resolution_alignment = encoder_info_requested_resolution_alignment_;
+ adapter.OnSinkWants(wants);
+ }
int cropped_width, cropped_height;
int out_width = 0, out_height = 0;
if (!adapter.AdaptFrameResolution(frame_width, frame_height, 0,
diff --git a/video/config/encoder_stream_factory.h b/video/config/encoder_stream_factory.h
index 11910eb..37abb93 100644
--- a/video/config/encoder_stream_factory.h
+++ b/video/config/encoder_stream_factory.h
@@ -16,6 +16,7 @@
#include "api/transport/field_trial_based_config.h"
#include "api/units/data_rate.h"
#include "api/video_codecs/video_encoder.h"
+#include "call/adaptation/video_source_restrictions.h"
#include "video/config/video_encoder_config.h"
namespace cricket {
@@ -34,14 +35,16 @@
bool is_screenshare,
bool conference_mode,
const webrtc::VideoEncoder::EncoderInfo& encoder_info,
+ absl::optional<webrtc::VideoSourceRestrictions>
+ restrictions = absl::nullopt,
const webrtc::FieldTrialsView* trials = nullptr);
- private:
std::vector<webrtc::VideoStream> CreateEncoderStreams(
int width,
int height,
const webrtc::VideoEncoderConfig& encoder_config) override;
+ private:
std::vector<webrtc::VideoStream> CreateDefaultVideoStreams(
int width,
int height,
@@ -69,6 +72,7 @@
const webrtc::FieldTrialBasedConfig fallback_trials_;
const webrtc::FieldTrialsView& trials_;
const int encoder_info_requested_resolution_alignment_;
+ const absl::optional<webrtc::VideoSourceRestrictions> restrictions_;
};
} // namespace cricket
diff --git a/video/config/encoder_stream_factory_unittest.cc b/video/config/encoder_stream_factory_unittest.cc
new file mode 100644
index 0000000..7214f40
--- /dev/null
+++ b/video/config/encoder_stream_factory_unittest.cc
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/config/encoder_stream_factory.h"
+
+#include "call/adaptation/video_source_restrictions.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+using cricket::EncoderStreamFactory;
+constexpr int kMaxQp = 48;
+
+namespace {
+
+std::vector<Resolution> GetStreamResolutions(
+ const std::vector<VideoStream>& streams) {
+ std::vector<Resolution> res;
+ for (const auto& s : streams) {
+ if (s.active) {
+ res.push_back(
+ {rtc::checked_cast<int>(s.width), rtc::checked_cast<int>(s.height)});
+ }
+ }
+ return res;
+}
+
+VideoStream LayerWithRequestedResolution(Resolution res) {
+ VideoStream s;
+ s.requested_resolution = res;
+ return s;
+}
+
+} // namespace
+
+TEST(EncoderStreamFactory, SinglecastRequestedResolution) {
+ VideoEncoder::EncoderInfo encoder_info;
+ auto factory = rtc::make_ref_counted<EncoderStreamFactory>(
+ "VP8", kMaxQp,
+ /* is_screenshare= */ false,
+ /* conference_mode= */ false, encoder_info);
+ VideoEncoderConfig encoder_config;
+ encoder_config.number_of_streams = 1;
+ encoder_config.simulcast_layers.push_back(
+ LayerWithRequestedResolution({.width = 640, .height = 360}));
+ auto streams = factory->CreateEncoderStreams(1280, 720, encoder_config);
+ EXPECT_EQ(GetStreamResolutions(streams), (std::vector<Resolution>{
+ {.width = 640, .height = 360},
+ }));
+}
+
+TEST(EncoderStreamFactory, SinglecastRequestedResolutionWithAdaptation) {
+ VideoSourceRestrictions restrictions(
+ /* max_pixels_per_frame= */ (320 * 320),
+ /* target_pixels_per_frame= */ absl::nullopt,
+ /* max_frame_rate= */ absl::nullopt);
+ VideoEncoder::EncoderInfo encoder_info;
+ auto factory = rtc::make_ref_counted<EncoderStreamFactory>(
+ "VP8", kMaxQp,
+ /* is_screenshare= */ false,
+ /* conference_mode= */ false, encoder_info, restrictions);
+ VideoEncoderConfig encoder_config;
+ encoder_config.number_of_streams = 1;
+ encoder_config.simulcast_layers.push_back(
+ LayerWithRequestedResolution({.width = 640, .height = 360}));
+ auto streams = factory->CreateEncoderStreams(1280, 720, encoder_config);
+ EXPECT_EQ(GetStreamResolutions(streams), (std::vector<Resolution>{
+ {.width = 320, .height = 180},
+ }));
+}
+
+} // namespace webrtc
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index deebc87..3178d9c 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -33,6 +33,7 @@
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_encoder.h"
#include "call/adaptation/resource_adaptation_processor.h"
+#include "call/adaptation/video_source_restrictions.h"
#include "call/adaptation/video_stream_adapter.h"
#include "modules/video_coding/include/video_codec_initializer.h"
#include "modules/video_coding/svc/svc_rate_allocator.h"
@@ -508,7 +509,7 @@
}
absl::optional<int> ParseVp9LowTierCoreCountThreshold(
- const webrtc::FieldTrialsView& trials) {
+ const FieldTrialsView& trials) {
FieldTrialFlag disable_low_tier("Disabled");
FieldTrialParameter<int> max_core_count("max_core_count", 2);
ParseFieldTrial({&disable_low_tier, &max_core_count},
@@ -519,6 +520,22 @@
return max_core_count.Get();
}
+absl::optional<VideoSourceRestrictions> MergeRestrictions(
+ const std::vector<absl::optional<VideoSourceRestrictions>>& list) {
+ absl::optional<VideoSourceRestrictions> return_value;
+ for (const auto& res : list) {
+ if (!res) {
+ continue;
+ }
+ if (!return_value) {
+ return_value = *res;
+ continue;
+ }
+ return_value->UpdateMin(*res);
+ }
+ return return_value;
+}
+
} // namespace
VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings()
@@ -949,7 +966,9 @@
encoder_config_.video_format.name, encoder_config_.max_qp,
encoder_config_.content_type ==
webrtc::VideoEncoderConfig::ContentType::kScreen,
- encoder_config_.legacy_conference_mode, encoder_->GetEncoderInfo());
+ encoder_config_.legacy_conference_mode, encoder_->GetEncoderInfo(),
+ MergeRestrictions({latest_restrictions_, animate_restrictions_}),
+ &field_trials_);
streams = factory->CreateEncoderStreams(
last_frame_info_->width, last_frame_info_->height, encoder_config_);
@@ -2313,6 +2332,11 @@
RTC_LOG(LS_INFO) << "Updating sink restrictions from "
<< (reason ? reason->Name() : std::string("<null>"))
<< " to " << restrictions.ToString();
+
+ // TODO(webrtc:14451) Split video_source_sink_controller_
+ // so that ownership on restrictions/wants is kept on &encoder_queue_
+ latest_restrictions_ = restrictions;
+
worker_queue_->PostTask(SafeTask(
task_safety_.flag(), [this, restrictions = std::move(restrictions)]() {
RTC_DCHECK_RUN_ON(worker_queue_);
@@ -2454,6 +2478,17 @@
RTC_LOG(LS_INFO) << "Removing resolution cap due to no consistent "
"animation detection.";
}
+ // TODO(webrtc:14451) Split video_source_sink_controller_
+ // so that ownership on restrictions/wants is kept on &encoder_queue_
+ if (should_cap_resolution) {
+ animate_restrictions_ =
+ VideoSourceRestrictions(kMaxAnimationPixels,
+ /* target_pixels_per_frame= */ absl::nullopt,
+ /* max_frame_rate= */ absl::nullopt);
+ } else {
+ animate_restrictions_.reset();
+ }
+
worker_queue_->PostTask(
SafeTask(task_safety_.flag(), [this, should_cap_resolution]() {
RTC_DCHECK_RUN_ON(worker_queue_);
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h
index 7a4fc32..9af2e0b 100644
--- a/video/video_stream_encoder.h
+++ b/video/video_stream_encoder.h
@@ -457,6 +457,20 @@
const absl::optional<int> vp9_low_tier_core_threshold_;
+ // These are copies of restrictions (glorified max_pixel_count) set by
+ // a) OnVideoSourceRestrictionsUpdated
+ // b) CheckForAnimatedContent
+ // They are used to scale down encoding resolution if needed when using
+ // requested_resolution.
+ //
+ // TODO(webrtc:14451) Split video_source_sink_controller_
+ // so that ownership on restrictions/wants is kept on &encoder_queue_, that
+ // these extra copies would not be needed.
+ absl::optional<VideoSourceRestrictions> latest_restrictions_
+ RTC_GUARDED_BY(&encoder_queue_);
+ absl::optional<VideoSourceRestrictions> animate_restrictions_
+ RTC_GUARDED_BY(&encoder_queue_);
+
// Used to cancel any potentially pending tasks to the worker thread.
// Refrenced by tasks running on `encoder_queue_` so need to be destroyed
// after stopping that queue. Must be created and destroyed on
