FrameCadenceAdapter: improves performance under repeat and high load
This change ensures that the FCA now is informed about a new max fps
when VideoStreamEncoder::OnVideoSourceRestrictionsUpdated is called.
The latest restricted frame rate which is provided to the FCA will
only affect the cadence of repeated non-idle (quality has not
converged) frames and the main goal is to ensure that the FCA reduces
its repeat rate in situations where the video source is constrained.
UpdateVideoSourceRestrictions is added to the FrameCadenceAdapter API
and it is called from the VideoStreamEncoder when its source
parameters (resolution and/or frame rate) are restricted.
This modification has no effect on the flow driven by
ProcessOnDelayedCadence (non repeated frames).
Bug: webrtc:15539
Change-Id: I26dee6480e5137f82c5ccf57091b737cad82dbf6
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/328300
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Markus Handell <handellm@webrtc.org>
Commit-Queue: Henrik Andreassson <henrika@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#41308}
diff --git a/video/frame_cadence_adapter.cc b/video/frame_cadence_adapter.cc
index 59e84f9..2c4acdd 100644
--- a/video/frame_cadence_adapter.cc
+++ b/video/frame_cadence_adapter.cc
@@ -131,6 +131,10 @@
// Callback::RequestRefreshFrame.
void ProcessKeyFrameRequest();
+ // Updates the restrictions of max frame rate for the video source.
+ // Always called during construction using latest `restricted_frame_delay_`.
+ void UpdateVideoSourceRestrictions(absl::optional<double> max_frame_rate);
+
private:
// The tracking state of each spatial layer. Used for determining when to
// stop repeating frames.
@@ -173,10 +177,11 @@
// Processes incoming frames on a delayed cadence.
void ProcessOnDelayedCadence(Timestamp post_time)
RTC_RUN_ON(sequence_checker_);
- // Schedules a later repeat with delay depending on state of layer trackers.
+ // Schedules a later repeat with delay depending on state of layer trackers
+ // and if UpdateVideoSourceRestrictions has been called or not.
// If true is passed in `idle_repeat`, the repeat is going to be
- // kZeroHertzIdleRepeatRatePeriod. Otherwise it'll be the value of
- // `frame_delay`.
+ // kZeroHertzIdleRepeatRatePeriod. Otherwise it'll be the maximum value of
+ // `frame_delay` or `restricted_frame_delay_` if it has been set.
void ScheduleRepeat(int frame_id, bool idle_repeat)
RTC_RUN_ON(sequence_checker_);
// Repeats a frame in the absence of incoming frames. Slows down when quality
@@ -221,6 +226,10 @@
// they can be dropped in various places in the capture pipeline.
RepeatingTaskHandle refresh_frame_requester_
RTC_GUARDED_BY(sequence_checker_);
+ // Can be set by UpdateVideoSourceRestrictions when the video source restricts
+ // the max frame rate.
+ absl::optional<TimeDelta> restricted_frame_delay_
+ RTC_GUARDED_BY(sequence_checker_);
ScopedTaskSafety safety_;
};
@@ -241,6 +250,8 @@
void UpdateLayerQualityConvergence(size_t spatial_index,
bool quality_converged) override;
void UpdateLayerStatus(size_t spatial_index, bool enabled) override;
+ void UpdateVideoSourceRestrictions(
+ absl::optional<double> max_frame_rate) override;
void ProcessKeyFrameRequest() override;
// VideoFrameSink overrides.
@@ -292,6 +303,11 @@
absl::optional<VideoTrackSourceConstraints> source_constraints_
RTC_GUARDED_BY(queue_);
+ // Stores the latest restriction in max frame rate set by
+ // UpdateVideoSourceRestrictions. Ensures that a previously set restriction
+ // can be maintained during reconstructions of the adapter.
+ absl::optional<double> restricted_max_frame_rate_ RTC_GUARDED_BY(queue_);
+
// Race checker for incoming frames. This is the network thread in chromium,
// but may vary from test contexts.
rtc::RaceChecker incoming_frame_race_checker_;
@@ -412,6 +428,20 @@
return max_fps_;
}
+void ZeroHertzAdapterMode::UpdateVideoSourceRestrictions(
+ absl::optional<double> max_frame_rate) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
+ TRACE_EVENT_INSTANT1(TRACE_DISABLED_BY_DEFAULT("webrtc"), __func__,
+ "max_frame_rate", max_frame_rate.value_or(-1));
+ if (max_frame_rate.value_or(0) > 0) {
+ // Set new, validated (> 0) and restricted frame rate.
+ restricted_frame_delay_ = TimeDelta::Seconds(1) / *max_frame_rate;
+ } else {
+ // Source reports that the frame rate is now unrestricted.
+ restricted_frame_delay_ = absl::nullopt;
+ }
+}
+
void ZeroHertzAdapterMode::ProcessKeyFrameRequest() {
RTC_DCHECK_RUN_ON(&sequence_checker_);
TRACE_EVENT_INSTANT0("webrtc", __func__);
@@ -574,9 +604,14 @@
TimeDelta ZeroHertzAdapterMode::RepeatDuration(bool idle_repeat) const {
RTC_DCHECK_RUN_ON(&sequence_checker_);
+ // By default use `frame_delay_` in non-idle repeat mode but use the
+ // restricted frame delay instead if it is set in
+ // UpdateVideoSourceRestrictions.
+ TimeDelta frame_delay =
+ std::max(frame_delay_, restricted_frame_delay_.value_or(frame_delay_));
return idle_repeat
? FrameCadenceAdapterInterface::kZeroHertzIdleRepeatRatePeriod
- : frame_delay_;
+ : frame_delay;
}
void ZeroHertzAdapterMode::MaybeStartRefreshFrameRequester() {
@@ -649,6 +684,17 @@
zero_hertz_adapter_->UpdateLayerStatus(spatial_index, enabled);
}
+void FrameCadenceAdapterImpl::UpdateVideoSourceRestrictions(
+ absl::optional<double> max_frame_rate) {
+ RTC_DCHECK_RUN_ON(queue_);
+ // Store the restriction to ensure that it can be reapplied in possible
+ // future adapter creations on configuration changes.
+ restricted_max_frame_rate_ = max_frame_rate;
+ if (zero_hertz_adapter_) {
+ zero_hertz_adapter_->UpdateVideoSourceRestrictions(max_frame_rate);
+ }
+}
+
void FrameCadenceAdapterImpl::ProcessKeyFrameRequest() {
RTC_DCHECK_RUN_ON(queue_);
if (zero_hertz_adapter_)
@@ -742,11 +788,12 @@
bool max_fps_has_changed = GetInputFrameRateFps().value_or(-1) !=
source_constraints_->max_fps.value_or(-1);
if (!was_zero_hertz_enabled || max_fps_has_changed) {
- zero_hertz_adapter_.emplace(queue_, clock_, callback_,
- source_constraints_->max_fps.value());
- zero_hertz_adapter_is_active_.store(true, std::memory_order_relaxed);
RTC_LOG(LS_INFO) << "Zero hertz mode enabled (max_fps="
<< source_constraints_->max_fps.value() << ")";
+ zero_hertz_adapter_.emplace(queue_, clock_, callback_,
+ source_constraints_->max_fps.value());
+ zero_hertz_adapter_->UpdateVideoSourceRestrictions(
+ restricted_max_frame_rate_);
zero_hertz_adapter_created_timestamp_ = clock_->CurrentTime();
}
zero_hertz_adapter_->ReconfigureParameters(zero_hertz_params_.value());
diff --git a/video/frame_cadence_adapter.h b/video/frame_cadence_adapter.h
index 37c4a43..2b62bb2 100644
--- a/video/frame_cadence_adapter.h
+++ b/video/frame_cadence_adapter.h
@@ -109,6 +109,12 @@
// Updates spatial layer enabled status.
virtual void UpdateLayerStatus(size_t spatial_index, bool enabled) = 0;
+ // Updates the restrictions of max frame rate for the video source.
+ // The new `max_frame_rate` will only affect the cadence of Callback::OnFrame
+ // for non-idle (non converged) repeated frames.
+ virtual void UpdateVideoSourceRestrictions(
+ absl::optional<double> max_frame_rate) = 0;
+
// Conditionally requests a refresh frame via
// Callback::RequestRefreshFrame.
virtual void ProcessKeyFrameRequest() = 0;
diff --git a/video/frame_cadence_adapter_unittest.cc b/video/frame_cadence_adapter_unittest.cc
index 2fd40fe..0fef240 100644
--- a/video/frame_cadence_adapter_unittest.cc
+++ b/video/frame_cadence_adapter_unittest.cc
@@ -731,6 +731,8 @@
static constexpr TimeDelta kMinFrameDelay = TimeDelta::Millis(100);
static constexpr TimeDelta kIdleFrameDelay =
FrameCadenceAdapterInterface::kZeroHertzIdleRepeatRatePeriod;
+ // Restricts non-idle repeat rate to 5 fps (default is 10 fps);
+ static constexpr int kRestrictedMaxFps = 5;
ZeroHertzLayerQualityConvergenceTest() {
adapter_->Initialize(&callback_);
@@ -833,6 +835,100 @@
});
}
+TEST_F(ZeroHertzLayerQualityConvergenceTest,
+ UnconvergedRepeatRateAdaptsDownWhenRestricted) {
+ PassFrame();
+ ScheduleDelayed(1.5 * kMinFrameDelay, [&] {
+ adapter_->UpdateVideoSourceRestrictions(kRestrictedMaxFps);
+ });
+ ExpectFrameEntriesAtDelaysFromNow({
+ 1 * kMinFrameDelay, // Original frame emitted at non-restricted rate.
+
+ // 1.5 * kMinFrameDelay: restricts max fps to 5 fps which should result
+ // in a new non-idle repeat delay of 2 * kMinFrameDelay.
+ 2 * kMinFrameDelay, // Unconverged repeat at non-restricted rate.
+ 4 * kMinFrameDelay, // Unconverged repeats at restricted rate. This
+ // happens 2 * kMinFrameDelay after the last frame.
+ 6 * kMinFrameDelay, // ...
+ });
+}
+
+TEST_F(ZeroHertzLayerQualityConvergenceTest,
+ UnconvergedRepeatRateAdaptsUpWhenGoingFromRestrictedToUnrestricted) {
+ PassFrame();
+ ScheduleDelayed(1.5 * kMinFrameDelay, [&] {
+ adapter_->UpdateVideoSourceRestrictions(kRestrictedMaxFps);
+ });
+ ScheduleDelayed(5.5 * kMinFrameDelay, [&] {
+ adapter_->UpdateVideoSourceRestrictions(absl::nullopt);
+ });
+ ExpectFrameEntriesAtDelaysFromNow({
+ 1 * kMinFrameDelay, // Original frame emitted at non-restricted rate.
+
+ // 1.5 * kMinFrameDelay: restricts max fps to 5 fps which should result
+ // in a new non-idle repeat delay of 2 * kMinFrameDelay.
+ 2 * kMinFrameDelay, // Unconverged repeat at non-restricted rate.
+ 4 * kMinFrameDelay, // Unconverged repeat at restricted rate.
+
+ // 5.5 * kMinFrameDelay: removes frame-rate restriction and we should
+ // then go back to 10 fps as unconverged repeat rate.
+ 6 * kMinFrameDelay, // Last unconverged repeat at restricted rate.
+ 7 * kMinFrameDelay, // Back to unconverged repeat at non-restricted rate.
+ 8 * kMinFrameDelay, // We are now unrestricted.
+ 9 * kMinFrameDelay, // ...
+ });
+}
+
+TEST_F(ZeroHertzLayerQualityConvergenceTest,
+ UnconvergedRepeatRateMaintainsRestrictionOnReconfigureToHigherMaxFps) {
+ PassFrame();
+ ScheduleDelayed(1.5 * kMinFrameDelay, [&] {
+ adapter_->UpdateVideoSourceRestrictions(kRestrictedMaxFps);
+ });
+ ScheduleDelayed(2.5 * kMinFrameDelay, [&] {
+ adapter_->OnConstraintsChanged(VideoTrackSourceConstraints{
+ /*min_fps=*/0, /*max_fps=*/2 * TimeDelta::Seconds(1) / kMinFrameDelay});
+ });
+ ScheduleDelayed(3 * kMinFrameDelay, [&] { PassFrame(); });
+ ScheduleDelayed(8 * kMinFrameDelay, [&] {
+ adapter_->OnConstraintsChanged(VideoTrackSourceConstraints{
+ /*min_fps=*/0,
+ /*max_fps=*/0.2 * TimeDelta::Seconds(1) / kMinFrameDelay});
+ });
+ ScheduleDelayed(9 * kMinFrameDelay, [&] { PassFrame(); });
+ ExpectFrameEntriesAtDelaysFromNow({
+ 1 * kMinFrameDelay, // Original frame emitted at non-restricted rate.
+
+ // 1.5 * kMinFrameDelay: restricts max fps to 5 fps which should result
+ // in a new non-idle repeat delay of 2 * kMinFrameDelay.
+ 2 * kMinFrameDelay, // Unconverged repeat at non-restricted rate.
+
+ // 2.5 * kMinFrameDelay: new constraint asks for max rate of 20 fps.
+ // The 0Hz adapter is reconstructed for 20 fps but inherits the current
+ // restriction for rate of non-converged frames of 5 fps.
+
+ // A new frame is passed at 3 * kMinFrameDelay. The previous repeat
+ // cadence was stopped by the change in constraints.
+ 3.5 * kMinFrameDelay, // Original frame emitted at non-restricted 20 fps.
+ // The delay is 0.5 * kMinFrameDelay.
+ 5.5 * kMinFrameDelay, // Unconverged repeat at restricted rate.
+ // The delay is 2 * kMinFrameDelay when restricted.
+ 7.5 * kMinFrameDelay, // ...
+
+ // 8 * kMinFrameDelay: new constraint asks for max rate of 2 fps.
+ // The 0Hz adapter is reconstructed for 2 fps and will therefore not obey
+ // the current restriction for rate of non-converged frames of 5 fps
+ // since the new max rate is lower.
+
+ // A new frame is passed at 9 * kMinFrameDelay. The previous repeat
+ // cadence was stopped by the change in constraints.
+ 14 * kMinFrameDelay, // Original frame emitted at non-restricted 2 fps.
+ // The delay is 5 * kMinFrameDelay.
+ 19 * kMinFrameDelay, // Unconverged repeat at non-restricted rate.
+ 24 * kMinFrameDelay, // ...
+ });
+}
+
class FrameCadenceAdapterMetricsTest : public ::testing::Test {
public:
FrameCadenceAdapterMetricsTest() : time_controller_(Timestamp::Millis(1)) {
diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc
index 7144b57..552463e 100644
--- a/video/video_stream_encoder.cc
+++ b/video/video_stream_encoder.cc
@@ -2358,6 +2358,11 @@
<< (reason ? reason->Name() : std::string("<null>"))
<< " to " << restrictions.ToString();
+ if (frame_cadence_adapter_) {
+ frame_cadence_adapter_->UpdateVideoSourceRestrictions(
+ restrictions.max_frame_rate());
+ }
+
// TODO(webrtc:14451) Split video_source_sink_controller_
// so that ownership on restrictions/wants is kept on &encoder_queue_
latest_restrictions_ = restrictions;
diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h
index 2276d79..f2c21c1 100644
--- a/video/video_stream_encoder.h
+++ b/video/video_stream_encoder.h
@@ -132,6 +132,8 @@
double cwnd_reduce_ratio);
protected:
+ friend class VideoStreamEncoderFrameCadenceRestrictionTest;
+
// Used for testing. For example the `ScalingObserverInterface` methods must
// be called on `encoder_queue_`.
TaskQueueBase* encoder_queue() { return encoder_queue_.Get(); }
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index 06e772d..6fa9908 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -783,6 +783,10 @@
UpdateLayerStatus,
(size_t spatial_index, bool enabled),
(override));
+ MOCK_METHOD(void,
+ UpdateVideoSourceRestrictions,
+ (absl::optional<double>),
+ (override));
MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
};
@@ -9580,4 +9584,60 @@
kMaxFps);
}
+class VideoStreamEncoderFrameCadenceRestrictionTest : public ::testing::Test {
+ public:
+ VideoStreamEncoderFrameCadenceRestrictionTest()
+ : adapter_ptr_(adapter_.get()),
+ fake_resource_(FakeResource::Create("FakeResource")),
+ video_stream_encoder_(
+ factory_.Create(std::move(adapter_), &encoder_queue_)) {}
+
+ ~VideoStreamEncoderFrameCadenceRestrictionTest() {
+ factory_.DepleteTaskQueues();
+ }
+
+ void UpdateVideoSourceRestrictions(VideoSourceRestrictions restrictions) {
+ encoder_queue_->PostTask([this, restrictions] {
+ RTC_DCHECK_RUN_ON(encoder_queue_);
+ video_stream_encoder_->OnVideoSourceRestrictionsUpdated(
+ restrictions, VideoAdaptationCounters(), fake_resource_,
+ VideoSourceRestrictions());
+ });
+ }
+
+ protected:
+ SimpleVideoStreamEncoderFactory factory_;
+ std::unique_ptr<NiceMock<MockFrameCadenceAdapter>> adapter_{
+ std::make_unique<NiceMock<MockFrameCadenceAdapter>>()};
+ NiceMock<MockFrameCadenceAdapter>* adapter_ptr_;
+ TaskQueueBase* encoder_queue_{nullptr};
+ rtc::scoped_refptr<FakeResource> fake_resource_;
+ VideoSourceRestrictions restrictions_;
+ std::unique_ptr<SimpleVideoStreamEncoderFactory::AdaptedVideoStreamEncoder>
+ video_stream_encoder_;
+};
+
+TEST_F(VideoStreamEncoderFrameCadenceRestrictionTest,
+ UpdatesVideoSourceRestrictionsUnRestricted) {
+ EXPECT_CALL(*adapter_ptr_, UpdateVideoSourceRestrictions(Eq(absl::nullopt)));
+ UpdateVideoSourceRestrictions(VideoSourceRestrictions());
+}
+
+TEST_F(VideoStreamEncoderFrameCadenceRestrictionTest,
+ UpdatesVideoSourceRestrictionsWithMaxFrameRateRestriction) {
+ restrictions_.set_max_frame_rate(20);
+ EXPECT_CALL(*adapter_ptr_, UpdateVideoSourceRestrictions(Optional(20)));
+ UpdateVideoSourceRestrictions(restrictions_);
+}
+
+TEST_F(VideoStreamEncoderFrameCadenceRestrictionTest,
+ UpdatesVideoSourceRestrictionsWithoutMaxFrameRateRestriction) {
+ // Restrictions in resolution count as restriction updated, even though the
+ // FPS is unlimited.
+ restrictions_.set_max_pixels_per_frame(99);
+ restrictions_.set_target_pixels_per_frame(101);
+ EXPECT_CALL(*adapter_ptr_, UpdateVideoSourceRestrictions(Eq(absl::nullopt)));
+ UpdateVideoSourceRestrictions(restrictions_);
+}
+
} // namespace webrtc
