blob: 8a7852b369b52984a33bec38f306dba76bc525f5 [file] [log] [blame]
Markus Handellb4e96d42021-11-05 11:00:551/*
2 * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "video/frame_cadence_adapter.h"
12
Markus Handell8fa86192023-08-31 09:59:0613#include <algorithm>
Markus Handell9a478b52021-11-18 15:07:0114#include <atomic>
Per Ke975b442024-03-27 10:28:4415#include <cstdint>
Markus Handell29dd8d82021-12-15 11:19:1516#include <deque>
Markus Handellb4e96d42021-11-05 11:00:5517#include <memory>
Markus Handell28c71802021-11-08 09:11:5518#include <utility>
Markus Handell8d87c462021-12-16 10:37:1619#include <vector>
Markus Handellb4e96d42021-11-05 11:00:5520
Markus Handell8d87c462021-12-16 10:37:1621#include "absl/algorithm/container.h"
Markus Handell2e0f4f02021-12-21 18:14:5822#include "absl/base/attributes.h"
Zhaoliang Maf089d7e2024-01-08 02:44:1523#include "absl/cleanup/cleanup.h"
Markus Handellb4e96d42021-11-05 11:00:5524#include "api/sequence_checker.h"
Artem Titovc374d112022-06-16 19:27:4525#include "api/task_queue/pending_task_safety_flag.h"
Markus Handell28c71802021-11-08 09:11:5526#include "api/task_queue/task_queue_base.h"
Markus Handell29dd8d82021-12-15 11:19:1527#include "api/units/time_delta.h"
Markus Handell9d04a782022-05-12 16:38:5728#include "api/units/timestamp.h"
Markus Handell29dd8d82021-12-15 11:19:1529#include "api/video/video_frame.h"
Markus Handell8d87c462021-12-16 10:37:1630#include "rtc_base/checks.h"
Markus Handellb4e96d42021-11-05 11:00:5531#include "rtc_base/logging.h"
32#include "rtc_base/race_checker.h"
Markus Handellee225432021-11-29 11:35:1233#include "rtc_base/rate_statistics.h"
Markus Handellb4e96d42021-11-05 11:00:5534#include "rtc_base/synchronization/mutex.h"
Markus Handellee225432021-11-29 11:35:1235#include "rtc_base/system/no_unique_address.h"
henrika1b573a72023-10-20 10:39:4636#include "rtc_base/system/unused.h"
Markus Handellf7f0b212022-05-24 16:39:3937#include "rtc_base/task_utils/repeating_task.h"
Markus Handell29dd8d82021-12-15 11:19:1538#include "rtc_base/thread_annotations.h"
39#include "rtc_base/time_utils.h"
Markus Handellf2827c42023-09-02 07:41:1040#include "rtc_base/trace_event.h"
Markus Handell9a478b52021-11-18 15:07:0141#include "system_wrappers/include/clock.h"
Markus Handellb4e96d42021-11-05 11:00:5542#include "system_wrappers/include/metrics.h"
Markus Handell8d87c462021-12-16 10:37:1643#include "system_wrappers/include/ntp_time.h"
Markus Handellb4e96d42021-11-05 11:00:5544
45namespace webrtc {
46namespace {
47
Markus Handellee225432021-11-29 11:35:1248// Abstracts concrete modes of the cadence adapter.
49class AdapterMode {
50 public:
51 virtual ~AdapterMode() = default;
52
53 // Called on the worker thread for every frame that enters.
54 virtual void OnFrame(Timestamp post_time,
Markus Handell84c016a2023-11-23 12:41:4455 bool queue_overload,
Markus Handellee225432021-11-29 11:35:1256 const VideoFrame& frame) = 0;
57
58 // Returns the currently estimated input framerate.
59 virtual absl::optional<uint32_t> GetInputFrameRateFps() = 0;
60
61 // Updates the frame rate.
Per Ke975b442024-03-27 10:28:4462 virtual void UpdateFrameRate(Timestamp frame_timestamp) = 0;
Markus Handellee225432021-11-29 11:35:1263};
64
65// Implements a pass-through adapter. Single-threaded.
66class PassthroughAdapterMode : public AdapterMode {
67 public:
Per Ke975b442024-03-27 10:28:4468 explicit PassthroughAdapterMode(
69 FrameCadenceAdapterInterface::Callback* callback)
70 : callback_(callback) {
Markus Handellee225432021-11-29 11:35:1271 sequence_checker_.Detach();
72 }
73
74 // Adapter overrides.
75 void OnFrame(Timestamp post_time,
Markus Handell84c016a2023-11-23 12:41:4476 bool queue_overload,
Markus Handellee225432021-11-29 11:35:1277 const VideoFrame& frame) override {
78 RTC_DCHECK_RUN_ON(&sequence_checker_);
Markus Handell84c016a2023-11-23 12:41:4479 callback_->OnFrame(post_time, queue_overload, frame);
Markus Handellee225432021-11-29 11:35:1280 }
81
82 absl::optional<uint32_t> GetInputFrameRateFps() override {
83 RTC_DCHECK_RUN_ON(&sequence_checker_);
Per Ke975b442024-03-27 10:28:4484 return last_frame_rate_;
Markus Handellee225432021-11-29 11:35:1285 }
86
Per Ke975b442024-03-27 10:28:4487 void UpdateFrameRate(Timestamp frame_timestamp) override {
Markus Handellee225432021-11-29 11:35:1288 RTC_DCHECK_RUN_ON(&sequence_checker_);
Per Ke975b442024-03-27 10:28:4489 // RateStatistics will calculate a too high rate immediately after Update.
90 last_frame_rate_ = input_framerate_.Rate(frame_timestamp.ms());
91 input_framerate_.Update(1, frame_timestamp.ms());
Markus Handellee225432021-11-29 11:35:1292 }
93
94 private:
Per Ke975b442024-03-27 10:28:4495 absl::optional<uint64_t> last_frame_rate_;
Markus Handellee225432021-11-29 11:35:1296 FrameCadenceAdapterInterface::Callback* const callback_;
97 RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_;
98 // Input frame rate statistics for use when not in zero-hertz mode.
99 RateStatistics input_framerate_ RTC_GUARDED_BY(sequence_checker_){
100 FrameCadenceAdapterInterface::kFrameRateAveragingWindowSizeMs, 1000};
101};
102
103// Implements a frame cadence adapter supporting zero-hertz input.
104class ZeroHertzAdapterMode : public AdapterMode {
105 public:
Markus Handelle59fee82021-12-23 08:29:23106 ZeroHertzAdapterMode(TaskQueueBase* queue,
107 Clock* clock,
108 FrameCadenceAdapterInterface::Callback* callback,
henrikab7ec0572024-01-09 09:48:52109 double max_fps,
110 std::atomic<int>& frames_scheduled_for_processing,
111 bool zero_hertz_queue_overload);
Markus Handellfb98b012023-09-13 22:31:11112 ~ZeroHertzAdapterMode() { refresh_frame_requester_.Stop(); }
Markus Handelle59fee82021-12-23 08:29:23113
114 // Reconfigures according to parameters.
115 // All spatial layer trackers are initialized as unconverged by this method.
116 void ReconfigureParameters(
117 const FrameCadenceAdapterInterface::ZeroHertzModeParams& params);
Markus Handell8d87c462021-12-16 10:37:16118
119 // Updates spatial layer quality convergence status.
Markus Handell5a77e512022-09-01 12:51:50120 void UpdateLayerQualityConvergence(size_t spatial_index,
121 bool quality_converged);
Markus Handell8d87c462021-12-16 10:37:16122
123 // Updates spatial layer enabled status.
Markus Handell5a77e512022-09-01 12:51:50124 void UpdateLayerStatus(size_t spatial_index, bool enabled);
Markus Handellee225432021-11-29 11:35:12125
126 // Adapter overrides.
127 void OnFrame(Timestamp post_time,
Markus Handell84c016a2023-11-23 12:41:44128 bool queue_overload,
Markus Handellee225432021-11-29 11:35:12129 const VideoFrame& frame) override;
130 absl::optional<uint32_t> GetInputFrameRateFps() override;
Per Ke975b442024-03-27 10:28:44131 void UpdateFrameRate(Timestamp frame_timestamp) override {}
Markus Handellee225432021-11-29 11:35:12132
Markus Handellf5a50792022-06-16 12:25:15133 // Notified on dropped frames.
134 void OnDiscardedFrame();
135
Markus Handell818e7fb2021-12-30 12:01:33136 // Conditionally requests a refresh frame via
137 // Callback::RequestRefreshFrame.
138 void ProcessKeyFrameRequest();
Markus Handell2e0f4f02021-12-21 18:14:58139
henrika8f16ce92023-12-04 13:03:52140 // Updates the restrictions of max frame rate for the video source.
141 // Always called during construction using latest `restricted_frame_delay_`.
142 void UpdateVideoSourceRestrictions(absl::optional<double> max_frame_rate);
143
Markus Handellee225432021-11-29 11:35:12144 private:
Markus Handell8d87c462021-12-16 10:37:16145 // The tracking state of each spatial layer. Used for determining when to
146 // stop repeating frames.
147 struct SpatialLayerTracker {
148 // If unset, the layer is disabled. Otherwise carries the quality
149 // convergence status of the layer.
150 absl::optional<bool> quality_converged;
151 };
Markus Handell2e0f4f02021-12-21 18:14:58152 // The state of a scheduled repeat.
153 struct ScheduledRepeat {
Markus Handell90a7e2c2021-12-29 22:32:30154 ScheduledRepeat(Timestamp origin,
155 int64_t origin_timestamp_us,
156 int64_t origin_ntp_time_ms)
157 : scheduled(origin),
158 idle(false),
159 origin(origin),
160 origin_timestamp_us(origin_timestamp_us),
161 origin_ntp_time_ms(origin_ntp_time_ms) {}
Markus Handell2e0f4f02021-12-21 18:14:58162 // The instant when the repeat was scheduled.
163 Timestamp scheduled;
164 // True if the repeat was scheduled as an idle repeat (long), false
165 // otherwise.
166 bool idle;
Markus Handell90a7e2c2021-12-29 22:32:30167 // The moment we decided to start repeating.
168 Timestamp origin;
169 // The timestamp_us of the frame when we started repeating.
170 int64_t origin_timestamp_us;
171 // The ntp_times_ms of the frame when we started repeating.
172 int64_t origin_ntp_time_ms;
Markus Handell2e0f4f02021-12-21 18:14:58173 };
Markus Handell8d87c462021-12-16 10:37:16174
Markus Handell2e0f4f02021-12-21 18:14:58175 // Returns true if all spatial layers can be considered to be converged in
176 // terms of quality.
177 // Convergence means QP has dropped to a low-enough level to warrant ceasing
178 // to send identical frames at high frequency.
179 bool HasQualityConverged() const RTC_RUN_ON(sequence_checker_);
Markus Handelle59fee82021-12-23 08:29:23180 // Resets quality convergence information. HasQualityConverged() returns false
181 // after this call.
182 void ResetQualityConvergenceInfo() RTC_RUN_ON(sequence_checker_);
Markus Handell29dd8d82021-12-15 11:19:15183 // Processes incoming frames on a delayed cadence.
henrikaf7cdcbd2023-11-20 18:32:54184 void ProcessOnDelayedCadence(Timestamp post_time)
185 RTC_RUN_ON(sequence_checker_);
henrika8f16ce92023-12-04 13:03:52186 // Schedules a later repeat with delay depending on state of layer trackers
187 // and if UpdateVideoSourceRestrictions has been called or not.
Markus Handell2e0f4f02021-12-21 18:14:58188 // If true is passed in `idle_repeat`, the repeat is going to be
henrika8f16ce92023-12-04 13:03:52189 // kZeroHertzIdleRepeatRatePeriod. Otherwise it'll be the maximum value of
190 // `frame_delay` or `restricted_frame_delay_` if it has been set.
Markus Handell2e0f4f02021-12-21 18:14:58191 void ScheduleRepeat(int frame_id, bool idle_repeat)
192 RTC_RUN_ON(sequence_checker_);
henrikaf7cdcbd2023-11-20 18:32:54193 // Repeats a frame in the absence of incoming frames. Slows down when quality
Markus Handell29dd8d82021-12-15 11:19:15194 // convergence is attained, and stops the cadence terminally when new frames
Markus Handell2e0f4f02021-12-21 18:14:58195 // have arrived.
196 void ProcessRepeatedFrameOnDelayedCadence(int frame_id)
Markus Handell29dd8d82021-12-15 11:19:15197 RTC_RUN_ON(sequence_checker_);
henrikab7ec0572024-01-09 09:48:52198 // Sends a frame, updating the timestamp to the current time. Also updates
199 // `queue_overload_count_` based on the time it takes to encode a frame and
200 // the amount of received frames while encoding. The `queue_overload`
201 // parameter in the OnFrame callback will be true while
202 // `queue_overload_count_` is larger than zero to allow the client to drop
203 // frames and thereby mitigate delay buildups.
204 // Repeated frames are sent with `post_time` set to absl::nullopt.
205 void SendFrameNow(absl::optional<Timestamp> post_time,
206 const VideoFrame& frame) RTC_RUN_ON(sequence_checker_);
Markus Handell2e0f4f02021-12-21 18:14:58207 // Returns the repeat duration depending on if it's an idle repeat or not.
208 TimeDelta RepeatDuration(bool idle_repeat) const
209 RTC_RUN_ON(sequence_checker_);
henrikab7ec0572024-01-09 09:48:52210 // Returns the frame duration taking potential restrictions into account.
211 TimeDelta FrameDuration() const RTC_RUN_ON(sequence_checker_);
Markus Handellf5a50792022-06-16 12:25:15212 // Unless timer already running, starts repeatedly requesting refresh frames
213 // after a grace_period. If a frame appears before the grace_period has
214 // passed, the request is cancelled.
215 void MaybeStartRefreshFrameRequester() RTC_RUN_ON(sequence_checker_);
Markus Handell29dd8d82021-12-15 11:19:15216
217 TaskQueueBase* const queue_;
218 Clock* const clock_;
Markus Handellee225432021-11-29 11:35:12219 FrameCadenceAdapterInterface::Callback* const callback_;
Markus Handell9d04a782022-05-12 16:38:57220
Markus Handellee225432021-11-29 11:35:12221 // The configured max_fps.
222 // TODO(crbug.com/1255737): support max_fps updates.
223 const double max_fps_;
henrikab7ec0572024-01-09 09:48:52224
225 // Number of frames that are currently scheduled for processing on the
226 // `queue_`.
227 const std::atomic<int>& frames_scheduled_for_processing_;
228
229 // Can be used as kill-switch for the queue overload mechanism.
230 const bool zero_hertz_queue_overload_enabled_;
231
Markus Handell29dd8d82021-12-15 11:19:15232 // How much the incoming frame sequence is delayed by.
233 const TimeDelta frame_delay_ = TimeDelta::Seconds(1) / max_fps_;
234
Markus Handellee225432021-11-29 11:35:12235 RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_;
Markus Handell29dd8d82021-12-15 11:19:15236 // A queue of incoming frames and repeated frames.
237 std::deque<VideoFrame> queued_frames_ RTC_GUARDED_BY(sequence_checker_);
238 // The current frame ID to use when starting to repeat frames. This is used
239 // for cancelling deferred repeated frame processing happening.
240 int current_frame_id_ RTC_GUARDED_BY(sequence_checker_) = 0;
Markus Handell2e0f4f02021-12-21 18:14:58241 // Has content when we are repeating frames.
242 absl::optional<ScheduledRepeat> scheduled_repeat_
243 RTC_GUARDED_BY(sequence_checker_);
Markus Handell8d87c462021-12-16 10:37:16244 // Convergent state of each of the configured simulcast layers.
245 std::vector<SpatialLayerTracker> layer_trackers_
246 RTC_GUARDED_BY(sequence_checker_);
Markus Handellf7f0b212022-05-24 16:39:39247 // Repeating task handle used for requesting refresh frames until arrival, as
248 // they can be dropped in various places in the capture pipeline.
249 RepeatingTaskHandle refresh_frame_requester_
250 RTC_GUARDED_BY(sequence_checker_);
henrika8f16ce92023-12-04 13:03:52251 // Can be set by UpdateVideoSourceRestrictions when the video source restricts
252 // the max frame rate.
253 absl::optional<TimeDelta> restricted_frame_delay_
254 RTC_GUARDED_BY(sequence_checker_);
henrikab7ec0572024-01-09 09:48:52255 // Set in OnSendFrame to reflect how many future frames will be forwarded with
256 // the `queue_overload` flag set to true.
257 int queue_overload_count_ RTC_GUARDED_BY(sequence_checker_) = 0;
Markus Handell29dd8d82021-12-15 11:19:15258
259 ScopedTaskSafety safety_;
Markus Handellee225432021-11-29 11:35:12260};
261
Zhaoliang Maf089d7e2024-01-08 02:44:15262// Implements a frame cadence adapter supporting VSync aligned encoding.
263class VSyncEncodeAdapterMode : public AdapterMode {
264 public:
265 VSyncEncodeAdapterMode(
266 Clock* clock,
267 TaskQueueBase* queue,
268 rtc::scoped_refptr<PendingTaskSafetyFlag> queue_safety_flag,
269 Metronome* metronome,
270 TaskQueueBase* worker_queue,
271 FrameCadenceAdapterInterface::Callback* callback)
272 : clock_(clock),
273 queue_(queue),
274 queue_safety_flag_(queue_safety_flag),
275 callback_(callback),
276 metronome_(metronome),
277 worker_queue_(worker_queue) {
278 queue_sequence_checker_.Detach();
279 worker_sequence_checker_.Detach();
280 }
281
Markus Handelle864dec2024-08-07 14:11:18282 void PrepareShutdown() {
283 MutexLock lock(&queue_lock_);
284 queue_ = nullptr;
285 }
286
Zhaoliang Maf089d7e2024-01-08 02:44:15287 // Adapter overrides.
288 void OnFrame(Timestamp post_time,
289 bool queue_overload,
290 const VideoFrame& frame) override;
291
292 absl::optional<uint32_t> GetInputFrameRateFps() override {
293 RTC_DCHECK_RUN_ON(&queue_sequence_checker_);
Per Ke975b442024-03-27 10:28:44294 return last_frame_rate_;
Zhaoliang Maf089d7e2024-01-08 02:44:15295 }
296
Per Ke975b442024-03-27 10:28:44297 void UpdateFrameRate(Timestamp frame_timestamp) override {
Zhaoliang Maf089d7e2024-01-08 02:44:15298 RTC_DCHECK_RUN_ON(&queue_sequence_checker_);
Per Ke975b442024-03-27 10:28:44299 // RateStatistics will calculate a too high rate immediately after Update.
300 last_frame_rate_ = input_framerate_.Rate(frame_timestamp.ms());
301 input_framerate_.Update(1, frame_timestamp.ms());
Zhaoliang Maf089d7e2024-01-08 02:44:15302 }
303
304 void EncodeAllEnqueuedFrames();
305
306 private:
307 // Holds input frames coming from the client ready to be encoded.
308 struct InputFrameRef {
309 InputFrameRef(const VideoFrame& video_frame, Timestamp time_when_posted_us)
310 : time_when_posted_us(time_when_posted_us),
311 video_frame(std::move(video_frame)) {}
312 Timestamp time_when_posted_us;
313 const VideoFrame video_frame;
314 };
315
316 Clock* const clock_;
Markus Handelle864dec2024-08-07 14:11:18317 // Protects `queue_`.
318 // TODO: crbug.com/358040973 - We should eventually figure out a way to avoid
319 // lock protection.
320 Mutex queue_lock_;
321 TaskQueueBase* queue_ RTC_GUARDED_BY(queue_lock_)
322 RTC_PT_GUARDED_BY(queue_lock_);
Zhaoliang Maf089d7e2024-01-08 02:44:15323 RTC_NO_UNIQUE_ADDRESS SequenceChecker queue_sequence_checker_;
324 rtc::scoped_refptr<PendingTaskSafetyFlag> queue_safety_flag_;
325 // Input frame rate statistics for use when not in zero-hertz mode.
Per Ke975b442024-03-27 10:28:44326 absl::optional<uint64_t> last_frame_rate_
327 RTC_GUARDED_BY(queue_sequence_checker_);
Zhaoliang Maf089d7e2024-01-08 02:44:15328 RateStatistics input_framerate_ RTC_GUARDED_BY(queue_sequence_checker_){
329 FrameCadenceAdapterInterface::kFrameRateAveragingWindowSizeMs, 1000};
330 FrameCadenceAdapterInterface::Callback* const callback_;
331
332 Metronome* metronome_;
333 TaskQueueBase* const worker_queue_;
334 RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_sequence_checker_;
Per Ke975b442024-03-27 10:28:44335 // `worker_safety_` protects tasks on the worker queue related to
336 // `metronome_` since metronome usage must happen on worker thread.
Zhaoliang Maf089d7e2024-01-08 02:44:15337 ScopedTaskSafetyDetached worker_safety_;
338 Timestamp expected_next_tick_ RTC_GUARDED_BY(worker_sequence_checker_) =
339 Timestamp::PlusInfinity();
340 // Vector of input frames to be encoded.
341 std::vector<InputFrameRef> input_queue_
342 RTC_GUARDED_BY(worker_sequence_checker_);
343};
344
Markus Handellb4e96d42021-11-05 11:00:55345class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface {
346 public:
Jonas Oreland8ca06132022-03-14 11:52:48347 FrameCadenceAdapterImpl(Clock* clock,
348 TaskQueueBase* queue,
Zhaoliang Maf089d7e2024-01-08 02:44:15349 Metronome* metronome,
350 TaskQueueBase* worker_queue,
Jonas Orelande62c2f22022-03-29 09:04:48351 const FieldTrialsView& field_trials);
Markus Handelle59fee82021-12-23 08:29:23352 ~FrameCadenceAdapterImpl();
Markus Handellb4e96d42021-11-05 11:00:55353
354 // FrameCadenceAdapterInterface overrides.
355 void Initialize(Callback* callback) override;
Markus Handell8d87c462021-12-16 10:37:16356 void SetZeroHertzModeEnabled(
357 absl::optional<ZeroHertzModeParams> params) override;
Markus Handellee225432021-11-29 11:35:12358 absl::optional<uint32_t> GetInputFrameRateFps() override;
Markus Handell5a77e512022-09-01 12:51:50359 void UpdateLayerQualityConvergence(size_t spatial_index,
Markus Handell8d87c462021-12-16 10:37:16360 bool quality_converged) override;
Markus Handell5a77e512022-09-01 12:51:50361 void UpdateLayerStatus(size_t spatial_index, bool enabled) override;
henrika8f16ce92023-12-04 13:03:52362 void UpdateVideoSourceRestrictions(
363 absl::optional<double> max_frame_rate) override;
Markus Handell818e7fb2021-12-30 12:01:33364 void ProcessKeyFrameRequest() override;
Markus Handellb4e96d42021-11-05 11:00:55365
366 // VideoFrameSink overrides.
367 void OnFrame(const VideoFrame& frame) override;
Markus Handellf5a50792022-06-16 12:25:15368 void OnDiscardedFrame() override;
Markus Handellb4e96d42021-11-05 11:00:55369 void OnConstraintsChanged(
370 const VideoTrackSourceConstraints& constraints) override;
371
372 private:
Per Ke975b442024-03-27 10:28:44373 void UpdateFrameRate(Timestamp frame_timestamp);
henrika86f09ae2023-10-31 13:49:40374 // Called from OnFrame in both pass-through and zero-hertz mode.
Markus Handell9a478b52021-11-18 15:07:01375 void OnFrameOnMainQueue(Timestamp post_time,
Ilya Nikolaevskiy7a841ce2023-11-24 12:01:21376 bool queue_overload,
Markus Handell9a478b52021-11-18 15:07:01377 const VideoFrame& frame) RTC_RUN_ON(queue_);
Markus Handell28c71802021-11-08 09:11:55378
Markus Handellee225432021-11-29 11:35:12379 // Returns true under all of the following conditions:
380 // - constraints min fps set to 0
381 // - constraints max fps set and greater than 0,
382 // - field trial enabled
383 // - zero-hertz mode enabled
384 bool IsZeroHertzScreenshareEnabled() const RTC_RUN_ON(queue_);
385
Zhaoliang Maf089d7e2024-01-08 02:44:15386 // Configures current adapter on non-ZeroHertz mode, called when Initialize or
387 // MaybeReconfigureAdapters.
388 void ConfigureCurrentAdapterWithoutZeroHertz();
389
Markus Handellee225432021-11-29 11:35:12390 // Handles adapter creation on configuration changes.
391 void MaybeReconfigureAdapters(bool was_zero_hertz_enabled) RTC_RUN_ON(queue_);
392
Markus Handell9a478b52021-11-18 15:07:01393 Clock* const clock_;
394 TaskQueueBase* const queue_;
Markus Handellb4e96d42021-11-05 11:00:55395
henrikab7ec0572024-01-09 09:48:52396 // Kill-switch for the queue overload mechanism in zero-hertz mode.
397 const bool frame_cadence_adapter_zero_hertz_queue_overload_enabled_;
398
Per Ke975b442024-03-27 10:28:44399 // Field trial for using timestamp from video frames, rather than clock when
400 // calculating input frame rate.
401 const bool use_video_frame_timestamp_;
402 // Used for verifying that timestamps are monotonically increasing.
403 absl::optional<Timestamp> last_incoming_frame_timestamp_;
404 bool incoming_frame_timestamp_monotonically_increasing_ = true;
405
Zhaoliang Maf089d7e2024-01-08 02:44:15406 // The three possible modes we're under.
Markus Handellee225432021-11-29 11:35:12407 absl::optional<PassthroughAdapterMode> passthrough_adapter_;
408 absl::optional<ZeroHertzAdapterMode> zero_hertz_adapter_;
Zhaoliang Maf089d7e2024-01-08 02:44:15409 // The `vsync_encode_adapter_` must be destroyed on the worker queue since
410 // VSync metronome needs to happen on worker thread.
411 std::unique_ptr<VSyncEncodeAdapterMode> vsync_encode_adapter_;
Markus Handell8d87c462021-12-16 10:37:16412 // If set, zero-hertz mode has been enabled.
413 absl::optional<ZeroHertzModeParams> zero_hertz_params_;
Markus Handellee225432021-11-29 11:35:12414 // Cache for the current adapter mode.
415 AdapterMode* current_adapter_mode_ = nullptr;
416
Zhaoliang Maf089d7e2024-01-08 02:44:15417 // VSync encoding is used when this valid.
418 Metronome* const metronome_;
419 TaskQueueBase* const worker_queue_;
420
Markus Handell9d04a782022-05-12 16:38:57421 // Timestamp for statistics reporting.
422 absl::optional<Timestamp> zero_hertz_adapter_created_timestamp_
423 RTC_GUARDED_BY(queue_);
424
Markus Handellb4e96d42021-11-05 11:00:55425 // Set up during Initialize.
426 Callback* callback_ = nullptr;
427
Markus Handellb4e96d42021-11-05 11:00:55428 // The source's constraints.
429 absl::optional<VideoTrackSourceConstraints> source_constraints_
Markus Handell9a478b52021-11-18 15:07:01430 RTC_GUARDED_BY(queue_);
Markus Handellb4e96d42021-11-05 11:00:55431
henrika8f16ce92023-12-04 13:03:52432 // Stores the latest restriction in max frame rate set by
433 // UpdateVideoSourceRestrictions. Ensures that a previously set restriction
434 // can be maintained during reconstructions of the adapter.
435 absl::optional<double> restricted_max_frame_rate_ RTC_GUARDED_BY(queue_);
436
Markus Handellb4e96d42021-11-05 11:00:55437 // Race checker for incoming frames. This is the network thread in chromium,
438 // but may vary from test contexts.
439 rtc::RaceChecker incoming_frame_race_checker_;
Markus Handell28c71802021-11-08 09:11:55440
Markus Handell9a478b52021-11-18 15:07:01441 // Number of frames that are currently scheduled for processing on the
Markus Handell8d87c462021-12-16 10:37:16442 // `queue_`.
Markus Handell9a478b52021-11-18 15:07:01443 std::atomic<int> frames_scheduled_for_processing_{0};
444
445 ScopedTaskSafetyDetached safety_;
Markus Handellb4e96d42021-11-05 11:00:55446};
447
Markus Handellee225432021-11-29 11:35:12448ZeroHertzAdapterMode::ZeroHertzAdapterMode(
Markus Handell29dd8d82021-12-15 11:19:15449 TaskQueueBase* queue,
450 Clock* clock,
Markus Handellee225432021-11-29 11:35:12451 FrameCadenceAdapterInterface::Callback* callback,
henrikab7ec0572024-01-09 09:48:52452 double max_fps,
453 std::atomic<int>& frames_scheduled_for_processing,
454 bool zero_hertz_queue_overload_enabled)
455 : queue_(queue),
456 clock_(clock),
457 callback_(callback),
458 max_fps_(max_fps),
459 frames_scheduled_for_processing_(frames_scheduled_for_processing),
460 zero_hertz_queue_overload_enabled_(zero_hertz_queue_overload_enabled) {
Markus Handellee225432021-11-29 11:35:12461 sequence_checker_.Detach();
Markus Handellf5a50792022-06-16 12:25:15462 MaybeStartRefreshFrameRequester();
Markus Handellee225432021-11-29 11:35:12463}
464
Markus Handelle59fee82021-12-23 08:29:23465void ZeroHertzAdapterMode::ReconfigureParameters(
466 const FrameCadenceAdapterInterface::ZeroHertzModeParams& params) {
467 RTC_DCHECK_RUN_ON(&sequence_checker_);
Markus Handellf7f0b212022-05-24 16:39:39468 RTC_DLOG(LS_INFO) << __func__ << " this " << this << " num_simulcast_layers "
469 << params.num_simulcast_layers;
Markus Handelle59fee82021-12-23 08:29:23470
471 // Start as unconverged.
472 layer_trackers_.clear();
473 layer_trackers_.resize(params.num_simulcast_layers,
474 SpatialLayerTracker{false});
475}
476
Markus Handell8d87c462021-12-16 10:37:16477void ZeroHertzAdapterMode::UpdateLayerQualityConvergence(
Markus Handell5a77e512022-09-01 12:51:50478 size_t spatial_index,
Markus Handell8d87c462021-12-16 10:37:16479 bool quality_converged) {
480 RTC_DCHECK_RUN_ON(&sequence_checker_);
Markus Handellf2827c42023-09-02 07:41:10481 TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc"), __func__,
Evan Shrubsole5bfcc872024-04-23 08:51:23482 TRACE_EVENT_SCOPE_GLOBAL, "spatial_index", spatial_index,
483 "converged", quality_converged);
Markus Handell5a77e512022-09-01 12:51:50484 if (spatial_index >= layer_trackers_.size())
485 return;
Markus Handell8d87c462021-12-16 10:37:16486 if (layer_trackers_[spatial_index].quality_converged.has_value())
487 layer_trackers_[spatial_index].quality_converged = quality_converged;
488}
489
Markus Handell5a77e512022-09-01 12:51:50490void ZeroHertzAdapterMode::UpdateLayerStatus(size_t spatial_index,
491 bool enabled) {
Markus Handell8d87c462021-12-16 10:37:16492 RTC_DCHECK_RUN_ON(&sequence_checker_);
Markus Handellf2827c42023-09-02 07:41:10493 TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc"), __func__,
Evan Shrubsole5bfcc872024-04-23 08:51:23494 TRACE_EVENT_SCOPE_GLOBAL, "spatial_index", spatial_index,
495 "enabled", enabled);
Markus Handell5a77e512022-09-01 12:51:50496 if (spatial_index >= layer_trackers_.size())
497 return;
Markus Handell8d87c462021-12-16 10:37:16498 if (enabled) {
499 if (!layer_trackers_[spatial_index].quality_converged.has_value()) {
500 // Assume quality has not converged until hearing otherwise.
501 layer_trackers_[spatial_index].quality_converged = false;
502 }
503 } else {
504 layer_trackers_[spatial_index].quality_converged = absl::nullopt;
505 }
Markus Handell8d87c462021-12-16 10:37:16506}
507
Markus Handellee225432021-11-29 11:35:12508void ZeroHertzAdapterMode::OnFrame(Timestamp post_time,
Markus Handell84c016a2023-11-23 12:41:44509 bool queue_overload,
Markus Handellee225432021-11-29 11:35:12510 const VideoFrame& frame) {
511 RTC_DCHECK_RUN_ON(&sequence_checker_);
Markus Handellf2827c42023-09-02 07:41:10512 TRACE_EVENT0("webrtc", "ZeroHertzAdapterMode::OnFrame");
Markus Handellf7f0b212022-05-24 16:39:39513 refresh_frame_requester_.Stop();
Markus Handell8d87c462021-12-16 10:37:16514
515 // Assume all enabled layers are unconverged after frame entry.
Markus Handelle59fee82021-12-23 08:29:23516 ResetQualityConvergenceInfo();
Markus Handell29dd8d82021-12-15 11:19:15517
518 // Remove stored repeating frame if needed.
Markus Handell2e0f4f02021-12-21 18:14:58519 if (scheduled_repeat_.has_value()) {
Markus Handell29dd8d82021-12-15 11:19:15520 RTC_DCHECK(queued_frames_.size() == 1);
Markus Handell8d87c462021-12-16 10:37:16521 RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this
522 << " cancel repeat and restart with original";
Markus Handell29dd8d82021-12-15 11:19:15523 queued_frames_.pop_front();
524 }
525
526 // Store the frame in the queue and schedule deferred processing.
527 queued_frames_.push_back(frame);
528 current_frame_id_++;
Markus Handell2e0f4f02021-12-21 18:14:58529 scheduled_repeat_ = absl::nullopt;
henrika40ed3ff2023-10-03 08:35:52530 TimeDelta time_spent_since_post = clock_->CurrentTime() - post_time;
Henrik Boström2dd39152022-01-25 07:20:33531 queue_->PostDelayedHighPrecisionTask(
Danil Chapovalov95eeaa72022-07-06 08:14:29532 SafeTask(safety_.flag(),
henrika97439b92023-12-27 13:10:19533 [this, post_time] {
Danil Chapovalov95eeaa72022-07-06 08:14:29534 RTC_DCHECK_RUN_ON(&sequence_checker_);
henrikaf7cdcbd2023-11-20 18:32:54535 ProcessOnDelayedCadence(post_time);
Danil Chapovalov95eeaa72022-07-06 08:14:29536 }),
Markus Handell8fa86192023-08-31 09:59:06537 std::max(frame_delay_ - time_spent_since_post, TimeDelta::Zero()));
Markus Handellee225432021-11-29 11:35:12538}
539
Markus Handellf5a50792022-06-16 12:25:15540void ZeroHertzAdapterMode::OnDiscardedFrame() {
541 RTC_DCHECK_RUN_ON(&sequence_checker_);
Markus Handellf2827c42023-09-02 07:41:10542 TRACE_EVENT0("webrtc", __func__);
Markus Handellf5a50792022-06-16 12:25:15543
544 // Under zero hertz source delivery, a discarded frame ending a sequence of
545 // frames which happened to contain important information can be seen as a
546 // capture freeze. Avoid this by starting requesting refresh frames after a
547 // grace period.
548 MaybeStartRefreshFrameRequester();
549}
550
Markus Handellee225432021-11-29 11:35:12551absl::optional<uint32_t> ZeroHertzAdapterMode::GetInputFrameRateFps() {
552 RTC_DCHECK_RUN_ON(&sequence_checker_);
553 return max_fps_;
554}
555
henrika8f16ce92023-12-04 13:03:52556void ZeroHertzAdapterMode::UpdateVideoSourceRestrictions(
557 absl::optional<double> max_frame_rate) {
558 RTC_DCHECK_RUN_ON(&sequence_checker_);
559 TRACE_EVENT_INSTANT1(TRACE_DISABLED_BY_DEFAULT("webrtc"), __func__,
Evan Shrubsole5bfcc872024-04-23 08:51:23560 TRACE_EVENT_SCOPE_GLOBAL, "max_frame_rate",
561 max_frame_rate.value_or(-1));
henrika8f16ce92023-12-04 13:03:52562 if (max_frame_rate.value_or(0) > 0) {
563 // Set new, validated (> 0) and restricted frame rate.
564 restricted_frame_delay_ = TimeDelta::Seconds(1) / *max_frame_rate;
565 } else {
566 // Source reports that the frame rate is now unrestricted.
567 restricted_frame_delay_ = absl::nullopt;
568 }
569}
570
Markus Handell818e7fb2021-12-30 12:01:33571void ZeroHertzAdapterMode::ProcessKeyFrameRequest() {
Markus Handell2e0f4f02021-12-21 18:14:58572 RTC_DCHECK_RUN_ON(&sequence_checker_);
Evan Shrubsole5bfcc872024-04-23 08:51:23573 TRACE_EVENT_INSTANT0("webrtc", __func__, TRACE_EVENT_SCOPE_GLOBAL);
Markus Handellf7f0b212022-05-24 16:39:39574 // If we're new and don't have a frame, there's no need to request refresh
575 // frames as this was being triggered for us when zero-hz mode was set up.
576 //
Markus Handellcb237f82021-12-29 20:31:57577 // The next frame encoded will be a key frame. Reset quality convergence so we
578 // don't get idle repeats shortly after, because key frames need a lot of
579 // refinement frames.
580 ResetQualityConvergenceInfo();
581
Markus Handell90a7e2c2021-12-29 22:32:30582 // If we're not repeating, or we're repeating with short duration, we will
Markus Handell2e0f4f02021-12-21 18:14:58583 // very soon send out a frame and don't need a refresh frame.
584 if (!scheduled_repeat_.has_value() || !scheduled_repeat_->idle) {
585 RTC_LOG(LS_INFO) << __func__ << " this " << this
Markus Handelle59fee82021-12-23 08:29:23586 << " not requesting refresh frame because of recently "
Markus Handell2e0f4f02021-12-21 18:14:58587 "incoming frame or short repeating.";
Markus Handell818e7fb2021-12-30 12:01:33588 return;
Markus Handell2e0f4f02021-12-21 18:14:58589 }
590
591 // If the repeat is scheduled within a short (i.e. frame_delay_) interval, we
592 // will very soon send out a frame and don't need a refresh frame.
593 Timestamp now = clock_->CurrentTime();
594 if (scheduled_repeat_->scheduled + RepeatDuration(/*idle_repeat=*/true) -
595 now <=
596 frame_delay_) {
Markus Handelle59fee82021-12-23 08:29:23597 RTC_LOG(LS_INFO) << __func__ << " this " << this
598 << " not requesting refresh frame because of soon "
599 "happening idle repeat";
Markus Handell818e7fb2021-12-30 12:01:33600 return;
Markus Handell2e0f4f02021-12-21 18:14:58601 }
602
603 // Cancel the current repeat and reschedule a short repeat now. No need for a
604 // new refresh frame.
605 RTC_LOG(LS_INFO) << __func__ << " this " << this
Markus Handelle59fee82021-12-23 08:29:23606 << " not requesting refresh frame and scheduling a short "
607 "repeat due to key frame request";
Markus Handell2e0f4f02021-12-21 18:14:58608 ScheduleRepeat(++current_frame_id_, /*idle_repeat=*/false);
Markus Handell818e7fb2021-12-30 12:01:33609 return;
Markus Handell2e0f4f02021-12-21 18:14:58610}
611
Markus Handell2e0f4f02021-12-21 18:14:58612bool ZeroHertzAdapterMode::HasQualityConverged() const {
Danil Chapovalov6e7c2682022-07-25 13:58:28613 RTC_DCHECK_RUN_ON(&sequence_checker_);
Markus Handellcb237f82021-12-29 20:31:57614 // 1. Define ourselves as unconverged with no spatial layers configured. This
615 // is to keep short repeating until the layer configuration comes.
616 // 2. Unset layers implicitly imply that they're converged to support
617 // disabling layers when they're not needed.
Markus Handell2e0f4f02021-12-21 18:14:58618 const bool quality_converged =
Markus Handellcb237f82021-12-29 20:31:57619 !layer_trackers_.empty() &&
Markus Handell2e0f4f02021-12-21 18:14:58620 absl::c_all_of(layer_trackers_, [](const SpatialLayerTracker& tracker) {
621 return tracker.quality_converged.value_or(true);
622 });
623 return quality_converged;
624}
625
Markus Handelle59fee82021-12-23 08:29:23626void ZeroHertzAdapterMode::ResetQualityConvergenceInfo() {
Danil Chapovalov6e7c2682022-07-25 13:58:28627 RTC_DCHECK_RUN_ON(&sequence_checker_);
Markus Handelle59fee82021-12-23 08:29:23628 RTC_DLOG(LS_INFO) << __func__ << " this " << this;
629 for (auto& layer_tracker : layer_trackers_) {
630 if (layer_tracker.quality_converged.has_value())
631 layer_tracker.quality_converged = false;
632 }
633}
634
henrikaf7cdcbd2023-11-20 18:32:54635void ZeroHertzAdapterMode::ProcessOnDelayedCadence(Timestamp post_time) {
Danil Chapovalov6e7c2682022-07-25 13:58:28636 RTC_DCHECK_RUN_ON(&sequence_checker_);
Markus Handell29dd8d82021-12-15 11:19:15637 RTC_DCHECK(!queued_frames_.empty());
Markus Handellf2827c42023-09-02 07:41:10638 TRACE_EVENT0("webrtc", __func__);
Markus Handell29dd8d82021-12-15 11:19:15639
Markus Handellf2827c42023-09-02 07:41:10640 // Avoid sending the front frame for encoding (which could take a long time)
henrikaf7cdcbd2023-11-20 18:32:54641 // until we schedule a repeat.
Markus Handellf2827c42023-09-02 07:41:10642 VideoFrame front_frame = queued_frames_.front();
Markus Handell29dd8d82021-12-15 11:19:15643
644 // If there were two or more frames stored, we do not have to schedule repeats
645 // of the front frame.
646 if (queued_frames_.size() > 1) {
647 queued_frames_.pop_front();
Markus Handellf2827c42023-09-02 07:41:10648 } else {
649 // There's only one frame to send. Schedule a repeat sequence, which is
650 // cancelled by `current_frame_id_` getting incremented should new frames
651 // arrive.
652 ScheduleRepeat(current_frame_id_, HasQualityConverged());
Markus Handell29dd8d82021-12-15 11:19:15653 }
henrikaf7cdcbd2023-11-20 18:32:54654 SendFrameNow(post_time, front_frame);
Markus Handell8d87c462021-12-16 10:37:16655}
656
Markus Handell2e0f4f02021-12-21 18:14:58657void ZeroHertzAdapterMode::ScheduleRepeat(int frame_id, bool idle_repeat) {
Danil Chapovalov6e7c2682022-07-25 13:58:28658 RTC_DCHECK_RUN_ON(&sequence_checker_);
Markus Handell90a7e2c2021-12-29 22:32:30659 Timestamp now = clock_->CurrentTime();
660 if (!scheduled_repeat_.has_value()) {
661 scheduled_repeat_.emplace(now, queued_frames_.front().timestamp_us(),
662 queued_frames_.front().ntp_time_ms());
663 }
664 scheduled_repeat_->scheduled = now;
665 scheduled_repeat_->idle = idle_repeat;
666
Markus Handell2e0f4f02021-12-21 18:14:58667 TimeDelta repeat_delay = RepeatDuration(idle_repeat);
Henrik Boström2dd39152022-01-25 07:20:33668 queue_->PostDelayedHighPrecisionTask(
Danil Chapovalov95eeaa72022-07-06 08:14:29669 SafeTask(safety_.flag(),
670 [this, frame_id] {
671 RTC_DCHECK_RUN_ON(&sequence_checker_);
672 ProcessRepeatedFrameOnDelayedCadence(frame_id);
673 }),
674 repeat_delay);
Markus Handell29dd8d82021-12-15 11:19:15675}
676
Markus Handell2e0f4f02021-12-21 18:14:58677void ZeroHertzAdapterMode::ProcessRepeatedFrameOnDelayedCadence(int frame_id) {
Danil Chapovalov6e7c2682022-07-25 13:58:28678 RTC_DCHECK_RUN_ON(&sequence_checker_);
Markus Handellf2827c42023-09-02 07:41:10679 TRACE_EVENT0("webrtc", __func__);
Markus Handell29dd8d82021-12-15 11:19:15680 RTC_DCHECK(!queued_frames_.empty());
681
682 // Cancel this invocation if new frames turned up.
683 if (frame_id != current_frame_id_)
684 return;
Markus Handell2e0f4f02021-12-21 18:14:58685 RTC_DCHECK(scheduled_repeat_.has_value());
Markus Handell29dd8d82021-12-15 11:19:15686
687 VideoFrame& frame = queued_frames_.front();
688
689 // Since this is a repeated frame, nothing changed compared to before.
690 VideoFrame::UpdateRect empty_update_rect;
691 empty_update_rect.MakeEmptyUpdate();
692 frame.set_update_rect(empty_update_rect);
693
Markus Handell90a7e2c2021-12-29 22:32:30694 // Adjust timestamps of the frame of the repeat, accounting for the actual
695 // delay since we started repeating.
696 //
Markus Handell29dd8d82021-12-15 11:19:15697 // NOTE: No need to update the RTP timestamp as the VideoStreamEncoder
698 // overwrites it based on its chosen NTP timestamp source.
Markus Handell90a7e2c2021-12-29 22:32:30699 TimeDelta total_delay = clock_->CurrentTime() - scheduled_repeat_->origin;
700 if (frame.timestamp_us() > 0) {
701 frame.set_timestamp_us(scheduled_repeat_->origin_timestamp_us +
702 total_delay.us());
703 }
704 if (frame.ntp_time_ms()) {
705 frame.set_ntp_time_ms(scheduled_repeat_->origin_ntp_time_ms +
706 total_delay.ms());
707 }
Markus Handell29dd8d82021-12-15 11:19:15708
Markus Handellf2827c42023-09-02 07:41:10709 // Schedule another repeat before sending the frame off which could take time.
Markus Handell2e0f4f02021-12-21 18:14:58710 ScheduleRepeat(frame_id, HasQualityConverged());
henrikab7ec0572024-01-09 09:48:52711 SendFrameNow(absl::nullopt, frame);
Markus Handell29dd8d82021-12-15 11:19:15712}
713
henrikab7ec0572024-01-09 09:48:52714void ZeroHertzAdapterMode::SendFrameNow(absl::optional<Timestamp> post_time,
715 const VideoFrame& frame) {
Danil Chapovalov6e7c2682022-07-25 13:58:28716 RTC_DCHECK_RUN_ON(&sequence_checker_);
Markus Handellf2827c42023-09-02 07:41:10717 TRACE_EVENT0("webrtc", __func__);
henrikab7ec0572024-01-09 09:48:52718
719 Timestamp encode_start_time = clock_->CurrentTime();
720 if (post_time.has_value()) {
721 TimeDelta delay = (encode_start_time - *post_time);
henrikaf7cdcbd2023-11-20 18:32:54722 RTC_HISTOGRAM_COUNTS_10000("WebRTC.Screenshare.ZeroHz.DelayMs", delay.ms());
723 }
henrikab7ec0572024-01-09 09:48:52724
725 // Forward the frame and set `queue_overload` if is has been detected that it
726 // is not possible to deliver frames at the expected rate due to slow
727 // encoding.
728 callback_->OnFrame(/*post_time=*/encode_start_time, queue_overload_count_ > 0,
729 frame);
730
731 // WebRTC-ZeroHertzQueueOverload kill-switch.
732 if (!zero_hertz_queue_overload_enabled_)
733 return;
734
735 // `queue_overload_count_` determines for how many future frames the
736 // `queue_overload` flag will be set and it is only increased if:
737 // o We are not already in an overload state.
738 // o New frames have been scheduled for processing on the queue while encoding
739 // took place in OnFrame.
740 // o The duration of OnFrame is longer than the current frame duration.
741 // If all these conditions are fulfilled, `queue_overload_count_` is set to
742 // `frames_scheduled_for_processing_` and any pending repeat is canceled since
743 // new frames are available and the repeat is not needed.
744 // If the adapter is already in an overload state, simply decrease
745 // `queue_overload_count_` by one.
746 if (queue_overload_count_ == 0) {
747 const int frames_scheduled_for_processing =
748 frames_scheduled_for_processing_.load(std::memory_order_relaxed);
749 if (frames_scheduled_for_processing > 0) {
750 TimeDelta encode_time = clock_->CurrentTime() - encode_start_time;
751 if (encode_time > FrameDuration()) {
752 queue_overload_count_ = frames_scheduled_for_processing;
753 // Invalidates any outstanding repeat to avoid sending pending repeat
754 // directly after too long encode.
755 current_frame_id_++;
756 }
757 }
758 } else {
759 queue_overload_count_--;
760 }
761 RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.ZeroHz.QueueOverload",
762 queue_overload_count_ > 0);
763}
764
765TimeDelta ZeroHertzAdapterMode::FrameDuration() const {
766 RTC_DCHECK_RUN_ON(&sequence_checker_);
767 return std::max(frame_delay_, restricted_frame_delay_.value_or(frame_delay_));
Markus Handell29dd8d82021-12-15 11:19:15768}
769
Markus Handell2e0f4f02021-12-21 18:14:58770TimeDelta ZeroHertzAdapterMode::RepeatDuration(bool idle_repeat) const {
Danil Chapovalov6e7c2682022-07-25 13:58:28771 RTC_DCHECK_RUN_ON(&sequence_checker_);
Markus Handell2e0f4f02021-12-21 18:14:58772 return idle_repeat
773 ? FrameCadenceAdapterInterface::kZeroHertzIdleRepeatRatePeriod
henrikab7ec0572024-01-09 09:48:52774 : FrameDuration();
Markus Handell2e0f4f02021-12-21 18:14:58775}
776
Markus Handellf5a50792022-06-16 12:25:15777void ZeroHertzAdapterMode::MaybeStartRefreshFrameRequester() {
Danil Chapovalov6e7c2682022-07-25 13:58:28778 RTC_DCHECK_RUN_ON(&sequence_checker_);
Markus Handellf5a50792022-06-16 12:25:15779 if (!refresh_frame_requester_.Running()) {
780 refresh_frame_requester_ = RepeatingTaskHandle::DelayedStart(
781 queue_,
782 FrameCadenceAdapterInterface::kOnDiscardedFrameRefreshFramePeriod *
783 frame_delay_,
784 [this] {
785 RTC_DLOG(LS_VERBOSE) << __func__ << " RequestRefreshFrame";
786 if (callback_)
787 callback_->RequestRefreshFrame();
788 return frame_delay_;
789 });
790 }
791}
792
Zhaoliang Maf089d7e2024-01-08 02:44:15793void VSyncEncodeAdapterMode::OnFrame(Timestamp post_time,
794 bool queue_overload,
795 const VideoFrame& frame) {
796 // We expect `metronome_` and `EncodeAllEnqueuedFrames()` runs on
797 // `worker_queue_`.
798 if (!worker_queue_->IsCurrent()) {
799 worker_queue_->PostTask(SafeTask(
800 worker_safety_.flag(), [this, post_time, queue_overload, frame] {
801 OnFrame(post_time, queue_overload, frame);
802 }));
803 return;
804 }
805
806 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
807 TRACE_EVENT0("webrtc", "VSyncEncodeAdapterMode::OnFrame");
808
809 input_queue_.emplace_back(std::move(frame), post_time);
810
811 // The `metronome_` tick period maybe throttled in some case, so here we only
812 // align encode task to VSync event when `metronome_` tick period is less
813 // than 34ms (30Hz).
814 static constexpr TimeDelta kMaxAllowedDelay = TimeDelta::Millis(34);
815 if (metronome_->TickPeriod() <= kMaxAllowedDelay) {
816 // The metronome is ticking frequently enough that it is worth the extra
817 // delay.
818 metronome_->RequestCallOnNextTick(
819 SafeTask(worker_safety_.flag(), [this] { EncodeAllEnqueuedFrames(); }));
820 } else {
821 // The metronome is ticking too infrequently, encode immediately.
822 EncodeAllEnqueuedFrames();
823 }
824}
825
826void VSyncEncodeAdapterMode::EncodeAllEnqueuedFrames() {
827 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
828 TRACE_EVENT0("webrtc", "VSyncEncodeAdapterMode::EncodeAllEnqueuedFrames");
829
830 // Local time in webrtc time base.
831 Timestamp post_time = clock_->CurrentTime();
832
833 for (auto& input : input_queue_) {
834 TRACE_EVENT1("webrtc", "FrameCadenceAdapterImpl::EncodeAllEnqueuedFrames",
835 "VSyncEncodeDelay",
836 (post_time - input.time_when_posted_us).ms());
837
838 const VideoFrame frame = std::move(input.video_frame);
Markus Handelle864dec2024-08-07 14:11:18839 MutexLock lock(&queue_lock_);
840 if (queue_) {
841 queue_->PostTask(SafeTask(queue_safety_flag_, [this, post_time, frame] {
842 {
843 MutexLock lock(&queue_lock_);
844 if (!queue_) {
845 return;
846 }
847 RTC_DCHECK_RUN_ON(queue_);
848 }
Zhaoliang Maf089d7e2024-01-08 02:44:15849
Markus Handelle864dec2024-08-07 14:11:18850 // TODO(b/304158952): Support more refined queue overload control.
851 // Not running under mutex is safe since `callback_` existence is
852 // guaranteed to exist as long as running encode queue tasks exist.
853 callback_->OnFrame(post_time, /*queue_overload=*/false, frame);
854 }));
855 }
Zhaoliang Maf089d7e2024-01-08 02:44:15856 }
857
858 input_queue_.clear();
859}
860
Jonas Oreland8ca06132022-03-14 11:52:48861FrameCadenceAdapterImpl::FrameCadenceAdapterImpl(
862 Clock* clock,
863 TaskQueueBase* queue,
Zhaoliang Maf089d7e2024-01-08 02:44:15864 Metronome* metronome,
865 TaskQueueBase* worker_queue,
Jonas Orelande62c2f22022-03-29 09:04:48866 const FieldTrialsView& field_trials)
Markus Handell9a478b52021-11-18 15:07:01867 : clock_(clock),
868 queue_(queue),
henrikab7ec0572024-01-09 09:48:52869 frame_cadence_adapter_zero_hertz_queue_overload_enabled_(
870 !field_trials.IsDisabled("WebRTC-ZeroHertzQueueOverload")),
Per Ke975b442024-03-27 10:28:44871 use_video_frame_timestamp_(field_trials.IsEnabled(
872 "WebRTC-FrameCadenceAdapter-UseVideoFrameTimestamp")),
Zhaoliang Maf089d7e2024-01-08 02:44:15873 metronome_(metronome),
874 worker_queue_(worker_queue) {}
Markus Handellb4e96d42021-11-05 11:00:55875
Markus Handelle59fee82021-12-23 08:29:23876FrameCadenceAdapterImpl::~FrameCadenceAdapterImpl() {
877 RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this;
Zhaoliang Maf089d7e2024-01-08 02:44:15878
879 // VSync adapter needs to be destroyed on worker queue when metronome is
880 // valid.
881 if (metronome_) {
Markus Handelle864dec2024-08-07 14:11:18882 vsync_encode_adapter_->PrepareShutdown();
Zhaoliang Maf089d7e2024-01-08 02:44:15883 absl::Cleanup cleanup = [adapter = std::move(vsync_encode_adapter_)] {};
884 worker_queue_->PostTask([cleanup = std::move(cleanup)] {});
885 }
Per Ke975b442024-03-27 10:28:44886
887 RTC_HISTOGRAM_BOOLEAN(
888 "WebRTC.Video.InputFrameTimestampMonotonicallyIncreasing",
889 incoming_frame_timestamp_monotonically_increasing_);
Markus Handelle59fee82021-12-23 08:29:23890}
891
Markus Handellb4e96d42021-11-05 11:00:55892void FrameCadenceAdapterImpl::Initialize(Callback* callback) {
893 callback_ = callback;
Zhaoliang Maf089d7e2024-01-08 02:44:15894 // Use VSync encode mode if metronome is valid, otherwise passthrough mode
895 // would be used.
896 if (metronome_) {
897 vsync_encode_adapter_ = std::make_unique<VSyncEncodeAdapterMode>(
898 clock_, queue_, safety_.flag(), metronome_, worker_queue_, callback_);
899 } else {
Per Ke975b442024-03-27 10:28:44900 passthrough_adapter_.emplace(callback);
Zhaoliang Maf089d7e2024-01-08 02:44:15901 }
902 ConfigureCurrentAdapterWithoutZeroHertz();
Markus Handellb4e96d42021-11-05 11:00:55903}
904
Markus Handell8d87c462021-12-16 10:37:16905void FrameCadenceAdapterImpl::SetZeroHertzModeEnabled(
906 absl::optional<ZeroHertzModeParams> params) {
Markus Handell9a478b52021-11-18 15:07:01907 RTC_DCHECK_RUN_ON(queue_);
Markus Handell8d87c462021-12-16 10:37:16908 bool was_zero_hertz_enabled = zero_hertz_params_.has_value();
Markus Handell8d87c462021-12-16 10:37:16909 zero_hertz_params_ = params;
Markus Handellee225432021-11-29 11:35:12910 MaybeReconfigureAdapters(was_zero_hertz_enabled);
911}
912
913absl::optional<uint32_t> FrameCadenceAdapterImpl::GetInputFrameRateFps() {
914 RTC_DCHECK_RUN_ON(queue_);
915 return current_adapter_mode_->GetInputFrameRateFps();
916}
917
Per Ke975b442024-03-27 10:28:44918void FrameCadenceAdapterImpl::UpdateFrameRate(Timestamp frame_timestamp) {
Markus Handellee225432021-11-29 11:35:12919 RTC_DCHECK_RUN_ON(queue_);
920 // The frame rate need not be updated for the zero-hertz adapter. The
Zhaoliang Maf089d7e2024-01-08 02:44:15921 // vsync encode and passthrough adapter however uses it. Always pass frames
922 // into the vsync encode or passthrough to keep the estimation alive should
923 // there be an adapter switch.
924 if (metronome_) {
925 RTC_CHECK(vsync_encode_adapter_);
Per Ke975b442024-03-27 10:28:44926 vsync_encode_adapter_->UpdateFrameRate(frame_timestamp);
Zhaoliang Maf089d7e2024-01-08 02:44:15927 } else {
928 RTC_CHECK(passthrough_adapter_);
Per Ke975b442024-03-27 10:28:44929 passthrough_adapter_->UpdateFrameRate(frame_timestamp);
Zhaoliang Maf089d7e2024-01-08 02:44:15930 }
Markus Handellb4e96d42021-11-05 11:00:55931}
932
Markus Handell8d87c462021-12-16 10:37:16933void FrameCadenceAdapterImpl::UpdateLayerQualityConvergence(
Markus Handell5a77e512022-09-01 12:51:50934 size_t spatial_index,
Markus Handell8d87c462021-12-16 10:37:16935 bool quality_converged) {
936 if (zero_hertz_adapter_.has_value())
937 zero_hertz_adapter_->UpdateLayerQualityConvergence(spatial_index,
938 quality_converged);
939}
940
Markus Handell5a77e512022-09-01 12:51:50941void FrameCadenceAdapterImpl::UpdateLayerStatus(size_t spatial_index,
Markus Handell8d87c462021-12-16 10:37:16942 bool enabled) {
943 if (zero_hertz_adapter_.has_value())
944 zero_hertz_adapter_->UpdateLayerStatus(spatial_index, enabled);
945}
946
henrika8f16ce92023-12-04 13:03:52947void FrameCadenceAdapterImpl::UpdateVideoSourceRestrictions(
948 absl::optional<double> max_frame_rate) {
949 RTC_DCHECK_RUN_ON(queue_);
950 // Store the restriction to ensure that it can be reapplied in possible
951 // future adapter creations on configuration changes.
952 restricted_max_frame_rate_ = max_frame_rate;
953 if (zero_hertz_adapter_) {
954 zero_hertz_adapter_->UpdateVideoSourceRestrictions(max_frame_rate);
955 }
956}
957
Markus Handell818e7fb2021-12-30 12:01:33958void FrameCadenceAdapterImpl::ProcessKeyFrameRequest() {
Markus Handell2e0f4f02021-12-21 18:14:58959 RTC_DCHECK_RUN_ON(queue_);
960 if (zero_hertz_adapter_)
Markus Handell818e7fb2021-12-30 12:01:33961 zero_hertz_adapter_->ProcessKeyFrameRequest();
Markus Handell2e0f4f02021-12-21 18:14:58962}
963
Markus Handellb4e96d42021-11-05 11:00:55964void FrameCadenceAdapterImpl::OnFrame(const VideoFrame& frame) {
965 // This method is called on the network thread under Chromium, or other
966 // various contexts in test.
967 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_);
Markus Handellf2827c42023-09-02 07:41:10968 TRACE_EVENT0("webrtc", "FrameCadenceAdapterImpl::OnFrame");
Markus Handell9a478b52021-11-18 15:07:01969
970 // Local time in webrtc time base.
971 Timestamp post_time = clock_->CurrentTime();
972 frames_scheduled_for_processing_.fetch_add(1, std::memory_order_relaxed);
Danil Chapovalov95eeaa72022-07-06 08:14:29973 queue_->PostTask(SafeTask(safety_.flag(), [this, post_time, frame] {
Markus Handell9a478b52021-11-18 15:07:01974 RTC_DCHECK_RUN_ON(queue_);
Markus Handell9d04a782022-05-12 16:38:57975 if (zero_hertz_adapter_created_timestamp_.has_value()) {
976 TimeDelta time_until_first_frame =
henrika40ed3ff2023-10-03 08:35:52977 clock_->CurrentTime() - *zero_hertz_adapter_created_timestamp_;
Markus Handell9d04a782022-05-12 16:38:57978 zero_hertz_adapter_created_timestamp_ = absl::nullopt;
979 RTC_HISTOGRAM_COUNTS_10000(
980 "WebRTC.Screenshare.ZeroHz.TimeUntilFirstFrameMs",
981 time_until_first_frame.ms());
982 }
983
Markus Handell9a478b52021-11-18 15:07:01984 const int frames_scheduled_for_processing =
985 frames_scheduled_for_processing_.fetch_sub(1,
986 std::memory_order_relaxed);
Markus Handell84c016a2023-11-23 12:41:44987 OnFrameOnMainQueue(post_time, frames_scheduled_for_processing > 1,
Markus Handell9a478b52021-11-18 15:07:01988 std::move(frame));
Markus Handell28c71802021-11-08 09:11:55989 }));
Markus Handellb4e96d42021-11-05 11:00:55990}
991
Markus Handellf5a50792022-06-16 12:25:15992void FrameCadenceAdapterImpl::OnDiscardedFrame() {
993 callback_->OnDiscardedFrame();
Danil Chapovalov95eeaa72022-07-06 08:14:29994 queue_->PostTask(SafeTask(safety_.flag(), [this] {
Markus Handellf5a50792022-06-16 12:25:15995 RTC_DCHECK_RUN_ON(queue_);
996 if (zero_hertz_adapter_) {
997 zero_hertz_adapter_->OnDiscardedFrame();
998 }
999 }));
1000}
1001
Markus Handellb4e96d42021-11-05 11:00:551002void FrameCadenceAdapterImpl::OnConstraintsChanged(
1003 const VideoTrackSourceConstraints& constraints) {
Markus Handelle59fee82021-12-23 08:29:231004 RTC_LOG(LS_INFO) << __func__ << " this " << this << " min_fps "
Markus Handellb4e96d42021-11-05 11:00:551005 << constraints.min_fps.value_or(-1) << " max_fps "
1006 << constraints.max_fps.value_or(-1);
Danil Chapovalov95eeaa72022-07-06 08:14:291007 queue_->PostTask(SafeTask(safety_.flag(), [this, constraints] {
Markus Handell9a478b52021-11-18 15:07:011008 RTC_DCHECK_RUN_ON(queue_);
Markus Handellee225432021-11-29 11:35:121009 bool was_zero_hertz_enabled = IsZeroHertzScreenshareEnabled();
Markus Handell28c71802021-11-08 09:11:551010 source_constraints_ = constraints;
Markus Handellee225432021-11-29 11:35:121011 MaybeReconfigureAdapters(was_zero_hertz_enabled);
Markus Handell28c71802021-11-08 09:11:551012 }));
Markus Handellb4e96d42021-11-05 11:00:551013}
1014
Ilya Nikolaevskiy7a841ce2023-11-24 12:01:211015void FrameCadenceAdapterImpl::OnFrameOnMainQueue(Timestamp post_time,
1016 bool queue_overload,
1017 const VideoFrame& frame) {
Danil Chapovalov6e7c2682022-07-25 13:58:281018 RTC_DCHECK_RUN_ON(queue_);
Ilya Nikolaevskiy7a841ce2023-11-24 12:01:211019 current_adapter_mode_->OnFrame(post_time, queue_overload, frame);
Per Ke975b442024-03-27 10:28:441020 if (last_incoming_frame_timestamp_ &&
1021 last_incoming_frame_timestamp_ >=
1022 Timestamp::Micros(frame.timestamp_us())) {
1023 RTC_LOG(LS_ERROR)
1024 << "Incoming frame timestamp is not monotonically increasing"
1025 << " current: " << frame.timestamp_us()
1026 << " last: " << last_incoming_frame_timestamp_.value().us();
1027 incoming_frame_timestamp_monotonically_increasing_ = false;
1028 }
1029 last_incoming_frame_timestamp_ = Timestamp::Micros(frame.timestamp_us());
1030 Timestamp update_frame_rate_timestamp =
1031 use_video_frame_timestamp_ ? *last_incoming_frame_timestamp_ : post_time;
1032 UpdateFrameRate(update_frame_rate_timestamp);
Markus Handellee225432021-11-29 11:35:121033}
1034
Markus Handellee225432021-11-29 11:35:121035bool FrameCadenceAdapterImpl::IsZeroHertzScreenshareEnabled() const {
Danil Chapovalov6e7c2682022-07-25 13:58:281036 RTC_DCHECK_RUN_ON(queue_);
Markus Handella57229b2024-04-16 08:40:451037 return source_constraints_.has_value() &&
Markus Handellee225432021-11-29 11:35:121038 source_constraints_->max_fps.value_or(-1) > 0 &&
1039 source_constraints_->min_fps.value_or(-1) == 0 &&
Markus Handell8d87c462021-12-16 10:37:161040 zero_hertz_params_.has_value();
Markus Handellee225432021-11-29 11:35:121041}
1042
Zhaoliang Maf089d7e2024-01-08 02:44:151043void FrameCadenceAdapterImpl::ConfigureCurrentAdapterWithoutZeroHertz() {
1044 // Enable VSyncEncodeAdapterMode if metronome is valid.
1045 if (metronome_) {
1046 RTC_CHECK(vsync_encode_adapter_);
1047 current_adapter_mode_ = vsync_encode_adapter_.get();
1048 } else {
1049 RTC_CHECK(passthrough_adapter_);
1050 current_adapter_mode_ = &passthrough_adapter_.value();
1051 }
1052}
1053
Markus Handellee225432021-11-29 11:35:121054void FrameCadenceAdapterImpl::MaybeReconfigureAdapters(
1055 bool was_zero_hertz_enabled) {
Danil Chapovalov6e7c2682022-07-25 13:58:281056 RTC_DCHECK_RUN_ON(queue_);
Markus Handellee225432021-11-29 11:35:121057 bool is_zero_hertz_enabled = IsZeroHertzScreenshareEnabled();
1058 if (is_zero_hertz_enabled) {
henrika644025c2023-11-07 17:22:021059 bool max_fps_has_changed = GetInputFrameRateFps().value_or(-1) !=
1060 source_constraints_->max_fps.value_or(-1);
1061 if (!was_zero_hertz_enabled || max_fps_has_changed) {
henrika644025c2023-11-07 17:22:021062 RTC_LOG(LS_INFO) << "Zero hertz mode enabled (max_fps="
1063 << source_constraints_->max_fps.value() << ")";
henrikab7ec0572024-01-09 09:48:521064 zero_hertz_adapter_.emplace(
1065 queue_, clock_, callback_, source_constraints_->max_fps.value(),
1066 frames_scheduled_for_processing_,
1067 frame_cadence_adapter_zero_hertz_queue_overload_enabled_);
henrika8f16ce92023-12-04 13:03:521068 zero_hertz_adapter_->UpdateVideoSourceRestrictions(
1069 restricted_max_frame_rate_);
Markus Handell9d04a782022-05-12 16:38:571070 zero_hertz_adapter_created_timestamp_ = clock_->CurrentTime();
Markus Handellee225432021-11-29 11:35:121071 }
Markus Handelle59fee82021-12-23 08:29:231072 zero_hertz_adapter_->ReconfigureParameters(zero_hertz_params_.value());
Markus Handellee225432021-11-29 11:35:121073 current_adapter_mode_ = &zero_hertz_adapter_.value();
1074 } else {
henrika86f09ae2023-10-31 13:49:401075 if (was_zero_hertz_enabled) {
Markus Handellee225432021-11-29 11:35:121076 zero_hertz_adapter_ = absl::nullopt;
henrika644025c2023-11-07 17:22:021077 RTC_LOG(LS_INFO) << "Zero hertz mode disabled.";
henrika86f09ae2023-10-31 13:49:401078 }
Zhaoliang Maf089d7e2024-01-08 02:44:151079 ConfigureCurrentAdapterWithoutZeroHertz();
Markus Handellee225432021-11-29 11:35:121080 }
Markus Handell28c71802021-11-08 09:11:551081}
1082
Markus Handellb4e96d42021-11-05 11:00:551083} // namespace
1084
1085std::unique_ptr<FrameCadenceAdapterInterface>
Jonas Oreland8ca06132022-03-14 11:52:481086FrameCadenceAdapterInterface::Create(Clock* clock,
1087 TaskQueueBase* queue,
Zhaoliang Maf089d7e2024-01-08 02:44:151088 Metronome* metronome,
1089 TaskQueueBase* worker_queue,
Jonas Orelande62c2f22022-03-29 09:04:481090 const FieldTrialsView& field_trials) {
Zhaoliang Maf089d7e2024-01-08 02:44:151091 return std::make_unique<FrameCadenceAdapterImpl>(clock, queue, metronome,
1092 worker_queue, field_trials);
Markus Handellb4e96d42021-11-05 11:00:551093}
1094
1095} // namespace webrtc