Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
| 11 | #include "video/frame_cadence_adapter.h" |
| 12 | |
Markus Handell | 8fa8619 | 2023-08-31 09:59:06 | [diff] [blame] | 13 | #include <algorithm> |
Markus Handell | 9a478b5 | 2021-11-18 15:07:01 | [diff] [blame] | 14 | #include <atomic> |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 15 | #include <cstdint> |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 16 | #include <deque> |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 17 | #include <memory> |
Markus Handell | 28c7180 | 2021-11-08 09:11:55 | [diff] [blame] | 18 | #include <utility> |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 19 | #include <vector> |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 20 | |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 21 | #include "absl/algorithm/container.h" |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 22 | #include "absl/base/attributes.h" |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 23 | #include "absl/cleanup/cleanup.h" |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 24 | #include "api/sequence_checker.h" |
Artem Titov | c374d11 | 2022-06-16 19:27:45 | [diff] [blame] | 25 | #include "api/task_queue/pending_task_safety_flag.h" |
Markus Handell | 28c7180 | 2021-11-08 09:11:55 | [diff] [blame] | 26 | #include "api/task_queue/task_queue_base.h" |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 27 | #include "api/units/time_delta.h" |
Markus Handell | 9d04a78 | 2022-05-12 16:38:57 | [diff] [blame] | 28 | #include "api/units/timestamp.h" |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 29 | #include "api/video/video_frame.h" |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 30 | #include "rtc_base/checks.h" |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 31 | #include "rtc_base/logging.h" |
| 32 | #include "rtc_base/race_checker.h" |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 33 | #include "rtc_base/rate_statistics.h" |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 34 | #include "rtc_base/synchronization/mutex.h" |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 35 | #include "rtc_base/system/no_unique_address.h" |
henrika | 1b573a7 | 2023-10-20 10:39:46 | [diff] [blame] | 36 | #include "rtc_base/system/unused.h" |
Markus Handell | f7f0b21 | 2022-05-24 16:39:39 | [diff] [blame] | 37 | #include "rtc_base/task_utils/repeating_task.h" |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 38 | #include "rtc_base/thread_annotations.h" |
| 39 | #include "rtc_base/time_utils.h" |
Markus Handell | f2827c4 | 2023-09-02 07:41:10 | [diff] [blame] | 40 | #include "rtc_base/trace_event.h" |
Markus Handell | 9a478b5 | 2021-11-18 15:07:01 | [diff] [blame] | 41 | #include "system_wrappers/include/clock.h" |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 42 | #include "system_wrappers/include/metrics.h" |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 43 | #include "system_wrappers/include/ntp_time.h" |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 44 | |
| 45 | namespace webrtc { |
| 46 | namespace { |
| 47 | |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 48 | // Abstracts concrete modes of the cadence adapter. |
| 49 | class AdapterMode { |
| 50 | public: |
| 51 | virtual ~AdapterMode() = default; |
| 52 | |
| 53 | // Called on the worker thread for every frame that enters. |
| 54 | virtual void OnFrame(Timestamp post_time, |
Markus Handell | 84c016a | 2023-11-23 12:41:44 | [diff] [blame] | 55 | bool queue_overload, |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 56 | const VideoFrame& frame) = 0; |
| 57 | |
| 58 | // Returns the currently estimated input framerate. |
| 59 | virtual absl::optional<uint32_t> GetInputFrameRateFps() = 0; |
| 60 | |
| 61 | // Updates the frame rate. |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 62 | virtual void UpdateFrameRate(Timestamp frame_timestamp) = 0; |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 63 | }; |
| 64 | |
| 65 | // Implements a pass-through adapter. Single-threaded. |
| 66 | class PassthroughAdapterMode : public AdapterMode { |
| 67 | public: |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 68 | explicit PassthroughAdapterMode( |
| 69 | FrameCadenceAdapterInterface::Callback* callback) |
| 70 | : callback_(callback) { |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 71 | sequence_checker_.Detach(); |
| 72 | } |
| 73 | |
| 74 | // Adapter overrides. |
| 75 | void OnFrame(Timestamp post_time, |
Markus Handell | 84c016a | 2023-11-23 12:41:44 | [diff] [blame] | 76 | bool queue_overload, |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 77 | const VideoFrame& frame) override { |
| 78 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
Markus Handell | 84c016a | 2023-11-23 12:41:44 | [diff] [blame] | 79 | callback_->OnFrame(post_time, queue_overload, frame); |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 80 | } |
| 81 | |
| 82 | absl::optional<uint32_t> GetInputFrameRateFps() override { |
| 83 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 84 | return last_frame_rate_; |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 85 | } |
| 86 | |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 87 | void UpdateFrameRate(Timestamp frame_timestamp) override { |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 88 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 89 | // RateStatistics will calculate a too high rate immediately after Update. |
| 90 | last_frame_rate_ = input_framerate_.Rate(frame_timestamp.ms()); |
| 91 | input_framerate_.Update(1, frame_timestamp.ms()); |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 92 | } |
| 93 | |
| 94 | private: |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 95 | absl::optional<uint64_t> last_frame_rate_; |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 96 | FrameCadenceAdapterInterface::Callback* const callback_; |
| 97 | RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; |
| 98 | // Input frame rate statistics for use when not in zero-hertz mode. |
| 99 | RateStatistics input_framerate_ RTC_GUARDED_BY(sequence_checker_){ |
| 100 | FrameCadenceAdapterInterface::kFrameRateAveragingWindowSizeMs, 1000}; |
| 101 | }; |
| 102 | |
| 103 | // Implements a frame cadence adapter supporting zero-hertz input. |
| 104 | class ZeroHertzAdapterMode : public AdapterMode { |
| 105 | public: |
Markus Handell | e59fee8 | 2021-12-23 08:29:23 | [diff] [blame] | 106 | ZeroHertzAdapterMode(TaskQueueBase* queue, |
| 107 | Clock* clock, |
| 108 | FrameCadenceAdapterInterface::Callback* callback, |
henrika | b7ec057 | 2024-01-09 09:48:52 | [diff] [blame] | 109 | double max_fps, |
| 110 | std::atomic<int>& frames_scheduled_for_processing, |
| 111 | bool zero_hertz_queue_overload); |
Markus Handell | fb98b01 | 2023-09-13 22:31:11 | [diff] [blame] | 112 | ~ZeroHertzAdapterMode() { refresh_frame_requester_.Stop(); } |
Markus Handell | e59fee8 | 2021-12-23 08:29:23 | [diff] [blame] | 113 | |
| 114 | // Reconfigures according to parameters. |
| 115 | // All spatial layer trackers are initialized as unconverged by this method. |
| 116 | void ReconfigureParameters( |
| 117 | const FrameCadenceAdapterInterface::ZeroHertzModeParams& params); |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 118 | |
| 119 | // Updates spatial layer quality convergence status. |
Markus Handell | 5a77e51 | 2022-09-01 12:51:50 | [diff] [blame] | 120 | void UpdateLayerQualityConvergence(size_t spatial_index, |
| 121 | bool quality_converged); |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 122 | |
| 123 | // Updates spatial layer enabled status. |
Markus Handell | 5a77e51 | 2022-09-01 12:51:50 | [diff] [blame] | 124 | void UpdateLayerStatus(size_t spatial_index, bool enabled); |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 125 | |
| 126 | // Adapter overrides. |
| 127 | void OnFrame(Timestamp post_time, |
Markus Handell | 84c016a | 2023-11-23 12:41:44 | [diff] [blame] | 128 | bool queue_overload, |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 129 | const VideoFrame& frame) override; |
| 130 | absl::optional<uint32_t> GetInputFrameRateFps() override; |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 131 | void UpdateFrameRate(Timestamp frame_timestamp) override {} |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 132 | |
Markus Handell | f5a5079 | 2022-06-16 12:25:15 | [diff] [blame] | 133 | // Notified on dropped frames. |
| 134 | void OnDiscardedFrame(); |
| 135 | |
Markus Handell | 818e7fb | 2021-12-30 12:01:33 | [diff] [blame] | 136 | // Conditionally requests a refresh frame via |
| 137 | // Callback::RequestRefreshFrame. |
| 138 | void ProcessKeyFrameRequest(); |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 139 | |
henrika | 8f16ce9 | 2023-12-04 13:03:52 | [diff] [blame] | 140 | // Updates the restrictions of max frame rate for the video source. |
| 141 | // Always called during construction using latest `restricted_frame_delay_`. |
| 142 | void UpdateVideoSourceRestrictions(absl::optional<double> max_frame_rate); |
| 143 | |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 144 | private: |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 145 | // The tracking state of each spatial layer. Used for determining when to |
| 146 | // stop repeating frames. |
| 147 | struct SpatialLayerTracker { |
| 148 | // If unset, the layer is disabled. Otherwise carries the quality |
| 149 | // convergence status of the layer. |
| 150 | absl::optional<bool> quality_converged; |
| 151 | }; |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 152 | // The state of a scheduled repeat. |
| 153 | struct ScheduledRepeat { |
Markus Handell | 90a7e2c | 2021-12-29 22:32:30 | [diff] [blame] | 154 | ScheduledRepeat(Timestamp origin, |
| 155 | int64_t origin_timestamp_us, |
| 156 | int64_t origin_ntp_time_ms) |
| 157 | : scheduled(origin), |
| 158 | idle(false), |
| 159 | origin(origin), |
| 160 | origin_timestamp_us(origin_timestamp_us), |
| 161 | origin_ntp_time_ms(origin_ntp_time_ms) {} |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 162 | // The instant when the repeat was scheduled. |
| 163 | Timestamp scheduled; |
| 164 | // True if the repeat was scheduled as an idle repeat (long), false |
| 165 | // otherwise. |
| 166 | bool idle; |
Markus Handell | 90a7e2c | 2021-12-29 22:32:30 | [diff] [blame] | 167 | // The moment we decided to start repeating. |
| 168 | Timestamp origin; |
| 169 | // The timestamp_us of the frame when we started repeating. |
| 170 | int64_t origin_timestamp_us; |
| 171 | // The ntp_times_ms of the frame when we started repeating. |
| 172 | int64_t origin_ntp_time_ms; |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 173 | }; |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 174 | |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 175 | // Returns true if all spatial layers can be considered to be converged in |
| 176 | // terms of quality. |
| 177 | // Convergence means QP has dropped to a low-enough level to warrant ceasing |
| 178 | // to send identical frames at high frequency. |
| 179 | bool HasQualityConverged() const RTC_RUN_ON(sequence_checker_); |
Markus Handell | e59fee8 | 2021-12-23 08:29:23 | [diff] [blame] | 180 | // Resets quality convergence information. HasQualityConverged() returns false |
| 181 | // after this call. |
| 182 | void ResetQualityConvergenceInfo() RTC_RUN_ON(sequence_checker_); |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 183 | // Processes incoming frames on a delayed cadence. |
henrika | f7cdcbd | 2023-11-20 18:32:54 | [diff] [blame] | 184 | void ProcessOnDelayedCadence(Timestamp post_time) |
| 185 | RTC_RUN_ON(sequence_checker_); |
henrika | 8f16ce9 | 2023-12-04 13:03:52 | [diff] [blame] | 186 | // Schedules a later repeat with delay depending on state of layer trackers |
| 187 | // and if UpdateVideoSourceRestrictions has been called or not. |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 188 | // If true is passed in `idle_repeat`, the repeat is going to be |
henrika | 8f16ce9 | 2023-12-04 13:03:52 | [diff] [blame] | 189 | // kZeroHertzIdleRepeatRatePeriod. Otherwise it'll be the maximum value of |
| 190 | // `frame_delay` or `restricted_frame_delay_` if it has been set. |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 191 | void ScheduleRepeat(int frame_id, bool idle_repeat) |
| 192 | RTC_RUN_ON(sequence_checker_); |
henrika | f7cdcbd | 2023-11-20 18:32:54 | [diff] [blame] | 193 | // Repeats a frame in the absence of incoming frames. Slows down when quality |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 194 | // convergence is attained, and stops the cadence terminally when new frames |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 195 | // have arrived. |
| 196 | void ProcessRepeatedFrameOnDelayedCadence(int frame_id) |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 197 | RTC_RUN_ON(sequence_checker_); |
henrika | b7ec057 | 2024-01-09 09:48:52 | [diff] [blame] | 198 | // Sends a frame, updating the timestamp to the current time. Also updates |
| 199 | // `queue_overload_count_` based on the time it takes to encode a frame and |
| 200 | // the amount of received frames while encoding. The `queue_overload` |
| 201 | // parameter in the OnFrame callback will be true while |
| 202 | // `queue_overload_count_` is larger than zero to allow the client to drop |
| 203 | // frames and thereby mitigate delay buildups. |
| 204 | // Repeated frames are sent with `post_time` set to absl::nullopt. |
| 205 | void SendFrameNow(absl::optional<Timestamp> post_time, |
| 206 | const VideoFrame& frame) RTC_RUN_ON(sequence_checker_); |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 207 | // Returns the repeat duration depending on if it's an idle repeat or not. |
| 208 | TimeDelta RepeatDuration(bool idle_repeat) const |
| 209 | RTC_RUN_ON(sequence_checker_); |
henrika | b7ec057 | 2024-01-09 09:48:52 | [diff] [blame] | 210 | // Returns the frame duration taking potential restrictions into account. |
| 211 | TimeDelta FrameDuration() const RTC_RUN_ON(sequence_checker_); |
Markus Handell | f5a5079 | 2022-06-16 12:25:15 | [diff] [blame] | 212 | // Unless timer already running, starts repeatedly requesting refresh frames |
| 213 | // after a grace_period. If a frame appears before the grace_period has |
| 214 | // passed, the request is cancelled. |
| 215 | void MaybeStartRefreshFrameRequester() RTC_RUN_ON(sequence_checker_); |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 216 | |
| 217 | TaskQueueBase* const queue_; |
| 218 | Clock* const clock_; |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 219 | FrameCadenceAdapterInterface::Callback* const callback_; |
Markus Handell | 9d04a78 | 2022-05-12 16:38:57 | [diff] [blame] | 220 | |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 221 | // The configured max_fps. |
| 222 | // TODO(crbug.com/1255737): support max_fps updates. |
| 223 | const double max_fps_; |
henrika | b7ec057 | 2024-01-09 09:48:52 | [diff] [blame] | 224 | |
| 225 | // Number of frames that are currently scheduled for processing on the |
| 226 | // `queue_`. |
| 227 | const std::atomic<int>& frames_scheduled_for_processing_; |
| 228 | |
| 229 | // Can be used as kill-switch for the queue overload mechanism. |
| 230 | const bool zero_hertz_queue_overload_enabled_; |
| 231 | |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 232 | // How much the incoming frame sequence is delayed by. |
| 233 | const TimeDelta frame_delay_ = TimeDelta::Seconds(1) / max_fps_; |
| 234 | |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 235 | RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 236 | // A queue of incoming frames and repeated frames. |
| 237 | std::deque<VideoFrame> queued_frames_ RTC_GUARDED_BY(sequence_checker_); |
| 238 | // The current frame ID to use when starting to repeat frames. This is used |
| 239 | // for cancelling deferred repeated frame processing happening. |
| 240 | int current_frame_id_ RTC_GUARDED_BY(sequence_checker_) = 0; |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 241 | // Has content when we are repeating frames. |
| 242 | absl::optional<ScheduledRepeat> scheduled_repeat_ |
| 243 | RTC_GUARDED_BY(sequence_checker_); |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 244 | // Convergent state of each of the configured simulcast layers. |
| 245 | std::vector<SpatialLayerTracker> layer_trackers_ |
| 246 | RTC_GUARDED_BY(sequence_checker_); |
Markus Handell | f7f0b21 | 2022-05-24 16:39:39 | [diff] [blame] | 247 | // Repeating task handle used for requesting refresh frames until arrival, as |
| 248 | // they can be dropped in various places in the capture pipeline. |
| 249 | RepeatingTaskHandle refresh_frame_requester_ |
| 250 | RTC_GUARDED_BY(sequence_checker_); |
henrika | 8f16ce9 | 2023-12-04 13:03:52 | [diff] [blame] | 251 | // Can be set by UpdateVideoSourceRestrictions when the video source restricts |
| 252 | // the max frame rate. |
| 253 | absl::optional<TimeDelta> restricted_frame_delay_ |
| 254 | RTC_GUARDED_BY(sequence_checker_); |
henrika | b7ec057 | 2024-01-09 09:48:52 | [diff] [blame] | 255 | // Set in OnSendFrame to reflect how many future frames will be forwarded with |
| 256 | // the `queue_overload` flag set to true. |
| 257 | int queue_overload_count_ RTC_GUARDED_BY(sequence_checker_) = 0; |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 258 | |
| 259 | ScopedTaskSafety safety_; |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 260 | }; |
| 261 | |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 262 | // Implements a frame cadence adapter supporting VSync aligned encoding. |
| 263 | class VSyncEncodeAdapterMode : public AdapterMode { |
| 264 | public: |
| 265 | VSyncEncodeAdapterMode( |
| 266 | Clock* clock, |
| 267 | TaskQueueBase* queue, |
| 268 | rtc::scoped_refptr<PendingTaskSafetyFlag> queue_safety_flag, |
| 269 | Metronome* metronome, |
| 270 | TaskQueueBase* worker_queue, |
| 271 | FrameCadenceAdapterInterface::Callback* callback) |
| 272 | : clock_(clock), |
| 273 | queue_(queue), |
| 274 | queue_safety_flag_(queue_safety_flag), |
| 275 | callback_(callback), |
| 276 | metronome_(metronome), |
| 277 | worker_queue_(worker_queue) { |
| 278 | queue_sequence_checker_.Detach(); |
| 279 | worker_sequence_checker_.Detach(); |
| 280 | } |
| 281 | |
Markus Handell | e864dec | 2024-08-07 14:11:18 | [diff] [blame] | 282 | void PrepareShutdown() { |
| 283 | MutexLock lock(&queue_lock_); |
| 284 | queue_ = nullptr; |
| 285 | } |
| 286 | |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 287 | // Adapter overrides. |
| 288 | void OnFrame(Timestamp post_time, |
| 289 | bool queue_overload, |
| 290 | const VideoFrame& frame) override; |
| 291 | |
| 292 | absl::optional<uint32_t> GetInputFrameRateFps() override { |
| 293 | RTC_DCHECK_RUN_ON(&queue_sequence_checker_); |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 294 | return last_frame_rate_; |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 295 | } |
| 296 | |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 297 | void UpdateFrameRate(Timestamp frame_timestamp) override { |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 298 | RTC_DCHECK_RUN_ON(&queue_sequence_checker_); |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 299 | // RateStatistics will calculate a too high rate immediately after Update. |
| 300 | last_frame_rate_ = input_framerate_.Rate(frame_timestamp.ms()); |
| 301 | input_framerate_.Update(1, frame_timestamp.ms()); |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 302 | } |
| 303 | |
| 304 | void EncodeAllEnqueuedFrames(); |
| 305 | |
| 306 | private: |
| 307 | // Holds input frames coming from the client ready to be encoded. |
| 308 | struct InputFrameRef { |
| 309 | InputFrameRef(const VideoFrame& video_frame, Timestamp time_when_posted_us) |
| 310 | : time_when_posted_us(time_when_posted_us), |
| 311 | video_frame(std::move(video_frame)) {} |
| 312 | Timestamp time_when_posted_us; |
| 313 | const VideoFrame video_frame; |
| 314 | }; |
| 315 | |
| 316 | Clock* const clock_; |
Markus Handell | e864dec | 2024-08-07 14:11:18 | [diff] [blame] | 317 | // Protects `queue_`. |
| 318 | // TODO: crbug.com/358040973 - We should eventually figure out a way to avoid |
| 319 | // lock protection. |
| 320 | Mutex queue_lock_; |
| 321 | TaskQueueBase* queue_ RTC_GUARDED_BY(queue_lock_) |
| 322 | RTC_PT_GUARDED_BY(queue_lock_); |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 323 | RTC_NO_UNIQUE_ADDRESS SequenceChecker queue_sequence_checker_; |
| 324 | rtc::scoped_refptr<PendingTaskSafetyFlag> queue_safety_flag_; |
| 325 | // Input frame rate statistics for use when not in zero-hertz mode. |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 326 | absl::optional<uint64_t> last_frame_rate_ |
| 327 | RTC_GUARDED_BY(queue_sequence_checker_); |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 328 | RateStatistics input_framerate_ RTC_GUARDED_BY(queue_sequence_checker_){ |
| 329 | FrameCadenceAdapterInterface::kFrameRateAveragingWindowSizeMs, 1000}; |
| 330 | FrameCadenceAdapterInterface::Callback* const callback_; |
| 331 | |
| 332 | Metronome* metronome_; |
| 333 | TaskQueueBase* const worker_queue_; |
| 334 | RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_sequence_checker_; |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 335 | // `worker_safety_` protects tasks on the worker queue related to |
| 336 | // `metronome_` since metronome usage must happen on worker thread. |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 337 | ScopedTaskSafetyDetached worker_safety_; |
| 338 | Timestamp expected_next_tick_ RTC_GUARDED_BY(worker_sequence_checker_) = |
| 339 | Timestamp::PlusInfinity(); |
| 340 | // Vector of input frames to be encoded. |
| 341 | std::vector<InputFrameRef> input_queue_ |
| 342 | RTC_GUARDED_BY(worker_sequence_checker_); |
| 343 | }; |
| 344 | |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 345 | class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface { |
| 346 | public: |
Jonas Oreland | 8ca0613 | 2022-03-14 11:52:48 | [diff] [blame] | 347 | FrameCadenceAdapterImpl(Clock* clock, |
| 348 | TaskQueueBase* queue, |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 349 | Metronome* metronome, |
| 350 | TaskQueueBase* worker_queue, |
Jonas Oreland | e62c2f2 | 2022-03-29 09:04:48 | [diff] [blame] | 351 | const FieldTrialsView& field_trials); |
Markus Handell | e59fee8 | 2021-12-23 08:29:23 | [diff] [blame] | 352 | ~FrameCadenceAdapterImpl(); |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 353 | |
| 354 | // FrameCadenceAdapterInterface overrides. |
| 355 | void Initialize(Callback* callback) override; |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 356 | void SetZeroHertzModeEnabled( |
| 357 | absl::optional<ZeroHertzModeParams> params) override; |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 358 | absl::optional<uint32_t> GetInputFrameRateFps() override; |
Markus Handell | 5a77e51 | 2022-09-01 12:51:50 | [diff] [blame] | 359 | void UpdateLayerQualityConvergence(size_t spatial_index, |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 360 | bool quality_converged) override; |
Markus Handell | 5a77e51 | 2022-09-01 12:51:50 | [diff] [blame] | 361 | void UpdateLayerStatus(size_t spatial_index, bool enabled) override; |
henrika | 8f16ce9 | 2023-12-04 13:03:52 | [diff] [blame] | 362 | void UpdateVideoSourceRestrictions( |
| 363 | absl::optional<double> max_frame_rate) override; |
Markus Handell | 818e7fb | 2021-12-30 12:01:33 | [diff] [blame] | 364 | void ProcessKeyFrameRequest() override; |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 365 | |
| 366 | // VideoFrameSink overrides. |
| 367 | void OnFrame(const VideoFrame& frame) override; |
Markus Handell | f5a5079 | 2022-06-16 12:25:15 | [diff] [blame] | 368 | void OnDiscardedFrame() override; |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 369 | void OnConstraintsChanged( |
| 370 | const VideoTrackSourceConstraints& constraints) override; |
| 371 | |
| 372 | private: |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 373 | void UpdateFrameRate(Timestamp frame_timestamp); |
henrika | 86f09ae | 2023-10-31 13:49:40 | [diff] [blame] | 374 | // Called from OnFrame in both pass-through and zero-hertz mode. |
Markus Handell | 9a478b5 | 2021-11-18 15:07:01 | [diff] [blame] | 375 | void OnFrameOnMainQueue(Timestamp post_time, |
Ilya Nikolaevskiy | 7a841ce | 2023-11-24 12:01:21 | [diff] [blame] | 376 | bool queue_overload, |
Markus Handell | 9a478b5 | 2021-11-18 15:07:01 | [diff] [blame] | 377 | const VideoFrame& frame) RTC_RUN_ON(queue_); |
Markus Handell | 28c7180 | 2021-11-08 09:11:55 | [diff] [blame] | 378 | |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 379 | // Returns true under all of the following conditions: |
| 380 | // - constraints min fps set to 0 |
| 381 | // - constraints max fps set and greater than 0, |
| 382 | // - field trial enabled |
| 383 | // - zero-hertz mode enabled |
| 384 | bool IsZeroHertzScreenshareEnabled() const RTC_RUN_ON(queue_); |
| 385 | |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 386 | // Configures current adapter on non-ZeroHertz mode, called when Initialize or |
| 387 | // MaybeReconfigureAdapters. |
| 388 | void ConfigureCurrentAdapterWithoutZeroHertz(); |
| 389 | |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 390 | // Handles adapter creation on configuration changes. |
| 391 | void MaybeReconfigureAdapters(bool was_zero_hertz_enabled) RTC_RUN_ON(queue_); |
| 392 | |
Markus Handell | 9a478b5 | 2021-11-18 15:07:01 | [diff] [blame] | 393 | Clock* const clock_; |
| 394 | TaskQueueBase* const queue_; |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 395 | |
henrika | b7ec057 | 2024-01-09 09:48:52 | [diff] [blame] | 396 | // Kill-switch for the queue overload mechanism in zero-hertz mode. |
| 397 | const bool frame_cadence_adapter_zero_hertz_queue_overload_enabled_; |
| 398 | |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 399 | // Field trial for using timestamp from video frames, rather than clock when |
| 400 | // calculating input frame rate. |
| 401 | const bool use_video_frame_timestamp_; |
| 402 | // Used for verifying that timestamps are monotonically increasing. |
| 403 | absl::optional<Timestamp> last_incoming_frame_timestamp_; |
| 404 | bool incoming_frame_timestamp_monotonically_increasing_ = true; |
| 405 | |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 406 | // The three possible modes we're under. |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 407 | absl::optional<PassthroughAdapterMode> passthrough_adapter_; |
| 408 | absl::optional<ZeroHertzAdapterMode> zero_hertz_adapter_; |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 409 | // The `vsync_encode_adapter_` must be destroyed on the worker queue since |
| 410 | // VSync metronome needs to happen on worker thread. |
| 411 | std::unique_ptr<VSyncEncodeAdapterMode> vsync_encode_adapter_; |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 412 | // If set, zero-hertz mode has been enabled. |
| 413 | absl::optional<ZeroHertzModeParams> zero_hertz_params_; |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 414 | // Cache for the current adapter mode. |
| 415 | AdapterMode* current_adapter_mode_ = nullptr; |
| 416 | |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 417 | // VSync encoding is used when this valid. |
| 418 | Metronome* const metronome_; |
| 419 | TaskQueueBase* const worker_queue_; |
| 420 | |
Markus Handell | 9d04a78 | 2022-05-12 16:38:57 | [diff] [blame] | 421 | // Timestamp for statistics reporting. |
| 422 | absl::optional<Timestamp> zero_hertz_adapter_created_timestamp_ |
| 423 | RTC_GUARDED_BY(queue_); |
| 424 | |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 425 | // Set up during Initialize. |
| 426 | Callback* callback_ = nullptr; |
| 427 | |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 428 | // The source's constraints. |
| 429 | absl::optional<VideoTrackSourceConstraints> source_constraints_ |
Markus Handell | 9a478b5 | 2021-11-18 15:07:01 | [diff] [blame] | 430 | RTC_GUARDED_BY(queue_); |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 431 | |
henrika | 8f16ce9 | 2023-12-04 13:03:52 | [diff] [blame] | 432 | // Stores the latest restriction in max frame rate set by |
| 433 | // UpdateVideoSourceRestrictions. Ensures that a previously set restriction |
| 434 | // can be maintained during reconstructions of the adapter. |
| 435 | absl::optional<double> restricted_max_frame_rate_ RTC_GUARDED_BY(queue_); |
| 436 | |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 437 | // Race checker for incoming frames. This is the network thread in chromium, |
| 438 | // but may vary from test contexts. |
| 439 | rtc::RaceChecker incoming_frame_race_checker_; |
Markus Handell | 28c7180 | 2021-11-08 09:11:55 | [diff] [blame] | 440 | |
Markus Handell | 9a478b5 | 2021-11-18 15:07:01 | [diff] [blame] | 441 | // Number of frames that are currently scheduled for processing on the |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 442 | // `queue_`. |
Markus Handell | 9a478b5 | 2021-11-18 15:07:01 | [diff] [blame] | 443 | std::atomic<int> frames_scheduled_for_processing_{0}; |
| 444 | |
| 445 | ScopedTaskSafetyDetached safety_; |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 446 | }; |
| 447 | |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 448 | ZeroHertzAdapterMode::ZeroHertzAdapterMode( |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 449 | TaskQueueBase* queue, |
| 450 | Clock* clock, |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 451 | FrameCadenceAdapterInterface::Callback* callback, |
henrika | b7ec057 | 2024-01-09 09:48:52 | [diff] [blame] | 452 | double max_fps, |
| 453 | std::atomic<int>& frames_scheduled_for_processing, |
| 454 | bool zero_hertz_queue_overload_enabled) |
| 455 | : queue_(queue), |
| 456 | clock_(clock), |
| 457 | callback_(callback), |
| 458 | max_fps_(max_fps), |
| 459 | frames_scheduled_for_processing_(frames_scheduled_for_processing), |
| 460 | zero_hertz_queue_overload_enabled_(zero_hertz_queue_overload_enabled) { |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 461 | sequence_checker_.Detach(); |
Markus Handell | f5a5079 | 2022-06-16 12:25:15 | [diff] [blame] | 462 | MaybeStartRefreshFrameRequester(); |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 463 | } |
| 464 | |
Markus Handell | e59fee8 | 2021-12-23 08:29:23 | [diff] [blame] | 465 | void ZeroHertzAdapterMode::ReconfigureParameters( |
| 466 | const FrameCadenceAdapterInterface::ZeroHertzModeParams& params) { |
| 467 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
Markus Handell | f7f0b21 | 2022-05-24 16:39:39 | [diff] [blame] | 468 | RTC_DLOG(LS_INFO) << __func__ << " this " << this << " num_simulcast_layers " |
| 469 | << params.num_simulcast_layers; |
Markus Handell | e59fee8 | 2021-12-23 08:29:23 | [diff] [blame] | 470 | |
| 471 | // Start as unconverged. |
| 472 | layer_trackers_.clear(); |
| 473 | layer_trackers_.resize(params.num_simulcast_layers, |
| 474 | SpatialLayerTracker{false}); |
| 475 | } |
| 476 | |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 477 | void ZeroHertzAdapterMode::UpdateLayerQualityConvergence( |
Markus Handell | 5a77e51 | 2022-09-01 12:51:50 | [diff] [blame] | 478 | size_t spatial_index, |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 479 | bool quality_converged) { |
| 480 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
Markus Handell | f2827c4 | 2023-09-02 07:41:10 | [diff] [blame] | 481 | TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc"), __func__, |
Evan Shrubsole | 5bfcc87 | 2024-04-23 08:51:23 | [diff] [blame] | 482 | TRACE_EVENT_SCOPE_GLOBAL, "spatial_index", spatial_index, |
| 483 | "converged", quality_converged); |
Markus Handell | 5a77e51 | 2022-09-01 12:51:50 | [diff] [blame] | 484 | if (spatial_index >= layer_trackers_.size()) |
| 485 | return; |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 486 | if (layer_trackers_[spatial_index].quality_converged.has_value()) |
| 487 | layer_trackers_[spatial_index].quality_converged = quality_converged; |
| 488 | } |
| 489 | |
Markus Handell | 5a77e51 | 2022-09-01 12:51:50 | [diff] [blame] | 490 | void ZeroHertzAdapterMode::UpdateLayerStatus(size_t spatial_index, |
| 491 | bool enabled) { |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 492 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
Markus Handell | f2827c4 | 2023-09-02 07:41:10 | [diff] [blame] | 493 | TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc"), __func__, |
Evan Shrubsole | 5bfcc87 | 2024-04-23 08:51:23 | [diff] [blame] | 494 | TRACE_EVENT_SCOPE_GLOBAL, "spatial_index", spatial_index, |
| 495 | "enabled", enabled); |
Markus Handell | 5a77e51 | 2022-09-01 12:51:50 | [diff] [blame] | 496 | if (spatial_index >= layer_trackers_.size()) |
| 497 | return; |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 498 | if (enabled) { |
| 499 | if (!layer_trackers_[spatial_index].quality_converged.has_value()) { |
| 500 | // Assume quality has not converged until hearing otherwise. |
| 501 | layer_trackers_[spatial_index].quality_converged = false; |
| 502 | } |
| 503 | } else { |
| 504 | layer_trackers_[spatial_index].quality_converged = absl::nullopt; |
| 505 | } |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 506 | } |
| 507 | |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 508 | void ZeroHertzAdapterMode::OnFrame(Timestamp post_time, |
Markus Handell | 84c016a | 2023-11-23 12:41:44 | [diff] [blame] | 509 | bool queue_overload, |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 510 | const VideoFrame& frame) { |
| 511 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
Markus Handell | f2827c4 | 2023-09-02 07:41:10 | [diff] [blame] | 512 | TRACE_EVENT0("webrtc", "ZeroHertzAdapterMode::OnFrame"); |
Markus Handell | f7f0b21 | 2022-05-24 16:39:39 | [diff] [blame] | 513 | refresh_frame_requester_.Stop(); |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 514 | |
| 515 | // Assume all enabled layers are unconverged after frame entry. |
Markus Handell | e59fee8 | 2021-12-23 08:29:23 | [diff] [blame] | 516 | ResetQualityConvergenceInfo(); |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 517 | |
| 518 | // Remove stored repeating frame if needed. |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 519 | if (scheduled_repeat_.has_value()) { |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 520 | RTC_DCHECK(queued_frames_.size() == 1); |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 521 | RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this |
| 522 | << " cancel repeat and restart with original"; |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 523 | queued_frames_.pop_front(); |
| 524 | } |
| 525 | |
| 526 | // Store the frame in the queue and schedule deferred processing. |
| 527 | queued_frames_.push_back(frame); |
| 528 | current_frame_id_++; |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 529 | scheduled_repeat_ = absl::nullopt; |
henrika | 40ed3ff | 2023-10-03 08:35:52 | [diff] [blame] | 530 | TimeDelta time_spent_since_post = clock_->CurrentTime() - post_time; |
Henrik Boström | 2dd3915 | 2022-01-25 07:20:33 | [diff] [blame] | 531 | queue_->PostDelayedHighPrecisionTask( |
Danil Chapovalov | 95eeaa7 | 2022-07-06 08:14:29 | [diff] [blame] | 532 | SafeTask(safety_.flag(), |
henrika | 97439b9 | 2023-12-27 13:10:19 | [diff] [blame] | 533 | [this, post_time] { |
Danil Chapovalov | 95eeaa7 | 2022-07-06 08:14:29 | [diff] [blame] | 534 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
henrika | f7cdcbd | 2023-11-20 18:32:54 | [diff] [blame] | 535 | ProcessOnDelayedCadence(post_time); |
Danil Chapovalov | 95eeaa7 | 2022-07-06 08:14:29 | [diff] [blame] | 536 | }), |
Markus Handell | 8fa8619 | 2023-08-31 09:59:06 | [diff] [blame] | 537 | std::max(frame_delay_ - time_spent_since_post, TimeDelta::Zero())); |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 538 | } |
| 539 | |
Markus Handell | f5a5079 | 2022-06-16 12:25:15 | [diff] [blame] | 540 | void ZeroHertzAdapterMode::OnDiscardedFrame() { |
| 541 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
Markus Handell | f2827c4 | 2023-09-02 07:41:10 | [diff] [blame] | 542 | TRACE_EVENT0("webrtc", __func__); |
Markus Handell | f5a5079 | 2022-06-16 12:25:15 | [diff] [blame] | 543 | |
| 544 | // Under zero hertz source delivery, a discarded frame ending a sequence of |
| 545 | // frames which happened to contain important information can be seen as a |
| 546 | // capture freeze. Avoid this by starting requesting refresh frames after a |
| 547 | // grace period. |
| 548 | MaybeStartRefreshFrameRequester(); |
| 549 | } |
| 550 | |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 551 | absl::optional<uint32_t> ZeroHertzAdapterMode::GetInputFrameRateFps() { |
| 552 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
| 553 | return max_fps_; |
| 554 | } |
| 555 | |
henrika | 8f16ce9 | 2023-12-04 13:03:52 | [diff] [blame] | 556 | void ZeroHertzAdapterMode::UpdateVideoSourceRestrictions( |
| 557 | absl::optional<double> max_frame_rate) { |
| 558 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
| 559 | TRACE_EVENT_INSTANT1(TRACE_DISABLED_BY_DEFAULT("webrtc"), __func__, |
Evan Shrubsole | 5bfcc87 | 2024-04-23 08:51:23 | [diff] [blame] | 560 | TRACE_EVENT_SCOPE_GLOBAL, "max_frame_rate", |
| 561 | max_frame_rate.value_or(-1)); |
henrika | 8f16ce9 | 2023-12-04 13:03:52 | [diff] [blame] | 562 | if (max_frame_rate.value_or(0) > 0) { |
| 563 | // Set new, validated (> 0) and restricted frame rate. |
| 564 | restricted_frame_delay_ = TimeDelta::Seconds(1) / *max_frame_rate; |
| 565 | } else { |
| 566 | // Source reports that the frame rate is now unrestricted. |
| 567 | restricted_frame_delay_ = absl::nullopt; |
| 568 | } |
| 569 | } |
| 570 | |
Markus Handell | 818e7fb | 2021-12-30 12:01:33 | [diff] [blame] | 571 | void ZeroHertzAdapterMode::ProcessKeyFrameRequest() { |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 572 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
Evan Shrubsole | 5bfcc87 | 2024-04-23 08:51:23 | [diff] [blame] | 573 | TRACE_EVENT_INSTANT0("webrtc", __func__, TRACE_EVENT_SCOPE_GLOBAL); |
Markus Handell | f7f0b21 | 2022-05-24 16:39:39 | [diff] [blame] | 574 | // If we're new and don't have a frame, there's no need to request refresh |
| 575 | // frames as this was being triggered for us when zero-hz mode was set up. |
| 576 | // |
Markus Handell | cb237f8 | 2021-12-29 20:31:57 | [diff] [blame] | 577 | // The next frame encoded will be a key frame. Reset quality convergence so we |
| 578 | // don't get idle repeats shortly after, because key frames need a lot of |
| 579 | // refinement frames. |
| 580 | ResetQualityConvergenceInfo(); |
| 581 | |
Markus Handell | 90a7e2c | 2021-12-29 22:32:30 | [diff] [blame] | 582 | // If we're not repeating, or we're repeating with short duration, we will |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 583 | // very soon send out a frame and don't need a refresh frame. |
| 584 | if (!scheduled_repeat_.has_value() || !scheduled_repeat_->idle) { |
| 585 | RTC_LOG(LS_INFO) << __func__ << " this " << this |
Markus Handell | e59fee8 | 2021-12-23 08:29:23 | [diff] [blame] | 586 | << " not requesting refresh frame because of recently " |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 587 | "incoming frame or short repeating."; |
Markus Handell | 818e7fb | 2021-12-30 12:01:33 | [diff] [blame] | 588 | return; |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 589 | } |
| 590 | |
| 591 | // If the repeat is scheduled within a short (i.e. frame_delay_) interval, we |
| 592 | // will very soon send out a frame and don't need a refresh frame. |
| 593 | Timestamp now = clock_->CurrentTime(); |
| 594 | if (scheduled_repeat_->scheduled + RepeatDuration(/*idle_repeat=*/true) - |
| 595 | now <= |
| 596 | frame_delay_) { |
Markus Handell | e59fee8 | 2021-12-23 08:29:23 | [diff] [blame] | 597 | RTC_LOG(LS_INFO) << __func__ << " this " << this |
| 598 | << " not requesting refresh frame because of soon " |
| 599 | "happening idle repeat"; |
Markus Handell | 818e7fb | 2021-12-30 12:01:33 | [diff] [blame] | 600 | return; |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 601 | } |
| 602 | |
| 603 | // Cancel the current repeat and reschedule a short repeat now. No need for a |
| 604 | // new refresh frame. |
| 605 | RTC_LOG(LS_INFO) << __func__ << " this " << this |
Markus Handell | e59fee8 | 2021-12-23 08:29:23 | [diff] [blame] | 606 | << " not requesting refresh frame and scheduling a short " |
| 607 | "repeat due to key frame request"; |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 608 | ScheduleRepeat(++current_frame_id_, /*idle_repeat=*/false); |
Markus Handell | 818e7fb | 2021-12-30 12:01:33 | [diff] [blame] | 609 | return; |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 610 | } |
| 611 | |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 612 | bool ZeroHertzAdapterMode::HasQualityConverged() const { |
Danil Chapovalov | 6e7c268 | 2022-07-25 13:58:28 | [diff] [blame] | 613 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
Markus Handell | cb237f8 | 2021-12-29 20:31:57 | [diff] [blame] | 614 | // 1. Define ourselves as unconverged with no spatial layers configured. This |
| 615 | // is to keep short repeating until the layer configuration comes. |
| 616 | // 2. Unset layers implicitly imply that they're converged to support |
| 617 | // disabling layers when they're not needed. |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 618 | const bool quality_converged = |
Markus Handell | cb237f8 | 2021-12-29 20:31:57 | [diff] [blame] | 619 | !layer_trackers_.empty() && |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 620 | absl::c_all_of(layer_trackers_, [](const SpatialLayerTracker& tracker) { |
| 621 | return tracker.quality_converged.value_or(true); |
| 622 | }); |
| 623 | return quality_converged; |
| 624 | } |
| 625 | |
Markus Handell | e59fee8 | 2021-12-23 08:29:23 | [diff] [blame] | 626 | void ZeroHertzAdapterMode::ResetQualityConvergenceInfo() { |
Danil Chapovalov | 6e7c268 | 2022-07-25 13:58:28 | [diff] [blame] | 627 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
Markus Handell | e59fee8 | 2021-12-23 08:29:23 | [diff] [blame] | 628 | RTC_DLOG(LS_INFO) << __func__ << " this " << this; |
| 629 | for (auto& layer_tracker : layer_trackers_) { |
| 630 | if (layer_tracker.quality_converged.has_value()) |
| 631 | layer_tracker.quality_converged = false; |
| 632 | } |
| 633 | } |
| 634 | |
henrika | f7cdcbd | 2023-11-20 18:32:54 | [diff] [blame] | 635 | void ZeroHertzAdapterMode::ProcessOnDelayedCadence(Timestamp post_time) { |
Danil Chapovalov | 6e7c268 | 2022-07-25 13:58:28 | [diff] [blame] | 636 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 637 | RTC_DCHECK(!queued_frames_.empty()); |
Markus Handell | f2827c4 | 2023-09-02 07:41:10 | [diff] [blame] | 638 | TRACE_EVENT0("webrtc", __func__); |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 639 | |
Markus Handell | f2827c4 | 2023-09-02 07:41:10 | [diff] [blame] | 640 | // Avoid sending the front frame for encoding (which could take a long time) |
henrika | f7cdcbd | 2023-11-20 18:32:54 | [diff] [blame] | 641 | // until we schedule a repeat. |
Markus Handell | f2827c4 | 2023-09-02 07:41:10 | [diff] [blame] | 642 | VideoFrame front_frame = queued_frames_.front(); |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 643 | |
| 644 | // If there were two or more frames stored, we do not have to schedule repeats |
| 645 | // of the front frame. |
| 646 | if (queued_frames_.size() > 1) { |
| 647 | queued_frames_.pop_front(); |
Markus Handell | f2827c4 | 2023-09-02 07:41:10 | [diff] [blame] | 648 | } else { |
| 649 | // There's only one frame to send. Schedule a repeat sequence, which is |
| 650 | // cancelled by `current_frame_id_` getting incremented should new frames |
| 651 | // arrive. |
| 652 | ScheduleRepeat(current_frame_id_, HasQualityConverged()); |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 653 | } |
henrika | f7cdcbd | 2023-11-20 18:32:54 | [diff] [blame] | 654 | SendFrameNow(post_time, front_frame); |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 655 | } |
| 656 | |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 657 | void ZeroHertzAdapterMode::ScheduleRepeat(int frame_id, bool idle_repeat) { |
Danil Chapovalov | 6e7c268 | 2022-07-25 13:58:28 | [diff] [blame] | 658 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
Markus Handell | 90a7e2c | 2021-12-29 22:32:30 | [diff] [blame] | 659 | Timestamp now = clock_->CurrentTime(); |
| 660 | if (!scheduled_repeat_.has_value()) { |
| 661 | scheduled_repeat_.emplace(now, queued_frames_.front().timestamp_us(), |
| 662 | queued_frames_.front().ntp_time_ms()); |
| 663 | } |
| 664 | scheduled_repeat_->scheduled = now; |
| 665 | scheduled_repeat_->idle = idle_repeat; |
| 666 | |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 667 | TimeDelta repeat_delay = RepeatDuration(idle_repeat); |
Henrik Boström | 2dd3915 | 2022-01-25 07:20:33 | [diff] [blame] | 668 | queue_->PostDelayedHighPrecisionTask( |
Danil Chapovalov | 95eeaa7 | 2022-07-06 08:14:29 | [diff] [blame] | 669 | SafeTask(safety_.flag(), |
| 670 | [this, frame_id] { |
| 671 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
| 672 | ProcessRepeatedFrameOnDelayedCadence(frame_id); |
| 673 | }), |
| 674 | repeat_delay); |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 675 | } |
| 676 | |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 677 | void ZeroHertzAdapterMode::ProcessRepeatedFrameOnDelayedCadence(int frame_id) { |
Danil Chapovalov | 6e7c268 | 2022-07-25 13:58:28 | [diff] [blame] | 678 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
Markus Handell | f2827c4 | 2023-09-02 07:41:10 | [diff] [blame] | 679 | TRACE_EVENT0("webrtc", __func__); |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 680 | RTC_DCHECK(!queued_frames_.empty()); |
| 681 | |
| 682 | // Cancel this invocation if new frames turned up. |
| 683 | if (frame_id != current_frame_id_) |
| 684 | return; |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 685 | RTC_DCHECK(scheduled_repeat_.has_value()); |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 686 | |
| 687 | VideoFrame& frame = queued_frames_.front(); |
| 688 | |
| 689 | // Since this is a repeated frame, nothing changed compared to before. |
| 690 | VideoFrame::UpdateRect empty_update_rect; |
| 691 | empty_update_rect.MakeEmptyUpdate(); |
| 692 | frame.set_update_rect(empty_update_rect); |
| 693 | |
Markus Handell | 90a7e2c | 2021-12-29 22:32:30 | [diff] [blame] | 694 | // Adjust timestamps of the frame of the repeat, accounting for the actual |
| 695 | // delay since we started repeating. |
| 696 | // |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 697 | // NOTE: No need to update the RTP timestamp as the VideoStreamEncoder |
| 698 | // overwrites it based on its chosen NTP timestamp source. |
Markus Handell | 90a7e2c | 2021-12-29 22:32:30 | [diff] [blame] | 699 | TimeDelta total_delay = clock_->CurrentTime() - scheduled_repeat_->origin; |
| 700 | if (frame.timestamp_us() > 0) { |
| 701 | frame.set_timestamp_us(scheduled_repeat_->origin_timestamp_us + |
| 702 | total_delay.us()); |
| 703 | } |
| 704 | if (frame.ntp_time_ms()) { |
| 705 | frame.set_ntp_time_ms(scheduled_repeat_->origin_ntp_time_ms + |
| 706 | total_delay.ms()); |
| 707 | } |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 708 | |
Markus Handell | f2827c4 | 2023-09-02 07:41:10 | [diff] [blame] | 709 | // Schedule another repeat before sending the frame off which could take time. |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 710 | ScheduleRepeat(frame_id, HasQualityConverged()); |
henrika | b7ec057 | 2024-01-09 09:48:52 | [diff] [blame] | 711 | SendFrameNow(absl::nullopt, frame); |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 712 | } |
| 713 | |
henrika | b7ec057 | 2024-01-09 09:48:52 | [diff] [blame] | 714 | void ZeroHertzAdapterMode::SendFrameNow(absl::optional<Timestamp> post_time, |
| 715 | const VideoFrame& frame) { |
Danil Chapovalov | 6e7c268 | 2022-07-25 13:58:28 | [diff] [blame] | 716 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
Markus Handell | f2827c4 | 2023-09-02 07:41:10 | [diff] [blame] | 717 | TRACE_EVENT0("webrtc", __func__); |
henrika | b7ec057 | 2024-01-09 09:48:52 | [diff] [blame] | 718 | |
| 719 | Timestamp encode_start_time = clock_->CurrentTime(); |
| 720 | if (post_time.has_value()) { |
| 721 | TimeDelta delay = (encode_start_time - *post_time); |
henrika | f7cdcbd | 2023-11-20 18:32:54 | [diff] [blame] | 722 | RTC_HISTOGRAM_COUNTS_10000("WebRTC.Screenshare.ZeroHz.DelayMs", delay.ms()); |
| 723 | } |
henrika | b7ec057 | 2024-01-09 09:48:52 | [diff] [blame] | 724 | |
| 725 | // Forward the frame and set `queue_overload` if is has been detected that it |
| 726 | // is not possible to deliver frames at the expected rate due to slow |
| 727 | // encoding. |
| 728 | callback_->OnFrame(/*post_time=*/encode_start_time, queue_overload_count_ > 0, |
| 729 | frame); |
| 730 | |
| 731 | // WebRTC-ZeroHertzQueueOverload kill-switch. |
| 732 | if (!zero_hertz_queue_overload_enabled_) |
| 733 | return; |
| 734 | |
| 735 | // `queue_overload_count_` determines for how many future frames the |
| 736 | // `queue_overload` flag will be set and it is only increased if: |
| 737 | // o We are not already in an overload state. |
| 738 | // o New frames have been scheduled for processing on the queue while encoding |
| 739 | // took place in OnFrame. |
| 740 | // o The duration of OnFrame is longer than the current frame duration. |
| 741 | // If all these conditions are fulfilled, `queue_overload_count_` is set to |
| 742 | // `frames_scheduled_for_processing_` and any pending repeat is canceled since |
| 743 | // new frames are available and the repeat is not needed. |
| 744 | // If the adapter is already in an overload state, simply decrease |
| 745 | // `queue_overload_count_` by one. |
| 746 | if (queue_overload_count_ == 0) { |
| 747 | const int frames_scheduled_for_processing = |
| 748 | frames_scheduled_for_processing_.load(std::memory_order_relaxed); |
| 749 | if (frames_scheduled_for_processing > 0) { |
| 750 | TimeDelta encode_time = clock_->CurrentTime() - encode_start_time; |
| 751 | if (encode_time > FrameDuration()) { |
| 752 | queue_overload_count_ = frames_scheduled_for_processing; |
| 753 | // Invalidates any outstanding repeat to avoid sending pending repeat |
| 754 | // directly after too long encode. |
| 755 | current_frame_id_++; |
| 756 | } |
| 757 | } |
| 758 | } else { |
| 759 | queue_overload_count_--; |
| 760 | } |
| 761 | RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.ZeroHz.QueueOverload", |
| 762 | queue_overload_count_ > 0); |
| 763 | } |
| 764 | |
| 765 | TimeDelta ZeroHertzAdapterMode::FrameDuration() const { |
| 766 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
| 767 | return std::max(frame_delay_, restricted_frame_delay_.value_or(frame_delay_)); |
Markus Handell | 29dd8d8 | 2021-12-15 11:19:15 | [diff] [blame] | 768 | } |
| 769 | |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 770 | TimeDelta ZeroHertzAdapterMode::RepeatDuration(bool idle_repeat) const { |
Danil Chapovalov | 6e7c268 | 2022-07-25 13:58:28 | [diff] [blame] | 771 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 772 | return idle_repeat |
| 773 | ? FrameCadenceAdapterInterface::kZeroHertzIdleRepeatRatePeriod |
henrika | b7ec057 | 2024-01-09 09:48:52 | [diff] [blame] | 774 | : FrameDuration(); |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 775 | } |
| 776 | |
Markus Handell | f5a5079 | 2022-06-16 12:25:15 | [diff] [blame] | 777 | void ZeroHertzAdapterMode::MaybeStartRefreshFrameRequester() { |
Danil Chapovalov | 6e7c268 | 2022-07-25 13:58:28 | [diff] [blame] | 778 | RTC_DCHECK_RUN_ON(&sequence_checker_); |
Markus Handell | f5a5079 | 2022-06-16 12:25:15 | [diff] [blame] | 779 | if (!refresh_frame_requester_.Running()) { |
| 780 | refresh_frame_requester_ = RepeatingTaskHandle::DelayedStart( |
| 781 | queue_, |
| 782 | FrameCadenceAdapterInterface::kOnDiscardedFrameRefreshFramePeriod * |
| 783 | frame_delay_, |
| 784 | [this] { |
| 785 | RTC_DLOG(LS_VERBOSE) << __func__ << " RequestRefreshFrame"; |
| 786 | if (callback_) |
| 787 | callback_->RequestRefreshFrame(); |
| 788 | return frame_delay_; |
| 789 | }); |
| 790 | } |
| 791 | } |
| 792 | |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 793 | void VSyncEncodeAdapterMode::OnFrame(Timestamp post_time, |
| 794 | bool queue_overload, |
| 795 | const VideoFrame& frame) { |
| 796 | // We expect `metronome_` and `EncodeAllEnqueuedFrames()` runs on |
| 797 | // `worker_queue_`. |
| 798 | if (!worker_queue_->IsCurrent()) { |
| 799 | worker_queue_->PostTask(SafeTask( |
| 800 | worker_safety_.flag(), [this, post_time, queue_overload, frame] { |
| 801 | OnFrame(post_time, queue_overload, frame); |
| 802 | })); |
| 803 | return; |
| 804 | } |
| 805 | |
| 806 | RTC_DCHECK_RUN_ON(&worker_sequence_checker_); |
| 807 | TRACE_EVENT0("webrtc", "VSyncEncodeAdapterMode::OnFrame"); |
| 808 | |
| 809 | input_queue_.emplace_back(std::move(frame), post_time); |
| 810 | |
| 811 | // The `metronome_` tick period maybe throttled in some case, so here we only |
| 812 | // align encode task to VSync event when `metronome_` tick period is less |
| 813 | // than 34ms (30Hz). |
| 814 | static constexpr TimeDelta kMaxAllowedDelay = TimeDelta::Millis(34); |
| 815 | if (metronome_->TickPeriod() <= kMaxAllowedDelay) { |
| 816 | // The metronome is ticking frequently enough that it is worth the extra |
| 817 | // delay. |
| 818 | metronome_->RequestCallOnNextTick( |
| 819 | SafeTask(worker_safety_.flag(), [this] { EncodeAllEnqueuedFrames(); })); |
| 820 | } else { |
| 821 | // The metronome is ticking too infrequently, encode immediately. |
| 822 | EncodeAllEnqueuedFrames(); |
| 823 | } |
| 824 | } |
| 825 | |
| 826 | void VSyncEncodeAdapterMode::EncodeAllEnqueuedFrames() { |
| 827 | RTC_DCHECK_RUN_ON(&worker_sequence_checker_); |
| 828 | TRACE_EVENT0("webrtc", "VSyncEncodeAdapterMode::EncodeAllEnqueuedFrames"); |
| 829 | |
| 830 | // Local time in webrtc time base. |
| 831 | Timestamp post_time = clock_->CurrentTime(); |
| 832 | |
| 833 | for (auto& input : input_queue_) { |
| 834 | TRACE_EVENT1("webrtc", "FrameCadenceAdapterImpl::EncodeAllEnqueuedFrames", |
| 835 | "VSyncEncodeDelay", |
| 836 | (post_time - input.time_when_posted_us).ms()); |
| 837 | |
| 838 | const VideoFrame frame = std::move(input.video_frame); |
Markus Handell | e864dec | 2024-08-07 14:11:18 | [diff] [blame] | 839 | MutexLock lock(&queue_lock_); |
| 840 | if (queue_) { |
| 841 | queue_->PostTask(SafeTask(queue_safety_flag_, [this, post_time, frame] { |
| 842 | { |
| 843 | MutexLock lock(&queue_lock_); |
| 844 | if (!queue_) { |
| 845 | return; |
| 846 | } |
| 847 | RTC_DCHECK_RUN_ON(queue_); |
| 848 | } |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 849 | |
Markus Handell | e864dec | 2024-08-07 14:11:18 | [diff] [blame] | 850 | // TODO(b/304158952): Support more refined queue overload control. |
| 851 | // Not running under mutex is safe since `callback_` existence is |
| 852 | // guaranteed to exist as long as running encode queue tasks exist. |
| 853 | callback_->OnFrame(post_time, /*queue_overload=*/false, frame); |
| 854 | })); |
| 855 | } |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 856 | } |
| 857 | |
| 858 | input_queue_.clear(); |
| 859 | } |
| 860 | |
Jonas Oreland | 8ca0613 | 2022-03-14 11:52:48 | [diff] [blame] | 861 | FrameCadenceAdapterImpl::FrameCadenceAdapterImpl( |
| 862 | Clock* clock, |
| 863 | TaskQueueBase* queue, |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 864 | Metronome* metronome, |
| 865 | TaskQueueBase* worker_queue, |
Jonas Oreland | e62c2f2 | 2022-03-29 09:04:48 | [diff] [blame] | 866 | const FieldTrialsView& field_trials) |
Markus Handell | 9a478b5 | 2021-11-18 15:07:01 | [diff] [blame] | 867 | : clock_(clock), |
| 868 | queue_(queue), |
henrika | b7ec057 | 2024-01-09 09:48:52 | [diff] [blame] | 869 | frame_cadence_adapter_zero_hertz_queue_overload_enabled_( |
| 870 | !field_trials.IsDisabled("WebRTC-ZeroHertzQueueOverload")), |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 871 | use_video_frame_timestamp_(field_trials.IsEnabled( |
| 872 | "WebRTC-FrameCadenceAdapter-UseVideoFrameTimestamp")), |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 873 | metronome_(metronome), |
| 874 | worker_queue_(worker_queue) {} |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 875 | |
Markus Handell | e59fee8 | 2021-12-23 08:29:23 | [diff] [blame] | 876 | FrameCadenceAdapterImpl::~FrameCadenceAdapterImpl() { |
| 877 | RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this; |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 878 | |
| 879 | // VSync adapter needs to be destroyed on worker queue when metronome is |
| 880 | // valid. |
| 881 | if (metronome_) { |
Markus Handell | e864dec | 2024-08-07 14:11:18 | [diff] [blame] | 882 | vsync_encode_adapter_->PrepareShutdown(); |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 883 | absl::Cleanup cleanup = [adapter = std::move(vsync_encode_adapter_)] {}; |
| 884 | worker_queue_->PostTask([cleanup = std::move(cleanup)] {}); |
| 885 | } |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 886 | |
| 887 | RTC_HISTOGRAM_BOOLEAN( |
| 888 | "WebRTC.Video.InputFrameTimestampMonotonicallyIncreasing", |
| 889 | incoming_frame_timestamp_monotonically_increasing_); |
Markus Handell | e59fee8 | 2021-12-23 08:29:23 | [diff] [blame] | 890 | } |
| 891 | |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 892 | void FrameCadenceAdapterImpl::Initialize(Callback* callback) { |
| 893 | callback_ = callback; |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 894 | // Use VSync encode mode if metronome is valid, otherwise passthrough mode |
| 895 | // would be used. |
| 896 | if (metronome_) { |
| 897 | vsync_encode_adapter_ = std::make_unique<VSyncEncodeAdapterMode>( |
| 898 | clock_, queue_, safety_.flag(), metronome_, worker_queue_, callback_); |
| 899 | } else { |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 900 | passthrough_adapter_.emplace(callback); |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 901 | } |
| 902 | ConfigureCurrentAdapterWithoutZeroHertz(); |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 903 | } |
| 904 | |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 905 | void FrameCadenceAdapterImpl::SetZeroHertzModeEnabled( |
| 906 | absl::optional<ZeroHertzModeParams> params) { |
Markus Handell | 9a478b5 | 2021-11-18 15:07:01 | [diff] [blame] | 907 | RTC_DCHECK_RUN_ON(queue_); |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 908 | bool was_zero_hertz_enabled = zero_hertz_params_.has_value(); |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 909 | zero_hertz_params_ = params; |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 910 | MaybeReconfigureAdapters(was_zero_hertz_enabled); |
| 911 | } |
| 912 | |
| 913 | absl::optional<uint32_t> FrameCadenceAdapterImpl::GetInputFrameRateFps() { |
| 914 | RTC_DCHECK_RUN_ON(queue_); |
| 915 | return current_adapter_mode_->GetInputFrameRateFps(); |
| 916 | } |
| 917 | |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 918 | void FrameCadenceAdapterImpl::UpdateFrameRate(Timestamp frame_timestamp) { |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 919 | RTC_DCHECK_RUN_ON(queue_); |
| 920 | // The frame rate need not be updated for the zero-hertz adapter. The |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 921 | // vsync encode and passthrough adapter however uses it. Always pass frames |
| 922 | // into the vsync encode or passthrough to keep the estimation alive should |
| 923 | // there be an adapter switch. |
| 924 | if (metronome_) { |
| 925 | RTC_CHECK(vsync_encode_adapter_); |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 926 | vsync_encode_adapter_->UpdateFrameRate(frame_timestamp); |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 927 | } else { |
| 928 | RTC_CHECK(passthrough_adapter_); |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 929 | passthrough_adapter_->UpdateFrameRate(frame_timestamp); |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 930 | } |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 931 | } |
| 932 | |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 933 | void FrameCadenceAdapterImpl::UpdateLayerQualityConvergence( |
Markus Handell | 5a77e51 | 2022-09-01 12:51:50 | [diff] [blame] | 934 | size_t spatial_index, |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 935 | bool quality_converged) { |
| 936 | if (zero_hertz_adapter_.has_value()) |
| 937 | zero_hertz_adapter_->UpdateLayerQualityConvergence(spatial_index, |
| 938 | quality_converged); |
| 939 | } |
| 940 | |
Markus Handell | 5a77e51 | 2022-09-01 12:51:50 | [diff] [blame] | 941 | void FrameCadenceAdapterImpl::UpdateLayerStatus(size_t spatial_index, |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 942 | bool enabled) { |
| 943 | if (zero_hertz_adapter_.has_value()) |
| 944 | zero_hertz_adapter_->UpdateLayerStatus(spatial_index, enabled); |
| 945 | } |
| 946 | |
henrika | 8f16ce9 | 2023-12-04 13:03:52 | [diff] [blame] | 947 | void FrameCadenceAdapterImpl::UpdateVideoSourceRestrictions( |
| 948 | absl::optional<double> max_frame_rate) { |
| 949 | RTC_DCHECK_RUN_ON(queue_); |
| 950 | // Store the restriction to ensure that it can be reapplied in possible |
| 951 | // future adapter creations on configuration changes. |
| 952 | restricted_max_frame_rate_ = max_frame_rate; |
| 953 | if (zero_hertz_adapter_) { |
| 954 | zero_hertz_adapter_->UpdateVideoSourceRestrictions(max_frame_rate); |
| 955 | } |
| 956 | } |
| 957 | |
Markus Handell | 818e7fb | 2021-12-30 12:01:33 | [diff] [blame] | 958 | void FrameCadenceAdapterImpl::ProcessKeyFrameRequest() { |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 959 | RTC_DCHECK_RUN_ON(queue_); |
| 960 | if (zero_hertz_adapter_) |
Markus Handell | 818e7fb | 2021-12-30 12:01:33 | [diff] [blame] | 961 | zero_hertz_adapter_->ProcessKeyFrameRequest(); |
Markus Handell | 2e0f4f0 | 2021-12-21 18:14:58 | [diff] [blame] | 962 | } |
| 963 | |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 964 | void FrameCadenceAdapterImpl::OnFrame(const VideoFrame& frame) { |
| 965 | // This method is called on the network thread under Chromium, or other |
| 966 | // various contexts in test. |
| 967 | RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); |
Markus Handell | f2827c4 | 2023-09-02 07:41:10 | [diff] [blame] | 968 | TRACE_EVENT0("webrtc", "FrameCadenceAdapterImpl::OnFrame"); |
Markus Handell | 9a478b5 | 2021-11-18 15:07:01 | [diff] [blame] | 969 | |
| 970 | // Local time in webrtc time base. |
| 971 | Timestamp post_time = clock_->CurrentTime(); |
| 972 | frames_scheduled_for_processing_.fetch_add(1, std::memory_order_relaxed); |
Danil Chapovalov | 95eeaa7 | 2022-07-06 08:14:29 | [diff] [blame] | 973 | queue_->PostTask(SafeTask(safety_.flag(), [this, post_time, frame] { |
Markus Handell | 9a478b5 | 2021-11-18 15:07:01 | [diff] [blame] | 974 | RTC_DCHECK_RUN_ON(queue_); |
Markus Handell | 9d04a78 | 2022-05-12 16:38:57 | [diff] [blame] | 975 | if (zero_hertz_adapter_created_timestamp_.has_value()) { |
| 976 | TimeDelta time_until_first_frame = |
henrika | 40ed3ff | 2023-10-03 08:35:52 | [diff] [blame] | 977 | clock_->CurrentTime() - *zero_hertz_adapter_created_timestamp_; |
Markus Handell | 9d04a78 | 2022-05-12 16:38:57 | [diff] [blame] | 978 | zero_hertz_adapter_created_timestamp_ = absl::nullopt; |
| 979 | RTC_HISTOGRAM_COUNTS_10000( |
| 980 | "WebRTC.Screenshare.ZeroHz.TimeUntilFirstFrameMs", |
| 981 | time_until_first_frame.ms()); |
| 982 | } |
| 983 | |
Markus Handell | 9a478b5 | 2021-11-18 15:07:01 | [diff] [blame] | 984 | const int frames_scheduled_for_processing = |
| 985 | frames_scheduled_for_processing_.fetch_sub(1, |
| 986 | std::memory_order_relaxed); |
Markus Handell | 84c016a | 2023-11-23 12:41:44 | [diff] [blame] | 987 | OnFrameOnMainQueue(post_time, frames_scheduled_for_processing > 1, |
Markus Handell | 9a478b5 | 2021-11-18 15:07:01 | [diff] [blame] | 988 | std::move(frame)); |
Markus Handell | 28c7180 | 2021-11-08 09:11:55 | [diff] [blame] | 989 | })); |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 990 | } |
| 991 | |
Markus Handell | f5a5079 | 2022-06-16 12:25:15 | [diff] [blame] | 992 | void FrameCadenceAdapterImpl::OnDiscardedFrame() { |
| 993 | callback_->OnDiscardedFrame(); |
Danil Chapovalov | 95eeaa7 | 2022-07-06 08:14:29 | [diff] [blame] | 994 | queue_->PostTask(SafeTask(safety_.flag(), [this] { |
Markus Handell | f5a5079 | 2022-06-16 12:25:15 | [diff] [blame] | 995 | RTC_DCHECK_RUN_ON(queue_); |
| 996 | if (zero_hertz_adapter_) { |
| 997 | zero_hertz_adapter_->OnDiscardedFrame(); |
| 998 | } |
| 999 | })); |
| 1000 | } |
| 1001 | |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 1002 | void FrameCadenceAdapterImpl::OnConstraintsChanged( |
| 1003 | const VideoTrackSourceConstraints& constraints) { |
Markus Handell | e59fee8 | 2021-12-23 08:29:23 | [diff] [blame] | 1004 | RTC_LOG(LS_INFO) << __func__ << " this " << this << " min_fps " |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 1005 | << constraints.min_fps.value_or(-1) << " max_fps " |
| 1006 | << constraints.max_fps.value_or(-1); |
Danil Chapovalov | 95eeaa7 | 2022-07-06 08:14:29 | [diff] [blame] | 1007 | queue_->PostTask(SafeTask(safety_.flag(), [this, constraints] { |
Markus Handell | 9a478b5 | 2021-11-18 15:07:01 | [diff] [blame] | 1008 | RTC_DCHECK_RUN_ON(queue_); |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 1009 | bool was_zero_hertz_enabled = IsZeroHertzScreenshareEnabled(); |
Markus Handell | 28c7180 | 2021-11-08 09:11:55 | [diff] [blame] | 1010 | source_constraints_ = constraints; |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 1011 | MaybeReconfigureAdapters(was_zero_hertz_enabled); |
Markus Handell | 28c7180 | 2021-11-08 09:11:55 | [diff] [blame] | 1012 | })); |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 1013 | } |
| 1014 | |
Ilya Nikolaevskiy | 7a841ce | 2023-11-24 12:01:21 | [diff] [blame] | 1015 | void FrameCadenceAdapterImpl::OnFrameOnMainQueue(Timestamp post_time, |
| 1016 | bool queue_overload, |
| 1017 | const VideoFrame& frame) { |
Danil Chapovalov | 6e7c268 | 2022-07-25 13:58:28 | [diff] [blame] | 1018 | RTC_DCHECK_RUN_ON(queue_); |
Ilya Nikolaevskiy | 7a841ce | 2023-11-24 12:01:21 | [diff] [blame] | 1019 | current_adapter_mode_->OnFrame(post_time, queue_overload, frame); |
Per K | e975b44 | 2024-03-27 10:28:44 | [diff] [blame] | 1020 | if (last_incoming_frame_timestamp_ && |
| 1021 | last_incoming_frame_timestamp_ >= |
| 1022 | Timestamp::Micros(frame.timestamp_us())) { |
| 1023 | RTC_LOG(LS_ERROR) |
| 1024 | << "Incoming frame timestamp is not monotonically increasing" |
| 1025 | << " current: " << frame.timestamp_us() |
| 1026 | << " last: " << last_incoming_frame_timestamp_.value().us(); |
| 1027 | incoming_frame_timestamp_monotonically_increasing_ = false; |
| 1028 | } |
| 1029 | last_incoming_frame_timestamp_ = Timestamp::Micros(frame.timestamp_us()); |
| 1030 | Timestamp update_frame_rate_timestamp = |
| 1031 | use_video_frame_timestamp_ ? *last_incoming_frame_timestamp_ : post_time; |
| 1032 | UpdateFrameRate(update_frame_rate_timestamp); |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 1033 | } |
| 1034 | |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 1035 | bool FrameCadenceAdapterImpl::IsZeroHertzScreenshareEnabled() const { |
Danil Chapovalov | 6e7c268 | 2022-07-25 13:58:28 | [diff] [blame] | 1036 | RTC_DCHECK_RUN_ON(queue_); |
Markus Handell | a57229b | 2024-04-16 08:40:45 | [diff] [blame] | 1037 | return source_constraints_.has_value() && |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 1038 | source_constraints_->max_fps.value_or(-1) > 0 && |
| 1039 | source_constraints_->min_fps.value_or(-1) == 0 && |
Markus Handell | 8d87c46 | 2021-12-16 10:37:16 | [diff] [blame] | 1040 | zero_hertz_params_.has_value(); |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 1041 | } |
| 1042 | |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 1043 | void FrameCadenceAdapterImpl::ConfigureCurrentAdapterWithoutZeroHertz() { |
| 1044 | // Enable VSyncEncodeAdapterMode if metronome is valid. |
| 1045 | if (metronome_) { |
| 1046 | RTC_CHECK(vsync_encode_adapter_); |
| 1047 | current_adapter_mode_ = vsync_encode_adapter_.get(); |
| 1048 | } else { |
| 1049 | RTC_CHECK(passthrough_adapter_); |
| 1050 | current_adapter_mode_ = &passthrough_adapter_.value(); |
| 1051 | } |
| 1052 | } |
| 1053 | |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 1054 | void FrameCadenceAdapterImpl::MaybeReconfigureAdapters( |
| 1055 | bool was_zero_hertz_enabled) { |
Danil Chapovalov | 6e7c268 | 2022-07-25 13:58:28 | [diff] [blame] | 1056 | RTC_DCHECK_RUN_ON(queue_); |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 1057 | bool is_zero_hertz_enabled = IsZeroHertzScreenshareEnabled(); |
| 1058 | if (is_zero_hertz_enabled) { |
henrika | 644025c | 2023-11-07 17:22:02 | [diff] [blame] | 1059 | bool max_fps_has_changed = GetInputFrameRateFps().value_or(-1) != |
| 1060 | source_constraints_->max_fps.value_or(-1); |
| 1061 | if (!was_zero_hertz_enabled || max_fps_has_changed) { |
henrika | 644025c | 2023-11-07 17:22:02 | [diff] [blame] | 1062 | RTC_LOG(LS_INFO) << "Zero hertz mode enabled (max_fps=" |
| 1063 | << source_constraints_->max_fps.value() << ")"; |
henrika | b7ec057 | 2024-01-09 09:48:52 | [diff] [blame] | 1064 | zero_hertz_adapter_.emplace( |
| 1065 | queue_, clock_, callback_, source_constraints_->max_fps.value(), |
| 1066 | frames_scheduled_for_processing_, |
| 1067 | frame_cadence_adapter_zero_hertz_queue_overload_enabled_); |
henrika | 8f16ce9 | 2023-12-04 13:03:52 | [diff] [blame] | 1068 | zero_hertz_adapter_->UpdateVideoSourceRestrictions( |
| 1069 | restricted_max_frame_rate_); |
Markus Handell | 9d04a78 | 2022-05-12 16:38:57 | [diff] [blame] | 1070 | zero_hertz_adapter_created_timestamp_ = clock_->CurrentTime(); |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 1071 | } |
Markus Handell | e59fee8 | 2021-12-23 08:29:23 | [diff] [blame] | 1072 | zero_hertz_adapter_->ReconfigureParameters(zero_hertz_params_.value()); |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 1073 | current_adapter_mode_ = &zero_hertz_adapter_.value(); |
| 1074 | } else { |
henrika | 86f09ae | 2023-10-31 13:49:40 | [diff] [blame] | 1075 | if (was_zero_hertz_enabled) { |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 1076 | zero_hertz_adapter_ = absl::nullopt; |
henrika | 644025c | 2023-11-07 17:22:02 | [diff] [blame] | 1077 | RTC_LOG(LS_INFO) << "Zero hertz mode disabled."; |
henrika | 86f09ae | 2023-10-31 13:49:40 | [diff] [blame] | 1078 | } |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 1079 | ConfigureCurrentAdapterWithoutZeroHertz(); |
Markus Handell | ee22543 | 2021-11-29 11:35:12 | [diff] [blame] | 1080 | } |
Markus Handell | 28c7180 | 2021-11-08 09:11:55 | [diff] [blame] | 1081 | } |
| 1082 | |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 1083 | } // namespace |
| 1084 | |
| 1085 | std::unique_ptr<FrameCadenceAdapterInterface> |
Jonas Oreland | 8ca0613 | 2022-03-14 11:52:48 | [diff] [blame] | 1086 | FrameCadenceAdapterInterface::Create(Clock* clock, |
| 1087 | TaskQueueBase* queue, |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 1088 | Metronome* metronome, |
| 1089 | TaskQueueBase* worker_queue, |
Jonas Oreland | e62c2f2 | 2022-03-29 09:04:48 | [diff] [blame] | 1090 | const FieldTrialsView& field_trials) { |
Zhaoliang Ma | f089d7e | 2024-01-08 02:44:15 | [diff] [blame] | 1091 | return std::make_unique<FrameCadenceAdapterImpl>(clock, queue, metronome, |
| 1092 | worker_queue, field_trials); |
Markus Handell | b4e96d4 | 2021-11-05 11:00:55 | [diff] [blame] | 1093 | } |
| 1094 | |
| 1095 | } // namespace webrtc |