blob: 59c07ddaab763cd483f42bd95d4282cc066cb8d3 [file] [log] [blame]
Evan Shrubsole476f18d22022-08-15 15:21:161/*
2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "video/video_stream_buffer_controller.h"
12
13#include <algorithm>
14#include <memory>
15#include <utility>
16
17#include "absl/base/attributes.h"
18#include "absl/functional/bind_front.h"
Rasmus Brandt65a6eca2023-03-03 08:22:1819#include "absl/types/optional.h"
Evan Shrubsole476f18d22022-08-15 15:21:1620#include "api/sequence_checker.h"
21#include "api/task_queue/task_queue_base.h"
22#include "api/units/data_size.h"
Rasmus Brandt65a6eca2023-03-03 08:22:1823#include "api/units/time_delta.h"
Rasmus Brandt621cb292023-05-24 11:29:1424#include "api/units/timestamp.h"
Evan Shrubsole476f18d22022-08-15 15:21:1625#include "api/video/encoded_frame.h"
26#include "api/video/frame_buffer.h"
27#include "api/video/video_content_type.h"
28#include "modules/video_coding/frame_helpers.h"
Rasmus Brandt65a6eca2023-03-03 08:22:1829#include "modules/video_coding/timing/inter_frame_delay_variation_calculator.h"
Evan Shrubsole476f18d22022-08-15 15:21:1630#include "modules/video_coding/timing/jitter_estimator.h"
31#include "rtc_base/checks.h"
32#include "rtc_base/logging.h"
33#include "rtc_base/thread_annotations.h"
Evan Shrubsolea006ba12022-09-05 10:09:0834#include "video/frame_decode_scheduler.h"
Evan Shrubsole476f18d22022-08-15 15:21:1635#include "video/frame_decode_timing.h"
36#include "video/task_queue_frame_decode_scheduler.h"
37#include "video/video_receive_stream_timeout_tracker.h"
38
39namespace webrtc {
40
41namespace {
42
43// Max number of frames the buffer will hold.
44static constexpr size_t kMaxFramesBuffered = 800;
45// Max number of decoded frame info that will be saved.
46static constexpr int kMaxFramesHistory = 1 << 13;
47
48// Default value for the maximum decode queue size that is used when the
49// low-latency renderer is used.
50static constexpr size_t kZeroPlayoutDelayDefaultMaxDecodeQueueSize = 8;
51
52struct FrameMetadata {
53 explicit FrameMetadata(const EncodedFrame& frame)
54 : is_last_spatial_layer(frame.is_last_spatial_layer),
55 is_keyframe(frame.is_keyframe()),
56 size(frame.size()),
57 contentType(frame.contentType()),
58 delayed_by_retransmission(frame.delayed_by_retransmission()),
Danil Chapovalov9c584832023-09-18 13:48:4959 rtp_timestamp(frame.RtpTimestamp()),
Evan Shrubsole476f18d22022-08-15 15:21:1660 receive_time(frame.ReceivedTimestamp()) {}
61
62 const bool is_last_spatial_layer;
63 const bool is_keyframe;
64 const size_t size;
65 const VideoContentType contentType;
66 const bool delayed_by_retransmission;
67 const uint32_t rtp_timestamp;
68 const absl::optional<Timestamp> receive_time;
69};
70
Rasmus Brandt621cb292023-05-24 11:29:1471Timestamp MinReceiveTime(const EncodedFrame& frame) {
72 Timestamp first_recv_time = Timestamp::PlusInfinity();
73 for (const auto& packet_info : frame.PacketInfos()) {
74 if (packet_info.receive_time().IsFinite()) {
75 first_recv_time = std::min(first_recv_time, packet_info.receive_time());
76 }
77 }
78 return first_recv_time;
79}
80
Evan Shrubsole476f18d22022-08-15 15:21:1681Timestamp ReceiveTime(const EncodedFrame& frame) {
82 absl::optional<Timestamp> ts = frame.ReceivedTimestamp();
83 RTC_DCHECK(ts.has_value()) << "Received frame must have a timestamp set!";
84 return *ts;
85}
86
Evan Shrubsole476f18d22022-08-15 15:21:1687} // namespace
88
Evan Shrubsole476f18d22022-08-15 15:21:1689VideoStreamBufferController::VideoStreamBufferController(
90 Clock* clock,
91 TaskQueueBase* worker_queue,
92 VCMTiming* timing,
Rasmus Brandt39250a42023-05-09 12:33:4393 VideoStreamBufferControllerStatsObserver* stats_proxy,
Evan Shrubsole476f18d22022-08-15 15:21:1694 FrameSchedulingReceiver* receiver,
95 TimeDelta max_wait_for_keyframe,
96 TimeDelta max_wait_for_frame,
97 std::unique_ptr<FrameDecodeScheduler> frame_decode_scheduler,
98 const FieldTrialsView& field_trials)
99 : field_trials_(field_trials),
100 clock_(clock),
Evan Shrubsole476f18d22022-08-15 15:21:16101 stats_proxy_(stats_proxy),
102 receiver_(receiver),
103 timing_(timing),
104 frame_decode_scheduler_(std::move(frame_decode_scheduler)),
105 jitter_estimator_(clock_, field_trials),
106 buffer_(std::make_unique<FrameBuffer>(kMaxFramesBuffered,
107 kMaxFramesHistory,
108 field_trials)),
109 decode_timing_(clock_, timing_),
110 timeout_tracker_(
111 clock_,
Evan Shrubsole214cab52022-08-16 09:48:23112 worker_queue,
Evan Shrubsole476f18d22022-08-15 15:21:16113 VideoReceiveStreamTimeoutTracker::Timeouts{
114 .max_wait_for_keyframe = max_wait_for_keyframe,
115 .max_wait_for_frame = max_wait_for_frame},
116 absl::bind_front(&VideoStreamBufferController::OnTimeout, this)),
117 zero_playout_delay_max_decode_queue_size_(
118 "max_decode_queue_size",
119 kZeroPlayoutDelayDefaultMaxDecodeQueueSize) {
Evan Shrubsole476f18d22022-08-15 15:21:16120 RTC_DCHECK(stats_proxy_);
121 RTC_DCHECK(receiver_);
122 RTC_DCHECK(timing_);
Evan Shrubsole476f18d22022-08-15 15:21:16123 RTC_DCHECK(clock_);
124 RTC_DCHECK(frame_decode_scheduler_);
Evan Shrubsole476f18d22022-08-15 15:21:16125
126 ParseFieldTrial({&zero_playout_delay_max_decode_queue_size_},
127 field_trials.Lookup("WebRTC-ZeroPlayoutDelay"));
128}
129
Evan Shrubsole214cab52022-08-16 09:48:23130void VideoStreamBufferController::Stop() {
Evan Shrubsole476f18d22022-08-15 15:21:16131 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
132 frame_decode_scheduler_->Stop();
133 timeout_tracker_.Stop();
134 decoder_ready_for_new_frame_ = false;
Evan Shrubsole476f18d22022-08-15 15:21:16135}
136
137void VideoStreamBufferController::SetProtectionMode(
138 VCMVideoProtection protection_mode) {
139 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
140 protection_mode_ = protection_mode;
141}
142
143void VideoStreamBufferController::Clear() {
144 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
145 stats_proxy_->OnDroppedFrames(buffer_->CurrentSize());
146 buffer_ = std::make_unique<FrameBuffer>(kMaxFramesBuffered, kMaxFramesHistory,
147 field_trials_);
148 frame_decode_scheduler_->CancelOutstanding();
149}
150
151absl::optional<int64_t> VideoStreamBufferController::InsertFrame(
152 std::unique_ptr<EncodedFrame> frame) {
153 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
154 FrameMetadata metadata(*frame);
155 int complete_units = buffer_->GetTotalNumberOfContinuousTemporalUnits();
156 if (buffer_->InsertFrame(std::move(frame))) {
157 RTC_DCHECK(metadata.receive_time) << "Frame receive time must be set!";
philipel7446b602022-10-06 13:49:17158 if (!metadata.delayed_by_retransmission && metadata.receive_time &&
159 (field_trials_.IsDisabled("WebRTC-IncomingTimestampOnMarkerBitOnly") ||
160 metadata.is_last_spatial_layer)) {
Evan Shrubsole476f18d22022-08-15 15:21:16161 timing_->IncomingTimestamp(metadata.rtp_timestamp,
162 *metadata.receive_time);
philipel7446b602022-10-06 13:49:17163 }
Evan Shrubsole476f18d22022-08-15 15:21:16164 if (complete_units < buffer_->GetTotalNumberOfContinuousTemporalUnits()) {
165 stats_proxy_->OnCompleteFrame(metadata.is_keyframe, metadata.size,
166 metadata.contentType);
167 MaybeScheduleFrameForRelease();
168 }
169 }
170
171 return buffer_->LastContinuousFrameId();
172}
173
174void VideoStreamBufferController::UpdateRtt(int64_t max_rtt_ms) {
175 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
176 jitter_estimator_.UpdateRtt(TimeDelta::Millis(max_rtt_ms));
177}
178
179void VideoStreamBufferController::SetMaxWaits(TimeDelta max_wait_for_keyframe,
180 TimeDelta max_wait_for_frame) {
181 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
182 timeout_tracker_.SetTimeouts({.max_wait_for_keyframe = max_wait_for_keyframe,
183 .max_wait_for_frame = max_wait_for_frame});
184}
185
186void VideoStreamBufferController::StartNextDecode(bool keyframe_required) {
Evan Shrubsole476f18d22022-08-15 15:21:16187 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
188 if (!timeout_tracker_.Running())
189 timeout_tracker_.Start(keyframe_required);
190 keyframe_required_ = keyframe_required;
191 if (keyframe_required_) {
192 timeout_tracker_.SetWaitingForKeyframe();
193 }
194 decoder_ready_for_new_frame_ = true;
195 MaybeScheduleFrameForRelease();
196}
197
198int VideoStreamBufferController::Size() {
199 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
200 return buffer_->CurrentSize();
201}
202
203void VideoStreamBufferController::OnFrameReady(
204 absl::InlinedVector<std::unique_ptr<EncodedFrame>, 4> frames,
205 Timestamp render_time) {
206 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
Evan Shrubsole8fe55792022-10-27 11:05:44207 RTC_CHECK(!frames.empty())
208 << "Callers must ensure there is at least one frame to decode.";
Evan Shrubsole476f18d22022-08-15 15:21:16209
210 timeout_tracker_.OnEncodedFrameReleased();
211
212 Timestamp now = clock_->CurrentTime();
213 bool superframe_delayed_by_retransmission = false;
214 DataSize superframe_size = DataSize::Zero();
215 const EncodedFrame& first_frame = *frames.front();
Rasmus Brandt621cb292023-05-24 11:29:14216 Timestamp min_receive_time = MinReceiveTime(first_frame);
217 Timestamp max_receive_time = ReceiveTime(first_frame);
Evan Shrubsole476f18d22022-08-15 15:21:16218
219 if (first_frame.is_keyframe())
220 keyframe_required_ = false;
221
222 // Gracefully handle bad RTP timestamps and render time issues.
Rasmus Brandtfb3bd4a2022-10-13 11:43:27223 if (FrameHasBadRenderTiming(render_time, now) ||
224 TargetVideoDelayIsTooLarge(timing_->TargetVideoDelay())) {
225 RTC_LOG(LS_WARNING) << "Resetting jitter estimator and timing module due "
226 "to bad render timing for rtp_timestamp="
Danil Chapovalov9c584832023-09-18 13:48:49227 << first_frame.RtpTimestamp();
Evan Shrubsole476f18d22022-08-15 15:21:16228 jitter_estimator_.Reset();
229 timing_->Reset();
Danil Chapovalov9c584832023-09-18 13:48:49230 render_time = timing_->RenderTime(first_frame.RtpTimestamp(), now);
Evan Shrubsole476f18d22022-08-15 15:21:16231 }
232
233 for (std::unique_ptr<EncodedFrame>& frame : frames) {
234 frame->SetRenderTime(render_time.ms());
235
236 superframe_delayed_by_retransmission |= frame->delayed_by_retransmission();
Rasmus Brandt621cb292023-05-24 11:29:14237 min_receive_time = std::min(min_receive_time, MinReceiveTime(*frame));
238 max_receive_time = std::max(max_receive_time, ReceiveTime(*frame));
Evan Shrubsole476f18d22022-08-15 15:21:16239 superframe_size += DataSize::Bytes(frame->size());
240 }
241
242 if (!superframe_delayed_by_retransmission) {
Rasmus Brandt65a6eca2023-03-03 08:22:18243 absl::optional<TimeDelta> inter_frame_delay_variation =
Danil Chapovalov9c584832023-09-18 13:48:49244 ifdv_calculator_.Calculate(first_frame.RtpTimestamp(),
245 max_receive_time);
Rasmus Brandt65a6eca2023-03-03 08:22:18246 if (inter_frame_delay_variation) {
247 jitter_estimator_.UpdateEstimate(*inter_frame_delay_variation,
248 superframe_size);
Evan Shrubsole476f18d22022-08-15 15:21:16249 }
250
251 float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0;
252 absl::optional<TimeDelta> rtt_mult_add_cap_ms = absl::nullopt;
253 if (rtt_mult_settings_.has_value()) {
254 rtt_mult = rtt_mult_settings_->rtt_mult_setting;
255 rtt_mult_add_cap_ms =
256 TimeDelta::Millis(rtt_mult_settings_->rtt_mult_add_cap_ms);
257 }
258 timing_->SetJitterDelay(
259 jitter_estimator_.GetJitterEstimate(rtt_mult, rtt_mult_add_cap_ms));
260 timing_->UpdateCurrentDelay(render_time, now);
261 } else if (RttMultExperiment::RttMultEnabled()) {
262 jitter_estimator_.FrameNacked();
263 }
264
265 // Update stats.
266 UpdateDroppedFrames();
Rasmus Brandt621cb292023-05-24 11:29:14267 UpdateFrameBufferTimings(min_receive_time, now);
Evan Shrubsole476f18d22022-08-15 15:21:16268 UpdateTimingFrameInfo();
269
270 std::unique_ptr<EncodedFrame> frame =
271 CombineAndDeleteFrames(std::move(frames));
272
273 timing_->SetLastDecodeScheduledTimestamp(now);
274
275 decoder_ready_for_new_frame_ = false;
Evan Shrubsole214cab52022-08-16 09:48:23276 receiver_->OnEncodedFrame(std::move(frame));
Evan Shrubsole476f18d22022-08-15 15:21:16277}
278
279void VideoStreamBufferController::OnTimeout(TimeDelta delay) {
280 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
Evan Shrubsole214cab52022-08-16 09:48:23281
282 // Stop sending timeouts until receiver starts waiting for a new frame.
283 timeout_tracker_.Stop();
284
Evan Shrubsole476f18d22022-08-15 15:21:16285 // If the stream is paused then ignore the timeout.
286 if (!decoder_ready_for_new_frame_) {
Evan Shrubsole476f18d22022-08-15 15:21:16287 return;
288 }
Evan Shrubsole476f18d22022-08-15 15:21:16289 decoder_ready_for_new_frame_ = false;
Evan Shrubsole214cab52022-08-16 09:48:23290 receiver_->OnDecodableFrameTimeout(delay);
Evan Shrubsole476f18d22022-08-15 15:21:16291}
292
293void VideoStreamBufferController::FrameReadyForDecode(uint32_t rtp_timestamp,
294 Timestamp render_time) {
295 RTC_DCHECK_RUN_ON(&worker_sequence_checker_);
Evan Shrubsole8fe55792022-10-27 11:05:44296 // Check that the frame to decode is still valid before passing the frame for
297 // decoding.
298 auto decodable_tu_info = buffer_->DecodableTemporalUnitsInfo();
299 if (!decodable_tu_info) {
300 RTC_LOG(LS_ERROR)
301 << "The frame buffer became undecodable during the wait "
302 "to decode frame with rtp-timestamp "
303 << rtp_timestamp
304 << ". Cancelling the decode of this frame, decoding "
305 "will resume when the frame buffers become decodable again.";
306 return;
307 }
308 RTC_DCHECK_EQ(rtp_timestamp, decodable_tu_info->next_rtp_timestamp)
Evan Shrubsole476f18d22022-08-15 15:21:16309 << "Frame buffer's next decodable frame was not the one sent for "
Evan Shrubsole8fe55792022-10-27 11:05:44310 "extraction.";
311 auto frames = buffer_->ExtractNextDecodableTemporalUnit();
312 if (frames.empty()) {
313 RTC_LOG(LS_ERROR)
314 << "The frame buffer should never return an empty temporal until list "
315 "when there is a decodable temporal unit.";
316 RTC_DCHECK_NOTREACHED();
317 return;
318 }
Evan Shrubsole476f18d22022-08-15 15:21:16319 OnFrameReady(std::move(frames), render_time);
320}
321
322void VideoStreamBufferController::UpdateDroppedFrames()
323 RTC_RUN_ON(&worker_sequence_checker_) {
324 const int dropped_frames = buffer_->GetTotalNumberOfDroppedFrames() -
325 frames_dropped_before_last_new_frame_;
326 if (dropped_frames > 0)
327 stats_proxy_->OnDroppedFrames(dropped_frames);
328 frames_dropped_before_last_new_frame_ =
329 buffer_->GetTotalNumberOfDroppedFrames();
330}
331
Rasmus Brandt621cb292023-05-24 11:29:14332void VideoStreamBufferController::UpdateFrameBufferTimings(
333 Timestamp min_receive_time,
334 Timestamp now) {
335 // Update instantaneous delays.
Evan Shrubsole476f18d22022-08-15 15:21:16336 auto timings = timing_->GetTimings();
337 if (timings.num_decoded_frames) {
338 stats_proxy_->OnFrameBufferTimingsUpdated(
Rasmus Brandt24f9a8b2023-05-08 14:32:48339 timings.estimated_max_decode_time.ms(), timings.current_delay.ms(),
Rasmus Brandtf0820ff2023-05-25 07:37:16340 timings.target_delay.ms(), timings.minimum_delay.ms(),
Evan Shrubsole476f18d22022-08-15 15:21:16341 timings.min_playout_delay.ms(), timings.render_delay.ms());
342 }
Rasmus Brandt621cb292023-05-24 11:29:14343
344 // The spec mandates that `jitterBufferDelay` is the "time the first
345 // packet is received by the jitter buffer (ingest timestamp) to the time it
346 // exits the jitter buffer (emit timestamp)". Since the "jitter buffer"
347 // is not a monolith in the webrtc.org implementation, we take the freedom to
348 // define "ingest timestamp" as "first packet received by
349 // RtpVideoStreamReceiver2" and "emit timestamp" as "decodable frame released
350 // by VideoStreamBufferController".
351 //
352 // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-jitterbufferdelay
353 TimeDelta jitter_buffer_delay =
354 std::max(TimeDelta::Zero(), now - min_receive_time);
Rasmus Brandtf0820ff2023-05-25 07:37:16355 stats_proxy_->OnDecodableFrame(jitter_buffer_delay, timings.target_delay,
356 timings.minimum_delay);
Evan Shrubsole476f18d22022-08-15 15:21:16357}
358
359void VideoStreamBufferController::UpdateTimingFrameInfo() {
360 absl::optional<TimingFrameInfo> info = timing_->GetTimingFrameInfo();
361 if (info)
362 stats_proxy_->OnTimingFrameInfoUpdated(*info);
363}
364
365bool VideoStreamBufferController::IsTooManyFramesQueued() const
366 RTC_RUN_ON(&worker_sequence_checker_) {
367 return buffer_->CurrentSize() > zero_playout_delay_max_decode_queue_size_;
368}
369
370void VideoStreamBufferController::ForceKeyFrameReleaseImmediately()
371 RTC_RUN_ON(&worker_sequence_checker_) {
372 RTC_DCHECK(keyframe_required_);
373 // Iterate through the frame buffer until there is a complete keyframe and
374 // release this right away.
375 while (buffer_->DecodableTemporalUnitsInfo()) {
376 auto next_frame = buffer_->ExtractNextDecodableTemporalUnit();
377 if (next_frame.empty()) {
378 RTC_DCHECK_NOTREACHED()
379 << "Frame buffer should always return at least 1 frame.";
380 continue;
381 }
382 // Found keyframe - decode right away.
383 if (next_frame.front()->is_keyframe()) {
Danil Chapovalov9c584832023-09-18 13:48:49384 auto render_time = timing_->RenderTime(next_frame.front()->RtpTimestamp(),
Evan Shrubsole476f18d22022-08-15 15:21:16385 clock_->CurrentTime());
386 OnFrameReady(std::move(next_frame), render_time);
387 return;
388 }
389 }
390}
391
392void VideoStreamBufferController::MaybeScheduleFrameForRelease()
393 RTC_RUN_ON(&worker_sequence_checker_) {
394 auto decodable_tu_info = buffer_->DecodableTemporalUnitsInfo();
395 if (!decoder_ready_for_new_frame_ || !decodable_tu_info) {
396 return;
397 }
398
399 if (keyframe_required_) {
400 return ForceKeyFrameReleaseImmediately();
401 }
402
403 // If already scheduled then abort.
404 if (frame_decode_scheduler_->ScheduledRtpTimestamp() ==
405 decodable_tu_info->next_rtp_timestamp) {
406 return;
407 }
408
409 TimeDelta max_wait = timeout_tracker_.TimeUntilTimeout();
410 // Ensures the frame is scheduled for decode before the stream times out.
411 // This is otherwise a race condition.
412 max_wait = std::max(max_wait - TimeDelta::Millis(1), TimeDelta::Zero());
413 absl::optional<FrameDecodeTiming::FrameSchedule> schedule;
414 while (decodable_tu_info) {
415 schedule = decode_timing_.OnFrameBufferUpdated(
416 decodable_tu_info->next_rtp_timestamp,
417 decodable_tu_info->last_rtp_timestamp, max_wait,
418 IsTooManyFramesQueued());
419 if (schedule) {
420 // Don't schedule if already waiting for the same frame.
421 if (frame_decode_scheduler_->ScheduledRtpTimestamp() !=
422 decodable_tu_info->next_rtp_timestamp) {
423 frame_decode_scheduler_->CancelOutstanding();
424 frame_decode_scheduler_->ScheduleFrame(
425 decodable_tu_info->next_rtp_timestamp, *schedule,
426 absl::bind_front(&VideoStreamBufferController::FrameReadyForDecode,
427 this));
428 }
429 return;
430 }
431 // If no schedule for current rtp, drop and try again.
432 buffer_->DropNextDecodableTemporalUnit();
433 decodable_tu_info = buffer_->DecodableTemporalUnitsInfo();
434 }
435}
436
437} // namespace webrtc