blob: 602cf9d82a8e14817d10a1d967a4095c0571f867 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:251/*
pwestin@webrtc.orgf6bb77a2012-01-24 17:16:592 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:253 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 04:47:3111#include "modules/rtp_rtcp/source/rtp_sender_video.h"
niklase@google.com470e71d2011-07-07 08:21:2512
pbos@webrtc.orga048d7c2013-05-29 14:27:3813#include <stdlib.h>
14#include <string.h>
niklase@google.com470e71d2011-07-07 08:21:2515
Erik Språng56e611b2020-02-06 16:10:0816#include <algorithm>
spranga8ae6f22017-09-04 14:23:5617#include <limits>
kwiberg84be5112016-04-27 08:19:5818#include <memory>
Elad Alon8b60e8b2019-04-08 12:14:0519#include <string>
danilchap74110612016-10-02 17:54:2920#include <utility>
mflodmanfcf54bd2015-04-14 19:28:0821
Danil Chapovalov4c3a7db2020-04-20 07:53:0322#include "absl/algorithm/container.h"
Danil Chapovalov670af262020-01-30 15:28:5323#include "absl/memory/memory.h"
Niels Mölleraa3c1cc2018-11-02 09:54:5624#include "absl/strings/match.h"
Steve Anton10542f22019-01-11 17:11:0025#include "api/crypto/frame_encryptor_interface.h"
Danil Chapovalov670af262020-01-30 15:28:5326#include "api/transport/rtp/dependency_descriptor.h"
Niels Möller59ab1cf2019-02-06 21:48:1127#include "modules/remote_bitrate_estimator/test/bwe_test_logging.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3128#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
Minyue Li5bb9adc2020-01-22 12:02:3429#include "modules/rtp_rtcp/source/absolute_capture_time_sender.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3130#include "modules/rtp_rtcp/source/byte_io.h"
Danil Chapovalov670af262020-01-30 15:28:5331#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
Danil Chapovalov95800f62020-02-20 14:01:1532#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
Mirta Dvornicicfe68daa2019-05-23 11:21:1233#include "modules/rtp_rtcp/source/rtp_format.h"
philipelb3e42a42018-09-13 08:57:1434#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3135#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
36#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
Per Kjellander4f350ba2020-10-14 06:41:5237#include "modules/rtp_rtcp/source/rtp_video_layers_allocation_extension.h"
Minyue Li5bb9adc2020-01-22 12:02:3438#include "modules/rtp_rtcp/source/time_util.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3139#include "rtc_base/checks.h"
Johannes Kron4da8e4c2020-09-22 14:23:3040#include "rtc_base/experiments/field_trial_parser.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3141#include "rtc_base/logging.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3142#include "rtc_base/trace_event.h"
ilniked9b9ff2017-06-02 14:30:2043
niklase@google.com470e71d2011-07-07 08:21:2544namespace webrtc {
Sergey Ulanovec4f0682016-07-28 22:19:1045
brandtr6631e8a2016-09-13 10:23:2946namespace {
47constexpr size_t kRedForFecHeaderLength = 1;
spranga8ae6f22017-09-04 14:23:5648constexpr int64_t kMaxUnretransmittableFrameIntervalMs = 33 * 4;
Minyue Lie64b3d02020-08-20 14:18:3149constexpr char kIncludeCaptureClockOffset[] =
50 "WebRTC-IncludeCaptureClockOffset";
danilchap74110612016-10-02 17:54:2951
52void BuildRedPayload(const RtpPacketToSend& media_packet,
53 RtpPacketToSend* red_packet) {
54 uint8_t* red_payload = red_packet->AllocatePayload(
55 kRedForFecHeaderLength + media_packet.payload_size());
56 RTC_DCHECK(red_payload);
57 red_payload[0] = media_packet.PayloadType();
danilchap96c15872016-11-21 09:35:2958
59 auto media_payload = media_packet.payload();
60 memcpy(&red_payload[kRedForFecHeaderLength], media_payload.data(),
61 media_payload.size());
danilchap74110612016-10-02 17:54:2962}
Danil Chapovalove4f8b382018-09-07 15:30:2663
Danil Chapovalov51bf2002019-10-11 08:53:2764bool MinimizeDescriptor(RTPVideoHeader* video_header) {
65 if (auto* vp8 =
66 absl::get_if<RTPVideoHeaderVP8>(&video_header->video_type_header)) {
Danil Chapovalov84ffb352018-09-25 16:59:0967 // Set minimum fields the RtpPacketizer is using to create vp8 packets.
Danil Chapovalov51bf2002019-10-11 08:53:2768 // nonReference is the only field that doesn't require extra space.
69 bool non_reference = vp8->nonReference;
70 vp8->InitRTPVideoHeaderVP8();
71 vp8->nonReference = non_reference;
Danil Chapovalov84ffb352018-09-25 16:59:0972 return true;
73 }
74 // TODO(danilchap): Reduce vp9 codec specific descriptor too.
75 return false;
76}
77
Johannes Krond0b69a82018-12-03 13:18:5378bool IsBaseLayer(const RTPVideoHeader& video_header) {
79 switch (video_header.codec) {
80 case kVideoCodecVP8: {
81 const auto& vp8 =
82 absl::get<RTPVideoHeaderVP8>(video_header.video_type_header);
83 return (vp8.temporalIdx == 0 || vp8.temporalIdx == kNoTemporalIdx);
84 }
85 case kVideoCodecVP9: {
86 const auto& vp9 =
87 absl::get<RTPVideoHeaderVP9>(video_header.video_type_header);
88 return (vp9.temporal_idx == 0 || vp9.temporal_idx == kNoTemporalIdx);
89 }
90 case kVideoCodecH264:
91 // TODO(kron): Implement logic for H264 once WebRTC supports temporal
92 // layers for H264.
93 break;
94 default:
95 break;
96 }
97 return true;
98}
99
Doudou Kisabaka2dec4962019-11-28 13:24:31100#if RTC_TRACE_EVENTS_ENABLED
Niels Möller87e2d782019-03-07 09:18:23101const char* FrameTypeToString(VideoFrameType frame_type) {
Niels Möller59ab1cf2019-02-06 21:48:11102 switch (frame_type) {
Niels Möller8f7ce222019-03-21 14:43:58103 case VideoFrameType::kEmptyFrame:
Niels Möller59ab1cf2019-02-06 21:48:11104 return "empty";
Niels Möller8f7ce222019-03-21 14:43:58105 case VideoFrameType::kVideoFrameKey:
Niels Möller59ab1cf2019-02-06 21:48:11106 return "video_key";
Niels Möller8f7ce222019-03-21 14:43:58107 case VideoFrameType::kVideoFrameDelta:
Niels Möller59ab1cf2019-02-06 21:48:11108 return "video_delta";
109 default:
110 RTC_NOTREACHED();
111 return "";
112 }
113}
Doudou Kisabaka2dec4962019-11-28 13:24:31114#endif
Niels Möller59ab1cf2019-02-06 21:48:11115
Niels Möllerd381eed2020-09-02 13:34:40116bool IsNoopDelay(const VideoPlayoutDelay& delay) {
Erik Språng56e611b2020-02-06 16:10:08117 return delay.min_ms == -1 && delay.max_ms == -1;
118}
119
Johannes Kron4da8e4c2020-09-22 14:23:30120absl::optional<VideoPlayoutDelay> LoadVideoPlayoutDelayOverride(
121 const WebRtcKeyValueConfig* key_value_config) {
122 RTC_DCHECK(key_value_config);
123 FieldTrialOptional<int> playout_delay_min_ms("min_ms", absl::nullopt);
124 FieldTrialOptional<int> playout_delay_max_ms("max_ms", absl::nullopt);
125 ParseFieldTrial({&playout_delay_max_ms, &playout_delay_min_ms},
126 key_value_config->Lookup("WebRTC-ForceSendPlayoutDelay"));
127 return playout_delay_max_ms && playout_delay_min_ms
128 ? absl::make_optional<VideoPlayoutDelay>(*playout_delay_min_ms,
129 *playout_delay_max_ms)
130 : absl::nullopt;
131}
132
Per Kjellander4f350ba2020-10-14 06:41:52133// Some packets can be skipped and the stream can still be decoded. Those
134// packets are less likely to be retransmitted if they are lost.
135bool PacketWillLikelyBeRequestedForRestransmitionIfLost(
136 const RTPVideoHeader& video_header) {
137 return IsBaseLayer(video_header) &&
138 !(video_header.generic.has_value()
139 ? absl::c_linear_search(
140 video_header.generic->decode_target_indications,
141 DecodeTargetIndication::kDiscardable)
142 : false);
143}
144
brandtr6631e8a2016-09-13 10:23:29145} // namespace
niklase@google.com470e71d2011-07-07 08:21:25146
Erik Språngdc34a252019-10-04 13:17:29147RTPSenderVideo::RTPSenderVideo(const Config& config)
148 : rtp_sender_(config.rtp_sender),
149 clock_(config.clock),
Andrei Dumitru09872732019-08-29 19:00:29150 retransmission_settings_(
Erik Språngdc34a252019-10-04 13:17:29151 config.enable_retransmit_all_layers
Andrei Dumitru09872732019-08-29 19:00:29152 ? kRetransmitAllLayers
153 : (kRetransmitBaseLayer | kConditionallyRetransmitHigherLayers)),
brandtrd8048952016-11-07 10:08:51154 last_rotation_(kVideoRotation_0),
Johannes Krond0b69a82018-12-03 13:18:53155 transmit_color_space_next_frame_(false),
Per Kjellanderdbf95492020-12-22 15:12:15156 send_allocation_(SendVideoLayersAllocation::kDontSend),
Erik Språng56e611b2020-02-06 16:10:08157 current_playout_delay_{-1, -1},
158 playout_delay_pending_(false),
Johannes Kron4da8e4c2020-09-22 14:23:30159 forced_playout_delay_(LoadVideoPlayoutDelayOverride(config.field_trials)),
Erik Språngdc34a252019-10-04 13:17:29160 red_payload_type_(config.red_payload_type),
Erik Språng42108882020-05-05 12:51:09161 fec_type_(config.fec_type),
162 fec_overhead_bytes_(config.fec_overhead_bytes),
Erik Språng482b3ef2019-01-08 15:19:11163 packetization_overhead_bitrate_(1000, RateStatistics::kBpsScale),
Erik Språngdc34a252019-10-04 13:17:29164 frame_encryptor_(config.frame_encryptor),
165 require_frame_encryption_(config.require_frame_encryption),
Mirko Bonadei06d35592020-04-01 11:43:08166 generic_descriptor_auth_experiment_(!absl::StartsWith(
167 config.field_trials->Lookup("WebRTC-GenericDescriptorAuth"),
168 "Disabled")),
Marina Cioceae77912b2020-02-27 15:16:55169 absolute_capture_time_sender_(config.clock),
Marina Ciocea3a087a82020-03-03 04:39:28170 frame_transformer_delegate_(
171 config.frame_transformer
172 ? new rtc::RefCountedObject<
173 RTPSenderVideoFrameTransformerDelegate>(
Marina Cioceadc69fd22020-04-10 18:19:14174 this,
175 config.frame_transformer,
Marina Ciocea2e696602020-05-29 10:37:01176 rtp_sender_->SSRC(),
Marina Cioceaf1c5b952020-06-01 10:03:09177 config.send_transport_queue)
Minyue Lie64b3d02020-08-20 14:18:31178 : nullptr),
179 include_capture_clock_offset_(absl::StartsWith(
180 config.field_trials->Lookup(kIncludeCaptureClockOffset),
181 "Enabled")) {
Marina Ciocea3a087a82020-03-03 04:39:28182 if (frame_transformer_delegate_)
183 frame_transformer_delegate_->Init();
184}
niklase@google.com470e71d2011-07-07 08:21:25185
Marina Ciocea3a087a82020-03-03 04:39:28186RTPSenderVideo::~RTPSenderVideo() {
187 if (frame_transformer_delegate_)
188 frame_transformer_delegate_->Reset();
189}
niklase@google.com470e71d2011-07-07 08:21:25190
Erik Språngf1e97b92019-09-26 10:48:47191void RTPSenderVideo::LogAndSendToNetwork(
192 std::vector<std::unique_ptr<RtpPacketToSend>> packets,
193 size_t unpacketized_payload_size) {
Erik Språngf1e97b92019-09-26 10:48:47194 {
Markus Handelle7c015e2020-07-07 09:44:28195 MutexLock lock(&stats_mutex_);
Erik Språngf1e97b92019-09-26 10:48:47196 size_t packetized_payload_size = 0;
197 for (const auto& packet : packets) {
Erik Språngf87536c2020-03-05 09:14:04198 if (*packet->packet_type() == RtpPacketMediaType::kVideo) {
Erik Språngf87536c2020-03-05 09:14:04199 packetized_payload_size += packet->payload_size();
Erik Språngf1e97b92019-09-26 10:48:47200 }
201 }
Danil Chapovalov31cb3ab2020-07-21 13:01:50202 // AV1 and H264 packetizers may produce less packetized bytes than
203 // unpacketized.
Danil Chapovalov0682ca92019-11-28 15:50:02204 if (packetized_payload_size >= unpacketized_payload_size) {
205 packetization_overhead_bitrate_.Update(
206 packetized_payload_size - unpacketized_payload_size,
207 clock_->TimeInMilliseconds());
208 }
Erik Språngf1e97b92019-09-26 10:48:47209 }
210
Erik Språngfc78aac2019-10-24 10:55:30211 rtp_sender_->EnqueuePackets(std::move(packets));
Niels Möller59ab1cf2019-02-06 21:48:11212}
213
Danil Chapovalov0deef722019-10-14 12:20:20214size_t RTPSenderVideo::FecPacketOverhead() const {
Erik Språng42108882020-05-05 12:51:09215 size_t overhead = fec_overhead_bytes_;
brandtrf1bb4762016-11-07 11:05:06216 if (red_enabled()) {
brandtr131bc492016-11-10 13:01:11217 // The RED overhead is due to a small header.
218 overhead += kRedForFecHeaderLength;
Erik Språngf87536c2020-03-05 09:14:04219
Erik Språng42108882020-05-05 12:51:09220 if (fec_type_ == VideoFecGenerator::FecType::kUlpFec) {
Erik Språngf87536c2020-03-05 09:14:04221 // For ULPFEC, the overhead is the FEC headers plus RED for FEC header
222 // (see above) plus anything in RTP header beyond the 12 bytes base header
223 // (CSRC list, extensions...)
224 // This reason for the header extensions to be included here is that
225 // from an FEC viewpoint, they are part of the payload to be protected.
226 // (The base RTP header is already protected by the FEC header.)
Erik Språng04e1bab2020-05-07 16:18:32227 overhead +=
228 rtp_sender_->FecOrPaddingPacketMaxRtpHeaderLength() - kRtpHeaderSize;
Erik Språngf87536c2020-03-05 09:14:04229 }
pbos@webrtc.orgb5e6bfc2014-09-12 11:05:55230 }
stefan8f4c77f2016-06-03 07:16:45231 return overhead;
niklase@google.com470e71d2011-07-07 08:21:25232}
233
Danil Chapovalov670af262020-01-30 15:28:53234void RTPSenderVideo::SetVideoStructure(
235 const FrameDependencyStructure* video_structure) {
Marina Ciocea3a087a82020-03-03 04:39:28236 if (frame_transformer_delegate_) {
237 frame_transformer_delegate_->SetVideoStructureUnderLock(video_structure);
238 return;
239 }
Per Kjellander4f350ba2020-10-14 06:41:52240 SetVideoStructureInternal(video_structure);
Marina Ciocea3a087a82020-03-03 04:39:28241}
242
Per Kjellander4f350ba2020-10-14 06:41:52243void RTPSenderVideo::SetVideoStructureAfterTransformation(
244 const FrameDependencyStructure* video_structure) {
245 SetVideoStructureInternal(video_structure);
246}
247
248void RTPSenderVideo::SetVideoStructureInternal(
Marina Ciocea3a087a82020-03-03 04:39:28249 const FrameDependencyStructure* video_structure) {
Danil Chapovalov670af262020-01-30 15:28:53250 RTC_DCHECK_RUNS_SERIALIZED(&send_checker_);
251 if (video_structure == nullptr) {
252 video_structure_ = nullptr;
253 return;
254 }
255 // Simple sanity checks video structure is set up.
256 RTC_DCHECK_GT(video_structure->num_decode_targets, 0);
257 RTC_DCHECK_GT(video_structure->templates.size(), 0);
258
259 int structure_id = 0;
260 if (video_structure_) {
261 if (*video_structure_ == *video_structure) {
262 // Same structure (just a new key frame), no update required.
263 return;
264 }
265 // When setting different video structure make sure structure_id is updated
266 // so that templates from different structures do not collide.
267 static constexpr int kMaxTemplates = 64;
268 structure_id =
269 (video_structure_->structure_id + video_structure_->templates.size()) %
270 kMaxTemplates;
271 }
272
273 video_structure_ =
274 std::make_unique<FrameDependencyStructure>(*video_structure);
275 video_structure_->structure_id = structure_id;
Danil Chapovalov670af262020-01-30 15:28:53276}
277
Per Kjellander4f350ba2020-10-14 06:41:52278void RTPSenderVideo::SetVideoLayersAllocation(
279 VideoLayersAllocation allocation) {
280 if (frame_transformer_delegate_) {
281 frame_transformer_delegate_->SetVideoLayersAllocationUnderLock(
282 std::move(allocation));
283 return;
284 }
285 SetVideoLayersAllocationInternal(std::move(allocation));
286}
287
288void RTPSenderVideo::SetVideoLayersAllocationAfterTransformation(
289 VideoLayersAllocation allocation) {
290 SetVideoLayersAllocationInternal(std::move(allocation));
291}
292
293void RTPSenderVideo::SetVideoLayersAllocationInternal(
294 VideoLayersAllocation allocation) {
295 RTC_DCHECK_RUNS_SERIALIZED(&send_checker_);
Per Kjellanderdbf95492020-12-22 15:12:15296 if (!allocation_ || allocation.active_spatial_layers.size() >
297 allocation_->active_spatial_layers.size()) {
298 send_allocation_ = SendVideoLayersAllocation::kSendWithResolution;
299 } else if (send_allocation_ == SendVideoLayersAllocation::kDontSend) {
300 send_allocation_ = SendVideoLayersAllocation::kSendWithoutResolution;
301 }
Per Kjellander4f350ba2020-10-14 06:41:52302 allocation_ = std::move(allocation);
Per Kjellander4f350ba2020-10-14 06:41:52303}
304
Danil Chapovalov5efb02b2020-03-13 16:29:12305void RTPSenderVideo::AddRtpHeaderExtensions(
306 const RTPVideoHeader& video_header,
307 const absl::optional<AbsoluteCaptureTime>& absolute_capture_time,
308 bool first_packet,
309 bool last_packet,
310 RtpPacketToSend* packet) const {
311 // Send color space when changed or if the frame is a key frame. Keep
312 // sending color space information until the first base layer frame to
313 // guarantee that the information is retrieved by the receiver.
314 bool set_color_space =
315 video_header.color_space != last_color_space_ ||
316 video_header.frame_type == VideoFrameType::kVideoFrameKey ||
317 transmit_color_space_next_frame_;
318 // Color space requires two-byte header extensions if HDR metadata is
319 // included. Therefore, it's best to add this extension first so that the
320 // other extensions in the same packet are written as two-byte headers at
321 // once.
322 if (last_packet && set_color_space && video_header.color_space)
323 packet->SetExtension<ColorSpaceExtension>(video_header.color_space.value());
324
325 // According to
326 // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
327 // ts_126114v120700p.pdf Section 7.4.5:
328 // The MTSI client shall add the payload bytes as defined in this clause
329 // onto the last RTP packet in each group of packets which make up a key
330 // frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265
331 // (HEVC)). The MTSI client may also add the payload bytes onto the last RTP
332 // packet in each group of packets which make up another type of frame
333 // (e.g. a P-Frame) only if the current value is different from the previous
334 // value sent.
335 // Set rotation when key frame or when changed (to follow standard).
336 // Or when different from 0 (to follow current receiver implementation).
337 bool set_video_rotation =
338 video_header.frame_type == VideoFrameType::kVideoFrameKey ||
339 video_header.rotation != last_rotation_ ||
340 video_header.rotation != kVideoRotation_0;
341 if (last_packet && set_video_rotation)
342 packet->SetExtension<VideoOrientation>(video_header.rotation);
343
344 // Report content type only for key frames.
345 if (last_packet &&
346 video_header.frame_type == VideoFrameType::kVideoFrameKey &&
347 video_header.content_type != VideoContentType::UNSPECIFIED)
348 packet->SetExtension<VideoContentTypeExtension>(video_header.content_type);
349
350 if (last_packet &&
351 video_header.video_timing.flags != VideoSendTiming::kInvalid)
352 packet->SetExtension<VideoTimingExtension>(video_header.video_timing);
353
354 // If transmitted, add to all packets; ack logic depends on this.
355 if (playout_delay_pending_) {
356 packet->SetExtension<PlayoutDelayLimits>(current_playout_delay_);
357 }
358
359 if (first_packet && absolute_capture_time) {
360 packet->SetExtension<AbsoluteCaptureTimeExtension>(*absolute_capture_time);
361 }
362
Danil Chapovalov5efb02b2020-03-13 16:29:12363 if (video_header.generic) {
364 bool extension_is_set = false;
365 if (video_structure_ != nullptr) {
366 DependencyDescriptor descriptor;
367 descriptor.first_packet_in_frame = first_packet;
368 descriptor.last_packet_in_frame = last_packet;
369 descriptor.frame_number = video_header.generic->frame_id & 0xFFFF;
370 descriptor.frame_dependencies.spatial_id =
371 video_header.generic->spatial_index;
372 descriptor.frame_dependencies.temporal_id =
373 video_header.generic->temporal_index;
374 for (int64_t dep : video_header.generic->dependencies) {
375 descriptor.frame_dependencies.frame_diffs.push_back(
376 video_header.generic->frame_id - dep);
377 }
Danil Chapovalov37120ab2020-05-14 14:35:23378 descriptor.frame_dependencies.chain_diffs =
379 video_header.generic->chain_diffs;
Danil Chapovalov5efb02b2020-03-13 16:29:12380 descriptor.frame_dependencies.decode_target_indications =
381 video_header.generic->decode_target_indications;
382 RTC_DCHECK_EQ(
383 descriptor.frame_dependencies.decode_target_indications.size(),
384 video_structure_->num_decode_targets);
385
Danil Chapovalove6ac8ff2020-06-26 11:51:08386 if (first_packet) {
387 descriptor.active_decode_targets_bitmask =
388 active_decode_targets_tracker_.ActiveDecodeTargetsBitmask();
389 }
Danil Chapovalov5ad731a2020-10-15 15:26:46390 // VP9 mark all layer frames of the first picture as kVideoFrameKey,
391 // Structure should be attached to the descriptor to lowest spatial layer
392 // when inter layer dependency is used, i.e. L structures; or to all
393 // layers when inter layer dependency is not used, i.e. S structures.
394 // Distinguish these two cases by checking if there are any dependencies.
Danil Chapovalov5efb02b2020-03-13 16:29:12395 if (video_header.frame_type == VideoFrameType::kVideoFrameKey &&
Danil Chapovalov5ad731a2020-10-15 15:26:46396 video_header.generic->dependencies.empty() && first_packet) {
397 // To avoid extra structure copy, temporary share ownership of the
398 // video_structure with the dependency descriptor.
Danil Chapovalov5efb02b2020-03-13 16:29:12399 descriptor.attached_structure =
400 absl::WrapUnique(video_structure_.get());
401 }
402 extension_is_set = packet->SetExtension<RtpDependencyDescriptorExtension>(
Danil Chapovalove6ac8ff2020-06-26 11:51:08403 *video_structure_,
404 active_decode_targets_tracker_.ActiveChainsBitmask(), descriptor);
Danil Chapovalov5efb02b2020-03-13 16:29:12405
406 // Remove the temporary shared ownership.
407 descriptor.attached_structure.release();
408 }
409
Danil Chapovalovec9fc222020-04-14 14:11:59410 // Do not use generic frame descriptor when dependency descriptor is stored.
Danil Chapovalov5efb02b2020-03-13 16:29:12411 if (!extension_is_set) {
412 RtpGenericFrameDescriptor generic_descriptor;
413 generic_descriptor.SetFirstPacketInSubFrame(first_packet);
414 generic_descriptor.SetLastPacketInSubFrame(last_packet);
Danil Chapovalov5efb02b2020-03-13 16:29:12415
416 if (first_packet) {
417 generic_descriptor.SetFrameId(
418 static_cast<uint16_t>(video_header.generic->frame_id));
419 for (int64_t dep : video_header.generic->dependencies) {
420 generic_descriptor.AddFrameDependencyDiff(
421 video_header.generic->frame_id - dep);
422 }
423
424 uint8_t spatial_bimask = 1 << video_header.generic->spatial_index;
425 generic_descriptor.SetSpatialLayersBitmask(spatial_bimask);
426
427 generic_descriptor.SetTemporalLayer(
428 video_header.generic->temporal_index);
429
430 if (video_header.frame_type == VideoFrameType::kVideoFrameKey) {
431 generic_descriptor.SetResolution(video_header.width,
432 video_header.height);
433 }
434 }
435
Danil Chapovalov9d287bf2020-04-06 09:34:34436 packet->SetExtension<RtpGenericFrameDescriptorExtension00>(
437 generic_descriptor);
Danil Chapovalov5efb02b2020-03-13 16:29:12438 }
439 }
Per Kjellander4f350ba2020-10-14 06:41:52440
Per Kjellanderdbf95492020-12-22 15:12:15441 if (first_packet &&
442 send_allocation_ != SendVideoLayersAllocation::kDontSend &&
443 (video_header.frame_type == VideoFrameType::kVideoFrameKey ||
444 PacketWillLikelyBeRequestedForRestransmitionIfLost(video_header))) {
445 VideoLayersAllocation allocation = allocation_.value();
446 allocation.resolution_and_frame_rate_is_valid =
447 send_allocation_ == SendVideoLayersAllocation::kSendWithResolution;
448 packet->SetExtension<RtpVideoLayersAllocationExtension>(allocation);
Per Kjellander4f350ba2020-10-14 06:41:52449 }
Danil Chapovalov5efb02b2020-03-13 16:29:12450}
451
Elad Alonb64af4b2019-06-05 09:39:37452bool RTPSenderVideo::SendVideo(
Danil Chapovalov51bf2002019-10-11 08:53:27453 int payload_type,
454 absl::optional<VideoCodecType> codec_type,
455 uint32_t rtp_timestamp,
456 int64_t capture_time_ms,
457 rtc::ArrayView<const uint8_t> payload,
Danil Chapovalov51bf2002019-10-11 08:53:27458 RTPVideoHeader video_header,
Minyue Lid37b0ec2020-08-21 14:28:07459 absl::optional<int64_t> expected_retransmission_time_ms,
460 absl::optional<int64_t> estimated_capture_clock_offset_ms) {
Marina Ciocea3a087a82020-03-03 04:39:28461#if RTC_TRACE_EVENTS_ENABLED
Niels Möller59ab1cf2019-02-06 21:48:11462 TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", capture_time_ms, "Send", "type",
Danil Chapovalov51bf2002019-10-11 08:53:27463 FrameTypeToString(video_header.frame_type));
Marina Ciocea3a087a82020-03-03 04:39:28464#endif
Erik Språng6cf554e2019-10-02 18:55:39465 RTC_CHECK_RUNS_SERIALIZED(&send_checker_);
Niels Möller59ab1cf2019-02-06 21:48:11466
Danil Chapovalov51bf2002019-10-11 08:53:27467 if (video_header.frame_type == VideoFrameType::kEmptyFrame)
Niels Möller59ab1cf2019-02-06 21:48:11468 return true;
469
Danil Chapovalov51bf2002019-10-11 08:53:27470 if (payload.empty())
Sergey Ulanov525df3f2016-08-03 00:46:41471 return false;
niklase@google.com470e71d2011-07-07 08:21:25472
Erik Språng6cf554e2019-10-02 18:55:39473 int32_t retransmission_settings = retransmission_settings_;
474 if (codec_type == VideoCodecType::kVideoCodecH264) {
475 // Backward compatibility for older receivers without temporal layer logic.
476 retransmission_settings = kRetransmitBaseLayer | kRetransmitHigherLayers;
477 }
478
Erik Språng56e611b2020-02-06 16:10:08479 MaybeUpdateCurrentPlayoutDelay(video_header);
Per Kjellander4f350ba2020-10-14 06:41:52480 if (video_header.frame_type == VideoFrameType::kVideoFrameKey) {
481 if (!IsNoopDelay(current_playout_delay_)) {
482 // Force playout delay on key-frames, if set.
483 playout_delay_pending_ = true;
484 }
485 if (allocation_) {
486 // Send the bitrate allocation on every key frame.
Per Kjellanderdbf95492020-12-22 15:12:15487 send_allocation_ = SendVideoLayersAllocation::kSendWithResolution;
Per Kjellander4f350ba2020-10-14 06:41:52488 }
Erik Språng56e611b2020-02-06 16:10:08489 }
Erik Språng6cf554e2019-10-02 18:55:39490
Danil Chapovalove6ac8ff2020-06-26 11:51:08491 if (video_structure_ != nullptr && video_header.generic) {
492 active_decode_targets_tracker_.OnFrame(
493 video_structure_->decode_target_protected_by_chain,
494 video_header.generic->active_decode_targets,
495 video_header.frame_type == VideoFrameType::kVideoFrameKey,
496 video_header.generic->frame_id, video_header.generic->chain_diffs);
497 }
498
Erik Språngb6477852020-10-05 12:20:51499 const uint8_t temporal_id = GetTemporalId(video_header);
500 // No FEC protection for upper temporal layers, if used.
501 const bool use_fec = fec_type_.has_value() &&
502 (temporal_id == 0 || temporal_id == kNoTemporalIdx);
503
Danil Chapovalove4f8b382018-09-07 15:30:26504 // Maximum size of packet including rtp headers.
505 // Extra space left in case packet will be resent using fec or rtx.
Erik Språngb6477852020-10-05 12:20:51506 int packet_capacity = rtp_sender_->MaxRtpPacketSize() -
507 (use_fec ? FecPacketOverhead() : 0) -
Danil Chapovalovfa5ec8d2018-09-07 08:57:26508 (rtp_sender_->RtxStatus() ? kRtxHeaderSize : 0);
Danil Chapovalove4f8b382018-09-07 15:30:26509
Danil Chapovalov6026f052018-10-16 14:22:33510 std::unique_ptr<RtpPacketToSend> single_packet =
511 rtp_sender_->AllocatePacket();
512 RTC_DCHECK_LE(packet_capacity, single_packet->capacity());
513 single_packet->SetPayloadType(payload_type);
514 single_packet->SetTimestamp(rtp_timestamp);
515 single_packet->set_capture_time_ms(capture_time_ms);
Danil Chapovalove4f8b382018-09-07 15:30:26516
Minyue Li5bb9adc2020-01-22 12:02:34517 const absl::optional<AbsoluteCaptureTime> absolute_capture_time =
518 absolute_capture_time_sender_.OnSendPacket(
519 AbsoluteCaptureTimeSender::GetSource(single_packet->Ssrc(),
520 single_packet->Csrcs()),
521 single_packet->Timestamp(), kVideoPayloadTypeFrequency,
522 Int64MsToUQ32x32(single_packet->capture_time_ms() + NtpOffsetMs()),
Minyue Lie64b3d02020-08-20 14:18:31523 /*estimated_capture_clock_offset=*/
Minyue Lid37b0ec2020-08-21 14:28:07524 include_capture_clock_offset_ ? estimated_capture_clock_offset_ms
Minyue Lie64b3d02020-08-20 14:18:31525 : absl::nullopt);
Minyue Li5bb9adc2020-01-22 12:02:34526
Mirko Bonadei317a1f02019-09-17 15:06:18527 auto first_packet = std::make_unique<RtpPacketToSend>(*single_packet);
528 auto middle_packet = std::make_unique<RtpPacketToSend>(*single_packet);
529 auto last_packet = std::make_unique<RtpPacketToSend>(*single_packet);
Danil Chapovalove4f8b382018-09-07 15:30:26530 // Simplest way to estimate how much extensions would occupy is to set them.
Danil Chapovalov5efb02b2020-03-13 16:29:12531 AddRtpHeaderExtensions(video_header, absolute_capture_time,
532 /*first_packet=*/true, /*last_packet=*/true,
533 single_packet.get());
534 AddRtpHeaderExtensions(video_header, absolute_capture_time,
535 /*first_packet=*/true, /*last_packet=*/false,
536 first_packet.get());
537 AddRtpHeaderExtensions(video_header, absolute_capture_time,
538 /*first_packet=*/false, /*last_packet=*/false,
539 middle_packet.get());
540 AddRtpHeaderExtensions(video_header, absolute_capture_time,
541 /*first_packet=*/false, /*last_packet=*/true,
542 last_packet.get());
Johnny Lee1a1c52b2019-02-08 19:25:40543
Danil Chapovalov6026f052018-10-16 14:22:33544 RTC_DCHECK_GT(packet_capacity, single_packet->headers_size());
Danil Chapovalove4f8b382018-09-07 15:30:26545 RTC_DCHECK_GT(packet_capacity, first_packet->headers_size());
546 RTC_DCHECK_GT(packet_capacity, middle_packet->headers_size());
ilnik7a3006b2017-05-23 16:34:21547 RTC_DCHECK_GT(packet_capacity, last_packet->headers_size());
Danil Chapovalovfa5ec8d2018-09-07 08:57:26548 RtpPacketizer::PayloadSizeLimits limits;
Danil Chapovalove4f8b382018-09-07 15:30:26549 limits.max_payload_len = packet_capacity - middle_packet->headers_size();
Danil Chapovalovfa5ec8d2018-09-07 08:57:26550
Danil Chapovalov6026f052018-10-16 14:22:33551 RTC_DCHECK_GE(single_packet->headers_size(), middle_packet->headers_size());
552 limits.single_packet_reduction_len =
553 single_packet->headers_size() - middle_packet->headers_size();
554
Danil Chapovalove4f8b382018-09-07 15:30:26555 RTC_DCHECK_GE(first_packet->headers_size(), middle_packet->headers_size());
556 limits.first_packet_reduction_len =
557 first_packet->headers_size() - middle_packet->headers_size();
558
559 RTC_DCHECK_GE(last_packet->headers_size(), middle_packet->headers_size());
Danil Chapovalovfa5ec8d2018-09-07 08:57:26560 limits.last_packet_reduction_len =
Danil Chapovalove4f8b382018-09-07 15:30:26561 last_packet->headers_size() - middle_packet->headers_size();
danilchap74110612016-10-02 17:54:29562
Danil Chapovalov9d287bf2020-04-06 09:34:34563 bool has_generic_descriptor =
564 first_packet->HasExtension<RtpGenericFrameDescriptorExtension00>() ||
Danil Chapovalov95800f62020-02-20 14:01:15565 first_packet->HasExtension<RtpDependencyDescriptorExtension>();
Elad Alonccb9b752019-02-19 12:01:31566
Erik Språngb6477852020-10-05 12:20:51567 // Minimization of the vp8 descriptor may erase temporal_id, so use
568 // |temporal_id| rather than reference |video_header| beyond this point.
Danil Chapovalov95800f62020-02-20 14:01:15569 if (has_generic_descriptor) {
Danil Chapovalov51bf2002019-10-11 08:53:27570 MinimizeDescriptor(&video_header);
Benjamin Wright192eeec2018-10-18 00:27:25571 }
572
573 // TODO(benwright@webrtc.org) - Allocate enough to always encrypt inline.
574 rtc::Buffer encrypted_video_payload;
575 if (frame_encryptor_ != nullptr) {
Danil Chapovalov95800f62020-02-20 14:01:15576 if (!has_generic_descriptor) {
Benjamin Wright192eeec2018-10-18 00:27:25577 return false;
578 }
579
580 const size_t max_ciphertext_size =
581 frame_encryptor_->GetMaxCiphertextByteSize(cricket::MEDIA_TYPE_VIDEO,
Danil Chapovalov51bf2002019-10-11 08:53:27582 payload.size());
Benjamin Wright192eeec2018-10-18 00:27:25583 encrypted_video_payload.SetSize(max_ciphertext_size);
584
585 size_t bytes_written = 0;
Benjamin Wright168456c2018-12-07 19:31:25586
Markus Handellc1cbf6b2020-02-17 19:03:57587 // Enable header authentication if the field trial isn't disabled.
Danil Chapovalov95800f62020-02-20 14:01:15588 std::vector<uint8_t> additional_data;
Benjamin Wright168456c2018-12-07 19:31:25589 if (generic_descriptor_auth_experiment_) {
Danil Chapovalov95800f62020-02-20 14:01:15590 additional_data = RtpDescriptorAuthentication(video_header);
Benjamin Wright168456c2018-12-07 19:31:25591 }
592
Benjamin Wright192eeec2018-10-18 00:27:25593 if (frame_encryptor_->Encrypt(
Benjamin Wright168456c2018-12-07 19:31:25594 cricket::MEDIA_TYPE_VIDEO, first_packet->Ssrc(), additional_data,
Danil Chapovalov51bf2002019-10-11 08:53:27595 payload, encrypted_video_payload, &bytes_written) != 0) {
Benjamin Wright192eeec2018-10-18 00:27:25596 return false;
597 }
598
599 encrypted_video_payload.SetSize(bytes_written);
Danil Chapovalov51bf2002019-10-11 08:53:27600 payload = encrypted_video_payload;
Benjamin Wright192eeec2018-10-18 00:27:25601 } else if (require_frame_encryption_) {
602 RTC_LOG(LS_WARNING)
603 << "No FrameEncryptor is attached to this video sending stream but "
Jonas Olssonb2b20312020-01-14 11:11:31604 "one is required since require_frame_encryptor is set";
Danil Chapovalov84ffb352018-09-25 16:59:09605 }
606
Danil Chapovalov31cb3ab2020-07-21 13:01:50607 std::unique_ptr<RtpPacketizer> packetizer =
608 RtpPacketizer::Create(codec_type, payload, limits, video_header);
spranga8ae6f22017-09-04 14:23:56609
Elad Alonb64af4b2019-06-05 09:39:37610 // TODO(bugs.webrtc.org/10714): retransmission_settings_ should generally be
611 // replaced by expected_retransmission_time_ms.has_value(). For now, though,
612 // only VP8 with an injected frame buffer controller actually controls it.
Erik Språng70768f42019-08-27 16:16:26613 const bool allow_retransmission =
Elad Alonb64af4b2019-06-05 09:39:37614 expected_retransmission_time_ms.has_value()
Erik Språng70768f42019-08-27 16:16:26615 ? AllowRetransmission(temporal_id, retransmission_settings,
616 expected_retransmission_time_ms.value())
617 : false;
Elad Alon8b60e8b2019-04-08 12:14:05618 const size_t num_packets = packetizer->NumPackets();
ilnik7a3006b2017-05-23 16:34:21619
620 if (num_packets == 0)
621 return false;
pbos@webrtc.orgb5e6bfc2014-09-12 11:05:55622
danilchap2a615fc2016-11-11 10:27:35623 bool first_frame = first_frame_sent_();
Erik Språngf1e97b92019-09-26 10:48:47624 std::vector<std::unique_ptr<RtpPacketToSend>> rtp_packets;
ilnik7a3006b2017-05-23 16:34:21625 for (size_t i = 0; i < num_packets; ++i) {
Danil Chapovalove4f8b382018-09-07 15:30:26626 std::unique_ptr<RtpPacketToSend> packet;
627 int expected_payload_capacity;
628 // Choose right packet template:
629 if (num_packets == 1) {
Danil Chapovalov6026f052018-10-16 14:22:33630 packet = std::move(single_packet);
Danil Chapovalovfcebe0e2018-10-12 15:51:22631 expected_payload_capacity =
632 limits.max_payload_len - limits.single_packet_reduction_len;
Danil Chapovalove4f8b382018-09-07 15:30:26633 } else if (i == 0) {
634 packet = std::move(first_packet);
635 expected_payload_capacity =
636 limits.max_payload_len - limits.first_packet_reduction_len;
637 } else if (i == num_packets - 1) {
638 packet = std::move(last_packet);
639 expected_payload_capacity =
640 limits.max_payload_len - limits.last_packet_reduction_len;
641 } else {
Mirko Bonadei317a1f02019-09-17 15:06:18642 packet = std::make_unique<RtpPacketToSend>(*middle_packet);
Danil Chapovalove4f8b382018-09-07 15:30:26643 expected_payload_capacity = limits.max_payload_len;
644 }
645
Erik Språng3663f942020-02-07 09:05:15646 packet->set_first_packet_of_frame(i == 0);
647
ilnik7a3006b2017-05-23 16:34:21648 if (!packetizer->NextPacket(packet.get()))
Sergey Ulanov525df3f2016-08-03 00:46:41649 return false;
Danil Chapovalove4f8b382018-09-07 15:30:26650 RTC_DCHECK_LE(packet->payload_size(), expected_payload_capacity);
danilchap74110612016-10-02 17:54:29651
Erik Språng70768f42019-08-27 16:16:26652 packet->set_allow_retransmission(allow_retransmission);
Erik Språngcf15cb52020-12-07 12:18:34653 packet->set_is_key_frame(video_header.frame_type ==
654 VideoFrameType::kVideoFrameKey);
Erik Språng70768f42019-08-27 16:16:26655
ilnik04f4d122017-06-19 14:18:55656 // Put packetization finish timestamp into extension.
ilnike4350192017-06-29 09:27:44657 if (packet->HasExtension<VideoTimingExtension>()) {
ilnik04f4d122017-06-19 14:18:55658 packet->set_packetization_finish_time_ms(clock_->TimeInMilliseconds());
659 }
660
Erik Språngb6477852020-10-05 12:20:51661 packet->set_fec_protect_packet(use_fec);
Erik Språngf87536c2020-03-05 09:14:04662
Danil Chapovalov0deef722019-10-14 12:20:20663 if (red_enabled()) {
Erik Språngb6477852020-10-05 12:20:51664 // TODO(sprang): Consider packetizing directly into packets with the RED
665 // header already in place, to avoid this copy.
Erik Språngf87536c2020-03-05 09:14:04666 std::unique_ptr<RtpPacketToSend> red_packet(new RtpPacketToSend(*packet));
667 BuildRedPayload(*packet, red_packet.get());
668 red_packet->SetPayloadType(*red_payload_type_);
Erik Språng1d50cb62020-07-02 15:41:32669 red_packet->set_is_red(true);
Erik Språngf87536c2020-03-05 09:14:04670
Erik Språng0f718712021-02-17 14:19:06671 // Append |red_packet| instead of |packet| to output.
Erik Språngf87536c2020-03-05 09:14:04672 red_packet->set_packet_type(RtpPacketMediaType::kVideo);
673 red_packet->set_allow_retransmission(packet->allow_retransmission());
674 rtp_packets.emplace_back(std::move(red_packet));
mflodmanfcf54bd2015-04-14 19:28:08675 } else {
Björn Terelius31d0f7c2020-02-06 15:35:46676 packet->set_packet_type(RtpPacketMediaType::kVideo);
Erik Språngf1e97b92019-09-26 10:48:47677 rtp_packets.emplace_back(std::move(packet));
stefan@webrtc.org2ec56062014-07-31 14:59:24678 }
skvlad98bb6642016-04-07 22:36:45679
680 if (first_frame) {
ilnik7a3006b2017-05-23 16:34:21681 if (i == 0) {
Mirko Bonadei675513b2017-11-09 10:09:25682 RTC_LOG(LS_INFO)
skvlad98bb6642016-04-07 22:36:45683 << "Sent first RTP packet of the first video frame (pre-pacer)";
684 }
Danil Chapovalove4f8b382018-09-07 15:30:26685 if (i == num_packets - 1) {
Mirko Bonadei675513b2017-11-09 10:09:25686 RTC_LOG(LS_INFO)
skvlad98bb6642016-04-07 22:36:45687 << "Sent last RTP packet of the first video frame (pre-pacer)";
688 }
689 }
stefan@webrtc.org2ec56062014-07-31 14:59:24690 }
pbos@webrtc.orgb5e6bfc2014-09-12 11:05:55691
Erik Språng0f718712021-02-17 14:19:06692 if (!rtp_sender_->AssignSequenceNumbersAndStoreLastPacketState(rtp_packets)) {
693 // Media not being sent.
694 return false;
695 }
696
Danil Chapovalov31cb3ab2020-07-21 13:01:50697 LogAndSendToNetwork(std::move(rtp_packets), payload.size());
Erik Språng482b3ef2019-01-08 15:19:11698
Danil Chapovalov5efb02b2020-03-13 16:29:12699 // Update details about the last sent frame.
700 last_rotation_ = video_header.rotation;
701
702 if (video_header.color_space != last_color_space_) {
703 last_color_space_ = video_header.color_space;
704 transmit_color_space_next_frame_ = !IsBaseLayer(video_header);
705 } else {
706 transmit_color_space_next_frame_ =
707 transmit_color_space_next_frame_ ? !IsBaseLayer(video_header) : false;
708 }
709
710 if (video_header.frame_type == VideoFrameType::kVideoFrameKey ||
Per Kjellander4f350ba2020-10-14 06:41:52711 PacketWillLikelyBeRequestedForRestransmitionIfLost(video_header)) {
712 // This frame will likely be delivered, no need to populate playout
Danil Chapovalov5efb02b2020-03-13 16:29:12713 // delay extensions until it changes again.
714 playout_delay_pending_ = false;
Per Kjellanderdbf95492020-12-22 15:12:15715 send_allocation_ = SendVideoLayersAllocation::kDontSend;
Danil Chapovalov5efb02b2020-03-13 16:29:12716 }
717
Sergey Ulanovec4f0682016-07-28 22:19:10718 TRACE_EVENT_ASYNC_END1("webrtc", "Video", capture_time_ms, "timestamp",
danilchape5b41412016-08-22 10:39:23719 rtp_timestamp);
Sergey Ulanov525df3f2016-08-03 00:46:41720 return true;
mflodmanfcf54bd2015-04-14 19:28:08721}
722
Marina Ciocea3a087a82020-03-03 04:39:28723bool RTPSenderVideo::SendEncodedImage(
724 int payload_type,
725 absl::optional<VideoCodecType> codec_type,
726 uint32_t rtp_timestamp,
727 const EncodedImage& encoded_image,
Marina Ciocea3a087a82020-03-03 04:39:28728 RTPVideoHeader video_header,
729 absl::optional<int64_t> expected_retransmission_time_ms) {
730 if (frame_transformer_delegate_) {
731 // The frame will be sent async once transformed.
732 return frame_transformer_delegate_->TransformFrame(
Danil Chapovalov31cb3ab2020-07-21 13:01:50733 payload_type, codec_type, rtp_timestamp, encoded_image, video_header,
734 expected_retransmission_time_ms);
Marina Ciocea3a087a82020-03-03 04:39:28735 }
736 return SendVideo(payload_type, codec_type, rtp_timestamp,
Danil Chapovalov31cb3ab2020-07-21 13:01:50737 encoded_image.capture_time_ms_, encoded_image, video_header,
738 expected_retransmission_time_ms);
Marina Ciocea3a087a82020-03-03 04:39:28739}
740
Erik Språng482b3ef2019-01-08 15:19:11741uint32_t RTPSenderVideo::PacketizationOverheadBps() const {
Markus Handelle7c015e2020-07-07 09:44:28742 MutexLock lock(&stats_mutex_);
Erik Språng482b3ef2019-01-08 15:19:11743 return packetization_overhead_bitrate_.Rate(clock_->TimeInMilliseconds())
744 .value_or(0);
745}
746
Erik Språng70768f42019-08-27 16:16:26747bool RTPSenderVideo::AllowRetransmission(
spranga8ae6f22017-09-04 14:23:56748 uint8_t temporal_id,
749 int32_t retransmission_settings,
750 int64_t expected_retransmission_time_ms) {
751 if (retransmission_settings == kRetransmitOff)
Erik Språng70768f42019-08-27 16:16:26752 return false;
spranga8ae6f22017-09-04 14:23:56753
Markus Handelle7c015e2020-07-07 09:44:28754 MutexLock lock(&stats_mutex_);
spranga8ae6f22017-09-04 14:23:56755 // Media packet storage.
756 if ((retransmission_settings & kConditionallyRetransmitHigherLayers) &&
757 UpdateConditionalRetransmit(temporal_id,
758 expected_retransmission_time_ms)) {
759 retransmission_settings |= kRetransmitHigherLayers;
760 }
761
762 if (temporal_id == kNoTemporalIdx)
Erik Språng70768f42019-08-27 16:16:26763 return true;
spranga8ae6f22017-09-04 14:23:56764
765 if ((retransmission_settings & kRetransmitBaseLayer) && temporal_id == 0)
Erik Språng70768f42019-08-27 16:16:26766 return true;
spranga8ae6f22017-09-04 14:23:56767
768 if ((retransmission_settings & kRetransmitHigherLayers) && temporal_id > 0)
Erik Språng70768f42019-08-27 16:16:26769 return true;
spranga8ae6f22017-09-04 14:23:56770
Erik Språng70768f42019-08-27 16:16:26771 return false;
spranga8ae6f22017-09-04 14:23:56772}
773
774uint8_t RTPSenderVideo::GetTemporalId(const RTPVideoHeader& header) {
Philip Eliassond52a1a62018-09-07 13:03:55775 struct TemporalIdGetter {
776 uint8_t operator()(const RTPVideoHeaderVP8& vp8) { return vp8.temporalIdx; }
777 uint8_t operator()(const RTPVideoHeaderVP9& vp9) {
778 return vp9.temporal_idx;
779 }
780 uint8_t operator()(const RTPVideoHeaderH264&) { return kNoTemporalIdx; }
Danil Chapovalovb6bf0b22020-01-28 17:36:57781 uint8_t operator()(const RTPVideoHeaderLegacyGeneric&) {
782 return kNoTemporalIdx;
783 }
Philip Eliassond52a1a62018-09-07 13:03:55784 uint8_t operator()(const absl::monostate&) { return kNoTemporalIdx; }
785 };
philipel94659782020-06-15 10:26:39786 return absl::visit(TemporalIdGetter(), header.video_type_header);
spranga8ae6f22017-09-04 14:23:56787}
788
789bool RTPSenderVideo::UpdateConditionalRetransmit(
790 uint8_t temporal_id,
791 int64_t expected_retransmission_time_ms) {
792 int64_t now_ms = clock_->TimeInMilliseconds();
793 // Update stats for any temporal layer.
794 TemporalLayerStats* current_layer_stats =
795 &frame_stats_by_temporal_layer_[temporal_id];
796 current_layer_stats->frame_rate_fp1000s.Update(1, now_ms);
797 int64_t tl_frame_interval = now_ms - current_layer_stats->last_frame_time_ms;
798 current_layer_stats->last_frame_time_ms = now_ms;
799
800 // Conditional retransmit only applies to upper layers.
801 if (temporal_id != kNoTemporalIdx && temporal_id > 0) {
802 if (tl_frame_interval >= kMaxUnretransmittableFrameIntervalMs) {
803 // Too long since a retransmittable frame in this layer, enable NACK
804 // protection.
805 return true;
806 } else {
807 // Estimate when the next frame of any lower layer will be sent.
808 const int64_t kUndefined = std::numeric_limits<int64_t>::max();
809 int64_t expected_next_frame_time = kUndefined;
810 for (int i = temporal_id - 1; i >= 0; --i) {
811 TemporalLayerStats* stats = &frame_stats_by_temporal_layer_[i];
Danil Chapovalovd264df52018-06-14 10:59:38812 absl::optional<uint32_t> rate = stats->frame_rate_fp1000s.Rate(now_ms);
spranga8ae6f22017-09-04 14:23:56813 if (rate) {
814 int64_t tl_next = stats->last_frame_time_ms + 1000000 / *rate;
815 if (tl_next - now_ms > -expected_retransmission_time_ms &&
816 tl_next < expected_next_frame_time) {
817 expected_next_frame_time = tl_next;
818 }
819 }
820 }
821
822 if (expected_next_frame_time == kUndefined ||
823 expected_next_frame_time - now_ms > expected_retransmission_time_ms) {
824 // The next frame in a lower layer is expected at a later time (or
825 // unable to tell due to lack of data) than a retransmission is
826 // estimated to be able to arrive, so allow this packet to be nacked.
827 return true;
828 }
829 }
830 }
831
832 return false;
833}
834
Erik Språng56e611b2020-02-06 16:10:08835void RTPSenderVideo::MaybeUpdateCurrentPlayoutDelay(
836 const RTPVideoHeader& header) {
Johannes Kron4da8e4c2020-09-22 14:23:30837 VideoPlayoutDelay requested_delay =
838 forced_playout_delay_.value_or(header.playout_delay);
839
840 if (IsNoopDelay(requested_delay)) {
Erik Språng56e611b2020-02-06 16:10:08841 return;
842 }
843
Erik Språng56e611b2020-02-06 16:10:08844 if (requested_delay.min_ms > PlayoutDelayLimits::kMaxMs ||
845 requested_delay.max_ms > PlayoutDelayLimits::kMaxMs) {
846 RTC_DLOG(LS_ERROR)
847 << "Requested playout delay values out of range, ignored";
848 return;
849 }
850 if (requested_delay.max_ms != -1 &&
851 requested_delay.min_ms > requested_delay.max_ms) {
852 RTC_DLOG(LS_ERROR) << "Requested playout delay values out of order";
853 return;
854 }
855
856 if (!playout_delay_pending_) {
857 current_playout_delay_ = requested_delay;
858 playout_delay_pending_ = true;
859 return;
860 }
861
862 if ((requested_delay.min_ms == -1 ||
863 requested_delay.min_ms == current_playout_delay_.min_ms) &&
864 (requested_delay.max_ms == -1 ||
865 requested_delay.max_ms == current_playout_delay_.max_ms)) {
866 // No change, ignore.
867 return;
868 }
869
870 if (requested_delay.min_ms == -1) {
871 RTC_DCHECK_GE(requested_delay.max_ms, 0);
872 requested_delay.min_ms =
873 std::min(current_playout_delay_.min_ms, requested_delay.max_ms);
874 }
875 if (requested_delay.max_ms == -1) {
876 requested_delay.max_ms =
877 std::max(current_playout_delay_.max_ms, requested_delay.min_ms);
878 }
879
880 current_playout_delay_ = requested_delay;
881 playout_delay_pending_ = true;
882}
883
pbos@webrtc.orgd900e8b2013-07-03 15:12:26884} // namespace webrtc