blob: ebf109eecb283ccefd04746d24a16ed81a467ab8 [file] [log] [blame]
Danil Chapovalov096a46f2019-11-28 12:42:501/*
2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10#include "modules/rtp_rtcp/source/rtp_packetizer_av1.h"
11
12#include <stddef.h>
13#include <stdint.h>
14
15#include <algorithm>
16
17#include "api/array_view.h"
18#include "api/video/video_frame_type.h"
Danil Chapovalove6b3f482023-01-13 11:33:4119#include "modules/rtp_rtcp/source/leb128.h"
Danil Chapovalov096a46f2019-11-28 12:42:5020#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
21#include "rtc_base/byte_buffer.h"
22#include "rtc_base/checks.h"
23#include "rtc_base/logging.h"
24
25namespace webrtc {
26namespace {
Danil Chapovalov096a46f2019-11-28 12:42:5027constexpr int kAggregationHeaderSize = 1;
28// when there are 3 or less OBU (fragments) in a packet, size of the last one
29// can be omited.
30constexpr int kMaxNumObusToOmitSize = 3;
31constexpr uint8_t kObuSizePresentBit = 0b0'0000'010;
32constexpr int kObuTypeSequenceHeader = 1;
33constexpr int kObuTypeTemporalDelimiter = 2;
34constexpr int kObuTypeTileList = 8;
35constexpr int kObuTypePadding = 15;
36
Philipp Hanckeacfd2792024-04-30 14:09:3737// Overhead introduced by "even distribution" of packet sizes.
38constexpr size_t kBytesOverheadEvenDistribution = 1;
39// Experimentally determined minimum amount of potential savings per packet to
40// make "even distribution" of packet sizes worthwhile.
41constexpr size_t kMinBytesSavedPerPacketWithEvenDistribution = 10;
42
Danil Chapovalov096a46f2019-11-28 12:42:5043bool ObuHasExtension(uint8_t obu_header) {
44 return obu_header & 0b0'0000'100;
45}
46
47bool ObuHasSize(uint8_t obu_header) {
48 return obu_header & kObuSizePresentBit;
49}
50
51int ObuType(uint8_t obu_header) {
52 return (obu_header & 0b0'1111'000) >> 3;
53}
54
Artem Titov913cfa72021-07-28 21:57:3355// Given `remaining_bytes` free bytes left in a packet, returns max size of an
Danil Chapovalov096a46f2019-11-28 12:42:5056// OBU fragment that can fit into the packet.
57// i.e. MaxFragmentSize + Leb128Size(MaxFragmentSize) <= remaining_bytes.
58int MaxFragmentSize(int remaining_bytes) {
59 if (remaining_bytes <= 1) {
60 return 0;
61 }
62 for (int i = 1;; ++i) {
63 if (remaining_bytes < (1 << 7 * i) + i) {
64 return remaining_bytes - i;
65 }
66 }
67}
68
69} // namespace
70
71RtpPacketizerAv1::RtpPacketizerAv1(rtc::ArrayView<const uint8_t> payload,
72 RtpPacketizer::PayloadSizeLimits limits,
Danil Chapovalov62a9a322020-11-11 15:15:0773 VideoFrameType frame_type,
Philipp Hanckeacfd2792024-04-30 14:09:3774 bool is_last_frame_in_picture,
75 bool even_distribution)
Danil Chapovalov096a46f2019-11-28 12:42:5076 : frame_type_(frame_type),
77 obus_(ParseObus(payload)),
Philipp Hanckeacfd2792024-04-30 14:09:3778 packets_(even_distribution ? PacketizeAboutEqually(obus_, limits)
79 : Packetize(obus_, limits)),
Danil Chapovalov62a9a322020-11-11 15:15:0780 is_last_frame_in_picture_(is_last_frame_in_picture) {}
Danil Chapovalov096a46f2019-11-28 12:42:5081
82std::vector<RtpPacketizerAv1::Obu> RtpPacketizerAv1::ParseObus(
83 rtc::ArrayView<const uint8_t> payload) {
84 std::vector<Obu> result;
Harald Alvestrand572502c2023-11-22 10:19:0685 rtc::ByteBufferReader payload_reader(payload);
Danil Chapovalov096a46f2019-11-28 12:42:5086 while (payload_reader.Length() > 0) {
87 Obu obu;
88 payload_reader.ReadUInt8(&obu.header);
89 obu.size = 1;
90 if (ObuHasExtension(obu.header)) {
91 if (payload_reader.Length() == 0) {
92 RTC_DLOG(LS_ERROR) << "Malformed AV1 input: expected extension_header, "
93 "no more bytes in the buffer. Offset: "
94 << (payload.size() - payload_reader.Length());
95 return {};
96 }
97 payload_reader.ReadUInt8(&obu.extension_header);
98 ++obu.size;
99 }
100 if (!ObuHasSize(obu.header)) {
101 obu.payload = rtc::MakeArrayView(
102 reinterpret_cast<const uint8_t*>(payload_reader.Data()),
103 payload_reader.Length());
104 payload_reader.Consume(payload_reader.Length());
105 } else {
106 uint64_t size = 0;
107 if (!payload_reader.ReadUVarint(&size) ||
108 size > payload_reader.Length()) {
109 RTC_DLOG(LS_ERROR) << "Malformed AV1 input: declared size " << size
110 << " is larger than remaining buffer size "
111 << payload_reader.Length();
112 return {};
113 }
114 obu.payload = rtc::MakeArrayView(
115 reinterpret_cast<const uint8_t*>(payload_reader.Data()), size);
116 payload_reader.Consume(size);
117 }
118 obu.size += obu.payload.size();
119 // Skip obus that shouldn't be transfered over rtp.
120 int obu_type = ObuType(obu.header);
121 if (obu_type != kObuTypeTemporalDelimiter && //
122 obu_type != kObuTypeTileList && //
123 obu_type != kObuTypePadding) {
124 result.push_back(obu);
125 }
126 }
127 return result;
128}
129
130int RtpPacketizerAv1::AdditionalBytesForPreviousObuElement(
131 const Packet& packet) {
132 if (packet.packet_size == 0) {
133 // Packet is still empty => no last OBU element, no need to reserve space
134 // for it.
135 return 0;
136 }
137 if (packet.num_obu_elements > kMaxNumObusToOmitSize) {
138 // There is so many obu elements in the packet, all of them must be
139 // prepended with the length field. That imply space for the length of the
140 // last obu element is already reserved.
141 return 0;
142 }
143 // No space was reserved for length field of the last OBU element, but that
144 // element becoming non-last, so it now requires explicit length field.
145 // Calculate how many bytes are needed to store the length in leb128 format.
146 return Leb128Size(packet.last_obu_size);
147}
148
149std::vector<RtpPacketizerAv1::Packet> RtpPacketizerAv1::Packetize(
150 rtc::ArrayView<const Obu> obus,
151 PayloadSizeLimits limits) {
152 std::vector<Packet> packets;
153 if (obus.empty()) {
154 return packets;
155 }
156 // Ignore certian edge cases where packets should be very small. They are
157 // inpractical but adds complexity to handle.
158 if (limits.max_payload_len - limits.last_packet_reduction_len < 3 ||
159 limits.max_payload_len - limits.first_packet_reduction_len < 3) {
160 RTC_DLOG(LS_ERROR) << "Failed to packetize AV1 frame: requested packet "
161 "size is unreasonable small.";
162 return packets;
163 }
164 // Aggregation header is present in all packets.
165 limits.max_payload_len -= kAggregationHeaderSize;
166
167 // Assemble packets. Push to current packet as much as it can hold before
168 // considering next one. That would normally cause uneven distribution across
169 // packets, specifically last one would be generally smaller.
170 packets.emplace_back(/*first_obu_index=*/0);
171 int packet_remaining_bytes =
172 limits.max_payload_len - limits.first_packet_reduction_len;
173 for (size_t obu_index = 0; obu_index < obus.size(); ++obu_index) {
174 const bool is_last_obu = obu_index == obus.size() - 1;
175 const Obu& obu = obus[obu_index];
176
Artem Titov913cfa72021-07-28 21:57:33177 // Putting `obu` into the last packet would make last obu element stored in
Danil Chapovalov096a46f2019-11-28 12:42:50178 // that packet not last. All not last OBU elements must be prepend with the
179 // element length. AdditionalBytesForPreviousObuElement calculates how many
180 // bytes are needed to store that length.
181 int previous_obu_extra_size =
182 AdditionalBytesForPreviousObuElement(packets.back());
183 int min_required_size =
184 packets.back().num_obu_elements >= kMaxNumObusToOmitSize ? 2 : 1;
185 if (packet_remaining_bytes < previous_obu_extra_size + min_required_size) {
186 // Start a new packet.
187 packets.emplace_back(/*first_obu_index=*/obu_index);
188 packet_remaining_bytes = limits.max_payload_len;
189 previous_obu_extra_size = 0;
190 }
191 Packet& packet = packets.back();
192 // Start inserting current obu into the packet.
193 packet.packet_size += previous_obu_extra_size;
194 packet_remaining_bytes -= previous_obu_extra_size;
195 packet.num_obu_elements++;
196
197 bool must_write_obu_element_size =
198 packet.num_obu_elements > kMaxNumObusToOmitSize;
199 // Can fit all of the obu into the packet?
200 int required_bytes = obu.size;
201 if (must_write_obu_element_size) {
202 required_bytes += Leb128Size(obu.size);
203 }
204 int available_bytes = packet_remaining_bytes;
205 if (is_last_obu) {
206 // If this packet would be the last packet, available size is smaller.
207 if (packets.size() == 1) {
208 available_bytes += limits.first_packet_reduction_len;
209 available_bytes -= limits.single_packet_reduction_len;
210 } else {
211 available_bytes -= limits.last_packet_reduction_len;
212 }
213 }
214 if (required_bytes <= available_bytes) {
215 // Insert the obu into the packet unfragmented.
216 packet.last_obu_size = obu.size;
217 packet.packet_size += required_bytes;
218 packet_remaining_bytes -= required_bytes;
219 continue;
220 }
221
222 // Fragment the obu.
223 int max_first_fragment_size = must_write_obu_element_size
224 ? MaxFragmentSize(packet_remaining_bytes)
225 : packet_remaining_bytes;
226 // Because available_bytes might be different than
227 // packet_remaining_bytes it might happen that max_first_fragment_size >=
Artem Titov913cfa72021-07-28 21:57:33228 // obu.size. Also, since checks above verified `obu` should not be put
229 // completely into the `packet`, leave at least 1 byte for later packet.
Danil Chapovalov096a46f2019-11-28 12:42:50230 int first_fragment_size = std::min(obu.size - 1, max_first_fragment_size);
231 if (first_fragment_size == 0) {
232 // Rather than writing 0-size element at the tail of the packet,
Artem Titov913cfa72021-07-28 21:57:33233 // 'uninsert' the `obu` from the `packet`.
Danil Chapovalov096a46f2019-11-28 12:42:50234 packet.num_obu_elements--;
235 packet.packet_size -= previous_obu_extra_size;
236 } else {
237 packet.packet_size += first_fragment_size;
238 if (must_write_obu_element_size) {
239 packet.packet_size += Leb128Size(first_fragment_size);
240 }
241 packet.last_obu_size = first_fragment_size;
242 }
243
244 // Add middle fragments that occupy all of the packet.
245 // These are easy because
246 // - one obu per packet imply no need to store the size of the obu.
247 // - this packets are nor the first nor the last packets of the frame, so
248 // packet capacity is always limits.max_payload_len.
249 int obu_offset;
250 for (obu_offset = first_fragment_size;
251 obu_offset + limits.max_payload_len < obu.size;
252 obu_offset += limits.max_payload_len) {
253 packets.emplace_back(/*first_obu_index=*/obu_index);
254 Packet& packet = packets.back();
255 packet.num_obu_elements = 1;
256 packet.first_obu_offset = obu_offset;
257 int middle_fragment_size = limits.max_payload_len;
258 packet.last_obu_size = middle_fragment_size;
259 packet.packet_size = middle_fragment_size;
260 }
261
262 // Add the last fragment of the obu.
263 int last_fragment_size = obu.size - obu_offset;
264 // Check for corner case where last fragment of the last obu is too large
265 // to fit into last packet, but may fully fit into semi-last packet.
266 if (is_last_obu &&
267 last_fragment_size >
268 limits.max_payload_len - limits.last_packet_reduction_len) {
269 // Split last fragments into two.
270 RTC_DCHECK_GE(last_fragment_size, 2);
271 // Try to even packet sizes rather than payload sizes across the last
272 // two packets.
273 int semi_last_fragment_size =
274 (last_fragment_size + limits.last_packet_reduction_len) / 2;
275 // But leave at least one payload byte for the last packet to avoid
276 // weird scenarios where size of the fragment is zero and rtp payload has
277 // nothing except for an aggregation header.
278 if (semi_last_fragment_size >= last_fragment_size) {
279 semi_last_fragment_size = last_fragment_size - 1;
280 }
281 last_fragment_size -= semi_last_fragment_size;
282
283 packets.emplace_back(/*first_obu_index=*/obu_index);
284 Packet& packet = packets.back();
285 packet.num_obu_elements = 1;
286 packet.first_obu_offset = obu_offset;
287 packet.last_obu_size = semi_last_fragment_size;
288 packet.packet_size = semi_last_fragment_size;
289 obu_offset += semi_last_fragment_size;
290 }
291 packets.emplace_back(/*first_obu_index=*/obu_index);
292 Packet& last_packet = packets.back();
293 last_packet.num_obu_elements = 1;
294 last_packet.first_obu_offset = obu_offset;
295 last_packet.last_obu_size = last_fragment_size;
296 last_packet.packet_size = last_fragment_size;
297 packet_remaining_bytes = limits.max_payload_len - last_fragment_size;
298 }
299 return packets;
300}
301
Philipp Hanckeacfd2792024-04-30 14:09:37302std::vector<RtpPacketizerAv1::Packet> RtpPacketizerAv1::PacketizeAboutEqually(
303 rtc::ArrayView<const Obu> obus,
304 PayloadSizeLimits limits) {
305 std::vector<Packet> packets = Packetize(obus, limits);
306 if (packets.size() <= 1) {
307 return packets;
308 }
309 size_t packet_index = 0;
310 size_t packet_size_left_unused = 0;
311 for (const auto& packet : packets) {
312 // Every packet has to have an aggregation header of size
313 // kAggregationHeaderSize.
314 int available_bytes = limits.max_payload_len - kAggregationHeaderSize;
315
316 if (packet_index == 0) {
317 available_bytes -= limits.first_packet_reduction_len;
318 } else if (packet_index == packets.size() - 1) {
319 available_bytes -= limits.last_packet_reduction_len;
320 }
321 if (available_bytes >= packet.packet_size) {
322 packet_size_left_unused += (available_bytes - packet.packet_size);
323 }
324 packet_index++;
325 }
326 if (packet_size_left_unused >
327 packets.size() * kMinBytesSavedPerPacketWithEvenDistribution) {
328 // Calculate new limits with a reduced max_payload_len.
329 size_t size_reduction = packet_size_left_unused / packets.size();
330 RTC_DCHECK_GT(limits.max_payload_len, size_reduction);
331 RTC_DCHECK_GT(size_reduction, kBytesOverheadEvenDistribution);
332 limits.max_payload_len -= (size_reduction - kBytesOverheadEvenDistribution);
333 if (limits.max_payload_len - limits.last_packet_reduction_len < 3 ||
334 limits.max_payload_len - limits.first_packet_reduction_len < 3) {
335 return packets;
336 }
337 std::vector<Packet> packets_even = Packetize(obus, limits);
338 // The number of packets should not change in the second pass. If it does,
339 // conservatively return the original packets.
340 if (packets_even.size() == packets.size()) {
341 return packets_even;
342 }
343 RTC_LOG(LS_WARNING) << "AV1 even distribution caused a regression in "
344 "number of packets from "
345 << packets.size() << " to " << packets_even.size();
346 }
347 return packets;
348}
349
Danil Chapovalov096a46f2019-11-28 12:42:50350uint8_t RtpPacketizerAv1::AggregationHeader() const {
351 const Packet& packet = packets_[packet_index_];
352 uint8_t aggregation_header = 0;
353
354 // Set Z flag: first obu element is continuation of the previous OBU.
355 bool first_obu_element_is_fragment = packet.first_obu_offset > 0;
356 if (first_obu_element_is_fragment)
357 aggregation_header |= (1 << 7);
358
359 // Set Y flag: last obu element will be continuated in the next packet.
360 int last_obu_offset =
361 packet.num_obu_elements == 1 ? packet.first_obu_offset : 0;
362 bool last_obu_is_fragment =
363 last_obu_offset + packet.last_obu_size <
364 obus_[packet.first_obu + packet.num_obu_elements - 1].size;
365 if (last_obu_is_fragment)
366 aggregation_header |= (1 << 6);
367
368 // Set W field: number of obu elements in the packet (when not too large).
369 if (packet.num_obu_elements <= kMaxNumObusToOmitSize)
370 aggregation_header |= packet.num_obu_elements << 4;
371
372 // Set N flag: beginning of a new coded video sequence.
373 // Encoder may produce key frame without a sequence header, thus double check
374 // incoming frame includes the sequence header. Since Temporal delimiter is
375 // already filtered out, sequence header should be the first obu when present.
376 if (frame_type_ == VideoFrameType::kVideoFrameKey && packet_index_ == 0 &&
377 ObuType(obus_.front().header) == kObuTypeSequenceHeader) {
378 aggregation_header |= (1 << 3);
379 }
380 return aggregation_header;
381}
382
383bool RtpPacketizerAv1::NextPacket(RtpPacketToSend* packet) {
384 if (packet_index_ >= packets_.size()) {
385 return false;
386 }
387 const Packet& next_packet = packets_[packet_index_];
388
389 RTC_DCHECK_GT(next_packet.num_obu_elements, 0);
390 RTC_DCHECK_LT(next_packet.first_obu_offset,
391 obus_[next_packet.first_obu].size);
392 RTC_DCHECK_LE(
393 next_packet.last_obu_size,
394 obus_[next_packet.first_obu + next_packet.num_obu_elements - 1].size);
395
396 uint8_t* const rtp_payload =
397 packet->AllocatePayload(kAggregationHeaderSize + next_packet.packet_size);
398 uint8_t* write_at = rtp_payload;
399
400 *write_at++ = AggregationHeader();
401
402 int obu_offset = next_packet.first_obu_offset;
403 // Store all OBU elements except the last one.
404 for (int i = 0; i < next_packet.num_obu_elements - 1; ++i) {
405 const Obu& obu = obus_[next_packet.first_obu + i];
406 size_t fragment_size = obu.size - obu_offset;
407 write_at += WriteLeb128(fragment_size, write_at);
408 if (obu_offset == 0) {
409 *write_at++ = obu.header & ~kObuSizePresentBit;
410 }
411 if (obu_offset <= 1 && ObuHasExtension(obu.header)) {
412 *write_at++ = obu.extension_header;
413 }
414 int payload_offset =
415 std::max(0, obu_offset - (ObuHasExtension(obu.header) ? 2 : 1));
416 size_t payload_size = obu.payload.size() - payload_offset;
Byoungchan Lee8c725f32022-09-13 08:53:49417 if (!obu.payload.empty() && payload_size > 0) {
418 memcpy(write_at, obu.payload.data() + payload_offset, payload_size);
419 }
Danil Chapovalov096a46f2019-11-28 12:42:50420 write_at += payload_size;
421 // All obus are stored from the beginning, except, may be, the first one.
422 obu_offset = 0;
423 }
424 // Store the last OBU element.
425 const Obu& last_obu =
426 obus_[next_packet.first_obu + next_packet.num_obu_elements - 1];
427 int fragment_size = next_packet.last_obu_size;
428 RTC_DCHECK_GT(fragment_size, 0);
429 if (next_packet.num_obu_elements > kMaxNumObusToOmitSize) {
430 write_at += WriteLeb128(fragment_size, write_at);
431 }
432 if (obu_offset == 0 && fragment_size > 0) {
433 *write_at++ = last_obu.header & ~kObuSizePresentBit;
434 --fragment_size;
435 }
436 if (obu_offset <= 1 && ObuHasExtension(last_obu.header) &&
437 fragment_size > 0) {
438 *write_at++ = last_obu.extension_header;
439 --fragment_size;
440 }
441 RTC_DCHECK_EQ(write_at - rtp_payload + fragment_size,
442 kAggregationHeaderSize + next_packet.packet_size);
443 int payload_offset =
444 std::max(0, obu_offset - (ObuHasExtension(last_obu.header) ? 2 : 1));
445 memcpy(write_at, last_obu.payload.data() + payload_offset, fragment_size);
446 write_at += fragment_size;
447
448 RTC_DCHECK_EQ(write_at - rtp_payload,
449 kAggregationHeaderSize + next_packet.packet_size);
450
451 ++packet_index_;
Danil Chapovalov62a9a322020-11-11 15:15:07452 bool is_last_packet_in_frame = packet_index_ == packets_.size();
453 packet->SetMarker(is_last_packet_in_frame && is_last_frame_in_picture_);
Danil Chapovalov096a46f2019-11-28 12:42:50454 return true;
455}
456
457} // namespace webrtc