blob: 0f64ab144991be53e187d677520fcb8bd0fca000 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:251/*
mflodman@webrtc.orgc80d9d92012-02-06 10:11:252 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:253 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 04:47:3111#include "modules/video_coding/frame_buffer.h"
niklase@google.com470e71d2011-07-07 08:21:2512
pbos@webrtc.org12dc1a32013-08-05 16:22:5313#include <assert.h>
niklase@google.com470e71d2011-07-07 08:21:2514#include <string.h>
15
Yves Gerey3e707812018-11-28 15:47:4916#include "api/video/encoded_image.h"
17#include "api/video/video_timing.h"
Yves Gerey3e707812018-11-28 15:47:4918#include "modules/video_coding/include/video_codec_interface.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3119#include "modules/video_coding/packet.h"
20#include "rtc_base/checks.h"
21#include "rtc_base/logging.h"
22#include "rtc_base/trace_event.h"
agalusza@google.comd818dcb2013-07-29 21:48:1123
niklase@google.com470e71d2011-07-07 08:21:2524namespace webrtc {
25
stefan@webrtc.orgc3d89102011-09-08 06:50:2826VCMFrameBuffer::VCMFrameBuffer()
philipel9d3ab612015-12-21 12:12:3927 : _state(kStateEmpty), _nackCount(0), _latestPacketTimeMs(-1) {}
niklase@google.com470e71d2011-07-07 08:21:2528
philipel9d3ab612015-12-21 12:12:3929VCMFrameBuffer::~VCMFrameBuffer() {}
niklase@google.com470e71d2011-07-07 08:21:2530
Niels Möller87e2d782019-03-07 09:18:2331webrtc::VideoFrameType VCMFrameBuffer::FrameType() const {
philipel9d3ab612015-12-21 12:12:3932 return _sessionInfo.FrameType();
niklase@google.com470e71d2011-07-07 08:21:2533}
34
philipel9d3ab612015-12-21 12:12:3935int32_t VCMFrameBuffer::GetLowSeqNum() const {
36 return _sessionInfo.LowSequenceNumber();
niklase@google.com470e71d2011-07-07 08:21:2537}
38
philipel9d3ab612015-12-21 12:12:3939int32_t VCMFrameBuffer::GetHighSeqNum() const {
40 return _sessionInfo.HighSequenceNumber();
niklase@google.com470e71d2011-07-07 08:21:2541}
42
stefan@webrtc.orgffd28f92011-10-19 15:55:3943int VCMFrameBuffer::PictureId() const {
44 return _sessionInfo.PictureId();
45}
46
mikhal@webrtc.orgf5ee1dc2011-12-08 19:04:4747int VCMFrameBuffer::TemporalId() const {
48 return _sessionInfo.TemporalId();
49}
50
henrik.lundin@webrtc.orgeda86dc2011-12-13 14:11:0651bool VCMFrameBuffer::LayerSync() const {
52 return _sessionInfo.LayerSync();
53}
54
mikhal@webrtc.orgf5ee1dc2011-12-08 19:04:4755int VCMFrameBuffer::Tl0PicId() const {
56 return _sessionInfo.Tl0PicId();
57}
58
stefana669a3a2016-10-06 12:04:5259std::vector<NaluInfo> VCMFrameBuffer::GetNaluInfos() const {
60 return _sessionInfo.GetNaluInfos();
61}
62
asapersson9a4cd872015-10-23 07:27:1463void VCMFrameBuffer::SetGofInfo(const GofInfoVP9& gof_info, size_t idx) {
tommidb23ea62017-03-03 15:21:1864 TRACE_EVENT0("webrtc", "VCMFrameBuffer::SetGofInfo");
asapersson9a4cd872015-10-23 07:27:1465 _sessionInfo.SetGofInfo(gof_info, idx);
66 // TODO(asapersson): Consider adding hdr->VP9.ref_picture_id for testing.
67 _codecSpecificInfo.codecSpecific.VP9.temporal_idx =
68 gof_info.temporal_idx[idx];
69 _codecSpecificInfo.codecSpecific.VP9.temporal_up_switch =
70 gof_info.temporal_up_switch[idx];
71}
72
niklase@google.com470e71d2011-07-07 08:21:2573// Insert packet
Jonas Olssona4d87372019-07-05 17:08:3374VCMFrameBufferEnum VCMFrameBuffer::InsertPacket(const VCMPacket& packet,
75 int64_t timeInMs,
76 const FrameData& frame_data) {
tommidb23ea62017-03-03 15:21:1877 TRACE_EVENT0("webrtc", "VCMFrameBuffer::InsertPacket");
philipel9d3ab612015-12-21 12:12:3978 assert(!(NULL == packet.dataPtr && packet.sizeBytes > 0));
79 if (packet.dataPtr != NULL) {
80 _payloadType = packet.payloadType;
81 }
82
83 if (kStateEmpty == _state) {
84 // First packet (empty and/or media) inserted into this frame.
85 // store some info and set some initial values.
Niels Möller23775882018-08-16 08:24:1286 SetTimestamp(packet.timestamp);
philipel9d3ab612015-12-21 12:12:3987 // We only take the ntp timestamp of the first packet of a frame.
88 ntp_time_ms_ = packet.ntp_time_ms_;
Niels Möllerd5e02f02019-02-20 12:12:2189 _codec = packet.codec();
Niels Möllerabbc50e2019-04-24 07:41:1690 if (packet.video_header.frame_type != VideoFrameType::kEmptyFrame) {
philipel9d3ab612015-12-21 12:12:3991 // first media packet
92 SetState(kStateIncomplete);
niklase@google.com470e71d2011-07-07 08:21:2593 }
philipel9d3ab612015-12-21 12:12:3994 }
niklase@google.com470e71d2011-07-07 08:21:2595
Niels Möllerd60e32a2020-09-23 12:26:4296 size_t oldSize = encoded_image_buffer_ ? encoded_image_buffer_->size() : 0;
philipel9d3ab612015-12-21 12:12:3997 uint32_t requiredSizeBytes =
Niels Möllerf0eee002018-11-28 15:31:2998 size() + packet.sizeBytes +
Niels Möller009ab3c2019-03-08 10:26:5899 (packet.insertStartCode ? kH264StartCodeLengthBytes : 0);
Niels Möllerd60e32a2020-09-23 12:26:42100 if (requiredSizeBytes > oldSize) {
Niels Möller24871e42019-01-17 10:31:13101 const uint8_t* prevBuffer = data();
philipel9d3ab612015-12-21 12:12:39102 const uint32_t increments =
103 requiredSizeBytes / kBufferIncStepSizeBytes +
104 (requiredSizeBytes % kBufferIncStepSizeBytes > 0);
Niels Möllerd60e32a2020-09-23 12:26:42105 const uint32_t newSize = oldSize + increments * kBufferIncStepSizeBytes;
philipel9d3ab612015-12-21 12:12:39106 if (newSize > kMaxJBFrameSizeBytes) {
Mirko Bonadei675513b2017-11-09 10:09:25107 RTC_LOG(LS_ERROR) << "Failed to insert packet due to frame being too "
108 "big.";
philipel9d3ab612015-12-21 12:12:39109 return kSizeError;
niklase@google.com470e71d2011-07-07 08:21:25110 }
Niels Möller2449d7a2019-09-30 08:18:16111 if (data() == nullptr) {
112 encoded_image_buffer_ = EncodedImageBuffer::Create(newSize);
113 SetEncodedData(encoded_image_buffer_);
114 set_size(0);
115 } else {
116 RTC_CHECK(encoded_image_buffer_ != nullptr);
117 RTC_DCHECK_EQ(encoded_image_buffer_->data(), data());
118 encoded_image_buffer_->Realloc(newSize);
119 }
Niels Möller24871e42019-01-17 10:31:13120 _sessionInfo.UpdateDataPointers(prevBuffer, data());
philipel9d3ab612015-12-21 12:12:39121 }
niklase@google.com470e71d2011-07-07 08:21:25122
Niels Möllerd5e02f02019-02-20 12:12:21123 if (packet.width() > 0 && packet.height() > 0) {
124 _encodedWidth = packet.width();
125 _encodedHeight = packet.height();
philipel9d3ab612015-12-21 12:12:39126 }
henrik.lundin@webrtc.org473bac82011-08-17 09:47:33127
philipel9d3ab612015-12-21 12:12:39128 // Don't copy payload specific data for empty packets (e.g padding packets).
129 if (packet.sizeBytes > 0)
isheriff6b4b5f32016-06-08 07:24:21130 CopyCodecSpecific(&packet.video_header);
stefan@webrtc.org3417eb42013-05-21 15:25:53131
Niels Möller08ae7ce2020-09-23 13:58:12132 int retVal = _sessionInfo.InsertPacket(
133 packet, encoded_image_buffer_ ? encoded_image_buffer_->data() : nullptr,
134 frame_data);
philipel9d3ab612015-12-21 12:12:39135 if (retVal == -1) {
136 return kSizeError;
137 } else if (retVal == -2) {
138 return kDuplicatePacket;
139 } else if (retVal == -3) {
140 return kOutOfBoundsPacket;
141 }
Niels Möller77536a22019-01-15 07:50:01142 // update size
143 set_size(size() + static_cast<uint32_t>(retVal));
henrik.lundin@webrtc.org473bac82011-08-17 09:47:33144
philipel9d3ab612015-12-21 12:12:39145 _latestPacketTimeMs = timeInMs;
niklase@google.com470e71d2011-07-07 08:21:25146
philipel9d3ab612015-12-21 12:12:39147 // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
148 // ts_126114v120700p.pdf Section 7.4.5.
149 // The MTSI client shall add the payload bytes as defined in this clause
150 // onto the last RTP packet in each group of packets which make up a key
151 // frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265
152 // (HEVC)).
153 if (packet.markerBit) {
perkj414dda12016-07-04 08:45:23154 rotation_ = packet.video_header.rotation;
ilnik00d802b2017-04-11 17:34:31155 content_type_ = packet.video_header.content_type;
Ilya Nikolaevskiyb6c462d2018-06-05 13:21:32156 if (packet.video_header.video_timing.flags != VideoSendTiming::kInvalid) {
ilnik04f4d122017-06-19 14:18:55157 timing_.encode_start_ms =
158 ntp_time_ms_ + packet.video_header.video_timing.encode_start_delta_ms;
159 timing_.encode_finish_ms =
160 ntp_time_ms_ +
161 packet.video_header.video_timing.encode_finish_delta_ms;
162 timing_.packetization_finish_ms =
163 ntp_time_ms_ +
164 packet.video_header.video_timing.packetization_finish_delta_ms;
165 timing_.pacer_exit_ms =
166 ntp_time_ms_ + packet.video_header.video_timing.pacer_exit_delta_ms;
167 timing_.network_timestamp_ms =
168 ntp_time_ms_ +
Danil Chapovalov996eb9e2017-10-30 16:14:41169 packet.video_header.video_timing.network_timestamp_delta_ms;
ilnik04f4d122017-06-19 14:18:55170 timing_.network2_timestamp_ms =
171 ntp_time_ms_ +
Danil Chapovalov996eb9e2017-10-30 16:14:41172 packet.video_header.video_timing.network2_timestamp_delta_ms;
ilnik04f4d122017-06-19 14:18:55173 }
sprangba050a62017-08-18 09:51:12174 timing_.flags = packet.video_header.video_timing.flags;
philipel9d3ab612015-12-21 12:12:39175 }
niklase@google.com470e71d2011-07-07 08:21:25176
Niels Möllerd5e02f02019-02-20 12:12:21177 if (packet.is_first_packet_in_frame()) {
isheriff6b4b5f32016-06-08 07:24:21178 playout_delay_ = packet.video_header.playout_delay;
179 }
180
philipel9d3ab612015-12-21 12:12:39181 if (_sessionInfo.complete()) {
182 SetState(kStateComplete);
183 return kCompleteSession;
philipel9d3ab612015-12-21 12:12:39184 }
185 return kIncomplete;
niklase@google.com470e71d2011-07-07 08:21:25186}
187
philipel9d3ab612015-12-21 12:12:39188int64_t VCMFrameBuffer::LatestPacketTimeMs() const {
tommidb23ea62017-03-03 15:21:18189 TRACE_EVENT0("webrtc", "VCMFrameBuffer::LatestPacketTimeMs");
philipel9d3ab612015-12-21 12:12:39190 return _latestPacketTimeMs;
niklase@google.com470e71d2011-07-07 08:21:25191}
192
philipel9d3ab612015-12-21 12:12:39193void VCMFrameBuffer::IncrementNackCount() {
tommidb23ea62017-03-03 15:21:18194 TRACE_EVENT0("webrtc", "VCMFrameBuffer::IncrementNackCount");
philipel9d3ab612015-12-21 12:12:39195 _nackCount++;
niklase@google.com470e71d2011-07-07 08:21:25196}
197
philipel9d3ab612015-12-21 12:12:39198int16_t VCMFrameBuffer::GetNackCount() const {
tommidb23ea62017-03-03 15:21:18199 TRACE_EVENT0("webrtc", "VCMFrameBuffer::GetNackCount");
philipel9d3ab612015-12-21 12:12:39200 return _nackCount;
niklase@google.com470e71d2011-07-07 08:21:25201}
202
philipel9d3ab612015-12-21 12:12:39203bool VCMFrameBuffer::HaveFirstPacket() const {
tommidb23ea62017-03-03 15:21:18204 TRACE_EVENT0("webrtc", "VCMFrameBuffer::HaveFirstPacket");
philipel9d3ab612015-12-21 12:12:39205 return _sessionInfo.HaveFirstPacket();
stefan@webrtc.org885cd132013-04-16 09:38:26206}
207
philipel9d3ab612015-12-21 12:12:39208int VCMFrameBuffer::NumPackets() const {
tommidb23ea62017-03-03 15:21:18209 TRACE_EVENT0("webrtc", "VCMFrameBuffer::NumPackets");
philipel9d3ab612015-12-21 12:12:39210 return _sessionInfo.NumPackets();
agalusza@google.comd818dcb2013-07-29 21:48:11211}
212
philipel9d3ab612015-12-21 12:12:39213void VCMFrameBuffer::Reset() {
tommidb23ea62017-03-03 15:21:18214 TRACE_EVENT0("webrtc", "VCMFrameBuffer::Reset");
Niels Möller77536a22019-01-15 07:50:01215 set_size(0);
philipel9d3ab612015-12-21 12:12:39216 _sessionInfo.Reset();
217 _payloadType = 0;
218 _nackCount = 0;
219 _latestPacketTimeMs = -1;
220 _state = kStateEmpty;
221 VCMEncodedFrame::Reset();
niklase@google.com470e71d2011-07-07 08:21:25222}
223
niklase@google.com470e71d2011-07-07 08:21:25224// Set state of frame
philipel9d3ab612015-12-21 12:12:39225void VCMFrameBuffer::SetState(VCMFrameBufferStateEnum state) {
tommidb23ea62017-03-03 15:21:18226 TRACE_EVENT0("webrtc", "VCMFrameBuffer::SetState");
philipel9d3ab612015-12-21 12:12:39227 if (_state == state) {
228 return;
229 }
230 switch (state) {
niklase@google.com470e71d2011-07-07 08:21:25231 case kStateIncomplete:
philipel9d3ab612015-12-21 12:12:39232 // we can go to this state from state kStateEmpty
233 assert(_state == kStateEmpty);
niklase@google.com470e71d2011-07-07 08:21:25234
philipel9d3ab612015-12-21 12:12:39235 // Do nothing, we received a packet
236 break;
niklase@google.com470e71d2011-07-07 08:21:25237
238 case kStateComplete:
Niels Möller375b3462019-01-10 14:35:56239 assert(_state == kStateEmpty || _state == kStateIncomplete);
niklase@google.com470e71d2011-07-07 08:21:25240
philipel9d3ab612015-12-21 12:12:39241 break;
niklase@google.com470e71d2011-07-07 08:21:25242
243 case kStateEmpty:
philipel9d3ab612015-12-21 12:12:39244 // Should only be set to empty through Reset().
245 assert(false);
246 break;
philipel9d3ab612015-12-21 12:12:39247 }
248 _state = state;
niklase@google.com470e71d2011-07-07 08:21:25249}
250
niklase@google.com470e71d2011-07-07 08:21:25251// Get current state of frame
philipel9d3ab612015-12-21 12:12:39252VCMFrameBufferStateEnum VCMFrameBuffer::GetState() const {
253 return _state;
niklase@google.com470e71d2011-07-07 08:21:25254}
255
philipel9d3ab612015-12-21 12:12:39256void VCMFrameBuffer::PrepareForDecode(bool continuous) {
tommidb23ea62017-03-03 15:21:18257 TRACE_EVENT0("webrtc", "VCMFrameBuffer::PrepareForDecode");
philipel9d3ab612015-12-21 12:12:39258 size_t bytes_removed = _sessionInfo.MakeDecodable();
Niels Möller77536a22019-01-15 07:50:01259 set_size(size() - bytes_removed);
philipel9d3ab612015-12-21 12:12:39260 // Transfer frame information to EncodedFrame and create any codec
261 // specific information.
262 _frameType = _sessionInfo.FrameType();
philipel9d3ab612015-12-21 12:12:39263 _missingFrame = !continuous;
niklase@google.com470e71d2011-07-07 08:21:25264}
265
agalusza@google.comd818dcb2013-07-29 21:48:11266} // namespace webrtc