blob: e67ac664b1f85dbc403f363a3476b786db0ed4c9 [file] [log] [blame]
philipel02447bc2016-05-13 13:01:031/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 04:47:3111#include "modules/video_coding/rtp_frame_reference_finder.h"
philipel02447bc2016-05-13 13:01:0312
13#include <algorithm>
14#include <limits>
15
Danil Chapovalov64f1f3f2020-01-16 13:41:1016#include "absl/base/macros.h"
philipel1a4746a2018-07-09 13:52:2917#include "absl/types/variant.h"
Mirko Bonadei92ea95e2017-09-15 04:47:3118#include "modules/video_coding/frame_object.h"
19#include "modules/video_coding/packet_buffer.h"
20#include "rtc_base/checks.h"
21#include "rtc_base/logging.h"
philipel02447bc2016-05-13 13:01:0322
23namespace webrtc {
24namespace video_coding {
25
26RtpFrameReferenceFinder::RtpFrameReferenceFinder(
27 OnCompleteFrameCallback* frame_callback)
philipel7acc4a42019-09-26 09:25:5228 : RtpFrameReferenceFinder(frame_callback, 0) {}
29
30RtpFrameReferenceFinder::RtpFrameReferenceFinder(
31 OnCompleteFrameCallback* frame_callback,
32 int64_t picture_id_offset)
philipel02447bc2016-05-13 13:01:0333 : last_picture_id_(-1),
philipel02447bc2016-05-13 13:01:0334 current_ss_idx_(0),
philipel463d3012016-09-09 10:32:4435 cleared_to_seq_num_(-1),
philipel7acc4a42019-09-26 09:25:5236 frame_callback_(frame_callback),
37 picture_id_offset_(picture_id_offset) {}
philipel02447bc2016-05-13 13:01:0338
Mirko Bonadei8fdcac32018-08-28 14:30:1839RtpFrameReferenceFinder::~RtpFrameReferenceFinder() = default;
40
philipel02447bc2016-05-13 13:01:0341void RtpFrameReferenceFinder::ManageFrame(
42 std::unique_ptr<RtpFrameObject> frame) {
philipel463d3012016-09-09 10:32:4443 // If we have cleared past this frame, drop it.
44 if (cleared_to_seq_num_ != -1 &&
45 AheadOf<uint16_t>(cleared_to_seq_num_, frame->first_seq_num())) {
46 return;
47 }
48
philipelafcf7f52017-04-26 15:17:3549 FrameDecision decision = ManageFrameInternal(frame.get());
50
51 switch (decision) {
52 case kStash:
53 if (stashed_frames_.size() > kMaxStashedFrames)
54 stashed_frames_.pop_back();
55 stashed_frames_.push_front(std::move(frame));
56 break;
57 case kHandOff:
philipel7acc4a42019-09-26 09:25:5258 HandOffFrame(std::move(frame));
philipelafcf7f52017-04-26 15:17:3559 RetryStashedFrames();
60 break;
61 case kDrop:
62 break;
63 }
64}
65
66void RtpFrameReferenceFinder::RetryStashedFrames() {
67 bool complete_frame = false;
68 do {
69 complete_frame = false;
70 for (auto frame_it = stashed_frames_.begin();
71 frame_it != stashed_frames_.end();) {
72 FrameDecision decision = ManageFrameInternal(frame_it->get());
73
74 switch (decision) {
75 case kStash:
76 ++frame_it;
77 break;
78 case kHandOff:
79 complete_frame = true;
philipel7acc4a42019-09-26 09:25:5280 HandOffFrame(std::move(*frame_it));
Danil Chapovalov64f1f3f2020-01-16 13:41:1081 ABSL_FALLTHROUGH_INTENDED;
philipelafcf7f52017-04-26 15:17:3582 case kDrop:
83 frame_it = stashed_frames_.erase(frame_it);
84 }
85 }
86 } while (complete_frame);
87}
88
philipel7acc4a42019-09-26 09:25:5289void RtpFrameReferenceFinder::HandOffFrame(
90 std::unique_ptr<RtpFrameObject> frame) {
91 frame->id.picture_id += picture_id_offset_;
92 for (size_t i = 0; i < frame->num_references; ++i) {
93 frame->references[i] += picture_id_offset_;
94 }
95
96 frame_callback_->OnCompleteFrame(std::move(frame));
97}
98
philipelafcf7f52017-04-26 15:17:3599RtpFrameReferenceFinder::FrameDecision
100RtpFrameReferenceFinder::ManageFrameInternal(RtpFrameObject* frame) {
Danil Chapovalov26b4cb32020-01-29 10:21:38101 if (const absl::optional<RTPVideoHeader::GenericDescriptorInfo>&
102 generic_descriptor = frame->GetRtpVideoHeader().generic) {
philipel2837edc2018-10-02 11:55:47103 return ManageFrameGeneric(frame, *generic_descriptor);
philipeldabfcae2018-09-25 10:54:37104 }
105
philipel02447bc2016-05-13 13:01:03106 switch (frame->codec_type()) {
philipel02447bc2016-05-13 13:01:03107 case kVideoCodecVP8:
philipelafcf7f52017-04-26 15:17:35108 return ManageFrameVp8(frame);
philipel02447bc2016-05-13 13:01:03109 case kVideoCodecVP9:
philipelafcf7f52017-04-26 15:17:35110 return ManageFrameVp9(frame);
Johnny Leebc7f41b2019-05-01 18:41:32111 case kVideoCodecH264:
112 return ManageFrameH264(frame);
Danil Chapovalovb6bf0b22020-01-28 17:36:57113 case kVideoCodecGeneric:
114 if (auto* generic_header = absl::get_if<RTPVideoHeaderLegacyGeneric>(
115 &frame->GetRtpVideoHeader().video_type_header)) {
116 return ManageFramePidOrSeqNum(frame, generic_header->picture_id);
117 }
118 ABSL_FALLTHROUGH_INTENDED;
119 default:
120 return ManageFramePidOrSeqNum(frame, kNoPictureId);
philipel02447bc2016-05-13 13:01:03121 }
122}
123
philipel9b2ce6b2016-07-05 12:04:46124void RtpFrameReferenceFinder::PaddingReceived(uint16_t seq_num) {
philipel9b2ce6b2016-07-05 12:04:46125 auto clean_padding_to =
126 stashed_padding_.lower_bound(seq_num - kMaxPaddingAge);
127 stashed_padding_.erase(stashed_padding_.begin(), clean_padding_to);
128 stashed_padding_.insert(seq_num);
129 UpdateLastPictureIdWithPadding(seq_num);
130 RetryStashedFrames();
131}
132
philipel463d3012016-09-09 10:32:44133void RtpFrameReferenceFinder::ClearTo(uint16_t seq_num) {
philipel463d3012016-09-09 10:32:44134 cleared_to_seq_num_ = seq_num;
135
136 auto it = stashed_frames_.begin();
137 while (it != stashed_frames_.end()) {
138 if (AheadOf<uint16_t>(cleared_to_seq_num_, (*it)->first_seq_num())) {
139 it = stashed_frames_.erase(it);
140 } else {
141 ++it;
142 }
143 }
144}
145
philipel9b2ce6b2016-07-05 12:04:46146void RtpFrameReferenceFinder::UpdateLastPictureIdWithPadding(uint16_t seq_num) {
147 auto gop_seq_num_it = last_seq_num_gop_.upper_bound(seq_num);
148
149 // If this padding packet "belongs" to a group of pictures that we don't track
150 // anymore, do nothing.
151 if (gop_seq_num_it == last_seq_num_gop_.begin())
152 return;
153 --gop_seq_num_it;
154
155 // Calculate the next contiuous sequence number and search for it in
156 // the padding packets we have stashed.
157 uint16_t next_seq_num_with_padding = gop_seq_num_it->second.second + 1;
158 auto padding_seq_num_it =
159 stashed_padding_.lower_bound(next_seq_num_with_padding);
160
161 // While there still are padding packets and those padding packets are
162 // continuous, then advance the "last-picture-id-with-padding" and remove
163 // the stashed padding packet.
164 while (padding_seq_num_it != stashed_padding_.end() &&
165 *padding_seq_num_it == next_seq_num_with_padding) {
166 gop_seq_num_it->second.second = next_seq_num_with_padding;
167 ++next_seq_num_with_padding;
168 padding_seq_num_it = stashed_padding_.erase(padding_seq_num_it);
169 }
philipel41bb7922017-02-20 15:53:23170
171 // In the case where the stream has been continuous without any new keyframes
172 // for a while there is a risk that new frames will appear to be older than
173 // the keyframe they belong to due to wrapping sequence number. In order
174 // to prevent this we advance the picture id of the keyframe every so often.
175 if (ForwardDiff(gop_seq_num_it->first, seq_num) > 10000) {
176 RTC_DCHECK_EQ(1ul, last_seq_num_gop_.size());
177 last_seq_num_gop_[seq_num] = gop_seq_num_it->second;
178 last_seq_num_gop_.erase(gop_seq_num_it);
179 }
philipel9b2ce6b2016-07-05 12:04:46180}
181
philipelafcf7f52017-04-26 15:17:35182RtpFrameReferenceFinder::FrameDecision
philipeldabfcae2018-09-25 10:54:37183RtpFrameReferenceFinder::ManageFrameGeneric(
184 RtpFrameObject* frame,
Danil Chapovalov26b4cb32020-01-29 10:21:38185 const RTPVideoHeader::GenericDescriptorInfo& descriptor) {
186 frame->id.picture_id = descriptor.frame_id;
187 frame->id.spatial_layer = descriptor.spatial_index;
philipel2837edc2018-10-02 11:55:47188
Danil Chapovalov26b4cb32020-01-29 10:21:38189 if (EncodedFrame::kMaxFrameReferences < descriptor.dependencies.size()) {
philipeldabfcae2018-09-25 10:54:37190 RTC_LOG(LS_WARNING) << "Too many dependencies in generic descriptor.";
191 return kDrop;
192 }
193
Danil Chapovalov26b4cb32020-01-29 10:21:38194 frame->num_references = descriptor.dependencies.size();
195 for (size_t i = 0; i < descriptor.dependencies.size(); ++i)
196 frame->references[i] = descriptor.dependencies[i];
philipeldabfcae2018-09-25 10:54:37197
198 return kHandOff;
199}
200
201RtpFrameReferenceFinder::FrameDecision
202RtpFrameReferenceFinder::ManageFramePidOrSeqNum(RtpFrameObject* frame,
203 int picture_id) {
philipel647998c2016-06-03 16:40:16204 // If |picture_id| is specified then we use that to set the frame references,
205 // otherwise we use sequence number.
206 if (picture_id != kNoPictureId) {
Danil Chapovalov95cb56b2020-01-30 14:03:26207 frame->id.picture_id = unwrapper_.Unwrap(picture_id & 0x7FFF);
Niels Möller8f7ce222019-03-21 14:43:58208 frame->num_references =
209 frame->frame_type() == VideoFrameType::kVideoFrameKey ? 0 : 1;
philipel0fa82a62018-03-19 14:34:53210 frame->references[0] = frame->id.picture_id - 1;
philipelafcf7f52017-04-26 15:17:35211 return kHandOff;
philipel647998c2016-06-03 16:40:16212 }
213
Niels Möller8f7ce222019-03-21 14:43:58214 if (frame->frame_type() == VideoFrameType::kVideoFrameKey) {
philipel9b2ce6b2016-07-05 12:04:46215 last_seq_num_gop_.insert(std::make_pair(
216 frame->last_seq_num(),
217 std::make_pair(frame->last_seq_num(), frame->last_seq_num())));
218 }
philipel02447bc2016-05-13 13:01:03219
220 // We have received a frame but not yet a keyframe, stash this frame.
philipelafcf7f52017-04-26 15:17:35221 if (last_seq_num_gop_.empty())
222 return kStash;
philipel02447bc2016-05-13 13:01:03223
224 // Clean up info for old keyframes but make sure to keep info
225 // for the last keyframe.
226 auto clean_to = last_seq_num_gop_.lower_bound(frame->last_seq_num() - 100);
philipel41bb7922017-02-20 15:53:23227 for (auto it = last_seq_num_gop_.begin();
228 it != clean_to && last_seq_num_gop_.size() > 1;) {
229 it = last_seq_num_gop_.erase(it);
230 }
philipel02447bc2016-05-13 13:01:03231
232 // Find the last sequence number of the last frame for the keyframe
233 // that this frame indirectly references.
234 auto seq_num_it = last_seq_num_gop_.upper_bound(frame->last_seq_num());
philipel9b2ce6b2016-07-05 12:04:46235 if (seq_num_it == last_seq_num_gop_.begin()) {
Mirko Bonadei675513b2017-11-09 10:09:25236 RTC_LOG(LS_WARNING) << "Generic frame with packet range ["
237 << frame->first_seq_num() << ", "
238 << frame->last_seq_num()
239 << "] has no GoP, dropping frame.";
philipelafcf7f52017-04-26 15:17:35240 return kDrop;
philipel9b2ce6b2016-07-05 12:04:46241 }
philipel02447bc2016-05-13 13:01:03242 seq_num_it--;
243
244 // Make sure the packet sequence numbers are continuous, otherwise stash
245 // this frame.
philipel9b2ce6b2016-07-05 12:04:46246 uint16_t last_picture_id_gop = seq_num_it->second.first;
247 uint16_t last_picture_id_with_padding_gop = seq_num_it->second.second;
Niels Möller8f7ce222019-03-21 14:43:58248 if (frame->frame_type() == VideoFrameType::kVideoFrameDelta) {
philipel9b2ce6b2016-07-05 12:04:46249 uint16_t prev_seq_num = frame->first_seq_num() - 1;
philipelafcf7f52017-04-26 15:17:35250
251 if (prev_seq_num != last_picture_id_with_padding_gop)
252 return kStash;
philipel02447bc2016-05-13 13:01:03253 }
254
255 RTC_DCHECK(AheadOrAt(frame->last_seq_num(), seq_num_it->first));
256
257 // Since keyframes can cause reordering we can't simply assign the
258 // picture id according to some incrementing counter.
philipel0fa82a62018-03-19 14:34:53259 frame->id.picture_id = frame->last_seq_num();
Niels Möller8f7ce222019-03-21 14:43:58260 frame->num_references =
261 frame->frame_type() == VideoFrameType::kVideoFrameDelta;
philipeldabfcae2018-09-25 10:54:37262 frame->references[0] = rtp_seq_num_unwrapper_.Unwrap(last_picture_id_gop);
philipel0fa82a62018-03-19 14:34:53263 if (AheadOf<uint16_t>(frame->id.picture_id, last_picture_id_gop)) {
264 seq_num_it->second.first = frame->id.picture_id;
265 seq_num_it->second.second = frame->id.picture_id;
philipel9b2ce6b2016-07-05 12:04:46266 }
philipel02447bc2016-05-13 13:01:03267
philipel0fa82a62018-03-19 14:34:53268 UpdateLastPictureIdWithPadding(frame->id.picture_id);
philipeldabfcae2018-09-25 10:54:37269 frame->id.picture_id = rtp_seq_num_unwrapper_.Unwrap(frame->id.picture_id);
philipelafcf7f52017-04-26 15:17:35270 return kHandOff;
philipel02447bc2016-05-13 13:01:03271}
272
philipelafcf7f52017-04-26 15:17:35273RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameVp8(
274 RtpFrameObject* frame) {
Johannes Krona3705562019-08-26 14:37:11275 const RTPVideoHeader& video_header = frame->GetRtpVideoHeader();
philipel1a4746a2018-07-09 13:52:29276 const RTPVideoHeaderVP8& codec_header =
Danil Chapovalov95cb56b2020-01-30 14:03:26277 absl::get<RTPVideoHeaderVP8>(video_header.video_type_header);
philipel02447bc2016-05-13 13:01:03278
279 if (codec_header.pictureId == kNoPictureId ||
280 codec_header.temporalIdx == kNoTemporalIdx ||
281 codec_header.tl0PicIdx == kNoTl0PicIdx) {
Mirko Bonadei05cf6be2019-01-31 20:38:12282 return ManageFramePidOrSeqNum(frame, codec_header.pictureId);
philipel02447bc2016-05-13 13:01:03283 }
284
Ilya Nikolaevskiy7a709c02020-01-17 15:40:02285 // Protect against corrupted packets with arbitrary large temporal idx.
286 if (codec_header.temporalIdx >= kMaxTemporalLayers)
287 return kDrop;
288
Danil Chapovalov95cb56b2020-01-30 14:03:26289 frame->id.picture_id = codec_header.pictureId & 0x7FFF;
philipel02447bc2016-05-13 13:01:03290
philipel02447bc2016-05-13 13:01:03291 if (last_picture_id_ == -1)
philipel0fa82a62018-03-19 14:34:53292 last_picture_id_ = frame->id.picture_id;
philipel02447bc2016-05-13 13:01:03293
Danil Chapovalov545c53e2020-01-31 17:08:54294 // Clean up info about not yet received frames that are too old.
295 uint16_t old_picture_id =
296 Subtract<kPicIdLength>(frame->id.picture_id, kMaxNotYetReceivedFrames);
297 auto clean_frames_to = not_yet_received_frames_.lower_bound(old_picture_id);
298 not_yet_received_frames_.erase(not_yet_received_frames_.begin(),
299 clean_frames_to);
300 // Avoid re-adding picture ids that were just erased.
301 if (AheadOf<uint16_t, kPicIdLength>(old_picture_id, last_picture_id_)) {
302 last_picture_id_ = old_picture_id;
303 }
philipel02447bc2016-05-13 13:01:03304 // Find if there has been a gap in fully received frames and save the picture
305 // id of those frames in |not_yet_received_frames_|.
philipel0fa82a62018-03-19 14:34:53306 if (AheadOf<uint16_t, kPicIdLength>(frame->id.picture_id, last_picture_id_)) {
philipel9bd1d662017-07-14 11:52:01307 do {
philipel02447bc2016-05-13 13:01:03308 last_picture_id_ = Add<kPicIdLength>(last_picture_id_, 1);
philipel9bd1d662017-07-14 11:52:01309 not_yet_received_frames_.insert(last_picture_id_);
philipel0fa82a62018-03-19 14:34:53310 } while (last_picture_id_ != frame->id.picture_id);
philipel02447bc2016-05-13 13:01:03311 }
312
Danil Chapovalov95cb56b2020-01-30 14:03:26313 int64_t unwrapped_tl0 = tl0_unwrapper_.Unwrap(codec_header.tl0PicIdx & 0xFF);
philipel57ec6852018-07-03 16:09:32314
philipel02447bc2016-05-13 13:01:03315 // Clean up info for base layers that are too old.
philipel57ec6852018-07-03 16:09:32316 int64_t old_tl0_pic_idx = unwrapped_tl0 - kMaxLayerInfo;
philipel02447bc2016-05-13 13:01:03317 auto clean_layer_info_to = layer_info_.lower_bound(old_tl0_pic_idx);
318 layer_info_.erase(layer_info_.begin(), clean_layer_info_to);
319
Niels Möller8f7ce222019-03-21 14:43:58320 if (frame->frame_type() == VideoFrameType::kVideoFrameKey) {
Danil Chapovalov95cb56b2020-01-30 14:03:26321 if (codec_header.temporalIdx != 0) {
322 return kDrop;
323 }
philipel02447bc2016-05-13 13:01:03324 frame->num_references = 0;
philipel57ec6852018-07-03 16:09:32325 layer_info_[unwrapped_tl0].fill(-1);
326 UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx);
philipelafcf7f52017-04-26 15:17:35327 return kHandOff;
philipel02447bc2016-05-13 13:01:03328 }
329
philipel57ec6852018-07-03 16:09:32330 auto layer_info_it = layer_info_.find(
331 codec_header.temporalIdx == 0 ? unwrapped_tl0 - 1 : unwrapped_tl0);
philipel02447bc2016-05-13 13:01:03332
333 // If we don't have the base layer frame yet, stash this frame.
philipelafcf7f52017-04-26 15:17:35334 if (layer_info_it == layer_info_.end())
335 return kStash;
philipel02447bc2016-05-13 13:01:03336
337 // A non keyframe base layer frame has been received, copy the layer info
338 // from the previous base layer frame and set a reference to the previous
339 // base layer frame.
340 if (codec_header.temporalIdx == 0) {
341 layer_info_it =
philipel57ec6852018-07-03 16:09:32342 layer_info_.emplace(unwrapped_tl0, layer_info_it->second).first;
philipel02447bc2016-05-13 13:01:03343 frame->num_references = 1;
344 frame->references[0] = layer_info_it->second[0];
philipel57ec6852018-07-03 16:09:32345 UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx);
philipelafcf7f52017-04-26 15:17:35346 return kHandOff;
philipel02447bc2016-05-13 13:01:03347 }
348
349 // Layer sync frame, this frame only references its base layer frame.
350 if (codec_header.layerSync) {
351 frame->num_references = 1;
352 frame->references[0] = layer_info_it->second[0];
353
philipel57ec6852018-07-03 16:09:32354 UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx);
philipelafcf7f52017-04-26 15:17:35355 return kHandOff;
philipel02447bc2016-05-13 13:01:03356 }
357
358 // Find all references for this frame.
359 frame->num_references = 0;
360 for (uint8_t layer = 0; layer <= codec_header.temporalIdx; ++layer) {
philipeld268d6f2016-09-15 11:43:13361 // If we have not yet received a previous frame on this temporal layer,
362 // stash this frame.
philipelafcf7f52017-04-26 15:17:35363 if (layer_info_it->second[layer] == -1)
364 return kStash;
philipel02447bc2016-05-13 13:01:03365
philipel86b92e02016-10-24 14:11:53366 // If the last frame on this layer is ahead of this frame it means that
367 // a layer sync frame has been received after this frame for the same
368 // base layer frame, drop this frame.
369 if (AheadOf<uint16_t, kPicIdLength>(layer_info_it->second[layer],
philipel0fa82a62018-03-19 14:34:53370 frame->id.picture_id)) {
philipelafcf7f52017-04-26 15:17:35371 return kDrop;
philipel86b92e02016-10-24 14:11:53372 }
373
philipel02447bc2016-05-13 13:01:03374 // If we have not yet received a frame between this frame and the referenced
375 // frame then we have to wait for that frame to be completed first.
376 auto not_received_frame_it =
377 not_yet_received_frames_.upper_bound(layer_info_it->second[layer]);
378 if (not_received_frame_it != not_yet_received_frames_.end() &&
philipel0fa82a62018-03-19 14:34:53379 AheadOf<uint16_t, kPicIdLength>(frame->id.picture_id,
philipel02447bc2016-05-13 13:01:03380 *not_received_frame_it)) {
philipelafcf7f52017-04-26 15:17:35381 return kStash;
philipel02447bc2016-05-13 13:01:03382 }
383
philipel0fa82a62018-03-19 14:34:53384 if (!(AheadOf<uint16_t, kPicIdLength>(frame->id.picture_id,
philipel57f19cc02017-03-07 11:54:05385 layer_info_it->second[layer]))) {
philipel0fa82a62018-03-19 14:34:53386 RTC_LOG(LS_WARNING) << "Frame with picture id " << frame->id.picture_id
Mirko Bonadei675513b2017-11-09 10:09:25387 << " and packet range [" << frame->first_seq_num()
388 << ", " << frame->last_seq_num()
389 << "] already received, "
Jonas Olssonb2b20312020-01-14 11:11:31390 " dropping frame.";
philipelafcf7f52017-04-26 15:17:35391 return kDrop;
philipel57f19cc02017-03-07 11:54:05392 }
393
philipel02447bc2016-05-13 13:01:03394 ++frame->num_references;
395 frame->references[layer] = layer_info_it->second[layer];
396 }
397
philipel57ec6852018-07-03 16:09:32398 UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx);
philipelafcf7f52017-04-26 15:17:35399 return kHandOff;
philipel02447bc2016-05-13 13:01:03400}
401
philipel57ec6852018-07-03 16:09:32402void RtpFrameReferenceFinder::UpdateLayerInfoVp8(RtpFrameObject* frame,
403 int64_t unwrapped_tl0,
404 uint8_t temporal_idx) {
405 auto layer_info_it = layer_info_.find(unwrapped_tl0);
philipel02447bc2016-05-13 13:01:03406
407 // Update this layer info and newer.
408 while (layer_info_it != layer_info_.end()) {
philipel57ec6852018-07-03 16:09:32409 if (layer_info_it->second[temporal_idx] != -1 &&
410 AheadOf<uint16_t, kPicIdLength>(layer_info_it->second[temporal_idx],
philipel0fa82a62018-03-19 14:34:53411 frame->id.picture_id)) {
philipel02447bc2016-05-13 13:01:03412 // The frame was not newer, then no subsequent layer info have to be
413 // update.
414 break;
415 }
416
philipel57ec6852018-07-03 16:09:32417 layer_info_it->second[temporal_idx] = frame->id.picture_id;
418 ++unwrapped_tl0;
419 layer_info_it = layer_info_.find(unwrapped_tl0);
philipel02447bc2016-05-13 13:01:03420 }
philipel0fa82a62018-03-19 14:34:53421 not_yet_received_frames_.erase(frame->id.picture_id);
philipel02447bc2016-05-13 13:01:03422
philipelafcf7f52017-04-26 15:17:35423 UnwrapPictureIds(frame);
philipel02447bc2016-05-13 13:01:03424}
425
philipelafcf7f52017-04-26 15:17:35426RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameVp9(
427 RtpFrameObject* frame) {
Johannes Krona3705562019-08-26 14:37:11428 const RTPVideoHeader& video_header = frame->GetRtpVideoHeader();
philipel1a4746a2018-07-09 13:52:29429 const RTPVideoHeaderVP9& codec_header =
Danil Chapovalov95cb56b2020-01-30 14:03:26430 absl::get<RTPVideoHeaderVP9>(video_header.video_type_header);
philipel02447bc2016-05-13 13:01:03431
philipel647998c2016-06-03 16:40:16432 if (codec_header.picture_id == kNoPictureId ||
Sergey Silkind34a1882018-08-20 14:46:05433 codec_header.temporal_idx == kNoTemporalIdx) {
Mirko Bonadei05cf6be2019-01-31 20:38:12434 return ManageFramePidOrSeqNum(frame, codec_header.picture_id);
philipel02447bc2016-05-13 13:01:03435 }
436
Ilya Nikolaevskiy7a709c02020-01-17 15:40:02437 // Protect against corrupted packets with arbitrary large temporal idx.
Ilya Nikolaevskiyef0d76a2020-02-05 17:01:31438 if (codec_header.temporal_idx >= kMaxTemporalLayers ||
439 codec_header.spatial_idx >= kMaxSpatialLayers)
Ilya Nikolaevskiy7a709c02020-01-17 15:40:02440 return kDrop;
441
philipel0fa82a62018-03-19 14:34:53442 frame->id.spatial_layer = codec_header.spatial_idx;
philipel02447bc2016-05-13 13:01:03443 frame->inter_layer_predicted = codec_header.inter_layer_predicted;
Danil Chapovalov95cb56b2020-01-30 14:03:26444 frame->id.picture_id = codec_header.picture_id & 0x7FFF;
philipel02447bc2016-05-13 13:01:03445
philipel02447bc2016-05-13 13:01:03446 if (last_picture_id_ == -1)
philipel0fa82a62018-03-19 14:34:53447 last_picture_id_ = frame->id.picture_id;
philipel02447bc2016-05-13 13:01:03448
449 if (codec_header.flexible_mode) {
Danil Chapovalov95cb56b2020-01-30 14:03:26450 if (codec_header.num_ref_pics > EncodedFrame::kMaxFrameReferences) {
451 return kDrop;
452 }
philipel02447bc2016-05-13 13:01:03453 frame->num_references = codec_header.num_ref_pics;
454 for (size_t i = 0; i < frame->num_references; ++i) {
philipel0fa82a62018-03-19 14:34:53455 frame->references[i] = Subtract<kPicIdLength>(frame->id.picture_id,
456 codec_header.pid_diff[i]);
philipel02447bc2016-05-13 13:01:03457 }
458
philipelafcf7f52017-04-26 15:17:35459 UnwrapPictureIds(frame);
460 return kHandOff;
philipel02447bc2016-05-13 13:01:03461 }
462
Sergey Silkind34a1882018-08-20 14:46:05463 if (codec_header.tl0_pic_idx == kNoTl0PicIdx) {
464 RTC_LOG(LS_WARNING) << "TL0PICIDX is expected to be present in "
465 "non-flexible mode.";
466 return kDrop;
467 }
468
philipel15643602018-05-03 14:14:13469 GofInfo* info;
Danil Chapovalov95cb56b2020-01-30 14:03:26470 int64_t unwrapped_tl0 =
471 tl0_unwrapper_.Unwrap(codec_header.tl0_pic_idx & 0xFF);
philipel02447bc2016-05-13 13:01:03472 if (codec_header.ss_data_available) {
philipel02447bc2016-05-13 13:01:03473 if (codec_header.temporal_idx != 0) {
philipel15643602018-05-03 14:14:13474 RTC_LOG(LS_WARNING) << "Received scalability structure on a non base "
475 "layer frame. Scalability structure ignored.";
philipel02447bc2016-05-13 13:01:03476 } else {
Sergey Silkin2f864fb2018-09-07 09:49:38477 if (codec_header.gof.num_frames_in_gof > kMaxVp9FramesInGof) {
Natalie Silvanovich3ea3e302018-05-16 18:03:12478 return kDrop;
479 }
480
Ilya Nikolaevskiyef0d76a2020-02-05 17:01:31481 for (size_t i = 0; i < codec_header.gof.num_frames_in_gof; ++i) {
482 if (codec_header.gof.num_ref_pics[i] > kMaxVp9RefPics) {
483 return kDrop;
484 }
485 }
486
Sergey Silkin2f864fb2018-09-07 09:49:38487 GofInfoVP9 gof = codec_header.gof;
488 if (gof.num_frames_in_gof == 0) {
489 RTC_LOG(LS_WARNING) << "Number of frames in GOF is zero. Assume "
490 "that stream has only one temporal layer.";
491 gof.SetGofInfoVP9(kTemporalStructureMode1);
492 }
493
494 current_ss_idx_ = Add<kMaxGofSaved>(current_ss_idx_, 1);
495 scalability_structures_[current_ss_idx_] = gof;
philipel0fa82a62018-03-19 14:34:53496 scalability_structures_[current_ss_idx_].pid_start = frame->id.picture_id;
philipel57ec6852018-07-03 16:09:32497 gof_info_.emplace(unwrapped_tl0,
philipel15643602018-05-03 14:14:13498 GofInfo(&scalability_structures_[current_ss_idx_],
499 frame->id.picture_id));
philipel02447bc2016-05-13 13:01:03500 }
philipel15643602018-05-03 14:14:13501
philipel57ec6852018-07-03 16:09:32502 const auto gof_info_it = gof_info_.find(unwrapped_tl0);
philipel15643602018-05-03 14:14:13503 if (gof_info_it == gof_info_.end())
504 return kStash;
505
506 info = &gof_info_it->second;
507
Niels Möller8f7ce222019-03-21 14:43:58508 if (frame->frame_type() == VideoFrameType::kVideoFrameKey) {
philipel15643602018-05-03 14:14:13509 frame->num_references = 0;
510 FrameReceivedVp9(frame->id.picture_id, info);
511 UnwrapPictureIds(frame);
512 return kHandOff;
513 }
Niels Möller8f7ce222019-03-21 14:43:58514 } else if (frame->frame_type() == VideoFrameType::kVideoFrameKey) {
Ilya Nikolaevskiy5546aef2018-12-04 14:54:52515 if (frame->id.spatial_layer == 0) {
philipel15643602018-05-03 14:14:13516 RTC_LOG(LS_WARNING) << "Received keyframe without scalability structure";
517 return kDrop;
518 }
Ilya Nikolaevskiy5546aef2018-12-04 14:54:52519 const auto gof_info_it = gof_info_.find(unwrapped_tl0);
520 if (gof_info_it == gof_info_.end())
521 return kStash;
philipel15643602018-05-03 14:14:13522
Ilya Nikolaevskiy5546aef2018-12-04 14:54:52523 info = &gof_info_it->second;
524
Niels Möller8f7ce222019-03-21 14:43:58525 if (frame->frame_type() == VideoFrameType::kVideoFrameKey) {
Ilya Nikolaevskiy5546aef2018-12-04 14:54:52526 frame->num_references = 0;
527 FrameReceivedVp9(frame->id.picture_id, info);
528 UnwrapPictureIds(frame);
529 return kHandOff;
530 }
531 } else {
philipel57ec6852018-07-03 16:09:32532 auto gof_info_it = gof_info_.find(
533 (codec_header.temporal_idx == 0) ? unwrapped_tl0 - 1 : unwrapped_tl0);
philipel15643602018-05-03 14:14:13534
535 // Gof info for this frame is not available yet, stash this frame.
536 if (gof_info_it == gof_info_.end())
537 return kStash;
538
539 if (codec_header.temporal_idx == 0) {
philipel57ec6852018-07-03 16:09:32540 gof_info_it = gof_info_
541 .emplace(unwrapped_tl0, GofInfo(gof_info_it->second.gof,
542 frame->id.picture_id))
543 .first;
philipel15643602018-05-03 14:14:13544 }
545
546 info = &gof_info_it->second;
philipel02447bc2016-05-13 13:01:03547 }
548
549 // Clean up info for base layers that are too old.
philipel57ec6852018-07-03 16:09:32550 int64_t old_tl0_pic_idx = unwrapped_tl0 - kMaxGofSaved;
philipel02447bc2016-05-13 13:01:03551 auto clean_gof_info_to = gof_info_.lower_bound(old_tl0_pic_idx);
552 gof_info_.erase(gof_info_.begin(), clean_gof_info_to);
553
philipel0fa82a62018-03-19 14:34:53554 FrameReceivedVp9(frame->id.picture_id, info);
philipel02447bc2016-05-13 13:01:03555
556 // Make sure we don't miss any frame that could potentially have the
557 // up switch flag set.
philipel0fa82a62018-03-19 14:34:53558 if (MissingRequiredFrameVp9(frame->id.picture_id, *info))
philipelafcf7f52017-04-26 15:17:35559 return kStash;
philipel02447bc2016-05-13 13:01:03560
philipel15643602018-05-03 14:14:13561 if (codec_header.temporal_up_switch)
562 up_switch_.emplace(frame->id.picture_id, codec_header.temporal_idx);
philipel02447bc2016-05-13 13:01:03563
564 // Clean out old info about up switch frames.
philipel0fa82a62018-03-19 14:34:53565 uint16_t old_picture_id = Subtract<kPicIdLength>(frame->id.picture_id, 50);
philipel02447bc2016-05-13 13:01:03566 auto up_switch_erase_to = up_switch_.lower_bound(old_picture_id);
567 up_switch_.erase(up_switch_.begin(), up_switch_erase_to);
568
philipelc9b27d52016-07-15 13:50:27569 size_t diff = ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start,
philipel0fa82a62018-03-19 14:34:53570 frame->id.picture_id);
philipelc9b27d52016-07-15 13:50:27571 size_t gof_idx = diff % info->gof->num_frames_in_gof;
philipel02447bc2016-05-13 13:01:03572
Danil Chapovalova1187022020-02-03 09:16:48573 if (info->gof->num_ref_pics[gof_idx] > EncodedFrame::kMaxFrameReferences) {
574 return kDrop;
575 }
philipel02447bc2016-05-13 13:01:03576 // Populate references according to the scalability structure.
philipelc9b27d52016-07-15 13:50:27577 frame->num_references = info->gof->num_ref_pics[gof_idx];
philipel02447bc2016-05-13 13:01:03578 for (size_t i = 0; i < frame->num_references; ++i) {
philipelc9b27d52016-07-15 13:50:27579 frame->references[i] = Subtract<kPicIdLength>(
philipel0fa82a62018-03-19 14:34:53580 frame->id.picture_id, info->gof->pid_diff[gof_idx][i]);
philipel02447bc2016-05-13 13:01:03581
582 // If this is a reference to a frame earlier than the last up switch point,
583 // then ignore this reference.
philipel0fa82a62018-03-19 14:34:53584 if (UpSwitchInIntervalVp9(frame->id.picture_id, codec_header.temporal_idx,
philipel02447bc2016-05-13 13:01:03585 frame->references[i])) {
586 --frame->num_references;
587 }
588 }
589
Ilya Nikolaevskiy2ec0c652019-01-18 10:56:48590 // Override GOF references.
591 if (!codec_header.inter_pic_predicted) {
592 frame->num_references = 0;
593 }
594
philipelafcf7f52017-04-26 15:17:35595 UnwrapPictureIds(frame);
596 return kHandOff;
philipel02447bc2016-05-13 13:01:03597}
598
599bool RtpFrameReferenceFinder::MissingRequiredFrameVp9(uint16_t picture_id,
philipelc9b27d52016-07-15 13:50:27600 const GofInfo& info) {
601 size_t diff =
602 ForwardDiff<uint16_t, kPicIdLength>(info.gof->pid_start, picture_id);
603 size_t gof_idx = diff % info.gof->num_frames_in_gof;
604 size_t temporal_idx = info.gof->temporal_idx[gof_idx];
philipel02447bc2016-05-13 13:01:03605
philipela157e082018-05-02 13:19:01606 if (temporal_idx >= kMaxTemporalLayers) {
Jonas Olssonb2b20312020-01-14 11:11:31607 RTC_LOG(LS_WARNING) << "At most " << kMaxTemporalLayers
608 << " temporal "
609 "layers are supported.";
philipela157e082018-05-02 13:19:01610 return true;
611 }
612
philipel02447bc2016-05-13 13:01:03613 // For every reference this frame has, check if there is a frame missing in
614 // the interval (|ref_pid|, |picture_id|) in any of the lower temporal
615 // layers. If so, we are missing a required frame.
philipelc9b27d52016-07-15 13:50:27616 uint8_t num_references = info.gof->num_ref_pics[gof_idx];
philipel02447bc2016-05-13 13:01:03617 for (size_t i = 0; i < num_references; ++i) {
618 uint16_t ref_pid =
philipelc9b27d52016-07-15 13:50:27619 Subtract<kPicIdLength>(picture_id, info.gof->pid_diff[gof_idx][i]);
philipel02447bc2016-05-13 13:01:03620 for (size_t l = 0; l < temporal_idx; ++l) {
621 auto missing_frame_it = missing_frames_for_layer_[l].lower_bound(ref_pid);
622 if (missing_frame_it != missing_frames_for_layer_[l].end() &&
623 AheadOf<uint16_t, kPicIdLength>(picture_id, *missing_frame_it)) {
624 return true;
625 }
626 }
627 }
628 return false;
629}
630
631void RtpFrameReferenceFinder::FrameReceivedVp9(uint16_t picture_id,
philipelc9b27d52016-07-15 13:50:27632 GofInfo* info) {
633 int last_picture_id = info->last_picture_id;
philipel459f4e32018-03-02 09:55:12634 size_t gof_size = std::min(info->gof->num_frames_in_gof, kMaxVp9FramesInGof);
philipel02447bc2016-05-13 13:01:03635
636 // If there is a gap, find which temporal layer the missing frames
637 // belong to and add the frame as missing for that temporal layer.
638 // Otherwise, remove this frame from the set of missing frames.
philipelc9b27d52016-07-15 13:50:27639 if (AheadOf<uint16_t, kPicIdLength>(picture_id, last_picture_id)) {
640 size_t diff = ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start,
641 last_picture_id);
philipel459f4e32018-03-02 09:55:12642 size_t gof_idx = diff % gof_size;
philipel02447bc2016-05-13 13:01:03643
philipelc9b27d52016-07-15 13:50:27644 last_picture_id = Add<kPicIdLength>(last_picture_id, 1);
645 while (last_picture_id != picture_id) {
Yves Gerey665174f2018-06-19 13:03:05646 gof_idx = (gof_idx + 1) % gof_size;
philipel459f4e32018-03-02 09:55:12647 RTC_CHECK(gof_idx < kMaxVp9FramesInGof);
648
philipelc9b27d52016-07-15 13:50:27649 size_t temporal_idx = info->gof->temporal_idx[gof_idx];
philipel459f4e32018-03-02 09:55:12650 if (temporal_idx >= kMaxTemporalLayers) {
Jonas Olssonb2b20312020-01-14 11:11:31651 RTC_LOG(LS_WARNING) << "At most " << kMaxTemporalLayers
652 << " temporal "
653 "layers are supported.";
philipel459f4e32018-03-02 09:55:12654 return;
655 }
656
philipelc9b27d52016-07-15 13:50:27657 missing_frames_for_layer_[temporal_idx].insert(last_picture_id);
658 last_picture_id = Add<kPicIdLength>(last_picture_id, 1);
philipel02447bc2016-05-13 13:01:03659 }
philipel459f4e32018-03-02 09:55:12660
philipelc9b27d52016-07-15 13:50:27661 info->last_picture_id = last_picture_id;
philipel02447bc2016-05-13 13:01:03662 } else {
663 size_t diff =
philipelc9b27d52016-07-15 13:50:27664 ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start, picture_id);
philipel459f4e32018-03-02 09:55:12665 size_t gof_idx = diff % gof_size;
666 RTC_CHECK(gof_idx < kMaxVp9FramesInGof);
667
philipelc9b27d52016-07-15 13:50:27668 size_t temporal_idx = info->gof->temporal_idx[gof_idx];
philipel459f4e32018-03-02 09:55:12669 if (temporal_idx >= kMaxTemporalLayers) {
Jonas Olssonb2b20312020-01-14 11:11:31670 RTC_LOG(LS_WARNING) << "At most " << kMaxTemporalLayers
671 << " temporal "
672 "layers are supported.";
philipel459f4e32018-03-02 09:55:12673 return;
674 }
675
philipel02447bc2016-05-13 13:01:03676 missing_frames_for_layer_[temporal_idx].erase(picture_id);
677 }
678}
679
680bool RtpFrameReferenceFinder::UpSwitchInIntervalVp9(uint16_t picture_id,
681 uint8_t temporal_idx,
682 uint16_t pid_ref) {
683 for (auto up_switch_it = up_switch_.upper_bound(pid_ref);
684 up_switch_it != up_switch_.end() &&
685 AheadOf<uint16_t, kPicIdLength>(picture_id, up_switch_it->first);
686 ++up_switch_it) {
687 if (up_switch_it->second < temporal_idx)
688 return true;
689 }
690
691 return false;
692}
693
philipelafcf7f52017-04-26 15:17:35694void RtpFrameReferenceFinder::UnwrapPictureIds(RtpFrameObject* frame) {
philipel02447bc2016-05-13 13:01:03695 for (size_t i = 0; i < frame->num_references; ++i)
philipeld4fac692017-09-04 14:03:46696 frame->references[i] = unwrapper_.Unwrap(frame->references[i]);
philipel0fa82a62018-03-19 14:34:53697 frame->id.picture_id = unwrapper_.Unwrap(frame->id.picture_id);
philipel02447bc2016-05-13 13:01:03698}
699
Johnny Leebc7f41b2019-05-01 18:41:32700RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameH264(
701 RtpFrameObject* frame) {
Johannes Krona3705562019-08-26 14:37:11702 const FrameMarking& rtp_frame_marking = frame->GetFrameMarking();
Johnny Leebc7f41b2019-05-01 18:41:32703
Johannes Krona3705562019-08-26 14:37:11704 uint8_t tid = rtp_frame_marking.temporal_id;
705 bool blSync = rtp_frame_marking.base_layer_sync;
Johnny Leebc7f41b2019-05-01 18:41:32706
707 if (tid == kNoTemporalIdx)
708 return ManageFramePidOrSeqNum(std::move(frame), kNoPictureId);
709
Ilya Nikolaevskiy7a709c02020-01-17 15:40:02710 // Protect against corrupted packets with arbitrary large temporal idx.
711 if (tid >= kMaxTemporalLayers)
712 return kDrop;
713
Johnny Leebc7f41b2019-05-01 18:41:32714 frame->id.picture_id = frame->last_seq_num();
715
716 if (frame->frame_type() == VideoFrameType::kVideoFrameKey) {
717 // For H264, use last_seq_num_gop_ to simply store last picture id
718 // as a pair of unpadded and padded sequence numbers.
719 if (last_seq_num_gop_.empty()) {
720 last_seq_num_gop_.insert(std::make_pair(
721 0, std::make_pair(frame->id.picture_id, frame->id.picture_id)));
722 }
723 }
724
725 // Stash if we have no keyframe yet.
726 if (last_seq_num_gop_.empty())
727 return kStash;
728
729 // Check for gap in sequence numbers. Store in |not_yet_received_seq_num_|.
730 if (frame->frame_type() == VideoFrameType::kVideoFrameDelta) {
731 uint16_t last_pic_id_padded = last_seq_num_gop_.begin()->second.second;
732 if (AheadOf<uint16_t>(frame->id.picture_id, last_pic_id_padded)) {
733 do {
734 last_pic_id_padded = last_pic_id_padded + 1;
735 not_yet_received_seq_num_.insert(last_pic_id_padded);
736 } while (last_pic_id_padded != frame->id.picture_id);
737 }
738 }
739
Johannes Krona3705562019-08-26 14:37:11740 int64_t unwrapped_tl0 = tl0_unwrapper_.Unwrap(rtp_frame_marking.tl0_pic_idx);
Johnny Leebc7f41b2019-05-01 18:41:32741
742 // Clean up info for base layers that are too old.
743 int64_t old_tl0_pic_idx = unwrapped_tl0 - kMaxLayerInfo;
744 auto clean_layer_info_to = layer_info_.lower_bound(old_tl0_pic_idx);
745 layer_info_.erase(layer_info_.begin(), clean_layer_info_to);
746
747 // Clean up info about not yet received frames that are too old.
748 uint16_t old_picture_id = frame->id.picture_id - kMaxNotYetReceivedFrames * 2;
749 auto clean_frames_to = not_yet_received_seq_num_.lower_bound(old_picture_id);
750 not_yet_received_seq_num_.erase(not_yet_received_seq_num_.begin(),
751 clean_frames_to);
752
753 if (frame->frame_type() == VideoFrameType::kVideoFrameKey) {
754 frame->num_references = 0;
755 layer_info_[unwrapped_tl0].fill(-1);
756 UpdateDataH264(frame, unwrapped_tl0, tid);
757 return kHandOff;
758 }
759
Jonas Olssona4d87372019-07-05 17:08:33760 auto layer_info_it =
761 layer_info_.find(tid == 0 ? unwrapped_tl0 - 1 : unwrapped_tl0);
Johnny Leebc7f41b2019-05-01 18:41:32762
763 // Stash if we have no base layer frame yet.
764 if (layer_info_it == layer_info_.end())
765 return kStash;
766
767 // Base layer frame. Copy layer info from previous base layer frame.
768 if (tid == 0) {
Jonas Olssona4d87372019-07-05 17:08:33769 layer_info_it =
770 layer_info_.insert(std::make_pair(unwrapped_tl0, layer_info_it->second))
771 .first;
Johnny Leebc7f41b2019-05-01 18:41:32772 frame->num_references = 1;
773 frame->references[0] = layer_info_it->second[0];
774 UpdateDataH264(frame, unwrapped_tl0, tid);
775 return kHandOff;
776 }
777
778 // This frame only references its base layer frame.
779 if (blSync) {
780 frame->num_references = 1;
781 frame->references[0] = layer_info_it->second[0];
782 UpdateDataH264(frame, unwrapped_tl0, tid);
783 return kHandOff;
784 }
785
786 // Find all references for general frame.
787 frame->num_references = 0;
788 for (uint8_t layer = 0; layer <= tid; ++layer) {
789 // Stash if we have not yet received frames on this temporal layer.
790 if (layer_info_it->second[layer] == -1)
791 return kStash;
792
793 // Drop if the last frame on this layer is ahead of this frame. A layer sync
794 // frame was received after this frame for the same base layer frame.
795 uint16_t last_frame_in_layer = layer_info_it->second[layer];
796 if (AheadOf<uint16_t>(last_frame_in_layer, frame->id.picture_id))
797 return kDrop;
798
799 // Stash and wait for missing frame between this frame and the reference
800 auto not_received_seq_num_it =
801 not_yet_received_seq_num_.upper_bound(last_frame_in_layer);
802 if (not_received_seq_num_it != not_yet_received_seq_num_.end() &&
803 AheadOf<uint16_t>(frame->id.picture_id, *not_received_seq_num_it)) {
804 return kStash;
805 }
806
807 if (!(AheadOf<uint16_t>(frame->id.picture_id, last_frame_in_layer))) {
808 RTC_LOG(LS_WARNING) << "Frame with picture id " << frame->id.picture_id
809 << " and packet range [" << frame->first_seq_num()
810 << ", " << frame->last_seq_num()
811 << "] already received, "
Jonas Olssonb2b20312020-01-14 11:11:31812 " dropping frame.";
Johnny Leebc7f41b2019-05-01 18:41:32813 return kDrop;
814 }
815
816 ++frame->num_references;
817 frame->references[layer] = last_frame_in_layer;
818 }
819
820 UpdateDataH264(frame, unwrapped_tl0, tid);
821 return kHandOff;
822}
823
824void RtpFrameReferenceFinder::UpdateLastPictureIdWithPaddingH264() {
825 auto seq_num_it = last_seq_num_gop_.begin();
826
827 // Check if next sequence number is in a stashed padding packet.
828 uint16_t next_padded_seq_num = seq_num_it->second.second + 1;
829 auto padding_seq_num_it = stashed_padding_.lower_bound(next_padded_seq_num);
830
831 // Check for more consecutive padding packets to increment
832 // the "last-picture-id-with-padding" and remove the stashed packets.
833 while (padding_seq_num_it != stashed_padding_.end() &&
834 *padding_seq_num_it == next_padded_seq_num) {
835 seq_num_it->second.second = next_padded_seq_num;
836 ++next_padded_seq_num;
837 padding_seq_num_it = stashed_padding_.erase(padding_seq_num_it);
838 }
839}
840
841void RtpFrameReferenceFinder::UpdateLayerInfoH264(RtpFrameObject* frame,
842 int64_t unwrapped_tl0,
843 uint8_t temporal_idx) {
844 auto layer_info_it = layer_info_.find(unwrapped_tl0);
845
846 // Update this layer info and newer.
847 while (layer_info_it != layer_info_.end()) {
848 if (layer_info_it->second[temporal_idx] != -1 &&
849 AheadOf<uint16_t>(layer_info_it->second[temporal_idx],
850 frame->id.picture_id)) {
851 // Not a newer frame. No subsequent layer info needs update.
852 break;
853 }
854
855 layer_info_it->second[temporal_idx] = frame->id.picture_id;
856 ++unwrapped_tl0;
857 layer_info_it = layer_info_.find(unwrapped_tl0);
858 }
859
860 for (size_t i = 0; i < frame->num_references; ++i)
861 frame->references[i] = rtp_seq_num_unwrapper_.Unwrap(frame->references[i]);
862 frame->id.picture_id = rtp_seq_num_unwrapper_.Unwrap(frame->id.picture_id);
863}
864
865void RtpFrameReferenceFinder::UpdateDataH264(RtpFrameObject* frame,
866 int64_t unwrapped_tl0,
867 uint8_t temporal_idx) {
868 // Update last_seq_num_gop_ entry for last picture id.
869 auto seq_num_it = last_seq_num_gop_.begin();
870 uint16_t last_pic_id = seq_num_it->second.first;
871 if (AheadOf<uint16_t>(frame->id.picture_id, last_pic_id)) {
872 seq_num_it->second.first = frame->id.picture_id;
873 seq_num_it->second.second = frame->id.picture_id;
874 }
875 UpdateLastPictureIdWithPaddingH264();
876
877 UpdateLayerInfoH264(frame, unwrapped_tl0, temporal_idx);
878
879 // Remove any current packets from |not_yet_received_seq_num_|.
880 uint16_t last_seq_num_padded = seq_num_it->second.second;
Jonas Olssona4d87372019-07-05 17:08:33881 for (uint16_t n = frame->first_seq_num(); AheadOrAt(last_seq_num_padded, n);
882 ++n) {
Johnny Leebc7f41b2019-05-01 18:41:32883 not_yet_received_seq_num_.erase(n);
884 }
885}
886
philipel02447bc2016-05-13 13:01:03887} // namespace video_coding
888} // namespace webrtc