philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 11 | #include "modules/video_coding/rtp_frame_reference_finder.h" |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 12 | |
| 13 | #include <algorithm> |
| 14 | #include <limits> |
| 15 | |
Danil Chapovalov | 64f1f3f | 2020-01-16 13:41:10 | [diff] [blame] | 16 | #include "absl/base/macros.h" |
philipel | 1a4746a | 2018-07-09 13:52:29 | [diff] [blame] | 17 | #include "absl/types/variant.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 04:47:31 | [diff] [blame] | 18 | #include "modules/video_coding/frame_object.h" |
| 19 | #include "modules/video_coding/packet_buffer.h" |
| 20 | #include "rtc_base/checks.h" |
| 21 | #include "rtc_base/logging.h" |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 22 | |
| 23 | namespace webrtc { |
| 24 | namespace video_coding { |
| 25 | |
| 26 | RtpFrameReferenceFinder::RtpFrameReferenceFinder( |
| 27 | OnCompleteFrameCallback* frame_callback) |
philipel | 7acc4a4 | 2019-09-26 09:25:52 | [diff] [blame] | 28 | : RtpFrameReferenceFinder(frame_callback, 0) {} |
| 29 | |
| 30 | RtpFrameReferenceFinder::RtpFrameReferenceFinder( |
| 31 | OnCompleteFrameCallback* frame_callback, |
| 32 | int64_t picture_id_offset) |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 33 | : last_picture_id_(-1), |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 34 | current_ss_idx_(0), |
philipel | 463d301 | 2016-09-09 10:32:44 | [diff] [blame] | 35 | cleared_to_seq_num_(-1), |
philipel | 7acc4a4 | 2019-09-26 09:25:52 | [diff] [blame] | 36 | frame_callback_(frame_callback), |
| 37 | picture_id_offset_(picture_id_offset) {} |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 38 | |
Mirko Bonadei | 8fdcac3 | 2018-08-28 14:30:18 | [diff] [blame] | 39 | RtpFrameReferenceFinder::~RtpFrameReferenceFinder() = default; |
| 40 | |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 41 | void RtpFrameReferenceFinder::ManageFrame( |
| 42 | std::unique_ptr<RtpFrameObject> frame) { |
philipel | 463d301 | 2016-09-09 10:32:44 | [diff] [blame] | 43 | // If we have cleared past this frame, drop it. |
| 44 | if (cleared_to_seq_num_ != -1 && |
| 45 | AheadOf<uint16_t>(cleared_to_seq_num_, frame->first_seq_num())) { |
| 46 | return; |
| 47 | } |
| 48 | |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 49 | FrameDecision decision = ManageFrameInternal(frame.get()); |
| 50 | |
| 51 | switch (decision) { |
| 52 | case kStash: |
| 53 | if (stashed_frames_.size() > kMaxStashedFrames) |
| 54 | stashed_frames_.pop_back(); |
| 55 | stashed_frames_.push_front(std::move(frame)); |
| 56 | break; |
| 57 | case kHandOff: |
philipel | 7acc4a4 | 2019-09-26 09:25:52 | [diff] [blame] | 58 | HandOffFrame(std::move(frame)); |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 59 | RetryStashedFrames(); |
| 60 | break; |
| 61 | case kDrop: |
| 62 | break; |
| 63 | } |
| 64 | } |
| 65 | |
| 66 | void RtpFrameReferenceFinder::RetryStashedFrames() { |
| 67 | bool complete_frame = false; |
| 68 | do { |
| 69 | complete_frame = false; |
| 70 | for (auto frame_it = stashed_frames_.begin(); |
| 71 | frame_it != stashed_frames_.end();) { |
| 72 | FrameDecision decision = ManageFrameInternal(frame_it->get()); |
| 73 | |
| 74 | switch (decision) { |
| 75 | case kStash: |
| 76 | ++frame_it; |
| 77 | break; |
| 78 | case kHandOff: |
| 79 | complete_frame = true; |
philipel | 7acc4a4 | 2019-09-26 09:25:52 | [diff] [blame] | 80 | HandOffFrame(std::move(*frame_it)); |
Danil Chapovalov | 64f1f3f | 2020-01-16 13:41:10 | [diff] [blame] | 81 | ABSL_FALLTHROUGH_INTENDED; |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 82 | case kDrop: |
| 83 | frame_it = stashed_frames_.erase(frame_it); |
| 84 | } |
| 85 | } |
| 86 | } while (complete_frame); |
| 87 | } |
| 88 | |
philipel | 7acc4a4 | 2019-09-26 09:25:52 | [diff] [blame] | 89 | void RtpFrameReferenceFinder::HandOffFrame( |
| 90 | std::unique_ptr<RtpFrameObject> frame) { |
| 91 | frame->id.picture_id += picture_id_offset_; |
| 92 | for (size_t i = 0; i < frame->num_references; ++i) { |
| 93 | frame->references[i] += picture_id_offset_; |
| 94 | } |
| 95 | |
| 96 | frame_callback_->OnCompleteFrame(std::move(frame)); |
| 97 | } |
| 98 | |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 99 | RtpFrameReferenceFinder::FrameDecision |
| 100 | RtpFrameReferenceFinder::ManageFrameInternal(RtpFrameObject* frame) { |
Danil Chapovalov | 26b4cb3 | 2020-01-29 10:21:38 | [diff] [blame] | 101 | if (const absl::optional<RTPVideoHeader::GenericDescriptorInfo>& |
| 102 | generic_descriptor = frame->GetRtpVideoHeader().generic) { |
philipel | 2837edc | 2018-10-02 11:55:47 | [diff] [blame] | 103 | return ManageFrameGeneric(frame, *generic_descriptor); |
philipel | dabfcae | 2018-09-25 10:54:37 | [diff] [blame] | 104 | } |
| 105 | |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 106 | switch (frame->codec_type()) { |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 107 | case kVideoCodecVP8: |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 108 | return ManageFrameVp8(frame); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 109 | case kVideoCodecVP9: |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 110 | return ManageFrameVp9(frame); |
Johnny Lee | bc7f41b | 2019-05-01 18:41:32 | [diff] [blame] | 111 | case kVideoCodecH264: |
| 112 | return ManageFrameH264(frame); |
Danil Chapovalov | b6bf0b2 | 2020-01-28 17:36:57 | [diff] [blame] | 113 | case kVideoCodecGeneric: |
| 114 | if (auto* generic_header = absl::get_if<RTPVideoHeaderLegacyGeneric>( |
| 115 | &frame->GetRtpVideoHeader().video_type_header)) { |
| 116 | return ManageFramePidOrSeqNum(frame, generic_header->picture_id); |
| 117 | } |
| 118 | ABSL_FALLTHROUGH_INTENDED; |
| 119 | default: |
| 120 | return ManageFramePidOrSeqNum(frame, kNoPictureId); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 121 | } |
| 122 | } |
| 123 | |
philipel | 9b2ce6b | 2016-07-05 12:04:46 | [diff] [blame] | 124 | void RtpFrameReferenceFinder::PaddingReceived(uint16_t seq_num) { |
philipel | 9b2ce6b | 2016-07-05 12:04:46 | [diff] [blame] | 125 | auto clean_padding_to = |
| 126 | stashed_padding_.lower_bound(seq_num - kMaxPaddingAge); |
| 127 | stashed_padding_.erase(stashed_padding_.begin(), clean_padding_to); |
| 128 | stashed_padding_.insert(seq_num); |
| 129 | UpdateLastPictureIdWithPadding(seq_num); |
| 130 | RetryStashedFrames(); |
| 131 | } |
| 132 | |
philipel | 463d301 | 2016-09-09 10:32:44 | [diff] [blame] | 133 | void RtpFrameReferenceFinder::ClearTo(uint16_t seq_num) { |
philipel | 463d301 | 2016-09-09 10:32:44 | [diff] [blame] | 134 | cleared_to_seq_num_ = seq_num; |
| 135 | |
| 136 | auto it = stashed_frames_.begin(); |
| 137 | while (it != stashed_frames_.end()) { |
| 138 | if (AheadOf<uint16_t>(cleared_to_seq_num_, (*it)->first_seq_num())) { |
| 139 | it = stashed_frames_.erase(it); |
| 140 | } else { |
| 141 | ++it; |
| 142 | } |
| 143 | } |
| 144 | } |
| 145 | |
philipel | 9b2ce6b | 2016-07-05 12:04:46 | [diff] [blame] | 146 | void RtpFrameReferenceFinder::UpdateLastPictureIdWithPadding(uint16_t seq_num) { |
| 147 | auto gop_seq_num_it = last_seq_num_gop_.upper_bound(seq_num); |
| 148 | |
| 149 | // If this padding packet "belongs" to a group of pictures that we don't track |
| 150 | // anymore, do nothing. |
| 151 | if (gop_seq_num_it == last_seq_num_gop_.begin()) |
| 152 | return; |
| 153 | --gop_seq_num_it; |
| 154 | |
| 155 | // Calculate the next contiuous sequence number and search for it in |
| 156 | // the padding packets we have stashed. |
| 157 | uint16_t next_seq_num_with_padding = gop_seq_num_it->second.second + 1; |
| 158 | auto padding_seq_num_it = |
| 159 | stashed_padding_.lower_bound(next_seq_num_with_padding); |
| 160 | |
| 161 | // While there still are padding packets and those padding packets are |
| 162 | // continuous, then advance the "last-picture-id-with-padding" and remove |
| 163 | // the stashed padding packet. |
| 164 | while (padding_seq_num_it != stashed_padding_.end() && |
| 165 | *padding_seq_num_it == next_seq_num_with_padding) { |
| 166 | gop_seq_num_it->second.second = next_seq_num_with_padding; |
| 167 | ++next_seq_num_with_padding; |
| 168 | padding_seq_num_it = stashed_padding_.erase(padding_seq_num_it); |
| 169 | } |
philipel | 41bb792 | 2017-02-20 15:53:23 | [diff] [blame] | 170 | |
| 171 | // In the case where the stream has been continuous without any new keyframes |
| 172 | // for a while there is a risk that new frames will appear to be older than |
| 173 | // the keyframe they belong to due to wrapping sequence number. In order |
| 174 | // to prevent this we advance the picture id of the keyframe every so often. |
| 175 | if (ForwardDiff(gop_seq_num_it->first, seq_num) > 10000) { |
| 176 | RTC_DCHECK_EQ(1ul, last_seq_num_gop_.size()); |
| 177 | last_seq_num_gop_[seq_num] = gop_seq_num_it->second; |
| 178 | last_seq_num_gop_.erase(gop_seq_num_it); |
| 179 | } |
philipel | 9b2ce6b | 2016-07-05 12:04:46 | [diff] [blame] | 180 | } |
| 181 | |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 182 | RtpFrameReferenceFinder::FrameDecision |
philipel | dabfcae | 2018-09-25 10:54:37 | [diff] [blame] | 183 | RtpFrameReferenceFinder::ManageFrameGeneric( |
| 184 | RtpFrameObject* frame, |
Danil Chapovalov | 26b4cb3 | 2020-01-29 10:21:38 | [diff] [blame] | 185 | const RTPVideoHeader::GenericDescriptorInfo& descriptor) { |
| 186 | frame->id.picture_id = descriptor.frame_id; |
| 187 | frame->id.spatial_layer = descriptor.spatial_index; |
philipel | 2837edc | 2018-10-02 11:55:47 | [diff] [blame] | 188 | |
Danil Chapovalov | 26b4cb3 | 2020-01-29 10:21:38 | [diff] [blame] | 189 | if (EncodedFrame::kMaxFrameReferences < descriptor.dependencies.size()) { |
philipel | dabfcae | 2018-09-25 10:54:37 | [diff] [blame] | 190 | RTC_LOG(LS_WARNING) << "Too many dependencies in generic descriptor."; |
| 191 | return kDrop; |
| 192 | } |
| 193 | |
Danil Chapovalov | 26b4cb3 | 2020-01-29 10:21:38 | [diff] [blame] | 194 | frame->num_references = descriptor.dependencies.size(); |
| 195 | for (size_t i = 0; i < descriptor.dependencies.size(); ++i) |
| 196 | frame->references[i] = descriptor.dependencies[i]; |
philipel | dabfcae | 2018-09-25 10:54:37 | [diff] [blame] | 197 | |
| 198 | return kHandOff; |
| 199 | } |
| 200 | |
| 201 | RtpFrameReferenceFinder::FrameDecision |
| 202 | RtpFrameReferenceFinder::ManageFramePidOrSeqNum(RtpFrameObject* frame, |
| 203 | int picture_id) { |
philipel | 647998c | 2016-06-03 16:40:16 | [diff] [blame] | 204 | // If |picture_id| is specified then we use that to set the frame references, |
| 205 | // otherwise we use sequence number. |
| 206 | if (picture_id != kNoPictureId) { |
Danil Chapovalov | 95cb56b | 2020-01-30 14:03:26 | [diff] [blame] | 207 | frame->id.picture_id = unwrapper_.Unwrap(picture_id & 0x7FFF); |
Niels Möller | 8f7ce22 | 2019-03-21 14:43:58 | [diff] [blame] | 208 | frame->num_references = |
| 209 | frame->frame_type() == VideoFrameType::kVideoFrameKey ? 0 : 1; |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 210 | frame->references[0] = frame->id.picture_id - 1; |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 211 | return kHandOff; |
philipel | 647998c | 2016-06-03 16:40:16 | [diff] [blame] | 212 | } |
| 213 | |
Niels Möller | 8f7ce22 | 2019-03-21 14:43:58 | [diff] [blame] | 214 | if (frame->frame_type() == VideoFrameType::kVideoFrameKey) { |
philipel | 9b2ce6b | 2016-07-05 12:04:46 | [diff] [blame] | 215 | last_seq_num_gop_.insert(std::make_pair( |
| 216 | frame->last_seq_num(), |
| 217 | std::make_pair(frame->last_seq_num(), frame->last_seq_num()))); |
| 218 | } |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 219 | |
| 220 | // We have received a frame but not yet a keyframe, stash this frame. |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 221 | if (last_seq_num_gop_.empty()) |
| 222 | return kStash; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 223 | |
| 224 | // Clean up info for old keyframes but make sure to keep info |
| 225 | // for the last keyframe. |
| 226 | auto clean_to = last_seq_num_gop_.lower_bound(frame->last_seq_num() - 100); |
philipel | 41bb792 | 2017-02-20 15:53:23 | [diff] [blame] | 227 | for (auto it = last_seq_num_gop_.begin(); |
| 228 | it != clean_to && last_seq_num_gop_.size() > 1;) { |
| 229 | it = last_seq_num_gop_.erase(it); |
| 230 | } |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 231 | |
| 232 | // Find the last sequence number of the last frame for the keyframe |
| 233 | // that this frame indirectly references. |
| 234 | auto seq_num_it = last_seq_num_gop_.upper_bound(frame->last_seq_num()); |
philipel | 9b2ce6b | 2016-07-05 12:04:46 | [diff] [blame] | 235 | if (seq_num_it == last_seq_num_gop_.begin()) { |
Mirko Bonadei | 675513b | 2017-11-09 10:09:25 | [diff] [blame] | 236 | RTC_LOG(LS_WARNING) << "Generic frame with packet range [" |
| 237 | << frame->first_seq_num() << ", " |
| 238 | << frame->last_seq_num() |
| 239 | << "] has no GoP, dropping frame."; |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 240 | return kDrop; |
philipel | 9b2ce6b | 2016-07-05 12:04:46 | [diff] [blame] | 241 | } |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 242 | seq_num_it--; |
| 243 | |
| 244 | // Make sure the packet sequence numbers are continuous, otherwise stash |
| 245 | // this frame. |
philipel | 9b2ce6b | 2016-07-05 12:04:46 | [diff] [blame] | 246 | uint16_t last_picture_id_gop = seq_num_it->second.first; |
| 247 | uint16_t last_picture_id_with_padding_gop = seq_num_it->second.second; |
Niels Möller | 8f7ce22 | 2019-03-21 14:43:58 | [diff] [blame] | 248 | if (frame->frame_type() == VideoFrameType::kVideoFrameDelta) { |
philipel | 9b2ce6b | 2016-07-05 12:04:46 | [diff] [blame] | 249 | uint16_t prev_seq_num = frame->first_seq_num() - 1; |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 250 | |
| 251 | if (prev_seq_num != last_picture_id_with_padding_gop) |
| 252 | return kStash; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 253 | } |
| 254 | |
| 255 | RTC_DCHECK(AheadOrAt(frame->last_seq_num(), seq_num_it->first)); |
| 256 | |
| 257 | // Since keyframes can cause reordering we can't simply assign the |
| 258 | // picture id according to some incrementing counter. |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 259 | frame->id.picture_id = frame->last_seq_num(); |
Niels Möller | 8f7ce22 | 2019-03-21 14:43:58 | [diff] [blame] | 260 | frame->num_references = |
| 261 | frame->frame_type() == VideoFrameType::kVideoFrameDelta; |
philipel | dabfcae | 2018-09-25 10:54:37 | [diff] [blame] | 262 | frame->references[0] = rtp_seq_num_unwrapper_.Unwrap(last_picture_id_gop); |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 263 | if (AheadOf<uint16_t>(frame->id.picture_id, last_picture_id_gop)) { |
| 264 | seq_num_it->second.first = frame->id.picture_id; |
| 265 | seq_num_it->second.second = frame->id.picture_id; |
philipel | 9b2ce6b | 2016-07-05 12:04:46 | [diff] [blame] | 266 | } |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 267 | |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 268 | UpdateLastPictureIdWithPadding(frame->id.picture_id); |
philipel | dabfcae | 2018-09-25 10:54:37 | [diff] [blame] | 269 | frame->id.picture_id = rtp_seq_num_unwrapper_.Unwrap(frame->id.picture_id); |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 270 | return kHandOff; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 271 | } |
| 272 | |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 273 | RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameVp8( |
| 274 | RtpFrameObject* frame) { |
Johannes Kron | a370556 | 2019-08-26 14:37:11 | [diff] [blame] | 275 | const RTPVideoHeader& video_header = frame->GetRtpVideoHeader(); |
philipel | 1a4746a | 2018-07-09 13:52:29 | [diff] [blame] | 276 | const RTPVideoHeaderVP8& codec_header = |
Danil Chapovalov | 95cb56b | 2020-01-30 14:03:26 | [diff] [blame] | 277 | absl::get<RTPVideoHeaderVP8>(video_header.video_type_header); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 278 | |
| 279 | if (codec_header.pictureId == kNoPictureId || |
| 280 | codec_header.temporalIdx == kNoTemporalIdx || |
| 281 | codec_header.tl0PicIdx == kNoTl0PicIdx) { |
Mirko Bonadei | 05cf6be | 2019-01-31 20:38:12 | [diff] [blame] | 282 | return ManageFramePidOrSeqNum(frame, codec_header.pictureId); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 283 | } |
| 284 | |
Ilya Nikolaevskiy | 7a709c0 | 2020-01-17 15:40:02 | [diff] [blame] | 285 | // Protect against corrupted packets with arbitrary large temporal idx. |
| 286 | if (codec_header.temporalIdx >= kMaxTemporalLayers) |
| 287 | return kDrop; |
| 288 | |
Danil Chapovalov | 95cb56b | 2020-01-30 14:03:26 | [diff] [blame] | 289 | frame->id.picture_id = codec_header.pictureId & 0x7FFF; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 290 | |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 291 | if (last_picture_id_ == -1) |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 292 | last_picture_id_ = frame->id.picture_id; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 293 | |
Danil Chapovalov | 545c53e | 2020-01-31 17:08:54 | [diff] [blame] | 294 | // Clean up info about not yet received frames that are too old. |
| 295 | uint16_t old_picture_id = |
| 296 | Subtract<kPicIdLength>(frame->id.picture_id, kMaxNotYetReceivedFrames); |
| 297 | auto clean_frames_to = not_yet_received_frames_.lower_bound(old_picture_id); |
| 298 | not_yet_received_frames_.erase(not_yet_received_frames_.begin(), |
| 299 | clean_frames_to); |
| 300 | // Avoid re-adding picture ids that were just erased. |
| 301 | if (AheadOf<uint16_t, kPicIdLength>(old_picture_id, last_picture_id_)) { |
| 302 | last_picture_id_ = old_picture_id; |
| 303 | } |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 304 | // Find if there has been a gap in fully received frames and save the picture |
| 305 | // id of those frames in |not_yet_received_frames_|. |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 306 | if (AheadOf<uint16_t, kPicIdLength>(frame->id.picture_id, last_picture_id_)) { |
philipel | 9bd1d66 | 2017-07-14 11:52:01 | [diff] [blame] | 307 | do { |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 308 | last_picture_id_ = Add<kPicIdLength>(last_picture_id_, 1); |
philipel | 9bd1d66 | 2017-07-14 11:52:01 | [diff] [blame] | 309 | not_yet_received_frames_.insert(last_picture_id_); |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 310 | } while (last_picture_id_ != frame->id.picture_id); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 311 | } |
| 312 | |
Danil Chapovalov | 95cb56b | 2020-01-30 14:03:26 | [diff] [blame] | 313 | int64_t unwrapped_tl0 = tl0_unwrapper_.Unwrap(codec_header.tl0PicIdx & 0xFF); |
philipel | 57ec685 | 2018-07-03 16:09:32 | [diff] [blame] | 314 | |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 315 | // Clean up info for base layers that are too old. |
philipel | 57ec685 | 2018-07-03 16:09:32 | [diff] [blame] | 316 | int64_t old_tl0_pic_idx = unwrapped_tl0 - kMaxLayerInfo; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 317 | auto clean_layer_info_to = layer_info_.lower_bound(old_tl0_pic_idx); |
| 318 | layer_info_.erase(layer_info_.begin(), clean_layer_info_to); |
| 319 | |
Niels Möller | 8f7ce22 | 2019-03-21 14:43:58 | [diff] [blame] | 320 | if (frame->frame_type() == VideoFrameType::kVideoFrameKey) { |
Danil Chapovalov | 95cb56b | 2020-01-30 14:03:26 | [diff] [blame] | 321 | if (codec_header.temporalIdx != 0) { |
| 322 | return kDrop; |
| 323 | } |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 324 | frame->num_references = 0; |
philipel | 57ec685 | 2018-07-03 16:09:32 | [diff] [blame] | 325 | layer_info_[unwrapped_tl0].fill(-1); |
| 326 | UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx); |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 327 | return kHandOff; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 328 | } |
| 329 | |
philipel | 57ec685 | 2018-07-03 16:09:32 | [diff] [blame] | 330 | auto layer_info_it = layer_info_.find( |
| 331 | codec_header.temporalIdx == 0 ? unwrapped_tl0 - 1 : unwrapped_tl0); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 332 | |
| 333 | // If we don't have the base layer frame yet, stash this frame. |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 334 | if (layer_info_it == layer_info_.end()) |
| 335 | return kStash; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 336 | |
| 337 | // A non keyframe base layer frame has been received, copy the layer info |
| 338 | // from the previous base layer frame and set a reference to the previous |
| 339 | // base layer frame. |
| 340 | if (codec_header.temporalIdx == 0) { |
| 341 | layer_info_it = |
philipel | 57ec685 | 2018-07-03 16:09:32 | [diff] [blame] | 342 | layer_info_.emplace(unwrapped_tl0, layer_info_it->second).first; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 343 | frame->num_references = 1; |
| 344 | frame->references[0] = layer_info_it->second[0]; |
philipel | 57ec685 | 2018-07-03 16:09:32 | [diff] [blame] | 345 | UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx); |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 346 | return kHandOff; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 347 | } |
| 348 | |
| 349 | // Layer sync frame, this frame only references its base layer frame. |
| 350 | if (codec_header.layerSync) { |
| 351 | frame->num_references = 1; |
| 352 | frame->references[0] = layer_info_it->second[0]; |
| 353 | |
philipel | 57ec685 | 2018-07-03 16:09:32 | [diff] [blame] | 354 | UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx); |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 355 | return kHandOff; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 356 | } |
| 357 | |
| 358 | // Find all references for this frame. |
| 359 | frame->num_references = 0; |
| 360 | for (uint8_t layer = 0; layer <= codec_header.temporalIdx; ++layer) { |
philipel | d268d6f | 2016-09-15 11:43:13 | [diff] [blame] | 361 | // If we have not yet received a previous frame on this temporal layer, |
| 362 | // stash this frame. |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 363 | if (layer_info_it->second[layer] == -1) |
| 364 | return kStash; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 365 | |
philipel | 86b92e0 | 2016-10-24 14:11:53 | [diff] [blame] | 366 | // If the last frame on this layer is ahead of this frame it means that |
| 367 | // a layer sync frame has been received after this frame for the same |
| 368 | // base layer frame, drop this frame. |
| 369 | if (AheadOf<uint16_t, kPicIdLength>(layer_info_it->second[layer], |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 370 | frame->id.picture_id)) { |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 371 | return kDrop; |
philipel | 86b92e0 | 2016-10-24 14:11:53 | [diff] [blame] | 372 | } |
| 373 | |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 374 | // If we have not yet received a frame between this frame and the referenced |
| 375 | // frame then we have to wait for that frame to be completed first. |
| 376 | auto not_received_frame_it = |
| 377 | not_yet_received_frames_.upper_bound(layer_info_it->second[layer]); |
| 378 | if (not_received_frame_it != not_yet_received_frames_.end() && |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 379 | AheadOf<uint16_t, kPicIdLength>(frame->id.picture_id, |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 380 | *not_received_frame_it)) { |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 381 | return kStash; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 382 | } |
| 383 | |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 384 | if (!(AheadOf<uint16_t, kPicIdLength>(frame->id.picture_id, |
philipel | 57f19cc0 | 2017-03-07 11:54:05 | [diff] [blame] | 385 | layer_info_it->second[layer]))) { |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 386 | RTC_LOG(LS_WARNING) << "Frame with picture id " << frame->id.picture_id |
Mirko Bonadei | 675513b | 2017-11-09 10:09:25 | [diff] [blame] | 387 | << " and packet range [" << frame->first_seq_num() |
| 388 | << ", " << frame->last_seq_num() |
| 389 | << "] already received, " |
Jonas Olsson | b2b2031 | 2020-01-14 11:11:31 | [diff] [blame] | 390 | " dropping frame."; |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 391 | return kDrop; |
philipel | 57f19cc0 | 2017-03-07 11:54:05 | [diff] [blame] | 392 | } |
| 393 | |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 394 | ++frame->num_references; |
| 395 | frame->references[layer] = layer_info_it->second[layer]; |
| 396 | } |
| 397 | |
philipel | 57ec685 | 2018-07-03 16:09:32 | [diff] [blame] | 398 | UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx); |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 399 | return kHandOff; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 400 | } |
| 401 | |
philipel | 57ec685 | 2018-07-03 16:09:32 | [diff] [blame] | 402 | void RtpFrameReferenceFinder::UpdateLayerInfoVp8(RtpFrameObject* frame, |
| 403 | int64_t unwrapped_tl0, |
| 404 | uint8_t temporal_idx) { |
| 405 | auto layer_info_it = layer_info_.find(unwrapped_tl0); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 406 | |
| 407 | // Update this layer info and newer. |
| 408 | while (layer_info_it != layer_info_.end()) { |
philipel | 57ec685 | 2018-07-03 16:09:32 | [diff] [blame] | 409 | if (layer_info_it->second[temporal_idx] != -1 && |
| 410 | AheadOf<uint16_t, kPicIdLength>(layer_info_it->second[temporal_idx], |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 411 | frame->id.picture_id)) { |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 412 | // The frame was not newer, then no subsequent layer info have to be |
| 413 | // update. |
| 414 | break; |
| 415 | } |
| 416 | |
philipel | 57ec685 | 2018-07-03 16:09:32 | [diff] [blame] | 417 | layer_info_it->second[temporal_idx] = frame->id.picture_id; |
| 418 | ++unwrapped_tl0; |
| 419 | layer_info_it = layer_info_.find(unwrapped_tl0); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 420 | } |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 421 | not_yet_received_frames_.erase(frame->id.picture_id); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 422 | |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 423 | UnwrapPictureIds(frame); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 424 | } |
| 425 | |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 426 | RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameVp9( |
| 427 | RtpFrameObject* frame) { |
Johannes Kron | a370556 | 2019-08-26 14:37:11 | [diff] [blame] | 428 | const RTPVideoHeader& video_header = frame->GetRtpVideoHeader(); |
philipel | 1a4746a | 2018-07-09 13:52:29 | [diff] [blame] | 429 | const RTPVideoHeaderVP9& codec_header = |
Danil Chapovalov | 95cb56b | 2020-01-30 14:03:26 | [diff] [blame] | 430 | absl::get<RTPVideoHeaderVP9>(video_header.video_type_header); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 431 | |
philipel | 647998c | 2016-06-03 16:40:16 | [diff] [blame] | 432 | if (codec_header.picture_id == kNoPictureId || |
Sergey Silkin | d34a188 | 2018-08-20 14:46:05 | [diff] [blame] | 433 | codec_header.temporal_idx == kNoTemporalIdx) { |
Mirko Bonadei | 05cf6be | 2019-01-31 20:38:12 | [diff] [blame] | 434 | return ManageFramePidOrSeqNum(frame, codec_header.picture_id); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 435 | } |
| 436 | |
Ilya Nikolaevskiy | 7a709c0 | 2020-01-17 15:40:02 | [diff] [blame] | 437 | // Protect against corrupted packets with arbitrary large temporal idx. |
Ilya Nikolaevskiy | ef0d76a | 2020-02-05 17:01:31 | [diff] [blame] | 438 | if (codec_header.temporal_idx >= kMaxTemporalLayers || |
| 439 | codec_header.spatial_idx >= kMaxSpatialLayers) |
Ilya Nikolaevskiy | 7a709c0 | 2020-01-17 15:40:02 | [diff] [blame] | 440 | return kDrop; |
| 441 | |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 442 | frame->id.spatial_layer = codec_header.spatial_idx; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 443 | frame->inter_layer_predicted = codec_header.inter_layer_predicted; |
Danil Chapovalov | 95cb56b | 2020-01-30 14:03:26 | [diff] [blame] | 444 | frame->id.picture_id = codec_header.picture_id & 0x7FFF; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 445 | |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 446 | if (last_picture_id_ == -1) |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 447 | last_picture_id_ = frame->id.picture_id; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 448 | |
| 449 | if (codec_header.flexible_mode) { |
Danil Chapovalov | 95cb56b | 2020-01-30 14:03:26 | [diff] [blame] | 450 | if (codec_header.num_ref_pics > EncodedFrame::kMaxFrameReferences) { |
| 451 | return kDrop; |
| 452 | } |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 453 | frame->num_references = codec_header.num_ref_pics; |
| 454 | for (size_t i = 0; i < frame->num_references; ++i) { |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 455 | frame->references[i] = Subtract<kPicIdLength>(frame->id.picture_id, |
| 456 | codec_header.pid_diff[i]); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 457 | } |
| 458 | |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 459 | UnwrapPictureIds(frame); |
| 460 | return kHandOff; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 461 | } |
| 462 | |
Sergey Silkin | d34a188 | 2018-08-20 14:46:05 | [diff] [blame] | 463 | if (codec_header.tl0_pic_idx == kNoTl0PicIdx) { |
| 464 | RTC_LOG(LS_WARNING) << "TL0PICIDX is expected to be present in " |
| 465 | "non-flexible mode."; |
| 466 | return kDrop; |
| 467 | } |
| 468 | |
philipel | 1564360 | 2018-05-03 14:14:13 | [diff] [blame] | 469 | GofInfo* info; |
Danil Chapovalov | 95cb56b | 2020-01-30 14:03:26 | [diff] [blame] | 470 | int64_t unwrapped_tl0 = |
| 471 | tl0_unwrapper_.Unwrap(codec_header.tl0_pic_idx & 0xFF); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 472 | if (codec_header.ss_data_available) { |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 473 | if (codec_header.temporal_idx != 0) { |
philipel | 1564360 | 2018-05-03 14:14:13 | [diff] [blame] | 474 | RTC_LOG(LS_WARNING) << "Received scalability structure on a non base " |
| 475 | "layer frame. Scalability structure ignored."; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 476 | } else { |
Sergey Silkin | 2f864fb | 2018-09-07 09:49:38 | [diff] [blame] | 477 | if (codec_header.gof.num_frames_in_gof > kMaxVp9FramesInGof) { |
Natalie Silvanovich | 3ea3e30 | 2018-05-16 18:03:12 | [diff] [blame] | 478 | return kDrop; |
| 479 | } |
| 480 | |
Ilya Nikolaevskiy | ef0d76a | 2020-02-05 17:01:31 | [diff] [blame] | 481 | for (size_t i = 0; i < codec_header.gof.num_frames_in_gof; ++i) { |
| 482 | if (codec_header.gof.num_ref_pics[i] > kMaxVp9RefPics) { |
| 483 | return kDrop; |
| 484 | } |
| 485 | } |
| 486 | |
Sergey Silkin | 2f864fb | 2018-09-07 09:49:38 | [diff] [blame] | 487 | GofInfoVP9 gof = codec_header.gof; |
| 488 | if (gof.num_frames_in_gof == 0) { |
| 489 | RTC_LOG(LS_WARNING) << "Number of frames in GOF is zero. Assume " |
| 490 | "that stream has only one temporal layer."; |
| 491 | gof.SetGofInfoVP9(kTemporalStructureMode1); |
| 492 | } |
| 493 | |
| 494 | current_ss_idx_ = Add<kMaxGofSaved>(current_ss_idx_, 1); |
| 495 | scalability_structures_[current_ss_idx_] = gof; |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 496 | scalability_structures_[current_ss_idx_].pid_start = frame->id.picture_id; |
philipel | 57ec685 | 2018-07-03 16:09:32 | [diff] [blame] | 497 | gof_info_.emplace(unwrapped_tl0, |
philipel | 1564360 | 2018-05-03 14:14:13 | [diff] [blame] | 498 | GofInfo(&scalability_structures_[current_ss_idx_], |
| 499 | frame->id.picture_id)); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 500 | } |
philipel | 1564360 | 2018-05-03 14:14:13 | [diff] [blame] | 501 | |
philipel | 57ec685 | 2018-07-03 16:09:32 | [diff] [blame] | 502 | const auto gof_info_it = gof_info_.find(unwrapped_tl0); |
philipel | 1564360 | 2018-05-03 14:14:13 | [diff] [blame] | 503 | if (gof_info_it == gof_info_.end()) |
| 504 | return kStash; |
| 505 | |
| 506 | info = &gof_info_it->second; |
| 507 | |
Niels Möller | 8f7ce22 | 2019-03-21 14:43:58 | [diff] [blame] | 508 | if (frame->frame_type() == VideoFrameType::kVideoFrameKey) { |
philipel | 1564360 | 2018-05-03 14:14:13 | [diff] [blame] | 509 | frame->num_references = 0; |
| 510 | FrameReceivedVp9(frame->id.picture_id, info); |
| 511 | UnwrapPictureIds(frame); |
| 512 | return kHandOff; |
| 513 | } |
Niels Möller | 8f7ce22 | 2019-03-21 14:43:58 | [diff] [blame] | 514 | } else if (frame->frame_type() == VideoFrameType::kVideoFrameKey) { |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 14:54:52 | [diff] [blame] | 515 | if (frame->id.spatial_layer == 0) { |
philipel | 1564360 | 2018-05-03 14:14:13 | [diff] [blame] | 516 | RTC_LOG(LS_WARNING) << "Received keyframe without scalability structure"; |
| 517 | return kDrop; |
| 518 | } |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 14:54:52 | [diff] [blame] | 519 | const auto gof_info_it = gof_info_.find(unwrapped_tl0); |
| 520 | if (gof_info_it == gof_info_.end()) |
| 521 | return kStash; |
philipel | 1564360 | 2018-05-03 14:14:13 | [diff] [blame] | 522 | |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 14:54:52 | [diff] [blame] | 523 | info = &gof_info_it->second; |
| 524 | |
Niels Möller | 8f7ce22 | 2019-03-21 14:43:58 | [diff] [blame] | 525 | if (frame->frame_type() == VideoFrameType::kVideoFrameKey) { |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 14:54:52 | [diff] [blame] | 526 | frame->num_references = 0; |
| 527 | FrameReceivedVp9(frame->id.picture_id, info); |
| 528 | UnwrapPictureIds(frame); |
| 529 | return kHandOff; |
| 530 | } |
| 531 | } else { |
philipel | 57ec685 | 2018-07-03 16:09:32 | [diff] [blame] | 532 | auto gof_info_it = gof_info_.find( |
| 533 | (codec_header.temporal_idx == 0) ? unwrapped_tl0 - 1 : unwrapped_tl0); |
philipel | 1564360 | 2018-05-03 14:14:13 | [diff] [blame] | 534 | |
| 535 | // Gof info for this frame is not available yet, stash this frame. |
| 536 | if (gof_info_it == gof_info_.end()) |
| 537 | return kStash; |
| 538 | |
| 539 | if (codec_header.temporal_idx == 0) { |
philipel | 57ec685 | 2018-07-03 16:09:32 | [diff] [blame] | 540 | gof_info_it = gof_info_ |
| 541 | .emplace(unwrapped_tl0, GofInfo(gof_info_it->second.gof, |
| 542 | frame->id.picture_id)) |
| 543 | .first; |
philipel | 1564360 | 2018-05-03 14:14:13 | [diff] [blame] | 544 | } |
| 545 | |
| 546 | info = &gof_info_it->second; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 547 | } |
| 548 | |
| 549 | // Clean up info for base layers that are too old. |
philipel | 57ec685 | 2018-07-03 16:09:32 | [diff] [blame] | 550 | int64_t old_tl0_pic_idx = unwrapped_tl0 - kMaxGofSaved; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 551 | auto clean_gof_info_to = gof_info_.lower_bound(old_tl0_pic_idx); |
| 552 | gof_info_.erase(gof_info_.begin(), clean_gof_info_to); |
| 553 | |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 554 | FrameReceivedVp9(frame->id.picture_id, info); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 555 | |
| 556 | // Make sure we don't miss any frame that could potentially have the |
| 557 | // up switch flag set. |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 558 | if (MissingRequiredFrameVp9(frame->id.picture_id, *info)) |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 559 | return kStash; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 560 | |
philipel | 1564360 | 2018-05-03 14:14:13 | [diff] [blame] | 561 | if (codec_header.temporal_up_switch) |
| 562 | up_switch_.emplace(frame->id.picture_id, codec_header.temporal_idx); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 563 | |
| 564 | // Clean out old info about up switch frames. |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 565 | uint16_t old_picture_id = Subtract<kPicIdLength>(frame->id.picture_id, 50); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 566 | auto up_switch_erase_to = up_switch_.lower_bound(old_picture_id); |
| 567 | up_switch_.erase(up_switch_.begin(), up_switch_erase_to); |
| 568 | |
philipel | c9b27d5 | 2016-07-15 13:50:27 | [diff] [blame] | 569 | size_t diff = ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start, |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 570 | frame->id.picture_id); |
philipel | c9b27d5 | 2016-07-15 13:50:27 | [diff] [blame] | 571 | size_t gof_idx = diff % info->gof->num_frames_in_gof; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 572 | |
Danil Chapovalov | a118702 | 2020-02-03 09:16:48 | [diff] [blame] | 573 | if (info->gof->num_ref_pics[gof_idx] > EncodedFrame::kMaxFrameReferences) { |
| 574 | return kDrop; |
| 575 | } |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 576 | // Populate references according to the scalability structure. |
philipel | c9b27d5 | 2016-07-15 13:50:27 | [diff] [blame] | 577 | frame->num_references = info->gof->num_ref_pics[gof_idx]; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 578 | for (size_t i = 0; i < frame->num_references; ++i) { |
philipel | c9b27d5 | 2016-07-15 13:50:27 | [diff] [blame] | 579 | frame->references[i] = Subtract<kPicIdLength>( |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 580 | frame->id.picture_id, info->gof->pid_diff[gof_idx][i]); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 581 | |
| 582 | // If this is a reference to a frame earlier than the last up switch point, |
| 583 | // then ignore this reference. |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 584 | if (UpSwitchInIntervalVp9(frame->id.picture_id, codec_header.temporal_idx, |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 585 | frame->references[i])) { |
| 586 | --frame->num_references; |
| 587 | } |
| 588 | } |
| 589 | |
Ilya Nikolaevskiy | 2ec0c65 | 2019-01-18 10:56:48 | [diff] [blame] | 590 | // Override GOF references. |
| 591 | if (!codec_header.inter_pic_predicted) { |
| 592 | frame->num_references = 0; |
| 593 | } |
| 594 | |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 595 | UnwrapPictureIds(frame); |
| 596 | return kHandOff; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 597 | } |
| 598 | |
| 599 | bool RtpFrameReferenceFinder::MissingRequiredFrameVp9(uint16_t picture_id, |
philipel | c9b27d5 | 2016-07-15 13:50:27 | [diff] [blame] | 600 | const GofInfo& info) { |
| 601 | size_t diff = |
| 602 | ForwardDiff<uint16_t, kPicIdLength>(info.gof->pid_start, picture_id); |
| 603 | size_t gof_idx = diff % info.gof->num_frames_in_gof; |
| 604 | size_t temporal_idx = info.gof->temporal_idx[gof_idx]; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 605 | |
philipel | a157e08 | 2018-05-02 13:19:01 | [diff] [blame] | 606 | if (temporal_idx >= kMaxTemporalLayers) { |
Jonas Olsson | b2b2031 | 2020-01-14 11:11:31 | [diff] [blame] | 607 | RTC_LOG(LS_WARNING) << "At most " << kMaxTemporalLayers |
| 608 | << " temporal " |
| 609 | "layers are supported."; |
philipel | a157e08 | 2018-05-02 13:19:01 | [diff] [blame] | 610 | return true; |
| 611 | } |
| 612 | |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 613 | // For every reference this frame has, check if there is a frame missing in |
| 614 | // the interval (|ref_pid|, |picture_id|) in any of the lower temporal |
| 615 | // layers. If so, we are missing a required frame. |
philipel | c9b27d5 | 2016-07-15 13:50:27 | [diff] [blame] | 616 | uint8_t num_references = info.gof->num_ref_pics[gof_idx]; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 617 | for (size_t i = 0; i < num_references; ++i) { |
| 618 | uint16_t ref_pid = |
philipel | c9b27d5 | 2016-07-15 13:50:27 | [diff] [blame] | 619 | Subtract<kPicIdLength>(picture_id, info.gof->pid_diff[gof_idx][i]); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 620 | for (size_t l = 0; l < temporal_idx; ++l) { |
| 621 | auto missing_frame_it = missing_frames_for_layer_[l].lower_bound(ref_pid); |
| 622 | if (missing_frame_it != missing_frames_for_layer_[l].end() && |
| 623 | AheadOf<uint16_t, kPicIdLength>(picture_id, *missing_frame_it)) { |
| 624 | return true; |
| 625 | } |
| 626 | } |
| 627 | } |
| 628 | return false; |
| 629 | } |
| 630 | |
| 631 | void RtpFrameReferenceFinder::FrameReceivedVp9(uint16_t picture_id, |
philipel | c9b27d5 | 2016-07-15 13:50:27 | [diff] [blame] | 632 | GofInfo* info) { |
| 633 | int last_picture_id = info->last_picture_id; |
philipel | 459f4e3 | 2018-03-02 09:55:12 | [diff] [blame] | 634 | size_t gof_size = std::min(info->gof->num_frames_in_gof, kMaxVp9FramesInGof); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 635 | |
| 636 | // If there is a gap, find which temporal layer the missing frames |
| 637 | // belong to and add the frame as missing for that temporal layer. |
| 638 | // Otherwise, remove this frame from the set of missing frames. |
philipel | c9b27d5 | 2016-07-15 13:50:27 | [diff] [blame] | 639 | if (AheadOf<uint16_t, kPicIdLength>(picture_id, last_picture_id)) { |
| 640 | size_t diff = ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start, |
| 641 | last_picture_id); |
philipel | 459f4e3 | 2018-03-02 09:55:12 | [diff] [blame] | 642 | size_t gof_idx = diff % gof_size; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 643 | |
philipel | c9b27d5 | 2016-07-15 13:50:27 | [diff] [blame] | 644 | last_picture_id = Add<kPicIdLength>(last_picture_id, 1); |
| 645 | while (last_picture_id != picture_id) { |
Yves Gerey | 665174f | 2018-06-19 13:03:05 | [diff] [blame] | 646 | gof_idx = (gof_idx + 1) % gof_size; |
philipel | 459f4e3 | 2018-03-02 09:55:12 | [diff] [blame] | 647 | RTC_CHECK(gof_idx < kMaxVp9FramesInGof); |
| 648 | |
philipel | c9b27d5 | 2016-07-15 13:50:27 | [diff] [blame] | 649 | size_t temporal_idx = info->gof->temporal_idx[gof_idx]; |
philipel | 459f4e3 | 2018-03-02 09:55:12 | [diff] [blame] | 650 | if (temporal_idx >= kMaxTemporalLayers) { |
Jonas Olsson | b2b2031 | 2020-01-14 11:11:31 | [diff] [blame] | 651 | RTC_LOG(LS_WARNING) << "At most " << kMaxTemporalLayers |
| 652 | << " temporal " |
| 653 | "layers are supported."; |
philipel | 459f4e3 | 2018-03-02 09:55:12 | [diff] [blame] | 654 | return; |
| 655 | } |
| 656 | |
philipel | c9b27d5 | 2016-07-15 13:50:27 | [diff] [blame] | 657 | missing_frames_for_layer_[temporal_idx].insert(last_picture_id); |
| 658 | last_picture_id = Add<kPicIdLength>(last_picture_id, 1); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 659 | } |
philipel | 459f4e3 | 2018-03-02 09:55:12 | [diff] [blame] | 660 | |
philipel | c9b27d5 | 2016-07-15 13:50:27 | [diff] [blame] | 661 | info->last_picture_id = last_picture_id; |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 662 | } else { |
| 663 | size_t diff = |
philipel | c9b27d5 | 2016-07-15 13:50:27 | [diff] [blame] | 664 | ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start, picture_id); |
philipel | 459f4e3 | 2018-03-02 09:55:12 | [diff] [blame] | 665 | size_t gof_idx = diff % gof_size; |
| 666 | RTC_CHECK(gof_idx < kMaxVp9FramesInGof); |
| 667 | |
philipel | c9b27d5 | 2016-07-15 13:50:27 | [diff] [blame] | 668 | size_t temporal_idx = info->gof->temporal_idx[gof_idx]; |
philipel | 459f4e3 | 2018-03-02 09:55:12 | [diff] [blame] | 669 | if (temporal_idx >= kMaxTemporalLayers) { |
Jonas Olsson | b2b2031 | 2020-01-14 11:11:31 | [diff] [blame] | 670 | RTC_LOG(LS_WARNING) << "At most " << kMaxTemporalLayers |
| 671 | << " temporal " |
| 672 | "layers are supported."; |
philipel | 459f4e3 | 2018-03-02 09:55:12 | [diff] [blame] | 673 | return; |
| 674 | } |
| 675 | |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 676 | missing_frames_for_layer_[temporal_idx].erase(picture_id); |
| 677 | } |
| 678 | } |
| 679 | |
| 680 | bool RtpFrameReferenceFinder::UpSwitchInIntervalVp9(uint16_t picture_id, |
| 681 | uint8_t temporal_idx, |
| 682 | uint16_t pid_ref) { |
| 683 | for (auto up_switch_it = up_switch_.upper_bound(pid_ref); |
| 684 | up_switch_it != up_switch_.end() && |
| 685 | AheadOf<uint16_t, kPicIdLength>(picture_id, up_switch_it->first); |
| 686 | ++up_switch_it) { |
| 687 | if (up_switch_it->second < temporal_idx) |
| 688 | return true; |
| 689 | } |
| 690 | |
| 691 | return false; |
| 692 | } |
| 693 | |
philipel | afcf7f5 | 2017-04-26 15:17:35 | [diff] [blame] | 694 | void RtpFrameReferenceFinder::UnwrapPictureIds(RtpFrameObject* frame) { |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 695 | for (size_t i = 0; i < frame->num_references; ++i) |
philipel | d4fac69 | 2017-09-04 14:03:46 | [diff] [blame] | 696 | frame->references[i] = unwrapper_.Unwrap(frame->references[i]); |
philipel | 0fa82a6 | 2018-03-19 14:34:53 | [diff] [blame] | 697 | frame->id.picture_id = unwrapper_.Unwrap(frame->id.picture_id); |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 698 | } |
| 699 | |
Johnny Lee | bc7f41b | 2019-05-01 18:41:32 | [diff] [blame] | 700 | RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameH264( |
| 701 | RtpFrameObject* frame) { |
Johannes Kron | a370556 | 2019-08-26 14:37:11 | [diff] [blame] | 702 | const FrameMarking& rtp_frame_marking = frame->GetFrameMarking(); |
Johnny Lee | bc7f41b | 2019-05-01 18:41:32 | [diff] [blame] | 703 | |
Johannes Kron | a370556 | 2019-08-26 14:37:11 | [diff] [blame] | 704 | uint8_t tid = rtp_frame_marking.temporal_id; |
| 705 | bool blSync = rtp_frame_marking.base_layer_sync; |
Johnny Lee | bc7f41b | 2019-05-01 18:41:32 | [diff] [blame] | 706 | |
| 707 | if (tid == kNoTemporalIdx) |
| 708 | return ManageFramePidOrSeqNum(std::move(frame), kNoPictureId); |
| 709 | |
Ilya Nikolaevskiy | 7a709c0 | 2020-01-17 15:40:02 | [diff] [blame] | 710 | // Protect against corrupted packets with arbitrary large temporal idx. |
| 711 | if (tid >= kMaxTemporalLayers) |
| 712 | return kDrop; |
| 713 | |
Johnny Lee | bc7f41b | 2019-05-01 18:41:32 | [diff] [blame] | 714 | frame->id.picture_id = frame->last_seq_num(); |
| 715 | |
| 716 | if (frame->frame_type() == VideoFrameType::kVideoFrameKey) { |
| 717 | // For H264, use last_seq_num_gop_ to simply store last picture id |
| 718 | // as a pair of unpadded and padded sequence numbers. |
| 719 | if (last_seq_num_gop_.empty()) { |
| 720 | last_seq_num_gop_.insert(std::make_pair( |
| 721 | 0, std::make_pair(frame->id.picture_id, frame->id.picture_id))); |
| 722 | } |
| 723 | } |
| 724 | |
| 725 | // Stash if we have no keyframe yet. |
| 726 | if (last_seq_num_gop_.empty()) |
| 727 | return kStash; |
| 728 | |
| 729 | // Check for gap in sequence numbers. Store in |not_yet_received_seq_num_|. |
| 730 | if (frame->frame_type() == VideoFrameType::kVideoFrameDelta) { |
| 731 | uint16_t last_pic_id_padded = last_seq_num_gop_.begin()->second.second; |
| 732 | if (AheadOf<uint16_t>(frame->id.picture_id, last_pic_id_padded)) { |
| 733 | do { |
| 734 | last_pic_id_padded = last_pic_id_padded + 1; |
| 735 | not_yet_received_seq_num_.insert(last_pic_id_padded); |
| 736 | } while (last_pic_id_padded != frame->id.picture_id); |
| 737 | } |
| 738 | } |
| 739 | |
Johannes Kron | a370556 | 2019-08-26 14:37:11 | [diff] [blame] | 740 | int64_t unwrapped_tl0 = tl0_unwrapper_.Unwrap(rtp_frame_marking.tl0_pic_idx); |
Johnny Lee | bc7f41b | 2019-05-01 18:41:32 | [diff] [blame] | 741 | |
| 742 | // Clean up info for base layers that are too old. |
| 743 | int64_t old_tl0_pic_idx = unwrapped_tl0 - kMaxLayerInfo; |
| 744 | auto clean_layer_info_to = layer_info_.lower_bound(old_tl0_pic_idx); |
| 745 | layer_info_.erase(layer_info_.begin(), clean_layer_info_to); |
| 746 | |
| 747 | // Clean up info about not yet received frames that are too old. |
| 748 | uint16_t old_picture_id = frame->id.picture_id - kMaxNotYetReceivedFrames * 2; |
| 749 | auto clean_frames_to = not_yet_received_seq_num_.lower_bound(old_picture_id); |
| 750 | not_yet_received_seq_num_.erase(not_yet_received_seq_num_.begin(), |
| 751 | clean_frames_to); |
| 752 | |
| 753 | if (frame->frame_type() == VideoFrameType::kVideoFrameKey) { |
| 754 | frame->num_references = 0; |
| 755 | layer_info_[unwrapped_tl0].fill(-1); |
| 756 | UpdateDataH264(frame, unwrapped_tl0, tid); |
| 757 | return kHandOff; |
| 758 | } |
| 759 | |
Jonas Olsson | a4d8737 | 2019-07-05 17:08:33 | [diff] [blame] | 760 | auto layer_info_it = |
| 761 | layer_info_.find(tid == 0 ? unwrapped_tl0 - 1 : unwrapped_tl0); |
Johnny Lee | bc7f41b | 2019-05-01 18:41:32 | [diff] [blame] | 762 | |
| 763 | // Stash if we have no base layer frame yet. |
| 764 | if (layer_info_it == layer_info_.end()) |
| 765 | return kStash; |
| 766 | |
| 767 | // Base layer frame. Copy layer info from previous base layer frame. |
| 768 | if (tid == 0) { |
Jonas Olsson | a4d8737 | 2019-07-05 17:08:33 | [diff] [blame] | 769 | layer_info_it = |
| 770 | layer_info_.insert(std::make_pair(unwrapped_tl0, layer_info_it->second)) |
| 771 | .first; |
Johnny Lee | bc7f41b | 2019-05-01 18:41:32 | [diff] [blame] | 772 | frame->num_references = 1; |
| 773 | frame->references[0] = layer_info_it->second[0]; |
| 774 | UpdateDataH264(frame, unwrapped_tl0, tid); |
| 775 | return kHandOff; |
| 776 | } |
| 777 | |
| 778 | // This frame only references its base layer frame. |
| 779 | if (blSync) { |
| 780 | frame->num_references = 1; |
| 781 | frame->references[0] = layer_info_it->second[0]; |
| 782 | UpdateDataH264(frame, unwrapped_tl0, tid); |
| 783 | return kHandOff; |
| 784 | } |
| 785 | |
| 786 | // Find all references for general frame. |
| 787 | frame->num_references = 0; |
| 788 | for (uint8_t layer = 0; layer <= tid; ++layer) { |
| 789 | // Stash if we have not yet received frames on this temporal layer. |
| 790 | if (layer_info_it->second[layer] == -1) |
| 791 | return kStash; |
| 792 | |
| 793 | // Drop if the last frame on this layer is ahead of this frame. A layer sync |
| 794 | // frame was received after this frame for the same base layer frame. |
| 795 | uint16_t last_frame_in_layer = layer_info_it->second[layer]; |
| 796 | if (AheadOf<uint16_t>(last_frame_in_layer, frame->id.picture_id)) |
| 797 | return kDrop; |
| 798 | |
| 799 | // Stash and wait for missing frame between this frame and the reference |
| 800 | auto not_received_seq_num_it = |
| 801 | not_yet_received_seq_num_.upper_bound(last_frame_in_layer); |
| 802 | if (not_received_seq_num_it != not_yet_received_seq_num_.end() && |
| 803 | AheadOf<uint16_t>(frame->id.picture_id, *not_received_seq_num_it)) { |
| 804 | return kStash; |
| 805 | } |
| 806 | |
| 807 | if (!(AheadOf<uint16_t>(frame->id.picture_id, last_frame_in_layer))) { |
| 808 | RTC_LOG(LS_WARNING) << "Frame with picture id " << frame->id.picture_id |
| 809 | << " and packet range [" << frame->first_seq_num() |
| 810 | << ", " << frame->last_seq_num() |
| 811 | << "] already received, " |
Jonas Olsson | b2b2031 | 2020-01-14 11:11:31 | [diff] [blame] | 812 | " dropping frame."; |
Johnny Lee | bc7f41b | 2019-05-01 18:41:32 | [diff] [blame] | 813 | return kDrop; |
| 814 | } |
| 815 | |
| 816 | ++frame->num_references; |
| 817 | frame->references[layer] = last_frame_in_layer; |
| 818 | } |
| 819 | |
| 820 | UpdateDataH264(frame, unwrapped_tl0, tid); |
| 821 | return kHandOff; |
| 822 | } |
| 823 | |
| 824 | void RtpFrameReferenceFinder::UpdateLastPictureIdWithPaddingH264() { |
| 825 | auto seq_num_it = last_seq_num_gop_.begin(); |
| 826 | |
| 827 | // Check if next sequence number is in a stashed padding packet. |
| 828 | uint16_t next_padded_seq_num = seq_num_it->second.second + 1; |
| 829 | auto padding_seq_num_it = stashed_padding_.lower_bound(next_padded_seq_num); |
| 830 | |
| 831 | // Check for more consecutive padding packets to increment |
| 832 | // the "last-picture-id-with-padding" and remove the stashed packets. |
| 833 | while (padding_seq_num_it != stashed_padding_.end() && |
| 834 | *padding_seq_num_it == next_padded_seq_num) { |
| 835 | seq_num_it->second.second = next_padded_seq_num; |
| 836 | ++next_padded_seq_num; |
| 837 | padding_seq_num_it = stashed_padding_.erase(padding_seq_num_it); |
| 838 | } |
| 839 | } |
| 840 | |
| 841 | void RtpFrameReferenceFinder::UpdateLayerInfoH264(RtpFrameObject* frame, |
| 842 | int64_t unwrapped_tl0, |
| 843 | uint8_t temporal_idx) { |
| 844 | auto layer_info_it = layer_info_.find(unwrapped_tl0); |
| 845 | |
| 846 | // Update this layer info and newer. |
| 847 | while (layer_info_it != layer_info_.end()) { |
| 848 | if (layer_info_it->second[temporal_idx] != -1 && |
| 849 | AheadOf<uint16_t>(layer_info_it->second[temporal_idx], |
| 850 | frame->id.picture_id)) { |
| 851 | // Not a newer frame. No subsequent layer info needs update. |
| 852 | break; |
| 853 | } |
| 854 | |
| 855 | layer_info_it->second[temporal_idx] = frame->id.picture_id; |
| 856 | ++unwrapped_tl0; |
| 857 | layer_info_it = layer_info_.find(unwrapped_tl0); |
| 858 | } |
| 859 | |
| 860 | for (size_t i = 0; i < frame->num_references; ++i) |
| 861 | frame->references[i] = rtp_seq_num_unwrapper_.Unwrap(frame->references[i]); |
| 862 | frame->id.picture_id = rtp_seq_num_unwrapper_.Unwrap(frame->id.picture_id); |
| 863 | } |
| 864 | |
| 865 | void RtpFrameReferenceFinder::UpdateDataH264(RtpFrameObject* frame, |
| 866 | int64_t unwrapped_tl0, |
| 867 | uint8_t temporal_idx) { |
| 868 | // Update last_seq_num_gop_ entry for last picture id. |
| 869 | auto seq_num_it = last_seq_num_gop_.begin(); |
| 870 | uint16_t last_pic_id = seq_num_it->second.first; |
| 871 | if (AheadOf<uint16_t>(frame->id.picture_id, last_pic_id)) { |
| 872 | seq_num_it->second.first = frame->id.picture_id; |
| 873 | seq_num_it->second.second = frame->id.picture_id; |
| 874 | } |
| 875 | UpdateLastPictureIdWithPaddingH264(); |
| 876 | |
| 877 | UpdateLayerInfoH264(frame, unwrapped_tl0, temporal_idx); |
| 878 | |
| 879 | // Remove any current packets from |not_yet_received_seq_num_|. |
| 880 | uint16_t last_seq_num_padded = seq_num_it->second.second; |
Jonas Olsson | a4d8737 | 2019-07-05 17:08:33 | [diff] [blame] | 881 | for (uint16_t n = frame->first_seq_num(); AheadOrAt(last_seq_num_padded, n); |
| 882 | ++n) { |
Johnny Lee | bc7f41b | 2019-05-01 18:41:32 | [diff] [blame] | 883 | not_yet_received_seq_num_.erase(n); |
| 884 | } |
| 885 | } |
| 886 | |
philipel | 02447bc | 2016-05-13 13:01:03 | [diff] [blame] | 887 | } // namespace video_coding |
| 888 | } // namespace webrtc |