philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #include "modules/video_coding/rtp_frame_reference_finder.h" |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 12 | |
| 13 | #include <algorithm> |
| 14 | #include <limits> |
| 15 | |
philipel | 1a4746a | 2018-07-09 15:52:29 +0200 | [diff] [blame] | 16 | #include "absl/types/variant.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 17 | #include "modules/video_coding/frame_object.h" |
| 18 | #include "modules/video_coding/packet_buffer.h" |
| 19 | #include "rtc_base/checks.h" |
| 20 | #include "rtc_base/logging.h" |
Karl Wiberg | 80ba333 | 2018-02-05 10:33:35 +0100 | [diff] [blame] | 21 | #include "rtc_base/system/fallthrough.h" |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 22 | |
| 23 | namespace webrtc { |
| 24 | namespace video_coding { |
| 25 | |
| 26 | RtpFrameReferenceFinder::RtpFrameReferenceFinder( |
| 27 | OnCompleteFrameCallback* frame_callback) |
| 28 | : last_picture_id_(-1), |
| 29 | last_unwrap_(-1), |
| 30 | current_ss_idx_(0), |
philipel | 463d301 | 2016-09-09 03:32:44 -0700 | [diff] [blame] | 31 | cleared_to_seq_num_(-1), |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 32 | frame_callback_(frame_callback) {} |
| 33 | |
Mirko Bonadei | 8fdcac3 | 2018-08-28 16:30:18 +0200 | [diff] [blame^] | 34 | RtpFrameReferenceFinder::~RtpFrameReferenceFinder() = default; |
| 35 | |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 36 | void RtpFrameReferenceFinder::ManageFrame( |
| 37 | std::unique_ptr<RtpFrameObject> frame) { |
| 38 | rtc::CritScope lock(&crit_); |
philipel | 463d301 | 2016-09-09 03:32:44 -0700 | [diff] [blame] | 39 | |
| 40 | // If we have cleared past this frame, drop it. |
| 41 | if (cleared_to_seq_num_ != -1 && |
| 42 | AheadOf<uint16_t>(cleared_to_seq_num_, frame->first_seq_num())) { |
| 43 | return; |
| 44 | } |
| 45 | |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 46 | FrameDecision decision = ManageFrameInternal(frame.get()); |
| 47 | |
| 48 | switch (decision) { |
| 49 | case kStash: |
| 50 | if (stashed_frames_.size() > kMaxStashedFrames) |
| 51 | stashed_frames_.pop_back(); |
| 52 | stashed_frames_.push_front(std::move(frame)); |
| 53 | break; |
| 54 | case kHandOff: |
| 55 | frame_callback_->OnCompleteFrame(std::move(frame)); |
| 56 | RetryStashedFrames(); |
| 57 | break; |
| 58 | case kDrop: |
| 59 | break; |
| 60 | } |
| 61 | } |
| 62 | |
| 63 | void RtpFrameReferenceFinder::RetryStashedFrames() { |
| 64 | bool complete_frame = false; |
| 65 | do { |
| 66 | complete_frame = false; |
| 67 | for (auto frame_it = stashed_frames_.begin(); |
| 68 | frame_it != stashed_frames_.end();) { |
| 69 | FrameDecision decision = ManageFrameInternal(frame_it->get()); |
| 70 | |
| 71 | switch (decision) { |
| 72 | case kStash: |
| 73 | ++frame_it; |
| 74 | break; |
| 75 | case kHandOff: |
| 76 | complete_frame = true; |
| 77 | frame_callback_->OnCompleteFrame(std::move(*frame_it)); |
Karl Wiberg | 80ba333 | 2018-02-05 10:33:35 +0100 | [diff] [blame] | 78 | RTC_FALLTHROUGH(); |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 79 | case kDrop: |
| 80 | frame_it = stashed_frames_.erase(frame_it); |
| 81 | } |
| 82 | } |
| 83 | } while (complete_frame); |
| 84 | } |
| 85 | |
| 86 | RtpFrameReferenceFinder::FrameDecision |
| 87 | RtpFrameReferenceFinder::ManageFrameInternal(RtpFrameObject* frame) { |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 88 | switch (frame->codec_type()) { |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 89 | case kVideoCodecVP8: |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 90 | return ManageFrameVp8(frame); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 91 | case kVideoCodecVP9: |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 92 | return ManageFrameVp9(frame); |
Sami Kalliomäki | 9882495 | 2018-08-28 14:39:21 +0200 | [diff] [blame] | 93 | default: { |
| 94 | // Use 15 first bits of frame ID as picture ID if available. |
| 95 | absl::optional<RTPVideoHeader> video_header = frame->GetCodecHeader(); |
| 96 | absl::optional<RTPVideoHeader::GenericDescriptorInfo> generic_info = |
| 97 | video_header ? video_header->generic : absl::nullopt; |
| 98 | return ManageFrameGeneric( |
| 99 | frame, generic_info ? generic_info->frame_id & 0x7fff : kNoPictureId); |
| 100 | } |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 101 | } |
| 102 | } |
| 103 | |
philipel | 9b2ce6b | 2016-07-05 05:04:46 -0700 | [diff] [blame] | 104 | void RtpFrameReferenceFinder::PaddingReceived(uint16_t seq_num) { |
| 105 | rtc::CritScope lock(&crit_); |
| 106 | auto clean_padding_to = |
| 107 | stashed_padding_.lower_bound(seq_num - kMaxPaddingAge); |
| 108 | stashed_padding_.erase(stashed_padding_.begin(), clean_padding_to); |
| 109 | stashed_padding_.insert(seq_num); |
| 110 | UpdateLastPictureIdWithPadding(seq_num); |
| 111 | RetryStashedFrames(); |
| 112 | } |
| 113 | |
philipel | 463d301 | 2016-09-09 03:32:44 -0700 | [diff] [blame] | 114 | void RtpFrameReferenceFinder::ClearTo(uint16_t seq_num) { |
| 115 | rtc::CritScope lock(&crit_); |
| 116 | cleared_to_seq_num_ = seq_num; |
| 117 | |
| 118 | auto it = stashed_frames_.begin(); |
| 119 | while (it != stashed_frames_.end()) { |
| 120 | if (AheadOf<uint16_t>(cleared_to_seq_num_, (*it)->first_seq_num())) { |
| 121 | it = stashed_frames_.erase(it); |
| 122 | } else { |
| 123 | ++it; |
| 124 | } |
| 125 | } |
| 126 | } |
| 127 | |
philipel | 9b2ce6b | 2016-07-05 05:04:46 -0700 | [diff] [blame] | 128 | void RtpFrameReferenceFinder::UpdateLastPictureIdWithPadding(uint16_t seq_num) { |
| 129 | auto gop_seq_num_it = last_seq_num_gop_.upper_bound(seq_num); |
| 130 | |
| 131 | // If this padding packet "belongs" to a group of pictures that we don't track |
| 132 | // anymore, do nothing. |
| 133 | if (gop_seq_num_it == last_seq_num_gop_.begin()) |
| 134 | return; |
| 135 | --gop_seq_num_it; |
| 136 | |
| 137 | // Calculate the next contiuous sequence number and search for it in |
| 138 | // the padding packets we have stashed. |
| 139 | uint16_t next_seq_num_with_padding = gop_seq_num_it->second.second + 1; |
| 140 | auto padding_seq_num_it = |
| 141 | stashed_padding_.lower_bound(next_seq_num_with_padding); |
| 142 | |
| 143 | // While there still are padding packets and those padding packets are |
| 144 | // continuous, then advance the "last-picture-id-with-padding" and remove |
| 145 | // the stashed padding packet. |
| 146 | while (padding_seq_num_it != stashed_padding_.end() && |
| 147 | *padding_seq_num_it == next_seq_num_with_padding) { |
| 148 | gop_seq_num_it->second.second = next_seq_num_with_padding; |
| 149 | ++next_seq_num_with_padding; |
| 150 | padding_seq_num_it = stashed_padding_.erase(padding_seq_num_it); |
| 151 | } |
philipel | 41bb792 | 2017-02-20 07:53:23 -0800 | [diff] [blame] | 152 | |
| 153 | // In the case where the stream has been continuous without any new keyframes |
| 154 | // for a while there is a risk that new frames will appear to be older than |
| 155 | // the keyframe they belong to due to wrapping sequence number. In order |
| 156 | // to prevent this we advance the picture id of the keyframe every so often. |
| 157 | if (ForwardDiff(gop_seq_num_it->first, seq_num) > 10000) { |
| 158 | RTC_DCHECK_EQ(1ul, last_seq_num_gop_.size()); |
| 159 | last_seq_num_gop_[seq_num] = gop_seq_num_it->second; |
| 160 | last_seq_num_gop_.erase(gop_seq_num_it); |
| 161 | } |
philipel | 9b2ce6b | 2016-07-05 05:04:46 -0700 | [diff] [blame] | 162 | } |
| 163 | |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 164 | RtpFrameReferenceFinder::FrameDecision |
| 165 | RtpFrameReferenceFinder::ManageFrameGeneric(RtpFrameObject* frame, |
| 166 | int picture_id) { |
philipel | 647998c | 2016-06-03 09:40:16 -0700 | [diff] [blame] | 167 | // If |picture_id| is specified then we use that to set the frame references, |
| 168 | // otherwise we use sequence number. |
| 169 | if (picture_id != kNoPictureId) { |
| 170 | if (last_unwrap_ == -1) |
| 171 | last_unwrap_ = picture_id; |
| 172 | |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 173 | frame->id.picture_id = unwrapper_.Unwrap(picture_id); |
philipel | 647998c | 2016-06-03 09:40:16 -0700 | [diff] [blame] | 174 | frame->num_references = frame->frame_type() == kVideoFrameKey ? 0 : 1; |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 175 | frame->references[0] = frame->id.picture_id - 1; |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 176 | return kHandOff; |
philipel | 647998c | 2016-06-03 09:40:16 -0700 | [diff] [blame] | 177 | } |
| 178 | |
philipel | 9b2ce6b | 2016-07-05 05:04:46 -0700 | [diff] [blame] | 179 | if (frame->frame_type() == kVideoFrameKey) { |
| 180 | last_seq_num_gop_.insert(std::make_pair( |
| 181 | frame->last_seq_num(), |
| 182 | std::make_pair(frame->last_seq_num(), frame->last_seq_num()))); |
| 183 | } |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 184 | |
| 185 | // We have received a frame but not yet a keyframe, stash this frame. |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 186 | if (last_seq_num_gop_.empty()) |
| 187 | return kStash; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 188 | |
| 189 | // Clean up info for old keyframes but make sure to keep info |
| 190 | // for the last keyframe. |
| 191 | auto clean_to = last_seq_num_gop_.lower_bound(frame->last_seq_num() - 100); |
philipel | 41bb792 | 2017-02-20 07:53:23 -0800 | [diff] [blame] | 192 | for (auto it = last_seq_num_gop_.begin(); |
| 193 | it != clean_to && last_seq_num_gop_.size() > 1;) { |
| 194 | it = last_seq_num_gop_.erase(it); |
| 195 | } |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 196 | |
| 197 | // Find the last sequence number of the last frame for the keyframe |
| 198 | // that this frame indirectly references. |
| 199 | auto seq_num_it = last_seq_num_gop_.upper_bound(frame->last_seq_num()); |
philipel | 9b2ce6b | 2016-07-05 05:04:46 -0700 | [diff] [blame] | 200 | if (seq_num_it == last_seq_num_gop_.begin()) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 201 | RTC_LOG(LS_WARNING) << "Generic frame with packet range [" |
| 202 | << frame->first_seq_num() << ", " |
| 203 | << frame->last_seq_num() |
| 204 | << "] has no GoP, dropping frame."; |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 205 | return kDrop; |
philipel | 9b2ce6b | 2016-07-05 05:04:46 -0700 | [diff] [blame] | 206 | } |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 207 | seq_num_it--; |
| 208 | |
| 209 | // Make sure the packet sequence numbers are continuous, otherwise stash |
| 210 | // this frame. |
philipel | 9b2ce6b | 2016-07-05 05:04:46 -0700 | [diff] [blame] | 211 | uint16_t last_picture_id_gop = seq_num_it->second.first; |
| 212 | uint16_t last_picture_id_with_padding_gop = seq_num_it->second.second; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 213 | if (frame->frame_type() == kVideoFrameDelta) { |
philipel | 9b2ce6b | 2016-07-05 05:04:46 -0700 | [diff] [blame] | 214 | uint16_t prev_seq_num = frame->first_seq_num() - 1; |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 215 | |
| 216 | if (prev_seq_num != last_picture_id_with_padding_gop) |
| 217 | return kStash; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 218 | } |
| 219 | |
| 220 | RTC_DCHECK(AheadOrAt(frame->last_seq_num(), seq_num_it->first)); |
| 221 | |
| 222 | // Since keyframes can cause reordering we can't simply assign the |
| 223 | // picture id according to some incrementing counter. |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 224 | frame->id.picture_id = frame->last_seq_num(); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 225 | frame->num_references = frame->frame_type() == kVideoFrameDelta; |
philipel | d4fac69 | 2017-09-04 07:03:46 -0700 | [diff] [blame] | 226 | frame->references[0] = generic_unwrapper_.Unwrap(last_picture_id_gop); |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 227 | if (AheadOf<uint16_t>(frame->id.picture_id, last_picture_id_gop)) { |
| 228 | seq_num_it->second.first = frame->id.picture_id; |
| 229 | seq_num_it->second.second = frame->id.picture_id; |
philipel | 9b2ce6b | 2016-07-05 05:04:46 -0700 | [diff] [blame] | 230 | } |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 231 | |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 232 | last_picture_id_ = frame->id.picture_id; |
| 233 | UpdateLastPictureIdWithPadding(frame->id.picture_id); |
| 234 | frame->id.picture_id = generic_unwrapper_.Unwrap(frame->id.picture_id); |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 235 | return kHandOff; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 236 | } |
| 237 | |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 238 | RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameVp8( |
| 239 | RtpFrameObject* frame) { |
Sami Kalliomäki | 9882495 | 2018-08-28 14:39:21 +0200 | [diff] [blame] | 240 | absl::optional<RTPVideoHeader> video_header = frame->GetCodecHeader(); |
| 241 | if (!video_header) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 242 | RTC_LOG(LS_WARNING) |
| 243 | << "Failed to get codec header from frame, dropping frame."; |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 244 | return kDrop; |
philipel | d4fac69 | 2017-09-04 07:03:46 -0700 | [diff] [blame] | 245 | } |
Sami Kalliomäki | 9882495 | 2018-08-28 14:39:21 +0200 | [diff] [blame] | 246 | RTPVideoTypeHeader rtp_codec_header = video_header->video_type_header; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 247 | |
philipel | 1a4746a | 2018-07-09 15:52:29 +0200 | [diff] [blame] | 248 | const RTPVideoHeaderVP8& codec_header = |
Sami Kalliomäki | 9882495 | 2018-08-28 14:39:21 +0200 | [diff] [blame] | 249 | absl::get<RTPVideoHeaderVP8>(rtp_codec_header); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 250 | |
| 251 | if (codec_header.pictureId == kNoPictureId || |
| 252 | codec_header.temporalIdx == kNoTemporalIdx || |
| 253 | codec_header.tl0PicIdx == kNoTl0PicIdx) { |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 254 | return ManageFrameGeneric(std::move(frame), codec_header.pictureId); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 255 | } |
| 256 | |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 257 | frame->id.picture_id = codec_header.pictureId % kPicIdLength; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 258 | |
| 259 | if (last_unwrap_ == -1) |
| 260 | last_unwrap_ = codec_header.pictureId; |
| 261 | |
| 262 | if (last_picture_id_ == -1) |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 263 | last_picture_id_ = frame->id.picture_id; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 264 | |
| 265 | // Find if there has been a gap in fully received frames and save the picture |
| 266 | // id of those frames in |not_yet_received_frames_|. |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 267 | if (AheadOf<uint16_t, kPicIdLength>(frame->id.picture_id, last_picture_id_)) { |
philipel | 9bd1d66 | 2017-07-14 04:52:01 -0700 | [diff] [blame] | 268 | do { |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 269 | last_picture_id_ = Add<kPicIdLength>(last_picture_id_, 1); |
philipel | 9bd1d66 | 2017-07-14 04:52:01 -0700 | [diff] [blame] | 270 | not_yet_received_frames_.insert(last_picture_id_); |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 271 | } while (last_picture_id_ != frame->id.picture_id); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 272 | } |
| 273 | |
philipel | 57ec685 | 2018-07-03 18:09:32 +0200 | [diff] [blame] | 274 | int64_t unwrapped_tl0 = tl0_unwrapper_.Unwrap(codec_header.tl0PicIdx); |
| 275 | |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 276 | // Clean up info for base layers that are too old. |
philipel | 57ec685 | 2018-07-03 18:09:32 +0200 | [diff] [blame] | 277 | int64_t old_tl0_pic_idx = unwrapped_tl0 - kMaxLayerInfo; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 278 | auto clean_layer_info_to = layer_info_.lower_bound(old_tl0_pic_idx); |
| 279 | layer_info_.erase(layer_info_.begin(), clean_layer_info_to); |
| 280 | |
| 281 | // Clean up info about not yet received frames that are too old. |
| 282 | uint16_t old_picture_id = |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 283 | Subtract<kPicIdLength>(frame->id.picture_id, kMaxNotYetReceivedFrames); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 284 | auto clean_frames_to = not_yet_received_frames_.lower_bound(old_picture_id); |
| 285 | not_yet_received_frames_.erase(not_yet_received_frames_.begin(), |
| 286 | clean_frames_to); |
| 287 | |
| 288 | if (frame->frame_type() == kVideoFrameKey) { |
| 289 | frame->num_references = 0; |
philipel | 57ec685 | 2018-07-03 18:09:32 +0200 | [diff] [blame] | 290 | layer_info_[unwrapped_tl0].fill(-1); |
| 291 | UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx); |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 292 | return kHandOff; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 293 | } |
| 294 | |
philipel | 57ec685 | 2018-07-03 18:09:32 +0200 | [diff] [blame] | 295 | auto layer_info_it = layer_info_.find( |
| 296 | codec_header.temporalIdx == 0 ? unwrapped_tl0 - 1 : unwrapped_tl0); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 297 | |
| 298 | // If we don't have the base layer frame yet, stash this frame. |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 299 | if (layer_info_it == layer_info_.end()) |
| 300 | return kStash; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 301 | |
| 302 | // A non keyframe base layer frame has been received, copy the layer info |
| 303 | // from the previous base layer frame and set a reference to the previous |
| 304 | // base layer frame. |
| 305 | if (codec_header.temporalIdx == 0) { |
| 306 | layer_info_it = |
philipel | 57ec685 | 2018-07-03 18:09:32 +0200 | [diff] [blame] | 307 | layer_info_.emplace(unwrapped_tl0, layer_info_it->second).first; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 308 | frame->num_references = 1; |
| 309 | frame->references[0] = layer_info_it->second[0]; |
philipel | 57ec685 | 2018-07-03 18:09:32 +0200 | [diff] [blame] | 310 | UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx); |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 311 | return kHandOff; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 312 | } |
| 313 | |
| 314 | // Layer sync frame, this frame only references its base layer frame. |
| 315 | if (codec_header.layerSync) { |
| 316 | frame->num_references = 1; |
| 317 | frame->references[0] = layer_info_it->second[0]; |
| 318 | |
philipel | 57ec685 | 2018-07-03 18:09:32 +0200 | [diff] [blame] | 319 | UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx); |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 320 | return kHandOff; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 321 | } |
| 322 | |
| 323 | // Find all references for this frame. |
| 324 | frame->num_references = 0; |
| 325 | for (uint8_t layer = 0; layer <= codec_header.temporalIdx; ++layer) { |
philipel | d268d6f | 2016-09-15 13:43:13 +0200 | [diff] [blame] | 326 | // If we have not yet received a previous frame on this temporal layer, |
| 327 | // stash this frame. |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 328 | if (layer_info_it->second[layer] == -1) |
| 329 | return kStash; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 330 | |
philipel | 86b92e0 | 2016-10-24 07:11:53 -0700 | [diff] [blame] | 331 | // If the last frame on this layer is ahead of this frame it means that |
| 332 | // a layer sync frame has been received after this frame for the same |
| 333 | // base layer frame, drop this frame. |
| 334 | if (AheadOf<uint16_t, kPicIdLength>(layer_info_it->second[layer], |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 335 | frame->id.picture_id)) { |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 336 | return kDrop; |
philipel | 86b92e0 | 2016-10-24 07:11:53 -0700 | [diff] [blame] | 337 | } |
| 338 | |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 339 | // If we have not yet received a frame between this frame and the referenced |
| 340 | // frame then we have to wait for that frame to be completed first. |
| 341 | auto not_received_frame_it = |
| 342 | not_yet_received_frames_.upper_bound(layer_info_it->second[layer]); |
| 343 | if (not_received_frame_it != not_yet_received_frames_.end() && |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 344 | AheadOf<uint16_t, kPicIdLength>(frame->id.picture_id, |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 345 | *not_received_frame_it)) { |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 346 | return kStash; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 347 | } |
| 348 | |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 349 | if (!(AheadOf<uint16_t, kPicIdLength>(frame->id.picture_id, |
philipel | 57f19cc | 2017-03-07 03:54:05 -0800 | [diff] [blame] | 350 | layer_info_it->second[layer]))) { |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 351 | RTC_LOG(LS_WARNING) << "Frame with picture id " << frame->id.picture_id |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 352 | << " and packet range [" << frame->first_seq_num() |
| 353 | << ", " << frame->last_seq_num() |
| 354 | << "] already received, " |
| 355 | << " dropping frame."; |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 356 | return kDrop; |
philipel | 57f19cc | 2017-03-07 03:54:05 -0800 | [diff] [blame] | 357 | } |
| 358 | |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 359 | ++frame->num_references; |
| 360 | frame->references[layer] = layer_info_it->second[layer]; |
| 361 | } |
| 362 | |
philipel | 57ec685 | 2018-07-03 18:09:32 +0200 | [diff] [blame] | 363 | UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx); |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 364 | return kHandOff; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 365 | } |
| 366 | |
philipel | 57ec685 | 2018-07-03 18:09:32 +0200 | [diff] [blame] | 367 | void RtpFrameReferenceFinder::UpdateLayerInfoVp8(RtpFrameObject* frame, |
| 368 | int64_t unwrapped_tl0, |
| 369 | uint8_t temporal_idx) { |
| 370 | auto layer_info_it = layer_info_.find(unwrapped_tl0); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 371 | |
| 372 | // Update this layer info and newer. |
| 373 | while (layer_info_it != layer_info_.end()) { |
philipel | 57ec685 | 2018-07-03 18:09:32 +0200 | [diff] [blame] | 374 | if (layer_info_it->second[temporal_idx] != -1 && |
| 375 | AheadOf<uint16_t, kPicIdLength>(layer_info_it->second[temporal_idx], |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 376 | frame->id.picture_id)) { |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 377 | // The frame was not newer, then no subsequent layer info have to be |
| 378 | // update. |
| 379 | break; |
| 380 | } |
| 381 | |
philipel | 57ec685 | 2018-07-03 18:09:32 +0200 | [diff] [blame] | 382 | layer_info_it->second[temporal_idx] = frame->id.picture_id; |
| 383 | ++unwrapped_tl0; |
| 384 | layer_info_it = layer_info_.find(unwrapped_tl0); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 385 | } |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 386 | not_yet_received_frames_.erase(frame->id.picture_id); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 387 | |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 388 | UnwrapPictureIds(frame); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 389 | } |
| 390 | |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 391 | RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameVp9( |
| 392 | RtpFrameObject* frame) { |
Sami Kalliomäki | 9882495 | 2018-08-28 14:39:21 +0200 | [diff] [blame] | 393 | absl::optional<RTPVideoHeader> video_header = frame->GetCodecHeader(); |
| 394 | if (!video_header) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 395 | RTC_LOG(LS_WARNING) |
| 396 | << "Failed to get codec header from frame, dropping frame."; |
philipel | 4c14009 | 2017-08-31 08:31:45 -0700 | [diff] [blame] | 397 | return kDrop; |
philipel | d4fac69 | 2017-09-04 07:03:46 -0700 | [diff] [blame] | 398 | } |
Sami Kalliomäki | 9882495 | 2018-08-28 14:39:21 +0200 | [diff] [blame] | 399 | RTPVideoTypeHeader rtp_codec_header = video_header->video_type_header; |
philipel | 4c14009 | 2017-08-31 08:31:45 -0700 | [diff] [blame] | 400 | |
philipel | 1a4746a | 2018-07-09 15:52:29 +0200 | [diff] [blame] | 401 | const RTPVideoHeaderVP9& codec_header = |
Sami Kalliomäki | 9882495 | 2018-08-28 14:39:21 +0200 | [diff] [blame] | 402 | absl::get<RTPVideoHeaderVP9>(rtp_codec_header); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 403 | |
philipel | 647998c | 2016-06-03 09:40:16 -0700 | [diff] [blame] | 404 | if (codec_header.picture_id == kNoPictureId || |
Sergey Silkin | d34a188 | 2018-08-20 16:46:05 +0200 | [diff] [blame] | 405 | codec_header.temporal_idx == kNoTemporalIdx) { |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 406 | return ManageFrameGeneric(std::move(frame), codec_header.picture_id); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 407 | } |
| 408 | |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 409 | frame->id.spatial_layer = codec_header.spatial_idx; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 410 | frame->inter_layer_predicted = codec_header.inter_layer_predicted; |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 411 | frame->id.picture_id = codec_header.picture_id % kPicIdLength; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 412 | |
| 413 | if (last_unwrap_ == -1) |
| 414 | last_unwrap_ = codec_header.picture_id; |
| 415 | |
| 416 | if (last_picture_id_ == -1) |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 417 | last_picture_id_ = frame->id.picture_id; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 418 | |
| 419 | if (codec_header.flexible_mode) { |
| 420 | frame->num_references = codec_header.num_ref_pics; |
| 421 | for (size_t i = 0; i < frame->num_references; ++i) { |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 422 | frame->references[i] = Subtract<kPicIdLength>(frame->id.picture_id, |
| 423 | codec_header.pid_diff[i]); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 424 | } |
| 425 | |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 426 | UnwrapPictureIds(frame); |
| 427 | return kHandOff; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 428 | } |
| 429 | |
Sergey Silkin | d34a188 | 2018-08-20 16:46:05 +0200 | [diff] [blame] | 430 | if (codec_header.tl0_pic_idx == kNoTl0PicIdx) { |
| 431 | RTC_LOG(LS_WARNING) << "TL0PICIDX is expected to be present in " |
| 432 | "non-flexible mode."; |
| 433 | return kDrop; |
| 434 | } |
| 435 | |
philipel | 1564360 | 2018-05-03 16:14:13 +0200 | [diff] [blame] | 436 | GofInfo* info; |
philipel | 57ec685 | 2018-07-03 18:09:32 +0200 | [diff] [blame] | 437 | int64_t unwrapped_tl0 = tl0_unwrapper_.Unwrap(codec_header.tl0_pic_idx); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 438 | if (codec_header.ss_data_available) { |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 439 | if (codec_header.temporal_idx != 0) { |
philipel | 1564360 | 2018-05-03 16:14:13 +0200 | [diff] [blame] | 440 | RTC_LOG(LS_WARNING) << "Received scalability structure on a non base " |
| 441 | "layer frame. Scalability structure ignored."; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 442 | } else { |
| 443 | current_ss_idx_ = Add<kMaxGofSaved>(current_ss_idx_, 1); |
Natalie Silvanovich | 3ea3e30 | 2018-05-16 11:03:12 -0700 | [diff] [blame] | 444 | if (codec_header.gof.num_frames_in_gof == 0 || |
| 445 | codec_header.gof.num_frames_in_gof > kMaxVp9FramesInGof) { |
| 446 | return kDrop; |
| 447 | } |
| 448 | |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 449 | scalability_structures_[current_ss_idx_] = codec_header.gof; |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 450 | scalability_structures_[current_ss_idx_].pid_start = frame->id.picture_id; |
philipel | 57ec685 | 2018-07-03 18:09:32 +0200 | [diff] [blame] | 451 | gof_info_.emplace(unwrapped_tl0, |
philipel | 1564360 | 2018-05-03 16:14:13 +0200 | [diff] [blame] | 452 | GofInfo(&scalability_structures_[current_ss_idx_], |
| 453 | frame->id.picture_id)); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 454 | } |
philipel | 1564360 | 2018-05-03 16:14:13 +0200 | [diff] [blame] | 455 | |
philipel | 57ec685 | 2018-07-03 18:09:32 +0200 | [diff] [blame] | 456 | const auto gof_info_it = gof_info_.find(unwrapped_tl0); |
philipel | 1564360 | 2018-05-03 16:14:13 +0200 | [diff] [blame] | 457 | if (gof_info_it == gof_info_.end()) |
| 458 | return kStash; |
| 459 | |
| 460 | info = &gof_info_it->second; |
| 461 | |
| 462 | if (frame->frame_type() == kVideoFrameKey) { |
| 463 | frame->num_references = 0; |
| 464 | FrameReceivedVp9(frame->id.picture_id, info); |
| 465 | UnwrapPictureIds(frame); |
| 466 | return kHandOff; |
| 467 | } |
| 468 | } else { |
| 469 | if (frame->frame_type() == kVideoFrameKey) { |
| 470 | RTC_LOG(LS_WARNING) << "Received keyframe without scalability structure"; |
| 471 | return kDrop; |
| 472 | } |
| 473 | |
philipel | 57ec685 | 2018-07-03 18:09:32 +0200 | [diff] [blame] | 474 | auto gof_info_it = gof_info_.find( |
| 475 | (codec_header.temporal_idx == 0) ? unwrapped_tl0 - 1 : unwrapped_tl0); |
philipel | 1564360 | 2018-05-03 16:14:13 +0200 | [diff] [blame] | 476 | |
| 477 | // Gof info for this frame is not available yet, stash this frame. |
| 478 | if (gof_info_it == gof_info_.end()) |
| 479 | return kStash; |
| 480 | |
| 481 | if (codec_header.temporal_idx == 0) { |
philipel | 57ec685 | 2018-07-03 18:09:32 +0200 | [diff] [blame] | 482 | gof_info_it = gof_info_ |
| 483 | .emplace(unwrapped_tl0, GofInfo(gof_info_it->second.gof, |
| 484 | frame->id.picture_id)) |
| 485 | .first; |
philipel | 1564360 | 2018-05-03 16:14:13 +0200 | [diff] [blame] | 486 | } |
| 487 | |
| 488 | info = &gof_info_it->second; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 489 | } |
| 490 | |
| 491 | // Clean up info for base layers that are too old. |
philipel | 57ec685 | 2018-07-03 18:09:32 +0200 | [diff] [blame] | 492 | int64_t old_tl0_pic_idx = unwrapped_tl0 - kMaxGofSaved; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 493 | auto clean_gof_info_to = gof_info_.lower_bound(old_tl0_pic_idx); |
| 494 | gof_info_.erase(gof_info_.begin(), clean_gof_info_to); |
| 495 | |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 496 | FrameReceivedVp9(frame->id.picture_id, info); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 497 | |
| 498 | // Make sure we don't miss any frame that could potentially have the |
| 499 | // up switch flag set. |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 500 | if (MissingRequiredFrameVp9(frame->id.picture_id, *info)) |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 501 | return kStash; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 502 | |
philipel | 1564360 | 2018-05-03 16:14:13 +0200 | [diff] [blame] | 503 | if (codec_header.temporal_up_switch) |
| 504 | up_switch_.emplace(frame->id.picture_id, codec_header.temporal_idx); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 505 | |
| 506 | // Clean out old info about up switch frames. |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 507 | uint16_t old_picture_id = Subtract<kPicIdLength>(frame->id.picture_id, 50); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 508 | auto up_switch_erase_to = up_switch_.lower_bound(old_picture_id); |
| 509 | up_switch_.erase(up_switch_.begin(), up_switch_erase_to); |
| 510 | |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 511 | size_t diff = ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start, |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 512 | frame->id.picture_id); |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 513 | size_t gof_idx = diff % info->gof->num_frames_in_gof; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 514 | |
| 515 | // Populate references according to the scalability structure. |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 516 | frame->num_references = info->gof->num_ref_pics[gof_idx]; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 517 | for (size_t i = 0; i < frame->num_references; ++i) { |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 518 | frame->references[i] = Subtract<kPicIdLength>( |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 519 | frame->id.picture_id, info->gof->pid_diff[gof_idx][i]); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 520 | |
| 521 | // If this is a reference to a frame earlier than the last up switch point, |
| 522 | // then ignore this reference. |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 523 | if (UpSwitchInIntervalVp9(frame->id.picture_id, codec_header.temporal_idx, |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 524 | frame->references[i])) { |
| 525 | --frame->num_references; |
| 526 | } |
| 527 | } |
| 528 | |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 529 | UnwrapPictureIds(frame); |
| 530 | return kHandOff; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 531 | } |
| 532 | |
| 533 | bool RtpFrameReferenceFinder::MissingRequiredFrameVp9(uint16_t picture_id, |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 534 | const GofInfo& info) { |
| 535 | size_t diff = |
| 536 | ForwardDiff<uint16_t, kPicIdLength>(info.gof->pid_start, picture_id); |
| 537 | size_t gof_idx = diff % info.gof->num_frames_in_gof; |
| 538 | size_t temporal_idx = info.gof->temporal_idx[gof_idx]; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 539 | |
philipel | a157e08 | 2018-05-02 15:19:01 +0200 | [diff] [blame] | 540 | if (temporal_idx >= kMaxTemporalLayers) { |
| 541 | RTC_LOG(LS_WARNING) << "At most " << kMaxTemporalLayers << " temporal " |
| 542 | << "layers are supported."; |
| 543 | return true; |
| 544 | } |
| 545 | |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 546 | // For every reference this frame has, check if there is a frame missing in |
| 547 | // the interval (|ref_pid|, |picture_id|) in any of the lower temporal |
| 548 | // layers. If so, we are missing a required frame. |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 549 | uint8_t num_references = info.gof->num_ref_pics[gof_idx]; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 550 | for (size_t i = 0; i < num_references; ++i) { |
| 551 | uint16_t ref_pid = |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 552 | Subtract<kPicIdLength>(picture_id, info.gof->pid_diff[gof_idx][i]); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 553 | for (size_t l = 0; l < temporal_idx; ++l) { |
| 554 | auto missing_frame_it = missing_frames_for_layer_[l].lower_bound(ref_pid); |
| 555 | if (missing_frame_it != missing_frames_for_layer_[l].end() && |
| 556 | AheadOf<uint16_t, kPicIdLength>(picture_id, *missing_frame_it)) { |
| 557 | return true; |
| 558 | } |
| 559 | } |
| 560 | } |
| 561 | return false; |
| 562 | } |
| 563 | |
| 564 | void RtpFrameReferenceFinder::FrameReceivedVp9(uint16_t picture_id, |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 565 | GofInfo* info) { |
| 566 | int last_picture_id = info->last_picture_id; |
philipel | 459f4e3 | 2018-03-02 10:55:12 +0100 | [diff] [blame] | 567 | size_t gof_size = std::min(info->gof->num_frames_in_gof, kMaxVp9FramesInGof); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 568 | |
| 569 | // If there is a gap, find which temporal layer the missing frames |
| 570 | // belong to and add the frame as missing for that temporal layer. |
| 571 | // Otherwise, remove this frame from the set of missing frames. |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 572 | if (AheadOf<uint16_t, kPicIdLength>(picture_id, last_picture_id)) { |
| 573 | size_t diff = ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start, |
| 574 | last_picture_id); |
philipel | 459f4e3 | 2018-03-02 10:55:12 +0100 | [diff] [blame] | 575 | size_t gof_idx = diff % gof_size; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 576 | |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 577 | last_picture_id = Add<kPicIdLength>(last_picture_id, 1); |
| 578 | while (last_picture_id != picture_id) { |
Yves Gerey | 665174f | 2018-06-19 15:03:05 +0200 | [diff] [blame] | 579 | gof_idx = (gof_idx + 1) % gof_size; |
philipel | 459f4e3 | 2018-03-02 10:55:12 +0100 | [diff] [blame] | 580 | RTC_CHECK(gof_idx < kMaxVp9FramesInGof); |
| 581 | |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 582 | size_t temporal_idx = info->gof->temporal_idx[gof_idx]; |
philipel | 459f4e3 | 2018-03-02 10:55:12 +0100 | [diff] [blame] | 583 | if (temporal_idx >= kMaxTemporalLayers) { |
| 584 | RTC_LOG(LS_WARNING) << "At most " << kMaxTemporalLayers << " temporal " |
| 585 | << "layers are supported."; |
| 586 | return; |
| 587 | } |
| 588 | |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 589 | missing_frames_for_layer_[temporal_idx].insert(last_picture_id); |
| 590 | last_picture_id = Add<kPicIdLength>(last_picture_id, 1); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 591 | } |
philipel | 459f4e3 | 2018-03-02 10:55:12 +0100 | [diff] [blame] | 592 | |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 593 | info->last_picture_id = last_picture_id; |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 594 | } else { |
| 595 | size_t diff = |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 596 | ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start, picture_id); |
philipel | 459f4e3 | 2018-03-02 10:55:12 +0100 | [diff] [blame] | 597 | size_t gof_idx = diff % gof_size; |
| 598 | RTC_CHECK(gof_idx < kMaxVp9FramesInGof); |
| 599 | |
philipel | c9b27d5 | 2016-07-15 06:50:27 -0700 | [diff] [blame] | 600 | size_t temporal_idx = info->gof->temporal_idx[gof_idx]; |
philipel | 459f4e3 | 2018-03-02 10:55:12 +0100 | [diff] [blame] | 601 | if (temporal_idx >= kMaxTemporalLayers) { |
| 602 | RTC_LOG(LS_WARNING) << "At most " << kMaxTemporalLayers << " temporal " |
| 603 | << "layers are supported."; |
| 604 | return; |
| 605 | } |
| 606 | |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 607 | missing_frames_for_layer_[temporal_idx].erase(picture_id); |
| 608 | } |
| 609 | } |
| 610 | |
| 611 | bool RtpFrameReferenceFinder::UpSwitchInIntervalVp9(uint16_t picture_id, |
| 612 | uint8_t temporal_idx, |
| 613 | uint16_t pid_ref) { |
| 614 | for (auto up_switch_it = up_switch_.upper_bound(pid_ref); |
| 615 | up_switch_it != up_switch_.end() && |
| 616 | AheadOf<uint16_t, kPicIdLength>(picture_id, up_switch_it->first); |
| 617 | ++up_switch_it) { |
| 618 | if (up_switch_it->second < temporal_idx) |
| 619 | return true; |
| 620 | } |
| 621 | |
| 622 | return false; |
| 623 | } |
| 624 | |
philipel | afcf7f5 | 2017-04-26 08:17:35 -0700 | [diff] [blame] | 625 | void RtpFrameReferenceFinder::UnwrapPictureIds(RtpFrameObject* frame) { |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 626 | for (size_t i = 0; i < frame->num_references; ++i) |
philipel | d4fac69 | 2017-09-04 07:03:46 -0700 | [diff] [blame] | 627 | frame->references[i] = unwrapper_.Unwrap(frame->references[i]); |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 628 | frame->id.picture_id = unwrapper_.Unwrap(frame->id.picture_id); |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 629 | } |
| 630 | |
philipel | 02447bc | 2016-05-13 06:01:03 -0700 | [diff] [blame] | 631 | } // namespace video_coding |
| 632 | } // namespace webrtc |