philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #include "modules/video_coding/frame_buffer2.h" |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 12 | |
| 13 | #include <algorithm> |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 14 | #include <cstdlib> |
| 15 | #include <iterator> |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 16 | #include <queue> |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 17 | #include <utility> |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 18 | #include <vector> |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 19 | |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 20 | #include "absl/memory/memory.h" |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 21 | #include "api/video/encoded_image.h" |
| 22 | #include "api/video/video_timing.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 23 | #include "modules/video_coding/include/video_coding_defines.h" |
| 24 | #include "modules/video_coding/jitter_estimator.h" |
| 25 | #include "modules/video_coding/timing.h" |
| 26 | #include "rtc_base/checks.h" |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 27 | #include "rtc_base/experiments/rtt_mult_experiment.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 28 | #include "rtc_base/logging.h" |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 29 | #include "rtc_base/numerics/sequence_number_util.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 30 | #include "rtc_base/trace_event.h" |
| 31 | #include "system_wrappers/include/clock.h" |
philipel | 707f278 | 2017-10-02 14:10:28 +0200 | [diff] [blame] | 32 | #include "system_wrappers/include/field_trial.h" |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 33 | |
| 34 | namespace webrtc { |
| 35 | namespace video_coding { |
| 36 | |
| 37 | namespace { |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 38 | // Max number of frames the buffer will hold. |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 39 | constexpr size_t kMaxFramesBuffered = 800; |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 40 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 41 | // Max number of decoded frame info that will be saved. |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 42 | constexpr int kMaxFramesHistory = 1 << 13; |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 43 | |
Ilya Nikolaevskiy | 8c4fe16 | 2018-02-27 15:49:47 +0100 | [diff] [blame] | 44 | // The time it's allowed for a frame to be late to its rendering prediction and |
| 45 | // still be rendered. |
Ilya Nikolaevskiy | 7eef007 | 2018-02-28 09:59:26 +0100 | [diff] [blame] | 46 | constexpr int kMaxAllowedFrameDelayMs = 5; |
Ilya Nikolaevskiy | 8c4fe16 | 2018-02-27 15:49:47 +0100 | [diff] [blame] | 47 | |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 48 | constexpr int64_t kLogNonDecodedIntervalMs = 5000; |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 49 | } // namespace |
| 50 | |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 51 | FrameBuffer::FrameBuffer(Clock* clock, |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 52 | VCMTiming* timing, |
| 53 | VCMReceiveStatisticsCallback* stats_callback) |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 54 | : decoded_frames_history_(kMaxFramesHistory), |
| 55 | clock_(clock), |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 56 | callback_queue_(nullptr), |
Niels Möller | d9c2d94 | 2019-04-30 09:16:36 +0200 | [diff] [blame] | 57 | jitter_estimator_(clock), |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 58 | timing_(timing), |
philipel | 4f6cd6a | 2016-08-03 10:59:32 +0200 | [diff] [blame] | 59 | inter_frame_delay_(clock_->TimeInMilliseconds()), |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 60 | stopped_(false), |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 61 | protection_mode_(kProtectionNack), |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 62 | stats_callback_(stats_callback), |
Elad Alon | e4b5023 | 2019-01-14 18:56:14 +0100 | [diff] [blame] | 63 | last_log_non_decoded_ms_(-kLogNonDecodedIntervalMs), |
| 64 | add_rtt_to_playout_delay_( |
“Michael | d3a4ebe | 2019-06-07 03:55:01 -0500 | [diff] [blame] | 65 | webrtc::field_trial::IsEnabled("WebRTC-AddRttToPlayoutDelay")), |
| 66 | rtt_mult_settings_(RttMultExperiment::GetRttMultValue()) {} |
philipel | 266f0a4 | 2016-11-28 08:49:07 -0800 | [diff] [blame] | 67 | |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 68 | FrameBuffer::~FrameBuffer() {} |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 69 | |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 70 | void FrameBuffer::NextFrame( |
| 71 | int64_t max_wait_time_ms, |
| 72 | bool keyframe_required, |
| 73 | rtc::TaskQueue* callback_queue, |
| 74 | std::function<void(std::unique_ptr<EncodedFrame>, ReturnReason)> handler) { |
| 75 | RTC_DCHECK_RUN_ON(callback_queue); |
| 76 | TRACE_EVENT0("webrtc", "FrameBuffer::NextFrame"); |
| 77 | int64_t latest_return_time_ms = |
| 78 | clock_->TimeInMilliseconds() + max_wait_time_ms; |
| 79 | rtc::CritScope lock(&crit_); |
| 80 | if (stopped_) { |
| 81 | return; |
| 82 | } |
| 83 | latest_return_time_ms_ = latest_return_time_ms; |
| 84 | keyframe_required_ = keyframe_required; |
| 85 | frame_handler_ = handler; |
| 86 | callback_queue_ = callback_queue; |
| 87 | StartWaitForNextFrameOnQueue(); |
| 88 | } |
| 89 | |
| 90 | void FrameBuffer::StartWaitForNextFrameOnQueue() { |
| 91 | RTC_DCHECK(callback_queue_); |
| 92 | RTC_DCHECK(!callback_task_.Running()); |
| 93 | int64_t wait_ms = FindNextFrame(clock_->TimeInMilliseconds()); |
| 94 | callback_task_ = RepeatingTaskHandle::DelayedStart( |
| 95 | callback_queue_->Get(), TimeDelta::ms(wait_ms), [this] { |
| 96 | // If this task has not been cancelled, we did not get any new frames |
| 97 | // while waiting. Continue with frame delivery. |
| 98 | rtc::CritScope lock(&crit_); |
| 99 | if (!frames_to_decode_.empty()) { |
| 100 | // We have frames, deliver! |
| 101 | frame_handler_(absl::WrapUnique(GetNextFrame()), kFrameFound); |
| 102 | CancelCallback(); |
| 103 | return TimeDelta::Zero(); // Ignored. |
| 104 | } else if (clock_->TimeInMilliseconds() >= latest_return_time_ms_) { |
| 105 | // We have timed out, signal this and stop repeating. |
| 106 | frame_handler_(nullptr, kTimeout); |
| 107 | CancelCallback(); |
| 108 | return TimeDelta::Zero(); // Ignored. |
| 109 | } else { |
| 110 | // If there's no frames to decode and there is still time left, it |
| 111 | // means that the frame buffer was cleared between creation and |
| 112 | // execution of this task. Continue waiting for the remaining time. |
| 113 | int64_t wait_ms = FindNextFrame(clock_->TimeInMilliseconds()); |
| 114 | return TimeDelta::ms(wait_ms); |
| 115 | } |
| 116 | }); |
| 117 | } |
| 118 | |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 119 | int64_t FrameBuffer::FindNextFrame(int64_t now_ms) { |
| 120 | int64_t wait_ms = latest_return_time_ms_ - now_ms; |
| 121 | frames_to_decode_.clear(); |
| 122 | |
| 123 | // |last_continuous_frame_| may be empty below, but nullopt is smaller |
| 124 | // than everything else and loop will immediately terminate as expected. |
| 125 | for (auto frame_it = frames_.begin(); |
| 126 | frame_it != frames_.end() && frame_it->first <= last_continuous_frame_; |
| 127 | ++frame_it) { |
| 128 | if (!frame_it->second.continuous || |
| 129 | frame_it->second.num_missing_decodable > 0) { |
| 130 | continue; |
| 131 | } |
| 132 | |
| 133 | EncodedFrame* frame = frame_it->second.frame.get(); |
| 134 | |
| 135 | if (keyframe_required_ && !frame->is_keyframe()) |
| 136 | continue; |
| 137 | |
| 138 | auto last_decoded_frame_timestamp = |
| 139 | decoded_frames_history_.GetLastDecodedFrameTimestamp(); |
| 140 | |
| 141 | // TODO(https://bugs.webrtc.org/9974): consider removing this check |
| 142 | // as it may make a stream undecodable after a very long delay between |
| 143 | // frames. |
| 144 | if (last_decoded_frame_timestamp && |
| 145 | AheadOf(*last_decoded_frame_timestamp, frame->Timestamp())) { |
| 146 | continue; |
| 147 | } |
| 148 | |
| 149 | // Only ever return all parts of a superframe. Therefore skip this |
| 150 | // frame if it's not a beginning of a superframe. |
| 151 | if (frame->inter_layer_predicted) { |
| 152 | continue; |
| 153 | } |
| 154 | |
| 155 | // Gather all remaining frames for the same superframe. |
| 156 | std::vector<FrameMap::iterator> current_superframe; |
| 157 | current_superframe.push_back(frame_it); |
| 158 | bool last_layer_completed = frame_it->second.frame->is_last_spatial_layer; |
| 159 | FrameMap::iterator next_frame_it = frame_it; |
| 160 | while (true) { |
| 161 | ++next_frame_it; |
| 162 | if (next_frame_it == frames_.end() || |
| 163 | next_frame_it->first.picture_id != frame->id.picture_id || |
| 164 | !next_frame_it->second.continuous) { |
| 165 | break; |
| 166 | } |
| 167 | // Check if the next frame has some undecoded references other than |
| 168 | // the previous frame in the same superframe. |
| 169 | size_t num_allowed_undecoded_refs = |
| 170 | (next_frame_it->second.frame->inter_layer_predicted) ? 1 : 0; |
| 171 | if (next_frame_it->second.num_missing_decodable > |
| 172 | num_allowed_undecoded_refs) { |
| 173 | break; |
| 174 | } |
| 175 | // All frames in the superframe should have the same timestamp. |
| 176 | if (frame->Timestamp() != next_frame_it->second.frame->Timestamp()) { |
| 177 | RTC_LOG(LS_WARNING) << "Frames in a single superframe have different" |
| 178 | " timestamps. Skipping undecodable superframe."; |
| 179 | break; |
| 180 | } |
| 181 | current_superframe.push_back(next_frame_it); |
| 182 | last_layer_completed = next_frame_it->second.frame->is_last_spatial_layer; |
| 183 | } |
| 184 | // Check if the current superframe is complete. |
| 185 | // TODO(bugs.webrtc.org/10064): consider returning all available to |
| 186 | // decode frames even if the superframe is not complete yet. |
| 187 | if (!last_layer_completed) { |
| 188 | continue; |
| 189 | } |
| 190 | |
| 191 | frames_to_decode_ = std::move(current_superframe); |
| 192 | |
| 193 | if (frame->RenderTime() == -1) { |
| 194 | frame->SetRenderTime(timing_->RenderTimeMs(frame->Timestamp(), now_ms)); |
| 195 | } |
| 196 | wait_ms = timing_->MaxWaitingTime(frame->RenderTime(), now_ms); |
| 197 | |
| 198 | // This will cause the frame buffer to prefer high framerate rather |
| 199 | // than high resolution in the case of the decoder not decoding fast |
| 200 | // enough and the stream has multiple spatial and temporal layers. |
| 201 | // For multiple temporal layers it may cause non-base layer frames to be |
| 202 | // skipped if they are late. |
| 203 | if (wait_ms < -kMaxAllowedFrameDelayMs) |
| 204 | continue; |
| 205 | |
| 206 | break; |
| 207 | } |
| 208 | wait_ms = std::min<int64_t>(wait_ms, latest_return_time_ms_ - now_ms); |
| 209 | wait_ms = std::max<int64_t>(wait_ms, 0); |
| 210 | return wait_ms; |
| 211 | } |
| 212 | |
| 213 | EncodedFrame* FrameBuffer::GetNextFrame() { |
| 214 | int64_t now_ms = clock_->TimeInMilliseconds(); |
| 215 | // TODO(ilnik): remove |frames_out| use frames_to_decode_ directly. |
| 216 | std::vector<EncodedFrame*> frames_out; |
| 217 | |
| 218 | RTC_DCHECK(!frames_to_decode_.empty()); |
| 219 | bool superframe_delayed_by_retransmission = false; |
| 220 | size_t superframe_size = 0; |
| 221 | EncodedFrame* first_frame = frames_to_decode_[0]->second.frame.get(); |
| 222 | int64_t render_time_ms = first_frame->RenderTime(); |
| 223 | int64_t receive_time_ms = first_frame->ReceivedTime(); |
| 224 | // Gracefully handle bad RTP timestamps and render time issues. |
| 225 | if (HasBadRenderTiming(*first_frame, now_ms)) { |
Niels Möller | d9c2d94 | 2019-04-30 09:16:36 +0200 | [diff] [blame] | 226 | jitter_estimator_.Reset(); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 227 | timing_->Reset(); |
| 228 | render_time_ms = timing_->RenderTimeMs(first_frame->Timestamp(), now_ms); |
| 229 | } |
| 230 | |
| 231 | for (FrameMap::iterator& frame_it : frames_to_decode_) { |
| 232 | RTC_DCHECK(frame_it != frames_.end()); |
| 233 | EncodedFrame* frame = frame_it->second.frame.release(); |
| 234 | |
| 235 | frame->SetRenderTime(render_time_ms); |
| 236 | |
| 237 | superframe_delayed_by_retransmission |= frame->delayed_by_retransmission(); |
| 238 | receive_time_ms = std::max(receive_time_ms, frame->ReceivedTime()); |
| 239 | superframe_size += frame->size(); |
| 240 | |
| 241 | PropagateDecodability(frame_it->second); |
| 242 | decoded_frames_history_.InsertDecoded(frame_it->first, frame->Timestamp()); |
| 243 | |
| 244 | // Remove decoded frame and all undecoded frames before it. |
Johannes Kron | 0c141c5 | 2019-08-26 15:04:43 +0200 | [diff] [blame] | 245 | if (stats_callback_) { |
| 246 | unsigned int dropped_frames = std::count_if( |
| 247 | frames_.begin(), frame_it, |
| 248 | [](const std::pair<const VideoLayerFrameId, FrameInfo>& frame) { |
| 249 | return frame.second.frame != nullptr; |
| 250 | }); |
| 251 | if (dropped_frames > 0) { |
| 252 | stats_callback_->OnDroppedFrames(dropped_frames); |
| 253 | } |
| 254 | } |
| 255 | |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 256 | frames_.erase(frames_.begin(), ++frame_it); |
| 257 | |
| 258 | frames_out.push_back(frame); |
| 259 | } |
| 260 | |
| 261 | if (!superframe_delayed_by_retransmission) { |
| 262 | int64_t frame_delay; |
| 263 | |
| 264 | if (inter_frame_delay_.CalculateDelay(first_frame->Timestamp(), |
| 265 | &frame_delay, receive_time_ms)) { |
Niels Möller | d9c2d94 | 2019-04-30 09:16:36 +0200 | [diff] [blame] | 266 | jitter_estimator_.UpdateEstimate(frame_delay, superframe_size); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 267 | } |
| 268 | |
| 269 | float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0; |
“Michael | d3a4ebe | 2019-06-07 03:55:01 -0500 | [diff] [blame] | 270 | absl::optional<float> rtt_mult_add_cap_ms = absl::nullopt; |
| 271 | if (rtt_mult_settings_.has_value()) { |
| 272 | rtt_mult = rtt_mult_settings_->rtt_mult_setting; |
| 273 | rtt_mult_add_cap_ms = rtt_mult_settings_->rtt_mult_add_cap_ms; |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 274 | } |
“Michael | e0f3704 | 2019-06-04 10:04:12 -0500 | [diff] [blame] | 275 | timing_->SetJitterDelay( |
| 276 | jitter_estimator_.GetJitterEstimate(rtt_mult, rtt_mult_add_cap_ms)); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 277 | timing_->UpdateCurrentDelay(render_time_ms, now_ms); |
| 278 | } else { |
| 279 | if (RttMultExperiment::RttMultEnabled() || add_rtt_to_playout_delay_) |
Niels Möller | d9c2d94 | 2019-04-30 09:16:36 +0200 | [diff] [blame] | 280 | jitter_estimator_.FrameNacked(); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 281 | } |
| 282 | |
| 283 | UpdateJitterDelay(); |
| 284 | UpdateTimingFrameInfo(); |
| 285 | |
| 286 | if (frames_out.size() == 1) { |
| 287 | return frames_out[0]; |
| 288 | } else { |
| 289 | return CombineAndDeleteFrames(frames_out); |
| 290 | } |
| 291 | } |
| 292 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 293 | bool FrameBuffer::HasBadRenderTiming(const EncodedFrame& frame, |
| 294 | int64_t now_ms) { |
stefan | 95e9754 | 2017-05-23 09:52:18 -0700 | [diff] [blame] | 295 | // Assume that render timing errors are due to changes in the video stream. |
| 296 | int64_t render_time_ms = frame.RenderTimeMs(); |
Stefan Holmer | 812ceaf | 2018-05-15 13:00:10 +0200 | [diff] [blame] | 297 | // Zero render time means render immediately. |
| 298 | if (render_time_ms == 0) { |
| 299 | return false; |
| 300 | } |
stefan | 95e9754 | 2017-05-23 09:52:18 -0700 | [diff] [blame] | 301 | if (render_time_ms < 0) { |
| 302 | return true; |
| 303 | } |
Stefan Holmer | 812ceaf | 2018-05-15 13:00:10 +0200 | [diff] [blame] | 304 | const int64_t kMaxVideoDelayMs = 10000; |
stefan | 95e9754 | 2017-05-23 09:52:18 -0700 | [diff] [blame] | 305 | if (std::abs(render_time_ms - now_ms) > kMaxVideoDelayMs) { |
| 306 | int frame_delay = static_cast<int>(std::abs(render_time_ms - now_ms)); |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 307 | RTC_LOG(LS_WARNING) |
| 308 | << "A frame about to be decoded is out of the configured " |
Jonas Olsson | b2b2031 | 2020-01-14 12:11:31 +0100 | [diff] [blame] | 309 | "delay bounds (" |
| 310 | << frame_delay << " > " << kMaxVideoDelayMs |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 311 | << "). Resetting the video jitter buffer."; |
stefan | 95e9754 | 2017-05-23 09:52:18 -0700 | [diff] [blame] | 312 | return true; |
| 313 | } |
| 314 | if (static_cast<int>(timing_->TargetVideoDelay()) > kMaxVideoDelayMs) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 315 | RTC_LOG(LS_WARNING) << "The video target delay has grown larger than " |
| 316 | << kMaxVideoDelayMs << " ms."; |
stefan | 95e9754 | 2017-05-23 09:52:18 -0700 | [diff] [blame] | 317 | return true; |
| 318 | } |
| 319 | return false; |
| 320 | } |
| 321 | |
philipel | 4f6cd6a | 2016-08-03 10:59:32 +0200 | [diff] [blame] | 322 | void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 323 | TRACE_EVENT0("webrtc", "FrameBuffer::SetProtectionMode"); |
Henrik Boström | c680c4a | 2019-04-03 10:27:36 +0000 | [diff] [blame] | 324 | rtc::CritScope lock(&crit_); |
| 325 | protection_mode_ = mode; |
philipel | 4f6cd6a | 2016-08-03 10:59:32 +0200 | [diff] [blame] | 326 | } |
| 327 | |
philipel | 504c47d | 2016-06-30 17:33:02 +0200 | [diff] [blame] | 328 | void FrameBuffer::Start() { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 329 | TRACE_EVENT0("webrtc", "FrameBuffer::Start"); |
Henrik Boström | c680c4a | 2019-04-03 10:27:36 +0000 | [diff] [blame] | 330 | rtc::CritScope lock(&crit_); |
| 331 | stopped_ = false; |
philipel | 504c47d | 2016-06-30 17:33:02 +0200 | [diff] [blame] | 332 | } |
| 333 | |
| 334 | void FrameBuffer::Stop() { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 335 | TRACE_EVENT0("webrtc", "FrameBuffer::Stop"); |
Henrik Boström | c680c4a | 2019-04-03 10:27:36 +0000 | [diff] [blame] | 336 | rtc::CritScope lock(&crit_); |
| 337 | stopped_ = true; |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 338 | CancelCallback(); |
philipel | 504c47d | 2016-06-30 17:33:02 +0200 | [diff] [blame] | 339 | } |
| 340 | |
Ilya Nikolaevskiy | e6a2d94 | 2018-11-07 14:32:28 +0100 | [diff] [blame] | 341 | void FrameBuffer::Clear() { |
Henrik Boström | c680c4a | 2019-04-03 10:27:36 +0000 | [diff] [blame] | 342 | rtc::CritScope lock(&crit_); |
| 343 | ClearFramesAndHistory(); |
Ilya Nikolaevskiy | e6a2d94 | 2018-11-07 14:32:28 +0100 | [diff] [blame] | 344 | } |
| 345 | |
philipel | e21be1d | 2017-09-25 06:37:12 -0700 | [diff] [blame] | 346 | void FrameBuffer::UpdateRtt(int64_t rtt_ms) { |
Henrik Boström | c680c4a | 2019-04-03 10:27:36 +0000 | [diff] [blame] | 347 | rtc::CritScope lock(&crit_); |
Niels Möller | d9c2d94 | 2019-04-30 09:16:36 +0200 | [diff] [blame] | 348 | jitter_estimator_.UpdateRtt(rtt_ms); |
philipel | e21be1d | 2017-09-25 06:37:12 -0700 | [diff] [blame] | 349 | } |
| 350 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 351 | bool FrameBuffer::ValidReferences(const EncodedFrame& frame) const { |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 352 | for (size_t i = 0; i < frame.num_references; ++i) { |
Philip Eliasson | 1f850a6 | 2019-03-19 12:15:00 +0000 | [diff] [blame] | 353 | if (frame.references[i] >= frame.id.picture_id) |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 354 | return false; |
philipel | 3b3c9c4 | 2017-09-11 09:38:36 -0700 | [diff] [blame] | 355 | |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 356 | for (size_t j = i + 1; j < frame.num_references; ++j) { |
| 357 | if (frame.references[i] == frame.references[j]) |
| 358 | return false; |
| 359 | } |
| 360 | } |
| 361 | |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 362 | if (frame.inter_layer_predicted && frame.id.spatial_layer == 0) |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 363 | return false; |
| 364 | |
| 365 | return true; |
| 366 | } |
| 367 | |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 368 | void FrameBuffer::CancelCallback() { |
| 369 | frame_handler_ = {}; |
| 370 | callback_task_.Stop(); |
| 371 | callback_queue_ = nullptr; |
| 372 | } |
| 373 | |
Ilya Nikolaevskiy | 48193b0 | 2019-03-25 11:40:34 +0100 | [diff] [blame] | 374 | bool FrameBuffer::IsCompleteSuperFrame(const EncodedFrame& frame) { |
| 375 | if (frame.inter_layer_predicted) { |
| 376 | // Check that all previous spatial layers are already inserted. |
| 377 | VideoLayerFrameId id = frame.id; |
| 378 | RTC_DCHECK_GT(id.spatial_layer, 0); |
| 379 | --id.spatial_layer; |
| 380 | FrameMap::iterator prev_frame = frames_.find(id); |
Ilya Nikolaevskiy | b4a70ed | 2019-04-23 10:37:06 +0200 | [diff] [blame] | 381 | if (prev_frame == frames_.end() || !prev_frame->second.frame) |
Ilya Nikolaevskiy | 48193b0 | 2019-03-25 11:40:34 +0100 | [diff] [blame] | 382 | return false; |
| 383 | while (prev_frame->second.frame->inter_layer_predicted) { |
Ilya Nikolaevskiy | f87a7df | 2019-04-09 19:23:36 +0200 | [diff] [blame] | 384 | if (prev_frame == frames_.begin()) |
| 385 | return false; |
Ilya Nikolaevskiy | 48193b0 | 2019-03-25 11:40:34 +0100 | [diff] [blame] | 386 | --prev_frame; |
| 387 | --id.spatial_layer; |
Ilya Nikolaevskiy | b4a70ed | 2019-04-23 10:37:06 +0200 | [diff] [blame] | 388 | if (!prev_frame->second.frame || |
| 389 | prev_frame->first.picture_id != id.picture_id || |
Ilya Nikolaevskiy | 48193b0 | 2019-03-25 11:40:34 +0100 | [diff] [blame] | 390 | prev_frame->first.spatial_layer != id.spatial_layer) { |
| 391 | return false; |
| 392 | } |
| 393 | } |
| 394 | } |
| 395 | |
| 396 | if (!frame.is_last_spatial_layer) { |
| 397 | // Check that all following spatial layers are already inserted. |
| 398 | VideoLayerFrameId id = frame.id; |
| 399 | ++id.spatial_layer; |
| 400 | FrameMap::iterator next_frame = frames_.find(id); |
Ilya Nikolaevskiy | b4a70ed | 2019-04-23 10:37:06 +0200 | [diff] [blame] | 401 | if (next_frame == frames_.end() || !next_frame->second.frame) |
Ilya Nikolaevskiy | 48193b0 | 2019-03-25 11:40:34 +0100 | [diff] [blame] | 402 | return false; |
| 403 | while (!next_frame->second.frame->is_last_spatial_layer) { |
| 404 | ++next_frame; |
| 405 | ++id.spatial_layer; |
Ilya Nikolaevskiy | b4a70ed | 2019-04-23 10:37:06 +0200 | [diff] [blame] | 406 | if (next_frame == frames_.end() || !next_frame->second.frame || |
Ilya Nikolaevskiy | 48193b0 | 2019-03-25 11:40:34 +0100 | [diff] [blame] | 407 | next_frame->first.picture_id != id.picture_id || |
| 408 | next_frame->first.spatial_layer != id.spatial_layer) { |
| 409 | return false; |
| 410 | } |
| 411 | } |
| 412 | } |
| 413 | |
| 414 | return true; |
| 415 | } |
| 416 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 417 | int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 418 | TRACE_EVENT0("webrtc", "FrameBuffer::InsertFrame"); |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 419 | RTC_DCHECK(frame); |
tommi | 0a73564 | 2017-03-14 06:23:57 -0700 | [diff] [blame] | 420 | |
| 421 | rtc::CritScope lock(&crit_); |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 422 | |
Ilya Nikolaevskiy | 48193b0 | 2019-03-25 11:40:34 +0100 | [diff] [blame] | 423 | const VideoLayerFrameId& id = frame->id; |
philipel | 1610f94 | 2017-12-12 13:58:31 +0100 | [diff] [blame] | 424 | int64_t last_continuous_picture_id = |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 425 | !last_continuous_frame_ ? -1 : last_continuous_frame_->picture_id; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 426 | |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 427 | if (!ValidReferences(*frame)) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 428 | RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 429 | << id.picture_id << ":" |
| 430 | << static_cast<int>(id.spatial_layer) |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 431 | << ") has invalid frame references, dropping frame."; |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 432 | return last_continuous_picture_id; |
| 433 | } |
| 434 | |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 435 | if (frames_.size() >= kMaxFramesBuffered) { |
philipel | 9771c50 | 2018-03-02 11:06:27 +0100 | [diff] [blame] | 436 | if (frame->is_keyframe()) { |
| 437 | RTC_LOG(LS_WARNING) << "Inserting keyframe (picture_id:spatial_id) (" |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 438 | << id.picture_id << ":" |
| 439 | << static_cast<int>(id.spatial_layer) |
philipel | 9771c50 | 2018-03-02 11:06:27 +0100 | [diff] [blame] | 440 | << ") but buffer is full, clearing" |
Jonas Olsson | b2b2031 | 2020-01-14 12:11:31 +0100 | [diff] [blame] | 441 | " buffer and inserting the frame."; |
philipel | 9771c50 | 2018-03-02 11:06:27 +0100 | [diff] [blame] | 442 | ClearFramesAndHistory(); |
| 443 | } else { |
| 444 | RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 445 | << id.picture_id << ":" |
| 446 | << static_cast<int>(id.spatial_layer) |
philipel | 9771c50 | 2018-03-02 11:06:27 +0100 | [diff] [blame] | 447 | << ") could not be inserted due to the frame " |
Jonas Olsson | b2b2031 | 2020-01-14 12:11:31 +0100 | [diff] [blame] | 448 | "buffer being full, dropping frame."; |
philipel | 9771c50 | 2018-03-02 11:06:27 +0100 | [diff] [blame] | 449 | return last_continuous_picture_id; |
| 450 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 451 | } |
| 452 | |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 453 | auto last_decoded_frame = decoded_frames_history_.GetLastDecodedFrameId(); |
| 454 | auto last_decoded_frame_timestamp = |
| 455 | decoded_frames_history_.GetLastDecodedFrameTimestamp(); |
| 456 | if (last_decoded_frame && id <= *last_decoded_frame) { |
| 457 | if (AheadOf(frame->Timestamp(), *last_decoded_frame_timestamp) && |
philipel | 3042c2d | 2017-08-18 04:55:02 -0700 | [diff] [blame] | 458 | frame->is_keyframe()) { |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 459 | // If this frame has a newer timestamp but an earlier picture id then we |
| 460 | // assume there has been a jump in the picture id due to some encoder |
| 461 | // reconfiguration or some other reason. Even though this is not according |
| 462 | // to spec we can still continue to decode from this frame if it is a |
| 463 | // keyframe. |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 464 | RTC_LOG(LS_WARNING) |
| 465 | << "A jump in picture id was detected, clearing buffer."; |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 466 | ClearFramesAndHistory(); |
| 467 | last_continuous_picture_id = -1; |
| 468 | } else { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 469 | RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 470 | << id.picture_id << ":" |
| 471 | << static_cast<int>(id.spatial_layer) |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 472 | << ") inserted after frame (" |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 473 | << last_decoded_frame->picture_id << ":" |
| 474 | << static_cast<int>(last_decoded_frame->spatial_layer) |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 475 | << ") was handed off for decoding, dropping frame."; |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 476 | return last_continuous_picture_id; |
| 477 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 478 | } |
| 479 | |
philipel | 146a48b | 2017-04-20 04:04:38 -0700 | [diff] [blame] | 480 | // Test if inserting this frame would cause the order of the frames to become |
| 481 | // ambiguous (covering more than half the interval of 2^16). This can happen |
| 482 | // when the picture id make large jumps mid stream. |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 483 | if (!frames_.empty() && id < frames_.begin()->first && |
| 484 | frames_.rbegin()->first < id) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 485 | RTC_LOG(LS_WARNING) |
| 486 | << "A jump in picture id was detected, clearing buffer."; |
philipel | 146a48b | 2017-04-20 04:04:38 -0700 | [diff] [blame] | 487 | ClearFramesAndHistory(); |
| 488 | last_continuous_picture_id = -1; |
| 489 | } |
| 490 | |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 491 | auto info = frames_.emplace(id, FrameInfo()).first; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 492 | |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 493 | if (info->second.frame) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 494 | RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 495 | << id.picture_id << ":" |
| 496 | << static_cast<int>(id.spatial_layer) |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 497 | << ") already inserted, dropping frame."; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 498 | return last_continuous_picture_id; |
| 499 | } |
| 500 | |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 501 | if (!UpdateFrameInfoWithIncomingFrame(*frame, info)) |
| 502 | return last_continuous_picture_id; |
Ruslan Burakov | 493a650 | 2019-02-27 15:32:48 +0100 | [diff] [blame] | 503 | |
| 504 | if (!frame->delayed_by_retransmission()) |
| 505 | timing_->IncomingTimestamp(frame->Timestamp(), frame->ReceivedTime()); |
philipel | 0a9f6de | 2018-02-28 11:29:47 +0100 | [diff] [blame] | 506 | |
Johannes Kron | b88b44e | 2019-08-22 13:16:44 +0200 | [diff] [blame] | 507 | if (stats_callback_ && IsCompleteSuperFrame(*frame)) { |
| 508 | stats_callback_->OnCompleteFrame(frame->is_keyframe(), frame->size(), |
| 509 | frame->contentType()); |
| 510 | } |
| 511 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 512 | info->second.frame = std::move(frame); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 513 | |
| 514 | if (info->second.num_missing_continuous == 0) { |
| 515 | info->second.continuous = true; |
| 516 | PropagateContinuity(info); |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 517 | last_continuous_picture_id = last_continuous_frame_->picture_id; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 518 | |
| 519 | // Since we now have new continuous frames there might be a better frame |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 520 | // to return from NextFrame. |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 521 | if (callback_queue_) { |
| 522 | callback_queue_->PostTask([this] { |
| 523 | rtc::CritScope lock(&crit_); |
| 524 | if (!callback_task_.Running()) |
| 525 | return; |
| 526 | RTC_CHECK(frame_handler_); |
| 527 | callback_task_.Stop(); |
| 528 | StartWaitForNextFrameOnQueue(); |
| 529 | }); |
| 530 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 531 | } |
| 532 | |
| 533 | return last_continuous_picture_id; |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 534 | } |
| 535 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 536 | void FrameBuffer::PropagateContinuity(FrameMap::iterator start) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 537 | TRACE_EVENT0("webrtc", "FrameBuffer::PropagateContinuity"); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 538 | RTC_DCHECK(start->second.continuous); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 539 | |
| 540 | std::queue<FrameMap::iterator> continuous_frames; |
| 541 | continuous_frames.push(start); |
| 542 | |
| 543 | // A simple BFS to traverse continuous frames. |
| 544 | while (!continuous_frames.empty()) { |
| 545 | auto frame = continuous_frames.front(); |
| 546 | continuous_frames.pop(); |
| 547 | |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 548 | if (!last_continuous_frame_ || *last_continuous_frame_ < frame->first) { |
| 549 | last_continuous_frame_ = frame->first; |
| 550 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 551 | |
| 552 | // Loop through all dependent frames, and if that frame no longer has |
| 553 | // any unfulfilled dependencies then that frame is continuous as well. |
Elad Alon | 69321dd | 2019-01-10 15:02:54 +0100 | [diff] [blame] | 554 | for (size_t d = 0; d < frame->second.dependent_frames.size(); ++d) { |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 555 | auto frame_ref = frames_.find(frame->second.dependent_frames[d]); |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 556 | RTC_DCHECK(frame_ref != frames_.end()); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 557 | |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 558 | // TODO(philipel): Look into why we've seen this happen. |
| 559 | if (frame_ref != frames_.end()) { |
| 560 | --frame_ref->second.num_missing_continuous; |
| 561 | if (frame_ref->second.num_missing_continuous == 0) { |
| 562 | frame_ref->second.continuous = true; |
| 563 | continuous_frames.push(frame_ref); |
| 564 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 565 | } |
| 566 | } |
| 567 | } |
| 568 | } |
| 569 | |
| 570 | void FrameBuffer::PropagateDecodability(const FrameInfo& info) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 571 | TRACE_EVENT0("webrtc", "FrameBuffer::PropagateDecodability"); |
Elad Alon | 69321dd | 2019-01-10 15:02:54 +0100 | [diff] [blame] | 572 | for (size_t d = 0; d < info.dependent_frames.size(); ++d) { |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 573 | auto ref_info = frames_.find(info.dependent_frames[d]); |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 574 | RTC_DCHECK(ref_info != frames_.end()); |
tommi | e95b78b | 2017-05-14 07:23:11 -0700 | [diff] [blame] | 575 | // TODO(philipel): Look into why we've seen this happen. |
| 576 | if (ref_info != frames_.end()) { |
| 577 | RTC_DCHECK_GT(ref_info->second.num_missing_decodable, 0U); |
| 578 | --ref_info->second.num_missing_decodable; |
| 579 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 580 | } |
| 581 | } |
| 582 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 583 | bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame, |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 584 | FrameMap::iterator info) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 585 | TRACE_EVENT0("webrtc", "FrameBuffer::UpdateFrameInfoWithIncomingFrame"); |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 586 | const VideoLayerFrameId& id = frame.id; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 587 | |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 588 | auto last_decoded_frame = decoded_frames_history_.GetLastDecodedFrameId(); |
| 589 | RTC_DCHECK(!last_decoded_frame || *last_decoded_frame < info->first); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 590 | |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 591 | // In this function we determine how many missing dependencies this |frame| |
| 592 | // has to become continuous/decodable. If a frame that this |frame| depend |
| 593 | // on has already been decoded then we can ignore that dependency since it has |
| 594 | // already been fulfilled. |
| 595 | // |
| 596 | // For all other frames we will register a backwards reference to this |frame| |
| 597 | // so that |num_missing_continuous| and |num_missing_decodable| can be |
| 598 | // decremented as frames become continuous/are decoded. |
| 599 | struct Dependency { |
| 600 | VideoLayerFrameId id; |
| 601 | bool continuous; |
| 602 | }; |
| 603 | std::vector<Dependency> not_yet_fulfilled_dependencies; |
| 604 | |
| 605 | // Find all dependencies that have not yet been fulfilled. |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 606 | for (size_t i = 0; i < frame.num_references; ++i) { |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 607 | VideoLayerFrameId ref_key(frame.references[i], frame.id.spatial_layer); |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 608 | // Does |frame| depend on a frame earlier than the last decoded one? |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 609 | if (last_decoded_frame && ref_key <= *last_decoded_frame) { |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 610 | // Was that frame decoded? If not, this |frame| will never become |
| 611 | // decodable. |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 612 | if (!decoded_frames_history_.WasDecoded(ref_key)) { |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 613 | int64_t now_ms = clock_->TimeInMilliseconds(); |
| 614 | if (last_log_non_decoded_ms_ + kLogNonDecodedIntervalMs < now_ms) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 615 | RTC_LOG(LS_WARNING) |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 616 | << "Frame with (picture_id:spatial_id) (" << id.picture_id << ":" |
| 617 | << static_cast<int>(id.spatial_layer) |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 618 | << ") depends on a non-decoded frame more previous than" |
Jonas Olsson | b2b2031 | 2020-01-14 12:11:31 +0100 | [diff] [blame] | 619 | " the last decoded frame, dropping frame."; |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 620 | last_log_non_decoded_ms_ = now_ms; |
| 621 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 622 | return false; |
| 623 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 624 | } else { |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 625 | auto ref_info = frames_.find(ref_key); |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 626 | bool ref_continuous = |
| 627 | ref_info != frames_.end() && ref_info->second.continuous; |
| 628 | not_yet_fulfilled_dependencies.push_back({ref_key, ref_continuous}); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 629 | } |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 630 | } |
| 631 | |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 632 | // Does |frame| depend on the lower spatial layer? |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 633 | if (frame.inter_layer_predicted) { |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 634 | VideoLayerFrameId ref_key(frame.id.picture_id, frame.id.spatial_layer - 1); |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 635 | auto ref_info = frames_.find(ref_key); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 636 | |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 637 | bool lower_layer_decoded = |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 638 | last_decoded_frame && *last_decoded_frame == ref_key; |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 639 | bool lower_layer_continuous = |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 640 | lower_layer_decoded || |
| 641 | (ref_info != frames_.end() && ref_info->second.continuous); |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 642 | |
| 643 | if (!lower_layer_continuous || !lower_layer_decoded) { |
| 644 | not_yet_fulfilled_dependencies.push_back( |
| 645 | {ref_key, lower_layer_continuous}); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 646 | } |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 647 | } |
| 648 | |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 649 | info->second.num_missing_continuous = not_yet_fulfilled_dependencies.size(); |
| 650 | info->second.num_missing_decodable = not_yet_fulfilled_dependencies.size(); |
| 651 | |
| 652 | for (const Dependency& dep : not_yet_fulfilled_dependencies) { |
| 653 | if (dep.continuous) |
| 654 | --info->second.num_missing_continuous; |
| 655 | |
Elad Alon | 69321dd | 2019-01-10 15:02:54 +0100 | [diff] [blame] | 656 | frames_[dep.id].dependent_frames.push_back(id); |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 657 | } |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 658 | |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 659 | return true; |
| 660 | } |
| 661 | |
philipel | be74270 | 2016-11-30 01:31:40 -0800 | [diff] [blame] | 662 | void FrameBuffer::UpdateJitterDelay() { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 663 | TRACE_EVENT0("webrtc", "FrameBuffer::UpdateJitterDelay"); |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 664 | if (!stats_callback_) |
| 665 | return; |
philipel | be74270 | 2016-11-30 01:31:40 -0800 | [diff] [blame] | 666 | |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 667 | int max_decode_ms; |
| 668 | int current_delay_ms; |
| 669 | int target_delay_ms; |
| 670 | int jitter_buffer_ms; |
| 671 | int min_playout_delay_ms; |
| 672 | int render_delay_ms; |
Johannes Kron | bfd343b | 2019-07-01 10:07:50 +0200 | [diff] [blame] | 673 | if (timing_->GetTimings(&max_decode_ms, ¤t_delay_ms, &target_delay_ms, |
| 674 | &jitter_buffer_ms, &min_playout_delay_ms, |
| 675 | &render_delay_ms)) { |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 676 | stats_callback_->OnFrameBufferTimingsUpdated( |
Johannes Kron | bfd343b | 2019-07-01 10:07:50 +0200 | [diff] [blame] | 677 | max_decode_ms, current_delay_ms, target_delay_ms, jitter_buffer_ms, |
| 678 | min_playout_delay_ms, render_delay_ms); |
philipel | be74270 | 2016-11-30 01:31:40 -0800 | [diff] [blame] | 679 | } |
philipel | 266f0a4 | 2016-11-28 08:49:07 -0800 | [diff] [blame] | 680 | } |
| 681 | |
ilnik | 2edc684 | 2017-07-06 03:06:50 -0700 | [diff] [blame] | 682 | void FrameBuffer::UpdateTimingFrameInfo() { |
| 683 | TRACE_EVENT0("webrtc", "FrameBuffer::UpdateTimingFrameInfo"); |
Danil Chapovalov | 0040b66 | 2018-06-18 10:48:16 +0200 | [diff] [blame] | 684 | absl::optional<TimingFrameInfo> info = timing_->GetTimingFrameInfo(); |
philipel | 9718711 | 2018-03-23 10:43:21 +0100 | [diff] [blame] | 685 | if (info && stats_callback_) |
ilnik | 2edc684 | 2017-07-06 03:06:50 -0700 | [diff] [blame] | 686 | stats_callback_->OnTimingFrameInfoUpdated(*info); |
| 687 | } |
| 688 | |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 689 | void FrameBuffer::ClearFramesAndHistory() { |
ilnik | 2edc684 | 2017-07-06 03:06:50 -0700 | [diff] [blame] | 690 | TRACE_EVENT0("webrtc", "FrameBuffer::ClearFramesAndHistory"); |
Johannes Kron | 0c141c5 | 2019-08-26 15:04:43 +0200 | [diff] [blame] | 691 | if (stats_callback_) { |
| 692 | unsigned int dropped_frames = std::count_if( |
| 693 | frames_.begin(), frames_.end(), |
| 694 | [](const std::pair<const VideoLayerFrameId, FrameInfo>& frame) { |
| 695 | return frame.second.frame != nullptr; |
| 696 | }); |
| 697 | if (dropped_frames > 0) { |
| 698 | stats_callback_->OnDroppedFrames(dropped_frames); |
| 699 | } |
| 700 | } |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 701 | frames_.clear(); |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 702 | last_continuous_frame_.reset(); |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 703 | frames_to_decode_.clear(); |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 704 | decoded_frames_history_.Clear(); |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 705 | } |
| 706 | |
Niels Möller | ff2e215 | 2019-09-27 10:29:30 +0200 | [diff] [blame] | 707 | // TODO(philipel): Avoid the concatenation of frames here, by replacing |
| 708 | // NextFrame and GetNextFrame with methods returning multiple frames. |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 709 | EncodedFrame* FrameBuffer::CombineAndDeleteFrames( |
| 710 | const std::vector<EncodedFrame*>& frames) const { |
| 711 | RTC_DCHECK(!frames.empty()); |
Sergey Silkin | 61832dd | 2018-12-20 14:32:14 +0100 | [diff] [blame] | 712 | EncodedFrame* first_frame = frames[0]; |
| 713 | EncodedFrame* last_frame = frames.back(); |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 714 | size_t total_length = 0; |
| 715 | for (size_t i = 0; i < frames.size(); ++i) { |
| 716 | total_length += frames[i]->size(); |
| 717 | } |
Niels Möller | ff2e215 | 2019-09-27 10:29:30 +0200 | [diff] [blame] | 718 | auto encoded_image_buffer = EncodedImageBuffer::Create(total_length); |
| 719 | uint8_t* buffer = encoded_image_buffer->data(); |
Sergey Silkin | 2799e63 | 2019-05-17 09:51:39 +0200 | [diff] [blame] | 720 | first_frame->SetSpatialLayerFrameSize(first_frame->id.spatial_layer, |
| 721 | first_frame->size()); |
Niels Möller | ff2e215 | 2019-09-27 10:29:30 +0200 | [diff] [blame] | 722 | memcpy(buffer, first_frame->data(), first_frame->size()); |
| 723 | buffer += first_frame->size(); |
Sergey Silkin | 2799e63 | 2019-05-17 09:51:39 +0200 | [diff] [blame] | 724 | |
Sergey Silkin | 61832dd | 2018-12-20 14:32:14 +0100 | [diff] [blame] | 725 | // Spatial index of combined frame is set equal to spatial index of its top |
| 726 | // spatial layer. |
| 727 | first_frame->SetSpatialIndex(last_frame->id.spatial_layer); |
| 728 | first_frame->id.spatial_layer = last_frame->id.spatial_layer; |
| 729 | |
| 730 | first_frame->video_timing_mutable()->network2_timestamp_ms = |
| 731 | last_frame->video_timing().network2_timestamp_ms; |
| 732 | first_frame->video_timing_mutable()->receive_finish_ms = |
| 733 | last_frame->video_timing().receive_finish_ms; |
| 734 | |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 735 | // Append all remaining frames to the first one. |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 736 | for (size_t i = 1; i < frames.size(); ++i) { |
Sergey Silkin | 61832dd | 2018-12-20 14:32:14 +0100 | [diff] [blame] | 737 | EncodedFrame* next_frame = frames[i]; |
Sergey Silkin | 2799e63 | 2019-05-17 09:51:39 +0200 | [diff] [blame] | 738 | first_frame->SetSpatialLayerFrameSize(next_frame->id.spatial_layer, |
| 739 | next_frame->size()); |
Niels Möller | 9c84390 | 2019-01-11 10:21:35 +0100 | [diff] [blame] | 740 | memcpy(buffer, next_frame->data(), next_frame->size()); |
Sergey Silkin | 61832dd | 2018-12-20 14:32:14 +0100 | [diff] [blame] | 741 | buffer += next_frame->size(); |
| 742 | delete next_frame; |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 743 | } |
Niels Möller | ff2e215 | 2019-09-27 10:29:30 +0200 | [diff] [blame] | 744 | first_frame->SetEncodedData(encoded_image_buffer); |
Sergey Silkin | 61832dd | 2018-12-20 14:32:14 +0100 | [diff] [blame] | 745 | return first_frame; |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 746 | } |
| 747 | |
Niels Möller | be682d4 | 2018-03-27 08:31:45 +0200 | [diff] [blame] | 748 | FrameBuffer::FrameInfo::FrameInfo() = default; |
| 749 | FrameBuffer::FrameInfo::FrameInfo(FrameInfo&&) = default; |
| 750 | FrameBuffer::FrameInfo::~FrameInfo() = default; |
| 751 | |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 752 | } // namespace video_coding |
| 753 | } // namespace webrtc |