niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 1 | /* |
stefan@webrtc.org | 91c6308 | 2012-01-31 10:49:08 +0000 | [diff] [blame] | 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #include "modules/video_coding/receiver.h" |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 12 | |
| 13 | #include <assert.h> |
Jonas Olsson | a4d8737 | 2019-07-05 19:08:33 +0200 | [diff] [blame] | 14 | |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 15 | #include <cstdint> |
pbos@webrtc.org | 3f655aa | 2014-03-18 11:10:11 +0000 | [diff] [blame] | 16 | #include <cstdlib> |
kwiberg | 0eb15ed | 2015-12-17 03:04:15 -0800 | [diff] [blame] | 17 | #include <utility> |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 18 | #include <vector> |
pbos@webrtc.org | 3f655aa | 2014-03-18 11:10:11 +0000 | [diff] [blame] | 19 | |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 20 | #include "absl/memory/memory.h" |
| 21 | #include "api/video/encoded_image.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 22 | #include "modules/video_coding/encoded_frame.h" |
| 23 | #include "modules/video_coding/internal_defines.h" |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 24 | #include "modules/video_coding/jitter_buffer_common.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 25 | #include "rtc_base/logging.h" |
Ilya Nikolaevskiy | 8c4fe16 | 2018-02-27 15:49:47 +0100 | [diff] [blame] | 26 | #include "rtc_base/numerics/safe_conversions.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 27 | #include "rtc_base/trace_event.h" |
| 28 | #include "system_wrappers/include/clock.h" |
stefan@webrtc.org | 91c6308 | 2012-01-31 10:49:08 +0000 | [diff] [blame] | 29 | |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 30 | namespace webrtc { |
| 31 | |
mikhal@webrtc.org | ef9f76a | 2013-02-15 23:22:18 +0000 | [diff] [blame] | 32 | enum { kMaxReceiverDelayMs = 10000 }; |
| 33 | |
Niels Möller | 689983f | 2018-11-07 16:36:22 +0100 | [diff] [blame] | 34 | VCMReceiver::VCMReceiver(VCMTiming* timing, Clock* clock) |
philipel | 83f831a | 2016-03-12 03:30:23 -0800 | [diff] [blame] | 35 | : VCMReceiver::VCMReceiver(timing, |
| 36 | clock, |
Niels Möller | 689983f | 2018-11-07 16:36:22 +0100 | [diff] [blame] | 37 | absl::WrapUnique(EventWrapper::Create()), |
Niels Möller | db64d99 | 2019-03-29 14:30:53 +0100 | [diff] [blame] | 38 | absl::WrapUnique(EventWrapper::Create())) {} |
Qiang Chen | d4cec15 | 2015-06-19 09:17:00 -0700 | [diff] [blame] | 39 | |
| 40 | VCMReceiver::VCMReceiver(VCMTiming* timing, |
| 41 | Clock* clock, |
kwiberg | 3f55dea | 2016-02-29 05:51:59 -0800 | [diff] [blame] | 42 | std::unique_ptr<EventWrapper> receiver_event, |
| 43 | std::unique_ptr<EventWrapper> jitter_buffer_event) |
kthelgason | d701dfd | 2017-03-27 07:24:57 -0700 | [diff] [blame] | 44 | : clock_(clock), |
Niels Möller | db64d99 | 2019-03-29 14:30:53 +0100 | [diff] [blame] | 45 | jitter_buffer_(clock_, std::move(jitter_buffer_event)), |
stefan@webrtc.org | 1ea4b50 | 2013-01-07 08:49:41 +0000 | [diff] [blame] | 46 | timing_(timing), |
kwiberg | 0eb15ed | 2015-12-17 03:04:15 -0800 | [diff] [blame] | 47 | render_wait_event_(std::move(receiver_event)), |
Peter Boström | 5464a6e | 2015-04-21 16:35:50 +0200 | [diff] [blame] | 48 | max_video_delay_ms_(kMaxVideoDelayMs) { |
Niels Möller | 45b01c7 | 2019-09-10 13:02:28 +0200 | [diff] [blame] | 49 | jitter_buffer_.Start(); |
Peter Boström | 5464a6e | 2015-04-21 16:35:50 +0200 | [diff] [blame] | 50 | } |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 51 | |
stefan@webrtc.org | 1ea4b50 | 2013-01-07 08:49:41 +0000 | [diff] [blame] | 52 | VCMReceiver::~VCMReceiver() { |
stefan@webrtc.org | 2baf5f5 | 2013-03-13 08:46:25 +0000 | [diff] [blame] | 53 | render_wait_event_->Set(); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 54 | } |
| 55 | |
Johan Ahlers | 95348f7 | 2016-06-28 11:11:28 +0200 | [diff] [blame] | 56 | int32_t VCMReceiver::InsertPacket(const VCMPacket& packet) { |
stefan@webrtc.org | 3417eb4 | 2013-05-21 15:25:53 +0000 | [diff] [blame] | 57 | // Insert the packet into the jitter buffer. The packet can either be empty or |
| 58 | // contain media at this point. |
| 59 | bool retransmitted = false; |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 60 | const VCMFrameBufferEnum ret = |
| 61 | jitter_buffer_.InsertPacket(packet, &retransmitted); |
stefan@webrtc.org | 3417eb4 | 2013-05-21 15:25:53 +0000 | [diff] [blame] | 62 | if (ret == kOldPacket) { |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 63 | return VCM_OK; |
stefan@webrtc.org | 3417eb4 | 2013-05-21 15:25:53 +0000 | [diff] [blame] | 64 | } else if (ret == kFlushIndicator) { |
| 65 | return VCM_FLUSH_INDICATOR; |
| 66 | } else if (ret < 0) { |
stefan@webrtc.org | 3417eb4 | 2013-05-21 15:25:53 +0000 | [diff] [blame] | 67 | return VCM_JITTER_BUFFER_ERROR; |
stefan@webrtc.org | 1ea4b50 | 2013-01-07 08:49:41 +0000 | [diff] [blame] | 68 | } |
stefan@webrtc.org | 3417eb4 | 2013-05-21 15:25:53 +0000 | [diff] [blame] | 69 | if (ret == kCompleteSession && !retransmitted) { |
| 70 | // We don't want to include timestamps which have suffered from |
| 71 | // retransmission here, since we compensate with extra retransmission |
| 72 | // delay within the jitter estimate. |
| 73 | timing_->IncomingTimestamp(packet.timestamp, clock_->TimeInMilliseconds()); |
| 74 | } |
stefan@webrtc.org | 1ea4b50 | 2013-01-07 08:49:41 +0000 | [diff] [blame] | 75 | return VCM_OK; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 76 | } |
| 77 | |
pbos@webrtc.org | 4f16c87 | 2014-11-24 09:06:48 +0000 | [diff] [blame] | 78 | VCMEncodedFrame* VCMReceiver::FrameForDecoding(uint16_t max_wait_time_ms, |
perkj | 796cfaf | 2015-12-10 09:27:38 -0800 | [diff] [blame] | 79 | bool prefer_late_decoding) { |
stefan@webrtc.org | a678a3b | 2013-01-21 07:42:11 +0000 | [diff] [blame] | 80 | const int64_t start_time_ms = clock_->TimeInMilliseconds(); |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 81 | uint32_t frame_timestamp = 0; |
isheriff | 6b4b5f3 | 2016-06-08 00:24:21 -0700 | [diff] [blame] | 82 | int min_playout_delay_ms = -1; |
| 83 | int max_playout_delay_ms = -1; |
Johan Ahlers | 31b2ec4 | 2016-06-28 13:32:49 +0200 | [diff] [blame] | 84 | int64_t render_time_ms = 0; |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 85 | // Exhaust wait time to get a complete frame for decoding. |
isheriff | 6b4b5f3 | 2016-06-08 00:24:21 -0700 | [diff] [blame] | 86 | VCMEncodedFrame* found_frame = |
| 87 | jitter_buffer_.NextCompleteFrame(max_wait_time_ms); |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 88 | |
isheriff | 6b4b5f3 | 2016-06-08 00:24:21 -0700 | [diff] [blame] | 89 | if (found_frame) { |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 90 | frame_timestamp = found_frame->Timestamp(); |
isheriff | 6b4b5f3 | 2016-06-08 00:24:21 -0700 | [diff] [blame] | 91 | min_playout_delay_ms = found_frame->EncodedImage().playout_delay_.min_ms; |
| 92 | max_playout_delay_ms = found_frame->EncodedImage().playout_delay_.max_ms; |
| 93 | } else { |
Niels Möller | 375b346 | 2019-01-10 15:35:56 +0100 | [diff] [blame] | 94 | return nullptr; |
isheriff | 6b4b5f3 | 2016-06-08 00:24:21 -0700 | [diff] [blame] | 95 | } |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 96 | |
isheriff | 6b4b5f3 | 2016-06-08 00:24:21 -0700 | [diff] [blame] | 97 | if (min_playout_delay_ms >= 0) |
| 98 | timing_->set_min_playout_delay(min_playout_delay_ms); |
| 99 | |
| 100 | if (max_playout_delay_ms >= 0) |
| 101 | timing_->set_max_playout_delay(max_playout_delay_ms); |
mikhal@webrtc.org | d3cd565 | 2013-05-03 17:54:18 +0000 | [diff] [blame] | 102 | |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 103 | // We have a frame - Set timing and render timestamp. |
mikhal@webrtc.org | adc64a7 | 2013-05-30 16:20:18 +0000 | [diff] [blame] | 104 | timing_->SetJitterDelay(jitter_buffer_.EstimatedJitterMs()); |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 105 | const int64_t now_ms = clock_->TimeInMilliseconds(); |
| 106 | timing_->UpdateCurrentDelay(frame_timestamp); |
Johan Ahlers | 31b2ec4 | 2016-06-28 13:32:49 +0200 | [diff] [blame] | 107 | render_time_ms = timing_->RenderTimeMs(frame_timestamp, now_ms); |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 108 | // Check render timing. |
| 109 | bool timing_error = false; |
| 110 | // Assume that render timing errors are due to changes in the video stream. |
Johan Ahlers | 31b2ec4 | 2016-06-28 13:32:49 +0200 | [diff] [blame] | 111 | if (render_time_ms < 0) { |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 112 | timing_error = true; |
Johan Ahlers | 31b2ec4 | 2016-06-28 13:32:49 +0200 | [diff] [blame] | 113 | } else if (std::abs(render_time_ms - now_ms) > max_video_delay_ms_) { |
| 114 | int frame_delay = static_cast<int>(std::abs(render_time_ms - now_ms)); |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 115 | RTC_LOG(LS_WARNING) |
| 116 | << "A frame about to be decoded is out of the configured " |
Jonas Olsson | b2b2031 | 2020-01-14 12:11:31 +0100 | [diff] [blame] | 117 | "delay bounds (" |
| 118 | << frame_delay << " > " << max_video_delay_ms_ |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 119 | << "). Resetting the video jitter buffer."; |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 120 | timing_error = true; |
| 121 | } else if (static_cast<int>(timing_->TargetVideoDelay()) > |
| 122 | max_video_delay_ms_) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 123 | RTC_LOG(LS_WARNING) << "The video target delay has grown larger than " |
| 124 | << max_video_delay_ms_ |
| 125 | << " ms. Resetting jitter buffer."; |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 126 | timing_error = true; |
stefan@webrtc.org | 4ce19b1 | 2013-05-06 13:16:51 +0000 | [diff] [blame] | 127 | } |
| 128 | |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 129 | if (timing_error) { |
| 130 | // Timing error => reset timing and flush the jitter buffer. |
| 131 | jitter_buffer_.Flush(); |
stefan@webrtc.org | 9f557c1 | 2013-05-17 12:55:07 +0000 | [diff] [blame] | 132 | timing_->Reset(); |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 133 | return NULL; |
| 134 | } |
| 135 | |
perkj | 796cfaf | 2015-12-10 09:27:38 -0800 | [diff] [blame] | 136 | if (prefer_late_decoding) { |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 137 | // Decode frame as close as possible to the render timestamp. |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 138 | const int32_t available_wait_time = |
| 139 | max_wait_time_ms - |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 140 | static_cast<int32_t>(clock_->TimeInMilliseconds() - start_time_ms); |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 141 | uint16_t new_max_wait_time = |
| 142 | static_cast<uint16_t>(VCM_MAX(available_wait_time, 0)); |
Ilya Nikolaevskiy | 8c4fe16 | 2018-02-27 15:49:47 +0100 | [diff] [blame] | 143 | uint32_t wait_time_ms = rtc::saturated_cast<uint32_t>( |
| 144 | timing_->MaxWaitingTime(render_time_ms, clock_->TimeInMilliseconds())); |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 145 | if (new_max_wait_time < wait_time_ms) { |
| 146 | // We're not allowed to wait until the frame is supposed to be rendered, |
| 147 | // waiting as long as we're allowed to avoid busy looping, and then return |
| 148 | // NULL. Next call to this function might return the frame. |
Niklas Enbom | b4c5eaa | 2015-06-03 09:34:25 -0700 | [diff] [blame] | 149 | render_wait_event_->Wait(new_max_wait_time); |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 150 | return NULL; |
| 151 | } |
| 152 | // Wait until it's time to render. |
| 153 | render_wait_event_->Wait(wait_time_ms); |
| 154 | } |
| 155 | |
| 156 | // Extract the frame from the jitter buffer and set the render time. |
| 157 | VCMEncodedFrame* frame = jitter_buffer_.ExtractAndSetDecode(frame_timestamp); |
mikhal@webrtc.org | 8f86cc8 | 2013-05-07 18:05:21 +0000 | [diff] [blame] | 158 | if (frame == NULL) { |
| 159 | return NULL; |
| 160 | } |
Johan Ahlers | 31b2ec4 | 2016-06-28 13:32:49 +0200 | [diff] [blame] | 161 | frame->SetRenderTime(render_time_ms); |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 162 | TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame->Timestamp(), "SetRenderTS", |
Johan Ahlers | 31b2ec4 | 2016-06-28 13:32:49 +0200 | [diff] [blame] | 163 | "render_time", frame->RenderTimeMs()); |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 164 | if (!frame->Complete()) { |
| 165 | // Update stats for incomplete frames. |
stefan@webrtc.org | 4ce19b1 | 2013-05-06 13:16:51 +0000 | [diff] [blame] | 166 | bool retransmitted = false; |
| 167 | const int64_t last_packet_time_ms = |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 168 | jitter_buffer_.LastPacketTime(frame, &retransmitted); |
stefan@webrtc.org | 4ce19b1 | 2013-05-06 13:16:51 +0000 | [diff] [blame] | 169 | if (last_packet_time_ms >= 0 && !retransmitted) { |
| 170 | // We don't want to include timestamps which have suffered from |
| 171 | // retransmission here, since we compensate with extra retransmission |
| 172 | // delay within the jitter estimate. |
mikhal@webrtc.org | 759b041 | 2013-05-07 16:36:00 +0000 | [diff] [blame] | 173 | timing_->IncomingTimestamp(frame_timestamp, last_packet_time_ms); |
stefan@webrtc.org | 4ce19b1 | 2013-05-06 13:16:51 +0000 | [diff] [blame] | 174 | } |
stefan@webrtc.org | 1ea4b50 | 2013-01-07 08:49:41 +0000 | [diff] [blame] | 175 | } |
| 176 | return frame; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 177 | } |
| 178 | |
stefan@webrtc.org | 1ea4b50 | 2013-01-07 08:49:41 +0000 | [diff] [blame] | 179 | void VCMReceiver::ReleaseFrame(VCMEncodedFrame* frame) { |
| 180 | jitter_buffer_.ReleaseFrame(frame); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 181 | } |
| 182 | |
stefan@webrtc.org | becf9c8 | 2013-02-01 15:09:57 +0000 | [diff] [blame] | 183 | void VCMReceiver::SetNackSettings(size_t max_nack_list_size, |
stefan@webrtc.org | ef14488 | 2013-05-07 19:16:33 +0000 | [diff] [blame] | 184 | int max_packet_age_to_nack, |
| 185 | int max_incomplete_time_ms) { |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 186 | jitter_buffer_.SetNackSettings(max_nack_list_size, max_packet_age_to_nack, |
stefan@webrtc.org | ef14488 | 2013-05-07 19:16:33 +0000 | [diff] [blame] | 187 | max_incomplete_time_ms); |
stefan@webrtc.org | becf9c8 | 2013-02-01 15:09:57 +0000 | [diff] [blame] | 188 | } |
| 189 | |
Wan-Teh Chang | b1825a4 | 2015-06-03 15:03:35 -0700 | [diff] [blame] | 190 | std::vector<uint16_t> VCMReceiver::NackList(bool* request_key_frame) { |
| 191 | return jitter_buffer_.GetNackList(request_key_frame); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 192 | } |
| 193 | |
stefan@webrtc.org | 1ea4b50 | 2013-01-07 08:49:41 +0000 | [diff] [blame] | 194 | } // namespace webrtc |