pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #ifndef VIDEO_VIDEO_RECEIVE_STREAM_H_ |
| 12 | #define VIDEO_VIDEO_RECEIVE_STREAM_H_ |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 13 | |
kwiberg | 27f982b | 2016-03-01 11:52:33 -0800 | [diff] [blame] | 14 | #include <memory> |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 15 | #include <vector> |
| 16 | |
Niels Möller | 4687915 | 2019-01-07 15:54:47 +0100 | [diff] [blame] | 17 | #include "api/media_transport_interface.h" |
Sebastian Jansson | 74682c1 | 2019-03-01 11:50:20 +0100 | [diff] [blame] | 18 | #include "api/task_queue/task_queue_factory.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 19 | #include "call/rtp_packet_sink_interface.h" |
| 20 | #include "call/syncable.h" |
| 21 | #include "call/video_receive_stream.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 22 | #include "modules/rtp_rtcp/include/flexfec_receiver.h" |
| 23 | #include "modules/video_coding/frame_buffer2.h" |
| 24 | #include "modules/video_coding/video_coding_impl.h" |
Sebastian Jansson | b55015e | 2019-04-09 13:44:04 +0200 | [diff] [blame] | 25 | #include "rtc_base/synchronization/sequence_checker.h" |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 26 | #include "rtc_base/task_queue.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 27 | #include "system_wrappers/include/clock.h" |
| 28 | #include "video/receive_statistics_proxy.h" |
| 29 | #include "video/rtp_streams_synchronizer.h" |
| 30 | #include "video/rtp_video_stream_receiver.h" |
| 31 | #include "video/transport_adapter.h" |
| 32 | #include "video/video_stream_decoder.h" |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 33 | |
| 34 | namespace webrtc { |
| 35 | |
mflodman | e378702 | 2015-10-21 13:24:28 +0200 | [diff] [blame] | 36 | class CallStats; |
Peter Boström | d1d66ba | 2016-02-08 14:07:14 +0100 | [diff] [blame] | 37 | class ProcessThread; |
mflodman | 4cd2790 | 2016-08-05 06:28:45 -0700 | [diff] [blame] | 38 | class RTPFragmentationHeader; |
nisse | 0f15f92 | 2017-06-21 01:05:22 -0700 | [diff] [blame] | 39 | class RtpStreamReceiverInterface; |
| 40 | class RtpStreamReceiverControllerInterface; |
nisse | ca5706d | 2017-09-11 02:32:16 -0700 | [diff] [blame] | 41 | class RtxReceiveStream; |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 42 | class VCMTiming; |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 43 | |
| 44 | namespace internal { |
| 45 | |
pbos@webrtc.org | 74fa489 | 2013-08-23 09:19:30 +0000 | [diff] [blame] | 46 | class VideoReceiveStream : public webrtc::VideoReceiveStream, |
nisse | 30f118e | 2016-05-03 01:09:11 -0700 | [diff] [blame] | 47 | public rtc::VideoSinkInterface<VideoFrame>, |
philipel | 83f831a | 2016-03-12 03:30:23 -0800 | [diff] [blame] | 48 | public NackSender, |
solenberg | 3ebbcb5 | 2017-01-31 03:58:40 -0800 | [diff] [blame] | 49 | public video_coding::OnCompleteFrameCallback, |
philipel | e21be1d | 2017-09-25 06:37:12 -0700 | [diff] [blame] | 50 | public Syncable, |
Niels Möller | 4687915 | 2019-01-07 15:54:47 +0100 | [diff] [blame] | 51 | public CallStatsObserver, |
| 52 | public MediaTransportVideoSinkInterface, |
| 53 | public MediaTransportRttObserver { |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 54 | public: |
Sebastian Jansson | 74682c1 | 2019-03-01 11:50:20 +0100 | [diff] [blame] | 55 | VideoReceiveStream(TaskQueueFactory* task_queue_factory, |
| 56 | RtpStreamReceiverControllerInterface* receiver_controller, |
nisse | 0f15f92 | 2017-06-21 01:05:22 -0700 | [diff] [blame] | 57 | int num_cpu_cores, |
nisse | 0245da0 | 2016-11-30 03:35:20 -0800 | [diff] [blame] | 58 | PacketRouter* packet_router, |
Tommi | 733b547 | 2016-06-10 17:58:01 +0200 | [diff] [blame] | 59 | VideoReceiveStream::Config config, |
mflodman | e378702 | 2015-10-21 13:24:28 +0200 | [diff] [blame] | 60 | ProcessThread* process_thread, |
Ruslan Burakov | 493a650 | 2019-02-27 15:32:48 +0100 | [diff] [blame] | 61 | CallStats* call_stats, |
| 62 | Clock* clock, |
| 63 | VCMTiming* timing); |
Sebastian Jansson | 74682c1 | 2019-03-01 11:50:20 +0100 | [diff] [blame] | 64 | VideoReceiveStream(TaskQueueFactory* task_queue_factory, |
| 65 | RtpStreamReceiverControllerInterface* receiver_controller, |
Ruslan Burakov | 493a650 | 2019-02-27 15:32:48 +0100 | [diff] [blame] | 66 | int num_cpu_cores, |
| 67 | PacketRouter* packet_router, |
| 68 | VideoReceiveStream::Config config, |
| 69 | ProcessThread* process_thread, |
Sebastian Jansson | 8026d60 | 2019-03-04 19:39:01 +0100 | [diff] [blame] | 70 | CallStats* call_stats, |
| 71 | Clock* clock); |
Jelena Marusic | cd67022 | 2015-07-16 09:30:09 +0200 | [diff] [blame] | 72 | ~VideoReceiveStream() override; |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 73 | |
brandtr | 090c940 | 2017-01-25 08:28:02 -0800 | [diff] [blame] | 74 | const Config& config() const { return config_; } |
| 75 | |
pbos | 1ba8d39 | 2016-05-01 20:18:34 -0700 | [diff] [blame] | 76 | void SignalNetworkState(NetworkState state); |
| 77 | bool DeliverRtcp(const uint8_t* packet, size_t length); |
Jelena Marusic | cd67022 | 2015-07-16 09:30:09 +0200 | [diff] [blame] | 78 | |
solenberg | 3ebbcb5 | 2017-01-31 03:58:40 -0800 | [diff] [blame] | 79 | void SetSync(Syncable* audio_syncable); |
brandtr | 090c940 | 2017-01-25 08:28:02 -0800 | [diff] [blame] | 80 | |
| 81 | // Implements webrtc::VideoReceiveStream. |
pbos | 1ba8d39 | 2016-05-01 20:18:34 -0700 | [diff] [blame] | 82 | void Start() override; |
| 83 | void Stop() override; |
| 84 | |
Jelena Marusic | cd67022 | 2015-07-16 09:30:09 +0200 | [diff] [blame] | 85 | webrtc::VideoReceiveStream::Stats GetStats() const override; |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 86 | |
eladalon | c0d481a | 2017-08-02 07:39:07 -0700 | [diff] [blame] | 87 | void AddSecondarySink(RtpPacketSinkInterface* sink) override; |
| 88 | void RemoveSecondarySink(const RtpPacketSinkInterface* sink) override; |
| 89 | |
Ruslan Burakov | 493a650 | 2019-02-27 15:32:48 +0100 | [diff] [blame] | 90 | // SetBaseMinimumPlayoutDelayMs and GetBaseMinimumPlayoutDelayMs are called |
| 91 | // from webrtc/api level and requested by user code. For e.g. blink/js layer |
| 92 | // in Chromium. |
| 93 | bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override; |
| 94 | int GetBaseMinimumPlayoutDelayMs() const override; |
| 95 | |
Benjamin Wright | a556448 | 2019-04-03 10:44:18 -0700 | [diff] [blame] | 96 | void SetFrameDecryptor( |
| 97 | rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor) override; |
| 98 | |
brandtr | 090c940 | 2017-01-25 08:28:02 -0800 | [diff] [blame] | 99 | // Implements rtc::VideoSinkInterface<VideoFrame>. |
| 100 | void OnFrame(const VideoFrame& video_frame) override; |
| 101 | |
brandtr | 090c940 | 2017-01-25 08:28:02 -0800 | [diff] [blame] | 102 | // Implements NackSender. |
| 103 | void SendNack(const std::vector<uint16_t>& sequence_numbers) override; |
Elad Alon | ef09c5b | 2019-05-31 13:25:50 +0200 | [diff] [blame] | 104 | // For this particular override of the interface, |
| 105 | // only (buffering_allowed == true) is acceptable. |
| 106 | void SendNack(const std::vector<uint16_t>& sequence_numbers, |
| 107 | bool buffering_allowed) override; |
brandtr | 090c940 | 2017-01-25 08:28:02 -0800 | [diff] [blame] | 108 | |
brandtr | 090c940 | 2017-01-25 08:28:02 -0800 | [diff] [blame] | 109 | // Implements video_coding::OnCompleteFrameCallback. |
| 110 | void OnCompleteFrame( |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 111 | std::unique_ptr<video_coding::EncodedFrame> frame) override; |
brandtr | 090c940 | 2017-01-25 08:28:02 -0800 | [diff] [blame] | 112 | |
Niels Möller | 4687915 | 2019-01-07 15:54:47 +0100 | [diff] [blame] | 113 | // Implements MediaTransportVideoSinkInterface, converts the received frame to |
| 114 | // OnCompleteFrameCallback |
| 115 | void OnData(uint64_t channel_id, |
| 116 | MediaTransportEncodedVideoFrame frame) override; |
| 117 | |
philipel | e21be1d | 2017-09-25 06:37:12 -0700 | [diff] [blame] | 118 | // Implements CallStatsObserver::OnRttUpdate |
| 119 | void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override; |
| 120 | |
Niels Möller | 4687915 | 2019-01-07 15:54:47 +0100 | [diff] [blame] | 121 | // Implements MediaTransportRttObserver::OnRttUpdated |
| 122 | void OnRttUpdated(int64_t rtt_ms) override; |
| 123 | |
solenberg | 3ebbcb5 | 2017-01-31 03:58:40 -0800 | [diff] [blame] | 124 | // Implements Syncable. |
| 125 | int id() const override; |
Danil Chapovalov | b9b146c | 2018-06-15 12:28:07 +0200 | [diff] [blame] | 126 | absl::optional<Syncable::Info> GetInfo() const override; |
solenberg | 3ebbcb5 | 2017-01-31 03:58:40 -0800 | [diff] [blame] | 127 | uint32_t GetPlayoutTimestamp() const override; |
Ruslan Burakov | 493a650 | 2019-02-27 15:32:48 +0100 | [diff] [blame] | 128 | |
| 129 | // SetMinimumPlayoutDelay is only called by A/V sync. |
solenberg | 3ebbcb5 | 2017-01-31 03:58:40 -0800 | [diff] [blame] | 130 | void SetMinimumPlayoutDelay(int delay_ms) override; |
| 131 | |
Jonas Oreland | 49ac595 | 2018-09-26 16:04:32 +0200 | [diff] [blame] | 132 | std::vector<webrtc::RtpSource> GetSources() const override; |
| 133 | |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 134 | private: |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 135 | int64_t GetWaitMs() const; |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 136 | void StartNextDecode() RTC_RUN_ON(decode_queue_); |
tommi | c8ece43 | 2017-06-20 02:44:38 -0700 | [diff] [blame] | 137 | static void DecodeThreadFunction(void* ptr); |
philipel | 2dfea3e | 2017-02-28 07:19:43 -0800 | [diff] [blame] | 138 | bool Decode(); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 139 | void HandleEncodedFrame(std::unique_ptr<video_coding::EncodedFrame> frame); |
| 140 | void HandleFrameBufferTimeout(); |
| 141 | |
Ruslan Burakov | 493a650 | 2019-02-27 15:32:48 +0100 | [diff] [blame] | 142 | void UpdatePlayoutDelays() const |
| 143 | RTC_EXCLUSIVE_LOCKS_REQUIRED(playout_delay_lock_); |
Niels Möller | 479c055 | 2019-05-23 16:57:01 +0200 | [diff] [blame] | 144 | void RequestKeyFrame(); |
Peter Boström | ca83525 | 2016-02-11 15:59:46 +0100 | [diff] [blame] | 145 | |
Sebastian Jansson | b55015e | 2019-04-09 13:44:04 +0200 | [diff] [blame] | 146 | SequenceChecker worker_sequence_checker_; |
| 147 | SequenceChecker module_process_sequence_checker_; |
| 148 | SequenceChecker network_sequence_checker_; |
solenberg | 3ebbcb5 | 2017-01-31 03:58:40 -0800 | [diff] [blame] | 149 | |
Sebastian Jansson | 74682c1 | 2019-03-01 11:50:20 +0100 | [diff] [blame] | 150 | TaskQueueFactory* const task_queue_factory_; |
| 151 | |
pbos@webrtc.org | e75a1bf | 2013-09-18 11:52:42 +0000 | [diff] [blame] | 152 | TransportAdapter transport_adapter_; |
pbos@webrtc.org | 6ae48c6 | 2014-06-06 10:49:19 +0000 | [diff] [blame] | 153 | const VideoReceiveStream::Config config_; |
sprang | 113bdca | 2016-10-11 03:10:10 -0700 | [diff] [blame] | 154 | const int num_cpu_cores_; |
Peter Boström | 1d04ac6 | 2016-02-05 11:25:46 +0100 | [diff] [blame] | 155 | ProcessThread* const process_thread_; |
pbos@webrtc.org | de1429e | 2014-04-28 13:00:21 +0000 | [diff] [blame] | 156 | Clock* const clock_; |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 157 | |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 158 | const bool use_task_queue_; |
| 159 | |
Peter Boström | ca83525 | 2016-02-11 15:59:46 +0100 | [diff] [blame] | 160 | rtc::PlatformThread decode_thread_; |
| 161 | |
mflodman | e378702 | 2015-10-21 13:24:28 +0200 | [diff] [blame] | 162 | CallStats* const call_stats_; |
Peter Boström | 45553ae | 2015-05-08 13:54:38 +0200 | [diff] [blame] | 163 | |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 164 | bool decoder_running_ RTC_GUARDED_BY(worker_sequence_checker_) = false; |
| 165 | bool decoder_stopped_ RTC_GUARDED_BY(decode_queue_) = true; |
| 166 | |
Danil Chapovalov | 8ce0d2b | 2018-11-23 11:03:25 +0100 | [diff] [blame] | 167 | ReceiveStatisticsProxy stats_proxy_; |
nisse | ca5706d | 2017-09-11 02:32:16 -0700 | [diff] [blame] | 168 | // Shared by media and rtx stream receivers, since the latter has no RtpRtcp |
| 169 | // module of its own. |
| 170 | const std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_; |
| 171 | |
philipel | 721d402 | 2016-12-15 07:10:57 -0800 | [diff] [blame] | 172 | std::unique_ptr<VCMTiming> timing_; // Jitter buffer experiment. |
Peter Boström | 0b25072 | 2016-04-22 18:23:15 +0200 | [diff] [blame] | 173 | vcm::VideoReceiver video_receiver_; |
tommi | 2e82f38 | 2016-06-21 00:26:43 -0700 | [diff] [blame] | 174 | std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> incoming_video_stream_; |
nisse | b1f2ff9 | 2017-06-09 04:01:55 -0700 | [diff] [blame] | 175 | RtpVideoStreamReceiver rtp_video_stream_receiver_; |
tommi | 2e82f38 | 2016-06-21 00:26:43 -0700 | [diff] [blame] | 176 | std::unique_ptr<VideoStreamDecoder> video_stream_decoder_; |
mflodman | 4cd2790 | 2016-08-05 06:28:45 -0700 | [diff] [blame] | 177 | RtpStreamsSynchronizer rtp_stream_sync_; |
sprang | 3911c26 | 2016-04-15 01:24:14 -0700 | [diff] [blame] | 178 | |
Niels Möller | cbcbc22 | 2018-09-28 09:07:24 +0200 | [diff] [blame] | 179 | // TODO(nisse, philipel): Creation and ownership of video encoders should be |
| 180 | // moved to the new VideoStreamDecoder. |
| 181 | std::vector<std::unique_ptr<VideoDecoder>> video_decoders_; |
| 182 | |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 183 | // Members for the new jitter buffer experiment. |
Henrik Boström | c680c4a | 2019-04-03 10:27:36 +0000 | [diff] [blame] | 184 | std::unique_ptr<video_coding::FrameBuffer> frame_buffer_; |
nisse | 0f15f92 | 2017-06-21 01:05:22 -0700 | [diff] [blame] | 185 | |
| 186 | std::unique_ptr<RtpStreamReceiverInterface> media_receiver_; |
nisse | ca5706d | 2017-09-11 02:32:16 -0700 | [diff] [blame] | 187 | std::unique_ptr<RtxReceiveStream> rtx_receive_stream_; |
nisse | 0f15f92 | 2017-06-21 01:05:22 -0700 | [diff] [blame] | 188 | std::unique_ptr<RtpStreamReceiverInterface> rtx_receiver_; |
philipel | 3042c2d | 2017-08-18 04:55:02 -0700 | [diff] [blame] | 189 | |
| 190 | // Whenever we are in an undecodable state (stream has just started or due to |
| 191 | // a decoding error) we require a keyframe to restart the stream. |
| 192 | bool keyframe_required_ = true; |
| 193 | |
| 194 | // If we have successfully decoded any frame. |
| 195 | bool frame_decoded_ = false; |
philipel | 48462b6 | 2017-09-26 02:54:58 -0700 | [diff] [blame] | 196 | |
| 197 | int64_t last_keyframe_request_ms_ = 0; |
Ilya Nikolaevskiy | e6a2d94 | 2018-11-07 14:32:28 +0100 | [diff] [blame] | 198 | int64_t last_complete_frame_time_ms_ = 0; |
Ruslan Burakov | 493a650 | 2019-02-27 15:32:48 +0100 | [diff] [blame] | 199 | |
Rasmus Brandt | 3dde450 | 2019-03-21 11:46:17 +0100 | [diff] [blame] | 200 | // Keyframe request intervals are configurable through field trials. |
| 201 | const int max_wait_for_keyframe_ms_; |
| 202 | const int max_wait_for_frame_ms_; |
| 203 | |
Ruslan Burakov | 493a650 | 2019-02-27 15:32:48 +0100 | [diff] [blame] | 204 | rtc::CriticalSection playout_delay_lock_; |
| 205 | |
| 206 | // All of them tries to change current min_playout_delay on |timing_| but |
| 207 | // source of the change request is different in each case. Among them the |
| 208 | // biggest delay is used. -1 means use default value from the |timing_|. |
| 209 | // |
| 210 | // Minimum delay as decided by the RTP playout delay extension. |
| 211 | int frame_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; |
| 212 | // Minimum delay as decided by the setLatency function in "webrtc/api". |
| 213 | int base_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; |
| 214 | // Minimum delay as decided by the A/V synchronization feature. |
| 215 | int syncable_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = |
| 216 | -1; |
| 217 | |
| 218 | // Maximum delay as decided by the RTP playout delay extension. |
| 219 | int frame_maximum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 220 | |
| 221 | // Defined last so they are destroyed before all other members. |
| 222 | rtc::TaskQueue decode_queue_; |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 223 | }; |
mflodman@webrtc.org | f3973e8 | 2013-12-13 09:40:45 +0000 | [diff] [blame] | 224 | } // namespace internal |
| 225 | } // namespace webrtc |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 226 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 227 | #endif // VIDEO_VIDEO_RECEIVE_STREAM_H_ |