pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #ifndef VIDEO_VIDEO_RECEIVE_STREAM_H_ |
| 12 | #define VIDEO_VIDEO_RECEIVE_STREAM_H_ |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 13 | |
kwiberg | 27f982b | 2016-03-01 11:52:33 -0800 | [diff] [blame] | 14 | #include <memory> |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 15 | #include <vector> |
| 16 | |
Sebastian Jansson | 74682c1 | 2019-03-01 11:50:20 +0100 | [diff] [blame] | 17 | #include "api/task_queue/task_queue_factory.h" |
Markus Handell | 269ac81 | 2019-12-03 14:31:45 +0100 | [diff] [blame] | 18 | #include "api/transport/media/media_transport_interface.h" |
| 19 | #include "api/video/recordable_encoded_frame.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 20 | #include "call/rtp_packet_sink_interface.h" |
| 21 | #include "call/syncable.h" |
| 22 | #include "call/video_receive_stream.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 23 | #include "modules/rtp_rtcp/include/flexfec_receiver.h" |
Chen Xing | 90f3b89 | 2019-06-25 10:16:14 +0200 | [diff] [blame] | 24 | #include "modules/rtp_rtcp/source/source_tracker.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 25 | #include "modules/video_coding/frame_buffer2.h" |
Niels Möller | ee3d995 | 2019-09-09 12:51:55 +0200 | [diff] [blame] | 26 | #include "modules/video_coding/video_receiver2.h" |
Sebastian Jansson | b55015e | 2019-04-09 13:44:04 +0200 | [diff] [blame] | 27 | #include "rtc_base/synchronization/sequence_checker.h" |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 28 | #include "rtc_base/task_queue.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 29 | #include "system_wrappers/include/clock.h" |
| 30 | #include "video/receive_statistics_proxy.h" |
| 31 | #include "video/rtp_streams_synchronizer.h" |
| 32 | #include "video/rtp_video_stream_receiver.h" |
| 33 | #include "video/transport_adapter.h" |
| 34 | #include "video/video_stream_decoder.h" |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 35 | |
| 36 | namespace webrtc { |
| 37 | |
mflodman | e378702 | 2015-10-21 13:24:28 +0200 | [diff] [blame] | 38 | class CallStats; |
Peter Boström | d1d66ba | 2016-02-08 14:07:14 +0100 | [diff] [blame] | 39 | class ProcessThread; |
mflodman | 4cd2790 | 2016-08-05 06:28:45 -0700 | [diff] [blame] | 40 | class RTPFragmentationHeader; |
nisse | 0f15f92 | 2017-06-21 01:05:22 -0700 | [diff] [blame] | 41 | class RtpStreamReceiverInterface; |
| 42 | class RtpStreamReceiverControllerInterface; |
nisse | ca5706d | 2017-09-11 02:32:16 -0700 | [diff] [blame] | 43 | class RtxReceiveStream; |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 44 | class VCMTiming; |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 45 | |
| 46 | namespace internal { |
| 47 | |
pbos@webrtc.org | 74fa489 | 2013-08-23 09:19:30 +0000 | [diff] [blame] | 48 | class VideoReceiveStream : public webrtc::VideoReceiveStream, |
nisse | 30f118e | 2016-05-03 01:09:11 -0700 | [diff] [blame] | 49 | public rtc::VideoSinkInterface<VideoFrame>, |
philipel | 83f831a | 2016-03-12 03:30:23 -0800 | [diff] [blame] | 50 | public NackSender, |
solenberg | 3ebbcb5 | 2017-01-31 03:58:40 -0800 | [diff] [blame] | 51 | public video_coding::OnCompleteFrameCallback, |
philipel | e21be1d | 2017-09-25 06:37:12 -0700 | [diff] [blame] | 52 | public Syncable, |
Bjorn A Mellem | 7a9a092 | 2019-11-26 09:19:40 -0800 | [diff] [blame] | 53 | public CallStatsObserver { |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 54 | public: |
Markus Handell | 269ac81 | 2019-12-03 14:31:45 +0100 | [diff] [blame] | 55 | // The default number of milliseconds to pass before re-requesting a key frame |
| 56 | // to be sent. |
| 57 | static constexpr int kMaxWaitForKeyFrameMs = 200; |
| 58 | |
Sebastian Jansson | 74682c1 | 2019-03-01 11:50:20 +0100 | [diff] [blame] | 59 | VideoReceiveStream(TaskQueueFactory* task_queue_factory, |
| 60 | RtpStreamReceiverControllerInterface* receiver_controller, |
nisse | 0f15f92 | 2017-06-21 01:05:22 -0700 | [diff] [blame] | 61 | int num_cpu_cores, |
nisse | 0245da0 | 2016-11-30 03:35:20 -0800 | [diff] [blame] | 62 | PacketRouter* packet_router, |
Tommi | 733b547 | 2016-06-10 17:58:01 +0200 | [diff] [blame] | 63 | VideoReceiveStream::Config config, |
mflodman | e378702 | 2015-10-21 13:24:28 +0200 | [diff] [blame] | 64 | ProcessThread* process_thread, |
Ruslan Burakov | 493a650 | 2019-02-27 15:32:48 +0100 | [diff] [blame] | 65 | CallStats* call_stats, |
| 66 | Clock* clock, |
| 67 | VCMTiming* timing); |
Sebastian Jansson | 74682c1 | 2019-03-01 11:50:20 +0100 | [diff] [blame] | 68 | VideoReceiveStream(TaskQueueFactory* task_queue_factory, |
| 69 | RtpStreamReceiverControllerInterface* receiver_controller, |
Ruslan Burakov | 493a650 | 2019-02-27 15:32:48 +0100 | [diff] [blame] | 70 | int num_cpu_cores, |
| 71 | PacketRouter* packet_router, |
| 72 | VideoReceiveStream::Config config, |
| 73 | ProcessThread* process_thread, |
Sebastian Jansson | 8026d60 | 2019-03-04 19:39:01 +0100 | [diff] [blame] | 74 | CallStats* call_stats, |
| 75 | Clock* clock); |
Jelena Marusic | cd67022 | 2015-07-16 09:30:09 +0200 | [diff] [blame] | 76 | ~VideoReceiveStream() override; |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 77 | |
brandtr | 090c940 | 2017-01-25 08:28:02 -0800 | [diff] [blame] | 78 | const Config& config() const { return config_; } |
| 79 | |
pbos | 1ba8d39 | 2016-05-01 20:18:34 -0700 | [diff] [blame] | 80 | void SignalNetworkState(NetworkState state); |
| 81 | bool DeliverRtcp(const uint8_t* packet, size_t length); |
Jelena Marusic | cd67022 | 2015-07-16 09:30:09 +0200 | [diff] [blame] | 82 | |
solenberg | 3ebbcb5 | 2017-01-31 03:58:40 -0800 | [diff] [blame] | 83 | void SetSync(Syncable* audio_syncable); |
brandtr | 090c940 | 2017-01-25 08:28:02 -0800 | [diff] [blame] | 84 | |
| 85 | // Implements webrtc::VideoReceiveStream. |
pbos | 1ba8d39 | 2016-05-01 20:18:34 -0700 | [diff] [blame] | 86 | void Start() override; |
| 87 | void Stop() override; |
| 88 | |
Jelena Marusic | cd67022 | 2015-07-16 09:30:09 +0200 | [diff] [blame] | 89 | webrtc::VideoReceiveStream::Stats GetStats() const override; |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 90 | |
eladalon | c0d481a | 2017-08-02 07:39:07 -0700 | [diff] [blame] | 91 | void AddSecondarySink(RtpPacketSinkInterface* sink) override; |
| 92 | void RemoveSecondarySink(const RtpPacketSinkInterface* sink) override; |
| 93 | |
Ruslan Burakov | 493a650 | 2019-02-27 15:32:48 +0100 | [diff] [blame] | 94 | // SetBaseMinimumPlayoutDelayMs and GetBaseMinimumPlayoutDelayMs are called |
| 95 | // from webrtc/api level and requested by user code. For e.g. blink/js layer |
| 96 | // in Chromium. |
| 97 | bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override; |
| 98 | int GetBaseMinimumPlayoutDelayMs() const override; |
| 99 | |
Benjamin Wright | a556448 | 2019-04-03 10:44:18 -0700 | [diff] [blame] | 100 | void SetFrameDecryptor( |
| 101 | rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor) override; |
| 102 | |
brandtr | 090c940 | 2017-01-25 08:28:02 -0800 | [diff] [blame] | 103 | // Implements rtc::VideoSinkInterface<VideoFrame>. |
| 104 | void OnFrame(const VideoFrame& video_frame) override; |
| 105 | |
brandtr | 090c940 | 2017-01-25 08:28:02 -0800 | [diff] [blame] | 106 | // Implements NackSender. |
Elad Alon | ef09c5b | 2019-05-31 13:25:50 +0200 | [diff] [blame] | 107 | // For this particular override of the interface, |
| 108 | // only (buffering_allowed == true) is acceptable. |
| 109 | void SendNack(const std::vector<uint16_t>& sequence_numbers, |
| 110 | bool buffering_allowed) override; |
brandtr | 090c940 | 2017-01-25 08:28:02 -0800 | [diff] [blame] | 111 | |
brandtr | 090c940 | 2017-01-25 08:28:02 -0800 | [diff] [blame] | 112 | // Implements video_coding::OnCompleteFrameCallback. |
| 113 | void OnCompleteFrame( |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 114 | std::unique_ptr<video_coding::EncodedFrame> frame) override; |
brandtr | 090c940 | 2017-01-25 08:28:02 -0800 | [diff] [blame] | 115 | |
philipel | e21be1d | 2017-09-25 06:37:12 -0700 | [diff] [blame] | 116 | // Implements CallStatsObserver::OnRttUpdate |
| 117 | void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override; |
| 118 | |
solenberg | 3ebbcb5 | 2017-01-31 03:58:40 -0800 | [diff] [blame] | 119 | // Implements Syncable. |
| 120 | int id() const override; |
Danil Chapovalov | b9b146c | 2018-06-15 12:28:07 +0200 | [diff] [blame] | 121 | absl::optional<Syncable::Info> GetInfo() const override; |
Åsa Persson | fcf79cc | 2019-10-22 15:23:44 +0200 | [diff] [blame] | 122 | bool GetPlayoutRtpTimestamp(uint32_t* rtp_timestamp, |
| 123 | int64_t* time_ms) const override; |
| 124 | void SetEstimatedPlayoutNtpTimestampMs(int64_t ntp_timestamp_ms, |
| 125 | int64_t time_ms) override; |
Ruslan Burakov | 493a650 | 2019-02-27 15:32:48 +0100 | [diff] [blame] | 126 | |
| 127 | // SetMinimumPlayoutDelay is only called by A/V sync. |
solenberg | 3ebbcb5 | 2017-01-31 03:58:40 -0800 | [diff] [blame] | 128 | void SetMinimumPlayoutDelay(int delay_ms) override; |
| 129 | |
Jonas Oreland | 49ac595 | 2018-09-26 16:04:32 +0200 | [diff] [blame] | 130 | std::vector<webrtc::RtpSource> GetSources() const override; |
| 131 | |
Markus Handell | 269ac81 | 2019-12-03 14:31:45 +0100 | [diff] [blame] | 132 | RecordingState SetAndGetRecordingState(RecordingState state, |
| 133 | bool generate_key_frame) override; |
| 134 | void GenerateKeyFrame() override; |
| 135 | |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 136 | private: |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 137 | int64_t GetWaitMs() const; |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 138 | void StartNextDecode() RTC_RUN_ON(decode_queue_); |
Markus Handell | 269ac81 | 2019-12-03 14:31:45 +0100 | [diff] [blame] | 139 | void HandleEncodedFrame(std::unique_ptr<video_coding::EncodedFrame> frame) |
| 140 | RTC_RUN_ON(decode_queue_); |
| 141 | void HandleFrameBufferTimeout() RTC_RUN_ON(decode_queue_); |
Ruslan Burakov | 493a650 | 2019-02-27 15:32:48 +0100 | [diff] [blame] | 142 | void UpdatePlayoutDelays() const |
| 143 | RTC_EXCLUSIVE_LOCKS_REQUIRED(playout_delay_lock_); |
Markus Handell | 269ac81 | 2019-12-03 14:31:45 +0100 | [diff] [blame] | 144 | void RequestKeyFrame(int64_t timestamp_ms) RTC_RUN_ON(decode_queue_); |
| 145 | void HandleKeyFrameGeneration(bool received_frame_is_keyframe, int64_t now_ms) |
| 146 | RTC_RUN_ON(decode_queue_); |
| 147 | bool IsReceivingKeyFrame(int64_t timestamp_ms) const |
| 148 | RTC_RUN_ON(decode_queue_); |
Peter Boström | ca83525 | 2016-02-11 15:59:46 +0100 | [diff] [blame] | 149 | |
Niels Möller | 9a9f18a | 2019-08-02 13:52:37 +0200 | [diff] [blame] | 150 | void UpdateHistograms(); |
| 151 | |
Sebastian Jansson | b55015e | 2019-04-09 13:44:04 +0200 | [diff] [blame] | 152 | SequenceChecker worker_sequence_checker_; |
| 153 | SequenceChecker module_process_sequence_checker_; |
| 154 | SequenceChecker network_sequence_checker_; |
solenberg | 3ebbcb5 | 2017-01-31 03:58:40 -0800 | [diff] [blame] | 155 | |
Sebastian Jansson | 74682c1 | 2019-03-01 11:50:20 +0100 | [diff] [blame] | 156 | TaskQueueFactory* const task_queue_factory_; |
| 157 | |
pbos@webrtc.org | e75a1bf | 2013-09-18 11:52:42 +0000 | [diff] [blame] | 158 | TransportAdapter transport_adapter_; |
pbos@webrtc.org | 6ae48c6 | 2014-06-06 10:49:19 +0000 | [diff] [blame] | 159 | const VideoReceiveStream::Config config_; |
sprang | 113bdca | 2016-10-11 03:10:10 -0700 | [diff] [blame] | 160 | const int num_cpu_cores_; |
Peter Boström | 1d04ac6 | 2016-02-05 11:25:46 +0100 | [diff] [blame] | 161 | ProcessThread* const process_thread_; |
pbos@webrtc.org | de1429e | 2014-04-28 13:00:21 +0000 | [diff] [blame] | 162 | Clock* const clock_; |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 163 | |
mflodman | e378702 | 2015-10-21 13:24:28 +0200 | [diff] [blame] | 164 | CallStats* const call_stats_; |
Peter Boström | 45553ae | 2015-05-08 13:54:38 +0200 | [diff] [blame] | 165 | |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 166 | bool decoder_running_ RTC_GUARDED_BY(worker_sequence_checker_) = false; |
| 167 | bool decoder_stopped_ RTC_GUARDED_BY(decode_queue_) = true; |
| 168 | |
Chen Xing | 90f3b89 | 2019-06-25 10:16:14 +0200 | [diff] [blame] | 169 | SourceTracker source_tracker_; |
Danil Chapovalov | 8ce0d2b | 2018-11-23 11:03:25 +0100 | [diff] [blame] | 170 | ReceiveStatisticsProxy stats_proxy_; |
nisse | ca5706d | 2017-09-11 02:32:16 -0700 | [diff] [blame] | 171 | // Shared by media and rtx stream receivers, since the latter has no RtpRtcp |
| 172 | // module of its own. |
| 173 | const std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_; |
| 174 | |
philipel | 721d402 | 2016-12-15 07:10:57 -0800 | [diff] [blame] | 175 | std::unique_ptr<VCMTiming> timing_; // Jitter buffer experiment. |
Niels Möller | ee3d995 | 2019-09-09 12:51:55 +0200 | [diff] [blame] | 176 | VideoReceiver2 video_receiver_; |
tommi | 2e82f38 | 2016-06-21 00:26:43 -0700 | [diff] [blame] | 177 | std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> incoming_video_stream_; |
nisse | b1f2ff9 | 2017-06-09 04:01:55 -0700 | [diff] [blame] | 178 | RtpVideoStreamReceiver rtp_video_stream_receiver_; |
tommi | 2e82f38 | 2016-06-21 00:26:43 -0700 | [diff] [blame] | 179 | std::unique_ptr<VideoStreamDecoder> video_stream_decoder_; |
mflodman | 4cd2790 | 2016-08-05 06:28:45 -0700 | [diff] [blame] | 180 | RtpStreamsSynchronizer rtp_stream_sync_; |
sprang | 3911c26 | 2016-04-15 01:24:14 -0700 | [diff] [blame] | 181 | |
Niels Möller | cbcbc22 | 2018-09-28 09:07:24 +0200 | [diff] [blame] | 182 | // TODO(nisse, philipel): Creation and ownership of video encoders should be |
| 183 | // moved to the new VideoStreamDecoder. |
| 184 | std::vector<std::unique_ptr<VideoDecoder>> video_decoders_; |
| 185 | |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 186 | // Members for the new jitter buffer experiment. |
Henrik Boström | c680c4a | 2019-04-03 10:27:36 +0000 | [diff] [blame] | 187 | std::unique_ptr<video_coding::FrameBuffer> frame_buffer_; |
nisse | 0f15f92 | 2017-06-21 01:05:22 -0700 | [diff] [blame] | 188 | |
| 189 | std::unique_ptr<RtpStreamReceiverInterface> media_receiver_; |
nisse | ca5706d | 2017-09-11 02:32:16 -0700 | [diff] [blame] | 190 | std::unique_ptr<RtxReceiveStream> rtx_receive_stream_; |
nisse | 0f15f92 | 2017-06-21 01:05:22 -0700 | [diff] [blame] | 191 | std::unique_ptr<RtpStreamReceiverInterface> rtx_receiver_; |
philipel | 3042c2d | 2017-08-18 04:55:02 -0700 | [diff] [blame] | 192 | |
| 193 | // Whenever we are in an undecodable state (stream has just started or due to |
| 194 | // a decoding error) we require a keyframe to restart the stream. |
| 195 | bool keyframe_required_ = true; |
| 196 | |
| 197 | // If we have successfully decoded any frame. |
| 198 | bool frame_decoded_ = false; |
philipel | 48462b6 | 2017-09-26 02:54:58 -0700 | [diff] [blame] | 199 | |
| 200 | int64_t last_keyframe_request_ms_ = 0; |
Ilya Nikolaevskiy | e6a2d94 | 2018-11-07 14:32:28 +0100 | [diff] [blame] | 201 | int64_t last_complete_frame_time_ms_ = 0; |
Ruslan Burakov | 493a650 | 2019-02-27 15:32:48 +0100 | [diff] [blame] | 202 | |
Rasmus Brandt | 3dde450 | 2019-03-21 11:46:17 +0100 | [diff] [blame] | 203 | // Keyframe request intervals are configurable through field trials. |
| 204 | const int max_wait_for_keyframe_ms_; |
| 205 | const int max_wait_for_frame_ms_; |
| 206 | |
Ruslan Burakov | 493a650 | 2019-02-27 15:32:48 +0100 | [diff] [blame] | 207 | rtc::CriticalSection playout_delay_lock_; |
| 208 | |
| 209 | // All of them tries to change current min_playout_delay on |timing_| but |
| 210 | // source of the change request is different in each case. Among them the |
| 211 | // biggest delay is used. -1 means use default value from the |timing_|. |
| 212 | // |
| 213 | // Minimum delay as decided by the RTP playout delay extension. |
| 214 | int frame_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; |
| 215 | // Minimum delay as decided by the setLatency function in "webrtc/api". |
| 216 | int base_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; |
| 217 | // Minimum delay as decided by the A/V synchronization feature. |
| 218 | int syncable_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = |
| 219 | -1; |
| 220 | |
| 221 | // Maximum delay as decided by the RTP playout delay extension. |
| 222 | int frame_maximum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 223 | |
Markus Handell | 269ac81 | 2019-12-03 14:31:45 +0100 | [diff] [blame] | 224 | // Function that is triggered with encoded frames, if not empty. |
| 225 | std::function<void(const RecordableEncodedFrame&)> |
| 226 | encoded_frame_buffer_function_ RTC_GUARDED_BY(decode_queue_); |
| 227 | // Set to true while we're requesting keyframes but not yet received one. |
| 228 | bool keyframe_generation_requested_ RTC_GUARDED_BY(decode_queue_) = false; |
| 229 | |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 230 | // Defined last so they are destroyed before all other members. |
| 231 | rtc::TaskQueue decode_queue_; |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 232 | }; |
mflodman@webrtc.org | f3973e8 | 2013-12-13 09:40:45 +0000 | [diff] [blame] | 233 | } // namespace internal |
| 234 | } // namespace webrtc |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 235 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 236 | #endif // VIDEO_VIDEO_RECEIVE_STREAM_H_ |