niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 1 | /* |
leozwang@webrtc.org | 39e9659 | 2012-03-01 18:22:48 +0000 | [diff] [blame] | 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ |
| 12 | #define VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 13 | |
pwestin@webrtc.org | 1da1ce0 | 2011-10-13 15:19:55 +0000 | [diff] [blame] | 14 | #include <list> |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 15 | #include <map> |
kwiberg | 27f982b | 2016-03-01 11:52:33 -0800 | [diff] [blame] | 16 | #include <memory> |
Peter Boström | 9c01725 | 2016-02-26 16:26:20 +0100 | [diff] [blame] | 17 | #include <string> |
kjellander@webrtc.org | 0fcaf99 | 2015-11-26 15:24:52 +0100 | [diff] [blame] | 18 | #include <vector> |
pwestin@webrtc.org | 1da1ce0 | 2011-10-13 15:19:55 +0000 | [diff] [blame] | 19 | |
Niels Möller | 2ff1f2a | 2018-08-09 16:16:34 +0200 | [diff] [blame] | 20 | #include "absl/types/optional.h" |
| 21 | |
| 22 | #include "api/video_codecs/video_codec.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 23 | #include "call/rtp_packet_sink_interface.h" |
Niels Möller | df9e9ae | 2018-07-31 08:29:53 +0200 | [diff] [blame] | 24 | #include "call/syncable.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 25 | #include "call/video_receive_stream.h" |
| 26 | #include "modules/include/module_common_types.h" |
| 27 | #include "modules/rtp_rtcp/include/receive_statistics.h" |
| 28 | #include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" |
Niels Möller | b0573bc | 2017-09-25 10:47:00 +0200 | [diff] [blame] | 29 | #include "modules/rtp_rtcp/include/rtp_header_extension_map.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 30 | #include "modules/rtp_rtcp/include/rtp_rtcp.h" |
| 31 | #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" |
| 32 | #include "modules/video_coding/h264_sps_pps_tracker.h" |
| 33 | #include "modules/video_coding/include/video_coding_defines.h" |
| 34 | #include "modules/video_coding/packet_buffer.h" |
| 35 | #include "modules/video_coding/rtp_frame_reference_finder.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 36 | #include "rtc_base/constructormagic.h" |
| 37 | #include "rtc_base/criticalsection.h" |
Bjorn Terelius | a194e58 | 2017-10-25 13:07:09 +0200 | [diff] [blame] | 38 | #include "rtc_base/numerics/sequence_number_util.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 39 | #include "rtc_base/sequenced_task_checker.h" |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 40 | |
mflodman@webrtc.org | ad4ee36 | 2011-11-28 22:39:24 +0000 | [diff] [blame] | 41 | namespace webrtc { |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 42 | |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 43 | class NackModule; |
mflodman | c0e58a3 | 2016-04-25 01:26:26 -0700 | [diff] [blame] | 44 | class PacketRouter; |
mflodman | dc7d0d2 | 2016-05-06 05:32:22 -0700 | [diff] [blame] | 45 | class ProcessThread; |
wu@webrtc.org | 822fbd8 | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 46 | class ReceiveStatistics; |
mflodman | cfc8e3b | 2016-05-03 21:22:04 -0700 | [diff] [blame] | 47 | class ReceiveStatisticsProxy; |
mflodman | c0e58a3 | 2016-04-25 01:26:26 -0700 | [diff] [blame] | 48 | class RtcpRttStats; |
nisse | 38cc1d6 | 2017-02-13 05:59:46 -0800 | [diff] [blame] | 49 | class RtpPacketReceived; |
mflodman | c0e58a3 | 2016-04-25 01:26:26 -0700 | [diff] [blame] | 50 | class Transport; |
brandtr | d55c3f6 | 2016-10-31 04:51:33 -0700 | [diff] [blame] | 51 | class UlpfecReceiver; |
Peter Boström | 0b25072 | 2016-04-22 18:23:15 +0200 | [diff] [blame] | 52 | |
nisse | b1f2ff9 | 2017-06-09 04:01:55 -0700 | [diff] [blame] | 53 | class RtpVideoStreamReceiver : public RtpData, |
| 54 | public RecoveredPacketReceiver, |
nisse | 0f15f92 | 2017-06-21 01:05:22 -0700 | [diff] [blame] | 55 | public RtpPacketSinkInterface, |
nisse | b1f2ff9 | 2017-06-09 04:01:55 -0700 | [diff] [blame] | 56 | public VCMFrameTypeCallback, |
| 57 | public VCMPacketRequestCallback, |
| 58 | public video_coding::OnReceivedFrameCallback, |
Tommi | 81de14f | 2018-03-25 22:19:25 +0200 | [diff] [blame] | 59 | public video_coding::OnCompleteFrameCallback { |
mflodman@webrtc.org | ad4ee36 | 2011-11-28 22:39:24 +0000 | [diff] [blame] | 60 | public: |
nisse | b1f2ff9 | 2017-06-09 04:01:55 -0700 | [diff] [blame] | 61 | RtpVideoStreamReceiver( |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 62 | Transport* transport, |
| 63 | RtcpRttStats* rtt_stats, |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 64 | PacketRouter* packet_router, |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 65 | const VideoReceiveStream::Config* config, |
nisse | ca5706d | 2017-09-11 02:32:16 -0700 | [diff] [blame] | 66 | ReceiveStatistics* rtp_receive_statistics, |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 67 | ReceiveStatisticsProxy* receive_stats_proxy, |
| 68 | ProcessThread* process_thread, |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 69 | NackSender* nack_sender, |
| 70 | KeyFrameRequestSender* keyframe_request_sender, |
philipel | 0a9f6de | 2018-02-28 11:29:47 +0100 | [diff] [blame] | 71 | video_coding::OnCompleteFrameCallback* complete_frame_callback); |
Mirko Bonadei | 8fdcac3 | 2018-08-28 16:30:18 +0200 | [diff] [blame^] | 72 | ~RtpVideoStreamReceiver() override; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 73 | |
Niels Möller | 2ff1f2a | 2018-08-09 16:16:34 +0200 | [diff] [blame] | 74 | void AddReceiveCodec(const VideoCodec& video_codec, |
philipel | 022b54e | 2016-12-20 04:15:59 -0800 | [diff] [blame] | 75 | const std::map<std::string, std::string>& codec_params); |
wu@webrtc.org | 822fbd8 | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 76 | |
mflodman@webrtc.org | ad4ee36 | 2011-11-28 22:39:24 +0000 | [diff] [blame] | 77 | void StartReceive(); |
| 78 | void StopReceive(); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 79 | |
Niels Möller | df9e9ae | 2018-07-31 08:29:53 +0200 | [diff] [blame] | 80 | // Produces the transport-related timestamps; current_delay_ms is left unset. |
| 81 | absl::optional<Syncable::Info> GetSyncInfo() const; |
| 82 | |
Peter Boström | d1d66ba | 2016-02-08 14:07:14 +0100 | [diff] [blame] | 83 | bool DeliverRtcp(const uint8_t* rtcp_packet, size_t rtcp_packet_length); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 84 | |
philipel | d4fac69 | 2017-09-04 07:03:46 -0700 | [diff] [blame] | 85 | void FrameContinuous(int64_t seq_num); |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 86 | |
philipel | d4fac69 | 2017-09-04 07:03:46 -0700 | [diff] [blame] | 87 | void FrameDecoded(int64_t seq_num); |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 88 | |
mflodman | dc7d0d2 | 2016-05-06 05:32:22 -0700 | [diff] [blame] | 89 | void SignalNetworkState(NetworkState state); |
| 90 | |
Ilya Nikolaevskiy | d397a0d | 2018-02-21 15:57:09 +0100 | [diff] [blame] | 91 | // Returns number of different frames seen in the packet buffer. |
| 92 | int GetUniqueFramesSeen() const; |
| 93 | |
nisse | 0f15f92 | 2017-06-21 01:05:22 -0700 | [diff] [blame] | 94 | // Implements RtpPacketSinkInterface. |
| 95 | void OnRtpPacket(const RtpPacketReceived& packet) override; |
nisse | 38cc1d6 | 2017-02-13 05:59:46 -0800 | [diff] [blame] | 96 | |
mflodman@webrtc.org | ad4ee36 | 2011-11-28 22:39:24 +0000 | [diff] [blame] | 97 | // Implements RtpData. |
kjellander@webrtc.org | 14665ff | 2015-03-04 12:58:35 +0000 | [diff] [blame] | 98 | int32_t OnReceivedPayloadData(const uint8_t* payload_data, |
Peter Boström | 0208322 | 2016-06-14 12:52:54 +0200 | [diff] [blame] | 99 | size_t payload_size, |
kjellander@webrtc.org | 14665ff | 2015-03-04 12:58:35 +0000 | [diff] [blame] | 100 | const WebRtcRTPHeader* rtp_header) override; |
nisse | 30e8931 | 2017-05-29 08:16:37 -0700 | [diff] [blame] | 101 | // Implements RecoveredPacketReceiver. |
| 102 | void OnRecoveredPacket(const uint8_t* packet, size_t packet_length) override; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 103 | |
mflodman | cfc8e3b | 2016-05-03 21:22:04 -0700 | [diff] [blame] | 104 | // Implements VCMFrameTypeCallback. |
| 105 | int32_t RequestKeyFrame() override; |
mflodman | cfc8e3b | 2016-05-03 21:22:04 -0700 | [diff] [blame] | 106 | |
brandtr | f7c6d72 | 2016-12-08 08:25:47 -0800 | [diff] [blame] | 107 | bool IsUlpfecEnabled() const; |
mflodman | dc7d0d2 | 2016-05-06 05:32:22 -0700 | [diff] [blame] | 108 | bool IsRetransmissionsEnabled() const; |
| 109 | // Don't use, still experimental. |
| 110 | void RequestPacketRetransmit(const std::vector<uint16_t>& sequence_numbers); |
| 111 | |
mflodman | cfc8e3b | 2016-05-03 21:22:04 -0700 | [diff] [blame] | 112 | // Implements VCMPacketRequestCallback. |
| 113 | int32_t ResendPackets(const uint16_t* sequenceNumbers, |
| 114 | uint16_t length) override; |
| 115 | |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 116 | // Implements OnReceivedFrameCallback. |
| 117 | void OnReceivedFrame( |
| 118 | std::unique_ptr<video_coding::RtpFrameObject> frame) override; |
| 119 | |
| 120 | // Implements OnCompleteFrameCallback. |
| 121 | void OnCompleteFrame( |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 122 | std::unique_ptr<video_coding::EncodedFrame> frame) override; |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 123 | |
Tommi | 81de14f | 2018-03-25 22:19:25 +0200 | [diff] [blame] | 124 | // Called by VideoReceiveStream when stats are updated. |
| 125 | void UpdateRtt(int64_t max_rtt_ms); |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 126 | |
Danil Chapovalov | b9b146c | 2018-06-15 12:28:07 +0200 | [diff] [blame] | 127 | absl::optional<int64_t> LastReceivedPacketMs() const; |
| 128 | absl::optional<int64_t> LastReceivedKeyframePacketMs() const; |
philipel | 3184f8e | 2017-05-18 08:08:53 -0700 | [diff] [blame] | 129 | |
eladalon | c0d481a | 2017-08-02 07:39:07 -0700 | [diff] [blame] | 130 | // RtpDemuxer only forwards a given RTP packet to one sink. However, some |
| 131 | // sinks, such as FlexFEC, might wish to be informed of all of the packets |
| 132 | // a given sink receives (or any set of sinks). They may do so by registering |
| 133 | // themselves as secondary sinks. |
| 134 | void AddSecondarySink(RtpPacketSinkInterface* sink); |
| 135 | void RemoveSecondarySink(const RtpPacketSinkInterface* sink); |
| 136 | |
mflodman@webrtc.org | ad4ee36 | 2011-11-28 22:39:24 +0000 | [diff] [blame] | 137 | private: |
Niels Möller | 2ff1f2a | 2018-08-09 16:16:34 +0200 | [diff] [blame] | 138 | // Entry point doing non-stats work for a received packet. Called |
| 139 | // for the same packet both before and after RED decapsulation. |
| 140 | void ReceivePacket(const RtpPacketReceived& packet); |
| 141 | // Parses and handles RED headers. |
stefan@webrtc.org | 7bb8f02 | 2013-09-06 13:40:11 +0000 | [diff] [blame] | 142 | // This function assumes that it's being called from only one thread. |
nisse | 30e8931 | 2017-05-29 08:16:37 -0700 | [diff] [blame] | 143 | void ParseAndHandleEncapsulatingHeader(const uint8_t* packet, |
pkasting@chromium.org | 4591fbd | 2014-11-20 22:28:14 +0000 | [diff] [blame] | 144 | size_t packet_length, |
stefan@webrtc.org | 7bb8f02 | 2013-09-06 13:40:11 +0000 | [diff] [blame] | 145 | const RTPHeader& header); |
Niels Möller | bc01047 | 2018-03-23 13:22:29 +0100 | [diff] [blame] | 146 | void NotifyReceiverOfEmptyPacket(uint16_t seq_num); |
Danil Chapovalov | 64b17c2 | 2018-06-25 16:58:54 +0200 | [diff] [blame] | 147 | bool IsPacketRetransmitted(const RTPHeader& header) const; |
asapersson@webrtc.org | 0800db7 | 2015-01-15 07:40:20 +0000 | [diff] [blame] | 148 | void UpdateHistograms(); |
brandtr | e6f98c7 | 2016-11-11 03:28:30 -0800 | [diff] [blame] | 149 | bool IsRedEnabled() const; |
philipel | 022b54e | 2016-12-20 04:15:59 -0800 | [diff] [blame] | 150 | void InsertSpsPpsIntoTracker(uint8_t payload_type); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 151 | |
Peter Boström | 4fa7eca | 2016-03-02 15:05:53 +0100 | [diff] [blame] | 152 | Clock* const clock_; |
Tommi | 733b547 | 2016-06-10 17:58:01 +0200 | [diff] [blame] | 153 | // Ownership of this object lies with VideoReceiveStream, which owns |this|. |
| 154 | const VideoReceiveStream::Config& config_; |
mflodman | c0e58a3 | 2016-04-25 01:26:26 -0700 | [diff] [blame] | 155 | PacketRouter* const packet_router_; |
mflodman | dc7d0d2 | 2016-05-06 05:32:22 -0700 | [diff] [blame] | 156 | ProcessThread* const process_thread_; |
Peter Boström | 4fa7eca | 2016-03-02 15:05:53 +0100 | [diff] [blame] | 157 | |
| 158 | RemoteNtpTimeEstimator ntp_estimator_; |
Peter Boström | 4fa7eca | 2016-03-02 15:05:53 +0100 | [diff] [blame] | 159 | |
Niels Möller | b0573bc | 2017-09-25 10:47:00 +0200 | [diff] [blame] | 160 | RtpHeaderExtensionMap rtp_header_extensions_; |
nisse | ca5706d | 2017-09-11 02:32:16 -0700 | [diff] [blame] | 161 | ReceiveStatistics* const rtp_receive_statistics_; |
brandtr | d55c3f6 | 2016-10-31 04:51:33 -0700 | [diff] [blame] | 162 | std::unique_ptr<UlpfecReceiver> ulpfec_receiver_; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 163 | |
eladalon | 8b07305 | 2017-08-25 00:49:08 -0700 | [diff] [blame] | 164 | rtc::SequencedTaskChecker worker_task_checker_; |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 165 | bool receiving_ RTC_GUARDED_BY(worker_task_checker_); |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 166 | int64_t last_packet_log_ms_ RTC_GUARDED_BY(worker_task_checker_); |
mflodman | c0e58a3 | 2016-04-25 01:26:26 -0700 | [diff] [blame] | 167 | |
| 168 | const std::unique_ptr<RtpRtcp> rtp_rtcp_; |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 169 | |
| 170 | // Members for the new jitter buffer experiment. |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 171 | video_coding::OnCompleteFrameCallback* complete_frame_callback_; |
| 172 | KeyFrameRequestSender* keyframe_request_sender_; |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 173 | std::unique_ptr<NackModule> nack_module_; |
| 174 | rtc::scoped_refptr<video_coding::PacketBuffer> packet_buffer_; |
| 175 | std::unique_ptr<video_coding::RtpFrameReferenceFinder> reference_finder_; |
| 176 | rtc::CriticalSection last_seq_num_cs_; |
philipel | d4fac69 | 2017-09-04 07:03:46 -0700 | [diff] [blame] | 177 | std::map<int64_t, uint16_t> last_seq_num_for_pic_id_ |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 178 | RTC_GUARDED_BY(last_seq_num_cs_); |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 179 | video_coding::H264SpsPpsTracker tracker_; |
Niels Möller | 2ff1f2a | 2018-08-09 16:16:34 +0200 | [diff] [blame] | 180 | |
| 181 | absl::optional<uint32_t> last_received_rtp_timestamp_; |
| 182 | absl::optional<int64_t> last_received_rtp_system_time_ms_; |
| 183 | std::map<uint8_t, VideoCodecType> pt_codec_type_; |
philipel | 022b54e | 2016-12-20 04:15:59 -0800 | [diff] [blame] | 184 | // TODO(johan): Remove pt_codec_params_ once |
| 185 | // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved. |
| 186 | // Maps a payload type to a map of out-of-band supplied codec parameters. |
| 187 | std::map<uint8_t, std::map<std::string, std::string>> pt_codec_params_; |
| 188 | int16_t last_payload_type_ = -1; |
philipel | 2c53b13 | 2017-05-16 08:06:30 -0700 | [diff] [blame] | 189 | |
| 190 | bool has_received_frame_; |
eladalon | c0d481a | 2017-08-02 07:39:07 -0700 | [diff] [blame] | 191 | |
eladalon | 8b07305 | 2017-08-25 00:49:08 -0700 | [diff] [blame] | 192 | std::vector<RtpPacketSinkInterface*> secondary_sinks_ |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 193 | RTC_GUARDED_BY(worker_task_checker_); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 194 | }; |
mflodman@webrtc.org | ad4ee36 | 2011-11-28 22:39:24 +0000 | [diff] [blame] | 195 | |
kjellander@webrtc.org | 0fcaf99 | 2015-11-26 15:24:52 +0100 | [diff] [blame] | 196 | } // namespace webrtc |
mflodman@webrtc.org | ad4ee36 | 2011-11-28 22:39:24 +0000 | [diff] [blame] | 197 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 198 | #endif // VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ |