niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 1 | /* |
leozwang@webrtc.org | 39e9659 | 2012-03-01 18:22:48 +0000 | [diff] [blame] | 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ |
| 12 | #define VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 13 | |
pwestin@webrtc.org | 1da1ce0 | 2011-10-13 15:19:55 +0000 | [diff] [blame] | 14 | #include <list> |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 15 | #include <map> |
kwiberg | 27f982b | 2016-03-01 11:52:33 -0800 | [diff] [blame] | 16 | #include <memory> |
Peter Boström | 9c01725 | 2016-02-26 16:26:20 +0100 | [diff] [blame] | 17 | #include <string> |
kjellander@webrtc.org | 0fcaf99 | 2015-11-26 15:24:52 +0100 | [diff] [blame] | 18 | #include <vector> |
pwestin@webrtc.org | 1da1ce0 | 2011-10-13 15:19:55 +0000 | [diff] [blame] | 19 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 20 | #include "call/rtp_packet_sink_interface.h" |
| 21 | #include "call/video_receive_stream.h" |
| 22 | #include "modules/include/module_common_types.h" |
| 23 | #include "modules/rtp_rtcp/include/receive_statistics.h" |
| 24 | #include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" |
Niels Möller | b0573bc | 2017-09-25 10:47:00 +0200 | [diff] [blame] | 25 | #include "modules/rtp_rtcp/include/rtp_header_extension_map.h" |
Bjorn Terelius | a194e58 | 2017-10-25 13:07:09 +0200 | [diff] [blame] | 26 | #include "modules/rtp_rtcp/include/rtp_payload_registry.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 27 | #include "modules/rtp_rtcp/include/rtp_rtcp.h" |
| 28 | #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" |
| 29 | #include "modules/video_coding/h264_sps_pps_tracker.h" |
| 30 | #include "modules/video_coding/include/video_coding_defines.h" |
| 31 | #include "modules/video_coding/packet_buffer.h" |
| 32 | #include "modules/video_coding/rtp_frame_reference_finder.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 33 | #include "rtc_base/constructormagic.h" |
| 34 | #include "rtc_base/criticalsection.h" |
Bjorn Terelius | a194e58 | 2017-10-25 13:07:09 +0200 | [diff] [blame] | 35 | #include "rtc_base/numerics/sequence_number_util.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 36 | #include "rtc_base/sequenced_task_checker.h" |
Mirko Bonadei | 7120742 | 2017-09-15 13:58:09 +0200 | [diff] [blame] | 37 | #include "typedefs.h" // NOLINT(build/include) |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 38 | |
mflodman@webrtc.org | ad4ee36 | 2011-11-28 22:39:24 +0000 | [diff] [blame] | 39 | namespace webrtc { |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 40 | |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 41 | class NackModule; |
mflodman | c0e58a3 | 2016-04-25 01:26:26 -0700 | [diff] [blame] | 42 | class PacketRouter; |
mflodman | dc7d0d2 | 2016-05-06 05:32:22 -0700 | [diff] [blame] | 43 | class ProcessThread; |
wu@webrtc.org | 822fbd8 | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 44 | class ReceiveStatistics; |
mflodman | cfc8e3b | 2016-05-03 21:22:04 -0700 | [diff] [blame] | 45 | class ReceiveStatisticsProxy; |
mflodman | c0e58a3 | 2016-04-25 01:26:26 -0700 | [diff] [blame] | 46 | class RtcpRttStats; |
nisse | 38cc1d6 | 2017-02-13 05:59:46 -0800 | [diff] [blame] | 47 | class RtpPacketReceived; |
wu@webrtc.org | 822fbd8 | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 48 | class RtpReceiver; |
mflodman | c0e58a3 | 2016-04-25 01:26:26 -0700 | [diff] [blame] | 49 | class Transport; |
brandtr | d55c3f6 | 2016-10-31 04:51:33 -0700 | [diff] [blame] | 50 | class UlpfecReceiver; |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 51 | class VCMTiming; |
Peter Boström | 0b25072 | 2016-04-22 18:23:15 +0200 | [diff] [blame] | 52 | |
| 53 | namespace vcm { |
| 54 | class VideoReceiver; |
| 55 | } // namespace vcm |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 56 | |
nisse | b1f2ff9 | 2017-06-09 04:01:55 -0700 | [diff] [blame] | 57 | class RtpVideoStreamReceiver : public RtpData, |
| 58 | public RecoveredPacketReceiver, |
| 59 | public RtpFeedback, |
nisse | 0f15f92 | 2017-06-21 01:05:22 -0700 | [diff] [blame] | 60 | public RtpPacketSinkInterface, |
nisse | b1f2ff9 | 2017-06-09 04:01:55 -0700 | [diff] [blame] | 61 | public VCMFrameTypeCallback, |
| 62 | public VCMPacketRequestCallback, |
| 63 | public video_coding::OnReceivedFrameCallback, |
| 64 | public video_coding::OnCompleteFrameCallback, |
| 65 | public CallStatsObserver { |
mflodman@webrtc.org | ad4ee36 | 2011-11-28 22:39:24 +0000 | [diff] [blame] | 66 | public: |
nisse | b1f2ff9 | 2017-06-09 04:01:55 -0700 | [diff] [blame] | 67 | RtpVideoStreamReceiver( |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 68 | Transport* transport, |
| 69 | RtcpRttStats* rtt_stats, |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 70 | PacketRouter* packet_router, |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 71 | const VideoReceiveStream::Config* config, |
nisse | ca5706d | 2017-09-11 02:32:16 -0700 | [diff] [blame] | 72 | ReceiveStatistics* rtp_receive_statistics, |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 73 | ReceiveStatisticsProxy* receive_stats_proxy, |
| 74 | ProcessThread* process_thread, |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 75 | NackSender* nack_sender, |
| 76 | KeyFrameRequestSender* keyframe_request_sender, |
| 77 | video_coding::OnCompleteFrameCallback* complete_frame_callback, |
| 78 | VCMTiming* timing); |
nisse | b1f2ff9 | 2017-06-09 04:01:55 -0700 | [diff] [blame] | 79 | ~RtpVideoStreamReceiver(); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 80 | |
philipel | 022b54e | 2016-12-20 04:15:59 -0800 | [diff] [blame] | 81 | bool AddReceiveCodec(const VideoCodec& video_codec, |
| 82 | const std::map<std::string, std::string>& codec_params); |
wu@webrtc.org | 822fbd8 | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 83 | uint32_t GetRemoteSsrc() const; |
| 84 | int GetCsrcs(uint32_t* csrcs) const; |
| 85 | |
wu@webrtc.org | 822fbd8 | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 86 | RtpReceiver* GetRtpReceiver() const; |
mflodman | c0e58a3 | 2016-04-25 01:26:26 -0700 | [diff] [blame] | 87 | RtpRtcp* rtp_rtcp() const { return rtp_rtcp_.get(); } |
wu@webrtc.org | 822fbd8 | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 88 | |
mflodman@webrtc.org | ad4ee36 | 2011-11-28 22:39:24 +0000 | [diff] [blame] | 89 | void StartReceive(); |
| 90 | void StopReceive(); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 91 | |
Peter Boström | d1d66ba | 2016-02-08 14:07:14 +0100 | [diff] [blame] | 92 | bool DeliverRtcp(const uint8_t* rtcp_packet, size_t rtcp_packet_length); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 93 | |
philipel | d4fac69 | 2017-09-04 07:03:46 -0700 | [diff] [blame] | 94 | void FrameContinuous(int64_t seq_num); |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 95 | |
philipel | d4fac69 | 2017-09-04 07:03:46 -0700 | [diff] [blame] | 96 | void FrameDecoded(int64_t seq_num); |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 97 | |
mflodman | dc7d0d2 | 2016-05-06 05:32:22 -0700 | [diff] [blame] | 98 | void SignalNetworkState(NetworkState state); |
| 99 | |
Ilya Nikolaevskiy | d397a0d | 2018-02-21 15:57:09 +0100 | [diff] [blame] | 100 | // Returns number of different frames seen in the packet buffer. |
| 101 | int GetUniqueFramesSeen() const; |
| 102 | |
nisse | 0f15f92 | 2017-06-21 01:05:22 -0700 | [diff] [blame] | 103 | // Implements RtpPacketSinkInterface. |
| 104 | void OnRtpPacket(const RtpPacketReceived& packet) override; |
nisse | 38cc1d6 | 2017-02-13 05:59:46 -0800 | [diff] [blame] | 105 | |
mflodman@webrtc.org | ad4ee36 | 2011-11-28 22:39:24 +0000 | [diff] [blame] | 106 | // Implements RtpData. |
kjellander@webrtc.org | 14665ff | 2015-03-04 12:58:35 +0000 | [diff] [blame] | 107 | int32_t OnReceivedPayloadData(const uint8_t* payload_data, |
Peter Boström | 0208322 | 2016-06-14 12:52:54 +0200 | [diff] [blame] | 108 | size_t payload_size, |
kjellander@webrtc.org | 14665ff | 2015-03-04 12:58:35 +0000 | [diff] [blame] | 109 | const WebRtcRTPHeader* rtp_header) override; |
nisse | 30e8931 | 2017-05-29 08:16:37 -0700 | [diff] [blame] | 110 | // Implements RecoveredPacketReceiver. |
| 111 | void OnRecoveredPacket(const uint8_t* packet, size_t packet_length) override; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 112 | |
mflodman | fa66659 | 2016-04-28 23:15:33 -0700 | [diff] [blame] | 113 | // Implements RtpFeedback. |
Karl Wiberg | c62f6c7 | 2017-10-04 12:38:53 +0200 | [diff] [blame] | 114 | int32_t OnInitializeDecoder(int payload_type, |
| 115 | const SdpAudioFormat& audio_format, |
Sergey Ulanov | ec4f068 | 2016-07-28 15:19:10 -0700 | [diff] [blame] | 116 | uint32_t rate) override; |
stefan | b4ab381 | 2017-06-09 06:12:11 -0700 | [diff] [blame] | 117 | void OnIncomingSSRCChanged(uint32_t ssrc) override {} |
Sergey Ulanov | ec4f068 | 2016-07-28 15:19:10 -0700 | [diff] [blame] | 118 | void OnIncomingCSRCChanged(uint32_t CSRC, bool added) override {} |
mflodman | fa66659 | 2016-04-28 23:15:33 -0700 | [diff] [blame] | 119 | |
mflodman | cfc8e3b | 2016-05-03 21:22:04 -0700 | [diff] [blame] | 120 | // Implements VCMFrameTypeCallback. |
| 121 | int32_t RequestKeyFrame() override; |
mflodman | cfc8e3b | 2016-05-03 21:22:04 -0700 | [diff] [blame] | 122 | |
brandtr | f7c6d72 | 2016-12-08 08:25:47 -0800 | [diff] [blame] | 123 | bool IsUlpfecEnabled() const; |
mflodman | dc7d0d2 | 2016-05-06 05:32:22 -0700 | [diff] [blame] | 124 | bool IsRetransmissionsEnabled() const; |
| 125 | // Don't use, still experimental. |
| 126 | void RequestPacketRetransmit(const std::vector<uint16_t>& sequence_numbers); |
| 127 | |
mflodman | cfc8e3b | 2016-05-03 21:22:04 -0700 | [diff] [blame] | 128 | // Implements VCMPacketRequestCallback. |
| 129 | int32_t ResendPackets(const uint16_t* sequenceNumbers, |
| 130 | uint16_t length) override; |
| 131 | |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 132 | // Implements OnReceivedFrameCallback. |
| 133 | void OnReceivedFrame( |
| 134 | std::unique_ptr<video_coding::RtpFrameObject> frame) override; |
| 135 | |
| 136 | // Implements OnCompleteFrameCallback. |
| 137 | void OnCompleteFrame( |
| 138 | std::unique_ptr<video_coding::FrameObject> frame) override; |
| 139 | |
| 140 | void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override; |
| 141 | |
philipel | 3184f8e | 2017-05-18 08:08:53 -0700 | [diff] [blame] | 142 | rtc::Optional<int64_t> LastReceivedPacketMs() const; |
| 143 | rtc::Optional<int64_t> LastReceivedKeyframePacketMs() const; |
| 144 | |
eladalon | c0d481a | 2017-08-02 07:39:07 -0700 | [diff] [blame] | 145 | // RtpDemuxer only forwards a given RTP packet to one sink. However, some |
| 146 | // sinks, such as FlexFEC, might wish to be informed of all of the packets |
| 147 | // a given sink receives (or any set of sinks). They may do so by registering |
| 148 | // themselves as secondary sinks. |
| 149 | void AddSecondarySink(RtpPacketSinkInterface* sink); |
| 150 | void RemoveSecondarySink(const RtpPacketSinkInterface* sink); |
| 151 | |
mflodman@webrtc.org | ad4ee36 | 2011-11-28 22:39:24 +0000 | [diff] [blame] | 152 | private: |
johan | bfb11b2 | 2017-01-25 07:37:27 -0800 | [diff] [blame] | 153 | bool AddReceiveCodec(const VideoCodec& video_codec); |
nisse | 30e8931 | 2017-05-29 08:16:37 -0700 | [diff] [blame] | 154 | void ReceivePacket(const uint8_t* packet, |
pkasting@chromium.org | 4591fbd | 2014-11-20 22:28:14 +0000 | [diff] [blame] | 155 | size_t packet_length, |
Niels Möller | 22ec952 | 2017-10-05 08:39:15 +0200 | [diff] [blame] | 156 | const RTPHeader& header); |
stefan@webrtc.org | 7bb8f02 | 2013-09-06 13:40:11 +0000 | [diff] [blame] | 157 | // Parses and handles for instance RTX and RED headers. |
| 158 | // This function assumes that it's being called from only one thread. |
nisse | 30e8931 | 2017-05-29 08:16:37 -0700 | [diff] [blame] | 159 | void ParseAndHandleEncapsulatingHeader(const uint8_t* packet, |
pkasting@chromium.org | 4591fbd | 2014-11-20 22:28:14 +0000 | [diff] [blame] | 160 | size_t packet_length, |
stefan@webrtc.org | 7bb8f02 | 2013-09-06 13:40:11 +0000 | [diff] [blame] | 161 | const RTPHeader& header); |
asapersson@webrtc.org | 37c0559 | 2015-01-28 13:58:27 +0000 | [diff] [blame] | 162 | void NotifyReceiverOfFecPacket(const RTPHeader& header); |
stefan@webrtc.org | 7bb8f02 | 2013-09-06 13:40:11 +0000 | [diff] [blame] | 163 | bool IsPacketInOrder(const RTPHeader& header) const; |
stefan@webrtc.org | 48df381 | 2013-11-08 15:18:52 +0000 | [diff] [blame] | 164 | bool IsPacketRetransmitted(const RTPHeader& header, bool in_order) const; |
asapersson@webrtc.org | 0800db7 | 2015-01-15 07:40:20 +0000 | [diff] [blame] | 165 | void UpdateHistograms(); |
brandtr | e6f98c7 | 2016-11-11 03:28:30 -0800 | [diff] [blame] | 166 | bool IsRedEnabled() const; |
philipel | 022b54e | 2016-12-20 04:15:59 -0800 | [diff] [blame] | 167 | void InsertSpsPpsIntoTracker(uint8_t payload_type); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 168 | |
Peter Boström | 4fa7eca | 2016-03-02 15:05:53 +0100 | [diff] [blame] | 169 | Clock* const clock_; |
Tommi | 733b547 | 2016-06-10 17:58:01 +0200 | [diff] [blame] | 170 | // Ownership of this object lies with VideoReceiveStream, which owns |this|. |
| 171 | const VideoReceiveStream::Config& config_; |
mflodman | c0e58a3 | 2016-04-25 01:26:26 -0700 | [diff] [blame] | 172 | PacketRouter* const packet_router_; |
mflodman | dc7d0d2 | 2016-05-06 05:32:22 -0700 | [diff] [blame] | 173 | ProcessThread* const process_thread_; |
Peter Boström | 4fa7eca | 2016-03-02 15:05:53 +0100 | [diff] [blame] | 174 | |
| 175 | RemoteNtpTimeEstimator ntp_estimator_; |
| 176 | RTPPayloadRegistry rtp_payload_registry_; |
| 177 | |
Niels Möller | b0573bc | 2017-09-25 10:47:00 +0200 | [diff] [blame] | 178 | RtpHeaderExtensionMap rtp_header_extensions_; |
Peter Boström | 4fa7eca | 2016-03-02 15:05:53 +0100 | [diff] [blame] | 179 | const std::unique_ptr<RtpReceiver> rtp_receiver_; |
nisse | ca5706d | 2017-09-11 02:32:16 -0700 | [diff] [blame] | 180 | ReceiveStatistics* const rtp_receive_statistics_; |
brandtr | d55c3f6 | 2016-10-31 04:51:33 -0700 | [diff] [blame] | 181 | std::unique_ptr<UlpfecReceiver> ulpfec_receiver_; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 182 | |
eladalon | 8b07305 | 2017-08-25 00:49:08 -0700 | [diff] [blame] | 183 | rtc::SequencedTaskChecker worker_task_checker_; |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 184 | bool receiving_ RTC_GUARDED_BY(worker_task_checker_); |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 185 | int64_t last_packet_log_ms_ RTC_GUARDED_BY(worker_task_checker_); |
mflodman | c0e58a3 | 2016-04-25 01:26:26 -0700 | [diff] [blame] | 186 | |
| 187 | const std::unique_ptr<RtpRtcp> rtp_rtcp_; |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 188 | |
| 189 | // Members for the new jitter buffer experiment. |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 190 | video_coding::OnCompleteFrameCallback* complete_frame_callback_; |
| 191 | KeyFrameRequestSender* keyframe_request_sender_; |
| 192 | VCMTiming* timing_; |
| 193 | std::unique_ptr<NackModule> nack_module_; |
| 194 | rtc::scoped_refptr<video_coding::PacketBuffer> packet_buffer_; |
| 195 | std::unique_ptr<video_coding::RtpFrameReferenceFinder> reference_finder_; |
| 196 | rtc::CriticalSection last_seq_num_cs_; |
philipel | d4fac69 | 2017-09-04 07:03:46 -0700 | [diff] [blame] | 197 | std::map<int64_t, uint16_t> last_seq_num_for_pic_id_ |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 198 | RTC_GUARDED_BY(last_seq_num_cs_); |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 199 | video_coding::H264SpsPpsTracker tracker_; |
philipel | 022b54e | 2016-12-20 04:15:59 -0800 | [diff] [blame] | 200 | // TODO(johan): Remove pt_codec_params_ once |
| 201 | // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved. |
| 202 | // Maps a payload type to a map of out-of-band supplied codec parameters. |
| 203 | std::map<uint8_t, std::map<std::string, std::string>> pt_codec_params_; |
| 204 | int16_t last_payload_type_ = -1; |
philipel | 2c53b13 | 2017-05-16 08:06:30 -0700 | [diff] [blame] | 205 | |
| 206 | bool has_received_frame_; |
eladalon | c0d481a | 2017-08-02 07:39:07 -0700 | [diff] [blame] | 207 | |
eladalon | 8b07305 | 2017-08-25 00:49:08 -0700 | [diff] [blame] | 208 | std::vector<RtpPacketSinkInterface*> secondary_sinks_ |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 209 | RTC_GUARDED_BY(worker_task_checker_); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 210 | }; |
mflodman@webrtc.org | ad4ee36 | 2011-11-28 22:39:24 +0000 | [diff] [blame] | 211 | |
kjellander@webrtc.org | 0fcaf99 | 2015-11-26 15:24:52 +0100 | [diff] [blame] | 212 | } // namespace webrtc |
mflodman@webrtc.org | ad4ee36 | 2011-11-28 22:39:24 +0000 | [diff] [blame] | 213 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 214 | #endif // VIDEO_RTP_VIDEO_STREAM_RECEIVER_H_ |