mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 11 | #include "call/rtp_video_sender.h" |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 +0000 | [diff] [blame] | 12 | |
philipel | 25d31ec | 2018-08-08 16:33:01 +0200 | [diff] [blame] | 13 | #include <algorithm> |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 14 | #include <memory> |
| 15 | #include <string> |
| 16 | #include <utility> |
| 17 | |
Erik Språng | 845c6aa | 2019-05-29 13:02:24 +0200 | [diff] [blame] | 18 | #include "absl/algorithm/container.h" |
Erik Språng | 6cf554e | 2019-10-02 20:55:39 +0200 | [diff] [blame] | 19 | #include "absl/strings/match.h" |
Erik Språng | 490d76c | 2019-05-07 09:29:15 -0700 | [diff] [blame] | 20 | #include "api/array_view.h" |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 21 | #include "api/transport/field_trial_based_config.h" |
Danil Chapovalov | b529b7a | 2019-11-27 13:59:41 +0100 | [diff] [blame] | 22 | #include "api/video_codecs/video_codec.h" |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 23 | #include "call/rtp_transport_controller_send_interface.h" |
| 24 | #include "modules/pacing/packet_router.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 25 | #include "modules/rtp_rtcp/include/rtp_rtcp.h" |
| 26 | #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 27 | #include "modules/rtp_rtcp/source/playout_delay_oracle.h" |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 28 | #include "modules/rtp_rtcp/source/rtp_sender.h" |
| 29 | #include "modules/utility/include/process_thread.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 30 | #include "modules/video_coding/include/video_codec_interface.h" |
| 31 | #include "rtc_base/checks.h" |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 32 | #include "rtc_base/location.h" |
| 33 | #include "rtc_base/logging.h" |
| 34 | #include "system_wrappers/include/field_trial.h" |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 +0000 | [diff] [blame] | 35 | |
| 36 | namespace webrtc { |
| 37 | |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 38 | namespace webrtc_internal_rtp_video_sender { |
| 39 | |
| 40 | RtpStreamSender::RtpStreamSender( |
| 41 | std::unique_ptr<PlayoutDelayOracle> playout_delay_oracle, |
| 42 | std::unique_ptr<RtpRtcp> rtp_rtcp, |
| 43 | std::unique_ptr<RTPSenderVideo> sender_video) |
| 44 | : playout_delay_oracle(std::move(playout_delay_oracle)), |
| 45 | rtp_rtcp(std::move(rtp_rtcp)), |
| 46 | sender_video(std::move(sender_video)) {} |
| 47 | |
| 48 | RtpStreamSender::~RtpStreamSender() = default; |
| 49 | |
| 50 | } // namespace webrtc_internal_rtp_video_sender |
| 51 | |
kjellander | 02b3d27 | 2016-04-20 05:05:54 -0700 | [diff] [blame] | 52 | namespace { |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 53 | static const int kMinSendSidePacketHistorySize = 600; |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 54 | // We don't do MTU discovery, so assume that we have the standard ethernet MTU. |
| 55 | static const size_t kPathMTU = 1500; |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 56 | |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 57 | using webrtc_internal_rtp_video_sender::RtpStreamSender; |
| 58 | |
Erik Språng | dc34a25 | 2019-10-04 15:17:29 +0200 | [diff] [blame] | 59 | bool PayloadTypeSupportsSkippingFecPackets(const std::string& payload_name) { |
| 60 | const VideoCodecType codecType = PayloadStringToCodecType(payload_name); |
| 61 | if (codecType == kVideoCodecVP8 || codecType == kVideoCodecVP9) { |
| 62 | return true; |
| 63 | } |
| 64 | if (codecType == kVideoCodecGeneric && |
| 65 | field_trial::IsEnabled("WebRTC-GenericPictureId")) { |
| 66 | return true; |
| 67 | } |
| 68 | return false; |
| 69 | } |
| 70 | |
| 71 | bool ShouldDisableRedAndUlpfec(bool flexfec_enabled, |
| 72 | const RtpConfig& rtp_config) { |
| 73 | // Consistency of NACK and RED+ULPFEC parameters is checked in this function. |
| 74 | const bool nack_enabled = rtp_config.nack.rtp_history_ms > 0; |
| 75 | |
| 76 | // Shorthands. |
| 77 | auto IsRedEnabled = [&]() { return rtp_config.ulpfec.red_payload_type >= 0; }; |
| 78 | auto IsUlpfecEnabled = [&]() { |
| 79 | return rtp_config.ulpfec.ulpfec_payload_type >= 0; |
| 80 | }; |
| 81 | |
| 82 | bool should_disable_red_and_ulpfec = false; |
| 83 | |
| 84 | if (webrtc::field_trial::IsEnabled("WebRTC-DisableUlpFecExperiment")) { |
| 85 | RTC_LOG(LS_INFO) << "Experiment to disable sending ULPFEC is enabled."; |
| 86 | should_disable_red_and_ulpfec = true; |
| 87 | } |
| 88 | |
| 89 | // If enabled, FlexFEC takes priority over RED+ULPFEC. |
| 90 | if (flexfec_enabled) { |
| 91 | if (IsUlpfecEnabled()) { |
| 92 | RTC_LOG(LS_INFO) |
| 93 | << "Both FlexFEC and ULPFEC are configured. Disabling ULPFEC."; |
| 94 | } |
| 95 | should_disable_red_and_ulpfec = true; |
| 96 | } |
| 97 | |
| 98 | // Payload types without picture ID cannot determine that a stream is complete |
| 99 | // without retransmitting FEC, so using ULPFEC + NACK for H.264 (for instance) |
| 100 | // is a waste of bandwidth since FEC packets still have to be transmitted. |
| 101 | // Note that this is not the case with FlexFEC. |
| 102 | if (nack_enabled && IsUlpfecEnabled() && |
| 103 | !PayloadTypeSupportsSkippingFecPackets(rtp_config.payload_name)) { |
| 104 | RTC_LOG(LS_WARNING) |
| 105 | << "Transmitting payload type without picture ID using " |
| 106 | "NACK+ULPFEC is a waste of bandwidth since ULPFEC packets " |
| 107 | "also have to be retransmitted. Disabling ULPFEC."; |
| 108 | should_disable_red_and_ulpfec = true; |
| 109 | } |
| 110 | |
| 111 | // Verify payload types. |
| 112 | if (IsUlpfecEnabled() ^ IsRedEnabled()) { |
| 113 | RTC_LOG(LS_WARNING) |
| 114 | << "Only RED or only ULPFEC enabled, but not both. Disabling both."; |
| 115 | should_disable_red_and_ulpfec = true; |
| 116 | } |
| 117 | |
| 118 | return should_disable_red_and_ulpfec; |
| 119 | } |
| 120 | |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 121 | std::vector<RtpStreamSender> CreateRtpStreamSenders( |
Sebastian Jansson | 572c60f | 2019-03-04 18:30:41 +0100 | [diff] [blame] | 122 | Clock* clock, |
Johannes Kron | 9190b82 | 2018-10-29 11:22:05 +0100 | [diff] [blame] | 123 | const RtpConfig& rtp_config, |
Erik Språng | 7ea9b80 | 2019-10-16 19:03:38 +0200 | [diff] [blame] | 124 | const RtpSenderObservers& observers, |
Jiawei Ou | 5571812 | 2018-11-09 13:17:39 -0800 | [diff] [blame] | 125 | int rtcp_report_interval_ms, |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 126 | Transport* send_transport, |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 127 | RtcpBandwidthObserver* bandwidth_callback, |
| 128 | RtpTransportControllerSendInterface* transport, |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 129 | FlexfecSender* flexfec_sender, |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 130 | RtcEventLog* event_log, |
| 131 | RateLimiter* retransmission_rate_limiter, |
| 132 | OverheadObserver* overhead_observer, |
Benjamin Wright | 192eeec | 2018-10-17 17:27:25 -0700 | [diff] [blame] | 133 | FrameEncryptorInterface* frame_encryptor, |
| 134 | const CryptoOptions& crypto_options) { |
Amit Hilbuch | 0fc2843 | 2018-12-18 13:01:47 -0800 | [diff] [blame] | 135 | RTC_DCHECK_GT(rtp_config.ssrcs.size(), 0); |
Benjamin Wright | 192eeec | 2018-10-17 17:27:25 -0700 | [diff] [blame] | 136 | |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 137 | RtpRtcp::Configuration configuration; |
Sebastian Jansson | 572c60f | 2019-03-04 18:30:41 +0100 | [diff] [blame] | 138 | configuration.clock = clock; |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 139 | configuration.audio = false; |
| 140 | configuration.receiver_only = false; |
| 141 | configuration.outgoing_transport = send_transport; |
Erik Språng | 7ea9b80 | 2019-10-16 19:03:38 +0200 | [diff] [blame] | 142 | configuration.intra_frame_callback = observers.intra_frame_callback; |
Elad Alon | 0a8562e | 2019-04-09 11:55:13 +0200 | [diff] [blame] | 143 | configuration.rtcp_loss_notification_observer = |
Erik Språng | 7ea9b80 | 2019-10-16 19:03:38 +0200 | [diff] [blame] | 144 | observers.rtcp_loss_notification_observer; |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 145 | configuration.bandwidth_callback = bandwidth_callback; |
Sebastian Jansson | e1795f4 | 2019-07-24 11:38:03 +0200 | [diff] [blame] | 146 | configuration.network_state_estimate_observer = |
| 147 | transport->network_state_estimate_observer(); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 148 | configuration.transport_feedback_callback = |
| 149 | transport->transport_feedback_observer(); |
Erik Språng | 7ea9b80 | 2019-10-16 19:03:38 +0200 | [diff] [blame] | 150 | configuration.rtt_stats = observers.rtcp_rtt_stats; |
| 151 | configuration.rtcp_packet_type_counter_observer = |
| 152 | observers.rtcp_type_observer; |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 153 | configuration.paced_sender = transport->packet_sender(); |
Erik Språng | 7ea9b80 | 2019-10-16 19:03:38 +0200 | [diff] [blame] | 154 | configuration.send_bitrate_observer = observers.bitrate_observer; |
| 155 | configuration.send_side_delay_observer = observers.send_delay_observer; |
| 156 | configuration.send_packet_observer = observers.send_packet_observer; |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 157 | configuration.event_log = event_log; |
| 158 | configuration.retransmission_rate_limiter = retransmission_rate_limiter; |
| 159 | configuration.overhead_observer = overhead_observer; |
Erik Språng | 7ea9b80 | 2019-10-16 19:03:38 +0200 | [diff] [blame] | 160 | configuration.rtp_stats_callback = observers.rtp_stats; |
Benjamin Wright | 192eeec | 2018-10-17 17:27:25 -0700 | [diff] [blame] | 161 | configuration.frame_encryptor = frame_encryptor; |
| 162 | configuration.require_frame_encryption = |
| 163 | crypto_options.sframe.require_frame_encryption; |
Johannes Kron | 9190b82 | 2018-10-29 11:22:05 +0100 | [diff] [blame] | 164 | configuration.extmap_allow_mixed = rtp_config.extmap_allow_mixed; |
Jiawei Ou | 8b5d9d8 | 2018-11-15 16:44:37 -0800 | [diff] [blame] | 165 | configuration.rtcp_report_interval_ms = rtcp_report_interval_ms; |
Benjamin Wright | 192eeec | 2018-10-17 17:27:25 -0700 | [diff] [blame] | 166 | |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 167 | std::vector<RtpStreamSender> rtp_streams; |
Johannes Kron | 9190b82 | 2018-10-29 11:22:05 +0100 | [diff] [blame] | 168 | const std::vector<uint32_t>& flexfec_protected_ssrcs = |
| 169 | rtp_config.flexfec.protected_media_ssrcs; |
Erik Språng | 93f5189 | 2019-08-19 12:42:58 +0200 | [diff] [blame] | 170 | RTC_DCHECK(rtp_config.rtx.ssrcs.empty() || |
| 171 | rtp_config.rtx.ssrcs.size() == rtp_config.rtx.ssrcs.size()); |
| 172 | for (size_t i = 0; i < rtp_config.ssrcs.size(); ++i) { |
Erik Språng | 54d5d2c | 2019-08-20 17:22:36 +0200 | [diff] [blame] | 173 | configuration.local_media_ssrc = rtp_config.ssrcs[i]; |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 174 | bool enable_flexfec = flexfec_sender != nullptr && |
| 175 | std::find(flexfec_protected_ssrcs.begin(), |
| 176 | flexfec_protected_ssrcs.end(), |
Erik Språng | 6841d25 | 2019-10-15 14:29:11 +0200 | [diff] [blame] | 177 | configuration.local_media_ssrc) != |
Erik Språng | 93f5189 | 2019-08-19 12:42:58 +0200 | [diff] [blame] | 178 | flexfec_protected_ssrcs.end(); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 179 | configuration.flexfec_sender = enable_flexfec ? flexfec_sender : nullptr; |
Mirko Bonadei | 317a1f0 | 2019-09-17 17:06:18 +0200 | [diff] [blame] | 180 | auto playout_delay_oracle = std::make_unique<PlayoutDelayOracle>(); |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 181 | |
| 182 | configuration.ack_observer = playout_delay_oracle.get(); |
Erik Språng | 93f5189 | 2019-08-19 12:42:58 +0200 | [diff] [blame] | 183 | if (rtp_config.rtx.ssrcs.size() > i) { |
| 184 | configuration.rtx_send_ssrc = rtp_config.rtx.ssrcs[i]; |
| 185 | } |
| 186 | |
Danil Chapovalov | c44f6cc | 2019-03-06 11:31:09 +0100 | [diff] [blame] | 187 | auto rtp_rtcp = RtpRtcp::Create(configuration); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 188 | rtp_rtcp->SetSendingStatus(false); |
| 189 | rtp_rtcp->SetSendingMediaStatus(false); |
| 190 | rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound); |
Erik Språng | dc34a25 | 2019-10-04 15:17:29 +0200 | [diff] [blame] | 191 | // Set NACK. |
| 192 | rtp_rtcp->SetStorePacketsStatus(true, kMinSendSidePacketHistorySize); |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 193 | |
Erik Språng | dc34a25 | 2019-10-04 15:17:29 +0200 | [diff] [blame] | 194 | FieldTrialBasedConfig field_trial_config; |
| 195 | RTPSenderVideo::Config video_config; |
| 196 | video_config.clock = configuration.clock; |
| 197 | video_config.rtp_sender = rtp_rtcp->RtpSender(); |
| 198 | video_config.flexfec_sender = configuration.flexfec_sender; |
| 199 | video_config.playout_delay_oracle = playout_delay_oracle.get(); |
| 200 | video_config.frame_encryptor = frame_encryptor; |
| 201 | video_config.require_frame_encryption = |
| 202 | crypto_options.sframe.require_frame_encryption; |
| 203 | video_config.need_rtp_packet_infos = rtp_config.lntf.enabled; |
| 204 | video_config.enable_retransmit_all_layers = false; |
| 205 | video_config.field_trials = &field_trial_config; |
| 206 | const bool should_disable_red_and_ulpfec = |
| 207 | ShouldDisableRedAndUlpfec(enable_flexfec, rtp_config); |
| 208 | if (rtp_config.ulpfec.red_payload_type != -1 && |
| 209 | !should_disable_red_and_ulpfec) { |
| 210 | video_config.red_payload_type = rtp_config.ulpfec.red_payload_type; |
| 211 | } |
| 212 | if (rtp_config.ulpfec.ulpfec_payload_type != -1 && |
| 213 | !should_disable_red_and_ulpfec) { |
| 214 | video_config.ulpfec_payload_type = rtp_config.ulpfec.ulpfec_payload_type; |
| 215 | } |
| 216 | auto sender_video = std::make_unique<RTPSenderVideo>(video_config); |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 217 | rtp_streams.emplace_back(std::move(playout_delay_oracle), |
| 218 | std::move(rtp_rtcp), std::move(sender_video)); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 219 | } |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 220 | return rtp_streams; |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 221 | } |
| 222 | |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 223 | // TODO(brandtr): Update this function when we support multistream protection. |
| 224 | std::unique_ptr<FlexfecSender> MaybeCreateFlexfecSender( |
Sebastian Jansson | 572c60f | 2019-03-04 18:30:41 +0100 | [diff] [blame] | 225 | Clock* clock, |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 226 | const RtpConfig& rtp, |
| 227 | const std::map<uint32_t, RtpState>& suspended_ssrcs) { |
| 228 | if (rtp.flexfec.payload_type < 0) { |
| 229 | return nullptr; |
| 230 | } |
| 231 | RTC_DCHECK_GE(rtp.flexfec.payload_type, 0); |
| 232 | RTC_DCHECK_LE(rtp.flexfec.payload_type, 127); |
| 233 | if (rtp.flexfec.ssrc == 0) { |
| 234 | RTC_LOG(LS_WARNING) << "FlexFEC is enabled, but no FlexFEC SSRC given. " |
| 235 | "Therefore disabling FlexFEC."; |
| 236 | return nullptr; |
| 237 | } |
| 238 | if (rtp.flexfec.protected_media_ssrcs.empty()) { |
| 239 | RTC_LOG(LS_WARNING) |
| 240 | << "FlexFEC is enabled, but no protected media SSRC given. " |
| 241 | "Therefore disabling FlexFEC."; |
| 242 | return nullptr; |
| 243 | } |
| 244 | |
| 245 | if (rtp.flexfec.protected_media_ssrcs.size() > 1) { |
| 246 | RTC_LOG(LS_WARNING) |
| 247 | << "The supplied FlexfecConfig contained multiple protected " |
| 248 | "media streams, but our implementation currently only " |
| 249 | "supports protecting a single media stream. " |
| 250 | "To avoid confusion, disabling FlexFEC completely."; |
| 251 | return nullptr; |
| 252 | } |
| 253 | |
| 254 | const RtpState* rtp_state = nullptr; |
| 255 | auto it = suspended_ssrcs.find(rtp.flexfec.ssrc); |
| 256 | if (it != suspended_ssrcs.end()) { |
| 257 | rtp_state = &it->second; |
| 258 | } |
| 259 | |
| 260 | RTC_DCHECK_EQ(1U, rtp.flexfec.protected_media_ssrcs.size()); |
Mirko Bonadei | 317a1f0 | 2019-09-17 17:06:18 +0200 | [diff] [blame] | 261 | return std::make_unique<FlexfecSender>( |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 262 | rtp.flexfec.payload_type, rtp.flexfec.ssrc, |
| 263 | rtp.flexfec.protected_media_ssrcs[0], rtp.mid, rtp.extensions, |
Sebastian Jansson | 572c60f | 2019-03-04 18:30:41 +0100 | [diff] [blame] | 264 | RTPSender::FecExtensionSizes(), rtp_state, clock); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 265 | } |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 266 | |
Sebastian Jansson | cf41eb1 | 2019-06-10 11:30:59 +0200 | [diff] [blame] | 267 | DataRate CalculateOverheadRate(DataRate data_rate, |
| 268 | DataSize packet_size, |
| 269 | DataSize overhead_per_packet) { |
| 270 | Frequency packet_rate = data_rate / packet_size; |
| 271 | // TOSO(srte): We should not need to round to nearest whole packet per second |
| 272 | // rate here. |
| 273 | return packet_rate.RoundUpTo(Frequency::hertz(1)) * overhead_per_packet; |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 274 | } |
Erik Språng | 6cf554e | 2019-10-02 20:55:39 +0200 | [diff] [blame] | 275 | |
| 276 | absl::optional<VideoCodecType> GetVideoCodecType(const RtpConfig& config) { |
Danil Chapovalov | b529b7a | 2019-11-27 13:59:41 +0100 | [diff] [blame] | 277 | if (config.raw_payload) { |
| 278 | return absl::nullopt; |
Erik Språng | 6cf554e | 2019-10-02 20:55:39 +0200 | [diff] [blame] | 279 | } |
Danil Chapovalov | b529b7a | 2019-11-27 13:59:41 +0100 | [diff] [blame] | 280 | return PayloadStringToCodecType(config.payload_name); |
Erik Språng | 6cf554e | 2019-10-02 20:55:39 +0200 | [diff] [blame] | 281 | } |
Sebastian Jansson | c3eb9fd | 2020-01-29 17:42:52 +0100 | [diff] [blame] | 282 | bool TransportSeqNumExtensionConfigured(const RtpConfig& config_config) { |
| 283 | return absl::c_any_of(config_config.extensions, [](const RtpExtension& ext) { |
| 284 | return ext.uri == RtpExtension::kTransportSequenceNumberUri; |
| 285 | }); |
| 286 | } |
kjellander | 02b3d27 | 2016-04-20 05:05:54 -0700 | [diff] [blame] | 287 | } // namespace |
| 288 | |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 289 | RtpVideoSender::RtpVideoSender( |
Sebastian Jansson | 572c60f | 2019-03-04 18:30:41 +0100 | [diff] [blame] | 290 | Clock* clock, |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 291 | std::map<uint32_t, RtpState> suspended_ssrcs, |
| 292 | const std::map<uint32_t, RtpPayloadState>& states, |
| 293 | const RtpConfig& rtp_config, |
Jiawei Ou | 5571812 | 2018-11-09 13:17:39 -0800 | [diff] [blame] | 294 | int rtcp_report_interval_ms, |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 295 | Transport* send_transport, |
| 296 | const RtpSenderObservers& observers, |
| 297 | RtpTransportControllerSendInterface* transport, |
| 298 | RtcEventLog* event_log, |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 299 | RateLimiter* retransmission_limiter, |
Benjamin Wright | 192eeec | 2018-10-17 17:27:25 -0700 | [diff] [blame] | 300 | std::unique_ptr<FecController> fec_controller, |
| 301 | FrameEncryptorInterface* frame_encryptor, |
| 302 | const CryptoOptions& crypto_options) |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 303 | : send_side_bwe_with_overhead_( |
| 304 | webrtc::field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")), |
Erik Språng | c12d41b | 2019-01-09 09:55:31 +0100 | [diff] [blame] | 305 | account_for_packetization_overhead_(!webrtc::field_trial::IsDisabled( |
| 306 | "WebRTC-SubtractPacketizationOverhead")), |
Erik Språng | 845c6aa | 2019-05-29 13:02:24 +0200 | [diff] [blame] | 307 | use_early_loss_detection_( |
| 308 | !webrtc::field_trial::IsDisabled("WebRTC-UseEarlyLossDetection")), |
Sebastian Jansson | c3eb9fd | 2020-01-29 17:42:52 +0100 | [diff] [blame] | 309 | has_packet_feedback_(TransportSeqNumExtensionConfigured(rtp_config)), |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 310 | active_(false), |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 311 | module_process_thread_(nullptr), |
| 312 | suspended_ssrcs_(std::move(suspended_ssrcs)), |
Sebastian Jansson | 572c60f | 2019-03-04 18:30:41 +0100 | [diff] [blame] | 313 | flexfec_sender_( |
| 314 | MaybeCreateFlexfecSender(clock, rtp_config, suspended_ssrcs_)), |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 315 | fec_controller_(std::move(fec_controller)), |
Elad Alon | 67daf71 | 2019-06-28 18:14:36 +0200 | [diff] [blame] | 316 | fec_allowed_(true), |
Erik Språng | 7ea9b80 | 2019-10-16 19:03:38 +0200 | [diff] [blame] | 317 | rtp_streams_(CreateRtpStreamSenders(clock, |
| 318 | rtp_config, |
| 319 | observers, |
| 320 | rtcp_report_interval_ms, |
| 321 | send_transport, |
| 322 | transport->GetBandwidthObserver(), |
| 323 | transport, |
| 324 | flexfec_sender_.get(), |
| 325 | event_log, |
| 326 | retransmission_limiter, |
| 327 | this, |
| 328 | frame_encryptor, |
| 329 | crypto_options)), |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 330 | rtp_config_(rtp_config), |
Erik Språng | 6cf554e | 2019-10-02 20:55:39 +0200 | [diff] [blame] | 331 | codec_type_(GetVideoCodecType(rtp_config)), |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 332 | transport_(transport), |
| 333 | transport_overhead_bytes_per_packet_(0), |
| 334 | overhead_bytes_per_packet_(0), |
Niels Möller | 949f0fd | 2019-01-29 09:44:24 +0100 | [diff] [blame] | 335 | encoder_target_rate_bps_(0), |
| 336 | frame_counts_(rtp_config.ssrcs.size()), |
Oleh Prypin | e896490 | 2019-03-29 15:33:01 +0000 | [diff] [blame] | 337 | frame_count_observer_(observers.frame_count_observer) { |
Elad Alon | 62ce035 | 2019-05-23 16:58:53 +0200 | [diff] [blame] | 338 | RTC_DCHECK_EQ(rtp_config_.ssrcs.size(), rtp_streams_.size()); |
Sebastian Jansson | c3eb9fd | 2020-01-29 17:42:52 +0100 | [diff] [blame] | 339 | if (send_side_bwe_with_overhead_ && has_packet_feedback_) |
| 340 | transport_->IncludeOverheadInPacedSender(); |
Sebastian Jansson | c01367d | 2019-04-08 15:20:44 +0200 | [diff] [blame] | 341 | module_process_thread_checker_.Detach(); |
Åsa Persson | 4bece9a | 2017-10-06 10:04:04 +0200 | [diff] [blame] | 342 | // SSRCs are assumed to be sorted in the same order as |rtp_modules|. |
Elad Alon | 62ce035 | 2019-05-23 16:58:53 +0200 | [diff] [blame] | 343 | for (uint32_t ssrc : rtp_config_.ssrcs) { |
Åsa Persson | 4bece9a | 2017-10-06 10:04:04 +0200 | [diff] [blame] | 344 | // Restore state if it previously existed. |
| 345 | const RtpPayloadState* state = nullptr; |
| 346 | auto it = states.find(ssrc); |
| 347 | if (it != states.end()) { |
| 348 | state = &it->second; |
philipel | 25d31ec | 2018-08-08 16:33:01 +0200 | [diff] [blame] | 349 | shared_frame_id_ = std::max(shared_frame_id_, state->shared_frame_id); |
Åsa Persson | 4bece9a | 2017-10-06 10:04:04 +0200 | [diff] [blame] | 350 | } |
| 351 | params_.push_back(RtpPayloadParams(ssrc, state)); |
| 352 | } |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 353 | |
| 354 | // RTP/RTCP initialization. |
| 355 | |
| 356 | // We add the highest spatial layer first to ensure it'll be prioritized |
| 357 | // when sending padding, with the hope that the packet rate will be smaller, |
| 358 | // and that it's more important to protect than the lower layers. |
Niels Möller | 2ff1f2a | 2018-08-09 16:16:34 +0200 | [diff] [blame] | 359 | |
| 360 | // TODO(nisse): Consider moving registration with PacketRouter last, after the |
| 361 | // modules are fully configured. |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 362 | for (const RtpStreamSender& stream : rtp_streams_) { |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 363 | constexpr bool remb_candidate = true; |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 364 | transport->packet_router()->AddSendRtpModule(stream.rtp_rtcp.get(), |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 365 | remb_candidate); |
| 366 | } |
| 367 | |
| 368 | for (size_t i = 0; i < rtp_config_.extensions.size(); ++i) { |
| 369 | const std::string& extension = rtp_config_.extensions[i].uri; |
| 370 | int id = rtp_config_.extensions[i].id; |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 371 | RTC_DCHECK(RtpExtension::IsSupportedForVideo(extension)); |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 372 | for (const RtpStreamSender& stream : rtp_streams_) { |
Sebastian Jansson | f39c815 | 2019-10-14 17:32:21 +0200 | [diff] [blame] | 373 | stream.rtp_rtcp->RegisterRtpHeaderExtension(extension, id); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 374 | } |
| 375 | } |
| 376 | |
Elad Alon | 62ce035 | 2019-05-23 16:58:53 +0200 | [diff] [blame] | 377 | ConfigureSsrcs(); |
| 378 | ConfigureRids(); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 379 | |
Elad Alon | 62ce035 | 2019-05-23 16:58:53 +0200 | [diff] [blame] | 380 | if (!rtp_config_.mid.empty()) { |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 381 | for (const RtpStreamSender& stream : rtp_streams_) { |
Elad Alon | 62ce035 | 2019-05-23 16:58:53 +0200 | [diff] [blame] | 382 | stream.rtp_rtcp->SetMid(rtp_config_.mid); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 383 | } |
| 384 | } |
| 385 | |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 386 | for (const RtpStreamSender& stream : rtp_streams_) { |
Amit Hilbuch | 38e6c66 | 2019-03-08 16:17:21 -0800 | [diff] [blame] | 387 | // Simulcast has one module for each layer. Set the CNAME on all modules. |
Elad Alon | 62ce035 | 2019-05-23 16:58:53 +0200 | [diff] [blame] | 388 | stream.rtp_rtcp->SetCNAME(rtp_config_.c_name.c_str()); |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 389 | stream.rtp_rtcp->RegisterRtcpStatisticsCallback(observers.rtcp_stats); |
Henrik Boström | 87e3f9d | 2019-05-27 10:44:24 +0200 | [diff] [blame] | 390 | stream.rtp_rtcp->SetReportBlockDataObserver( |
| 391 | observers.report_block_data_observer); |
Elad Alon | 62ce035 | 2019-05-23 16:58:53 +0200 | [diff] [blame] | 392 | stream.rtp_rtcp->SetMaxRtpPacketSize(rtp_config_.max_packet_size); |
| 393 | stream.rtp_rtcp->RegisterSendPayloadFrequency(rtp_config_.payload_type, |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 394 | kVideoPayloadTypeFrequency); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 395 | } |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 396 | // Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic, |
| 397 | // so enable that logic if either of those FEC schemes are enabled. |
| 398 | fec_controller_->SetProtectionMethod(FecEnabled(), NackEnabled()); |
| 399 | |
| 400 | fec_controller_->SetProtectionCallback(this); |
| 401 | // Signal congestion controller this object is ready for OnPacket* callbacks. |
Sebastian Jansson | f298855 | 2019-10-29 17:18:51 +0100 | [diff] [blame] | 402 | transport_->GetStreamFeedbackProvider()->RegisterStreamFeedbackObserver( |
| 403 | rtp_config_.ssrcs, this); |
Per | 83d0910 | 2016-04-15 14:59:13 +0200 | [diff] [blame] | 404 | } |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 +0000 | [diff] [blame] | 405 | |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 406 | RtpVideoSender::~RtpVideoSender() { |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 407 | for (const RtpStreamSender& stream : rtp_streams_) { |
| 408 | transport_->packet_router()->RemoveSendRtpModule(stream.rtp_rtcp.get()); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 409 | } |
Sebastian Jansson | f298855 | 2019-10-29 17:18:51 +0100 | [diff] [blame] | 410 | transport_->GetStreamFeedbackProvider()->DeRegisterStreamFeedbackObserver( |
| 411 | this); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 412 | } |
| 413 | |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 414 | void RtpVideoSender::RegisterProcessThread( |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 415 | ProcessThread* module_process_thread) { |
| 416 | RTC_DCHECK_RUN_ON(&module_process_thread_checker_); |
| 417 | RTC_DCHECK(!module_process_thread_); |
| 418 | module_process_thread_ = module_process_thread; |
| 419 | |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 420 | for (const RtpStreamSender& stream : rtp_streams_) { |
| 421 | module_process_thread_->RegisterModule(stream.rtp_rtcp.get(), |
| 422 | RTC_FROM_HERE); |
| 423 | } |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 424 | } |
| 425 | |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 426 | void RtpVideoSender::DeRegisterProcessThread() { |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 427 | RTC_DCHECK_RUN_ON(&module_process_thread_checker_); |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 428 | for (const RtpStreamSender& stream : rtp_streams_) |
| 429 | module_process_thread_->DeRegisterModule(stream.rtp_rtcp.get()); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 430 | } |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 +0000 | [diff] [blame] | 431 | |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 432 | void RtpVideoSender::SetActive(bool active) { |
Tommi | 97888bd | 2016-01-21 23:24:59 +0100 | [diff] [blame] | 433 | rtc::CritScope lock(&crit_); |
Peter Boström | 8b79b07 | 2016-02-26 16:31:37 +0100 | [diff] [blame] | 434 | if (active_ == active) |
| 435 | return; |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 436 | const std::vector<bool> active_modules(rtp_streams_.size(), active); |
Seth Hampson | cc7125f | 2018-02-02 08:46:16 -0800 | [diff] [blame] | 437 | SetActiveModules(active_modules); |
| 438 | } |
Per | 512ecb3 | 2016-09-23 15:52:06 +0200 | [diff] [blame] | 439 | |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 440 | void RtpVideoSender::SetActiveModules(const std::vector<bool> active_modules) { |
Seth Hampson | cc7125f | 2018-02-02 08:46:16 -0800 | [diff] [blame] | 441 | rtc::CritScope lock(&crit_); |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 442 | RTC_DCHECK_EQ(rtp_streams_.size(), active_modules.size()); |
Seth Hampson | cc7125f | 2018-02-02 08:46:16 -0800 | [diff] [blame] | 443 | active_ = false; |
| 444 | for (size_t i = 0; i < active_modules.size(); ++i) { |
| 445 | if (active_modules[i]) { |
| 446 | active_ = true; |
| 447 | } |
| 448 | // Sends a kRtcpByeCode when going from true to false. |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 449 | rtp_streams_[i].rtp_rtcp->SetSendingStatus(active_modules[i]); |
Seth Hampson | cc7125f | 2018-02-02 08:46:16 -0800 | [diff] [blame] | 450 | // If set to false this module won't send media. |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 451 | rtp_streams_[i].rtp_rtcp->SetSendingMediaStatus(active_modules[i]); |
Per | 512ecb3 | 2016-09-23 15:52:06 +0200 | [diff] [blame] | 452 | } |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 +0000 | [diff] [blame] | 453 | } |
| 454 | |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 455 | bool RtpVideoSender::IsActive() { |
Tommi | 97888bd | 2016-01-21 23:24:59 +0100 | [diff] [blame] | 456 | rtc::CritScope lock(&crit_); |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 457 | return active_ && !rtp_streams_.empty(); |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 +0000 | [diff] [blame] | 458 | } |
| 459 | |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 460 | EncodedImageCallback::Result RtpVideoSender::OnEncodedImage( |
Sergey Ulanov | 525df3f | 2016-08-02 17:46:41 -0700 | [diff] [blame] | 461 | const EncodedImage& encoded_image, |
| 462 | const CodecSpecificInfo* codec_specific_info, |
| 463 | const RTPFragmentationHeader* fragmentation) { |
Niels Möller | 77536a2 | 2019-01-15 08:50:01 +0100 | [diff] [blame] | 464 | fec_controller_->UpdateWithEncodedData(encoded_image.size(), |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 465 | encoded_image._frameType); |
Tommi | 97888bd | 2016-01-21 23:24:59 +0100 | [diff] [blame] | 466 | rtc::CritScope lock(&crit_); |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 467 | RTC_DCHECK(!rtp_streams_.empty()); |
Per | 512ecb3 | 2016-09-23 15:52:06 +0200 | [diff] [blame] | 468 | if (!active_) |
Sergey Ulanov | 525df3f | 2016-08-02 17:46:41 -0700 | [diff] [blame] | 469 | return Result(Result::ERROR_SEND_FAILED); |
mflodman@webrtc.org | 50e2816 | 2015-02-23 07:45:11 +0000 | [diff] [blame] | 470 | |
philipel | bf2b620 | 2018-08-27 14:33:18 +0200 | [diff] [blame] | 471 | shared_frame_id_++; |
Niels Möller | d3b8c63 | 2018-08-27 15:33:42 +0200 | [diff] [blame] | 472 | size_t stream_index = 0; |
| 473 | if (codec_specific_info && |
| 474 | (codec_specific_info->codecType == kVideoCodecVP8 || |
| 475 | codec_specific_info->codecType == kVideoCodecH264 || |
| 476 | codec_specific_info->codecType == kVideoCodecGeneric)) { |
| 477 | // Map spatial index to simulcast. |
| 478 | stream_index = encoded_image.SpatialIndex().value_or(0); |
| 479 | } |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 480 | RTC_DCHECK_LT(stream_index, rtp_streams_.size()); |
Niels Möller | bb894ff | 2018-03-15 12:28:53 +0100 | [diff] [blame] | 481 | |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 482 | uint32_t rtp_timestamp = |
| 483 | encoded_image.Timestamp() + |
| 484 | rtp_streams_[stream_index].rtp_rtcp->StartTimestamp(); |
| 485 | |
| 486 | // RTCPSender has it's own copy of the timestamp offset, added in |
| 487 | // RTCPSender::BuildSR, hence we must not add the in the offset for this call. |
| 488 | // TODO(nisse): Delete RTCPSender:timestamp_offset_, and see if we can confine |
| 489 | // knowledge of the offset to a single place. |
| 490 | if (!rtp_streams_[stream_index].rtp_rtcp->OnSendingRtpFrame( |
| 491 | encoded_image.Timestamp(), encoded_image.capture_time_ms_, |
| 492 | rtp_config_.payload_type, |
Niels Möller | 8f7ce22 | 2019-03-21 15:43:58 +0100 | [diff] [blame] | 493 | encoded_image._frameType == VideoFrameType::kVideoFrameKey)) { |
Seth Hampson | cc7125f | 2018-02-02 08:46:16 -0800 | [diff] [blame] | 494 | // The payload router could be active but this module isn't sending. |
| 495 | return Result(Result::ERROR_SEND_FAILED); |
| 496 | } |
Elad Alon | b64af4b | 2019-06-05 11:39:37 +0200 | [diff] [blame] | 497 | |
| 498 | absl::optional<int64_t> expected_retransmission_time_ms; |
| 499 | if (encoded_image.RetransmissionAllowed()) { |
| 500 | expected_retransmission_time_ms = |
| 501 | rtp_streams_[stream_index].rtp_rtcp->ExpectedRetransmissionTimeMs(); |
| 502 | } |
Niels Möller | 949f0fd | 2019-01-29 09:44:24 +0100 | [diff] [blame] | 503 | |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 504 | bool send_result = rtp_streams_[stream_index].sender_video->SendVideo( |
Danil Chapovalov | 51bf200 | 2019-10-11 10:53:27 +0200 | [diff] [blame] | 505 | rtp_config_.payload_type, codec_type_, rtp_timestamp, |
| 506 | encoded_image.capture_time_ms_, encoded_image, fragmentation, |
| 507 | params_[stream_index].GetRtpVideoHeader( |
| 508 | encoded_image, codec_specific_info, shared_frame_id_), |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 509 | expected_retransmission_time_ms); |
Niels Möller | 949f0fd | 2019-01-29 09:44:24 +0100 | [diff] [blame] | 510 | if (frame_count_observer_) { |
| 511 | FrameCounts& counts = frame_counts_[stream_index]; |
Niels Möller | 8f7ce22 | 2019-03-21 15:43:58 +0100 | [diff] [blame] | 512 | if (encoded_image._frameType == VideoFrameType::kVideoFrameKey) { |
Niels Möller | 949f0fd | 2019-01-29 09:44:24 +0100 | [diff] [blame] | 513 | ++counts.key_frames; |
Niels Möller | 8f7ce22 | 2019-03-21 15:43:58 +0100 | [diff] [blame] | 514 | } else if (encoded_image._frameType == VideoFrameType::kVideoFrameDelta) { |
Niels Möller | 949f0fd | 2019-01-29 09:44:24 +0100 | [diff] [blame] | 515 | ++counts.delta_frames; |
| 516 | } else { |
Niels Möller | 8f7ce22 | 2019-03-21 15:43:58 +0100 | [diff] [blame] | 517 | RTC_DCHECK(encoded_image._frameType == VideoFrameType::kEmptyFrame); |
Niels Möller | 949f0fd | 2019-01-29 09:44:24 +0100 | [diff] [blame] | 518 | } |
| 519 | frame_count_observer_->FrameCountUpdated(counts, |
| 520 | rtp_config_.ssrcs[stream_index]); |
| 521 | } |
sergeyu | 7b9feee | 2016-11-17 16:16:14 -0800 | [diff] [blame] | 522 | if (!send_result) |
Sergey Ulanov | 525df3f | 2016-08-02 17:46:41 -0700 | [diff] [blame] | 523 | return Result(Result::ERROR_SEND_FAILED); |
| 524 | |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 525 | return Result(Result::OK, rtp_timestamp); |
mflodman@webrtc.org | a4ef2ce | 2015-02-12 09:54:18 +0000 | [diff] [blame] | 526 | } |
| 527 | |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 528 | void RtpVideoSender::OnBitrateAllocationUpdated( |
Erik Språng | 566124a | 2018-04-23 12:32:22 +0200 | [diff] [blame] | 529 | const VideoBitrateAllocation& bitrate) { |
sprang | 1a646ee | 2016-12-01 06:34:11 -0800 | [diff] [blame] | 530 | rtc::CritScope lock(&crit_); |
| 531 | if (IsActive()) { |
Oleh Prypin | e896490 | 2019-03-29 15:33:01 +0000 | [diff] [blame] | 532 | if (rtp_streams_.size() == 1) { |
sprang | 1a646ee | 2016-12-01 06:34:11 -0800 | [diff] [blame] | 533 | // If spatial scalability is enabled, it is covered by a single stream. |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 534 | rtp_streams_[0].rtp_rtcp->SetVideoBitrateAllocation(bitrate); |
sprang | 1a646ee | 2016-12-01 06:34:11 -0800 | [diff] [blame] | 535 | } else { |
Stefan Holmer | f704468 | 2018-07-17 10:16:41 +0200 | [diff] [blame] | 536 | std::vector<absl::optional<VideoBitrateAllocation>> layer_bitrates = |
| 537 | bitrate.GetSimulcastAllocations(); |
Erik Språng | 566124a | 2018-04-23 12:32:22 +0200 | [diff] [blame] | 538 | // Simulcast is in use, split the VideoBitrateAllocation into one struct |
| 539 | // per rtp stream, moving over the temporal layer allocation. |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 540 | for (size_t i = 0; i < rtp_streams_.size(); ++i) { |
Stefan Holmer | f704468 | 2018-07-17 10:16:41 +0200 | [diff] [blame] | 541 | // The next spatial layer could be used if the current one is |
| 542 | // inactive. |
| 543 | if (layer_bitrates[i]) { |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 544 | rtp_streams_[i].rtp_rtcp->SetVideoBitrateAllocation( |
| 545 | *layer_bitrates[i]); |
Ilya Nikolaevskiy | b0588e6 | 2018-08-27 14:12:27 +0200 | [diff] [blame] | 546 | } else { |
| 547 | // Signal a 0 bitrate on a simulcast stream. |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 548 | rtp_streams_[i].rtp_rtcp->SetVideoBitrateAllocation( |
| 549 | VideoBitrateAllocation()); |
Seth Hampson | 46e31ba | 2018-01-18 10:39:54 -0800 | [diff] [blame] | 550 | } |
sprang | 1a646ee | 2016-12-01 06:34:11 -0800 | [diff] [blame] | 551 | } |
| 552 | } |
| 553 | } |
| 554 | } |
| 555 | |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 556 | bool RtpVideoSender::FecEnabled() const { |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 557 | const bool flexfec_enabled = (flexfec_sender_ != nullptr); |
Emircan Uysaler | a7af021 | 2018-09-22 19:11:29 -0400 | [diff] [blame] | 558 | const bool ulpfec_enabled = |
| 559 | !webrtc::field_trial::IsEnabled("WebRTC-DisableUlpFecExperiment") && |
| 560 | (rtp_config_.ulpfec.ulpfec_payload_type >= 0); |
| 561 | return flexfec_enabled || ulpfec_enabled; |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 562 | } |
| 563 | |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 564 | bool RtpVideoSender::NackEnabled() const { |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 565 | const bool nack_enabled = rtp_config_.nack.rtp_history_ms > 0; |
| 566 | return nack_enabled; |
| 567 | } |
| 568 | |
Erik Språng | 482b3ef | 2019-01-08 16:19:11 +0100 | [diff] [blame] | 569 | uint32_t RtpVideoSender::GetPacketizationOverheadRate() const { |
| 570 | uint32_t packetization_overhead_bps = 0; |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 571 | for (size_t i = 0; i < rtp_streams_.size(); ++i) { |
| 572 | if (rtp_streams_[i].rtp_rtcp->SendingMedia()) { |
| 573 | packetization_overhead_bps += |
| 574 | rtp_streams_[i].sender_video->PacketizationOverheadBps(); |
Erik Språng | 482b3ef | 2019-01-08 16:19:11 +0100 | [diff] [blame] | 575 | } |
| 576 | } |
| 577 | return packetization_overhead_bps; |
| 578 | } |
| 579 | |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 580 | void RtpVideoSender::DeliverRtcp(const uint8_t* packet, size_t length) { |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 581 | // Runs on a network thread. |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 582 | for (const RtpStreamSender& stream : rtp_streams_) |
| 583 | stream.rtp_rtcp->IncomingRtcpPacket(packet, length); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 584 | } |
| 585 | |
Elad Alon | 62ce035 | 2019-05-23 16:58:53 +0200 | [diff] [blame] | 586 | void RtpVideoSender::ConfigureSsrcs() { |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 587 | // Configure regular SSRCs. |
Erik Språng | a922904 | 2019-10-24 12:39:32 +0200 | [diff] [blame] | 588 | RTC_CHECK(ssrc_to_rtp_module_.empty()); |
Elad Alon | 62ce035 | 2019-05-23 16:58:53 +0200 | [diff] [blame] | 589 | for (size_t i = 0; i < rtp_config_.ssrcs.size(); ++i) { |
| 590 | uint32_t ssrc = rtp_config_.ssrcs[i]; |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 591 | RtpRtcp* const rtp_rtcp = rtp_streams_[i].rtp_rtcp.get(); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 592 | |
| 593 | // Restore RTP state if previous existed. |
| 594 | auto it = suspended_ssrcs_.find(ssrc); |
| 595 | if (it != suspended_ssrcs_.end()) |
| 596 | rtp_rtcp->SetRtpState(it->second); |
Erik Språng | 490d76c | 2019-05-07 09:29:15 -0700 | [diff] [blame] | 597 | |
Erik Språng | a922904 | 2019-10-24 12:39:32 +0200 | [diff] [blame] | 598 | ssrc_to_rtp_module_[ssrc] = rtp_rtcp; |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 599 | } |
| 600 | |
| 601 | // Set up RTX if available. |
Elad Alon | 62ce035 | 2019-05-23 16:58:53 +0200 | [diff] [blame] | 602 | if (rtp_config_.rtx.ssrcs.empty()) |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 603 | return; |
| 604 | |
Elad Alon | 62ce035 | 2019-05-23 16:58:53 +0200 | [diff] [blame] | 605 | RTC_DCHECK_EQ(rtp_config_.rtx.ssrcs.size(), rtp_config_.ssrcs.size()); |
| 606 | for (size_t i = 0; i < rtp_config_.rtx.ssrcs.size(); ++i) { |
| 607 | uint32_t ssrc = rtp_config_.rtx.ssrcs[i]; |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 608 | RtpRtcp* const rtp_rtcp = rtp_streams_[i].rtp_rtcp.get(); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 609 | auto it = suspended_ssrcs_.find(ssrc); |
| 610 | if (it != suspended_ssrcs_.end()) |
| 611 | rtp_rtcp->SetRtxState(it->second); |
| 612 | } |
| 613 | |
| 614 | // Configure RTX payload types. |
Elad Alon | 62ce035 | 2019-05-23 16:58:53 +0200 | [diff] [blame] | 615 | RTC_DCHECK_GE(rtp_config_.rtx.payload_type, 0); |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 616 | for (const RtpStreamSender& stream : rtp_streams_) { |
Elad Alon | 62ce035 | 2019-05-23 16:58:53 +0200 | [diff] [blame] | 617 | stream.rtp_rtcp->SetRtxSendPayloadType(rtp_config_.rtx.payload_type, |
| 618 | rtp_config_.payload_type); |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 619 | stream.rtp_rtcp->SetRtxSendStatus(kRtxRetransmitted | |
| 620 | kRtxRedundantPayloads); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 621 | } |
Elad Alon | 62ce035 | 2019-05-23 16:58:53 +0200 | [diff] [blame] | 622 | if (rtp_config_.ulpfec.red_payload_type != -1 && |
| 623 | rtp_config_.ulpfec.red_rtx_payload_type != -1) { |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 624 | for (const RtpStreamSender& stream : rtp_streams_) { |
| 625 | stream.rtp_rtcp->SetRtxSendPayloadType( |
Elad Alon | 62ce035 | 2019-05-23 16:58:53 +0200 | [diff] [blame] | 626 | rtp_config_.ulpfec.red_rtx_payload_type, |
| 627 | rtp_config_.ulpfec.red_payload_type); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 628 | } |
| 629 | } |
| 630 | } |
| 631 | |
Elad Alon | 62ce035 | 2019-05-23 16:58:53 +0200 | [diff] [blame] | 632 | void RtpVideoSender::ConfigureRids() { |
Florent Castelli | 463d44a | 2019-07-12 15:35:53 +0200 | [diff] [blame] | 633 | if (rtp_config_.rids.empty()) |
| 634 | return; |
| 635 | |
| 636 | // Some streams could have been disabled, but the rids are still there. |
| 637 | // This will occur when simulcast has been disabled for a codec (e.g. VP9) |
| 638 | RTC_DCHECK(rtp_config_.rids.size() >= rtp_streams_.size()); |
| 639 | for (size_t i = 0; i < rtp_streams_.size(); ++i) { |
| 640 | rtp_streams_[i].rtp_rtcp->SetRid(rtp_config_.rids[i]); |
Amit Hilbuch | 77938e6 | 2018-12-21 09:23:38 -0800 | [diff] [blame] | 641 | } |
| 642 | } |
| 643 | |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 644 | void RtpVideoSender::OnNetworkAvailability(bool network_available) { |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 645 | for (const RtpStreamSender& stream : rtp_streams_) { |
| 646 | stream.rtp_rtcp->SetRTCPStatus(network_available ? rtp_config_.rtcp_mode |
| 647 | : RtcpMode::kOff); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 648 | } |
| 649 | } |
| 650 | |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 651 | std::map<uint32_t, RtpState> RtpVideoSender::GetRtpStates() const { |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 652 | std::map<uint32_t, RtpState> rtp_states; |
| 653 | |
| 654 | for (size_t i = 0; i < rtp_config_.ssrcs.size(); ++i) { |
| 655 | uint32_t ssrc = rtp_config_.ssrcs[i]; |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 656 | RTC_DCHECK_EQ(ssrc, rtp_streams_[i].rtp_rtcp->SSRC()); |
| 657 | rtp_states[ssrc] = rtp_streams_[i].rtp_rtcp->GetRtpState(); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 658 | } |
| 659 | |
| 660 | for (size_t i = 0; i < rtp_config_.rtx.ssrcs.size(); ++i) { |
| 661 | uint32_t ssrc = rtp_config_.rtx.ssrcs[i]; |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 662 | rtp_states[ssrc] = rtp_streams_[i].rtp_rtcp->GetRtxState(); |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 663 | } |
| 664 | |
| 665 | if (flexfec_sender_) { |
| 666 | uint32_t ssrc = rtp_config_.flexfec.ssrc; |
| 667 | rtp_states[ssrc] = flexfec_sender_->GetRtpState(); |
| 668 | } |
| 669 | |
| 670 | return rtp_states; |
| 671 | } |
| 672 | |
Stefan Holmer | 9416ef8 | 2018-07-19 10:34:38 +0200 | [diff] [blame] | 673 | std::map<uint32_t, RtpPayloadState> RtpVideoSender::GetRtpPayloadStates() |
| 674 | const { |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 675 | rtc::CritScope lock(&crit_); |
| 676 | std::map<uint32_t, RtpPayloadState> payload_states; |
| 677 | for (const auto& param : params_) { |
| 678 | payload_states[param.ssrc()] = param.state(); |
philipel | 25d31ec | 2018-08-08 16:33:01 +0200 | [diff] [blame] | 679 | payload_states[param.ssrc()].shared_frame_id = shared_frame_id_; |
Stefan Holmer | dbdb3a0 | 2018-07-17 16:03:46 +0200 | [diff] [blame] | 680 | } |
| 681 | return payload_states; |
| 682 | } |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 683 | |
| 684 | void RtpVideoSender::OnTransportOverheadChanged( |
| 685 | size_t transport_overhead_bytes_per_packet) { |
| 686 | rtc::CritScope lock(&crit_); |
| 687 | transport_overhead_bytes_per_packet_ = transport_overhead_bytes_per_packet; |
| 688 | |
| 689 | size_t max_rtp_packet_size = |
| 690 | std::min(rtp_config_.max_packet_size, |
| 691 | kPathMTU - transport_overhead_bytes_per_packet_); |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 692 | for (const RtpStreamSender& stream : rtp_streams_) { |
| 693 | stream.rtp_rtcp->SetMaxRtpPacketSize(max_rtp_packet_size); |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 694 | } |
| 695 | } |
| 696 | |
| 697 | void RtpVideoSender::OnOverheadChanged(size_t overhead_bytes_per_packet) { |
| 698 | rtc::CritScope lock(&crit_); |
| 699 | overhead_bytes_per_packet_ = overhead_bytes_per_packet; |
| 700 | } |
| 701 | |
Sebastian Jansson | 82ed2e8 | 2019-10-15 15:58:37 +0200 | [diff] [blame] | 702 | void RtpVideoSender::OnBitrateUpdated(BitrateAllocationUpdate update, |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 703 | int framerate) { |
| 704 | // Substract overhead from bitrate. |
| 705 | rtc::CritScope lock(&crit_); |
Sebastian Jansson | cf41eb1 | 2019-06-10 11:30:59 +0200 | [diff] [blame] | 706 | DataSize packet_overhead = DataSize::bytes( |
| 707 | overhead_bytes_per_packet_ + transport_overhead_bytes_per_packet_); |
| 708 | DataSize max_total_packet_size = DataSize::bytes( |
| 709 | rtp_config_.max_packet_size + transport_overhead_bytes_per_packet_); |
Sebastian Jansson | 82ed2e8 | 2019-10-15 15:58:37 +0200 | [diff] [blame] | 710 | uint32_t payload_bitrate_bps = update.target_bitrate.bps(); |
Sebastian Jansson | c3eb9fd | 2020-01-29 17:42:52 +0100 | [diff] [blame] | 711 | if (send_side_bwe_with_overhead_ && has_packet_feedback_) { |
Sebastian Jansson | cf41eb1 | 2019-06-10 11:30:59 +0200 | [diff] [blame] | 712 | DataRate overhead_rate = CalculateOverheadRate( |
Sebastian Jansson | 82ed2e8 | 2019-10-15 15:58:37 +0200 | [diff] [blame] | 713 | update.target_bitrate, max_total_packet_size, packet_overhead); |
Sebastian Jansson | cf41eb1 | 2019-06-10 11:30:59 +0200 | [diff] [blame] | 714 | // TODO(srte): We probably should not accept 0 payload bitrate here. |
Sebastian Jansson | 82ed2e8 | 2019-10-15 15:58:37 +0200 | [diff] [blame] | 715 | payload_bitrate_bps = rtc::saturated_cast<uint32_t>(payload_bitrate_bps - |
| 716 | overhead_rate.bps()); |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 717 | } |
| 718 | |
| 719 | // Get the encoder target rate. It is the estimated network rate - |
| 720 | // protection overhead. |
Sebastian Jansson | 82ed2e8 | 2019-10-15 15:58:37 +0200 | [diff] [blame] | 721 | // TODO(srte): We should multiply with 255 here. |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 722 | encoder_target_rate_bps_ = fec_controller_->UpdateFecRates( |
Sebastian Jansson | 82ed2e8 | 2019-10-15 15:58:37 +0200 | [diff] [blame] | 723 | payload_bitrate_bps, framerate, |
| 724 | rtc::saturated_cast<uint8_t>(update.packet_loss_ratio * 256), |
| 725 | loss_mask_vector_, update.round_trip_time.ms()); |
Elad Alon | 67daf71 | 2019-06-28 18:14:36 +0200 | [diff] [blame] | 726 | if (!fec_allowed_) { |
| 727 | encoder_target_rate_bps_ = payload_bitrate_bps; |
| 728 | // fec_controller_->UpdateFecRates() was still called so as to allow |
| 729 | // |fec_controller_| to update whatever internal state it might have, |
| 730 | // since |fec_allowed_| may be toggled back on at any moment. |
| 731 | } |
Erik Språng | 482b3ef | 2019-01-08 16:19:11 +0100 | [diff] [blame] | 732 | |
Erik Språng | d15687d | 2019-01-18 10:47:07 +0100 | [diff] [blame] | 733 | uint32_t packetization_rate_bps = 0; |
Erik Språng | c12d41b | 2019-01-09 09:55:31 +0100 | [diff] [blame] | 734 | if (account_for_packetization_overhead_) { |
Erik Språng | d15687d | 2019-01-18 10:47:07 +0100 | [diff] [blame] | 735 | // Subtract packetization overhead from the encoder target. If target rate |
| 736 | // is really low, cap the overhead at 50%. This also avoids the case where |
| 737 | // |encoder_target_rate_bps_| is 0 due to encoder pause event while the |
| 738 | // packetization rate is positive since packets are still flowing. |
| 739 | packetization_rate_bps = |
| 740 | std::min(GetPacketizationOverheadRate(), encoder_target_rate_bps_ / 2); |
| 741 | encoder_target_rate_bps_ -= packetization_rate_bps; |
Erik Språng | c12d41b | 2019-01-09 09:55:31 +0100 | [diff] [blame] | 742 | } |
Erik Språng | 482b3ef | 2019-01-08 16:19:11 +0100 | [diff] [blame] | 743 | |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 744 | loss_mask_vector_.clear(); |
| 745 | |
Sebastian Jansson | cf41eb1 | 2019-06-10 11:30:59 +0200 | [diff] [blame] | 746 | uint32_t encoder_overhead_rate_bps = 0; |
Sebastian Jansson | c3eb9fd | 2020-01-29 17:42:52 +0100 | [diff] [blame] | 747 | if (send_side_bwe_with_overhead_ && has_packet_feedback_) { |
Sebastian Jansson | cf41eb1 | 2019-06-10 11:30:59 +0200 | [diff] [blame] | 748 | // TODO(srte): The packet size should probably be the same as in the |
| 749 | // CalculateOverheadRate call above (just max_total_packet_size), it doesn't |
| 750 | // make sense to use different packet rates for different overhead |
| 751 | // calculations. |
| 752 | DataRate encoder_overhead_rate = CalculateOverheadRate( |
| 753 | DataRate::bps(encoder_target_rate_bps_), |
| 754 | max_total_packet_size - DataSize::bytes(overhead_bytes_per_packet_), |
| 755 | packet_overhead); |
Sebastian Jansson | 82ed2e8 | 2019-10-15 15:58:37 +0200 | [diff] [blame] | 756 | encoder_overhead_rate_bps = std::min( |
| 757 | encoder_overhead_rate.bps<uint32_t>(), |
| 758 | update.target_bitrate.bps<uint32_t>() - encoder_target_rate_bps_); |
Sebastian Jansson | cf41eb1 | 2019-06-10 11:30:59 +0200 | [diff] [blame] | 759 | } |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 760 | // When the field trial "WebRTC-SendSideBwe-WithOverhead" is enabled |
| 761 | // protection_bitrate includes overhead. |
Erik Språng | d15687d | 2019-01-18 10:47:07 +0100 | [diff] [blame] | 762 | const uint32_t media_rate = encoder_target_rate_bps_ + |
| 763 | encoder_overhead_rate_bps + |
| 764 | packetization_rate_bps; |
Sebastian Jansson | 82ed2e8 | 2019-10-15 15:58:37 +0200 | [diff] [blame] | 765 | RTC_DCHECK_GE(update.target_bitrate, DataRate::bps(media_rate)); |
| 766 | protection_bitrate_bps_ = update.target_bitrate.bps() - media_rate; |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 767 | } |
| 768 | |
| 769 | uint32_t RtpVideoSender::GetPayloadBitrateBps() const { |
| 770 | return encoder_target_rate_bps_; |
| 771 | } |
| 772 | |
| 773 | uint32_t RtpVideoSender::GetProtectionBitrateBps() const { |
| 774 | return protection_bitrate_bps_; |
| 775 | } |
| 776 | |
Elad Alon | 898395d | 2019-04-10 15:55:00 +0200 | [diff] [blame] | 777 | std::vector<RtpSequenceNumberMap::Info> RtpVideoSender::GetSentRtpPacketInfos( |
Elad Alon | 8b60e8b | 2019-04-08 14:14:05 +0200 | [diff] [blame] | 778 | uint32_t ssrc, |
Elad Alon | 898395d | 2019-04-10 15:55:00 +0200 | [diff] [blame] | 779 | rtc::ArrayView<const uint16_t> sequence_numbers) const { |
Elad Alon | 8b60e8b | 2019-04-08 14:14:05 +0200 | [diff] [blame] | 780 | for (const auto& rtp_stream : rtp_streams_) { |
| 781 | if (ssrc == rtp_stream.rtp_rtcp->SSRC()) { |
Elad Alon | 898395d | 2019-04-10 15:55:00 +0200 | [diff] [blame] | 782 | return rtp_stream.sender_video->GetSentRtpPacketInfos(sequence_numbers); |
Elad Alon | 8b60e8b | 2019-04-08 14:14:05 +0200 | [diff] [blame] | 783 | } |
| 784 | } |
Elad Alon | 898395d | 2019-04-10 15:55:00 +0200 | [diff] [blame] | 785 | return std::vector<RtpSequenceNumberMap::Info>(); |
Elad Alon | 8b60e8b | 2019-04-08 14:14:05 +0200 | [diff] [blame] | 786 | } |
| 787 | |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 788 | int RtpVideoSender::ProtectionRequest(const FecProtectionParams* delta_params, |
| 789 | const FecProtectionParams* key_params, |
| 790 | uint32_t* sent_video_rate_bps, |
| 791 | uint32_t* sent_nack_rate_bps, |
| 792 | uint32_t* sent_fec_rate_bps) { |
| 793 | *sent_video_rate_bps = 0; |
| 794 | *sent_nack_rate_bps = 0; |
| 795 | *sent_fec_rate_bps = 0; |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 796 | for (const RtpStreamSender& stream : rtp_streams_) { |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 797 | uint32_t not_used = 0; |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 798 | uint32_t module_nack_rate = 0; |
Niels Möller | 5fe9510 | 2019-03-04 16:49:25 +0100 | [diff] [blame] | 799 | stream.sender_video->SetFecParameters(*delta_params, *key_params); |
| 800 | *sent_video_rate_bps += stream.sender_video->VideoBitrateSent(); |
| 801 | *sent_fec_rate_bps += stream.sender_video->FecOverheadRate(); |
| 802 | stream.rtp_rtcp->BitrateSent(¬_used, /*video_rate=*/nullptr, |
| 803 | /*fec_rate=*/nullptr, &module_nack_rate); |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 804 | *sent_nack_rate_bps += module_nack_rate; |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 805 | } |
| 806 | return 0; |
| 807 | } |
| 808 | |
Elad Alon | 8f01c4e | 2019-06-28 15:19:43 +0200 | [diff] [blame] | 809 | void RtpVideoSender::SetFecAllowed(bool fec_allowed) { |
Elad Alon | 67daf71 | 2019-06-28 18:14:36 +0200 | [diff] [blame] | 810 | rtc::CritScope cs(&crit_); |
| 811 | fec_allowed_ = fec_allowed; |
Elad Alon | 8f01c4e | 2019-06-28 15:19:43 +0200 | [diff] [blame] | 812 | } |
| 813 | |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 814 | void RtpVideoSender::OnPacketFeedbackVector( |
Sebastian Jansson | f298855 | 2019-10-29 17:18:51 +0100 | [diff] [blame] | 815 | std::vector<StreamPacketInfo> packet_feedback_vector) { |
Erik Språng | 490d76c | 2019-05-07 09:29:15 -0700 | [diff] [blame] | 816 | if (fec_controller_->UseLossVectorMask()) { |
| 817 | rtc::CritScope cs(&crit_); |
Sebastian Jansson | f298855 | 2019-10-29 17:18:51 +0100 | [diff] [blame] | 818 | for (const StreamPacketInfo& packet : packet_feedback_vector) { |
| 819 | loss_mask_vector_.push_back(!packet.received); |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 820 | } |
| 821 | } |
Erik Språng | 490d76c | 2019-05-07 09:29:15 -0700 | [diff] [blame] | 822 | |
| 823 | // Map from SSRC to all acked packets for that RTP module. |
| 824 | std::map<uint32_t, std::vector<uint16_t>> acked_packets_per_ssrc; |
Sebastian Jansson | f298855 | 2019-10-29 17:18:51 +0100 | [diff] [blame] | 825 | for (const StreamPacketInfo& packet : packet_feedback_vector) { |
| 826 | if (packet.received) { |
| 827 | acked_packets_per_ssrc[packet.ssrc].push_back(packet.rtp_sequence_number); |
Erik Språng | 490d76c | 2019-05-07 09:29:15 -0700 | [diff] [blame] | 828 | } |
| 829 | } |
| 830 | |
Erik Språng | 845c6aa | 2019-05-29 13:02:24 +0200 | [diff] [blame] | 831 | if (use_early_loss_detection_) { |
| 832 | // Map from SSRC to vector of RTP sequence numbers that are indicated as |
| 833 | // lost by feedback, without being trailed by any received packets. |
| 834 | std::map<uint32_t, std::vector<uint16_t>> early_loss_detected_per_ssrc; |
| 835 | |
Sebastian Jansson | f298855 | 2019-10-29 17:18:51 +0100 | [diff] [blame] | 836 | for (const StreamPacketInfo& packet : packet_feedback_vector) { |
| 837 | if (!packet.received) { |
Erik Språng | 845c6aa | 2019-05-29 13:02:24 +0200 | [diff] [blame] | 838 | // Last known lost packet, might not be detectable as lost by remote |
| 839 | // jitter buffer. |
Sebastian Jansson | f298855 | 2019-10-29 17:18:51 +0100 | [diff] [blame] | 840 | early_loss_detected_per_ssrc[packet.ssrc].push_back( |
Erik Språng | 845c6aa | 2019-05-29 13:02:24 +0200 | [diff] [blame] | 841 | packet.rtp_sequence_number); |
| 842 | } else { |
| 843 | // Packet received, so any loss prior to this is already detectable. |
Sebastian Jansson | f298855 | 2019-10-29 17:18:51 +0100 | [diff] [blame] | 844 | early_loss_detected_per_ssrc.erase(packet.ssrc); |
Erik Språng | 845c6aa | 2019-05-29 13:02:24 +0200 | [diff] [blame] | 845 | } |
| 846 | } |
| 847 | |
| 848 | for (const auto& kv : early_loss_detected_per_ssrc) { |
| 849 | const uint32_t ssrc = kv.first; |
Erik Språng | a922904 | 2019-10-24 12:39:32 +0200 | [diff] [blame] | 850 | auto it = ssrc_to_rtp_module_.find(ssrc); |
| 851 | RTC_DCHECK(it != ssrc_to_rtp_module_.end()); |
| 852 | RTPSender* rtp_sender = it->second->RtpSender(); |
Erik Språng | 845c6aa | 2019-05-29 13:02:24 +0200 | [diff] [blame] | 853 | for (uint16_t sequence_number : kv.second) { |
| 854 | rtp_sender->ReSendPacket(sequence_number); |
| 855 | } |
| 856 | } |
| 857 | } |
| 858 | |
Erik Språng | 490d76c | 2019-05-07 09:29:15 -0700 | [diff] [blame] | 859 | for (const auto& kv : acked_packets_per_ssrc) { |
| 860 | const uint32_t ssrc = kv.first; |
Erik Språng | a922904 | 2019-10-24 12:39:32 +0200 | [diff] [blame] | 861 | auto it = ssrc_to_rtp_module_.find(ssrc); |
| 862 | if (it == ssrc_to_rtp_module_.end()) { |
Erik Språng | 490d76c | 2019-05-07 09:29:15 -0700 | [diff] [blame] | 863 | // Packets not for a media SSRC, so likely RTX or FEC. If so, ignore |
| 864 | // since there's no RTP history to clean up anyway. |
| 865 | continue; |
| 866 | } |
| 867 | rtc::ArrayView<const uint16_t> rtp_sequence_numbers(kv.second); |
Erik Språng | 845c6aa | 2019-05-29 13:02:24 +0200 | [diff] [blame] | 868 | it->second->OnPacketsAcknowledged(rtp_sequence_numbers); |
Erik Språng | 490d76c | 2019-05-07 09:29:15 -0700 | [diff] [blame] | 869 | } |
Stefan Holmer | 64be7fa | 2018-10-04 15:21:55 +0200 | [diff] [blame] | 870 | } |
| 871 | |
| 872 | void RtpVideoSender::SetEncodingData(size_t width, |
| 873 | size_t height, |
| 874 | size_t num_temporal_layers) { |
| 875 | fec_controller_->SetEncodingData(width, height, num_temporal_layers, |
| 876 | rtp_config_.max_packet_size); |
| 877 | } |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 +0000 | [diff] [blame] | 878 | } // namespace webrtc |