mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #include "video/payload_router.h" |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 +0000 | [diff] [blame] | 12 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 13 | #include "modules/rtp_rtcp/include/rtp_rtcp.h" |
| 14 | #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" |
| 15 | #include "modules/video_coding/include/video_codec_interface.h" |
| 16 | #include "rtc_base/checks.h" |
Åsa Persson | 4bece9a | 2017-10-06 10:04:04 +0200 | [diff] [blame] | 17 | #include "rtc_base/random.h" |
| 18 | #include "rtc_base/timeutils.h" |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 +0000 | [diff] [blame] | 19 | |
| 20 | namespace webrtc { |
| 21 | |
kjellander | 02b3d27 | 2016-04-20 05:05:54 -0700 | [diff] [blame] | 22 | namespace { |
| 23 | // Map information from info into rtp. |
| 24 | void CopyCodecSpecific(const CodecSpecificInfo* info, RTPVideoHeader* rtp) { |
| 25 | RTC_DCHECK(info); |
| 26 | switch (info->codecType) { |
| 27 | case kVideoCodecVP8: { |
| 28 | rtp->codec = kRtpVideoVp8; |
| 29 | rtp->codecHeader.VP8.InitRTPVideoHeaderVP8(); |
kjellander | 02b3d27 | 2016-04-20 05:05:54 -0700 | [diff] [blame] | 30 | rtp->codecHeader.VP8.nonReference = info->codecSpecific.VP8.nonReference; |
| 31 | rtp->codecHeader.VP8.temporalIdx = info->codecSpecific.VP8.temporalIdx; |
| 32 | rtp->codecHeader.VP8.layerSync = info->codecSpecific.VP8.layerSync; |
kjellander | 02b3d27 | 2016-04-20 05:05:54 -0700 | [diff] [blame] | 33 | rtp->codecHeader.VP8.keyIdx = info->codecSpecific.VP8.keyIdx; |
| 34 | rtp->simulcastIdx = info->codecSpecific.VP8.simulcastIdx; |
| 35 | return; |
| 36 | } |
| 37 | case kVideoCodecVP9: { |
| 38 | rtp->codec = kRtpVideoVp9; |
| 39 | rtp->codecHeader.VP9.InitRTPVideoHeaderVP9(); |
| 40 | rtp->codecHeader.VP9.inter_pic_predicted = |
| 41 | info->codecSpecific.VP9.inter_pic_predicted; |
| 42 | rtp->codecHeader.VP9.flexible_mode = |
| 43 | info->codecSpecific.VP9.flexible_mode; |
| 44 | rtp->codecHeader.VP9.ss_data_available = |
| 45 | info->codecSpecific.VP9.ss_data_available; |
kjellander | 02b3d27 | 2016-04-20 05:05:54 -0700 | [diff] [blame] | 46 | rtp->codecHeader.VP9.temporal_idx = info->codecSpecific.VP9.temporal_idx; |
| 47 | rtp->codecHeader.VP9.spatial_idx = info->codecSpecific.VP9.spatial_idx; |
| 48 | rtp->codecHeader.VP9.temporal_up_switch = |
| 49 | info->codecSpecific.VP9.temporal_up_switch; |
| 50 | rtp->codecHeader.VP9.inter_layer_predicted = |
| 51 | info->codecSpecific.VP9.inter_layer_predicted; |
| 52 | rtp->codecHeader.VP9.gof_idx = info->codecSpecific.VP9.gof_idx; |
| 53 | rtp->codecHeader.VP9.num_spatial_layers = |
| 54 | info->codecSpecific.VP9.num_spatial_layers; |
| 55 | |
| 56 | if (info->codecSpecific.VP9.ss_data_available) { |
| 57 | rtp->codecHeader.VP9.spatial_layer_resolution_present = |
| 58 | info->codecSpecific.VP9.spatial_layer_resolution_present; |
| 59 | if (info->codecSpecific.VP9.spatial_layer_resolution_present) { |
| 60 | for (size_t i = 0; i < info->codecSpecific.VP9.num_spatial_layers; |
| 61 | ++i) { |
| 62 | rtp->codecHeader.VP9.width[i] = info->codecSpecific.VP9.width[i]; |
| 63 | rtp->codecHeader.VP9.height[i] = info->codecSpecific.VP9.height[i]; |
| 64 | } |
| 65 | } |
| 66 | rtp->codecHeader.VP9.gof.CopyGofInfoVP9(info->codecSpecific.VP9.gof); |
| 67 | } |
| 68 | |
| 69 | rtp->codecHeader.VP9.num_ref_pics = info->codecSpecific.VP9.num_ref_pics; |
| 70 | for (int i = 0; i < info->codecSpecific.VP9.num_ref_pics; ++i) |
| 71 | rtp->codecHeader.VP9.pid_diff[i] = info->codecSpecific.VP9.p_diff[i]; |
| 72 | return; |
| 73 | } |
| 74 | case kVideoCodecH264: |
| 75 | rtp->codec = kRtpVideoH264; |
hta | 9aa9688 | 2016-12-06 05:36:03 -0800 | [diff] [blame] | 76 | rtp->codecHeader.H264.packetization_mode = |
| 77 | info->codecSpecific.H264.packetization_mode; |
kjellander | 02b3d27 | 2016-04-20 05:05:54 -0700 | [diff] [blame] | 78 | return; |
Emircan Uysaler | d7ae3c3 | 2018-01-25 13:01:09 -0800 | [diff] [blame] | 79 | case kVideoCodecMultiplex: |
kjellander | 02b3d27 | 2016-04-20 05:05:54 -0700 | [diff] [blame] | 80 | case kVideoCodecGeneric: |
| 81 | rtp->codec = kRtpVideoGeneric; |
| 82 | rtp->simulcastIdx = info->codecSpecific.generic.simulcast_idx; |
| 83 | return; |
| 84 | default: |
| 85 | return; |
| 86 | } |
| 87 | } |
perkj | bc75d97 | 2016-05-02 06:31:25 -0700 | [diff] [blame] | 88 | |
kjellander | 02b3d27 | 2016-04-20 05:05:54 -0700 | [diff] [blame] | 89 | } // namespace |
| 90 | |
Niels Möller | bb894ff | 2018-03-15 12:28:53 +0100 | [diff] [blame] | 91 | // State for setting picture id and tl0 pic idx, for VP8 and VP9 |
| 92 | // TODO(nisse): Make these properties not codec specific. |
Åsa Persson | 4bece9a | 2017-10-06 10:04:04 +0200 | [diff] [blame] | 93 | class PayloadRouter::RtpPayloadParams final { |
| 94 | public: |
| 95 | RtpPayloadParams(const uint32_t ssrc, const RtpPayloadState* state) |
| 96 | : ssrc_(ssrc) { |
| 97 | Random random(rtc::TimeMicros()); |
| 98 | state_.picture_id = |
| 99 | state ? state->picture_id : (random.Rand<int16_t>() & 0x7FFF); |
Niels Möller | bb894ff | 2018-03-15 12:28:53 +0100 | [diff] [blame] | 100 | state_.tl0_pic_idx = state ? state->tl0_pic_idx : (random.Rand<uint8_t>()); |
Åsa Persson | 4bece9a | 2017-10-06 10:04:04 +0200 | [diff] [blame] | 101 | } |
| 102 | ~RtpPayloadParams() {} |
| 103 | |
Niels Möller | bb894ff | 2018-03-15 12:28:53 +0100 | [diff] [blame] | 104 | void Set(RTPVideoHeader* rtp_video_header, bool first_frame_in_picture) { |
| 105 | // Always set picture id. Set tl0_pic_idx iff temporal index is set. |
| 106 | if (first_frame_in_picture) { |
| 107 | state_.picture_id = |
| 108 | (static_cast<uint16_t>(state_.picture_id) + 1) & 0x7FFF; |
| 109 | } |
| 110 | if (rtp_video_header->codec == kRtpVideoVp8) { |
Åsa Persson | 4bece9a | 2017-10-06 10:04:04 +0200 | [diff] [blame] | 111 | rtp_video_header->codecHeader.VP8.pictureId = state_.picture_id; |
Niels Möller | bb894ff | 2018-03-15 12:28:53 +0100 | [diff] [blame] | 112 | |
| 113 | if (rtp_video_header->codecHeader.VP8.temporalIdx != kNoTemporalIdx) { |
| 114 | if (rtp_video_header->codecHeader.VP8.temporalIdx == 0) { |
| 115 | ++state_.tl0_pic_idx; |
| 116 | } |
| 117 | rtp_video_header->codecHeader.VP8.tl0PicIdx = state_.tl0_pic_idx; |
| 118 | } |
| 119 | } |
| 120 | if (rtp_video_header->codec == kRtpVideoVp9) { |
| 121 | rtp_video_header->codecHeader.VP9.picture_id = state_.picture_id; |
| 122 | |
| 123 | // Note that in the case that we have no temporal layers but we do have |
| 124 | // spatial layers, packets will carry layering info with a temporal_idx of |
| 125 | // zero, and we then have to set and increment tl0_pic_idx. |
| 126 | if (rtp_video_header->codecHeader.VP9.temporal_idx != kNoTemporalIdx || |
| 127 | rtp_video_header->codecHeader.VP9.spatial_idx != kNoSpatialIdx) { |
| 128 | if (first_frame_in_picture && |
| 129 | (rtp_video_header->codecHeader.VP9.temporal_idx == 0 || |
| 130 | rtp_video_header->codecHeader.VP9.temporal_idx == |
| 131 | kNoTemporalIdx)) { |
| 132 | ++state_.tl0_pic_idx; |
| 133 | } |
| 134 | rtp_video_header->codecHeader.VP9.tl0_pic_idx = state_.tl0_pic_idx; |
| 135 | } |
Åsa Persson | 4bece9a | 2017-10-06 10:04:04 +0200 | [diff] [blame] | 136 | } |
| 137 | } |
| 138 | |
| 139 | uint32_t ssrc() const { return ssrc_; } |
| 140 | |
| 141 | RtpPayloadState state() const { return state_; } |
| 142 | |
| 143 | private: |
| 144 | const uint32_t ssrc_; |
| 145 | RtpPayloadState state_; |
| 146 | }; |
| 147 | |
kjellander | 02b3d27 | 2016-04-20 05:05:54 -0700 | [diff] [blame] | 148 | PayloadRouter::PayloadRouter(const std::vector<RtpRtcp*>& rtp_modules, |
Åsa Persson | 4bece9a | 2017-10-06 10:04:04 +0200 | [diff] [blame] | 149 | const std::vector<uint32_t>& ssrcs, |
| 150 | int payload_type, |
| 151 | const std::map<uint32_t, RtpPayloadState>& states) |
Niels Möller | bb894ff | 2018-03-15 12:28:53 +0100 | [diff] [blame] | 152 | : active_(false), rtp_modules_(rtp_modules), payload_type_(payload_type) { |
Åsa Persson | 4bece9a | 2017-10-06 10:04:04 +0200 | [diff] [blame] | 153 | RTC_DCHECK_EQ(ssrcs.size(), rtp_modules.size()); |
| 154 | // SSRCs are assumed to be sorted in the same order as |rtp_modules|. |
| 155 | for (uint32_t ssrc : ssrcs) { |
| 156 | // Restore state if it previously existed. |
| 157 | const RtpPayloadState* state = nullptr; |
| 158 | auto it = states.find(ssrc); |
| 159 | if (it != states.end()) { |
| 160 | state = &it->second; |
| 161 | } |
| 162 | params_.push_back(RtpPayloadParams(ssrc, state)); |
| 163 | } |
Per | 83d0910 | 2016-04-15 14:59:13 +0200 | [diff] [blame] | 164 | } |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 +0000 | [diff] [blame] | 165 | |
| 166 | PayloadRouter::~PayloadRouter() {} |
| 167 | |
sprang | 1a646ee | 2016-12-01 06:34:11 -0800 | [diff] [blame] | 168 | void PayloadRouter::SetActive(bool active) { |
Tommi | 97888bd | 2016-01-21 23:24:59 +0100 | [diff] [blame] | 169 | rtc::CritScope lock(&crit_); |
Peter Boström | 8b79b07 | 2016-02-26 16:31:37 +0100 | [diff] [blame] | 170 | if (active_ == active) |
| 171 | return; |
Seth Hampson | cc7125f | 2018-02-02 08:46:16 -0800 | [diff] [blame] | 172 | const std::vector<bool> active_modules(rtp_modules_.size(), active); |
| 173 | SetActiveModules(active_modules); |
| 174 | } |
Per | 512ecb3 | 2016-09-23 15:52:06 +0200 | [diff] [blame] | 175 | |
Seth Hampson | cc7125f | 2018-02-02 08:46:16 -0800 | [diff] [blame] | 176 | void PayloadRouter::SetActiveModules(const std::vector<bool> active_modules) { |
| 177 | rtc::CritScope lock(&crit_); |
| 178 | RTC_DCHECK_EQ(rtp_modules_.size(), active_modules.size()); |
| 179 | active_ = false; |
| 180 | for (size_t i = 0; i < active_modules.size(); ++i) { |
| 181 | if (active_modules[i]) { |
| 182 | active_ = true; |
| 183 | } |
| 184 | // Sends a kRtcpByeCode when going from true to false. |
| 185 | rtp_modules_[i]->SetSendingStatus(active_modules[i]); |
| 186 | // If set to false this module won't send media. |
| 187 | rtp_modules_[i]->SetSendingMediaStatus(active_modules[i]); |
Per | 512ecb3 | 2016-09-23 15:52:06 +0200 | [diff] [blame] | 188 | } |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 +0000 | [diff] [blame] | 189 | } |
| 190 | |
sprang | 1a646ee | 2016-12-01 06:34:11 -0800 | [diff] [blame] | 191 | bool PayloadRouter::IsActive() { |
Tommi | 97888bd | 2016-01-21 23:24:59 +0100 | [diff] [blame] | 192 | rtc::CritScope lock(&crit_); |
mflodman@webrtc.org | 47d657b | 2015-02-19 10:29:32 +0000 | [diff] [blame] | 193 | return active_ && !rtp_modules_.empty(); |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 +0000 | [diff] [blame] | 194 | } |
| 195 | |
Åsa Persson | 4bece9a | 2017-10-06 10:04:04 +0200 | [diff] [blame] | 196 | std::map<uint32_t, RtpPayloadState> PayloadRouter::GetRtpPayloadStates() const { |
| 197 | rtc::CritScope lock(&crit_); |
| 198 | std::map<uint32_t, RtpPayloadState> payload_states; |
| 199 | for (const auto& param : params_) { |
| 200 | payload_states[param.ssrc()] = param.state(); |
| 201 | } |
| 202 | return payload_states; |
| 203 | } |
| 204 | |
Sergey Ulanov | 525df3f | 2016-08-02 17:46:41 -0700 | [diff] [blame] | 205 | EncodedImageCallback::Result PayloadRouter::OnEncodedImage( |
| 206 | const EncodedImage& encoded_image, |
| 207 | const CodecSpecificInfo* codec_specific_info, |
| 208 | const RTPFragmentationHeader* fragmentation) { |
Tommi | 97888bd | 2016-01-21 23:24:59 +0100 | [diff] [blame] | 209 | rtc::CritScope lock(&crit_); |
Peter Boström | 8b79b07 | 2016-02-26 16:31:37 +0100 | [diff] [blame] | 210 | RTC_DCHECK(!rtp_modules_.empty()); |
Per | 512ecb3 | 2016-09-23 15:52:06 +0200 | [diff] [blame] | 211 | if (!active_) |
Sergey Ulanov | 525df3f | 2016-08-02 17:46:41 -0700 | [diff] [blame] | 212 | return Result(Result::ERROR_SEND_FAILED); |
mflodman@webrtc.org | 50e2816 | 2015-02-23 07:45:11 +0000 | [diff] [blame] | 213 | |
kjellander | 02b3d27 | 2016-04-20 05:05:54 -0700 | [diff] [blame] | 214 | RTPVideoHeader rtp_video_header; |
| 215 | memset(&rtp_video_header, 0, sizeof(RTPVideoHeader)); |
| 216 | if (codec_specific_info) |
| 217 | CopyCodecSpecific(codec_specific_info, &rtp_video_header); |
| 218 | rtp_video_header.rotation = encoded_image.rotation_; |
ilnik | 00d802b | 2017-04-11 10:34:31 -0700 | [diff] [blame] | 219 | rtp_video_header.content_type = encoded_image.content_type_; |
Niels Möller | c241af9 | 2017-11-10 08:51:29 +0100 | [diff] [blame] | 220 | if (encoded_image.timing_.flags != TimingFrameFlags::kInvalid && |
| 221 | encoded_image.timing_.flags != TimingFrameFlags::kNotTriggered) { |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 222 | rtp_video_header.video_timing.encode_start_delta_ms = |
ilnik | 2edc684 | 2017-07-06 03:06:50 -0700 | [diff] [blame] | 223 | VideoSendTiming::GetDeltaCappedMs( |
| 224 | encoded_image.capture_time_ms_, |
| 225 | encoded_image.timing_.encode_start_ms); |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 226 | rtp_video_header.video_timing.encode_finish_delta_ms = |
ilnik | 2edc684 | 2017-07-06 03:06:50 -0700 | [diff] [blame] | 227 | VideoSendTiming::GetDeltaCappedMs( |
| 228 | encoded_image.capture_time_ms_, |
| 229 | encoded_image.timing_.encode_finish_ms); |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 230 | rtp_video_header.video_timing.packetization_finish_delta_ms = 0; |
| 231 | rtp_video_header.video_timing.pacer_exit_delta_ms = 0; |
Danil Chapovalov | 996eb9e | 2017-10-30 17:14:41 +0100 | [diff] [blame] | 232 | rtp_video_header.video_timing.network_timestamp_delta_ms = 0; |
| 233 | rtp_video_header.video_timing.network2_timestamp_delta_ms = 0; |
Niels Möller | c241af9 | 2017-11-10 08:51:29 +0100 | [diff] [blame] | 234 | rtp_video_header.video_timing.flags = encoded_image.timing_.flags; |
| 235 | } else { |
| 236 | rtp_video_header.video_timing.flags = TimingFrameFlags::kInvalid; |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 237 | } |
isheriff | 6b4b5f3 | 2016-06-08 00:24:21 -0700 | [diff] [blame] | 238 | rtp_video_header.playout_delay = encoded_image.playout_delay_; |
kjellander | 02b3d27 | 2016-04-20 05:05:54 -0700 | [diff] [blame] | 239 | |
sergeyu | 7b9feee | 2016-11-17 16:16:14 -0800 | [diff] [blame] | 240 | int stream_index = rtp_video_header.simulcastIdx; |
| 241 | RTC_DCHECK_LT(stream_index, rtp_modules_.size()); |
Niels Möller | bb894ff | 2018-03-15 12:28:53 +0100 | [diff] [blame] | 242 | |
| 243 | // Sets picture id and tl0 pic idx. |
| 244 | const bool first_frame_in_picture = |
| 245 | (codec_specific_info && codec_specific_info->codecType == kVideoCodecVP9) |
| 246 | ? codec_specific_info->codecSpecific.VP9.first_frame_in_picture |
| 247 | : true; |
| 248 | params_[stream_index].Set(&rtp_video_header, first_frame_in_picture); |
| 249 | |
Sergey Ulanov | 525df3f | 2016-08-02 17:46:41 -0700 | [diff] [blame] | 250 | uint32_t frame_id; |
Seth Hampson | cc7125f | 2018-02-02 08:46:16 -0800 | [diff] [blame] | 251 | if (!rtp_modules_[stream_index]->Sending()) { |
| 252 | // The payload router could be active but this module isn't sending. |
| 253 | return Result(Result::ERROR_SEND_FAILED); |
| 254 | } |
sergeyu | 7b9feee | 2016-11-17 16:16:14 -0800 | [diff] [blame] | 255 | bool send_result = rtp_modules_[stream_index]->SendOutgoingData( |
kjellander | 02b3d27 | 2016-04-20 05:05:54 -0700 | [diff] [blame] | 256 | encoded_image._frameType, payload_type_, encoded_image._timeStamp, |
| 257 | encoded_image.capture_time_ms_, encoded_image._buffer, |
Sergey Ulanov | 525df3f | 2016-08-02 17:46:41 -0700 | [diff] [blame] | 258 | encoded_image._length, fragmentation, &rtp_video_header, &frame_id); |
sergeyu | 7b9feee | 2016-11-17 16:16:14 -0800 | [diff] [blame] | 259 | if (!send_result) |
Sergey Ulanov | 525df3f | 2016-08-02 17:46:41 -0700 | [diff] [blame] | 260 | return Result(Result::ERROR_SEND_FAILED); |
| 261 | |
| 262 | return Result(Result::OK, frame_id); |
mflodman@webrtc.org | a4ef2ce | 2015-02-12 09:54:18 +0000 | [diff] [blame] | 263 | } |
| 264 | |
sprang | 1a646ee | 2016-12-01 06:34:11 -0800 | [diff] [blame] | 265 | void PayloadRouter::OnBitrateAllocationUpdated( |
| 266 | const BitrateAllocation& bitrate) { |
| 267 | rtc::CritScope lock(&crit_); |
| 268 | if (IsActive()) { |
| 269 | if (rtp_modules_.size() == 1) { |
| 270 | // If spatial scalability is enabled, it is covered by a single stream. |
| 271 | rtp_modules_[0]->SetVideoBitrateAllocation(bitrate); |
| 272 | } else { |
| 273 | // Simulcast is in use, split the BitrateAllocation into one struct per |
| 274 | // rtp stream, moving over the temporal layer allocation. |
| 275 | for (size_t si = 0; si < rtp_modules_.size(); ++si) { |
sprang | d0fc37a | 2017-06-22 05:40:25 -0700 | [diff] [blame] | 276 | // Don't send empty TargetBitrate messages on streams not being relayed. |
Seth Hampson | 46e31ba | 2018-01-18 10:39:54 -0800 | [diff] [blame] | 277 | if (!bitrate.IsSpatialLayerUsed(si)) { |
| 278 | // The next spatial layer could be used if the current one is |
| 279 | // inactive. |
| 280 | continue; |
| 281 | } |
sprang | d0fc37a | 2017-06-22 05:40:25 -0700 | [diff] [blame] | 282 | |
sprang | 1a646ee | 2016-12-01 06:34:11 -0800 | [diff] [blame] | 283 | BitrateAllocation layer_bitrate; |
erikvarga@webrtc.org | 01f2ec3 | 2017-11-15 14:58:23 +0100 | [diff] [blame] | 284 | for (int tl = 0; tl < kMaxTemporalStreams; ++tl) { |
| 285 | if (bitrate.HasBitrate(si, tl)) |
| 286 | layer_bitrate.SetBitrate(0, tl, bitrate.GetBitrate(si, tl)); |
| 287 | } |
sprang | 1a646ee | 2016-12-01 06:34:11 -0800 | [diff] [blame] | 288 | rtp_modules_[si]->SetVideoBitrateAllocation(layer_bitrate); |
| 289 | } |
| 290 | } |
| 291 | } |
| 292 | } |
| 293 | |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 +0000 | [diff] [blame] | 294 | } // namespace webrtc |