Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2019 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
| 11 | #include "pc/video_rtp_receiver.h" |
| 12 | |
| 13 | #include <stddef.h> |
Jonas Olsson | a4d8737 | 2019-07-05 19:08:33 +0200 | [diff] [blame] | 14 | |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 15 | #include <utility> |
| 16 | #include <vector> |
| 17 | |
| 18 | #include "api/media_stream_proxy.h" |
| 19 | #include "api/media_stream_track_proxy.h" |
| 20 | #include "api/video_track_source_proxy.h" |
Ruslan Burakov | 428dcb2 | 2019-04-18 17:49:49 +0200 | [diff] [blame] | 21 | #include "pc/jitter_buffer_delay.h" |
| 22 | #include "pc/jitter_buffer_delay_proxy.h" |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 23 | #include "pc/media_stream.h" |
| 24 | #include "pc/video_track.h" |
| 25 | #include "rtc_base/checks.h" |
| 26 | #include "rtc_base/location.h" |
| 27 | #include "rtc_base/logging.h" |
| 28 | #include "rtc_base/trace_event.h" |
| 29 | |
| 30 | namespace webrtc { |
| 31 | |
| 32 | VideoRtpReceiver::VideoRtpReceiver(rtc::Thread* worker_thread, |
| 33 | std::string receiver_id, |
| 34 | std::vector<std::string> stream_ids) |
| 35 | : VideoRtpReceiver(worker_thread, |
| 36 | receiver_id, |
| 37 | CreateStreamsFromIds(std::move(stream_ids))) {} |
| 38 | |
| 39 | VideoRtpReceiver::VideoRtpReceiver( |
| 40 | rtc::Thread* worker_thread, |
| 41 | const std::string& receiver_id, |
| 42 | const std::vector<rtc::scoped_refptr<MediaStreamInterface>>& streams) |
| 43 | : worker_thread_(worker_thread), |
| 44 | id_(receiver_id), |
Markus Handell | d5e2f21 | 2019-11-26 09:30:08 +0100 | [diff] [blame] | 45 | source_(new RefCountedObject<VideoRtpTrackSource>(this)), |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 46 | track_(VideoTrackProxy::Create( |
| 47 | rtc::Thread::Current(), |
| 48 | worker_thread, |
| 49 | VideoTrack::Create( |
| 50 | receiver_id, |
| 51 | VideoTrackSourceProxy::Create(rtc::Thread::Current(), |
| 52 | worker_thread, |
| 53 | source_), |
| 54 | worker_thread))), |
Ruslan Burakov | 428dcb2 | 2019-04-18 17:49:49 +0200 | [diff] [blame] | 55 | attachment_id_(GenerateUniqueId()), |
| 56 | delay_(JitterBufferDelayProxy::Create( |
| 57 | rtc::Thread::Current(), |
| 58 | worker_thread, |
| 59 | new rtc::RefCountedObject<JitterBufferDelay>(worker_thread))) { |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 60 | RTC_DCHECK(worker_thread_); |
| 61 | SetStreams(streams); |
| 62 | source_->SetState(MediaSourceInterface::kLive); |
| 63 | } |
| 64 | |
| 65 | VideoRtpReceiver::~VideoRtpReceiver() { |
| 66 | // Since cricket::VideoRenderer is not reference counted, |
| 67 | // we need to remove it from the channel before we are deleted. |
| 68 | Stop(); |
Markus Handell | d5e2f21 | 2019-11-26 09:30:08 +0100 | [diff] [blame] | 69 | // Make sure we can't be called by the |source_| anymore. |
| 70 | worker_thread_->Invoke<void>(RTC_FROM_HERE, |
| 71 | [this] { source_->ClearCallback(); }); |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 72 | } |
| 73 | |
| 74 | std::vector<std::string> VideoRtpReceiver::stream_ids() const { |
| 75 | std::vector<std::string> stream_ids(streams_.size()); |
| 76 | for (size_t i = 0; i < streams_.size(); ++i) |
| 77 | stream_ids[i] = streams_[i]->id(); |
| 78 | return stream_ids; |
| 79 | } |
| 80 | |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 81 | RtpParameters VideoRtpReceiver::GetParameters() const { |
Saurav Das | 7262fc2 | 2019-09-11 16:23:05 -0700 | [diff] [blame] | 82 | if (!media_channel_ || stopped_) { |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 83 | return RtpParameters(); |
| 84 | } |
| 85 | return worker_thread_->Invoke<RtpParameters>(RTC_FROM_HERE, [&] { |
Saurav Das | 749f660 | 2019-12-04 09:31:36 -0800 | [diff] [blame] | 86 | return ssrc_ ? media_channel_->GetRtpReceiveParameters(*ssrc_) |
| 87 | : media_channel_->GetDefaultRtpReceiveParameters(); |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 88 | }); |
| 89 | } |
| 90 | |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 91 | void VideoRtpReceiver::SetFrameDecryptor( |
| 92 | rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor) { |
| 93 | frame_decryptor_ = std::move(frame_decryptor); |
| 94 | // Special Case: Set the frame decryptor to any value on any existing channel. |
| 95 | if (media_channel_ && ssrc_.has_value() && !stopped_) { |
| 96 | worker_thread_->Invoke<void>(RTC_FROM_HERE, [&] { |
| 97 | media_channel_->SetFrameDecryptor(*ssrc_, frame_decryptor_); |
| 98 | }); |
| 99 | } |
| 100 | } |
| 101 | |
| 102 | rtc::scoped_refptr<FrameDecryptorInterface> |
| 103 | VideoRtpReceiver::GetFrameDecryptor() const { |
| 104 | return frame_decryptor_; |
| 105 | } |
| 106 | |
Marina Ciocea | 412a31b | 2020-02-28 16:02:06 +0100 | [diff] [blame] | 107 | void VideoRtpReceiver::SetDepacketizerToDecoderFrameTransformer( |
| 108 | rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) { |
| 109 | worker_thread_->Invoke<void>(RTC_FROM_HERE, [&] { |
| 110 | RTC_DCHECK_RUN_ON(worker_thread_); |
| 111 | frame_transformer_ = std::move(frame_transformer); |
| 112 | if (media_channel_ && ssrc_.has_value() && !stopped_) { |
| 113 | media_channel_->SetDepacketizerToDecoderFrameTransformer( |
| 114 | *ssrc_, frame_transformer_); |
| 115 | } |
| 116 | }); |
| 117 | } |
| 118 | |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 119 | void VideoRtpReceiver::Stop() { |
| 120 | // TODO(deadbeef): Need to do more here to fully stop receiving packets. |
| 121 | if (stopped_) { |
| 122 | return; |
| 123 | } |
| 124 | source_->SetState(MediaSourceInterface::kEnded); |
Saurav Das | 7262fc2 | 2019-09-11 16:23:05 -0700 | [diff] [blame] | 125 | if (!media_channel_) { |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 126 | RTC_LOG(LS_WARNING) << "VideoRtpReceiver::Stop: No video channel exists."; |
| 127 | } else { |
Markus Handell | 9c27ed2 | 2019-12-04 12:57:58 +0100 | [diff] [blame] | 128 | // Allow that SetSink fails. This is the normal case when the underlying |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 129 | // media channel has already been deleted. |
Markus Handell | 9c27ed2 | 2019-12-04 12:57:58 +0100 | [diff] [blame] | 130 | worker_thread_->Invoke<void>(RTC_FROM_HERE, [&] { |
| 131 | RTC_DCHECK_RUN_ON(worker_thread_); |
| 132 | SetSink(nullptr); |
| 133 | }); |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 134 | } |
Ruslan Burakov | 428dcb2 | 2019-04-18 17:49:49 +0200 | [diff] [blame] | 135 | delay_->OnStop(); |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 136 | stopped_ = true; |
| 137 | } |
| 138 | |
Saurav Das | 7262fc2 | 2019-09-11 16:23:05 -0700 | [diff] [blame] | 139 | void VideoRtpReceiver::RestartMediaChannel(absl::optional<uint32_t> ssrc) { |
| 140 | RTC_DCHECK(media_channel_); |
| 141 | if (!stopped_ && ssrc_ == ssrc) { |
| 142 | return; |
| 143 | } |
Markus Handell | 9c27ed2 | 2019-12-04 12:57:58 +0100 | [diff] [blame] | 144 | worker_thread_->Invoke<void>(RTC_FROM_HERE, [&] { |
| 145 | RTC_DCHECK_RUN_ON(worker_thread_); |
| 146 | if (!stopped_) { |
| 147 | SetSink(nullptr); |
| 148 | } |
| 149 | bool encoded_sink_enabled = saved_encoded_sink_enabled_; |
| 150 | SetEncodedSinkEnabled(false); |
| 151 | stopped_ = false; |
| 152 | |
| 153 | ssrc_ = ssrc; |
| 154 | |
| 155 | SetSink(source_->sink()); |
| 156 | if (encoded_sink_enabled) { |
| 157 | SetEncodedSinkEnabled(true); |
| 158 | } |
Marina Ciocea | 412a31b | 2020-02-28 16:02:06 +0100 | [diff] [blame] | 159 | |
| 160 | if (frame_transformer_ && media_channel_ && ssrc_.has_value()) { |
| 161 | media_channel_->SetDepacketizerToDecoderFrameTransformer( |
| 162 | *ssrc_, frame_transformer_); |
| 163 | } |
Markus Handell | 9c27ed2 | 2019-12-04 12:57:58 +0100 | [diff] [blame] | 164 | }); |
Saurav Das | 7262fc2 | 2019-09-11 16:23:05 -0700 | [diff] [blame] | 165 | |
| 166 | // Attach any existing frame decryptor to the media channel. |
| 167 | MaybeAttachFrameDecryptorToMediaChannel( |
| 168 | ssrc, worker_thread_, frame_decryptor_, media_channel_, stopped_); |
| 169 | // TODO(bugs.webrtc.org/8694): Stop using 0 to mean unsignalled SSRC |
| 170 | // value. |
| 171 | delay_->OnStart(media_channel_, ssrc.value_or(0)); |
| 172 | } |
| 173 | |
Markus Handell | 9c27ed2 | 2019-12-04 12:57:58 +0100 | [diff] [blame] | 174 | void VideoRtpReceiver::SetSink(rtc::VideoSinkInterface<VideoFrame>* sink) { |
Markus Handell | fc90797 | 2019-12-05 12:41:40 +0100 | [diff] [blame] | 175 | RTC_DCHECK(media_channel_); |
Saurav Das | 749f660 | 2019-12-04 09:31:36 -0800 | [diff] [blame] | 176 | if (ssrc_) { |
| 177 | media_channel_->SetSink(*ssrc_, sink); |
| 178 | return; |
| 179 | } |
| 180 | media_channel_->SetDefaultSink(sink); |
Markus Handell | 9c27ed2 | 2019-12-04 12:57:58 +0100 | [diff] [blame] | 181 | } |
| 182 | |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 183 | void VideoRtpReceiver::SetupMediaChannel(uint32_t ssrc) { |
| 184 | if (!media_channel_) { |
| 185 | RTC_LOG(LS_ERROR) |
| 186 | << "VideoRtpReceiver::SetupMediaChannel: No video channel exists."; |
| 187 | } |
Saurav Das | 7262fc2 | 2019-09-11 16:23:05 -0700 | [diff] [blame] | 188 | RestartMediaChannel(ssrc); |
| 189 | } |
Ruslan Burakov | 493a650 | 2019-02-27 15:32:48 +0100 | [diff] [blame] | 190 | |
Saurav Das | 7262fc2 | 2019-09-11 16:23:05 -0700 | [diff] [blame] | 191 | void VideoRtpReceiver::SetupUnsignaledMediaChannel() { |
| 192 | if (!media_channel_) { |
| 193 | RTC_LOG(LS_ERROR) << "VideoRtpReceiver::SetupUnsignaledMediaChannel: No " |
| 194 | "video channel exists."; |
| 195 | } |
| 196 | RestartMediaChannel(absl::nullopt); |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 197 | } |
| 198 | |
| 199 | void VideoRtpReceiver::set_stream_ids(std::vector<std::string> stream_ids) { |
| 200 | SetStreams(CreateStreamsFromIds(std::move(stream_ids))); |
| 201 | } |
| 202 | |
| 203 | void VideoRtpReceiver::SetStreams( |
| 204 | const std::vector<rtc::scoped_refptr<MediaStreamInterface>>& streams) { |
| 205 | // Remove remote track from any streams that are going away. |
| 206 | for (const auto& existing_stream : streams_) { |
| 207 | bool removed = true; |
| 208 | for (const auto& stream : streams) { |
| 209 | if (existing_stream->id() == stream->id()) { |
| 210 | RTC_DCHECK_EQ(existing_stream.get(), stream.get()); |
| 211 | removed = false; |
| 212 | break; |
| 213 | } |
| 214 | } |
| 215 | if (removed) { |
| 216 | existing_stream->RemoveTrack(track_); |
| 217 | } |
| 218 | } |
| 219 | // Add remote track to any streams that are new. |
| 220 | for (const auto& stream : streams) { |
| 221 | bool added = true; |
| 222 | for (const auto& existing_stream : streams_) { |
| 223 | if (stream->id() == existing_stream->id()) { |
| 224 | RTC_DCHECK_EQ(stream.get(), existing_stream.get()); |
| 225 | added = false; |
| 226 | break; |
| 227 | } |
| 228 | } |
| 229 | if (added) { |
| 230 | stream->AddTrack(track_); |
| 231 | } |
| 232 | } |
| 233 | streams_ = streams; |
| 234 | } |
| 235 | |
| 236 | void VideoRtpReceiver::SetObserver(RtpReceiverObserverInterface* observer) { |
| 237 | observer_ = observer; |
| 238 | // Deliver any notifications the observer may have missed by being set late. |
| 239 | if (received_first_packet_ && observer_) { |
| 240 | observer_->OnFirstPacketReceived(media_type()); |
| 241 | } |
| 242 | } |
| 243 | |
Ruslan Burakov | 4bac79e | 2019-04-03 19:55:33 +0200 | [diff] [blame] | 244 | void VideoRtpReceiver::SetJitterBufferMinimumDelay( |
| 245 | absl::optional<double> delay_seconds) { |
Ruslan Burakov | 428dcb2 | 2019-04-18 17:49:49 +0200 | [diff] [blame] | 246 | delay_->Set(delay_seconds); |
Ruslan Burakov | 4bac79e | 2019-04-03 19:55:33 +0200 | [diff] [blame] | 247 | } |
| 248 | |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 249 | void VideoRtpReceiver::SetMediaChannel(cricket::MediaChannel* media_channel) { |
| 250 | RTC_DCHECK(media_channel == nullptr || |
| 251 | media_channel->media_type() == media_type()); |
Markus Handell | 9c27ed2 | 2019-12-04 12:57:58 +0100 | [diff] [blame] | 252 | worker_thread_->Invoke<void>(RTC_FROM_HERE, [&] { |
| 253 | RTC_DCHECK_RUN_ON(worker_thread_); |
| 254 | bool encoded_sink_enabled = saved_encoded_sink_enabled_; |
| 255 | if (encoded_sink_enabled && media_channel_) { |
| 256 | // Turn off the old sink, if any. |
| 257 | SetEncodedSinkEnabled(false); |
| 258 | } |
| 259 | |
| 260 | media_channel_ = static_cast<cricket::VideoMediaChannel*>(media_channel); |
| 261 | |
| 262 | if (media_channel_) { |
| 263 | if (saved_generate_keyframe_) { |
| 264 | // TODO(bugs.webrtc.org/8694): Stop using 0 to mean unsignalled SSRC |
| 265 | media_channel_->GenerateKeyFrame(ssrc_.value_or(0)); |
| 266 | saved_generate_keyframe_ = false; |
| 267 | } |
| 268 | if (encoded_sink_enabled) { |
| 269 | SetEncodedSinkEnabled(true); |
| 270 | } |
| 271 | } |
| 272 | }); |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 273 | } |
| 274 | |
| 275 | void VideoRtpReceiver::NotifyFirstPacketReceived() { |
| 276 | if (observer_) { |
| 277 | observer_->OnFirstPacketReceived(media_type()); |
| 278 | } |
| 279 | received_first_packet_ = true; |
| 280 | } |
| 281 | |
| 282 | std::vector<RtpSource> VideoRtpReceiver::GetSources() const { |
| 283 | if (!media_channel_ || !ssrc_ || stopped_) { |
| 284 | return {}; |
| 285 | } |
| 286 | return worker_thread_->Invoke<std::vector<RtpSource>>( |
| 287 | RTC_FROM_HERE, [&] { return media_channel_->GetSources(*ssrc_); }); |
| 288 | } |
| 289 | |
Markus Handell | d5e2f21 | 2019-11-26 09:30:08 +0100 | [diff] [blame] | 290 | void VideoRtpReceiver::OnGenerateKeyFrame() { |
| 291 | RTC_DCHECK_RUN_ON(worker_thread_); |
Markus Handell | fc90797 | 2019-12-05 12:41:40 +0100 | [diff] [blame] | 292 | if (!media_channel_) { |
| 293 | RTC_LOG(LS_ERROR) |
| 294 | << "VideoRtpReceiver::OnGenerateKeyFrame: No video channel exists."; |
| 295 | return; |
| 296 | } |
Markus Handell | 9c27ed2 | 2019-12-04 12:57:58 +0100 | [diff] [blame] | 297 | // TODO(bugs.webrtc.org/8694): Stop using 0 to mean unsignalled SSRC |
| 298 | media_channel_->GenerateKeyFrame(ssrc_.value_or(0)); |
| 299 | // We need to remember to request generation of a new key frame if the media |
| 300 | // channel changes, because there's no feedback whether the keyframe |
| 301 | // generation has completed on the channel. |
| 302 | saved_generate_keyframe_ = true; |
Markus Handell | d5e2f21 | 2019-11-26 09:30:08 +0100 | [diff] [blame] | 303 | } |
| 304 | |
| 305 | void VideoRtpReceiver::OnEncodedSinkEnabled(bool enable) { |
| 306 | RTC_DCHECK_RUN_ON(worker_thread_); |
Markus Handell | 9c27ed2 | 2019-12-04 12:57:58 +0100 | [diff] [blame] | 307 | SetEncodedSinkEnabled(enable); |
| 308 | // Always save the latest state of the callback in case the media_channel_ |
| 309 | // changes. |
| 310 | saved_encoded_sink_enabled_ = enable; |
| 311 | } |
| 312 | |
| 313 | void VideoRtpReceiver::SetEncodedSinkEnabled(bool enable) { |
| 314 | if (media_channel_) { |
| 315 | if (enable) { |
| 316 | // TODO(bugs.webrtc.org/8694): Stop using 0 to mean unsignalled SSRC |
| 317 | auto source = source_; |
| 318 | media_channel_->SetRecordableEncodedFrameCallback( |
| 319 | ssrc_.value_or(0), |
| 320 | [source = std::move(source)](const RecordableEncodedFrame& frame) { |
| 321 | source->BroadcastRecordableEncodedFrame(frame); |
| 322 | }); |
| 323 | } else { |
| 324 | // TODO(bugs.webrtc.org/8694): Stop using 0 to mean unsignalled SSRC |
| 325 | media_channel_->ClearRecordableEncodedFrameCallback(ssrc_.value_or(0)); |
| 326 | } |
| 327 | } |
Markus Handell | d5e2f21 | 2019-11-26 09:30:08 +0100 | [diff] [blame] | 328 | } |
| 329 | |
Ruslan Burakov | 501bfba | 2019-02-11 10:29:19 +0100 | [diff] [blame] | 330 | } // namespace webrtc |