pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 10 | #include "video/video_send_stream.h" |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 11 | |
perkj | 71ee44c | 2016-06-15 00:47:53 -0700 | [diff] [blame] | 12 | #include <utility> |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 13 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 14 | #include "modules/rtp_rtcp/source/rtp_sender.h" |
Sebastian Jansson | 8e0b15b | 2018-04-18 19:19:22 +0200 | [diff] [blame^] | 15 | #include "video/video_send_stream_impl.h" |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 16 | |
| 17 | namespace webrtc { |
mflodman | 949c2f0 | 2015-10-16 02:31:11 -0700 | [diff] [blame] | 18 | |
Ying Wang | 3b790f3 | 2018-01-19 17:58:57 +0100 | [diff] [blame] | 19 | |
Per | 83d0910 | 2016-04-15 14:59:13 +0200 | [diff] [blame] | 20 | namespace { |
| 21 | |
Ilya Nikolaevskiy | 1d037ae | 2018-03-15 15:46:17 +0100 | [diff] [blame] | 22 | size_t CalculateMaxHeaderSize(const VideoSendStream::Config::Rtp& config) { |
| 23 | size_t header_size = kRtpHeaderSize; |
| 24 | size_t extensions_size = 0; |
| 25 | size_t fec_extensions_size = 0; |
| 26 | if (config.extensions.size() > 0) { |
| 27 | RtpHeaderExtensionMap extensions_map(config.extensions); |
| 28 | extensions_size = |
| 29 | extensions_map.GetTotalLengthInBytes(RTPSender::VideoExtensionSizes()); |
| 30 | fec_extensions_size = |
| 31 | extensions_map.GetTotalLengthInBytes(RTPSender::FecExtensionSizes()); |
| 32 | } |
| 33 | header_size += extensions_size; |
| 34 | if (config.flexfec.payload_type >= 0) { |
| 35 | // All FEC extensions again plus maximum FlexFec overhead. |
| 36 | header_size += fec_extensions_size + 32; |
| 37 | } else { |
| 38 | if (config.ulpfec.ulpfec_payload_type >= 0) { |
| 39 | // Header with all the FEC extensions will be repeated plus maximum |
| 40 | // UlpFec overhead. |
| 41 | header_size += fec_extensions_size + 18; |
| 42 | } |
| 43 | if (config.ulpfec.red_payload_type >= 0) { |
| 44 | header_size += 1; // RED header. |
| 45 | } |
| 46 | } |
| 47 | // Additional room for Rtx. |
| 48 | if (config.rtx.payload_type >= 0) |
| 49 | header_size += kRtxHeaderSize; |
| 50 | return header_size; |
| 51 | } |
| 52 | |
Peter Boström | e449915 | 2016-02-05 11:13:28 +0100 | [diff] [blame] | 53 | } // namespace |
| 54 | |
pbos@webrtc.org | 024e4d5 | 2014-05-15 10:03:24 +0000 | [diff] [blame] | 55 | namespace internal { |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 56 | |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 57 | |
pbos@webrtc.org | 2bb1bda | 2014-07-07 13:06:48 +0000 | [diff] [blame] | 58 | VideoSendStream::VideoSendStream( |
Peter Boström | 45553ae | 2015-05-08 13:54:38 +0200 | [diff] [blame] | 59 | int num_cpu_cores, |
Peter Boström | f16fcbe | 2015-04-30 12:16:05 +0200 | [diff] [blame] | 60 | ProcessThread* module_process_thread, |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 61 | rtc::TaskQueue* worker_queue, |
mflodman | e378702 | 2015-10-21 13:24:28 +0200 | [diff] [blame] | 62 | CallStats* call_stats, |
nisse | b8f9a32 | 2017-03-27 05:36:15 -0700 | [diff] [blame] | 63 | RtpTransportControllerSendInterface* transport, |
mflodman | 0e7e259 | 2015-11-12 21:02:42 -0800 | [diff] [blame] | 64 | BitrateAllocator* bitrate_allocator, |
asapersson | 35151f3 | 2016-05-02 23:44:01 -0700 | [diff] [blame] | 65 | SendDelayStats* send_delay_stats, |
terelius | adafe0b | 2016-05-26 01:58:40 -0700 | [diff] [blame] | 66 | RtcEventLog* event_log, |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 67 | VideoSendStream::Config config, |
| 68 | VideoEncoderConfig encoder_config, |
Åsa Persson | 4bece9a | 2017-10-06 10:04:04 +0200 | [diff] [blame] | 69 | const std::map<uint32_t, RtpState>& suspended_ssrcs, |
Ying Wang | 3b790f3 | 2018-01-19 17:58:57 +0100 | [diff] [blame] | 70 | const std::map<uint32_t, RtpPayloadState>& suspended_payload_states, |
Sebastian Jansson | 25e5110 | 2018-03-01 15:56:47 +0100 | [diff] [blame] | 71 | std::unique_ptr<FecController> fec_controller, |
| 72 | RateLimiter* retransmission_limiter) |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 73 | : worker_queue_(worker_queue), |
| 74 | thread_sync_event_(false /* manual_reset */, false), |
| 75 | stats_proxy_(Clock::GetRealTimeClock(), |
sprang | b4a1ae5 | 2015-12-03 08:10:08 -0800 | [diff] [blame] | 76 | config, |
| 77 | encoder_config.content_type), |
sprang | f24a064 | 2017-02-28 13:23:26 -0800 | [diff] [blame] | 78 | config_(std::move(config)), |
| 79 | content_type_(encoder_config.content_type) { |
Niels Möller | 73f29cb | 2018-01-31 16:09:31 +0100 | [diff] [blame] | 80 | video_stream_encoder_ = rtc::MakeUnique<VideoStreamEncoder>( |
| 81 | num_cpu_cores, &stats_proxy_, |
| 82 | config_.encoder_settings, |
| 83 | config_.pre_encode_callback, |
Niels Möller | d1f7eb6 | 2018-03-28 16:40:58 +0200 | [diff] [blame] | 84 | rtc::MakeUnique<OveruseFrameDetector>(&stats_proxy_)); |
Sebastian Jansson | 06b83aa | 2018-02-28 13:03:46 +0100 | [diff] [blame] | 85 | // TODO(srte): Initialization should not be done posted on a task queue. |
| 86 | // Note that the posted task must not outlive this scope since the closure |
| 87 | // references local variables. |
| 88 | worker_queue_->PostTask(rtc::NewClosure( |
| 89 | [this, call_stats, transport, bitrate_allocator, send_delay_stats, |
| 90 | event_log, &suspended_ssrcs, &encoder_config, &suspended_payload_states, |
Sebastian Jansson | 25e5110 | 2018-03-01 15:56:47 +0100 | [diff] [blame] | 91 | &fec_controller, retransmission_limiter]() { |
Sebastian Jansson | 06b83aa | 2018-02-28 13:03:46 +0100 | [diff] [blame] | 92 | send_stream_.reset(new VideoSendStreamImpl( |
| 93 | &stats_proxy_, worker_queue_, call_stats, transport, |
| 94 | bitrate_allocator, send_delay_stats, video_stream_encoder_.get(), |
| 95 | event_log, &config_, encoder_config.max_bitrate_bps, |
| 96 | encoder_config.bitrate_priority, suspended_ssrcs, |
| 97 | suspended_payload_states, encoder_config.content_type, |
Sebastian Jansson | 25e5110 | 2018-03-01 15:56:47 +0100 | [diff] [blame] | 98 | std::move(fec_controller), retransmission_limiter)); |
Sebastian Jansson | 06b83aa | 2018-02-28 13:03:46 +0100 | [diff] [blame] | 99 | }, |
| 100 | [this]() { thread_sync_event_.Set(); })); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 101 | |
| 102 | // Wait for ConstructionTask to complete so that |send_stream_| can be used. |
| 103 | // |module_process_thread| must be registered and deregistered on the thread |
| 104 | // it was created on. |
| 105 | thread_sync_event_.Wait(rtc::Event::kForever); |
| 106 | send_stream_->RegisterProcessThread(module_process_thread); |
sprang | 44b3ef6 | 2017-01-13 07:30:25 -0800 | [diff] [blame] | 107 | // TODO(sprang): Enable this also for regular video calls if it works well. |
| 108 | if (encoder_config.content_type == VideoEncoderConfig::ContentType::kScreen) { |
| 109 | // Only signal target bitrate for screenshare streams, for now. |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 110 | video_stream_encoder_->SetBitrateObserver(send_stream_.get()); |
sprang | 44b3ef6 | 2017-01-13 07:30:25 -0800 | [diff] [blame] | 111 | } |
Ying Wang | 38a31b0 | 2017-12-21 12:26:19 +0000 | [diff] [blame] | 112 | |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 113 | ReconfigureVideoEncoder(std::move(encoder_config)); |
| 114 | } |
| 115 | |
| 116 | VideoSendStream::~VideoSendStream() { |
| 117 | RTC_DCHECK_RUN_ON(&thread_checker_); |
| 118 | RTC_DCHECK(!send_stream_); |
| 119 | } |
| 120 | |
Seth Hampson | cc7125f | 2018-02-02 08:46:16 -0800 | [diff] [blame] | 121 | void VideoSendStream::UpdateActiveSimulcastLayers( |
| 122 | const std::vector<bool> active_layers) { |
| 123 | RTC_DCHECK_RUN_ON(&thread_checker_); |
| 124 | RTC_LOG(LS_INFO) << "VideoSendStream::UpdateActiveSimulcastLayers"; |
| 125 | VideoSendStreamImpl* send_stream = send_stream_.get(); |
| 126 | worker_queue_->PostTask([this, send_stream, active_layers] { |
| 127 | send_stream->UpdateActiveSimulcastLayers(active_layers); |
| 128 | thread_sync_event_.Set(); |
| 129 | }); |
| 130 | |
| 131 | thread_sync_event_.Wait(rtc::Event::kForever); |
| 132 | } |
| 133 | |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 134 | void VideoSendStream::Start() { |
| 135 | RTC_DCHECK_RUN_ON(&thread_checker_); |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 136 | RTC_LOG(LS_INFO) << "VideoSendStream::Start"; |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 137 | VideoSendStreamImpl* send_stream = send_stream_.get(); |
| 138 | worker_queue_->PostTask([this, send_stream] { |
| 139 | send_stream->Start(); |
| 140 | thread_sync_event_.Set(); |
| 141 | }); |
| 142 | |
| 143 | // It is expected that after VideoSendStream::Start has been called, incoming |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 144 | // frames are not dropped in VideoStreamEncoder. To ensure this, Start has to |
| 145 | // be synchronized. |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 146 | thread_sync_event_.Wait(rtc::Event::kForever); |
| 147 | } |
| 148 | |
| 149 | void VideoSendStream::Stop() { |
| 150 | RTC_DCHECK_RUN_ON(&thread_checker_); |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 151 | RTC_LOG(LS_INFO) << "VideoSendStream::Stop"; |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 152 | VideoSendStreamImpl* send_stream = send_stream_.get(); |
| 153 | worker_queue_->PostTask([send_stream] { send_stream->Stop(); }); |
| 154 | } |
| 155 | |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 156 | void VideoSendStream::SetSource( |
perkj | 803d97f | 2016-11-01 11:45:46 -0700 | [diff] [blame] | 157 | rtc::VideoSourceInterface<webrtc::VideoFrame>* source, |
| 158 | const DegradationPreference& degradation_preference) { |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 159 | RTC_DCHECK_RUN_ON(&thread_checker_); |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 160 | video_stream_encoder_->SetSource(source, degradation_preference); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 161 | } |
| 162 | |
| 163 | void VideoSendStream::ReconfigureVideoEncoder(VideoEncoderConfig config) { |
perkj | fa10b55 | 2016-10-02 23:45:26 -0700 | [diff] [blame] | 164 | // TODO(perkj): Some test cases in VideoSendStreamTest call |
| 165 | // ReconfigureVideoEncoder from the network thread. |
| 166 | // RTC_DCHECK_RUN_ON(&thread_checker_); |
sprang | f24a064 | 2017-02-28 13:23:26 -0800 | [diff] [blame] | 167 | RTC_DCHECK(content_type_ == config.content_type); |
Ilya Nikolaevskiy | 1d037ae | 2018-03-15 15:46:17 +0100 | [diff] [blame] | 168 | video_stream_encoder_->ConfigureEncoder( |
| 169 | std::move(config), |
| 170 | config_.rtp.max_packet_size - CalculateMaxHeaderSize(config_.rtp), |
| 171 | config_.rtp.nack.rtp_history_ms > 0); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 172 | } |
| 173 | |
| 174 | VideoSendStream::Stats VideoSendStream::GetStats() { |
| 175 | // TODO(perkj, solenberg): Some test cases in EndToEndTest call GetStats from |
| 176 | // a network thread. See comment in Call::GetStats(). |
| 177 | // RTC_DCHECK_RUN_ON(&thread_checker_); |
| 178 | return stats_proxy_.GetStats(); |
| 179 | } |
| 180 | |
Sebastian Jansson | a45c8da | 2018-01-16 10:55:29 +0100 | [diff] [blame] | 181 | rtc::Optional<float> VideoSendStream::GetPacingFactorOverride() const { |
| 182 | return send_stream_->configured_pacing_factor_; |
| 183 | } |
| 184 | |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 185 | void VideoSendStream::SignalNetworkState(NetworkState state) { |
| 186 | RTC_DCHECK_RUN_ON(&thread_checker_); |
| 187 | VideoSendStreamImpl* send_stream = send_stream_.get(); |
| 188 | worker_queue_->PostTask( |
| 189 | [send_stream, state] { send_stream->SignalNetworkState(state); }); |
| 190 | } |
| 191 | |
Åsa Persson | 4bece9a | 2017-10-06 10:04:04 +0200 | [diff] [blame] | 192 | void VideoSendStream::StopPermanentlyAndGetRtpStates( |
| 193 | VideoSendStream::RtpStateMap* rtp_state_map, |
| 194 | VideoSendStream::RtpPayloadStateMap* payload_state_map) { |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 195 | RTC_DCHECK_RUN_ON(&thread_checker_); |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 196 | video_stream_encoder_->Stop(); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 197 | send_stream_->DeRegisterProcessThread(); |
Sebastian Jansson | 1b2e90b | 2018-03-05 19:09:11 +0100 | [diff] [blame] | 198 | worker_queue_->PostTask([this, rtp_state_map, payload_state_map]() { |
| 199 | send_stream_->Stop(); |
| 200 | *rtp_state_map = send_stream_->GetRtpStates(); |
| 201 | *payload_state_map = send_stream_->GetRtpPayloadStates(); |
| 202 | send_stream_.reset(); |
| 203 | thread_sync_event_.Set(); |
| 204 | }); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 205 | thread_sync_event_.Wait(rtc::Event::kForever); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 206 | } |
| 207 | |
nisse | 284542b | 2017-01-10 08:58:32 -0800 | [diff] [blame] | 208 | void VideoSendStream::SetTransportOverhead( |
| 209 | size_t transport_overhead_per_packet) { |
michaelt | 79e0588 | 2016-11-08 02:50:09 -0800 | [diff] [blame] | 210 | RTC_DCHECK_RUN_ON(&thread_checker_); |
| 211 | VideoSendStreamImpl* send_stream = send_stream_.get(); |
| 212 | worker_queue_->PostTask([send_stream, transport_overhead_per_packet] { |
| 213 | send_stream->SetTransportOverhead(transport_overhead_per_packet); |
| 214 | }); |
| 215 | } |
| 216 | |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 217 | bool VideoSendStream::DeliverRtcp(const uint8_t* packet, size_t length) { |
| 218 | // Called on a network thread. |
| 219 | return send_stream_->DeliverRtcp(packet, length); |
| 220 | } |
| 221 | |
palmkvist | e75f204 | 2016-09-28 06:19:48 -0700 | [diff] [blame] | 222 | void VideoSendStream::EnableEncodedFrameRecording( |
| 223 | const std::vector<rtc::PlatformFile>& files, |
| 224 | size_t byte_limit) { |
| 225 | send_stream_->EnableEncodedFrameRecording(files, byte_limit); |
| 226 | } |
| 227 | |
pbos@webrtc.org | 29d5839 | 2013-05-16 12:08:03 +0000 | [diff] [blame] | 228 | } // namespace internal |
| 229 | } // namespace webrtc |