blob: 12d9dac5e9d87d35aa99592787b56817a7139aaf [file] [log] [blame]
pbos@webrtc.org29d58392013-05-16 12:08:03 +00001/*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020010#include "video/video_send_stream.h"
pbos@webrtc.org29d58392013-05-16 12:08:03 +000011
perkj71ee44c2016-06-15 00:47:53 -070012#include <utility>
pbos@webrtc.org29d58392013-05-16 12:08:03 +000013
Yves Gerey3e707812018-11-28 16:47:49 +010014#include "api/array_view.h"
Niels Möller213618e2018-07-24 09:29:58 +020015#include "api/video/video_stream_encoder_create.h"
Yves Gerey3e707812018-11-28 16:47:49 +010016#include "api/video/video_stream_encoder_settings.h"
17#include "modules/rtp_rtcp/include/rtp_header_extension_map.h"
Danil Chapovalov7b189922018-10-03 10:15:36 +020018#include "modules/rtp_rtcp/source/rtp_header_extension_size.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020019#include "modules/rtp_rtcp/source/rtp_sender.h"
Yves Gerey3e707812018-11-28 16:47:49 +010020#include "rtc_base/checks.h"
Stefan Holmerdbdb3a02018-07-17 16:03:46 +020021#include "rtc_base/logging.h"
Danil Chapovalov1aa75812019-03-05 11:11:35 +010022#include "rtc_base/task_utils/to_queued_task.h"
Yves Gerey3e707812018-11-28 16:47:49 +010023#include "system_wrappers/include/clock.h"
Rasmus Brandtbdc6c402018-11-06 12:55:53 +010024#include "system_wrappers/include/field_trial.h"
Sebastian Jansson8e0b15b2018-04-18 19:19:22 +020025#include "video/video_send_stream_impl.h"
pbos@webrtc.org29d58392013-05-16 12:08:03 +000026
27namespace webrtc {
mflodman949c2f02015-10-16 02:31:11 -070028
Per83d09102016-04-15 14:59:13 +020029namespace {
30
Rasmus Brandtbdc6c402018-11-06 12:55:53 +010031constexpr char kTargetBitrateRtcpFieldTrial[] = "WebRTC-Target-Bitrate-Rtcp";
32
Stefan Holmerdbdb3a02018-07-17 16:03:46 +020033size_t CalculateMaxHeaderSize(const RtpConfig& config) {
Ilya Nikolaevskiy1d037ae2018-03-15 15:46:17 +010034 size_t header_size = kRtpHeaderSize;
35 size_t extensions_size = 0;
36 size_t fec_extensions_size = 0;
Benjamin Wright1f4173e2019-03-13 17:59:32 -070037 if (!config.extensions.empty()) {
Ilya Nikolaevskiy1d037ae2018-03-15 15:46:17 +010038 RtpHeaderExtensionMap extensions_map(config.extensions);
Danil Chapovalov7b189922018-10-03 10:15:36 +020039 extensions_size = RtpHeaderExtensionSize(RTPSender::VideoExtensionSizes(),
40 extensions_map);
Ilya Nikolaevskiy1d037ae2018-03-15 15:46:17 +010041 fec_extensions_size =
Danil Chapovalov7b189922018-10-03 10:15:36 +020042 RtpHeaderExtensionSize(RTPSender::FecExtensionSizes(), extensions_map);
Ilya Nikolaevskiy1d037ae2018-03-15 15:46:17 +010043 }
44 header_size += extensions_size;
45 if (config.flexfec.payload_type >= 0) {
46 // All FEC extensions again plus maximum FlexFec overhead.
47 header_size += fec_extensions_size + 32;
48 } else {
49 if (config.ulpfec.ulpfec_payload_type >= 0) {
50 // Header with all the FEC extensions will be repeated plus maximum
51 // UlpFec overhead.
52 header_size += fec_extensions_size + 18;
53 }
54 if (config.ulpfec.red_payload_type >= 0) {
55 header_size += 1; // RED header.
56 }
57 }
58 // Additional room for Rtx.
59 if (config.rtx.payload_type >= 0)
60 header_size += kRtxHeaderSize;
61 return header_size;
62}
63
Peter Boströme4499152016-02-05 11:13:28 +010064} // namespace
65
pbos@webrtc.org024e4d52014-05-15 10:03:24 +000066namespace internal {
perkj26091b12016-09-01 01:17:40 -070067
pbos@webrtc.org2bb1bda2014-07-07 13:06:48 +000068VideoSendStream::VideoSendStream(
Sebastian Jansson572c60f2019-03-04 18:30:41 +010069 Clock* clock,
Peter Boström45553ae2015-05-08 13:54:38 +020070 int num_cpu_cores,
Peter Boströmf16fcbe2015-04-30 12:16:05 +020071 ProcessThread* module_process_thread,
Sebastian Jansson74682c12019-03-01 11:50:20 +010072 TaskQueueFactory* task_queue_factory,
mflodmane3787022015-10-21 13:24:28 +020073 CallStats* call_stats,
nisseb8f9a322017-03-27 05:36:15 -070074 RtpTransportControllerSendInterface* transport,
Niels Möller67b011d2018-10-22 13:00:40 +020075 BitrateAllocatorInterface* bitrate_allocator,
asapersson35151f32016-05-02 23:44:01 -070076 SendDelayStats* send_delay_stats,
tereliusadafe0b2016-05-26 01:58:40 -070077 RtcEventLog* event_log,
perkj26091b12016-09-01 01:17:40 -070078 VideoSendStream::Config config,
79 VideoEncoderConfig encoder_config,
Åsa Persson4bece9a2017-10-06 10:04:04 +020080 const std::map<uint32_t, RtpState>& suspended_ssrcs,
Ying Wang3b790f32018-01-19 17:58:57 +010081 const std::map<uint32_t, RtpPayloadState>& suspended_payload_states,
Stefan Holmerdbdb3a02018-07-17 16:03:46 +020082 std::unique_ptr<FecController> fec_controller)
Sebastian Jansson0b698262019-03-07 09:17:19 +010083 : worker_queue_(transport->GetWorkerQueue()),
Sebastian Jansson572c60f2019-03-04 18:30:41 +010084 stats_proxy_(clock, config, encoder_config.content_type),
sprangf24a0642017-02-28 13:23:26 -080085 config_(std::move(config)),
86 content_type_(encoder_config.content_type) {
Niels Möller4db138e2018-04-19 09:04:13 +020087 RTC_DCHECK(config_.encoder_settings.encoder_factory);
Jiawei Ouc2ebe212018-11-08 10:02:56 -080088 RTC_DCHECK(config_.encoder_settings.bitrate_allocator_factory);
Niels Möller4db138e2018-04-19 09:04:13 +020089
Sebastian Jansson74682c12019-03-01 11:50:20 +010090 video_stream_encoder_ =
Sebastian Jansson572c60f2019-03-04 18:30:41 +010091 CreateVideoStreamEncoder(clock, task_queue_factory, num_cpu_cores,
92 &stats_proxy_, config_.encoder_settings);
Sebastian Jansson06b83aa2018-02-28 13:03:46 +010093 // TODO(srte): Initialization should not be done posted on a task queue.
94 // Note that the posted task must not outlive this scope since the closure
95 // references local variables.
Danil Chapovalov1aa75812019-03-05 11:11:35 +010096 worker_queue_->PostTask(ToQueuedTask(
Sebastian Jansson572c60f2019-03-04 18:30:41 +010097 [this, clock, call_stats, transport, bitrate_allocator, send_delay_stats,
Sebastian Jansson06b83aa2018-02-28 13:03:46 +010098 event_log, &suspended_ssrcs, &encoder_config, &suspended_payload_states,
Stefan Holmerdbdb3a02018-07-17 16:03:46 +020099 &fec_controller]() {
Sebastian Jansson06b83aa2018-02-28 13:03:46 +0100100 send_stream_.reset(new VideoSendStreamImpl(
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100101 clock, &stats_proxy_, worker_queue_, call_stats, transport,
Sebastian Jansson06b83aa2018-02-28 13:03:46 +0100102 bitrate_allocator, send_delay_stats, video_stream_encoder_.get(),
103 event_log, &config_, encoder_config.max_bitrate_bps,
104 encoder_config.bitrate_priority, suspended_ssrcs,
105 suspended_payload_states, encoder_config.content_type,
Niels Möller46879152019-01-07 15:54:47 +0100106 std::move(fec_controller), config_.media_transport));
Sebastian Jansson06b83aa2018-02-28 13:03:46 +0100107 },
108 [this]() { thread_sync_event_.Set(); }));
perkj26091b12016-09-01 01:17:40 -0700109
110 // Wait for ConstructionTask to complete so that |send_stream_| can be used.
111 // |module_process_thread| must be registered and deregistered on the thread
112 // it was created on.
113 thread_sync_event_.Wait(rtc::Event::kForever);
114 send_stream_->RegisterProcessThread(module_process_thread);
Rasmus Brandtbdc6c402018-11-06 12:55:53 +0100115 // TODO(sprang): Enable this also for regular video calls by default, if it
116 // works well.
117 if (encoder_config.content_type == VideoEncoderConfig::ContentType::kScreen ||
118 field_trial::IsEnabled(kTargetBitrateRtcpFieldTrial)) {
Niels Möller0327c2d2018-05-21 14:09:31 +0200119 video_stream_encoder_->SetBitrateAllocationObserver(send_stream_.get());
sprang44b3ef62017-01-13 07:30:25 -0800120 }
Ying Wang38a31b02017-12-21 12:26:19 +0000121
perkj26091b12016-09-01 01:17:40 -0700122 ReconfigureVideoEncoder(std::move(encoder_config));
123}
124
125VideoSendStream::~VideoSendStream() {
126 RTC_DCHECK_RUN_ON(&thread_checker_);
127 RTC_DCHECK(!send_stream_);
128}
129
Seth Hampsoncc7125f2018-02-02 08:46:16 -0800130void VideoSendStream::UpdateActiveSimulcastLayers(
131 const std::vector<bool> active_layers) {
132 RTC_DCHECK_RUN_ON(&thread_checker_);
133 RTC_LOG(LS_INFO) << "VideoSendStream::UpdateActiveSimulcastLayers";
134 VideoSendStreamImpl* send_stream = send_stream_.get();
135 worker_queue_->PostTask([this, send_stream, active_layers] {
136 send_stream->UpdateActiveSimulcastLayers(active_layers);
137 thread_sync_event_.Set();
138 });
139
140 thread_sync_event_.Wait(rtc::Event::kForever);
141}
142
perkj26091b12016-09-01 01:17:40 -0700143void VideoSendStream::Start() {
144 RTC_DCHECK_RUN_ON(&thread_checker_);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100145 RTC_LOG(LS_INFO) << "VideoSendStream::Start";
perkj26091b12016-09-01 01:17:40 -0700146 VideoSendStreamImpl* send_stream = send_stream_.get();
147 worker_queue_->PostTask([this, send_stream] {
148 send_stream->Start();
149 thread_sync_event_.Set();
150 });
151
152 // It is expected that after VideoSendStream::Start has been called, incoming
mflodmancc3d4422017-08-03 08:27:51 -0700153 // frames are not dropped in VideoStreamEncoder. To ensure this, Start has to
154 // be synchronized.
perkj26091b12016-09-01 01:17:40 -0700155 thread_sync_event_.Wait(rtc::Event::kForever);
156}
157
158void VideoSendStream::Stop() {
159 RTC_DCHECK_RUN_ON(&thread_checker_);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100160 RTC_LOG(LS_INFO) << "VideoSendStream::Stop";
perkj26091b12016-09-01 01:17:40 -0700161 VideoSendStreamImpl* send_stream = send_stream_.get();
162 worker_queue_->PostTask([send_stream] { send_stream->Stop(); });
163}
164
perkja49cbd32016-09-16 07:53:41 -0700165void VideoSendStream::SetSource(
perkj803d97f2016-11-01 11:45:46 -0700166 rtc::VideoSourceInterface<webrtc::VideoFrame>* source,
167 const DegradationPreference& degradation_preference) {
perkja49cbd32016-09-16 07:53:41 -0700168 RTC_DCHECK_RUN_ON(&thread_checker_);
mflodmancc3d4422017-08-03 08:27:51 -0700169 video_stream_encoder_->SetSource(source, degradation_preference);
perkj26091b12016-09-01 01:17:40 -0700170}
171
172void VideoSendStream::ReconfigureVideoEncoder(VideoEncoderConfig config) {
perkjfa10b552016-10-02 23:45:26 -0700173 // TODO(perkj): Some test cases in VideoSendStreamTest call
174 // ReconfigureVideoEncoder from the network thread.
175 // RTC_DCHECK_RUN_ON(&thread_checker_);
sprangf24a0642017-02-28 13:23:26 -0800176 RTC_DCHECK(content_type_ == config.content_type);
Ilya Nikolaevskiy1d037ae2018-03-15 15:46:17 +0100177 video_stream_encoder_->ConfigureEncoder(
178 std::move(config),
Niels Möllerf1338562018-04-26 09:51:47 +0200179 config_.rtp.max_packet_size - CalculateMaxHeaderSize(config_.rtp));
perkj26091b12016-09-01 01:17:40 -0700180}
181
182VideoSendStream::Stats VideoSendStream::GetStats() {
183 // TODO(perkj, solenberg): Some test cases in EndToEndTest call GetStats from
184 // a network thread. See comment in Call::GetStats().
185 // RTC_DCHECK_RUN_ON(&thread_checker_);
186 return stats_proxy_.GetStats();
187}
188
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200189absl::optional<float> VideoSendStream::GetPacingFactorOverride() const {
Sebastian Janssona45c8da2018-01-16 10:55:29 +0100190 return send_stream_->configured_pacing_factor_;
191}
192
Åsa Persson4bece9a2017-10-06 10:04:04 +0200193void VideoSendStream::StopPermanentlyAndGetRtpStates(
194 VideoSendStream::RtpStateMap* rtp_state_map,
195 VideoSendStream::RtpPayloadStateMap* payload_state_map) {
perkj26091b12016-09-01 01:17:40 -0700196 RTC_DCHECK_RUN_ON(&thread_checker_);
mflodmancc3d4422017-08-03 08:27:51 -0700197 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700198 send_stream_->DeRegisterProcessThread();
Sebastian Jansson1b2e90b2018-03-05 19:09:11 +0100199 worker_queue_->PostTask([this, rtp_state_map, payload_state_map]() {
200 send_stream_->Stop();
201 *rtp_state_map = send_stream_->GetRtpStates();
202 *payload_state_map = send_stream_->GetRtpPayloadStates();
203 send_stream_.reset();
204 thread_sync_event_.Set();
205 });
perkj26091b12016-09-01 01:17:40 -0700206 thread_sync_event_.Wait(rtc::Event::kForever);
perkj26091b12016-09-01 01:17:40 -0700207}
208
Niels Möller8fb1a6a2019-03-05 14:29:42 +0100209void VideoSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
perkj26091b12016-09-01 01:17:40 -0700210 // Called on a network thread.
Niels Möller8fb1a6a2019-03-05 14:29:42 +0100211 send_stream_->DeliverRtcp(packet, length);
perkj26091b12016-09-01 01:17:40 -0700212}
213
pbos@webrtc.org29d58392013-05-16 12:08:03 +0000214} // namespace internal
215} // namespace webrtc