Sebastian Jansson | 8e0b15b | 2018-04-18 19:19:22 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2018 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | #ifndef VIDEO_VIDEO_SEND_STREAM_IMPL_H_ |
| 11 | #define VIDEO_VIDEO_SEND_STREAM_IMPL_H_ |
| 12 | |
| 13 | #include <map> |
| 14 | #include <memory> |
| 15 | #include <unordered_set> |
| 16 | #include <vector> |
| 17 | |
| 18 | #include "call/bitrate_allocator.h" |
| 19 | #include "common_types.h" // NOLINT(build/include) |
| 20 | #include "common_video/include/video_bitrate_allocator.h" |
| 21 | #include "modules/rtp_rtcp/include/flexfec_sender.h" |
| 22 | #include "modules/utility/include/process_thread.h" |
| 23 | #include "modules/video_coding/utility/ivf_file_writer.h" |
| 24 | #include "rtc_base/weak_ptr.h" |
| 25 | #include "video/call_stats.h" |
| 26 | #include "video/encoder_rtcp_feedback.h" |
| 27 | #include "video/payload_router.h" |
| 28 | #include "video/send_delay_stats.h" |
| 29 | #include "video/send_statistics_proxy.h" |
| 30 | #include "video/video_send_stream.h" |
| 31 | #include "video/video_stream_encoder.h" |
| 32 | |
| 33 | namespace webrtc { |
| 34 | namespace internal { |
| 35 | |
| 36 | // VideoSendStreamImpl implements internal::VideoSendStream. |
| 37 | // It is created and destroyed on |worker_queue|. The intent is to decrease the |
| 38 | // need for locking and to ensure methods are called in sequence. |
| 39 | // Public methods except |DeliverRtcp| must be called on |worker_queue|. |
| 40 | // DeliverRtcp is called on the libjingle worker thread or a network thread. |
| 41 | // An encoder may deliver frames through the EncodedImageCallback on an |
| 42 | // arbitrary thread. |
| 43 | class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, |
| 44 | public webrtc::OverheadObserver, |
| 45 | public webrtc::VCMProtectionCallback, |
| 46 | public VideoStreamEncoder::EncoderSink, |
| 47 | public VideoBitrateAllocationObserver, |
| 48 | public webrtc::PacketFeedbackObserver { |
| 49 | public: |
| 50 | VideoSendStreamImpl( |
| 51 | SendStatisticsProxy* stats_proxy, |
| 52 | rtc::TaskQueue* worker_queue, |
| 53 | CallStats* call_stats, |
| 54 | RtpTransportControllerSendInterface* transport, |
Sebastian Jansson | 652dc91 | 2018-04-19 17:09:15 +0200 | [diff] [blame] | 55 | BitrateAllocatorInterface* bitrate_allocator, |
Sebastian Jansson | 8e0b15b | 2018-04-18 19:19:22 +0200 | [diff] [blame] | 56 | SendDelayStats* send_delay_stats, |
Sebastian Jansson | 652dc91 | 2018-04-19 17:09:15 +0200 | [diff] [blame] | 57 | VideoStreamEncoderInterface* video_stream_encoder, |
Sebastian Jansson | 8e0b15b | 2018-04-18 19:19:22 +0200 | [diff] [blame] | 58 | RtcEventLog* event_log, |
| 59 | const VideoSendStream::Config* config, |
| 60 | int initial_encoder_max_bitrate, |
| 61 | double initial_encoder_bitrate_priority, |
| 62 | std::map<uint32_t, RtpState> suspended_ssrcs, |
| 63 | std::map<uint32_t, RtpPayloadState> suspended_payload_states, |
| 64 | VideoEncoderConfig::ContentType content_type, |
| 65 | std::unique_ptr<FecController> fec_controller, |
| 66 | RateLimiter* retransmission_limiter); |
| 67 | ~VideoSendStreamImpl() override; |
| 68 | |
| 69 | // RegisterProcessThread register |module_process_thread| with those objects |
| 70 | // that use it. Registration has to happen on the thread were |
| 71 | // |module_process_thread| was created (libjingle's worker thread). |
| 72 | // TODO(perkj): Replace the use of |module_process_thread| with a TaskQueue, |
| 73 | // maybe |worker_queue|. |
| 74 | void RegisterProcessThread(ProcessThread* module_process_thread); |
| 75 | void DeRegisterProcessThread(); |
| 76 | |
| 77 | void SignalNetworkState(NetworkState state); |
| 78 | bool DeliverRtcp(const uint8_t* packet, size_t length); |
| 79 | void UpdateActiveSimulcastLayers(const std::vector<bool> active_layers); |
| 80 | void Start(); |
| 81 | void Stop(); |
| 82 | |
| 83 | VideoSendStream::RtpStateMap GetRtpStates() const; |
| 84 | VideoSendStream::RtpPayloadStateMap GetRtpPayloadStates() const; |
| 85 | |
| 86 | void EnableEncodedFrameRecording(const std::vector<rtc::PlatformFile>& files, |
| 87 | size_t byte_limit); |
| 88 | |
| 89 | void SetTransportOverhead(size_t transport_overhead_per_packet); |
| 90 | |
| 91 | rtc::Optional<float> configured_pacing_factor_; |
| 92 | |
| 93 | // From PacketFeedbackObserver. |
| 94 | void OnPacketAdded(uint32_t ssrc, uint16_t seq_num) override; |
| 95 | void OnPacketFeedbackVector( |
| 96 | const std::vector<PacketFeedback>& packet_feedback_vector) override; |
| 97 | |
| 98 | private: |
| 99 | class CheckEncoderActivityTask; |
| 100 | |
| 101 | // Implements BitrateAllocatorObserver. |
| 102 | uint32_t OnBitrateUpdated(uint32_t bitrate_bps, |
| 103 | uint8_t fraction_loss, |
| 104 | int64_t rtt, |
| 105 | int64_t probing_interval_ms) override; |
| 106 | |
| 107 | // Implements webrtc::VCMProtectionCallback. |
| 108 | int ProtectionRequest(const FecProtectionParams* delta_params, |
| 109 | const FecProtectionParams* key_params, |
| 110 | uint32_t* sent_video_rate_bps, |
| 111 | uint32_t* sent_nack_rate_bps, |
| 112 | uint32_t* sent_fec_rate_bps) override; |
| 113 | |
| 114 | // Implements OverheadObserver. |
| 115 | void OnOverheadChanged(size_t overhead_bytes_per_packet) override; |
| 116 | |
| 117 | void OnEncoderConfigurationChanged(std::vector<VideoStream> streams, |
| 118 | int min_transmit_bitrate_bps) override; |
| 119 | |
| 120 | // Implements EncodedImageCallback. The implementation routes encoded frames |
| 121 | // to the |payload_router_| and |config.pre_encode_callback| if set. |
| 122 | // Called on an arbitrary encoder callback thread. |
| 123 | EncodedImageCallback::Result OnEncodedImage( |
| 124 | const EncodedImage& encoded_image, |
| 125 | const CodecSpecificInfo* codec_specific_info, |
| 126 | const RTPFragmentationHeader* fragmentation) override; |
| 127 | |
| 128 | // Implements VideoBitrateAllocationObserver. |
Erik Språng | 566124a | 2018-04-23 12:32:22 +0200 | [diff] [blame] | 129 | void OnBitrateAllocationUpdated( |
| 130 | const VideoBitrateAllocation& allocation) override; |
Sebastian Jansson | 8e0b15b | 2018-04-18 19:19:22 +0200 | [diff] [blame] | 131 | |
| 132 | // Starts monitoring and sends a keyframe. |
| 133 | void StartupVideoSendStream(); |
| 134 | // Removes the bitrate observer, stops monitoring and notifies the video |
| 135 | // encoder of the bitrate update. |
| 136 | void StopVideoSendStream(); |
| 137 | |
| 138 | void ConfigureProtection(); |
| 139 | void ConfigureSsrcs(); |
| 140 | void SignalEncoderTimedOut(); |
| 141 | void SignalEncoderActive(); |
| 142 | |
| 143 | const bool send_side_bwe_with_overhead_; |
| 144 | |
| 145 | SendStatisticsProxy* const stats_proxy_; |
| 146 | const VideoSendStream::Config* const config_; |
| 147 | std::map<uint32_t, RtpState> suspended_ssrcs_; |
| 148 | |
| 149 | std::unique_ptr<FecController> fec_controller_; |
| 150 | ProcessThread* module_process_thread_; |
| 151 | rtc::ThreadChecker module_process_thread_checker_; |
| 152 | rtc::TaskQueue* const worker_queue_; |
| 153 | |
| 154 | rtc::CriticalSection encoder_activity_crit_sect_; |
| 155 | CheckEncoderActivityTask* check_encoder_activity_task_ |
| 156 | RTC_GUARDED_BY(encoder_activity_crit_sect_); |
| 157 | |
| 158 | CallStats* const call_stats_; |
| 159 | RtpTransportControllerSendInterface* const transport_; |
Sebastian Jansson | 652dc91 | 2018-04-19 17:09:15 +0200 | [diff] [blame] | 160 | BitrateAllocatorInterface* const bitrate_allocator_; |
Sebastian Jansson | 8e0b15b | 2018-04-18 19:19:22 +0200 | [diff] [blame] | 161 | |
| 162 | // TODO(brandtr): Move ownership to PayloadRouter. |
| 163 | std::unique_ptr<FlexfecSender> flexfec_sender_; |
| 164 | |
| 165 | rtc::CriticalSection ivf_writers_crit_; |
| 166 | std::unique_ptr<IvfFileWriter> |
| 167 | file_writers_[kMaxSimulcastStreams] RTC_GUARDED_BY(ivf_writers_crit_); |
| 168 | |
| 169 | int max_padding_bitrate_; |
| 170 | int encoder_min_bitrate_bps_; |
| 171 | uint32_t encoder_max_bitrate_bps_; |
| 172 | uint32_t encoder_target_rate_bps_; |
| 173 | double encoder_bitrate_priority_; |
| 174 | bool has_packet_feedback_; |
| 175 | |
Sebastian Jansson | 652dc91 | 2018-04-19 17:09:15 +0200 | [diff] [blame] | 176 | VideoStreamEncoderInterface* const video_stream_encoder_; |
Sebastian Jansson | 8e0b15b | 2018-04-18 19:19:22 +0200 | [diff] [blame] | 177 | EncoderRtcpFeedback encoder_feedback_; |
| 178 | |
| 179 | RtcpBandwidthObserver* const bandwidth_observer_; |
| 180 | // RtpRtcp modules, declared here as they use other members on construction. |
| 181 | const std::vector<RtpRtcp*> rtp_rtcp_modules_; |
| 182 | PayloadRouter payload_router_; |
| 183 | |
| 184 | // |weak_ptr_| to our self. This is used since we can not call |
| 185 | // |weak_ptr_factory_.GetWeakPtr| from multiple sequences but it is ok to copy |
| 186 | // an existing WeakPtr. |
| 187 | rtc::WeakPtr<VideoSendStreamImpl> weak_ptr_; |
| 188 | // |weak_ptr_factory_| must be declared last to make sure all WeakPtr's are |
| 189 | // invalidated before any other members are destroyed. |
| 190 | rtc::WeakPtrFactory<VideoSendStreamImpl> weak_ptr_factory_; |
| 191 | |
| 192 | rtc::CriticalSection overhead_bytes_per_packet_crit_; |
| 193 | size_t overhead_bytes_per_packet_ |
| 194 | RTC_GUARDED_BY(overhead_bytes_per_packet_crit_); |
| 195 | size_t transport_overhead_bytes_per_packet_; |
| 196 | |
| 197 | std::unordered_set<uint16_t> feedback_packet_seq_num_set_; |
| 198 | std::vector<bool> loss_mask_vector_; |
| 199 | }; |
| 200 | } // namespace internal |
| 201 | } // namespace webrtc |
| 202 | #endif // VIDEO_VIDEO_SEND_STREAM_IMPL_H_ |