solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #ifndef AUDIO_AUDIO_SEND_STREAM_H_ |
| 12 | #define AUDIO_AUDIO_SEND_STREAM_H_ |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 13 | |
kwiberg | fffa42b | 2016-02-23 10:46:32 -0800 | [diff] [blame] | 14 | #include <memory> |
Sebastian Jansson | 62aee93 | 2019-10-02 12:27:06 +0200 | [diff] [blame] | 15 | #include <utility> |
elad.alon | d12a8e1 | 2017-03-23 11:04:48 -0700 | [diff] [blame] | 16 | #include <vector> |
kwiberg | fffa42b | 2016-02-23 10:46:32 -0800 | [diff] [blame] | 17 | |
Henrik Boström | d2c336f | 2019-07-03 17:11:10 +0200 | [diff] [blame] | 18 | #include "audio/audio_level.h" |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 19 | #include "audio/channel_send.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 20 | #include "call/audio_send_stream.h" |
| 21 | #include "call/audio_state.h" |
| 22 | #include "call/bitrate_allocator.h" |
| 23 | #include "modules/rtp_rtcp/include/rtp_rtcp.h" |
Steve Anton | 10542f2 | 2019-01-11 09:11:00 -0800 | [diff] [blame] | 24 | #include "rtc_base/constructor_magic.h" |
Sebastian Jansson | f23131f | 2019-10-03 10:03:55 +0200 | [diff] [blame] | 25 | #include "rtc_base/experiments/struct_parameters_parser.h" |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 26 | #include "rtc_base/race_checker.h" |
Sebastian Jansson | 44dd9f2 | 2019-03-08 14:50:30 +0100 | [diff] [blame] | 27 | #include "rtc_base/task_queue.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 28 | #include "rtc_base/thread_checker.h" |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 29 | |
| 30 | namespace webrtc { |
terelius | e035e2d | 2016-09-21 06:51:47 -0700 | [diff] [blame] | 31 | class RtcEventLog; |
stefan | 7de8d64 | 2017-02-07 07:14:08 -0800 | [diff] [blame] | 32 | class RtcpBandwidthObserver; |
michaelt | 9332b7d | 2016-11-30 07:51:13 -0800 | [diff] [blame] | 33 | class RtcpRttStats; |
nisse | b8f9a32 | 2017-03-27 05:36:15 -0700 | [diff] [blame] | 34 | class RtpTransportControllerSendInterface; |
solenberg | 3a94154 | 2015-11-16 07:34:50 -0800 | [diff] [blame] | 35 | |
Sebastian Jansson | f23131f | 2019-10-03 10:03:55 +0200 | [diff] [blame] | 36 | struct AudioAllocationConfig { |
| 37 | static constexpr char kKey[] = "WebRTC-Audio-Allocation"; |
| 38 | // Field Trial configured bitrates to use as overrides over default/user |
| 39 | // configured bitrate range when audio bitrate allocation is enabled. |
| 40 | absl::optional<DataRate> min_bitrate; |
| 41 | absl::optional<DataRate> max_bitrate; |
| 42 | DataRate priority_bitrate = DataRate::Zero(); |
| 43 | // By default the priority_bitrate is compensated for packet overhead. |
| 44 | // Use this flag to configure a raw value instead. |
| 45 | absl::optional<DataRate> priority_bitrate_raw; |
| 46 | absl::optional<double> bitrate_priority; |
| 47 | |
| 48 | std::unique_ptr<StructParametersParser> Parser(); |
| 49 | AudioAllocationConfig(); |
| 50 | }; |
solenberg | 1372508 | 2015-11-25 08:16:52 -0800 | [diff] [blame] | 51 | namespace internal { |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 52 | class AudioState; |
| 53 | |
mflodman | 86cc6ff | 2016-07-26 04:44:06 -0700 | [diff] [blame] | 54 | class AudioSendStream final : public webrtc::AudioSendStream, |
elad.alon | d12a8e1 | 2017-03-23 11:04:48 -0700 | [diff] [blame] | 55 | public webrtc::BitrateAllocatorObserver, |
Anton Sukhanov | 626015d | 2019-02-04 15:16:06 -0800 | [diff] [blame] | 56 | public webrtc::OverheadObserver { |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 57 | public: |
Sebastian Jansson | 977b335 | 2019-03-04 17:43:34 +0100 | [diff] [blame] | 58 | AudioSendStream(Clock* clock, |
| 59 | const webrtc::AudioSendStream::Config& config, |
Stefan Holmer | b86d4e4 | 2015-12-07 10:26:18 +0100 | [diff] [blame] | 60 | const rtc::scoped_refptr<webrtc::AudioState>& audio_state, |
Sebastian Jansson | 44dd9f2 | 2019-03-08 14:50:30 +0100 | [diff] [blame] | 61 | TaskQueueFactory* task_queue_factory, |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 62 | ProcessThread* module_process_thread, |
Niels Möller | 7d76a31 | 2018-10-26 12:57:07 +0200 | [diff] [blame] | 63 | RtpTransportControllerSendInterface* rtp_transport, |
Niels Möller | 67b011d | 2018-10-22 13:00:40 +0200 | [diff] [blame] | 64 | BitrateAllocatorInterface* bitrate_allocator, |
michaelt | 9332b7d | 2016-11-30 07:51:13 -0800 | [diff] [blame] | 65 | RtcEventLog* event_log, |
ossu | c3d4b48 | 2017-05-23 06:07:11 -0700 | [diff] [blame] | 66 | RtcpRttStats* rtcp_rtt_stats, |
Sam Zackrisson | ff05816 | 2018-11-20 17:15:13 +0100 | [diff] [blame] | 67 | const absl::optional<RtpState>& suspended_rtp_state); |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 68 | // For unit tests, which need to supply a mock ChannelSend. |
Sebastian Jansson | 977b335 | 2019-03-04 17:43:34 +0100 | [diff] [blame] | 69 | AudioSendStream(Clock* clock, |
| 70 | const webrtc::AudioSendStream::Config& config, |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 71 | const rtc::scoped_refptr<webrtc::AudioState>& audio_state, |
Sebastian Jansson | 44dd9f2 | 2019-03-08 14:50:30 +0100 | [diff] [blame] | 72 | TaskQueueFactory* task_queue_factory, |
Niels Möller | 7d76a31 | 2018-10-26 12:57:07 +0200 | [diff] [blame] | 73 | RtpTransportControllerSendInterface* rtp_transport, |
Niels Möller | 67b011d | 2018-10-22 13:00:40 +0200 | [diff] [blame] | 74 | BitrateAllocatorInterface* bitrate_allocator, |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 75 | RtcEventLog* event_log, |
| 76 | RtcpRttStats* rtcp_rtt_stats, |
Danil Chapovalov | b9b146c | 2018-06-15 12:28:07 +0200 | [diff] [blame] | 77 | const absl::optional<RtpState>& suspended_rtp_state, |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 78 | std::unique_ptr<voe::ChannelSendInterface> channel_send); |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 79 | ~AudioSendStream() override; |
| 80 | |
pbos | 1ba8d39 | 2016-05-01 20:18:34 -0700 | [diff] [blame] | 81 | // webrtc::AudioSendStream implementation. |
eladalon | abbc430 | 2017-07-26 02:09:44 -0700 | [diff] [blame] | 82 | const webrtc::AudioSendStream::Config& GetConfig() const override; |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 83 | void Reconfigure(const webrtc::AudioSendStream::Config& config) override; |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 84 | void Start() override; |
| 85 | void Stop() override; |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 86 | void SendAudioData(std::unique_ptr<AudioFrame> audio_frame) override; |
Yves Gerey | 665174f | 2018-06-19 15:03:05 +0200 | [diff] [blame] | 87 | bool SendTelephoneEvent(int payload_type, |
| 88 | int payload_frequency, |
| 89 | int event, |
solenberg | 8842c3e | 2016-03-11 03:06:41 -0800 | [diff] [blame] | 90 | int duration_ms) override; |
solenberg | 9421853 | 2016-06-16 10:53:22 -0700 | [diff] [blame] | 91 | void SetMuted(bool muted) override; |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 92 | webrtc::AudioSendStream::Stats GetStats() const override; |
Ivo Creusen | 56d4609 | 2017-11-24 17:29:59 +0100 | [diff] [blame] | 93 | webrtc::AudioSendStream::Stats GetStats( |
| 94 | bool has_remote_tracks) const override; |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 95 | |
Niels Möller | 8fb1a6a | 2019-03-05 14:29:42 +0100 | [diff] [blame] | 96 | void DeliverRtcp(const uint8_t* packet, size_t length); |
mflodman | 86cc6ff | 2016-07-26 04:44:06 -0700 | [diff] [blame] | 97 | |
| 98 | // Implements BitrateAllocatorObserver. |
Sebastian Jansson | c0e4d45 | 2018-10-25 15:08:32 +0200 | [diff] [blame] | 99 | uint32_t OnBitrateUpdated(BitrateAllocationUpdate update) override; |
mflodman | 86cc6ff | 2016-07-26 04:44:06 -0700 | [diff] [blame] | 100 | |
Anton Sukhanov | 626015d | 2019-02-04 15:16:06 -0800 | [diff] [blame] | 101 | void SetTransportOverhead(int transport_overhead_per_packet_bytes); |
| 102 | |
| 103 | // OverheadObserver override reports audio packetization overhead from |
| 104 | // RTP/RTCP module or Media Transport. |
| 105 | void OnOverheadChanged(size_t overhead_bytes_per_packet_bytes) override; |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 106 | |
ossu | c3d4b48 | 2017-05-23 06:07:11 -0700 | [diff] [blame] | 107 | RtpState GetRtpState() const; |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 108 | const voe::ChannelSendInterface* GetChannel() const; |
ossu | c3d4b48 | 2017-05-23 06:07:11 -0700 | [diff] [blame] | 109 | |
Anton Sukhanov | 626015d | 2019-02-04 15:16:06 -0800 | [diff] [blame] | 110 | // Returns combined per-packet overhead. |
| 111 | size_t TestOnlyGetPerPacketOverheadBytes() const |
| 112 | RTC_LOCKS_EXCLUDED(overhead_per_packet_lock_); |
| 113 | |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 114 | private: |
saza | c58f8c0 | 2017-07-19 00:39:19 -0700 | [diff] [blame] | 115 | class TimedTransport; |
Daniel Lee | 9356252 | 2019-05-03 14:40:13 +0200 | [diff] [blame] | 116 | // Constraints including overhead. |
| 117 | struct TargetAudioBitrateConstraints { |
| 118 | DataRate min; |
| 119 | DataRate max; |
| 120 | }; |
saza | c58f8c0 | 2017-07-19 00:39:19 -0700 | [diff] [blame] | 121 | |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 122 | internal::AudioState* audio_state(); |
| 123 | const internal::AudioState* audio_state() const; |
solenberg | 3a94154 | 2015-11-16 07:34:50 -0800 | [diff] [blame] | 124 | |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 125 | void StoreEncoderProperties(int sample_rate_hz, size_t num_channels); |
| 126 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 127 | void ConfigureStream(const Config& new_config, bool first_time); |
| 128 | bool SetupSendCodec(const Config& new_config); |
| 129 | bool ReconfigureSendCodec(const Config& new_config); |
| 130 | void ReconfigureANA(const Config& new_config); |
| 131 | void ReconfigureCNG(const Config& new_config); |
| 132 | void ReconfigureBitrateObserver(const Config& new_config); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 133 | |
Sebastian Jansson | 8672cac | 2019-03-01 15:57:55 +0100 | [diff] [blame] | 134 | void ConfigureBitrateObserver() RTC_RUN_ON(worker_queue_); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 135 | void RemoveBitrateObserver(); |
minyue | 7a97344 | 2016-10-20 03:27:12 -0700 | [diff] [blame] | 136 | |
Daniel Lee | 9356252 | 2019-05-03 14:40:13 +0200 | [diff] [blame] | 137 | // Returns bitrate constraints, maybe including overhead when enabled by |
| 138 | // field trial. |
Sebastian Jansson | 62aee93 | 2019-10-02 12:27:06 +0200 | [diff] [blame] | 139 | TargetAudioBitrateConstraints GetMinMaxBitrateConstraints() const |
| 140 | RTC_RUN_ON(worker_queue_); |
Daniel Lee | 9356252 | 2019-05-03 14:40:13 +0200 | [diff] [blame] | 141 | |
Anton Sukhanov | 626015d | 2019-02-04 15:16:06 -0800 | [diff] [blame] | 142 | // Sets per-packet overhead on encoded (for ANA) based on current known values |
| 143 | // of transport and packetization overheads. |
| 144 | void UpdateOverheadForEncoder() |
| 145 | RTC_EXCLUSIVE_LOCKS_REQUIRED(overhead_per_packet_lock_); |
| 146 | |
| 147 | // Returns combined per-packet overhead. |
| 148 | size_t GetPerPacketOverheadBytes() const |
| 149 | RTC_EXCLUSIVE_LOCKS_REQUIRED(overhead_per_packet_lock_); |
| 150 | |
ossu | 3b9ff38 | 2017-04-27 08:03:42 -0700 | [diff] [blame] | 151 | void RegisterCngPayloadType(int payload_type, int clockrate_hz); |
Sebastian Jansson | 977b335 | 2019-03-04 17:43:34 +0100 | [diff] [blame] | 152 | Clock* clock_; |
ossu | 3b9ff38 | 2017-04-27 08:03:42 -0700 | [diff] [blame] | 153 | |
elad.alon | d12a8e1 | 2017-03-23 11:04:48 -0700 | [diff] [blame] | 154 | rtc::ThreadChecker worker_thread_checker_; |
| 155 | rtc::ThreadChecker pacer_thread_checker_; |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 156 | rtc::RaceChecker audio_capture_race_checker_; |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 157 | rtc::TaskQueue* worker_queue_; |
Sebastian Jansson | f23131f | 2019-10-03 10:03:55 +0200 | [diff] [blame] | 158 | |
| 159 | const bool audio_send_side_bwe_; |
| 160 | const bool allocate_audio_without_feedback_; |
| 161 | const bool force_no_audio_feedback_ = allocate_audio_without_feedback_; |
| 162 | const bool enable_audio_alr_probing_; |
| 163 | const bool send_side_bwe_with_overhead_; |
| 164 | const AudioAllocationConfig allocation_settings_; |
| 165 | |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 166 | webrtc::AudioSendStream::Config config_; |
solenberg | 566ef24 | 2015-11-06 15:34:49 -0800 | [diff] [blame] | 167 | rtc::scoped_refptr<webrtc::AudioState> audio_state_; |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 168 | const std::unique_ptr<voe::ChannelSendInterface> channel_send_; |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 169 | RtcEventLog* const event_log_; |
Sebastian Jansson | 62aee93 | 2019-10-02 12:27:06 +0200 | [diff] [blame] | 170 | const bool use_legacy_overhead_calculation_; |
solenberg | 85a0496 | 2015-10-27 03:35:21 -0700 | [diff] [blame] | 171 | |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 172 | int encoder_sample_rate_hz_ = 0; |
| 173 | size_t encoder_num_channels_ = 0; |
| 174 | bool sending_ = false; |
Henrik Boström | d2c336f | 2019-07-03 17:11:10 +0200 | [diff] [blame] | 175 | rtc::CriticalSection audio_level_lock_; |
| 176 | // Keeps track of audio level, total audio energy and total samples duration. |
| 177 | // https://w3c.github.io/webrtc-stats/#dom-rtcaudiohandlerstats-totalaudioenergy |
| 178 | webrtc::voe::AudioLevel audio_level_; |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 179 | |
Sebastian Jansson | 8672cac | 2019-03-01 15:57:55 +0100 | [diff] [blame] | 180 | BitrateAllocatorInterface* const bitrate_allocator_ |
| 181 | RTC_GUARDED_BY(worker_queue_); |
Niels Möller | 7d76a31 | 2018-10-26 12:57:07 +0200 | [diff] [blame] | 182 | RtpTransportControllerSendInterface* const rtp_transport_; |
mflodman | 86cc6ff | 2016-07-26 04:44:06 -0700 | [diff] [blame] | 183 | |
Sebastian Jansson | 6298b56 | 2020-01-14 17:55:19 +0100 | [diff] [blame] | 184 | RtpRtcp* const rtp_rtcp_module_; |
Danil Chapovalov | b9b146c | 2018-06-15 12:28:07 +0200 | [diff] [blame] | 185 | absl::optional<RtpState> const suspended_rtp_state_; |
ossu | c3d4b48 | 2017-05-23 06:07:11 -0700 | [diff] [blame] | 186 | |
Alex Narest | cedd351 | 2017-12-07 20:54:55 +0100 | [diff] [blame] | 187 | // RFC 5285: Each distinct extension MUST have a unique ID. The value 0 is |
| 188 | // reserved for padding and MUST NOT be used as a local identifier. |
| 189 | // So it should be safe to use 0 here to indicate "not configured". |
| 190 | struct ExtensionIds { |
| 191 | int audio_level = 0; |
Sebastian Jansson | 71c6b56 | 2019-08-14 11:31:02 +0200 | [diff] [blame] | 192 | int abs_send_time = 0; |
Minyue Li | 74dadc1 | 2020-03-05 11:33:13 +0100 | [diff] [blame] | 193 | int abs_capture_time = 0; |
Alex Narest | cedd351 | 2017-12-07 20:54:55 +0100 | [diff] [blame] | 194 | int transport_sequence_number = 0; |
Steve Anton | bb50ce5 | 2018-03-26 10:24:32 -0700 | [diff] [blame] | 195 | int mid = 0; |
Amit Hilbuch | 77938e6 | 2018-12-21 09:23:38 -0800 | [diff] [blame] | 196 | int rid = 0; |
| 197 | int repaired_rid = 0; |
Alex Narest | cedd351 | 2017-12-07 20:54:55 +0100 | [diff] [blame] | 198 | }; |
| 199 | static ExtensionIds FindExtensionIds( |
| 200 | const std::vector<RtpExtension>& extensions); |
Sebastian Jansson | 470a5ea | 2019-01-23 12:37:49 +0100 | [diff] [blame] | 201 | static int TransportSeqNumId(const Config& config); |
Alex Narest | cedd351 | 2017-12-07 20:54:55 +0100 | [diff] [blame] | 202 | |
Anton Sukhanov | 626015d | 2019-02-04 15:16:06 -0800 | [diff] [blame] | 203 | rtc::CriticalSection overhead_per_packet_lock_; |
| 204 | |
| 205 | // Current transport overhead (ICE, TURN, etc.) |
| 206 | size_t transport_overhead_per_packet_bytes_ |
| 207 | RTC_GUARDED_BY(overhead_per_packet_lock_) = 0; |
| 208 | |
| 209 | // Current audio packetization overhead (RTP or Media Transport). |
| 210 | size_t audio_overhead_per_packet_bytes_ |
| 211 | RTC_GUARDED_BY(overhead_per_packet_lock_) = 0; |
| 212 | |
Sebastian Jansson | 8672cac | 2019-03-01 15:57:55 +0100 | [diff] [blame] | 213 | bool registered_with_allocator_ RTC_GUARDED_BY(worker_queue_) = false; |
| 214 | size_t total_packet_overhead_bytes_ RTC_GUARDED_BY(worker_queue_) = 0; |
Sebastian Jansson | 62aee93 | 2019-10-02 12:27:06 +0200 | [diff] [blame] | 215 | absl::optional<std::pair<TimeDelta, TimeDelta>> frame_length_range_ |
| 216 | RTC_GUARDED_BY(worker_queue_); |
Sebastian Jansson | 8672cac | 2019-03-01 15:57:55 +0100 | [diff] [blame] | 217 | |
solenberg | 85a0496 | 2015-10-27 03:35:21 -0700 | [diff] [blame] | 218 | RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AudioSendStream); |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 219 | }; |
| 220 | } // namespace internal |
| 221 | } // namespace webrtc |
| 222 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 223 | #endif // AUDIO_AUDIO_SEND_STREAM_H_ |