solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #include "audio/audio_send_stream.h" |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 12 | |
Mirko Bonadei | 317a1f0 | 2019-09-17 17:06:18 +0200 | [diff] [blame] | 13 | #include <memory> |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 14 | #include <string> |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 15 | #include <utility> |
| 16 | #include <vector> |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 17 | |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 18 | #include "api/audio_codecs/audio_encoder.h" |
| 19 | #include "api/audio_codecs/audio_encoder_factory.h" |
| 20 | #include "api/audio_codecs/audio_format.h" |
| 21 | #include "api/call/transport.h" |
Steve Anton | 10542f2 | 2019-01-11 09:11:00 -0800 | [diff] [blame] | 22 | #include "api/crypto/frame_encryptor_interface.h" |
Artem Titov | 741daaf | 2019-03-21 14:37:36 +0100 | [diff] [blame] | 23 | #include "api/function_view.h" |
Danil Chapovalov | 83bbe91 | 2019-08-07 12:24:53 +0200 | [diff] [blame] | 24 | #include "api/rtc_event_log/rtc_event_log.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 25 | #include "audio/audio_state.h" |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 26 | #include "audio/channel_send.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 27 | #include "audio/conversion.h" |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 28 | #include "call/rtp_config.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 29 | #include "call/rtp_transport_controller_send_interface.h" |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 30 | #include "common_audio/vad/include/vad.h" |
Oskar Sundbom | 56ef305 | 2018-10-30 16:11:02 +0100 | [diff] [blame] | 31 | #include "logging/rtc_event_log/events/rtc_event_audio_send_stream_config.h" |
Oskar Sundbom | 56ef305 | 2018-10-30 16:11:02 +0100 | [diff] [blame] | 32 | #include "logging/rtc_event_log/rtc_stream_config.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 33 | #include "modules/audio_coding/codecs/cng/audio_encoder_cng.h" |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 34 | #include "modules/audio_processing/include/audio_processing.h" |
Sebastian Jansson | 6298b56 | 2020-01-14 17:55:19 +0100 | [diff] [blame] | 35 | #include "modules/rtp_rtcp/source/rtp_header_extensions.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 36 | #include "rtc_base/checks.h" |
| 37 | #include "rtc_base/event.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 38 | #include "rtc_base/logging.h" |
Jonas Olsson | abbe841 | 2018-04-03 13:40:05 +0200 | [diff] [blame] | 39 | #include "rtc_base/strings/audio_format_to_string.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 40 | #include "rtc_base/task_queue.h" |
Alex Narest | cedd351 | 2017-12-07 20:54:55 +0100 | [diff] [blame] | 41 | #include "system_wrappers/include/field_trial.h" |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 42 | |
| 43 | namespace webrtc { |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 44 | namespace { |
elad.alon | d12a8e1 | 2017-03-23 11:04:48 -0700 | [diff] [blame] | 45 | |
Oskar Sundbom | 56ef305 | 2018-10-30 16:11:02 +0100 | [diff] [blame] | 46 | void UpdateEventLogStreamConfig(RtcEventLog* event_log, |
| 47 | const AudioSendStream::Config& config, |
| 48 | const AudioSendStream::Config* old_config) { |
| 49 | using SendCodecSpec = AudioSendStream::Config::SendCodecSpec; |
| 50 | // Only update if any of the things we log have changed. |
| 51 | auto payload_types_equal = [](const absl::optional<SendCodecSpec>& a, |
| 52 | const absl::optional<SendCodecSpec>& b) { |
| 53 | if (a.has_value() && b.has_value()) { |
| 54 | return a->format.name == b->format.name && |
| 55 | a->payload_type == b->payload_type; |
| 56 | } |
| 57 | return !a.has_value() && !b.has_value(); |
| 58 | }; |
| 59 | |
| 60 | if (old_config && config.rtp.ssrc == old_config->rtp.ssrc && |
| 61 | config.rtp.extensions == old_config->rtp.extensions && |
| 62 | payload_types_equal(config.send_codec_spec, |
| 63 | old_config->send_codec_spec)) { |
| 64 | return; |
| 65 | } |
| 66 | |
Mirko Bonadei | 317a1f0 | 2019-09-17 17:06:18 +0200 | [diff] [blame] | 67 | auto rtclog_config = std::make_unique<rtclog::StreamConfig>(); |
Oskar Sundbom | 56ef305 | 2018-10-30 16:11:02 +0100 | [diff] [blame] | 68 | rtclog_config->local_ssrc = config.rtp.ssrc; |
| 69 | rtclog_config->rtp_extensions = config.rtp.extensions; |
| 70 | if (config.send_codec_spec) { |
| 71 | rtclog_config->codecs.emplace_back(config.send_codec_spec->format.name, |
| 72 | config.send_codec_spec->payload_type, 0); |
| 73 | } |
Mirko Bonadei | 317a1f0 | 2019-09-17 17:06:18 +0200 | [diff] [blame] | 74 | event_log->Log(std::make_unique<RtcEventAudioSendStreamConfig>( |
Oskar Sundbom | 56ef305 | 2018-10-30 16:11:02 +0100 | [diff] [blame] | 75 | std::move(rtclog_config))); |
| 76 | } |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 77 | } // namespace |
| 78 | |
Sebastian Jansson | f23131f | 2019-10-03 10:03:55 +0200 | [diff] [blame] | 79 | constexpr char AudioAllocationConfig::kKey[]; |
| 80 | |
| 81 | std::unique_ptr<StructParametersParser> AudioAllocationConfig::Parser() { |
| 82 | return StructParametersParser::Create( // |
| 83 | "min", &min_bitrate, // |
| 84 | "max", &max_bitrate, // |
| 85 | "prio_rate", &priority_bitrate, // |
| 86 | "prio_rate_raw", &priority_bitrate_raw, // |
| 87 | "rate_prio", &bitrate_priority); |
| 88 | } |
| 89 | |
| 90 | AudioAllocationConfig::AudioAllocationConfig() { |
| 91 | Parser()->Parse(field_trial::FindFullName(kKey)); |
| 92 | if (priority_bitrate_raw && !priority_bitrate.IsZero()) { |
| 93 | RTC_LOG(LS_WARNING) << "'priority_bitrate' and '_raw' are mutually " |
| 94 | "exclusive but both were configured."; |
| 95 | } |
| 96 | } |
| 97 | |
| 98 | namespace internal { |
solenberg | 566ef24 | 2015-11-06 15:34:49 -0800 | [diff] [blame] | 99 | AudioSendStream::AudioSendStream( |
Sebastian Jansson | 977b335 | 2019-03-04 17:43:34 +0100 | [diff] [blame] | 100 | Clock* clock, |
solenberg | 566ef24 | 2015-11-06 15:34:49 -0800 | [diff] [blame] | 101 | const webrtc::AudioSendStream::Config& config, |
Stefan Holmer | b86d4e4 | 2015-12-07 10:26:18 +0100 | [diff] [blame] | 102 | const rtc::scoped_refptr<webrtc::AudioState>& audio_state, |
Sebastian Jansson | 44dd9f2 | 2019-03-08 14:50:30 +0100 | [diff] [blame] | 103 | TaskQueueFactory* task_queue_factory, |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 104 | ProcessThread* module_process_thread, |
Niels Möller | 7d76a31 | 2018-10-26 12:57:07 +0200 | [diff] [blame] | 105 | RtpTransportControllerSendInterface* rtp_transport, |
Niels Möller | 67b011d | 2018-10-22 13:00:40 +0200 | [diff] [blame] | 106 | BitrateAllocatorInterface* bitrate_allocator, |
michaelt | 9332b7d | 2016-11-30 07:51:13 -0800 | [diff] [blame] | 107 | RtcEventLog* event_log, |
ossu | c3d4b48 | 2017-05-23 06:07:11 -0700 | [diff] [blame] | 108 | RtcpRttStats* rtcp_rtt_stats, |
Sam Zackrisson | ff05816 | 2018-11-20 17:15:13 +0100 | [diff] [blame] | 109 | const absl::optional<RtpState>& suspended_rtp_state) |
Sebastian Jansson | 977b335 | 2019-03-04 17:43:34 +0100 | [diff] [blame] | 110 | : AudioSendStream(clock, |
| 111 | config, |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 112 | audio_state, |
Sebastian Jansson | 44dd9f2 | 2019-03-08 14:50:30 +0100 | [diff] [blame] | 113 | task_queue_factory, |
Niels Möller | 7d76a31 | 2018-10-26 12:57:07 +0200 | [diff] [blame] | 114 | rtp_transport, |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 115 | bitrate_allocator, |
| 116 | event_log, |
| 117 | rtcp_rtt_stats, |
| 118 | suspended_rtp_state, |
Sebastian Jansson | 977b335 | 2019-03-04 17:43:34 +0100 | [diff] [blame] | 119 | voe::CreateChannelSend(clock, |
Sebastian Jansson | 44dd9f2 | 2019-03-08 14:50:30 +0100 | [diff] [blame] | 120 | task_queue_factory, |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 121 | module_process_thread, |
Anton Sukhanov | 626015d | 2019-02-04 15:16:06 -0800 | [diff] [blame] | 122 | /*overhead_observer=*/this, |
Niels Möller | e977199 | 2018-11-26 10:55:07 +0100 | [diff] [blame] | 123 | config.send_transport, |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 124 | rtcp_rtt_stats, |
| 125 | event_log, |
| 126 | config.frame_encryptor, |
| 127 | config.crypto_options, |
| 128 | config.rtp.extmap_allow_mixed, |
Erik Språng | 4c2c412 | 2019-07-11 15:20:15 +0200 | [diff] [blame] | 129 | config.rtcp_report_interval_ms, |
| 130 | config.rtp.ssrc)) {} |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 131 | |
| 132 | AudioSendStream::AudioSendStream( |
Sebastian Jansson | 977b335 | 2019-03-04 17:43:34 +0100 | [diff] [blame] | 133 | Clock* clock, |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 134 | const webrtc::AudioSendStream::Config& config, |
| 135 | const rtc::scoped_refptr<webrtc::AudioState>& audio_state, |
Sebastian Jansson | 44dd9f2 | 2019-03-08 14:50:30 +0100 | [diff] [blame] | 136 | TaskQueueFactory* task_queue_factory, |
Niels Möller | 7d76a31 | 2018-10-26 12:57:07 +0200 | [diff] [blame] | 137 | RtpTransportControllerSendInterface* rtp_transport, |
Niels Möller | 67b011d | 2018-10-22 13:00:40 +0200 | [diff] [blame] | 138 | BitrateAllocatorInterface* bitrate_allocator, |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 139 | RtcEventLog* event_log, |
| 140 | RtcpRttStats* rtcp_rtt_stats, |
Danil Chapovalov | b9b146c | 2018-06-15 12:28:07 +0200 | [diff] [blame] | 141 | const absl::optional<RtpState>& suspended_rtp_state, |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 142 | std::unique_ptr<voe::ChannelSendInterface> channel_send) |
Sebastian Jansson | 977b335 | 2019-03-04 17:43:34 +0100 | [diff] [blame] | 143 | : clock_(clock), |
Sebastian Jansson | 0b69826 | 2019-03-07 09:17:19 +0100 | [diff] [blame] | 144 | worker_queue_(rtp_transport->GetWorkerQueue()), |
Sebastian Jansson | f23131f | 2019-10-03 10:03:55 +0200 | [diff] [blame] | 145 | audio_send_side_bwe_(field_trial::IsEnabled("WebRTC-Audio-SendSideBwe")), |
| 146 | allocate_audio_without_feedback_( |
| 147 | field_trial::IsEnabled("WebRTC-Audio-ABWENoTWCC")), |
| 148 | enable_audio_alr_probing_( |
| 149 | !field_trial::IsDisabled("WebRTC-Audio-AlrProbing")), |
| 150 | send_side_bwe_with_overhead_( |
| 151 | field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")), |
Bjorn A Mellem | 7a9a092 | 2019-11-26 09:19:40 -0800 | [diff] [blame] | 152 | config_(Config(/*send_transport=*/nullptr)), |
mflodman | 86cc6ff | 2016-07-26 04:44:06 -0700 | [diff] [blame] | 153 | audio_state_(audio_state), |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 154 | channel_send_(std::move(channel_send)), |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 155 | event_log_(event_log), |
Sebastian Jansson | 62aee93 | 2019-10-02 12:27:06 +0200 | [diff] [blame] | 156 | use_legacy_overhead_calculation_( |
Sebastian Jansson | bef818d | 2020-01-30 14:09:48 +0100 | [diff] [blame] | 157 | field_trial::IsEnabled("WebRTC-Audio-LegacyOverhead")), |
michaelt | f4caaab | 2017-01-16 23:55:07 -0800 | [diff] [blame] | 158 | bitrate_allocator_(bitrate_allocator), |
Niels Möller | 7d76a31 | 2018-10-26 12:57:07 +0200 | [diff] [blame] | 159 | rtp_transport_(rtp_transport), |
Sebastian Jansson | 6298b56 | 2020-01-14 17:55:19 +0100 | [diff] [blame] | 160 | rtp_rtcp_module_(channel_send_->GetRtpRtcp()), |
Sam Zackrisson | ff05816 | 2018-11-20 17:15:13 +0100 | [diff] [blame] | 161 | suspended_rtp_state_(suspended_rtp_state) { |
Jonas Olsson | 24ea822 | 2018-01-25 10:14:29 +0100 | [diff] [blame] | 162 | RTC_LOG(LS_INFO) << "AudioSendStream: " << config.rtp.ssrc; |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 163 | RTC_DCHECK(worker_queue_); |
| 164 | RTC_DCHECK(audio_state_); |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 165 | RTC_DCHECK(channel_send_); |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 166 | RTC_DCHECK(bitrate_allocator_); |
Sebastian Jansson | 0b69826 | 2019-03-07 09:17:19 +0100 | [diff] [blame] | 167 | RTC_DCHECK(rtp_transport); |
| 168 | |
ossu | c3d4b48 | 2017-05-23 06:07:11 -0700 | [diff] [blame] | 169 | RTC_DCHECK(rtp_rtcp_module_); |
mflodman | 3d7db26 | 2016-04-29 00:57:13 -0700 | [diff] [blame] | 170 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 171 | ConfigureStream(config, true); |
elad.alon | d12a8e1 | 2017-03-23 11:04:48 -0700 | [diff] [blame] | 172 | |
Sebastian Jansson | c01367d | 2019-04-08 15:20:44 +0200 | [diff] [blame] | 173 | pacer_thread_checker_.Detach(); |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 174 | } |
| 175 | |
| 176 | AudioSendStream::~AudioSendStream() { |
Sebastian Jansson | c01367d | 2019-04-08 15:20:44 +0200 | [diff] [blame] | 177 | RTC_DCHECK(worker_thread_checker_.IsCurrent()); |
Jonas Olsson | 24ea822 | 2018-01-25 10:14:29 +0100 | [diff] [blame] | 178 | RTC_LOG(LS_INFO) << "~AudioSendStream: " << config_.rtp.ssrc; |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 179 | RTC_DCHECK(!sending_); |
Sebastian Jansson | 0a6510d | 2019-10-04 09:31:08 +0200 | [diff] [blame] | 180 | channel_send_->ResetSenderCongestionControlObjects(); |
Sebastian Jansson | 8672cac | 2019-03-01 15:57:55 +0100 | [diff] [blame] | 181 | // Blocking call to synchronize state with worker queue to ensure that there |
| 182 | // are no pending tasks left that keeps references to audio. |
| 183 | rtc::Event thread_sync_event; |
| 184 | worker_queue_->PostTask([&] { thread_sync_event.Set(); }); |
| 185 | thread_sync_event.Wait(rtc::Event::kForever); |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 186 | } |
| 187 | |
eladalon | abbc430 | 2017-07-26 02:09:44 -0700 | [diff] [blame] | 188 | const webrtc::AudioSendStream::Config& AudioSendStream::GetConfig() const { |
Sebastian Jansson | c01367d | 2019-04-08 15:20:44 +0200 | [diff] [blame] | 189 | RTC_DCHECK(worker_thread_checker_.IsCurrent()); |
eladalon | abbc430 | 2017-07-26 02:09:44 -0700 | [diff] [blame] | 190 | return config_; |
| 191 | } |
| 192 | |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 193 | void AudioSendStream::Reconfigure( |
| 194 | const webrtc::AudioSendStream::Config& new_config) { |
Sebastian Jansson | c01367d | 2019-04-08 15:20:44 +0200 | [diff] [blame] | 195 | RTC_DCHECK(worker_thread_checker_.IsCurrent()); |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 196 | ConfigureStream(new_config, false); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 197 | } |
| 198 | |
Alex Narest | cedd351 | 2017-12-07 20:54:55 +0100 | [diff] [blame] | 199 | AudioSendStream::ExtensionIds AudioSendStream::FindExtensionIds( |
| 200 | const std::vector<RtpExtension>& extensions) { |
| 201 | ExtensionIds ids; |
| 202 | for (const auto& extension : extensions) { |
| 203 | if (extension.uri == RtpExtension::kAudioLevelUri) { |
| 204 | ids.audio_level = extension.id; |
Sebastian Jansson | 71c6b56 | 2019-08-14 11:31:02 +0200 | [diff] [blame] | 205 | } else if (extension.uri == RtpExtension::kAbsSendTimeUri) { |
| 206 | ids.abs_send_time = extension.id; |
Alex Narest | cedd351 | 2017-12-07 20:54:55 +0100 | [diff] [blame] | 207 | } else if (extension.uri == RtpExtension::kTransportSequenceNumberUri) { |
| 208 | ids.transport_sequence_number = extension.id; |
Steve Anton | bb50ce5 | 2018-03-26 10:24:32 -0700 | [diff] [blame] | 209 | } else if (extension.uri == RtpExtension::kMidUri) { |
| 210 | ids.mid = extension.id; |
Amit Hilbuch | 77938e6 | 2018-12-21 09:23:38 -0800 | [diff] [blame] | 211 | } else if (extension.uri == RtpExtension::kRidUri) { |
| 212 | ids.rid = extension.id; |
| 213 | } else if (extension.uri == RtpExtension::kRepairedRidUri) { |
| 214 | ids.repaired_rid = extension.id; |
Minyue Li | 74dadc1 | 2020-03-05 11:33:13 +0100 | [diff] [blame] | 215 | } else if (extension.uri == RtpExtension::kAbsoluteCaptureTimeUri) { |
| 216 | ids.abs_capture_time = extension.id; |
Alex Narest | cedd351 | 2017-12-07 20:54:55 +0100 | [diff] [blame] | 217 | } |
| 218 | } |
| 219 | return ids; |
| 220 | } |
| 221 | |
Sebastian Jansson | 470a5ea | 2019-01-23 12:37:49 +0100 | [diff] [blame] | 222 | int AudioSendStream::TransportSeqNumId(const AudioSendStream::Config& config) { |
| 223 | return FindExtensionIds(config.rtp.extensions).transport_sequence_number; |
| 224 | } |
| 225 | |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 226 | void AudioSendStream::ConfigureStream( |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 227 | const webrtc::AudioSendStream::Config& new_config, |
| 228 | bool first_time) { |
Jonas Olsson | 24ea822 | 2018-01-25 10:14:29 +0100 | [diff] [blame] | 229 | RTC_LOG(LS_INFO) << "AudioSendStream::ConfigureStream: " |
| 230 | << new_config.ToString(); |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 231 | UpdateEventLogStreamConfig(event_log_, new_config, |
| 232 | first_time ? nullptr : &config_); |
Oskar Sundbom | 56ef305 | 2018-10-30 16:11:02 +0100 | [diff] [blame] | 233 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 234 | const auto& old_config = config_; |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 235 | |
Niels Möller | e977199 | 2018-11-26 10:55:07 +0100 | [diff] [blame] | 236 | // Configuration parameters which cannot be changed. |
| 237 | RTC_DCHECK(first_time || |
| 238 | old_config.send_transport == new_config.send_transport); |
Erik Språng | 70efdde | 2019-08-21 13:36:20 +0200 | [diff] [blame] | 239 | RTC_DCHECK(first_time || old_config.rtp.ssrc == new_config.rtp.ssrc); |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 240 | if (suspended_rtp_state_ && first_time) { |
| 241 | rtp_rtcp_module_->SetRtpState(*suspended_rtp_state_); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 242 | } |
| 243 | if (first_time || old_config.rtp.c_name != new_config.rtp.c_name) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 244 | channel_send_->SetRTCP_CNAME(new_config.rtp.c_name); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 245 | } |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 246 | |
Benjamin Wright | 84583f6 | 2018-10-04 14:22:34 -0700 | [diff] [blame] | 247 | // Enable the frame encryptor if a new frame encryptor has been provided. |
| 248 | if (first_time || new_config.frame_encryptor != old_config.frame_encryptor) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 249 | channel_send_->SetFrameEncryptor(new_config.frame_encryptor); |
Benjamin Wright | 84583f6 | 2018-10-04 14:22:34 -0700 | [diff] [blame] | 250 | } |
| 251 | |
Johannes Kron | 9190b82 | 2018-10-29 11:22:05 +0100 | [diff] [blame] | 252 | if (first_time || |
| 253 | new_config.rtp.extmap_allow_mixed != old_config.rtp.extmap_allow_mixed) { |
Sebastian Jansson | 6298b56 | 2020-01-14 17:55:19 +0100 | [diff] [blame] | 254 | rtp_rtcp_module_->SetExtmapAllowMixed(new_config.rtp.extmap_allow_mixed); |
Johannes Kron | 9190b82 | 2018-10-29 11:22:05 +0100 | [diff] [blame] | 255 | } |
| 256 | |
Alex Narest | cedd351 | 2017-12-07 20:54:55 +0100 | [diff] [blame] | 257 | const ExtensionIds old_ids = FindExtensionIds(old_config.rtp.extensions); |
| 258 | const ExtensionIds new_ids = FindExtensionIds(new_config.rtp.extensions); |
Yves Gerey | 1704801 | 2019-07-26 17:49:52 +0200 | [diff] [blame] | 259 | |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 260 | // Audio level indication |
| 261 | if (first_time || new_ids.audio_level != old_ids.audio_level) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 262 | channel_send_->SetSendAudioLevelIndicationStatus(new_ids.audio_level != 0, |
| 263 | new_ids.audio_level); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 264 | } |
Sebastian Jansson | 71c6b56 | 2019-08-14 11:31:02 +0200 | [diff] [blame] | 265 | |
| 266 | if (first_time || new_ids.abs_send_time != old_ids.abs_send_time) { |
Sebastian Jansson | 6298b56 | 2020-01-14 17:55:19 +0100 | [diff] [blame] | 267 | rtp_rtcp_module_->DeregisterSendRtpHeaderExtension( |
Sebastian Jansson | 71c6b56 | 2019-08-14 11:31:02 +0200 | [diff] [blame] | 268 | kRtpExtensionAbsoluteSendTime); |
| 269 | if (new_ids.abs_send_time) { |
Sebastian Jansson | f39c815 | 2019-10-14 17:32:21 +0200 | [diff] [blame] | 270 | rtp_rtcp_module_->RegisterRtpHeaderExtension(AbsoluteSendTime::kUri, |
| 271 | new_ids.abs_send_time); |
Sebastian Jansson | 71c6b56 | 2019-08-14 11:31:02 +0200 | [diff] [blame] | 272 | } |
| 273 | } |
| 274 | |
Sebastian Jansson | 8d9c540 | 2017-11-15 17:22:16 +0100 | [diff] [blame] | 275 | bool transport_seq_num_id_changed = |
| 276 | new_ids.transport_sequence_number != old_ids.transport_sequence_number; |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 277 | if (first_time || |
| 278 | (transport_seq_num_id_changed && !allocate_audio_without_feedback_)) { |
ossu | 1129df2 | 2017-06-30 01:38:56 -0700 | [diff] [blame] | 279 | if (!first_time) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 280 | channel_send_->ResetSenderCongestionControlObjects(); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 281 | } |
| 282 | |
Sebastian Jansson | 8d9c540 | 2017-11-15 17:22:16 +0100 | [diff] [blame] | 283 | RtcpBandwidthObserver* bandwidth_observer = nullptr; |
Sebastian Jansson | 470a5ea | 2019-01-23 12:37:49 +0100 | [diff] [blame] | 284 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 285 | if (audio_send_side_bwe_ && !allocate_audio_without_feedback_ && |
Sebastian Jansson | f23131f | 2019-10-03 10:03:55 +0200 | [diff] [blame] | 286 | new_ids.transport_sequence_number != 0) { |
Sebastian Jansson | 6298b56 | 2020-01-14 17:55:19 +0100 | [diff] [blame] | 287 | rtp_rtcp_module_->RegisterRtpHeaderExtension( |
| 288 | TransportSequenceNumber::kUri, new_ids.transport_sequence_number); |
Sebastian Jansson | 8d9c540 | 2017-11-15 17:22:16 +0100 | [diff] [blame] | 289 | // Probing in application limited region is only used in combination with |
| 290 | // send side congestion control, wich depends on feedback packets which |
| 291 | // requires transport sequence numbers to be enabled. |
Sebastian Jansson | 0a6510d | 2019-10-04 09:31:08 +0200 | [diff] [blame] | 292 | // Optionally request ALR probing but do not override any existing |
| 293 | // request from other streams. |
| 294 | if (enable_audio_alr_probing_) { |
| 295 | rtp_transport_->EnablePeriodicAlrProbing(true); |
Niels Möller | 7d76a31 | 2018-10-26 12:57:07 +0200 | [diff] [blame] | 296 | } |
Sebastian Jansson | 0a6510d | 2019-10-04 09:31:08 +0200 | [diff] [blame] | 297 | bandwidth_observer = rtp_transport_->GetBandwidthObserver(); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 298 | } |
Sebastian Jansson | 0a6510d | 2019-10-04 09:31:08 +0200 | [diff] [blame] | 299 | channel_send_->RegisterSenderCongestionControlObjects(rtp_transport_, |
| 300 | bandwidth_observer); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 301 | } |
Steve Anton | bb50ce5 | 2018-03-26 10:24:32 -0700 | [diff] [blame] | 302 | // MID RTP header extension. |
Steve Anton | 003930a | 2018-03-29 12:37:21 -0700 | [diff] [blame] | 303 | if ((first_time || new_ids.mid != old_ids.mid || |
| 304 | new_config.rtp.mid != old_config.rtp.mid) && |
| 305 | new_ids.mid != 0 && !new_config.rtp.mid.empty()) { |
Sebastian Jansson | 6298b56 | 2020-01-14 17:55:19 +0100 | [diff] [blame] | 306 | rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpMid::kUri, new_ids.mid); |
| 307 | rtp_rtcp_module_->SetMid(new_config.rtp.mid); |
Steve Anton | bb50ce5 | 2018-03-26 10:24:32 -0700 | [diff] [blame] | 308 | } |
| 309 | |
Amit Hilbuch | 77938e6 | 2018-12-21 09:23:38 -0800 | [diff] [blame] | 310 | // RID RTP header extension |
| 311 | if ((first_time || new_ids.rid != old_ids.rid || |
| 312 | new_ids.repaired_rid != old_ids.repaired_rid || |
| 313 | new_config.rtp.rid != old_config.rtp.rid)) { |
Sebastian Jansson | 6298b56 | 2020-01-14 17:55:19 +0100 | [diff] [blame] | 314 | if (new_ids.rid != 0 || new_ids.repaired_rid != 0) { |
| 315 | if (new_config.rtp.rid.empty()) { |
| 316 | rtp_rtcp_module_->DeregisterSendRtpHeaderExtension(RtpStreamId::kUri); |
| 317 | } else if (new_ids.repaired_rid != 0) { |
| 318 | rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpStreamId::kUri, |
| 319 | new_ids.repaired_rid); |
| 320 | } else { |
| 321 | rtp_rtcp_module_->RegisterRtpHeaderExtension(RtpStreamId::kUri, |
| 322 | new_ids.rid); |
| 323 | } |
| 324 | } |
| 325 | rtp_rtcp_module_->SetRid(new_config.rtp.rid); |
Amit Hilbuch | 77938e6 | 2018-12-21 09:23:38 -0800 | [diff] [blame] | 326 | } |
| 327 | |
Minyue Li | 74dadc1 | 2020-03-05 11:33:13 +0100 | [diff] [blame] | 328 | if (first_time || new_ids.abs_capture_time != old_ids.abs_capture_time) { |
| 329 | rtp_rtcp_module_->DeregisterSendRtpHeaderExtension( |
| 330 | kRtpExtensionAbsoluteCaptureTime); |
| 331 | if (new_ids.abs_capture_time) { |
| 332 | rtp_rtcp_module_->RegisterRtpHeaderExtension( |
| 333 | AbsoluteCaptureTimeExtension::kUri, new_ids.abs_capture_time); |
| 334 | } |
| 335 | } |
| 336 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 337 | if (!ReconfigureSendCodec(new_config)) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 338 | RTC_LOG(LS_ERROR) << "Failed to set up send codec state."; |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 339 | } |
| 340 | |
Jakob Ivarsson | d14525e | 2020-03-06 09:49:29 +0100 | [diff] [blame] | 341 | channel_send_->CallEncoder([this](AudioEncoder* encoder) { |
| 342 | if (!encoder) { |
| 343 | return; |
| 344 | } |
| 345 | worker_queue_->PostTask( |
| 346 | [this, length_range = encoder->GetFrameLengthRange()] { |
| 347 | RTC_DCHECK_RUN_ON(worker_queue_); |
| 348 | frame_length_range_ = length_range; |
| 349 | }); |
| 350 | }); |
| 351 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 352 | if (sending_) { |
| 353 | ReconfigureBitrateObserver(new_config); |
Oskar Sundbom | f85e31b | 2017-12-20 16:38:09 +0100 | [diff] [blame] | 354 | } |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 355 | config_ = new_config; |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 356 | } |
| 357 | |
solenberg | 3a94154 | 2015-11-16 07:34:50 -0800 | [diff] [blame] | 358 | void AudioSendStream::Start() { |
Sebastian Jansson | 8672cac | 2019-03-01 15:57:55 +0100 | [diff] [blame] | 359 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 360 | if (sending_) { |
| 361 | return; |
| 362 | } |
Sebastian Jansson | f23131f | 2019-10-03 10:03:55 +0200 | [diff] [blame] | 363 | if (!config_.has_dscp && config_.min_bitrate_bps != -1 && |
| 364 | config_.max_bitrate_bps != -1 && |
Sebastian Jansson | cd0eedb | 2019-10-10 13:52:26 +0200 | [diff] [blame] | 365 | (allocate_audio_without_feedback_ || TransportSeqNumId(config_) != 0)) { |
Erik Språng | aa59eca | 2019-07-24 14:52:55 +0200 | [diff] [blame] | 366 | rtp_transport_->AccountForAudioPacketsInPacedSender(true); |
Sebastian Jansson | c3eb9fd | 2020-01-29 17:42:52 +0100 | [diff] [blame] | 367 | if (send_side_bwe_with_overhead_) |
| 368 | rtp_transport_->IncludeOverheadInPacedSender(); |
Sebastian Jansson | b686396 | 2018-10-10 10:23:13 +0200 | [diff] [blame] | 369 | rtp_rtcp_module_->SetAsPartOfAllocation(true); |
Sebastian Jansson | 8672cac | 2019-03-01 15:57:55 +0100 | [diff] [blame] | 370 | rtc::Event thread_sync_event; |
| 371 | worker_queue_->PostTask([&] { |
| 372 | RTC_DCHECK_RUN_ON(worker_queue_); |
| 373 | ConfigureBitrateObserver(); |
| 374 | thread_sync_event.Set(); |
| 375 | }); |
| 376 | thread_sync_event.Wait(rtc::Event::kForever); |
Sebastian Jansson | b686396 | 2018-10-10 10:23:13 +0200 | [diff] [blame] | 377 | } else { |
| 378 | rtp_rtcp_module_->SetAsPartOfAllocation(false); |
mflodman | 86cc6ff | 2016-07-26 04:44:06 -0700 | [diff] [blame] | 379 | } |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 380 | channel_send_->StartSend(); |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 381 | sending_ = true; |
| 382 | audio_state()->AddSendingStream(this, encoder_sample_rate_hz_, |
| 383 | encoder_num_channels_); |
solenberg | 3a94154 | 2015-11-16 07:34:50 -0800 | [diff] [blame] | 384 | } |
| 385 | |
| 386 | void AudioSendStream::Stop() { |
Sebastian Jansson | c01367d | 2019-04-08 15:20:44 +0200 | [diff] [blame] | 387 | RTC_DCHECK(worker_thread_checker_.IsCurrent()); |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 388 | if (!sending_) { |
| 389 | return; |
| 390 | } |
| 391 | |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 392 | RemoveBitrateObserver(); |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 393 | channel_send_->StopSend(); |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 394 | sending_ = false; |
| 395 | audio_state()->RemoveSendingStream(this); |
| 396 | } |
| 397 | |
| 398 | void AudioSendStream::SendAudioData(std::unique_ptr<AudioFrame> audio_frame) { |
| 399 | RTC_CHECK_RUNS_SERIALIZED(&audio_capture_race_checker_); |
Henrik Boström | d2c336f | 2019-07-03 17:11:10 +0200 | [diff] [blame] | 400 | RTC_DCHECK_GT(audio_frame->sample_rate_hz_, 0); |
| 401 | double duration = static_cast<double>(audio_frame->samples_per_channel_) / |
| 402 | audio_frame->sample_rate_hz_; |
| 403 | { |
| 404 | // Note: SendAudioData() passes the frame further down the pipeline and it |
| 405 | // may eventually get sent. But this method is invoked even if we are not |
| 406 | // connected, as long as we have an AudioSendStream (created as a result of |
| 407 | // an O/A exchange). This means that we are calculating audio levels whether |
| 408 | // or not we are sending samples. |
| 409 | // TODO(https://crbug.com/webrtc/10771): All "media-source" related stats |
| 410 | // should move from send-streams to the local audio sources or tracks; a |
| 411 | // send-stream should not be required to read the microphone audio levels. |
| 412 | rtc::CritScope cs(&audio_level_lock_); |
| 413 | audio_level_.ComputeLevel(*audio_frame, duration); |
| 414 | } |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 415 | channel_send_->ProcessAndEncodeAudio(std::move(audio_frame)); |
solenberg | 3a94154 | 2015-11-16 07:34:50 -0800 | [diff] [blame] | 416 | } |
| 417 | |
solenberg | ffbbcac | 2016-11-17 05:25:37 -0800 | [diff] [blame] | 418 | bool AudioSendStream::SendTelephoneEvent(int payload_type, |
Yves Gerey | 665174f | 2018-06-19 15:03:05 +0200 | [diff] [blame] | 419 | int payload_frequency, |
| 420 | int event, |
solenberg | 8842c3e | 2016-03-11 03:06:41 -0800 | [diff] [blame] | 421 | int duration_ms) { |
Sebastian Jansson | c01367d | 2019-04-08 15:20:44 +0200 | [diff] [blame] | 422 | RTC_DCHECK(worker_thread_checker_.IsCurrent()); |
Niels Möller | 8fb1a6a | 2019-03-05 14:29:42 +0100 | [diff] [blame] | 423 | channel_send_->SetSendTelephoneEventPayloadType(payload_type, |
| 424 | payload_frequency); |
| 425 | return channel_send_->SendTelephoneEventOutband(event, duration_ms); |
Fredrik Solenberg | b572768 | 2015-12-04 15:22:19 +0100 | [diff] [blame] | 426 | } |
| 427 | |
solenberg | 9421853 | 2016-06-16 10:53:22 -0700 | [diff] [blame] | 428 | void AudioSendStream::SetMuted(bool muted) { |
Sebastian Jansson | c01367d | 2019-04-08 15:20:44 +0200 | [diff] [blame] | 429 | RTC_DCHECK(worker_thread_checker_.IsCurrent()); |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 430 | channel_send_->SetInputMute(muted); |
solenberg | 9421853 | 2016-06-16 10:53:22 -0700 | [diff] [blame] | 431 | } |
| 432 | |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 433 | webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const { |
Ivo Creusen | 56d4609 | 2017-11-24 17:29:59 +0100 | [diff] [blame] | 434 | return GetStats(true); |
| 435 | } |
| 436 | |
| 437 | webrtc::AudioSendStream::Stats AudioSendStream::GetStats( |
| 438 | bool has_remote_tracks) const { |
Sebastian Jansson | c01367d | 2019-04-08 15:20:44 +0200 | [diff] [blame] | 439 | RTC_DCHECK(worker_thread_checker_.IsCurrent()); |
solenberg | 85a0496 | 2015-10-27 03:35:21 -0700 | [diff] [blame] | 440 | webrtc::AudioSendStream::Stats stats; |
| 441 | stats.local_ssrc = config_.rtp.ssrc; |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 442 | stats.target_bitrate_bps = channel_send_->GetBitrate(); |
solenberg | 85a0496 | 2015-10-27 03:35:21 -0700 | [diff] [blame] | 443 | |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 444 | webrtc::CallSendStatistics call_stats = channel_send_->GetRTCPStatistics(); |
Niels Möller | ac0a4cb | 2019-10-09 15:01:33 +0200 | [diff] [blame] | 445 | stats.payload_bytes_sent = call_stats.payload_bytes_sent; |
| 446 | stats.header_and_padding_bytes_sent = |
| 447 | call_stats.header_and_padding_bytes_sent; |
Henrik Boström | cf96e0f | 2019-04-17 13:51:53 +0200 | [diff] [blame] | 448 | stats.retransmitted_bytes_sent = call_stats.retransmitted_bytes_sent; |
solenberg | 85a0496 | 2015-10-27 03:35:21 -0700 | [diff] [blame] | 449 | stats.packets_sent = call_stats.packetsSent; |
Henrik Boström | cf96e0f | 2019-04-17 13:51:53 +0200 | [diff] [blame] | 450 | stats.retransmitted_packets_sent = call_stats.retransmitted_packets_sent; |
solenberg | 8b85de2 | 2015-11-16 09:48:04 -0800 | [diff] [blame] | 451 | // RTT isn't known until a RTCP report is received. Until then, VoiceEngine |
| 452 | // returns 0 to indicate an error value. |
| 453 | if (call_stats.rttMs > 0) { |
| 454 | stats.rtt_ms = call_stats.rttMs; |
| 455 | } |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 456 | if (config_.send_codec_spec) { |
| 457 | const auto& spec = *config_.send_codec_spec; |
| 458 | stats.codec_name = spec.format.name; |
Oskar Sundbom | 2707fb2 | 2017-11-16 10:57:35 +0100 | [diff] [blame] | 459 | stats.codec_payload_type = spec.payload_type; |
solenberg | 85a0496 | 2015-10-27 03:35:21 -0700 | [diff] [blame] | 460 | |
| 461 | // Get data from the last remote RTCP report. |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 462 | for (const auto& block : channel_send_->GetRemoteRTCPReportBlocks()) { |
solenberg | 8b85de2 | 2015-11-16 09:48:04 -0800 | [diff] [blame] | 463 | // Lookup report for send ssrc only. |
| 464 | if (block.source_SSRC == stats.local_ssrc) { |
| 465 | stats.packets_lost = block.cumulative_num_packets_lost; |
| 466 | stats.fraction_lost = Q8ToFloat(block.fraction_lost); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 467 | // Convert timestamps to milliseconds. |
| 468 | if (spec.format.clockrate_hz / 1000 > 0) { |
solenberg | 8b85de2 | 2015-11-16 09:48:04 -0800 | [diff] [blame] | 469 | stats.jitter_ms = |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 470 | block.interarrival_jitter / (spec.format.clockrate_hz / 1000); |
solenberg | 85a0496 | 2015-10-27 03:35:21 -0700 | [diff] [blame] | 471 | } |
solenberg | 8b85de2 | 2015-11-16 09:48:04 -0800 | [diff] [blame] | 472 | break; |
solenberg | 85a0496 | 2015-10-27 03:35:21 -0700 | [diff] [blame] | 473 | } |
| 474 | } |
| 475 | } |
| 476 | |
Henrik Boström | d2c336f | 2019-07-03 17:11:10 +0200 | [diff] [blame] | 477 | { |
| 478 | rtc::CritScope cs(&audio_level_lock_); |
| 479 | stats.audio_level = audio_level_.LevelFullRange(); |
| 480 | stats.total_input_energy = audio_level_.TotalEnergy(); |
| 481 | stats.total_input_duration = audio_level_.TotalDuration(); |
| 482 | } |
solenberg | 796b8f9 | 2017-03-01 17:02:23 -0800 | [diff] [blame] | 483 | |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 484 | stats.typing_noise_detected = audio_state()->typing_noise_detected(); |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 485 | stats.ana_statistics = channel_send_->GetANAStatistics(); |
Ivo Creusen | 56d4609 | 2017-11-24 17:29:59 +0100 | [diff] [blame] | 486 | RTC_DCHECK(audio_state_->audio_processing()); |
| 487 | stats.apm_statistics = |
| 488 | audio_state_->audio_processing()->GetStatistics(has_remote_tracks); |
solenberg | 85a0496 | 2015-10-27 03:35:21 -0700 | [diff] [blame] | 489 | |
Henrik Boström | 6e436d1 | 2019-05-27 12:19:33 +0200 | [diff] [blame] | 490 | stats.report_block_datas = std::move(call_stats.report_block_datas); |
| 491 | |
solenberg | 85a0496 | 2015-10-27 03:35:21 -0700 | [diff] [blame] | 492 | return stats; |
| 493 | } |
| 494 | |
Niels Möller | 8fb1a6a | 2019-03-05 14:29:42 +0100 | [diff] [blame] | 495 | void AudioSendStream::DeliverRtcp(const uint8_t* packet, size_t length) { |
pbos | 1ba8d39 | 2016-05-01 20:18:34 -0700 | [diff] [blame] | 496 | // TODO(solenberg): Tests call this function on a network thread, libjingle |
| 497 | // calls on the worker thread. We should move towards always using a network |
| 498 | // thread. Then this check can be enabled. |
Sebastian Jansson | c01367d | 2019-04-08 15:20:44 +0200 | [diff] [blame] | 499 | // RTC_DCHECK(!worker_thread_checker_.IsCurrent()); |
Niels Möller | 8fb1a6a | 2019-03-05 14:29:42 +0100 | [diff] [blame] | 500 | channel_send_->ReceivedRTCPPacket(packet, length); |
pbos | 1ba8d39 | 2016-05-01 20:18:34 -0700 | [diff] [blame] | 501 | } |
| 502 | |
Sebastian Jansson | c0e4d45 | 2018-10-25 15:08:32 +0200 | [diff] [blame] | 503 | uint32_t AudioSendStream::OnBitrateUpdated(BitrateAllocationUpdate update) { |
Sebastian Jansson | 62aee93 | 2019-10-02 12:27:06 +0200 | [diff] [blame] | 504 | RTC_DCHECK_RUN_ON(worker_queue_); |
Daniel Lee | 9356252 | 2019-05-03 14:40:13 +0200 | [diff] [blame] | 505 | // Pick a target bitrate between the constraints. Overrules the allocator if |
| 506 | // it 1) allocated a bitrate of zero to disable the stream or 2) allocated a |
| 507 | // higher than max to allow for e.g. extra FEC. |
| 508 | auto constraints = GetMinMaxBitrateConstraints(); |
| 509 | update.target_bitrate.Clamp(constraints.min, constraints.max); |
mflodman | 86cc6ff | 2016-07-26 04:44:06 -0700 | [diff] [blame] | 510 | |
Sebastian Jansson | 254d869 | 2018-11-21 19:19:00 +0100 | [diff] [blame] | 511 | channel_send_->OnBitrateAllocation(update); |
mflodman | 86cc6ff | 2016-07-26 04:44:06 -0700 | [diff] [blame] | 512 | |
| 513 | // The amount of audio protection is not exposed by the encoder, hence |
| 514 | // always returning 0. |
| 515 | return 0; |
| 516 | } |
| 517 | |
Anton Sukhanov | 626015d | 2019-02-04 15:16:06 -0800 | [diff] [blame] | 518 | void AudioSendStream::SetTransportOverhead( |
| 519 | int transport_overhead_per_packet_bytes) { |
Sebastian Jansson | c01367d | 2019-04-08 15:20:44 +0200 | [diff] [blame] | 520 | RTC_DCHECK(worker_thread_checker_.IsCurrent()); |
Anton Sukhanov | 626015d | 2019-02-04 15:16:06 -0800 | [diff] [blame] | 521 | rtc::CritScope cs(&overhead_per_packet_lock_); |
| 522 | transport_overhead_per_packet_bytes_ = transport_overhead_per_packet_bytes; |
| 523 | UpdateOverheadForEncoder(); |
| 524 | } |
| 525 | |
| 526 | void AudioSendStream::OnOverheadChanged( |
| 527 | size_t overhead_bytes_per_packet_bytes) { |
| 528 | rtc::CritScope cs(&overhead_per_packet_lock_); |
| 529 | audio_overhead_per_packet_bytes_ = overhead_bytes_per_packet_bytes; |
| 530 | UpdateOverheadForEncoder(); |
| 531 | } |
| 532 | |
| 533 | void AudioSendStream::UpdateOverheadForEncoder() { |
| 534 | const size_t overhead_per_packet_bytes = GetPerPacketOverheadBytes(); |
Bjorn A Mellem | 413ccc4 | 2019-04-26 15:41:05 -0700 | [diff] [blame] | 535 | if (overhead_per_packet_bytes == 0) { |
| 536 | return; // Overhead is not known yet, do not tell the encoder. |
| 537 | } |
Sebastian Jansson | 14a7cf9 | 2019-02-13 15:11:42 +0100 | [diff] [blame] | 538 | channel_send_->CallEncoder([&](AudioEncoder* encoder) { |
| 539 | encoder->OnReceivedOverhead(overhead_per_packet_bytes); |
Anton Sukhanov | 626015d | 2019-02-04 15:16:06 -0800 | [diff] [blame] | 540 | }); |
Sebastian Jansson | 8672cac | 2019-03-01 15:57:55 +0100 | [diff] [blame] | 541 | worker_queue_->PostTask([this, overhead_per_packet_bytes] { |
| 542 | RTC_DCHECK_RUN_ON(worker_queue_); |
| 543 | if (total_packet_overhead_bytes_ != overhead_per_packet_bytes) { |
| 544 | total_packet_overhead_bytes_ = overhead_per_packet_bytes; |
| 545 | if (registered_with_allocator_) { |
| 546 | ConfigureBitrateObserver(); |
| 547 | } |
| 548 | } |
| 549 | }); |
Anton Sukhanov | 626015d | 2019-02-04 15:16:06 -0800 | [diff] [blame] | 550 | } |
| 551 | |
| 552 | size_t AudioSendStream::TestOnlyGetPerPacketOverheadBytes() const { |
| 553 | rtc::CritScope cs(&overhead_per_packet_lock_); |
| 554 | return GetPerPacketOverheadBytes(); |
| 555 | } |
| 556 | |
| 557 | size_t AudioSendStream::GetPerPacketOverheadBytes() const { |
| 558 | return transport_overhead_per_packet_bytes_ + |
| 559 | audio_overhead_per_packet_bytes_; |
michaelt | 79e0588 | 2016-11-08 02:50:09 -0800 | [diff] [blame] | 560 | } |
| 561 | |
ossu | c3d4b48 | 2017-05-23 06:07:11 -0700 | [diff] [blame] | 562 | RtpState AudioSendStream::GetRtpState() const { |
| 563 | return rtp_rtcp_module_->GetRtpState(); |
| 564 | } |
| 565 | |
Niels Möller | dced9f6 | 2018-11-19 10:27:07 +0100 | [diff] [blame] | 566 | const voe::ChannelSendInterface* AudioSendStream::GetChannel() const { |
| 567 | return channel_send_.get(); |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 568 | } |
| 569 | |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 570 | internal::AudioState* AudioSendStream::audio_state() { |
| 571 | internal::AudioState* audio_state = |
| 572 | static_cast<internal::AudioState*>(audio_state_.get()); |
| 573 | RTC_DCHECK(audio_state); |
| 574 | return audio_state; |
| 575 | } |
| 576 | |
| 577 | const internal::AudioState* AudioSendStream::audio_state() const { |
| 578 | internal::AudioState* audio_state = |
| 579 | static_cast<internal::AudioState*>(audio_state_.get()); |
| 580 | RTC_DCHECK(audio_state); |
| 581 | return audio_state; |
| 582 | } |
| 583 | |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 584 | void AudioSendStream::StoreEncoderProperties(int sample_rate_hz, |
| 585 | size_t num_channels) { |
Sebastian Jansson | c01367d | 2019-04-08 15:20:44 +0200 | [diff] [blame] | 586 | RTC_DCHECK(worker_thread_checker_.IsCurrent()); |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 587 | encoder_sample_rate_hz_ = sample_rate_hz; |
| 588 | encoder_num_channels_ = num_channels; |
| 589 | if (sending_) { |
| 590 | // Update AudioState's information about the stream. |
| 591 | audio_state()->AddSendingStream(this, sample_rate_hz, num_channels); |
| 592 | } |
| 593 | } |
| 594 | |
minyue | 7a97344 | 2016-10-20 03:27:12 -0700 | [diff] [blame] | 595 | // Apply current codec settings to a single voe::Channel used for sending. |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 596 | bool AudioSendStream::SetupSendCodec(const Config& new_config) { |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 597 | RTC_DCHECK(new_config.send_codec_spec); |
| 598 | const auto& spec = *new_config.send_codec_spec; |
minyue | 48368ad | 2017-05-10 04:06:11 -0700 | [diff] [blame] | 599 | |
| 600 | RTC_DCHECK(new_config.encoder_factory); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 601 | std::unique_ptr<AudioEncoder> encoder = |
Karl Wiberg | 77490b9 | 2018-03-21 15:18:42 +0100 | [diff] [blame] | 602 | new_config.encoder_factory->MakeAudioEncoder( |
| 603 | spec.payload_type, spec.format, new_config.codec_pair_id); |
minyue | 7a97344 | 2016-10-20 03:27:12 -0700 | [diff] [blame] | 604 | |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 605 | if (!encoder) { |
Jonas Olsson | abbe841 | 2018-04-03 13:40:05 +0200 | [diff] [blame] | 606 | RTC_DLOG(LS_ERROR) << "Unable to create encoder for " |
| 607 | << rtc::ToString(spec.format); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 608 | return false; |
| 609 | } |
Alex Narest | bbbe4e1 | 2018-07-13 10:32:58 +0200 | [diff] [blame] | 610 | |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 611 | // If a bitrate has been specified for the codec, use it over the |
| 612 | // codec's default. |
Christoffer Rodbro | 110c64b | 2019-03-06 09:51:08 +0100 | [diff] [blame] | 613 | if (spec.target_bitrate_bps) { |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 614 | encoder->OnReceivedTargetAudioBitrate(*spec.target_bitrate_bps); |
minyue | 7a97344 | 2016-10-20 03:27:12 -0700 | [diff] [blame] | 615 | } |
| 616 | |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 617 | // Enable ANA if configured (currently only used by Opus). |
Mirko Bonadei | 4356490 | 2020-01-29 15:29:36 +0000 | [diff] [blame] | 618 | if (new_config.audio_network_adaptor_config) { |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 619 | if (encoder->EnableAudioNetworkAdaptor( |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 620 | *new_config.audio_network_adaptor_config, event_log_)) { |
Jonas Olsson | 24ea822 | 2018-01-25 10:14:29 +0100 | [diff] [blame] | 621 | RTC_DLOG(LS_INFO) << "Audio network adaptor enabled on SSRC " |
| 622 | << new_config.rtp.ssrc; |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 623 | } else { |
Alejandro Luebs | c771084 | 2020-03-05 12:05:50 -0800 | [diff] [blame^] | 624 | RTC_DLOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC " |
| 625 | << new_config.rtp.ssrc; |
minyue | 6b825df | 2016-10-31 04:08:32 -0700 | [diff] [blame] | 626 | } |
minyue | 7a97344 | 2016-10-20 03:27:12 -0700 | [diff] [blame] | 627 | } |
| 628 | |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 629 | // Wrap the encoder in a an AudioEncoderCNG, if VAD is enabled. |
| 630 | if (spec.cng_payload_type) { |
Karl Wiberg | 2365936 | 2018-11-01 11:13:44 +0100 | [diff] [blame] | 631 | AudioEncoderCngConfig cng_config; |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 632 | cng_config.num_channels = encoder->NumChannels(); |
| 633 | cng_config.payload_type = *spec.cng_payload_type; |
| 634 | cng_config.speech_encoder = std::move(encoder); |
| 635 | cng_config.vad_mode = Vad::kVadNormal; |
Karl Wiberg | 2365936 | 2018-11-01 11:13:44 +0100 | [diff] [blame] | 636 | encoder = CreateComfortNoiseEncoder(std::move(cng_config)); |
ossu | 3b9ff38 | 2017-04-27 08:03:42 -0700 | [diff] [blame] | 637 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 638 | RegisterCngPayloadType(*spec.cng_payload_type, |
| 639 | new_config.send_codec_spec->format.clockrate_hz); |
minyue | 7a97344 | 2016-10-20 03:27:12 -0700 | [diff] [blame] | 640 | } |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 641 | |
Anton Sukhanov | 626015d | 2019-02-04 15:16:06 -0800 | [diff] [blame] | 642 | // Set currently known overhead (used in ANA, opus only). |
| 643 | // If overhead changes later, it will be updated in UpdateOverheadForEncoder. |
| 644 | { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 645 | rtc::CritScope cs(&overhead_per_packet_lock_); |
| 646 | if (GetPerPacketOverheadBytes() > 0) { |
| 647 | encoder->OnReceivedOverhead(GetPerPacketOverheadBytes()); |
Bjorn A Mellem | 413ccc4 | 2019-04-26 15:41:05 -0700 | [diff] [blame] | 648 | } |
Anton Sukhanov | 626015d | 2019-02-04 15:16:06 -0800 | [diff] [blame] | 649 | } |
| 650 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 651 | StoreEncoderProperties(encoder->SampleRateHz(), encoder->NumChannels()); |
| 652 | channel_send_->SetEncoder(new_config.send_codec_spec->payload_type, |
| 653 | std::move(encoder)); |
Anton Sukhanov | 626015d | 2019-02-04 15:16:06 -0800 | [diff] [blame] | 654 | |
minyue | 7a97344 | 2016-10-20 03:27:12 -0700 | [diff] [blame] | 655 | return true; |
| 656 | } |
| 657 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 658 | bool AudioSendStream::ReconfigureSendCodec(const Config& new_config) { |
| 659 | const auto& old_config = config_; |
minyue-webrtc | 8de1826 | 2017-07-26 14:18:40 +0200 | [diff] [blame] | 660 | |
| 661 | if (!new_config.send_codec_spec) { |
| 662 | // We cannot de-configure a send codec. So we will do nothing. |
| 663 | // By design, the send codec should have not been configured. |
| 664 | RTC_DCHECK(!old_config.send_codec_spec); |
| 665 | return true; |
| 666 | } |
| 667 | |
| 668 | if (new_config.send_codec_spec == old_config.send_codec_spec && |
| 669 | new_config.audio_network_adaptor_config == |
| 670 | old_config.audio_network_adaptor_config) { |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 671 | return true; |
| 672 | } |
| 673 | |
| 674 | // If we have no encoder, or the format or payload type's changed, create a |
| 675 | // new encoder. |
| 676 | if (!old_config.send_codec_spec || |
| 677 | new_config.send_codec_spec->format != |
| 678 | old_config.send_codec_spec->format || |
| 679 | new_config.send_codec_spec->payload_type != |
| 680 | old_config.send_codec_spec->payload_type) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 681 | return SetupSendCodec(new_config); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 682 | } |
| 683 | |
Danil Chapovalov | b9b146c | 2018-06-15 12:28:07 +0200 | [diff] [blame] | 684 | const absl::optional<int>& new_target_bitrate_bps = |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 685 | new_config.send_codec_spec->target_bitrate_bps; |
| 686 | // If a bitrate has been specified for the codec, use it over the |
| 687 | // codec's default. |
Christoffer Rodbro | 110c64b | 2019-03-06 09:51:08 +0100 | [diff] [blame] | 688 | if (new_target_bitrate_bps && |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 689 | new_target_bitrate_bps != |
| 690 | old_config.send_codec_spec->target_bitrate_bps) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 691 | channel_send_->CallEncoder([&](AudioEncoder* encoder) { |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 692 | encoder->OnReceivedTargetAudioBitrate(*new_target_bitrate_bps); |
| 693 | }); |
| 694 | } |
| 695 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 696 | ReconfigureANA(new_config); |
| 697 | ReconfigureCNG(new_config); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 698 | |
Anton Sukhanov | 626015d | 2019-02-04 15:16:06 -0800 | [diff] [blame] | 699 | // Set currently known overhead (used in ANA, opus only). |
| 700 | { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 701 | rtc::CritScope cs(&overhead_per_packet_lock_); |
| 702 | UpdateOverheadForEncoder(); |
Anton Sukhanov | 626015d | 2019-02-04 15:16:06 -0800 | [diff] [blame] | 703 | } |
| 704 | |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 705 | return true; |
| 706 | } |
| 707 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 708 | void AudioSendStream::ReconfigureANA(const Config& new_config) { |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 709 | if (new_config.audio_network_adaptor_config == |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 710 | config_.audio_network_adaptor_config) { |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 711 | return; |
| 712 | } |
Mirko Bonadei | 4356490 | 2020-01-29 15:29:36 +0000 | [diff] [blame] | 713 | if (new_config.audio_network_adaptor_config) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 714 | channel_send_->CallEncoder([&](AudioEncoder* encoder) { |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 715 | if (encoder->EnableAudioNetworkAdaptor( |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 716 | *new_config.audio_network_adaptor_config, event_log_)) { |
Jonas Olsson | 24ea822 | 2018-01-25 10:14:29 +0100 | [diff] [blame] | 717 | RTC_DLOG(LS_INFO) << "Audio network adaptor enabled on SSRC " |
| 718 | << new_config.rtp.ssrc; |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 719 | } else { |
Alejandro Luebs | c771084 | 2020-03-05 12:05:50 -0800 | [diff] [blame^] | 720 | RTC_DLOG(LS_INFO) << "Failed to enable Audio network adaptor on SSRC " |
| 721 | << new_config.rtp.ssrc; |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 722 | } |
| 723 | }); |
| 724 | } else { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 725 | channel_send_->CallEncoder( |
Sebastian Jansson | 14a7cf9 | 2019-02-13 15:11:42 +0100 | [diff] [blame] | 726 | [&](AudioEncoder* encoder) { encoder->DisableAudioNetworkAdaptor(); }); |
Jonas Olsson | 24ea822 | 2018-01-25 10:14:29 +0100 | [diff] [blame] | 727 | RTC_DLOG(LS_INFO) << "Audio network adaptor disabled on SSRC " |
| 728 | << new_config.rtp.ssrc; |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 729 | } |
| 730 | } |
| 731 | |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 732 | void AudioSendStream::ReconfigureCNG(const Config& new_config) { |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 733 | if (new_config.send_codec_spec->cng_payload_type == |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 734 | config_.send_codec_spec->cng_payload_type) { |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 735 | return; |
| 736 | } |
| 737 | |
ossu | 3b9ff38 | 2017-04-27 08:03:42 -0700 | [diff] [blame] | 738 | // Register the CNG payload type if it's been added, don't do anything if CNG |
| 739 | // is removed. Payload types must not be redefined. |
| 740 | if (new_config.send_codec_spec->cng_payload_type) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 741 | RegisterCngPayloadType(*new_config.send_codec_spec->cng_payload_type, |
| 742 | new_config.send_codec_spec->format.clockrate_hz); |
ossu | 3b9ff38 | 2017-04-27 08:03:42 -0700 | [diff] [blame] | 743 | } |
| 744 | |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 745 | // Wrap or unwrap the encoder in an AudioEncoderCNG. |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 746 | channel_send_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder_ptr) { |
| 747 | std::unique_ptr<AudioEncoder> old_encoder(std::move(*encoder_ptr)); |
| 748 | auto sub_encoders = old_encoder->ReclaimContainedEncoders(); |
| 749 | if (!sub_encoders.empty()) { |
| 750 | // Replace enc with its sub encoder. We need to put the sub |
| 751 | // encoder in a temporary first, since otherwise the old value |
| 752 | // of enc would be destroyed before the new value got assigned, |
| 753 | // which would be bad since the new value is a part of the old |
| 754 | // value. |
| 755 | auto tmp = std::move(sub_encoders[0]); |
| 756 | old_encoder = std::move(tmp); |
| 757 | } |
| 758 | if (new_config.send_codec_spec->cng_payload_type) { |
| 759 | AudioEncoderCngConfig config; |
| 760 | config.speech_encoder = std::move(old_encoder); |
| 761 | config.num_channels = config.speech_encoder->NumChannels(); |
| 762 | config.payload_type = *new_config.send_codec_spec->cng_payload_type; |
| 763 | config.vad_mode = Vad::kVadNormal; |
| 764 | *encoder_ptr = CreateComfortNoiseEncoder(std::move(config)); |
| 765 | } else { |
| 766 | *encoder_ptr = std::move(old_encoder); |
| 767 | } |
| 768 | }); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 769 | } |
| 770 | |
| 771 | void AudioSendStream::ReconfigureBitrateObserver( |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 772 | const webrtc::AudioSendStream::Config& new_config) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 773 | RTC_DCHECK_RUN_ON(&worker_thread_checker_); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 774 | // Since the Config's default is for both of these to be -1, this test will |
| 775 | // allow us to configure the bitrate observer if the new config has bitrate |
| 776 | // limits set, but would only have us call RemoveBitrateObserver if we were |
| 777 | // previously configured with bitrate limits. |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 778 | if (config_.min_bitrate_bps == new_config.min_bitrate_bps && |
| 779 | config_.max_bitrate_bps == new_config.max_bitrate_bps && |
| 780 | config_.bitrate_priority == new_config.bitrate_priority && |
| 781 | (TransportSeqNumId(config_) == TransportSeqNumId(new_config) || |
Jakob Ivarsson | d14525e | 2020-03-06 09:49:29 +0100 | [diff] [blame] | 782 | !audio_send_side_bwe_) && |
| 783 | config_.audio_network_adaptor_config == |
| 784 | new_config.audio_network_adaptor_config) { |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 785 | return; |
| 786 | } |
| 787 | |
Sebastian Jansson | f23131f | 2019-10-03 10:03:55 +0200 | [diff] [blame] | 788 | if (!new_config.has_dscp && new_config.min_bitrate_bps != -1 && |
Sebastian Jansson | cd0eedb | 2019-10-10 13:52:26 +0200 | [diff] [blame] | 789 | new_config.max_bitrate_bps != -1 && TransportSeqNumId(new_config) != 0) { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 790 | rtp_transport_->AccountForAudioPacketsInPacedSender(true); |
Sebastian Jansson | c3eb9fd | 2020-01-29 17:42:52 +0100 | [diff] [blame] | 791 | if (send_side_bwe_with_overhead_) |
| 792 | rtp_transport_->IncludeOverheadInPacedSender(); |
Sebastian Jansson | 8672cac | 2019-03-01 15:57:55 +0100 | [diff] [blame] | 793 | rtc::Event thread_sync_event; |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 794 | worker_queue_->PostTask([&] { |
| 795 | RTC_DCHECK_RUN_ON(worker_queue_); |
Sebastian Jansson | 8672cac | 2019-03-01 15:57:55 +0100 | [diff] [blame] | 796 | // We may get a callback immediately as the observer is registered, so |
| 797 | // make |
| 798 | // sure the bitrate limits in config_ are up-to-date. |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 799 | config_.min_bitrate_bps = new_config.min_bitrate_bps; |
| 800 | config_.max_bitrate_bps = new_config.max_bitrate_bps; |
| 801 | |
| 802 | config_.bitrate_priority = new_config.bitrate_priority; |
| 803 | ConfigureBitrateObserver(); |
Sebastian Jansson | 8672cac | 2019-03-01 15:57:55 +0100 | [diff] [blame] | 804 | thread_sync_event.Set(); |
| 805 | }); |
| 806 | thread_sync_event.Wait(rtc::Event::kForever); |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 807 | rtp_rtcp_module_->SetAsPartOfAllocation(true); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 808 | } else { |
Sebastian Jansson | 35cf9e7 | 2019-10-04 09:30:32 +0200 | [diff] [blame] | 809 | rtp_transport_->AccountForAudioPacketsInPacedSender(false); |
| 810 | RemoveBitrateObserver(); |
| 811 | rtp_rtcp_module_->SetAsPartOfAllocation(false); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 812 | } |
| 813 | } |
| 814 | |
Sebastian Jansson | 8672cac | 2019-03-01 15:57:55 +0100 | [diff] [blame] | 815 | void AudioSendStream::ConfigureBitrateObserver() { |
| 816 | // This either updates the current observer or adds a new observer. |
| 817 | // TODO(srte): Add overhead compensation here. |
Daniel Lee | 9356252 | 2019-05-03 14:40:13 +0200 | [diff] [blame] | 818 | auto constraints = GetMinMaxBitrateConstraints(); |
| 819 | |
Sebastian Jansson | 0429f78 | 2019-10-03 18:32:45 +0200 | [diff] [blame] | 820 | DataRate priority_bitrate = allocation_settings_.priority_bitrate; |
Sebastian Jansson | f23131f | 2019-10-03 10:03:55 +0200 | [diff] [blame] | 821 | if (send_side_bwe_with_overhead_) { |
Sebastian Jansson | 0429f78 | 2019-10-03 18:32:45 +0200 | [diff] [blame] | 822 | if (use_legacy_overhead_calculation_) { |
| 823 | // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12) |
| 824 | constexpr int kOverheadPerPacket = 20 + 8 + 10 + 12; |
Danil Chapovalov | 0c626af | 2020-02-10 11:16:00 +0100 | [diff] [blame] | 825 | const TimeDelta kMinPacketDuration = TimeDelta::Millis(20); |
Sebastian Jansson | 0429f78 | 2019-10-03 18:32:45 +0200 | [diff] [blame] | 826 | DataRate max_overhead = |
Danil Chapovalov | cad3e0e | 2020-02-17 18:46:07 +0100 | [diff] [blame] | 827 | DataSize::Bytes(kOverheadPerPacket) / kMinPacketDuration; |
Sebastian Jansson | 0429f78 | 2019-10-03 18:32:45 +0200 | [diff] [blame] | 828 | priority_bitrate += max_overhead; |
| 829 | } else { |
| 830 | RTC_DCHECK(frame_length_range_); |
| 831 | const DataSize kOverheadPerPacket = |
Danil Chapovalov | cad3e0e | 2020-02-17 18:46:07 +0100 | [diff] [blame] | 832 | DataSize::Bytes(total_packet_overhead_bytes_); |
Jakob Ivarsson | 01ab084 | 2020-03-06 09:59:56 +0100 | [diff] [blame] | 833 | DataRate min_overhead = kOverheadPerPacket / frame_length_range_->second; |
| 834 | priority_bitrate += min_overhead; |
Sebastian Jansson | 0429f78 | 2019-10-03 18:32:45 +0200 | [diff] [blame] | 835 | } |
Sebastian Jansson | f23131f | 2019-10-03 10:03:55 +0200 | [diff] [blame] | 836 | } |
Sebastian Jansson | f23131f | 2019-10-03 10:03:55 +0200 | [diff] [blame] | 837 | if (allocation_settings_.priority_bitrate_raw) |
| 838 | priority_bitrate = *allocation_settings_.priority_bitrate_raw; |
| 839 | |
Sebastian Jansson | 8672cac | 2019-03-01 15:57:55 +0100 | [diff] [blame] | 840 | bitrate_allocator_->AddObserver( |
Daniel Lee | 9356252 | 2019-05-03 14:40:13 +0200 | [diff] [blame] | 841 | this, |
| 842 | MediaStreamAllocationConfig{ |
| 843 | constraints.min.bps<uint32_t>(), constraints.max.bps<uint32_t>(), 0, |
Sebastian Jansson | f23131f | 2019-10-03 10:03:55 +0200 | [diff] [blame] | 844 | priority_bitrate.bps(), true, |
| 845 | allocation_settings_.bitrate_priority.value_or( |
Jonas Olsson | 8f119ca | 2019-05-08 10:56:23 +0200 | [diff] [blame] | 846 | config_.bitrate_priority)}); |
Jakob Ivarsson | d14525e | 2020-03-06 09:49:29 +0100 | [diff] [blame] | 847 | registered_with_allocator_ = true; |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 848 | } |
| 849 | |
| 850 | void AudioSendStream::RemoveBitrateObserver() { |
Sebastian Jansson | c01367d | 2019-04-08 15:20:44 +0200 | [diff] [blame] | 851 | RTC_DCHECK(worker_thread_checker_.IsCurrent()); |
Niels Möller | c572ff3 | 2018-11-07 08:43:50 +0100 | [diff] [blame] | 852 | rtc::Event thread_sync_event; |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 853 | worker_queue_->PostTask([this, &thread_sync_event] { |
Sebastian Jansson | 8672cac | 2019-03-01 15:57:55 +0100 | [diff] [blame] | 854 | RTC_DCHECK_RUN_ON(worker_queue_); |
| 855 | registered_with_allocator_ = false; |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 856 | bitrate_allocator_->RemoveObserver(this); |
| 857 | thread_sync_event.Set(); |
| 858 | }); |
| 859 | thread_sync_event.Wait(rtc::Event::kForever); |
| 860 | } |
| 861 | |
Daniel Lee | 9356252 | 2019-05-03 14:40:13 +0200 | [diff] [blame] | 862 | AudioSendStream::TargetAudioBitrateConstraints |
| 863 | AudioSendStream::GetMinMaxBitrateConstraints() const { |
| 864 | TargetAudioBitrateConstraints constraints{ |
Danil Chapovalov | cad3e0e | 2020-02-17 18:46:07 +0100 | [diff] [blame] | 865 | DataRate::BitsPerSec(config_.min_bitrate_bps), |
| 866 | DataRate::BitsPerSec(config_.max_bitrate_bps)}; |
Daniel Lee | 9356252 | 2019-05-03 14:40:13 +0200 | [diff] [blame] | 867 | |
| 868 | // If bitrates were explicitly overriden via field trial, use those values. |
Sebastian Jansson | f23131f | 2019-10-03 10:03:55 +0200 | [diff] [blame] | 869 | if (allocation_settings_.min_bitrate) |
| 870 | constraints.min = *allocation_settings_.min_bitrate; |
| 871 | if (allocation_settings_.max_bitrate) |
| 872 | constraints.max = *allocation_settings_.max_bitrate; |
Daniel Lee | 9356252 | 2019-05-03 14:40:13 +0200 | [diff] [blame] | 873 | |
Sebastian Jansson | 62aee93 | 2019-10-02 12:27:06 +0200 | [diff] [blame] | 874 | RTC_DCHECK_GE(constraints.min, DataRate::Zero()); |
| 875 | RTC_DCHECK_GE(constraints.max, DataRate::Zero()); |
| 876 | RTC_DCHECK_GE(constraints.max, constraints.min); |
Sebastian Jansson | f23131f | 2019-10-03 10:03:55 +0200 | [diff] [blame] | 877 | if (send_side_bwe_with_overhead_) { |
Sebastian Jansson | 62aee93 | 2019-10-02 12:27:06 +0200 | [diff] [blame] | 878 | if (use_legacy_overhead_calculation_) { |
| 879 | // OverheadPerPacket = Ipv4(20B) + UDP(8B) + SRTP(10B) + RTP(12) |
Danil Chapovalov | cad3e0e | 2020-02-17 18:46:07 +0100 | [diff] [blame] | 880 | const DataSize kOverheadPerPacket = DataSize::Bytes(20 + 8 + 10 + 12); |
Sebastian Jansson | 62aee93 | 2019-10-02 12:27:06 +0200 | [diff] [blame] | 881 | const TimeDelta kMaxFrameLength = |
Danil Chapovalov | 0c626af | 2020-02-10 11:16:00 +0100 | [diff] [blame] | 882 | TimeDelta::Millis(60); // Based on Opus spec |
Sebastian Jansson | 62aee93 | 2019-10-02 12:27:06 +0200 | [diff] [blame] | 883 | const DataRate kMinOverhead = kOverheadPerPacket / kMaxFrameLength; |
| 884 | constraints.min += kMinOverhead; |
| 885 | constraints.max += kMinOverhead; |
| 886 | } else { |
| 887 | RTC_DCHECK(frame_length_range_); |
| 888 | const DataSize kOverheadPerPacket = |
Danil Chapovalov | cad3e0e | 2020-02-17 18:46:07 +0100 | [diff] [blame] | 889 | DataSize::Bytes(total_packet_overhead_bytes_); |
Sebastian Jansson | 62aee93 | 2019-10-02 12:27:06 +0200 | [diff] [blame] | 890 | constraints.min += kOverheadPerPacket / frame_length_range_->second; |
| 891 | constraints.max += kOverheadPerPacket / frame_length_range_->first; |
| 892 | } |
Daniel Lee | 9356252 | 2019-05-03 14:40:13 +0200 | [diff] [blame] | 893 | } |
| 894 | return constraints; |
| 895 | } |
| 896 | |
ossu | 3b9ff38 | 2017-04-27 08:03:42 -0700 | [diff] [blame] | 897 | void AudioSendStream::RegisterCngPayloadType(int payload_type, |
| 898 | int clockrate_hz) { |
Niels Möller | ee5ccbc | 2019-03-06 16:47:29 +0100 | [diff] [blame] | 899 | channel_send_->RegisterCngPayloadType(payload_type, clockrate_hz); |
ossu | 3b9ff38 | 2017-04-27 08:03:42 -0700 | [diff] [blame] | 900 | } |
solenberg | c7a8b08 | 2015-10-16 14:35:07 -0700 | [diff] [blame] | 901 | } // namespace internal |
| 902 | } // namespace webrtc |