blob: c458fe465b07080d4f12632b81f817b62a3d1444 [file] [log] [blame]
Niels Möller530ead42018-10-04 14:28:39 +02001/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "audio/channel_send.h"
12
13#include <algorithm>
14#include <map>
15#include <memory>
16#include <string>
17#include <utility>
18#include <vector>
19
20#include "absl/memory/memory.h"
21#include "api/array_view.h"
Niels Möllerdced9f62018-11-19 10:27:07 +010022#include "api/call/transport.h"
Benjamin Wright84583f62018-10-04 14:22:34 -070023#include "api/crypto/frameencryptorinterface.h"
Niels Möller530ead42018-10-04 14:28:39 +020024#include "audio/utility/audio_frame_operations.h"
25#include "call/rtp_transport_controller_send_interface.h"
26#include "logging/rtc_event_log/events/rtc_event_audio_playout.h"
27#include "logging/rtc_event_log/rtc_event_log.h"
28#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h"
Niels Möllerdced9f62018-11-19 10:27:07 +010029#include "modules/audio_coding/include/audio_coding_module.h"
30#include "modules/audio_processing/rms_level.h"
Niels Möller530ead42018-10-04 14:28:39 +020031#include "modules/pacing/packet_router.h"
32#include "modules/utility/include/process_thread.h"
33#include "rtc_base/checks.h"
34#include "rtc_base/criticalsection.h"
Yves Gerey2e00abc2018-10-05 15:39:24 +020035#include "rtc_base/event.h"
Niels Möller530ead42018-10-04 14:28:39 +020036#include "rtc_base/format_macros.h"
37#include "rtc_base/location.h"
38#include "rtc_base/logging.h"
Niels Möller26815232018-11-16 09:32:40 +010039#include "rtc_base/numerics/safe_conversions.h"
Niels Möllerdced9f62018-11-19 10:27:07 +010040#include "rtc_base/race_checker.h"
Niels Möller530ead42018-10-04 14:28:39 +020041#include "rtc_base/rate_limiter.h"
42#include "rtc_base/task_queue.h"
43#include "rtc_base/thread_checker.h"
44#include "rtc_base/timeutils.h"
45#include "system_wrappers/include/field_trial.h"
46#include "system_wrappers/include/metrics.h"
47
48namespace webrtc {
49namespace voe {
50
51namespace {
52
53constexpr int64_t kMaxRetransmissionWindowMs = 1000;
54constexpr int64_t kMinRetransmissionWindowMs = 30;
55
Niels Möller7d76a312018-10-26 12:57:07 +020056MediaTransportEncodedAudioFrame::FrameType
57MediaTransportFrameTypeForWebrtcFrameType(webrtc::FrameType frame_type) {
58 switch (frame_type) {
59 case kAudioFrameSpeech:
60 return MediaTransportEncodedAudioFrame::FrameType::kSpeech;
61 break;
62
63 case kAudioFrameCN:
64 return MediaTransportEncodedAudioFrame::FrameType::
65 kDiscontinuousTransmission;
66 break;
67
68 default:
69 RTC_CHECK(false) << "Unexpected frame type=" << frame_type;
70 break;
71 }
72}
73
Niels Möllerdced9f62018-11-19 10:27:07 +010074class RtpPacketSenderProxy;
75class TransportFeedbackProxy;
76class TransportSequenceNumberProxy;
77class VoERtcpObserver;
78
Niels Möllerdced9f62018-11-19 10:27:07 +010079class ChannelSend
80 : public ChannelSendInterface,
81 public Transport,
82 public OverheadObserver,
83 public AudioPacketizationCallback, // receive encoded packets from the
84 // ACM
85 public TargetTransferRateObserver {
86 public:
87 // TODO(nisse): Make OnUplinkPacketLossRate public, and delete friend
88 // declaration.
89 friend class VoERtcpObserver;
90
91 ChannelSend(rtc::TaskQueue* encoder_queue,
92 ProcessThread* module_process_thread,
93 MediaTransportInterface* media_transport,
Niels Möllere9771992018-11-26 10:55:07 +010094 Transport* rtp_transport,
Niels Möllerdced9f62018-11-19 10:27:07 +010095 RtcpRttStats* rtcp_rtt_stats,
96 RtcEventLog* rtc_event_log,
97 FrameEncryptorInterface* frame_encryptor,
98 const webrtc::CryptoOptions& crypto_options,
99 bool extmap_allow_mixed,
100 int rtcp_report_interval_ms);
101
102 ~ChannelSend() override;
103
104 // Send using this encoder, with this payload type.
105 bool SetEncoder(int payload_type,
106 std::unique_ptr<AudioEncoder> encoder) override;
107 void ModifyEncoder(rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)>
108 modifier) override;
109
110 // API methods
Niels Möllerdced9f62018-11-19 10:27:07 +0100111 void StartSend() override;
112 void StopSend() override;
113
114 // Codecs
Sebastian Jansson254d8692018-11-21 19:19:00 +0100115 void OnBitrateAllocation(BitrateAllocationUpdate update) override;
Niels Möllerdced9f62018-11-19 10:27:07 +0100116 int GetBitrate() const override;
117
118 // Network
Niels Möllerdced9f62018-11-19 10:27:07 +0100119 bool ReceivedRTCPPacket(const uint8_t* data, size_t length) override;
120
121 // Muting, Volume and Level.
122 void SetInputMute(bool enable) override;
123
124 // Stats.
125 ANAStats GetANAStatistics() const override;
126
127 // Used by AudioSendStream.
128 RtpRtcp* GetRtpRtcp() const override;
129
130 // DTMF.
131 bool SendTelephoneEventOutband(int event, int duration_ms) override;
132 bool SetSendTelephoneEventPayloadType(int payload_type,
133 int payload_frequency) override;
134
135 // RTP+RTCP
136 void SetLocalSSRC(uint32_t ssrc) override;
Niels Möllerdced9f62018-11-19 10:27:07 +0100137 void SetMid(const std::string& mid, int extension_id) override;
138 void SetExtmapAllowMixed(bool extmap_allow_mixed) override;
139 void SetSendAudioLevelIndicationStatus(bool enable, int id) override;
140 void EnableSendTransportSequenceNumber(int id) override;
141
142 void RegisterSenderCongestionControlObjects(
143 RtpTransportControllerSendInterface* transport,
144 RtcpBandwidthObserver* bandwidth_observer) override;
145 void ResetSenderCongestionControlObjects() override;
146 void SetRTCP_CNAME(absl::string_view c_name) override;
147 std::vector<ReportBlock> GetRemoteRTCPReportBlocks() const override;
148 CallSendStatistics GetRTCPStatistics() const override;
Niels Möllerdced9f62018-11-19 10:27:07 +0100149
150 // ProcessAndEncodeAudio() posts a task on the shared encoder task queue,
151 // which in turn calls (on the queue) ProcessAndEncodeAudioOnTaskQueue() where
152 // the actual processing of the audio takes place. The processing mainly
153 // consists of encoding and preparing the result for sending by adding it to a
154 // send queue.
155 // The main reason for using a task queue here is to release the native,
156 // OS-specific, audio capture thread as soon as possible to ensure that it
157 // can go back to sleep and be prepared to deliver an new captured audio
158 // packet.
159 void ProcessAndEncodeAudio(std::unique_ptr<AudioFrame> audio_frame) override;
160
161 void SetTransportOverhead(size_t transport_overhead_per_packet) override;
162
163 // The existence of this function alongside OnUplinkPacketLossRate is
164 // a compromise. We want the encoder to be agnostic of the PLR source, but
165 // we also don't want it to receive conflicting information from TWCC and
166 // from RTCP-XR.
167 void OnTwccBasedUplinkPacketLossRate(float packet_loss_rate) override;
168
169 void OnRecoverableUplinkPacketLossRate(
170 float recoverable_packet_loss_rate) override;
171
172 int64_t GetRTT() const override;
173
174 // E2EE Custom Audio Frame Encryption
175 void SetFrameEncryptor(
176 rtc::scoped_refptr<FrameEncryptorInterface> frame_encryptor) override;
177
178 private:
179 class ProcessAndEncodeAudioTask;
180
181 // From AudioPacketizationCallback in the ACM
182 int32_t SendData(FrameType frameType,
183 uint8_t payloadType,
184 uint32_t timeStamp,
185 const uint8_t* payloadData,
186 size_t payloadSize,
187 const RTPFragmentationHeader* fragmentation) override;
188
189 // From Transport (called by the RTP/RTCP module)
190 bool SendRtp(const uint8_t* data,
191 size_t len,
192 const PacketOptions& packet_options) override;
193 bool SendRtcp(const uint8_t* data, size_t len) override;
194
Niels Möllerdced9f62018-11-19 10:27:07 +0100195 // From OverheadObserver in the RTP/RTCP module
196 void OnOverheadChanged(size_t overhead_bytes_per_packet) override;
197
198 void OnUplinkPacketLossRate(float packet_loss_rate);
199 bool InputMute() const;
200
Niels Möllerdced9f62018-11-19 10:27:07 +0100201 int SetSendRtpHeaderExtension(bool enable, RTPExtensionType type, int id);
202
203 void UpdateOverheadForEncoder()
204 RTC_EXCLUSIVE_LOCKS_REQUIRED(overhead_per_packet_lock_);
205
Niels Möllerdced9f62018-11-19 10:27:07 +0100206 int32_t SendRtpAudio(FrameType frameType,
207 uint8_t payloadType,
208 uint32_t timeStamp,
209 rtc::ArrayView<const uint8_t> payload,
210 const RTPFragmentationHeader* fragmentation);
211
212 int32_t SendMediaTransportAudio(FrameType frameType,
213 uint8_t payloadType,
214 uint32_t timeStamp,
215 rtc::ArrayView<const uint8_t> payload,
216 const RTPFragmentationHeader* fragmentation);
217
218 // Return media transport or nullptr if using RTP.
219 MediaTransportInterface* media_transport() { return media_transport_; }
220
221 // Called on the encoder task queue when a new input audio frame is ready
222 // for encoding.
223 void ProcessAndEncodeAudioOnTaskQueue(AudioFrame* audio_input);
224
225 void OnReceivedRtt(int64_t rtt_ms);
226
227 void OnTargetTransferRate(TargetTransferRate) override;
228
229 // Thread checkers document and lock usage of some methods on voe::Channel to
230 // specific threads we know about. The goal is to eventually split up
231 // voe::Channel into parts with single-threaded semantics, and thereby reduce
232 // the need for locks.
233 rtc::ThreadChecker worker_thread_checker_;
234 rtc::ThreadChecker module_process_thread_checker_;
235 // Methods accessed from audio and video threads are checked for sequential-
236 // only access. We don't necessarily own and control these threads, so thread
237 // checkers cannot be used. E.g. Chromium may transfer "ownership" from one
238 // audio thread to another, but access is still sequential.
239 rtc::RaceChecker audio_thread_race_checker_;
240
241 rtc::CriticalSection _callbackCritSect;
242 rtc::CriticalSection volume_settings_critsect_;
243
Niels Möller26e88b02018-11-19 15:08:13 +0100244 bool sending_ RTC_GUARDED_BY(&worker_thread_checker_) = false;
Niels Möllerdced9f62018-11-19 10:27:07 +0100245
246 RtcEventLog* const event_log_;
247
248 std::unique_ptr<RtpRtcp> _rtpRtcpModule;
249
250 std::unique_ptr<AudioCodingModule> audio_coding_;
251 uint32_t _timeStamp RTC_GUARDED_BY(encoder_queue_);
252
253 uint16_t send_sequence_number_;
254
255 // uses
Niels Möller985a1f32018-11-19 16:08:42 +0100256 ProcessThread* const _moduleProcessThreadPtr;
Niels Möllere9771992018-11-26 10:55:07 +0100257 Transport* const _transportPtr; // WebRtc socket or external transport
Niels Möllerdced9f62018-11-19 10:27:07 +0100258 RmsLevel rms_level_ RTC_GUARDED_BY(encoder_queue_);
259 bool input_mute_ RTC_GUARDED_BY(volume_settings_critsect_);
260 bool previous_frame_muted_ RTC_GUARDED_BY(encoder_queue_);
261 // VoeRTP_RTCP
262 // TODO(henrika): can today be accessed on the main thread and on the
263 // task queue; hence potential race.
264 bool _includeAudioLevelIndication;
265 size_t transport_overhead_per_packet_
266 RTC_GUARDED_BY(overhead_per_packet_lock_);
267 size_t rtp_overhead_per_packet_ RTC_GUARDED_BY(overhead_per_packet_lock_);
268 rtc::CriticalSection overhead_per_packet_lock_;
269 // RtcpBandwidthObserver
Niels Möller985a1f32018-11-19 16:08:42 +0100270 const std::unique_ptr<VoERtcpObserver> rtcp_observer_;
Niels Möllerdced9f62018-11-19 10:27:07 +0100271
Niels Möller985a1f32018-11-19 16:08:42 +0100272 PacketRouter* packet_router_ RTC_GUARDED_BY(&worker_thread_checker_) =
273 nullptr;
274 const std::unique_ptr<TransportFeedbackProxy> feedback_observer_proxy_;
275 const std::unique_ptr<TransportSequenceNumberProxy> seq_num_allocator_proxy_;
276 const std::unique_ptr<RtpPacketSenderProxy> rtp_packet_sender_proxy_;
277 const std::unique_ptr<RateLimiter> retransmission_rate_limiter_;
Niels Möllerdced9f62018-11-19 10:27:07 +0100278
279 rtc::ThreadChecker construction_thread_;
280
281 const bool use_twcc_plr_for_ana_;
282
283 rtc::CriticalSection encoder_queue_lock_;
284 bool encoder_queue_is_active_ RTC_GUARDED_BY(encoder_queue_lock_) = false;
Niels Möller985a1f32018-11-19 16:08:42 +0100285 rtc::TaskQueue* const encoder_queue_ = nullptr;
Niels Möllerdced9f62018-11-19 10:27:07 +0100286
287 MediaTransportInterface* const media_transport_;
288 int media_transport_sequence_number_ RTC_GUARDED_BY(encoder_queue_) = 0;
289
290 rtc::CriticalSection media_transport_lock_;
291 // Currently set by SetLocalSSRC.
292 uint64_t media_transport_channel_id_ RTC_GUARDED_BY(&media_transport_lock_) =
293 0;
294 // Cache payload type and sampling frequency from most recent call to
295 // SetEncoder. Needed to set MediaTransportEncodedAudioFrame metadata, and
296 // invalidate on encoder change.
297 int media_transport_payload_type_ RTC_GUARDED_BY(&media_transport_lock_);
298 int media_transport_sampling_frequency_
299 RTC_GUARDED_BY(&media_transport_lock_);
300
301 // E2EE Audio Frame Encryption
302 rtc::scoped_refptr<FrameEncryptorInterface> frame_encryptor_;
303 // E2EE Frame Encryption Options
Niels Möller985a1f32018-11-19 16:08:42 +0100304 const webrtc::CryptoOptions crypto_options_;
Niels Möllerdced9f62018-11-19 10:27:07 +0100305
306 rtc::CriticalSection bitrate_crit_section_;
307 int configured_bitrate_bps_ RTC_GUARDED_BY(bitrate_crit_section_) = 0;
308};
Niels Möller530ead42018-10-04 14:28:39 +0200309
310const int kTelephoneEventAttenuationdB = 10;
311
312class TransportFeedbackProxy : public TransportFeedbackObserver {
313 public:
314 TransportFeedbackProxy() : feedback_observer_(nullptr) {
315 pacer_thread_.DetachFromThread();
316 network_thread_.DetachFromThread();
317 }
318
319 void SetTransportFeedbackObserver(
320 TransportFeedbackObserver* feedback_observer) {
321 RTC_DCHECK(thread_checker_.CalledOnValidThread());
322 rtc::CritScope lock(&crit_);
323 feedback_observer_ = feedback_observer;
324 }
325
326 // Implements TransportFeedbackObserver.
327 void AddPacket(uint32_t ssrc,
328 uint16_t sequence_number,
329 size_t length,
330 const PacedPacketInfo& pacing_info) override {
331 RTC_DCHECK(pacer_thread_.CalledOnValidThread());
332 rtc::CritScope lock(&crit_);
333 if (feedback_observer_)
334 feedback_observer_->AddPacket(ssrc, sequence_number, length, pacing_info);
335 }
336
337 void OnTransportFeedback(const rtcp::TransportFeedback& feedback) override {
338 RTC_DCHECK(network_thread_.CalledOnValidThread());
339 rtc::CritScope lock(&crit_);
340 if (feedback_observer_)
341 feedback_observer_->OnTransportFeedback(feedback);
342 }
343
344 private:
345 rtc::CriticalSection crit_;
346 rtc::ThreadChecker thread_checker_;
347 rtc::ThreadChecker pacer_thread_;
348 rtc::ThreadChecker network_thread_;
349 TransportFeedbackObserver* feedback_observer_ RTC_GUARDED_BY(&crit_);
350};
351
352class TransportSequenceNumberProxy : public TransportSequenceNumberAllocator {
353 public:
354 TransportSequenceNumberProxy() : seq_num_allocator_(nullptr) {
355 pacer_thread_.DetachFromThread();
356 }
357
358 void SetSequenceNumberAllocator(
359 TransportSequenceNumberAllocator* seq_num_allocator) {
360 RTC_DCHECK(thread_checker_.CalledOnValidThread());
361 rtc::CritScope lock(&crit_);
362 seq_num_allocator_ = seq_num_allocator;
363 }
364
365 // Implements TransportSequenceNumberAllocator.
366 uint16_t AllocateSequenceNumber() override {
367 RTC_DCHECK(pacer_thread_.CalledOnValidThread());
368 rtc::CritScope lock(&crit_);
369 if (!seq_num_allocator_)
370 return 0;
371 return seq_num_allocator_->AllocateSequenceNumber();
372 }
373
374 private:
375 rtc::CriticalSection crit_;
376 rtc::ThreadChecker thread_checker_;
377 rtc::ThreadChecker pacer_thread_;
378 TransportSequenceNumberAllocator* seq_num_allocator_ RTC_GUARDED_BY(&crit_);
379};
380
381class RtpPacketSenderProxy : public RtpPacketSender {
382 public:
383 RtpPacketSenderProxy() : rtp_packet_sender_(nullptr) {}
384
385 void SetPacketSender(RtpPacketSender* rtp_packet_sender) {
386 RTC_DCHECK(thread_checker_.CalledOnValidThread());
387 rtc::CritScope lock(&crit_);
388 rtp_packet_sender_ = rtp_packet_sender;
389 }
390
391 // Implements RtpPacketSender.
392 void InsertPacket(Priority priority,
393 uint32_t ssrc,
394 uint16_t sequence_number,
395 int64_t capture_time_ms,
396 size_t bytes,
397 bool retransmission) override {
398 rtc::CritScope lock(&crit_);
399 if (rtp_packet_sender_) {
400 rtp_packet_sender_->InsertPacket(priority, ssrc, sequence_number,
401 capture_time_ms, bytes, retransmission);
402 }
403 }
404
405 void SetAccountForAudioPackets(bool account_for_audio) override {
406 RTC_NOTREACHED();
407 }
408
409 private:
410 rtc::ThreadChecker thread_checker_;
411 rtc::CriticalSection crit_;
412 RtpPacketSender* rtp_packet_sender_ RTC_GUARDED_BY(&crit_);
413};
414
415class VoERtcpObserver : public RtcpBandwidthObserver {
416 public:
417 explicit VoERtcpObserver(ChannelSend* owner)
418 : owner_(owner), bandwidth_observer_(nullptr) {}
419 virtual ~VoERtcpObserver() {}
420
421 void SetBandwidthObserver(RtcpBandwidthObserver* bandwidth_observer) {
422 rtc::CritScope lock(&crit_);
423 bandwidth_observer_ = bandwidth_observer;
424 }
425
426 void OnReceivedEstimatedBitrate(uint32_t bitrate) override {
427 rtc::CritScope lock(&crit_);
428 if (bandwidth_observer_) {
429 bandwidth_observer_->OnReceivedEstimatedBitrate(bitrate);
430 }
431 }
432
433 void OnReceivedRtcpReceiverReport(const ReportBlockList& report_blocks,
434 int64_t rtt,
435 int64_t now_ms) override {
436 {
437 rtc::CritScope lock(&crit_);
438 if (bandwidth_observer_) {
439 bandwidth_observer_->OnReceivedRtcpReceiverReport(report_blocks, rtt,
440 now_ms);
441 }
442 }
443 // TODO(mflodman): Do we need to aggregate reports here or can we jut send
444 // what we get? I.e. do we ever get multiple reports bundled into one RTCP
445 // report for VoiceEngine?
446 if (report_blocks.empty())
447 return;
448
449 int fraction_lost_aggregate = 0;
450 int total_number_of_packets = 0;
451
452 // If receiving multiple report blocks, calculate the weighted average based
453 // on the number of packets a report refers to.
454 for (ReportBlockList::const_iterator block_it = report_blocks.begin();
455 block_it != report_blocks.end(); ++block_it) {
456 // Find the previous extended high sequence number for this remote SSRC,
457 // to calculate the number of RTP packets this report refers to. Ignore if
458 // we haven't seen this SSRC before.
459 std::map<uint32_t, uint32_t>::iterator seq_num_it =
460 extended_max_sequence_number_.find(block_it->source_ssrc);
461 int number_of_packets = 0;
462 if (seq_num_it != extended_max_sequence_number_.end()) {
463 number_of_packets =
464 block_it->extended_highest_sequence_number - seq_num_it->second;
465 }
466 fraction_lost_aggregate += number_of_packets * block_it->fraction_lost;
467 total_number_of_packets += number_of_packets;
468
469 extended_max_sequence_number_[block_it->source_ssrc] =
470 block_it->extended_highest_sequence_number;
471 }
472 int weighted_fraction_lost = 0;
473 if (total_number_of_packets > 0) {
474 weighted_fraction_lost =
475 (fraction_lost_aggregate + total_number_of_packets / 2) /
476 total_number_of_packets;
477 }
478 owner_->OnUplinkPacketLossRate(weighted_fraction_lost / 255.0f);
479 }
480
481 private:
482 ChannelSend* owner_;
483 // Maps remote side ssrc to extended highest sequence number received.
484 std::map<uint32_t, uint32_t> extended_max_sequence_number_;
485 rtc::CriticalSection crit_;
486 RtcpBandwidthObserver* bandwidth_observer_ RTC_GUARDED_BY(crit_);
487};
488
489class ChannelSend::ProcessAndEncodeAudioTask : public rtc::QueuedTask {
490 public:
491 ProcessAndEncodeAudioTask(std::unique_ptr<AudioFrame> audio_frame,
492 ChannelSend* channel)
493 : audio_frame_(std::move(audio_frame)), channel_(channel) {
494 RTC_DCHECK(channel_);
495 }
496
497 private:
498 bool Run() override {
499 RTC_DCHECK_RUN_ON(channel_->encoder_queue_);
500 channel_->ProcessAndEncodeAudioOnTaskQueue(audio_frame_.get());
501 return true;
502 }
503
504 std::unique_ptr<AudioFrame> audio_frame_;
505 ChannelSend* const channel_;
506};
507
508int32_t ChannelSend::SendData(FrameType frameType,
509 uint8_t payloadType,
510 uint32_t timeStamp,
511 const uint8_t* payloadData,
512 size_t payloadSize,
513 const RTPFragmentationHeader* fragmentation) {
514 RTC_DCHECK_RUN_ON(encoder_queue_);
Niels Möller7d76a312018-10-26 12:57:07 +0200515 rtc::ArrayView<const uint8_t> payload(payloadData, payloadSize);
516
517 if (media_transport() != nullptr) {
518 return SendMediaTransportAudio(frameType, payloadType, timeStamp, payload,
519 fragmentation);
520 } else {
521 return SendRtpAudio(frameType, payloadType, timeStamp, payload,
522 fragmentation);
523 }
524}
525
526int32_t ChannelSend::SendRtpAudio(FrameType frameType,
527 uint8_t payloadType,
528 uint32_t timeStamp,
529 rtc::ArrayView<const uint8_t> payload,
530 const RTPFragmentationHeader* fragmentation) {
531 RTC_DCHECK_RUN_ON(encoder_queue_);
Niels Möller530ead42018-10-04 14:28:39 +0200532 if (_includeAudioLevelIndication) {
533 // Store current audio level in the RTP/RTCP module.
534 // The level will be used in combination with voice-activity state
535 // (frameType) to add an RTP header extension
536 _rtpRtcpModule->SetAudioLevel(rms_level_.Average());
537 }
538
Benjamin Wright84583f62018-10-04 14:22:34 -0700539 // E2EE Custom Audio Frame Encryption (This is optional).
540 // Keep this buffer around for the lifetime of the send call.
541 rtc::Buffer encrypted_audio_payload;
542 if (frame_encryptor_ != nullptr) {
543 // TODO(benwright@webrtc.org) - Allocate enough to always encrypt inline.
544 // Allocate a buffer to hold the maximum possible encrypted payload.
545 size_t max_ciphertext_size = frame_encryptor_->GetMaxCiphertextByteSize(
Niels Möller7d76a312018-10-26 12:57:07 +0200546 cricket::MEDIA_TYPE_AUDIO, payload.size());
Benjamin Wright84583f62018-10-04 14:22:34 -0700547 encrypted_audio_payload.SetSize(max_ciphertext_size);
548
549 // Encrypt the audio payload into the buffer.
550 size_t bytes_written = 0;
551 int encrypt_status = frame_encryptor_->Encrypt(
552 cricket::MEDIA_TYPE_AUDIO, _rtpRtcpModule->SSRC(),
Niels Möller7d76a312018-10-26 12:57:07 +0200553 /*additional_data=*/nullptr, payload, encrypted_audio_payload,
554 &bytes_written);
Benjamin Wright84583f62018-10-04 14:22:34 -0700555 if (encrypt_status != 0) {
556 RTC_DLOG(LS_ERROR) << "Channel::SendData() failed encrypt audio payload: "
557 << encrypt_status;
558 return -1;
559 }
560 // Resize the buffer to the exact number of bytes actually used.
561 encrypted_audio_payload.SetSize(bytes_written);
562 // Rewrite the payloadData and size to the new encrypted payload.
Niels Möller7d76a312018-10-26 12:57:07 +0200563 payload = encrypted_audio_payload;
Benjamin Wrightbfb444c2018-10-15 10:20:24 -0700564 } else if (crypto_options_.sframe.require_frame_encryption) {
565 RTC_DLOG(LS_ERROR) << "Channel::SendData() failed sending audio payload: "
566 << "A frame encryptor is required but one is not set.";
567 return -1;
Benjamin Wright84583f62018-10-04 14:22:34 -0700568 }
569
Niels Möller530ead42018-10-04 14:28:39 +0200570 // Push data from ACM to RTP/RTCP-module to deliver audio frame for
571 // packetization.
572 // This call will trigger Transport::SendPacket() from the RTP/RTCP module.
Niels Möller7d76a312018-10-26 12:57:07 +0200573 if (!_rtpRtcpModule->SendOutgoingData((FrameType&)frameType, payloadType,
574 timeStamp,
575 // Leaving the time when this frame was
576 // received from the capture device as
577 // undefined for voice for now.
578 -1, payload.data(), payload.size(),
579 fragmentation, nullptr, nullptr)) {
Niels Möller530ead42018-10-04 14:28:39 +0200580 RTC_DLOG(LS_ERROR)
581 << "ChannelSend::SendData() failed to send data to RTP/RTCP module";
582 return -1;
583 }
584
585 return 0;
586}
587
Niels Möller7d76a312018-10-26 12:57:07 +0200588int32_t ChannelSend::SendMediaTransportAudio(
589 FrameType frameType,
590 uint8_t payloadType,
591 uint32_t timeStamp,
592 rtc::ArrayView<const uint8_t> payload,
593 const RTPFragmentationHeader* fragmentation) {
594 RTC_DCHECK_RUN_ON(encoder_queue_);
595 // TODO(nisse): Use null _transportPtr for MediaTransport.
596 // RTC_DCHECK(_transportPtr == nullptr);
597 uint64_t channel_id;
598 int sampling_rate_hz;
599 {
600 rtc::CritScope cs(&media_transport_lock_);
601 if (media_transport_payload_type_ != payloadType) {
602 // Payload type is being changed, media_transport_sampling_frequency_,
603 // no longer current.
604 return -1;
605 }
606 sampling_rate_hz = media_transport_sampling_frequency_;
607 channel_id = media_transport_channel_id_;
608 }
609 const MediaTransportEncodedAudioFrame frame(
610 /*sampling_rate_hz=*/sampling_rate_hz,
611
612 // TODO(nisse): Timestamp and sample index are the same for all supported
613 // audio codecs except G722. Refactor audio coding module to only use
614 // sample index, and leave translation to RTP time, when needed, for
615 // RTP-specific code.
616 /*starting_sample_index=*/timeStamp,
617
618 // Sample count isn't conveniently available from the AudioCodingModule,
619 // and needs some refactoring to wire up in a good way. For now, left as
620 // zero.
621 /*sample_count=*/0,
622
623 /*sequence_number=*/media_transport_sequence_number_,
624 MediaTransportFrameTypeForWebrtcFrameType(frameType), payloadType,
625 std::vector<uint8_t>(payload.begin(), payload.end()));
626
627 // TODO(nisse): Introduce a MediaTransportSender object bound to a specific
628 // channel id.
629 RTCError rtc_error =
630 media_transport()->SendAudioFrame(channel_id, std::move(frame));
631
632 if (!rtc_error.ok()) {
633 RTC_LOG(LS_ERROR) << "Failed to send frame, rtc_error="
634 << ToString(rtc_error.type()) << ", "
635 << rtc_error.message();
636 return -1;
637 }
638
639 ++media_transport_sequence_number_;
640
641 return 0;
642}
643
Niels Möller530ead42018-10-04 14:28:39 +0200644bool ChannelSend::SendRtp(const uint8_t* data,
645 size_t len,
646 const PacketOptions& options) {
Niels Möller7d76a312018-10-26 12:57:07 +0200647 // We should not be sending RTP packets if media transport is available.
648 RTC_CHECK(!media_transport());
649
Niels Möller530ead42018-10-04 14:28:39 +0200650 rtc::CritScope cs(&_callbackCritSect);
651
652 if (_transportPtr == NULL) {
653 RTC_DLOG(LS_ERROR)
654 << "ChannelSend::SendPacket() failed to send RTP packet due to"
655 << " invalid transport object";
656 return false;
657 }
658
659 if (!_transportPtr->SendRtp(data, len, options)) {
660 RTC_DLOG(LS_ERROR) << "ChannelSend::SendPacket() RTP transmission failed";
661 return false;
662 }
663 return true;
664}
665
666bool ChannelSend::SendRtcp(const uint8_t* data, size_t len) {
667 rtc::CritScope cs(&_callbackCritSect);
668 if (_transportPtr == NULL) {
669 RTC_DLOG(LS_ERROR)
670 << "ChannelSend::SendRtcp() failed to send RTCP packet due to"
671 << " invalid transport object";
672 return false;
673 }
674
675 int n = _transportPtr->SendRtcp(data, len);
676 if (n < 0) {
677 RTC_DLOG(LS_ERROR) << "ChannelSend::SendRtcp() transmission failed";
678 return false;
679 }
680 return true;
681}
682
Niels Möller530ead42018-10-04 14:28:39 +0200683ChannelSend::ChannelSend(rtc::TaskQueue* encoder_queue,
684 ProcessThread* module_process_thread,
Niels Möller7d76a312018-10-26 12:57:07 +0200685 MediaTransportInterface* media_transport,
Niels Möllere9771992018-11-26 10:55:07 +0100686 Transport* rtp_transport,
Niels Möller530ead42018-10-04 14:28:39 +0200687 RtcpRttStats* rtcp_rtt_stats,
Benjamin Wright84583f62018-10-04 14:22:34 -0700688 RtcEventLog* rtc_event_log,
Benjamin Wrightbfb444c2018-10-15 10:20:24 -0700689 FrameEncryptorInterface* frame_encryptor,
Johannes Kron9190b822018-10-29 11:22:05 +0100690 const webrtc::CryptoOptions& crypto_options,
Jiawei Ou55718122018-11-09 13:17:39 -0800691 bool extmap_allow_mixed,
692 int rtcp_report_interval_ms)
Niels Möller530ead42018-10-04 14:28:39 +0200693 : event_log_(rtc_event_log),
694 _timeStamp(0), // This is just an offset, RTP module will add it's own
695 // random offset
696 send_sequence_number_(0),
697 _moduleProcessThreadPtr(module_process_thread),
Niels Möllere9771992018-11-26 10:55:07 +0100698 _transportPtr(rtp_transport),
Niels Möller530ead42018-10-04 14:28:39 +0200699 input_mute_(false),
700 previous_frame_muted_(false),
701 _includeAudioLevelIndication(false),
702 transport_overhead_per_packet_(0),
703 rtp_overhead_per_packet_(0),
704 rtcp_observer_(new VoERtcpObserver(this)),
705 feedback_observer_proxy_(new TransportFeedbackProxy()),
706 seq_num_allocator_proxy_(new TransportSequenceNumberProxy()),
707 rtp_packet_sender_proxy_(new RtpPacketSenderProxy()),
708 retransmission_rate_limiter_(new RateLimiter(Clock::GetRealTimeClock(),
709 kMaxRetransmissionWindowMs)),
710 use_twcc_plr_for_ana_(
711 webrtc::field_trial::FindFullName("UseTwccPlrForAna") == "Enabled"),
Benjamin Wright84583f62018-10-04 14:22:34 -0700712 encoder_queue_(encoder_queue),
Niels Möller7d76a312018-10-26 12:57:07 +0200713 media_transport_(media_transport),
Benjamin Wrightbfb444c2018-10-15 10:20:24 -0700714 frame_encryptor_(frame_encryptor),
715 crypto_options_(crypto_options) {
Niels Möller530ead42018-10-04 14:28:39 +0200716 RTC_DCHECK(module_process_thread);
717 RTC_DCHECK(encoder_queue);
Niels Möllerdced9f62018-11-19 10:27:07 +0100718 module_process_thread_checker_.DetachFromThread();
719
Niels Möller530ead42018-10-04 14:28:39 +0200720 audio_coding_.reset(AudioCodingModule::Create(AudioCodingModule::Config()));
721
722 RtpRtcp::Configuration configuration;
Piotr (Peter) Slatala179a3922018-11-16 09:57:58 -0800723
724 // We gradually remove codepaths that depend on RTP when using media
725 // transport. All of this logic should be moved to the future
726 // RTPMediaTransport. In this case it means that overhead and bandwidth
727 // observers should not be called when using media transport.
728 if (!media_transport_) {
729 configuration.overhead_observer = this;
730 configuration.bandwidth_callback = rtcp_observer_.get();
731 configuration.transport_feedback_callback = feedback_observer_proxy_.get();
732 }
733
Niels Möller530ead42018-10-04 14:28:39 +0200734 configuration.audio = true;
735 configuration.outgoing_transport = this;
Niels Möller530ead42018-10-04 14:28:39 +0200736
737 configuration.paced_sender = rtp_packet_sender_proxy_.get();
738 configuration.transport_sequence_number_allocator =
739 seq_num_allocator_proxy_.get();
Niels Möller530ead42018-10-04 14:28:39 +0200740
741 configuration.event_log = event_log_;
742 configuration.rtt_stats = rtcp_rtt_stats;
743 configuration.retransmission_rate_limiter =
744 retransmission_rate_limiter_.get();
Johannes Kron9190b822018-10-29 11:22:05 +0100745 configuration.extmap_allow_mixed = extmap_allow_mixed;
Jiawei Ou8b5d9d82018-11-15 16:44:37 -0800746 configuration.rtcp_report_interval_ms = rtcp_report_interval_ms;
Niels Möller530ead42018-10-04 14:28:39 +0200747
748 _rtpRtcpModule.reset(RtpRtcp::CreateRtpRtcp(configuration));
749 _rtpRtcpModule->SetSendingMediaStatus(false);
Niels Möller530ead42018-10-04 14:28:39 +0200750
Piotr (Peter) Slatala179a3922018-11-16 09:57:58 -0800751 // We want to invoke the 'TargetRateObserver' and |OnOverheadChanged|
752 // callbacks after the audio_coding_ is fully initialized.
753 if (media_transport_) {
754 RTC_DLOG(LS_INFO) << "Setting media_transport_ rate observers.";
755 media_transport_->AddTargetTransferRateObserver(this);
756 OnOverheadChanged(media_transport_->GetAudioPacketOverhead());
757 } else {
758 RTC_DLOG(LS_INFO) << "Not setting media_transport_ rate observers.";
759 }
760
Niels Möller530ead42018-10-04 14:28:39 +0200761 _moduleProcessThreadPtr->RegisterModule(_rtpRtcpModule.get(), RTC_FROM_HERE);
762
Niels Möller530ead42018-10-04 14:28:39 +0200763 // Ensure that RTCP is enabled by default for the created channel.
764 // Note that, the module will keep generating RTCP until it is explicitly
765 // disabled by the user.
766 // After StopListen (when no sockets exists), RTCP packets will no longer
767 // be transmitted since the Transport object will then be invalid.
768 // RTCP is enabled by default.
769 _rtpRtcpModule->SetRTCPStatus(RtcpMode::kCompound);
770
Fredrik Solenbergeb134842018-11-19 14:13:15 +0100771 int error = audio_coding_->RegisterTransportCallback(this);
Niels Möller530ead42018-10-04 14:28:39 +0200772 RTC_DCHECK_EQ(0, error);
773}
774
Fredrik Solenberg645a3af2018-11-16 12:51:15 +0100775ChannelSend::~ChannelSend() {
Niels Möller530ead42018-10-04 14:28:39 +0200776 RTC_DCHECK(construction_thread_.CalledOnValidThread());
Niels Möller530ead42018-10-04 14:28:39 +0200777
Piotr (Peter) Slatala179a3922018-11-16 09:57:58 -0800778 if (media_transport_) {
779 media_transport_->RemoveTargetTransferRateObserver(this);
780 }
781
Niels Möller530ead42018-10-04 14:28:39 +0200782 StopSend();
783
Niels Möller530ead42018-10-04 14:28:39 +0200784 int error = audio_coding_->RegisterTransportCallback(NULL);
785 RTC_DCHECK_EQ(0, error);
786
Niels Möller530ead42018-10-04 14:28:39 +0200787 if (_moduleProcessThreadPtr)
788 _moduleProcessThreadPtr->DeRegisterModule(_rtpRtcpModule.get());
Niels Möller530ead42018-10-04 14:28:39 +0200789}
790
Niels Möller26815232018-11-16 09:32:40 +0100791void ChannelSend::StartSend() {
Niels Möller26e88b02018-11-19 15:08:13 +0100792 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Fredrik Solenbergeb134842018-11-19 14:13:15 +0100793 RTC_DCHECK(!sending_);
794 sending_ = true;
Niels Möller530ead42018-10-04 14:28:39 +0200795
796 // Resume the previous sequence number which was reset by StopSend(). This
797 // needs to be done before |sending| is set to true on the RTP/RTCP module.
798 if (send_sequence_number_) {
799 _rtpRtcpModule->SetSequenceNumber(send_sequence_number_);
800 }
801 _rtpRtcpModule->SetSendingMediaStatus(true);
Niels Möller26815232018-11-16 09:32:40 +0100802 int ret = _rtpRtcpModule->SetSendingStatus(true);
803 RTC_DCHECK_EQ(0, ret);
Niels Möller530ead42018-10-04 14:28:39 +0200804 {
805 // It is now OK to start posting tasks to the encoder task queue.
806 rtc::CritScope cs(&encoder_queue_lock_);
807 encoder_queue_is_active_ = true;
808 }
Niels Möller530ead42018-10-04 14:28:39 +0200809}
810
811void ChannelSend::StopSend() {
Niels Möller26e88b02018-11-19 15:08:13 +0100812 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Fredrik Solenbergeb134842018-11-19 14:13:15 +0100813 if (!sending_) {
Niels Möller530ead42018-10-04 14:28:39 +0200814 return;
815 }
Fredrik Solenbergeb134842018-11-19 14:13:15 +0100816 sending_ = false;
Niels Möller530ead42018-10-04 14:28:39 +0200817
818 // Post a task to the encoder thread which sets an event when the task is
819 // executed. We know that no more encoding tasks will be added to the task
820 // queue for this channel since sending is now deactivated. It means that,
821 // if we wait for the event to bet set, we know that no more pending tasks
822 // exists and it is therfore guaranteed that the task queue will never try
823 // to acccess and invalid channel object.
824 RTC_DCHECK(encoder_queue_);
825
Niels Möllerc572ff32018-11-07 08:43:50 +0100826 rtc::Event flush;
Niels Möller530ead42018-10-04 14:28:39 +0200827 {
828 // Clear |encoder_queue_is_active_| under lock to prevent any other tasks
829 // than this final "flush task" to be posted on the queue.
830 rtc::CritScope cs(&encoder_queue_lock_);
831 encoder_queue_is_active_ = false;
832 encoder_queue_->PostTask([&flush]() { flush.Set(); });
833 }
834 flush.Wait(rtc::Event::kForever);
835
836 // Store the sequence number to be able to pick up the same sequence for
837 // the next StartSend(). This is needed for restarting device, otherwise
838 // it might cause libSRTP to complain about packets being replayed.
839 // TODO(xians): Remove this workaround after RtpRtcpModule's refactoring
840 // CL is landed. See issue
841 // https://code.google.com/p/webrtc/issues/detail?id=2111 .
842 send_sequence_number_ = _rtpRtcpModule->SequenceNumber();
843
844 // Reset sending SSRC and sequence number and triggers direct transmission
845 // of RTCP BYE
846 if (_rtpRtcpModule->SetSendingStatus(false) == -1) {
847 RTC_DLOG(LS_ERROR) << "StartSend() RTP/RTCP failed to stop sending";
848 }
849 _rtpRtcpModule->SetSendingMediaStatus(false);
850}
851
852bool ChannelSend::SetEncoder(int payload_type,
853 std::unique_ptr<AudioEncoder> encoder) {
Niels Möller26e88b02018-11-19 15:08:13 +0100854 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller530ead42018-10-04 14:28:39 +0200855 RTC_DCHECK_GE(payload_type, 0);
856 RTC_DCHECK_LE(payload_type, 127);
857 // TODO(ossu): Make CodecInsts up, for now: one for the RTP/RTCP module and
858 // one for for us to keep track of sample rate and number of channels, etc.
859
860 // The RTP/RTCP module needs to know the RTP timestamp rate (i.e. clockrate)
861 // as well as some other things, so we collect this info and send it along.
862 CodecInst rtp_codec;
863 rtp_codec.pltype = payload_type;
864 strncpy(rtp_codec.plname, "audio", sizeof(rtp_codec.plname));
865 rtp_codec.plname[sizeof(rtp_codec.plname) - 1] = 0;
866 // Seems unclear if it should be clock rate or sample rate. CodecInst
867 // supposedly carries the sample rate, but only clock rate seems sensible to
868 // send to the RTP/RTCP module.
869 rtp_codec.plfreq = encoder->RtpTimestampRateHz();
870 rtp_codec.pacsize = rtc::CheckedDivExact(
871 static_cast<int>(encoder->Max10MsFramesInAPacket() * rtp_codec.plfreq),
872 100);
873 rtp_codec.channels = encoder->NumChannels();
874 rtp_codec.rate = 0;
875
876 if (_rtpRtcpModule->RegisterSendPayload(rtp_codec) != 0) {
877 _rtpRtcpModule->DeRegisterSendPayload(payload_type);
878 if (_rtpRtcpModule->RegisterSendPayload(rtp_codec) != 0) {
879 RTC_DLOG(LS_ERROR)
880 << "SetEncoder() failed to register codec to RTP/RTCP module";
881 return false;
882 }
883 }
884
Niels Möller7d76a312018-10-26 12:57:07 +0200885 if (media_transport_) {
886 rtc::CritScope cs(&media_transport_lock_);
887 media_transport_payload_type_ = payload_type;
888 // TODO(nisse): Currently broken for G722, since timestamps passed through
889 // encoder use RTP clock rather than sample count, and they differ for G722.
890 media_transport_sampling_frequency_ = encoder->RtpTimestampRateHz();
891 }
Niels Möller530ead42018-10-04 14:28:39 +0200892 audio_coding_->SetEncoder(std::move(encoder));
893 return true;
894}
895
896void ChannelSend::ModifyEncoder(
897 rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier) {
Niels Möller26e88b02018-11-19 15:08:13 +0100898 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller530ead42018-10-04 14:28:39 +0200899 audio_coding_->ModifyEncoder(modifier);
900}
901
Sebastian Jansson254d8692018-11-21 19:19:00 +0100902void ChannelSend::OnBitrateAllocation(BitrateAllocationUpdate update) {
Niels Möllerdced9f62018-11-19 10:27:07 +0100903 // This method can be called on the worker thread, module process thread
904 // or on a TaskQueue via VideoSendStreamImpl::OnEncoderConfigurationChanged.
905 // TODO(solenberg): Figure out a good way to check this or enforce calling
906 // rules.
907 // RTC_DCHECK(worker_thread_checker_.CalledOnValidThread() ||
908 // module_process_thread_checker_.CalledOnValidThread());
Piotr (Peter) Slatala1eebec92018-11-16 09:03:35 -0800909 rtc::CritScope lock(&bitrate_crit_section_);
Niels Möllerdced9f62018-11-19 10:27:07 +0100910
Niels Möller530ead42018-10-04 14:28:39 +0200911 audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
912 if (*encoder) {
Sebastian Jansson254d8692018-11-21 19:19:00 +0100913 (*encoder)->OnReceivedUplinkAllocation(update);
Niels Möller530ead42018-10-04 14:28:39 +0200914 }
915 });
Sebastian Jansson254d8692018-11-21 19:19:00 +0100916 retransmission_rate_limiter_->SetMaxRate(update.target_bitrate.bps());
917 configured_bitrate_bps_ = update.target_bitrate.bps();
Sebastian Jansson359d60a2018-10-25 16:22:02 +0200918}
919
Niels Möllerdced9f62018-11-19 10:27:07 +0100920int ChannelSend::GetBitrate() const {
Piotr (Peter) Slatala1eebec92018-11-16 09:03:35 -0800921 rtc::CritScope lock(&bitrate_crit_section_);
Sebastian Jansson359d60a2018-10-25 16:22:02 +0200922 return configured_bitrate_bps_;
Niels Möller530ead42018-10-04 14:28:39 +0200923}
924
925void ChannelSend::OnTwccBasedUplinkPacketLossRate(float packet_loss_rate) {
Niels Möller26e88b02018-11-19 15:08:13 +0100926 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller530ead42018-10-04 14:28:39 +0200927 if (!use_twcc_plr_for_ana_)
928 return;
929 audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
930 if (*encoder) {
931 (*encoder)->OnReceivedUplinkPacketLossFraction(packet_loss_rate);
932 }
933 });
934}
935
936void ChannelSend::OnRecoverableUplinkPacketLossRate(
937 float recoverable_packet_loss_rate) {
Niels Möller26e88b02018-11-19 15:08:13 +0100938 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller530ead42018-10-04 14:28:39 +0200939 audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
940 if (*encoder) {
941 (*encoder)->OnReceivedUplinkRecoverablePacketLossFraction(
942 recoverable_packet_loss_rate);
943 }
944 });
945}
946
947void ChannelSend::OnUplinkPacketLossRate(float packet_loss_rate) {
948 if (use_twcc_plr_for_ana_)
949 return;
950 audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
951 if (*encoder) {
952 (*encoder)->OnReceivedUplinkPacketLossFraction(packet_loss_rate);
953 }
954 });
955}
956
Niels Möller26815232018-11-16 09:32:40 +0100957// TODO(nisse): Delete always-true return value.
958bool ChannelSend::ReceivedRTCPPacket(const uint8_t* data, size_t length) {
Niels Möllerdced9f62018-11-19 10:27:07 +0100959 // May be called on either worker thread or network thread.
Piotr (Peter) Slatala179a3922018-11-16 09:57:58 -0800960 if (media_transport_) {
961 // Ignore RTCP packets while media transport is used.
962 // Those packets should not arrive, but we are seeing occasional packets.
963 return 0;
964 }
965
Niels Möller530ead42018-10-04 14:28:39 +0200966 // Deliver RTCP packet to RTP/RTCP module for parsing
967 _rtpRtcpModule->IncomingRtcpPacket(data, length);
968
969 int64_t rtt = GetRTT();
970 if (rtt == 0) {
971 // Waiting for valid RTT.
Niels Möller26815232018-11-16 09:32:40 +0100972 return true;
Niels Möller530ead42018-10-04 14:28:39 +0200973 }
974
975 int64_t nack_window_ms = rtt;
976 if (nack_window_ms < kMinRetransmissionWindowMs) {
977 nack_window_ms = kMinRetransmissionWindowMs;
978 } else if (nack_window_ms > kMaxRetransmissionWindowMs) {
979 nack_window_ms = kMaxRetransmissionWindowMs;
980 }
981 retransmission_rate_limiter_->SetWindowSize(nack_window_ms);
982
Piotr (Peter) Slatala179a3922018-11-16 09:57:58 -0800983 OnReceivedRtt(rtt);
Niels Möller26815232018-11-16 09:32:40 +0100984 return true;
Niels Möller530ead42018-10-04 14:28:39 +0200985}
986
987void ChannelSend::SetInputMute(bool enable) {
Niels Möller26e88b02018-11-19 15:08:13 +0100988 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller530ead42018-10-04 14:28:39 +0200989 rtc::CritScope cs(&volume_settings_critsect_);
990 input_mute_ = enable;
991}
992
993bool ChannelSend::InputMute() const {
994 rtc::CritScope cs(&volume_settings_critsect_);
995 return input_mute_;
996}
997
Niels Möller26815232018-11-16 09:32:40 +0100998bool ChannelSend::SendTelephoneEventOutband(int event, int duration_ms) {
Niels Möller26e88b02018-11-19 15:08:13 +0100999 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller530ead42018-10-04 14:28:39 +02001000 RTC_DCHECK_LE(0, event);
1001 RTC_DCHECK_GE(255, event);
1002 RTC_DCHECK_LE(0, duration_ms);
1003 RTC_DCHECK_GE(65535, duration_ms);
Fredrik Solenbergeb134842018-11-19 14:13:15 +01001004 if (!sending_) {
Niels Möller26815232018-11-16 09:32:40 +01001005 return false;
Niels Möller530ead42018-10-04 14:28:39 +02001006 }
1007 if (_rtpRtcpModule->SendTelephoneEventOutband(
1008 event, duration_ms, kTelephoneEventAttenuationdB) != 0) {
1009 RTC_DLOG(LS_ERROR) << "SendTelephoneEventOutband() failed to send event";
Niels Möller26815232018-11-16 09:32:40 +01001010 return false;
Niels Möller530ead42018-10-04 14:28:39 +02001011 }
Niels Möller26815232018-11-16 09:32:40 +01001012 return true;
Niels Möller530ead42018-10-04 14:28:39 +02001013}
1014
Niels Möller26815232018-11-16 09:32:40 +01001015bool ChannelSend::SetSendTelephoneEventPayloadType(int payload_type,
1016 int payload_frequency) {
Niels Möller26e88b02018-11-19 15:08:13 +01001017 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller530ead42018-10-04 14:28:39 +02001018 RTC_DCHECK_LE(0, payload_type);
1019 RTC_DCHECK_GE(127, payload_type);
1020 CodecInst codec = {0};
1021 codec.pltype = payload_type;
1022 codec.plfreq = payload_frequency;
1023 memcpy(codec.plname, "telephone-event", 16);
1024 if (_rtpRtcpModule->RegisterSendPayload(codec) != 0) {
1025 _rtpRtcpModule->DeRegisterSendPayload(codec.pltype);
1026 if (_rtpRtcpModule->RegisterSendPayload(codec) != 0) {
1027 RTC_DLOG(LS_ERROR)
1028 << "SetSendTelephoneEventPayloadType() failed to register "
1029 "send payload type";
Niels Möller26815232018-11-16 09:32:40 +01001030 return false;
Niels Möller530ead42018-10-04 14:28:39 +02001031 }
1032 }
Niels Möller26815232018-11-16 09:32:40 +01001033 return true;
Niels Möller530ead42018-10-04 14:28:39 +02001034}
1035
Niels Möllerdced9f62018-11-19 10:27:07 +01001036void ChannelSend::SetLocalSSRC(uint32_t ssrc) {
Niels Möller26e88b02018-11-19 15:08:13 +01001037 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Fredrik Solenbergeb134842018-11-19 14:13:15 +01001038 RTC_DCHECK(!sending_);
Niels Möller26815232018-11-16 09:32:40 +01001039
Niels Möller7d76a312018-10-26 12:57:07 +02001040 if (media_transport_) {
1041 rtc::CritScope cs(&media_transport_lock_);
1042 media_transport_channel_id_ = ssrc;
1043 }
Niels Möller530ead42018-10-04 14:28:39 +02001044 _rtpRtcpModule->SetSSRC(ssrc);
Niels Möller530ead42018-10-04 14:28:39 +02001045}
1046
1047void ChannelSend::SetMid(const std::string& mid, int extension_id) {
Niels Möller26e88b02018-11-19 15:08:13 +01001048 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller530ead42018-10-04 14:28:39 +02001049 int ret = SetSendRtpHeaderExtension(true, kRtpExtensionMid, extension_id);
1050 RTC_DCHECK_EQ(0, ret);
1051 _rtpRtcpModule->SetMid(mid);
1052}
1053
Johannes Kron9190b822018-10-29 11:22:05 +01001054void ChannelSend::SetExtmapAllowMixed(bool extmap_allow_mixed) {
Niels Möller26e88b02018-11-19 15:08:13 +01001055 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Johannes Kron9190b822018-10-29 11:22:05 +01001056 _rtpRtcpModule->SetExtmapAllowMixed(extmap_allow_mixed);
1057}
1058
Niels Möller26815232018-11-16 09:32:40 +01001059void ChannelSend::SetSendAudioLevelIndicationStatus(bool enable, int id) {
Niels Möller26e88b02018-11-19 15:08:13 +01001060 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller530ead42018-10-04 14:28:39 +02001061 _includeAudioLevelIndication = enable;
Niels Möller26815232018-11-16 09:32:40 +01001062 int ret = SetSendRtpHeaderExtension(enable, kRtpExtensionAudioLevel, id);
1063 RTC_DCHECK_EQ(0, ret);
Niels Möller530ead42018-10-04 14:28:39 +02001064}
1065
1066void ChannelSend::EnableSendTransportSequenceNumber(int id) {
Niels Möller26e88b02018-11-19 15:08:13 +01001067 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller530ead42018-10-04 14:28:39 +02001068 int ret =
1069 SetSendRtpHeaderExtension(true, kRtpExtensionTransportSequenceNumber, id);
1070 RTC_DCHECK_EQ(0, ret);
1071}
1072
1073void ChannelSend::RegisterSenderCongestionControlObjects(
1074 RtpTransportControllerSendInterface* transport,
1075 RtcpBandwidthObserver* bandwidth_observer) {
Niels Möller26e88b02018-11-19 15:08:13 +01001076 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller530ead42018-10-04 14:28:39 +02001077 RtpPacketSender* rtp_packet_sender = transport->packet_sender();
1078 TransportFeedbackObserver* transport_feedback_observer =
1079 transport->transport_feedback_observer();
1080 PacketRouter* packet_router = transport->packet_router();
1081
1082 RTC_DCHECK(rtp_packet_sender);
1083 RTC_DCHECK(transport_feedback_observer);
1084 RTC_DCHECK(packet_router);
1085 RTC_DCHECK(!packet_router_);
1086 rtcp_observer_->SetBandwidthObserver(bandwidth_observer);
1087 feedback_observer_proxy_->SetTransportFeedbackObserver(
1088 transport_feedback_observer);
1089 seq_num_allocator_proxy_->SetSequenceNumberAllocator(packet_router);
1090 rtp_packet_sender_proxy_->SetPacketSender(rtp_packet_sender);
1091 _rtpRtcpModule->SetStorePacketsStatus(true, 600);
1092 constexpr bool remb_candidate = false;
1093 packet_router->AddSendRtpModule(_rtpRtcpModule.get(), remb_candidate);
1094 packet_router_ = packet_router;
1095}
1096
1097void ChannelSend::ResetSenderCongestionControlObjects() {
Niels Möller26e88b02018-11-19 15:08:13 +01001098 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller530ead42018-10-04 14:28:39 +02001099 RTC_DCHECK(packet_router_);
1100 _rtpRtcpModule->SetStorePacketsStatus(false, 600);
1101 rtcp_observer_->SetBandwidthObserver(nullptr);
1102 feedback_observer_proxy_->SetTransportFeedbackObserver(nullptr);
1103 seq_num_allocator_proxy_->SetSequenceNumberAllocator(nullptr);
1104 packet_router_->RemoveSendRtpModule(_rtpRtcpModule.get());
1105 packet_router_ = nullptr;
1106 rtp_packet_sender_proxy_->SetPacketSender(nullptr);
1107}
1108
Niels Möller26815232018-11-16 09:32:40 +01001109void ChannelSend::SetRTCP_CNAME(absl::string_view c_name) {
Niels Möller26e88b02018-11-19 15:08:13 +01001110 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller26815232018-11-16 09:32:40 +01001111 // Note: SetCNAME() accepts a c string of length at most 255.
1112 const std::string c_name_limited(c_name.substr(0, 255));
1113 int ret = _rtpRtcpModule->SetCNAME(c_name_limited.c_str()) != 0;
1114 RTC_DCHECK_EQ(0, ret) << "SetRTCP_CNAME() failed to set RTCP CNAME";
Niels Möller530ead42018-10-04 14:28:39 +02001115}
1116
Niels Möller26815232018-11-16 09:32:40 +01001117std::vector<ReportBlock> ChannelSend::GetRemoteRTCPReportBlocks() const {
Niels Möller26e88b02018-11-19 15:08:13 +01001118 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller530ead42018-10-04 14:28:39 +02001119 // Get the report blocks from the latest received RTCP Sender or Receiver
1120 // Report. Each element in the vector contains the sender's SSRC and a
1121 // report block according to RFC 3550.
1122 std::vector<RTCPReportBlock> rtcp_report_blocks;
Niels Möller530ead42018-10-04 14:28:39 +02001123
Niels Möller26815232018-11-16 09:32:40 +01001124 int ret = _rtpRtcpModule->RemoteRTCPStat(&rtcp_report_blocks);
1125 RTC_DCHECK_EQ(0, ret);
1126
1127 std::vector<ReportBlock> report_blocks;
Niels Möller530ead42018-10-04 14:28:39 +02001128
1129 std::vector<RTCPReportBlock>::const_iterator it = rtcp_report_blocks.begin();
1130 for (; it != rtcp_report_blocks.end(); ++it) {
1131 ReportBlock report_block;
1132 report_block.sender_SSRC = it->sender_ssrc;
1133 report_block.source_SSRC = it->source_ssrc;
1134 report_block.fraction_lost = it->fraction_lost;
1135 report_block.cumulative_num_packets_lost = it->packets_lost;
1136 report_block.extended_highest_sequence_number =
1137 it->extended_highest_sequence_number;
1138 report_block.interarrival_jitter = it->jitter;
1139 report_block.last_SR_timestamp = it->last_sender_report_timestamp;
1140 report_block.delay_since_last_SR = it->delay_since_last_sender_report;
Niels Möller26815232018-11-16 09:32:40 +01001141 report_blocks.push_back(report_block);
Niels Möller530ead42018-10-04 14:28:39 +02001142 }
Niels Möller26815232018-11-16 09:32:40 +01001143 return report_blocks;
Niels Möller530ead42018-10-04 14:28:39 +02001144}
1145
Niels Möller26815232018-11-16 09:32:40 +01001146CallSendStatistics ChannelSend::GetRTCPStatistics() const {
Niels Möller26e88b02018-11-19 15:08:13 +01001147 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller26815232018-11-16 09:32:40 +01001148 CallSendStatistics stats = {0};
Niels Möller530ead42018-10-04 14:28:39 +02001149 stats.rttMs = GetRTT();
1150
Niels Möller530ead42018-10-04 14:28:39 +02001151 size_t bytesSent(0);
1152 uint32_t packetsSent(0);
1153
1154 if (_rtpRtcpModule->DataCountersRTP(&bytesSent, &packetsSent) != 0) {
1155 RTC_DLOG(LS_WARNING)
1156 << "GetRTPStatistics() failed to retrieve RTP datacounters"
1157 << " => output will not be complete";
1158 }
1159
1160 stats.bytesSent = bytesSent;
1161 stats.packetsSent = packetsSent;
1162
Niels Möller26815232018-11-16 09:32:40 +01001163 return stats;
Niels Möller530ead42018-10-04 14:28:39 +02001164}
1165
Niels Möller530ead42018-10-04 14:28:39 +02001166void ChannelSend::ProcessAndEncodeAudio(
1167 std::unique_ptr<AudioFrame> audio_frame) {
Niels Möllerdced9f62018-11-19 10:27:07 +01001168 RTC_DCHECK_RUNS_SERIALIZED(&audio_thread_race_checker_);
Niels Möller530ead42018-10-04 14:28:39 +02001169 // Avoid posting any new tasks if sending was already stopped in StopSend().
1170 rtc::CritScope cs(&encoder_queue_lock_);
1171 if (!encoder_queue_is_active_) {
1172 return;
1173 }
1174 // Profile time between when the audio frame is added to the task queue and
1175 // when the task is actually executed.
1176 audio_frame->UpdateProfileTimeStamp();
1177 encoder_queue_->PostTask(std::unique_ptr<rtc::QueuedTask>(
1178 new ProcessAndEncodeAudioTask(std::move(audio_frame), this)));
1179}
1180
1181void ChannelSend::ProcessAndEncodeAudioOnTaskQueue(AudioFrame* audio_input) {
1182 RTC_DCHECK_RUN_ON(encoder_queue_);
1183 RTC_DCHECK_GT(audio_input->samples_per_channel_, 0);
1184 RTC_DCHECK_LE(audio_input->num_channels_, 2);
1185
1186 // Measure time between when the audio frame is added to the task queue and
1187 // when the task is actually executed. Goal is to keep track of unwanted
1188 // extra latency added by the task queue.
1189 RTC_HISTOGRAM_COUNTS_10000("WebRTC.Audio.EncodingTaskQueueLatencyMs",
1190 audio_input->ElapsedProfileTimeMs());
1191
1192 bool is_muted = InputMute();
1193 AudioFrameOperations::Mute(audio_input, previous_frame_muted_, is_muted);
1194
1195 if (_includeAudioLevelIndication) {
1196 size_t length =
1197 audio_input->samples_per_channel_ * audio_input->num_channels_;
1198 RTC_CHECK_LE(length, AudioFrame::kMaxDataSizeBytes);
1199 if (is_muted && previous_frame_muted_) {
1200 rms_level_.AnalyzeMuted(length);
1201 } else {
1202 rms_level_.Analyze(
1203 rtc::ArrayView<const int16_t>(audio_input->data(), length));
1204 }
1205 }
1206 previous_frame_muted_ = is_muted;
1207
1208 // Add 10ms of raw (PCM) audio data to the encoder @ 32kHz.
1209
1210 // The ACM resamples internally.
1211 audio_input->timestamp_ = _timeStamp;
1212 // This call will trigger AudioPacketizationCallback::SendData if encoding
1213 // is done and payload is ready for packetization and transmission.
1214 // Otherwise, it will return without invoking the callback.
1215 if (audio_coding_->Add10MsData(*audio_input) < 0) {
1216 RTC_DLOG(LS_ERROR) << "ACM::Add10MsData() failed.";
1217 return;
1218 }
1219
1220 _timeStamp += static_cast<uint32_t>(audio_input->samples_per_channel_);
1221}
1222
1223void ChannelSend::UpdateOverheadForEncoder() {
1224 size_t overhead_per_packet =
1225 transport_overhead_per_packet_ + rtp_overhead_per_packet_;
1226 audio_coding_->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder) {
1227 if (*encoder) {
1228 (*encoder)->OnReceivedOverhead(overhead_per_packet);
1229 }
1230 });
1231}
1232
1233void ChannelSend::SetTransportOverhead(size_t transport_overhead_per_packet) {
Niels Möller26e88b02018-11-19 15:08:13 +01001234 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller530ead42018-10-04 14:28:39 +02001235 rtc::CritScope cs(&overhead_per_packet_lock_);
1236 transport_overhead_per_packet_ = transport_overhead_per_packet;
1237 UpdateOverheadForEncoder();
1238}
1239
1240// TODO(solenberg): Make AudioSendStream an OverheadObserver instead.
1241void ChannelSend::OnOverheadChanged(size_t overhead_bytes_per_packet) {
1242 rtc::CritScope cs(&overhead_per_packet_lock_);
1243 rtp_overhead_per_packet_ = overhead_bytes_per_packet;
1244 UpdateOverheadForEncoder();
1245}
1246
1247ANAStats ChannelSend::GetANAStatistics() const {
Niels Möller26e88b02018-11-19 15:08:13 +01001248 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Niels Möller530ead42018-10-04 14:28:39 +02001249 return audio_coding_->GetANAStats();
1250}
1251
1252RtpRtcp* ChannelSend::GetRtpRtcp() const {
Niels Möllerdced9f62018-11-19 10:27:07 +01001253 RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread());
Niels Möller530ead42018-10-04 14:28:39 +02001254 return _rtpRtcpModule.get();
1255}
1256
1257int ChannelSend::SetSendRtpHeaderExtension(bool enable,
1258 RTPExtensionType type,
Niels Möller26815232018-11-16 09:32:40 +01001259 int id) {
Niels Möller530ead42018-10-04 14:28:39 +02001260 int error = 0;
1261 _rtpRtcpModule->DeregisterSendRtpHeaderExtension(type);
1262 if (enable) {
Niels Möller26815232018-11-16 09:32:40 +01001263 // TODO(nisse): RtpRtcp::RegisterSendRtpHeaderExtension to take an int
1264 // argument. Currently it wants an uint8_t.
1265 error = _rtpRtcpModule->RegisterSendRtpHeaderExtension(
1266 type, rtc::dchecked_cast<uint8_t>(id));
Niels Möller530ead42018-10-04 14:28:39 +02001267 }
1268 return error;
1269}
1270
Niels Möller530ead42018-10-04 14:28:39 +02001271int64_t ChannelSend::GetRTT() const {
Piotr (Peter) Slatala179a3922018-11-16 09:57:58 -08001272 if (media_transport_) {
1273 // GetRTT is generally used in the RTCP codepath, where media transport is
1274 // not present and so it shouldn't be needed. But it's also invoked in
1275 // 'GetStats' method, and for now returning media transport RTT here gives
1276 // us "free" rtt stats for media transport.
1277 auto target_rate = media_transport_->GetLatestTargetTransferRate();
1278 if (target_rate.has_value()) {
1279 return target_rate.value().network_estimate.round_trip_time.ms();
1280 }
1281
1282 return 0;
1283 }
Niels Möller530ead42018-10-04 14:28:39 +02001284 RtcpMode method = _rtpRtcpModule->RTCP();
1285 if (method == RtcpMode::kOff) {
1286 return 0;
1287 }
1288 std::vector<RTCPReportBlock> report_blocks;
1289 _rtpRtcpModule->RemoteRTCPStat(&report_blocks);
1290
1291 if (report_blocks.empty()) {
1292 return 0;
1293 }
1294
1295 int64_t rtt = 0;
1296 int64_t avg_rtt = 0;
1297 int64_t max_rtt = 0;
1298 int64_t min_rtt = 0;
1299 // We don't know in advance the remote ssrc used by the other end's receiver
1300 // reports, so use the SSRC of the first report block for calculating the RTT.
1301 if (_rtpRtcpModule->RTT(report_blocks[0].sender_ssrc, &rtt, &avg_rtt,
1302 &min_rtt, &max_rtt) != 0) {
1303 return 0;
1304 }
1305 return rtt;
1306}
1307
Benjamin Wright78410ad2018-10-25 09:52:57 -07001308void ChannelSend::SetFrameEncryptor(
1309 rtc::scoped_refptr<FrameEncryptorInterface> frame_encryptor) {
Niels Möller26e88b02018-11-19 15:08:13 +01001310 RTC_DCHECK_RUN_ON(&worker_thread_checker_);
Benjamin Wright84583f62018-10-04 14:22:34 -07001311 rtc::CritScope cs(&encoder_queue_lock_);
1312 if (encoder_queue_is_active_) {
1313 encoder_queue_->PostTask([this, frame_encryptor]() {
Benjamin Wright78410ad2018-10-25 09:52:57 -07001314 this->frame_encryptor_ = std::move(frame_encryptor);
Benjamin Wright84583f62018-10-04 14:22:34 -07001315 });
1316 } else {
Benjamin Wright78410ad2018-10-25 09:52:57 -07001317 frame_encryptor_ = std::move(frame_encryptor);
Benjamin Wright84583f62018-10-04 14:22:34 -07001318 }
1319}
1320
Piotr (Peter) Slatala179a3922018-11-16 09:57:58 -08001321void ChannelSend::OnTargetTransferRate(TargetTransferRate rate) {
1322 RTC_DCHECK(media_transport_);
1323 OnReceivedRtt(rate.network_estimate.round_trip_time.ms());
1324}
1325
1326void ChannelSend::OnReceivedRtt(int64_t rtt_ms) {
1327 // Invoke audio encoders OnReceivedRtt().
1328 audio_coding_->ModifyEncoder(
1329 [rtt_ms](std::unique_ptr<AudioEncoder>* encoder) {
1330 if (*encoder) {
1331 (*encoder)->OnReceivedRtt(rtt_ms);
1332 }
1333 });
1334}
1335
Niels Möllerdced9f62018-11-19 10:27:07 +01001336} // namespace
1337
1338std::unique_ptr<ChannelSendInterface> CreateChannelSend(
1339 rtc::TaskQueue* encoder_queue,
1340 ProcessThread* module_process_thread,
1341 MediaTransportInterface* media_transport,
Niels Möllere9771992018-11-26 10:55:07 +01001342 Transport* rtp_transport,
Niels Möllerdced9f62018-11-19 10:27:07 +01001343 RtcpRttStats* rtcp_rtt_stats,
1344 RtcEventLog* rtc_event_log,
1345 FrameEncryptorInterface* frame_encryptor,
1346 const webrtc::CryptoOptions& crypto_options,
1347 bool extmap_allow_mixed,
1348 int rtcp_report_interval_ms) {
1349 return absl::make_unique<ChannelSend>(
Niels Möllere9771992018-11-26 10:55:07 +01001350 encoder_queue, module_process_thread, media_transport, rtp_transport,
1351 rtcp_rtt_stats, rtc_event_log, frame_encryptor, crypto_options,
1352 extmap_allow_mixed, rtcp_report_interval_ms);
Niels Möllerdced9f62018-11-19 10:27:07 +01001353}
1354
Niels Möller530ead42018-10-04 14:28:39 +02001355} // namespace voe
1356} // namespace webrtc