niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 1 | /* |
leozwang@webrtc.org | 813e4b0 | 2012-03-01 18:34:25 +0000 | [diff] [blame] | 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Fredrik Solenberg | a8b7c7f | 2018-01-17 11:18:31 +0100 | [diff] [blame] | 11 | #ifndef AUDIO_CHANNEL_H_ |
| 12 | #define AUDIO_CHANNEL_H_ |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 13 | |
Fredrik Solenberg | a8b7c7f | 2018-01-17 11:18:31 +0100 | [diff] [blame] | 14 | #include <map> |
kwiberg | b7f89d6 | 2016-02-17 10:04:18 -0800 | [diff] [blame] | 15 | #include <memory> |
Fredrik Solenberg | a8b7c7f | 2018-01-17 11:18:31 +0100 | [diff] [blame] | 16 | #include <string> |
| 17 | #include <vector> |
kwiberg | b7f89d6 | 2016-02-17 10:04:18 -0800 | [diff] [blame] | 18 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 19 | #include "api/audio/audio_mixer.h" |
| 20 | #include "api/audio_codecs/audio_encoder.h" |
| 21 | #include "api/call/audio_sink.h" |
solenberg | 946d886 | 2017-09-21 04:02:53 -0700 | [diff] [blame] | 22 | #include "api/call/transport.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 23 | #include "api/optional.h" |
Fredrik Solenberg | a8b7c7f | 2018-01-17 11:18:31 +0100 | [diff] [blame] | 24 | #include "audio/audio_level.h" |
Mirko Bonadei | 7120742 | 2017-09-15 13:58:09 +0200 | [diff] [blame] | 25 | #include "common_types.h" // NOLINT(build/include) |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 26 | #include "modules/audio_coding/include/audio_coding_module.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 27 | #include "modules/audio_processing/rms_level.h" |
| 28 | #include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" |
| 29 | #include "modules/rtp_rtcp/include/rtp_header_parser.h" |
| 30 | #include "modules/rtp_rtcp/include/rtp_receiver.h" |
| 31 | #include "modules/rtp_rtcp/include/rtp_rtcp.h" |
| 32 | #include "rtc_base/criticalsection.h" |
| 33 | #include "rtc_base/event.h" |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 34 | #include "rtc_base/task_queue.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 35 | #include "rtc_base/thread_checker.h" |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 36 | |
wu@webrtc.org | 94454b7 | 2014-06-05 20:34:08 +0000 | [diff] [blame] | 37 | namespace rtc { |
wu@webrtc.org | 94454b7 | 2014-06-05 20:34:08 +0000 | [diff] [blame] | 38 | class TimestampWrapAroundHandler; |
| 39 | } |
| 40 | |
andrew@webrtc.org | eb524d9 | 2013-09-23 23:02:24 +0000 | [diff] [blame] | 41 | namespace webrtc { |
| 42 | |
tnakamura@webrtc.org | aa4d96a | 2013-07-16 19:25:04 +0000 | [diff] [blame] | 43 | class AudioDeviceModule; |
Stefan Holmer | b86d4e4 | 2015-12-07 10:26:18 +0100 | [diff] [blame] | 44 | class PacketRouter; |
wu@webrtc.org | 822fbd8 | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 45 | class ProcessThread; |
Erik Språng | 737336d | 2016-07-29 12:59:36 +0200 | [diff] [blame] | 46 | class RateLimiter; |
wu@webrtc.org | 822fbd8 | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 47 | class ReceiveStatistics; |
wu@webrtc.org | 82c4b85 | 2014-05-20 22:55:01 +0000 | [diff] [blame] | 48 | class RemoteNtpTimeEstimator; |
ivoc | b04965c | 2015-09-09 00:09:43 -0700 | [diff] [blame] | 49 | class RtcEventLog; |
wu@webrtc.org | 822fbd8 | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 50 | class RTPPayloadRegistry; |
wu@webrtc.org | 822fbd8 | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 51 | class RTPReceiverAudio; |
nisse | 657bab2 | 2017-02-21 06:28:10 -0800 | [diff] [blame] | 52 | class RtpPacketReceived; |
wu@webrtc.org | 822fbd8 | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 53 | class RtpRtcp; |
nisse | b8f9a32 | 2017-03-27 05:36:15 -0700 | [diff] [blame] | 54 | class RtpTransportControllerSendInterface; |
wu@webrtc.org | 822fbd8 | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 55 | class TelephoneEventHandler; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 56 | |
henrika@webrtc.org | 8a2fc88 | 2012-08-22 08:53:55 +0000 | [diff] [blame] | 57 | struct SenderInfo; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 58 | |
solenberg | dd3abbb | 2017-09-18 07:05:30 -0700 | [diff] [blame] | 59 | struct CallStatistics { |
| 60 | unsigned short fractionLost; |
| 61 | unsigned int cumulativeLost; |
| 62 | unsigned int extendedMax; |
| 63 | unsigned int jitterSamples; |
| 64 | int64_t rttMs; |
| 65 | size_t bytesSent; |
| 66 | int packetsSent; |
| 67 | size_t bytesReceived; |
| 68 | int packetsReceived; |
| 69 | // The capture ntp time (in local timebase) of the first played out audio |
| 70 | // frame. |
| 71 | int64_t capture_start_ntp_time_ms_; |
| 72 | }; |
| 73 | |
| 74 | // See section 6.4.2 in http://www.ietf.org/rfc/rfc3550.txt for details. |
| 75 | struct ReportBlock { |
| 76 | uint32_t sender_SSRC; // SSRC of sender |
| 77 | uint32_t source_SSRC; |
| 78 | uint8_t fraction_lost; |
| 79 | uint32_t cumulative_num_packets_lost; |
| 80 | uint32_t extended_highest_sequence_number; |
| 81 | uint32_t interarrival_jitter; |
| 82 | uint32_t last_SR_timestamp; |
| 83 | uint32_t delay_since_last_SR; |
| 84 | }; |
| 85 | |
andrew@webrtc.org | eb524d9 | 2013-09-23 23:02:24 +0000 | [diff] [blame] | 86 | namespace voe { |
| 87 | |
ivoc | 14d5dbe | 2016-07-04 07:06:55 -0700 | [diff] [blame] | 88 | class RtcEventLogProxy; |
michaelt | 9332b7d | 2016-11-30 07:51:13 -0800 | [diff] [blame] | 89 | class RtcpRttStatsProxy; |
Stefan Holmer | b86d4e4 | 2015-12-07 10:26:18 +0100 | [diff] [blame] | 90 | class RtpPacketSenderProxy; |
Stefan Holmer | b86d4e4 | 2015-12-07 10:26:18 +0100 | [diff] [blame] | 91 | class TransportFeedbackProxy; |
Stefan Holmer | b86d4e4 | 2015-12-07 10:26:18 +0100 | [diff] [blame] | 92 | class TransportSequenceNumberProxy; |
mflodman@webrtc.org | 0a7d4ee | 2015-02-17 12:57:14 +0000 | [diff] [blame] | 93 | class VoERtcpObserver; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 94 | |
henrika@webrtc.org | 944cbeb | 2014-03-18 10:32:33 +0000 | [diff] [blame] | 95 | // Helper class to simplify locking scheme for members that are accessed from |
| 96 | // multiple threads. |
| 97 | // Example: a member can be set on thread T1 and read by an internal audio |
| 98 | // thread T2. Accessing the member via this class ensures that we are |
| 99 | // safe and also avoid TSan v2 warnings. |
| 100 | class ChannelState { |
| 101 | public: |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 102 | struct State { |
solenberg | 11ace15 | 2016-09-15 04:29:13 -0700 | [diff] [blame] | 103 | bool playing = false; |
| 104 | bool sending = false; |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 105 | }; |
henrika@webrtc.org | 944cbeb | 2014-03-18 10:32:33 +0000 | [diff] [blame] | 106 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 107 | ChannelState() {} |
| 108 | virtual ~ChannelState() {} |
henrika@webrtc.org | 944cbeb | 2014-03-18 10:32:33 +0000 | [diff] [blame] | 109 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 110 | void Reset() { |
| 111 | rtc::CritScope lock(&lock_); |
| 112 | state_ = State(); |
| 113 | } |
henrika@webrtc.org | 944cbeb | 2014-03-18 10:32:33 +0000 | [diff] [blame] | 114 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 115 | State Get() const { |
| 116 | rtc::CritScope lock(&lock_); |
| 117 | return state_; |
| 118 | } |
henrika@webrtc.org | 944cbeb | 2014-03-18 10:32:33 +0000 | [diff] [blame] | 119 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 120 | void SetPlaying(bool enable) { |
| 121 | rtc::CritScope lock(&lock_); |
| 122 | state_.playing = enable; |
| 123 | } |
henrika@webrtc.org | 944cbeb | 2014-03-18 10:32:33 +0000 | [diff] [blame] | 124 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 125 | void SetSending(bool enable) { |
| 126 | rtc::CritScope lock(&lock_); |
| 127 | state_.sending = enable; |
| 128 | } |
henrika@webrtc.org | 944cbeb | 2014-03-18 10:32:33 +0000 | [diff] [blame] | 129 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 130 | private: |
pbos | d8de115 | 2016-02-01 09:00:51 -0800 | [diff] [blame] | 131 | rtc::CriticalSection lock_; |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 132 | State state_; |
henrika@webrtc.org | 944cbeb | 2014-03-18 10:32:33 +0000 | [diff] [blame] | 133 | }; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 134 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 135 | class Channel |
| 136 | : public RtpData, |
| 137 | public RtpFeedback, |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 138 | public Transport, |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 139 | public AudioPacketizationCallback, // receive encoded packets from the |
| 140 | // ACM |
michaelt | bf65be5 | 2016-12-15 06:24:49 -0800 | [diff] [blame] | 141 | public OverheadObserver { |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 142 | public: |
| 143 | friend class VoERtcpObserver; |
mflodman@webrtc.org | 0a7d4ee | 2015-02-17 12:57:14 +0000 | [diff] [blame] | 144 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 145 | enum { KNumSocketThreads = 1 }; |
| 146 | enum { KNumberOfSocketBuffers = 8 }; |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 147 | // Used for send streams. |
| 148 | Channel(rtc::TaskQueue* encoder_queue, |
| 149 | ProcessThread* module_process_thread, |
| 150 | AudioDeviceModule* audio_device_module); |
| 151 | // Used for receive streams. |
| 152 | Channel(ProcessThread* module_process_thread, |
| 153 | AudioDeviceModule* audio_device_module, |
| 154 | size_t jitter_buffer_max_packets, |
| 155 | bool jitter_buffer_fast_playout, |
| 156 | rtc::scoped_refptr<AudioDecoderFactory> decoder_factory); |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 157 | virtual ~Channel(); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 158 | |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 159 | void SetSink(AudioSinkInterface* sink); |
ossu | 29b1a8d | 2016-06-13 07:34:51 -0700 | [diff] [blame] | 160 | |
kwiberg | 1c07c70 | 2017-03-27 07:15:49 -0700 | [diff] [blame] | 161 | void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs); |
| 162 | |
ossu | 1ffbd6c | 2017-04-06 12:05:04 -0700 | [diff] [blame] | 163 | // Send using this encoder, with this payload type. |
| 164 | bool SetEncoder(int payload_type, std::unique_ptr<AudioEncoder> encoder); |
ossu | 20a4b3f | 2017-04-27 02:08:52 -0700 | [diff] [blame] | 165 | void ModifyEncoder( |
| 166 | rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier); |
ossu | 1ffbd6c | 2017-04-06 12:05:04 -0700 | [diff] [blame] | 167 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 168 | // API methods |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 169 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 170 | // VoEBase |
| 171 | int32_t StartPlayout(); |
| 172 | int32_t StopPlayout(); |
| 173 | int32_t StartSend(); |
henrika | ec6fbd2 | 2017-03-31 05:43:36 -0700 | [diff] [blame] | 174 | void StopSend(); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 175 | |
solenberg | 6dc2038 | 2017-09-18 05:22:39 -0700 | [diff] [blame] | 176 | // Codecs |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 177 | int32_t GetRecCodec(CodecInst& codec); |
minyue | 78b4d56 | 2016-11-30 04:47:39 -0800 | [diff] [blame] | 178 | void SetBitRate(int bitrate_bps, int64_t probing_interval_ms); |
minyue | 7e30432 | 2016-10-12 05:00:55 -0700 | [diff] [blame] | 179 | bool EnableAudioNetworkAdaptor(const std::string& config_string); |
| 180 | void DisableAudioNetworkAdaptor(); |
| 181 | void SetReceiverFrameLengthRange(int min_frame_length_ms, |
| 182 | int max_frame_length_ms); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 183 | |
solenberg | 946d886 | 2017-09-21 04:02:53 -0700 | [diff] [blame] | 184 | // Network |
solenberg | 1c239d4 | 2017-09-29 06:00:28 -0700 | [diff] [blame] | 185 | void RegisterTransport(Transport* transport); |
nisse | 657bab2 | 2017-02-21 06:28:10 -0800 | [diff] [blame] | 186 | // TODO(nisse, solenberg): Delete when VoENetwork is deleted. |
mflodman | 3d7db26 | 2016-04-29 00:57:13 -0700 | [diff] [blame] | 187 | int32_t ReceivedRTCPPacket(const uint8_t* data, size_t length); |
nisse | 657bab2 | 2017-02-21 06:28:10 -0800 | [diff] [blame] | 188 | void OnRtpPacket(const RtpPacketReceived& packet); |
pwestin@webrtc.org | 684f057 | 2013-03-13 23:20:57 +0000 | [diff] [blame] | 189 | |
solenberg | 8d73f8c | 2017-03-08 01:52:20 -0800 | [diff] [blame] | 190 | // Muting, Volume and Level. |
| 191 | void SetInputMute(bool enable); |
| 192 | void SetChannelOutputVolumeScaling(float scaling); |
| 193 | int GetSpeechOutputLevel() const; |
| 194 | int GetSpeechOutputLevelFullRange() const; |
zstein | e76bd3a | 2017-07-14 12:17:49 -0700 | [diff] [blame] | 195 | // See description of "totalAudioEnergy" in the WebRTC stats spec: |
| 196 | // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy |
| 197 | double GetTotalOutputEnergy() const; |
| 198 | double GetTotalOutputDuration() const; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 199 | |
solenberg | c6192a9 | 2017-03-13 02:36:19 -0700 | [diff] [blame] | 200 | // Stats. |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 201 | int GetNetworkStatistics(NetworkStatistics& stats); |
| 202 | void GetDecodingCallStatistics(AudioDecodingCallStats* stats) const; |
ivoc | e1198e0 | 2017-09-08 08:13:19 -0700 | [diff] [blame] | 203 | ANAStats GetANAStatistics() const; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 204 | |
solenberg | c6192a9 | 2017-03-13 02:36:19 -0700 | [diff] [blame] | 205 | // Audio+Video Sync. |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 206 | uint32_t GetDelayEstimate() const; |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 207 | int SetMinimumPlayoutDelay(int delayMs); |
| 208 | int GetPlayoutTimestamp(unsigned int& timestamp); |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 209 | int GetRtpRtcp(RtpRtcp** rtpRtcpModule, RtpReceiver** rtp_receiver) const; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 210 | |
solenberg | c6192a9 | 2017-03-13 02:36:19 -0700 | [diff] [blame] | 211 | // DTMF. |
solenberg | 8842c3e | 2016-03-11 03:06:41 -0800 | [diff] [blame] | 212 | int SendTelephoneEventOutband(int event, int duration_ms); |
solenberg | ffbbcac | 2016-11-17 05:25:37 -0800 | [diff] [blame] | 213 | int SetSendTelephoneEventPayloadType(int payload_type, int payload_frequency); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 214 | |
solenberg | dd3abbb | 2017-09-18 07:05:30 -0700 | [diff] [blame] | 215 | // RTP+RTCP |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 216 | int SetLocalSSRC(unsigned int ssrc); |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 217 | int SetSendAudioLevelIndicationStatus(bool enable, unsigned char id); |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 218 | void EnableSendTransportSequenceNumber(int id); |
Stefan Holmer | b86d4e4 | 2015-12-07 10:26:18 +0100 | [diff] [blame] | 219 | |
stefan | 7de8d64 | 2017-02-07 07:14:08 -0800 | [diff] [blame] | 220 | void RegisterSenderCongestionControlObjects( |
nisse | b8f9a32 | 2017-03-27 05:36:15 -0700 | [diff] [blame] | 221 | RtpTransportControllerSendInterface* transport, |
stefan | 7de8d64 | 2017-02-07 07:14:08 -0800 | [diff] [blame] | 222 | RtcpBandwidthObserver* bandwidth_observer); |
| 223 | void RegisterReceiverCongestionControlObjects(PacketRouter* packet_router); |
nisse | fdbfdc9 | 2017-03-31 05:44:52 -0700 | [diff] [blame] | 224 | void ResetSenderCongestionControlObjects(); |
| 225 | void ResetReceiverCongestionControlObjects(); |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 226 | void SetRTCPStatus(bool enable); |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 227 | int SetRTCP_CNAME(const char cName[256]); |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 228 | int GetRemoteRTCPReportBlocks(std::vector<ReportBlock>* report_blocks); |
| 229 | int GetRTPStatistics(CallStatistics& stats); |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 230 | void SetNACKStatus(bool enable, int maxNumberOfPackets); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 231 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 232 | // From AudioPacketizationCallback in the ACM |
| 233 | int32_t SendData(FrameType frameType, |
| 234 | uint8_t payloadType, |
| 235 | uint32_t timeStamp, |
| 236 | const uint8_t* payloadData, |
| 237 | size_t payloadSize, |
| 238 | const RTPFragmentationHeader* fragmentation) override; |
xians@webrtc.org | 3cefbc9 | 2014-10-10 09:42:53 +0000 | [diff] [blame] | 239 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 240 | // From RtpData in the RTP/RTCP module |
| 241 | int32_t OnReceivedPayloadData(const uint8_t* payloadData, |
| 242 | size_t payloadSize, |
| 243 | const WebRtcRTPHeader* rtpHeader) override; |
wu@webrtc.org | 822fbd8 | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 244 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 245 | // From RtpFeedback in the RTP/RTCP module |
Karl Wiberg | c62f6c7 | 2017-10-04 12:38:53 +0200 | [diff] [blame] | 246 | int32_t OnInitializeDecoder(int payload_type, |
| 247 | const SdpAudioFormat& audio_format, |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 248 | uint32_t rate) override; |
| 249 | void OnIncomingSSRCChanged(uint32_t ssrc) override; |
| 250 | void OnIncomingCSRCChanged(uint32_t CSRC, bool added) override; |
wu@webrtc.org | 822fbd8 | 2013-08-15 23:38:54 +0000 | [diff] [blame] | 251 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 252 | // From Transport (called by the RTP/RTCP module) |
| 253 | bool SendRtp(const uint8_t* data, |
| 254 | size_t len, |
| 255 | const PacketOptions& packet_options) override; |
| 256 | bool SendRtcp(const uint8_t* data, size_t len) override; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 257 | |
aleloi | aed581a | 2016-10-20 06:32:39 -0700 | [diff] [blame] | 258 | // From AudioMixer::Source. |
aleloi | 6c27849 | 2016-10-20 14:24:39 -0700 | [diff] [blame] | 259 | AudioMixer::Source::AudioFrameInfo GetAudioFrameWithInfo( |
| 260 | int sample_rate_hz, |
| 261 | AudioFrame* audio_frame); |
aleloi | aed581a | 2016-10-20 06:32:39 -0700 | [diff] [blame] | 262 | |
solenberg | 2397b9a | 2017-09-22 06:48:10 -0700 | [diff] [blame] | 263 | int PreferredSampleRate() const; |
| 264 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 265 | bool Playing() const { return channel_state_.Get().playing; } |
| 266 | bool Sending() const { return channel_state_.Get().sending; } |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 267 | RtpRtcp* RtpRtcpModulePtr() const { return _rtpRtcpModule.get(); } |
| 268 | int8_t OutputEnergyLevel() const { return _outputAudioLevel.Level(); } |
henrika | ec6fbd2 | 2017-03-31 05:43:36 -0700 | [diff] [blame] | 269 | |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 270 | // ProcessAndEncodeAudio() posts a task on the shared encoder task queue, |
| 271 | // which in turn calls (on the queue) ProcessAndEncodeAudioOnTaskQueue() where |
| 272 | // the actual processing of the audio takes place. The processing mainly |
| 273 | // consists of encoding and preparing the result for sending by adding it to a |
| 274 | // send queue. |
henrika | ec6fbd2 | 2017-03-31 05:43:36 -0700 | [diff] [blame] | 275 | // The main reason for using a task queue here is to release the native, |
| 276 | // OS-specific, audio capture thread as soon as possible to ensure that it |
| 277 | // can go back to sleep and be prepared to deliver an new captured audio |
| 278 | // packet. |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 279 | void ProcessAndEncodeAudio(std::unique_ptr<AudioFrame> audio_frame); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 280 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 281 | // Associate to a send channel. |
| 282 | // Used for obtaining RTT for a receive-only channel. |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 283 | void SetAssociatedSendChannel(Channel* channel); |
Minyue | 2013aec | 2015-05-13 14:14:42 +0200 | [diff] [blame] | 284 | |
ivoc | 14d5dbe | 2016-07-04 07:06:55 -0700 | [diff] [blame] | 285 | // Set a RtcEventLog logging object. |
| 286 | void SetRtcEventLog(RtcEventLog* event_log); |
| 287 | |
michaelt | 9332b7d | 2016-11-30 07:51:13 -0800 | [diff] [blame] | 288 | void SetRtcpRttStats(RtcpRttStats* rtcp_rtt_stats); |
nisse | 284542b | 2017-01-10 08:58:32 -0800 | [diff] [blame] | 289 | void SetTransportOverhead(size_t transport_overhead_per_packet); |
michaelt | 79e0588 | 2016-11-08 02:50:09 -0800 | [diff] [blame] | 290 | |
michaelt | bf65be5 | 2016-12-15 06:24:49 -0800 | [diff] [blame] | 291 | // From OverheadObserver in the RTP/RTCP module |
| 292 | void OnOverheadChanged(size_t overhead_bytes_per_packet) override; |
| 293 | |
elad.alon | d12a8e1 | 2017-03-23 11:04:48 -0700 | [diff] [blame] | 294 | // The existence of this function alongside OnUplinkPacketLossRate is |
| 295 | // a compromise. We want the encoder to be agnostic of the PLR source, but |
| 296 | // we also don't want it to receive conflicting information from TWCC and |
| 297 | // from RTCP-XR. |
| 298 | void OnTwccBasedUplinkPacketLossRate(float packet_loss_rate); |
minyue@webrtc.org | c1a40a7 | 2014-05-28 09:52:06 +0000 | [diff] [blame] | 299 | |
elad.alon | dadb4dc | 2017-03-23 15:29:50 -0700 | [diff] [blame] | 300 | void OnRecoverableUplinkPacketLossRate(float recoverable_packet_loss_rate); |
| 301 | |
hbos | 8d609f6 | 2017-04-10 07:39:05 -0700 | [diff] [blame] | 302 | std::vector<RtpSource> GetSources() const { |
| 303 | return rtp_receiver_->GetSources(); |
| 304 | } |
| 305 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 306 | private: |
henrika | ec6fbd2 | 2017-03-31 05:43:36 -0700 | [diff] [blame] | 307 | class ProcessAndEncodeAudioTask; |
elad.alon | d12a8e1 | 2017-03-23 11:04:48 -0700 | [diff] [blame] | 308 | |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 309 | void Init(); |
| 310 | void Terminate(); |
| 311 | |
solenberg | dd3abbb | 2017-09-18 07:05:30 -0700 | [diff] [blame] | 312 | int GetRemoteSSRC(unsigned int& ssrc); |
henrika | ec6fbd2 | 2017-03-31 05:43:36 -0700 | [diff] [blame] | 313 | void OnUplinkPacketLossRate(float packet_loss_rate); |
solenberg | 8d73f8c | 2017-03-08 01:52:20 -0800 | [diff] [blame] | 314 | bool InputMute() const; |
nisse | 30e8931 | 2017-05-29 08:16:37 -0700 | [diff] [blame] | 315 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 316 | bool ReceivePacket(const uint8_t* packet, |
| 317 | size_t packet_length, |
Niels Möller | 22ec952 | 2017-10-05 08:39:15 +0200 | [diff] [blame] | 318 | const RTPHeader& header); |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 319 | bool IsPacketInOrder(const RTPHeader& header) const; |
| 320 | bool IsPacketRetransmitted(const RTPHeader& header, bool in_order) const; |
| 321 | int ResendPackets(const uint16_t* sequence_numbers, int length); |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 322 | void UpdatePlayoutTimestamp(bool rtcp); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 323 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 324 | int SetSendRtpHeaderExtension(bool enable, |
| 325 | RTPExtensionType type, |
| 326 | unsigned char id); |
andrew@webrtc.org | eb524d9 | 2013-09-23 23:02:24 +0000 | [diff] [blame] | 327 | |
hbos | 3fd31fe | 2017-02-28 05:43:16 -0800 | [diff] [blame] | 328 | void UpdateOverheadForEncoder() |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 329 | RTC_EXCLUSIVE_LOCKS_REQUIRED(overhead_per_packet_lock_); |
nisse | 284542b | 2017-01-10 08:58:32 -0800 | [diff] [blame] | 330 | |
ossu | e280cde | 2016-10-12 11:04:10 -0700 | [diff] [blame] | 331 | int GetRtpTimestampRateHz() const; |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 332 | int64_t GetRTT(bool allow_associate_channel) const; |
wu@webrtc.org | 94454b7 | 2014-06-05 20:34:08 +0000 | [diff] [blame] | 333 | |
henrika | ec6fbd2 | 2017-03-31 05:43:36 -0700 | [diff] [blame] | 334 | // Called on the encoder task queue when a new input audio frame is ready |
| 335 | // for encoding. |
| 336 | void ProcessAndEncodeAudioOnTaskQueue(AudioFrame* audio_input); |
| 337 | |
pbos | d8de115 | 2016-02-01 09:00:51 -0800 | [diff] [blame] | 338 | rtc::CriticalSection _callbackCritSect; |
| 339 | rtc::CriticalSection volume_settings_critsect_; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 340 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 341 | ChannelState channel_state_; |
henrika@webrtc.org | 944cbeb | 2014-03-18 10:32:33 +0000 | [diff] [blame] | 342 | |
ivoc | 14d5dbe | 2016-07-04 07:06:55 -0700 | [diff] [blame] | 343 | std::unique_ptr<voe::RtcEventLogProxy> event_log_proxy_; |
michaelt | 9332b7d | 2016-11-30 07:51:13 -0800 | [diff] [blame] | 344 | std::unique_ptr<voe::RtcpRttStatsProxy> rtcp_rtt_stats_proxy_; |
Ivo Creusen | ae856f2 | 2015-09-17 16:30:16 +0200 | [diff] [blame] | 345 | |
kwiberg | b7f89d6 | 2016-02-17 10:04:18 -0800 | [diff] [blame] | 346 | std::unique_ptr<RTPPayloadRegistry> rtp_payload_registry_; |
| 347 | std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_; |
kwiberg | b7f89d6 | 2016-02-17 10:04:18 -0800 | [diff] [blame] | 348 | std::unique_ptr<RtpReceiver> rtp_receiver_; |
danilchap | 799a9d0 | 2016-09-22 03:36:27 -0700 | [diff] [blame] | 349 | TelephoneEventHandler* telephone_event_handler_; |
kwiberg | b7f89d6 | 2016-02-17 10:04:18 -0800 | [diff] [blame] | 350 | std::unique_ptr<RtpRtcp> _rtpRtcpModule; |
| 351 | std::unique_ptr<AudioCodingModule> audio_coding_; |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 352 | AudioSinkInterface* audio_sink_ = nullptr; |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 353 | AudioLevel _outputAudioLevel; |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 354 | uint32_t _timeStamp RTC_ACCESS_ON(encoder_queue_); |
turaj@webrtc.org | 167b6df | 2013-12-13 21:05:07 +0000 | [diff] [blame] | 355 | |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 356 | RemoteNtpTimeEstimator ntp_estimator_ RTC_GUARDED_BY(ts_stats_lock_); |
wu@webrtc.org | 82c4b85 | 2014-05-20 22:55:01 +0000 | [diff] [blame] | 357 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 358 | // Timestamp of the audio pulled from NetEq. |
henrik.lundin | 96bd502 | 2016-04-06 04:13:56 -0700 | [diff] [blame] | 359 | rtc::Optional<uint32_t> jitter_buffer_playout_timestamp_; |
solenberg | fe7dd6d | 2017-03-11 08:10:43 -0800 | [diff] [blame] | 360 | |
| 361 | rtc::CriticalSection video_sync_lock_; |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 362 | uint32_t playout_timestamp_rtp_ RTC_GUARDED_BY(video_sync_lock_); |
| 363 | uint32_t playout_delay_ms_ RTC_GUARDED_BY(video_sync_lock_); |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 364 | uint16_t send_sequence_number_; |
pwestin@webrtc.org | 1de0135 | 2013-04-11 20:23:35 +0000 | [diff] [blame] | 365 | |
pbos | d8de115 | 2016-02-01 09:00:51 -0800 | [diff] [blame] | 366 | rtc::CriticalSection ts_stats_lock_; |
wu@webrtc.org | cb711f7 | 2014-05-19 17:39:11 +0000 | [diff] [blame] | 367 | |
kwiberg | b7f89d6 | 2016-02-17 10:04:18 -0800 | [diff] [blame] | 368 | std::unique_ptr<rtc::TimestampWrapAroundHandler> rtp_ts_wraparound_handler_; |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 369 | // The rtp timestamp of the first played out audio frame. |
| 370 | int64_t capture_start_rtp_time_stamp_; |
| 371 | // The capture ntp time (in local timebase) of the first played out audio |
| 372 | // frame. |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 373 | int64_t capture_start_ntp_time_ms_ RTC_GUARDED_BY(ts_stats_lock_); |
wu@webrtc.org | cb711f7 | 2014-05-19 17:39:11 +0000 | [diff] [blame] | 374 | |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 375 | // uses |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 376 | ProcessThread* _moduleProcessThreadPtr; |
| 377 | AudioDeviceModule* _audioDeviceModulePtr; |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 378 | Transport* _transportPtr; // WebRtc socket or external transport |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 379 | RmsLevel rms_level_ RTC_ACCESS_ON(encoder_queue_); |
| 380 | bool input_mute_ RTC_GUARDED_BY(volume_settings_critsect_); |
| 381 | bool previous_frame_muted_ RTC_ACCESS_ON(encoder_queue_); |
| 382 | float _outputGain RTC_GUARDED_BY(volume_settings_critsect_); |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 383 | // VoeRTP_RTCP |
henrika | ec6fbd2 | 2017-03-31 05:43:36 -0700 | [diff] [blame] | 384 | // TODO(henrika): can today be accessed on the main thread and on the |
| 385 | // task queue; hence potential race. |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 386 | bool _includeAudioLevelIndication; |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 387 | size_t transport_overhead_per_packet_ |
| 388 | RTC_GUARDED_BY(overhead_per_packet_lock_); |
| 389 | size_t rtp_overhead_per_packet_ RTC_GUARDED_BY(overhead_per_packet_lock_); |
hbos | 3fd31fe | 2017-02-28 05:43:16 -0800 | [diff] [blame] | 390 | rtc::CriticalSection overhead_per_packet_lock_; |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 391 | // RtcpBandwidthObserver |
kwiberg | b7f89d6 | 2016-02-17 10:04:18 -0800 | [diff] [blame] | 392 | std::unique_ptr<VoERtcpObserver> rtcp_observer_; |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 393 | // An associated send channel. |
pbos | d8de115 | 2016-02-01 09:00:51 -0800 | [diff] [blame] | 394 | rtc::CriticalSection assoc_send_channel_lock_; |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 395 | Channel* associated_send_channel_ RTC_GUARDED_BY(assoc_send_channel_lock_); |
Stefan Holmer | b86d4e4 | 2015-12-07 10:26:18 +0100 | [diff] [blame] | 396 | |
Fredrik Solenberg | 8f5787a | 2018-01-11 13:52:30 +0100 | [diff] [blame] | 397 | bool pacing_enabled_ = true; |
kwiberg | 55b97fe | 2016-01-28 05:22:45 -0800 | [diff] [blame] | 398 | PacketRouter* packet_router_ = nullptr; |
kwiberg | b7f89d6 | 2016-02-17 10:04:18 -0800 | [diff] [blame] | 399 | std::unique_ptr<TransportFeedbackProxy> feedback_observer_proxy_; |
| 400 | std::unique_ptr<TransportSequenceNumberProxy> seq_num_allocator_proxy_; |
| 401 | std::unique_ptr<RtpPacketSenderProxy> rtp_packet_sender_proxy_; |
Erik Språng | 737336d | 2016-07-29 12:59:36 +0200 | [diff] [blame] | 402 | std::unique_ptr<RateLimiter> retransmission_rate_limiter_; |
ossu | 29b1a8d | 2016-06-13 07:34:51 -0700 | [diff] [blame] | 403 | |
tommi | 0a2391f | 2017-03-21 02:31:51 -0700 | [diff] [blame] | 404 | rtc::ThreadChecker construction_thread_; |
elad.alon | d12a8e1 | 2017-03-23 11:04:48 -0700 | [diff] [blame] | 405 | |
| 406 | const bool use_twcc_plr_for_ana_; |
henrika | ec6fbd2 | 2017-03-31 05:43:36 -0700 | [diff] [blame] | 407 | |
henrika | 4515fa0 | 2017-05-03 08:30:15 -0700 | [diff] [blame] | 408 | rtc::CriticalSection encoder_queue_lock_; |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 409 | bool encoder_queue_is_active_ RTC_GUARDED_BY(encoder_queue_lock_) = false; |
henrika | ec6fbd2 | 2017-03-31 05:43:36 -0700 | [diff] [blame] | 410 | rtc::TaskQueue* encoder_queue_ = nullptr; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 411 | }; |
| 412 | |
pbos@webrtc.org | d900e8b | 2013-07-03 15:12:26 +0000 | [diff] [blame] | 413 | } // namespace voe |
pbos@webrtc.org | d900e8b | 2013-07-03 15:12:26 +0000 | [diff] [blame] | 414 | } // namespace webrtc |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 415 | |
Fredrik Solenberg | a8b7c7f | 2018-01-17 11:18:31 +0100 | [diff] [blame] | 416 | #endif // AUDIO_CHANNEL_H_ |