Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
| 11 | #include "audio/audio_transport_impl.h" |
| 12 | |
| 13 | #include <algorithm> |
| 14 | #include <memory> |
| 15 | #include <utility> |
| 16 | |
Fredrik Solenberg | a8b7c7f | 2018-01-17 11:18:31 +0100 | [diff] [blame] | 17 | #include "audio/remix_resample.h" |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 18 | #include "audio/utility/audio_frame_operations.h" |
| 19 | #include "call/audio_send_stream.h" |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 20 | #include "rtc_base/checks.h" |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 21 | |
| 22 | namespace webrtc { |
| 23 | |
| 24 | namespace { |
| 25 | |
| 26 | // We want to process at the lowest sample rate and channel count possible |
| 27 | // without losing information. Choose the lowest native rate at least equal to |
| 28 | // the minimum of input and codec rates, choose lowest channel count, and |
| 29 | // configure the audio frame. |
| 30 | void InitializeCaptureFrame(int input_sample_rate, |
| 31 | int send_sample_rate_hz, |
| 32 | size_t input_num_channels, |
| 33 | size_t send_num_channels, |
| 34 | AudioFrame* audio_frame) { |
| 35 | RTC_DCHECK(audio_frame); |
| 36 | int min_processing_rate_hz = std::min(input_sample_rate, send_sample_rate_hz); |
| 37 | for (int native_rate_hz : AudioProcessing::kNativeSampleRatesHz) { |
| 38 | audio_frame->sample_rate_hz_ = native_rate_hz; |
| 39 | if (audio_frame->sample_rate_hz_ >= min_processing_rate_hz) { |
| 40 | break; |
| 41 | } |
| 42 | } |
| 43 | audio_frame->num_channels_ = std::min(input_num_channels, send_num_channels); |
| 44 | } |
| 45 | |
henrika | 649a385 | 2017-12-22 13:58:29 +0100 | [diff] [blame] | 46 | void ProcessCaptureFrame(uint32_t delay_ms, |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 47 | bool key_pressed, |
| 48 | bool swap_stereo_channels, |
| 49 | AudioProcessing* audio_processing, |
| 50 | AudioFrame* audio_frame) { |
| 51 | RTC_DCHECK(audio_processing); |
| 52 | RTC_DCHECK(audio_frame); |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 53 | audio_processing->set_stream_delay_ms(delay_ms); |
| 54 | audio_processing->set_stream_key_pressed(key_pressed); |
henrika | 649a385 | 2017-12-22 13:58:29 +0100 | [diff] [blame] | 55 | int error = audio_processing->ProcessStream(audio_frame); |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 56 | RTC_DCHECK_EQ(0, error) << "ProcessStream() error: " << error; |
| 57 | if (swap_stereo_channels) { |
| 58 | AudioFrameOperations::SwapStereoChannels(audio_frame); |
| 59 | } |
| 60 | } |
| 61 | |
| 62 | // Resample audio in |frame| to given sample rate preserving the |
| 63 | // channel count and place the result in |destination|. |
| 64 | int Resample(const AudioFrame& frame, |
| 65 | const int destination_sample_rate, |
| 66 | PushResampler<int16_t>* resampler, |
| 67 | int16_t* destination) { |
| 68 | const int number_of_channels = static_cast<int>(frame.num_channels_); |
| 69 | const int target_number_of_samples_per_channel = |
| 70 | destination_sample_rate / 100; |
| 71 | resampler->InitializeIfNeeded(frame.sample_rate_hz_, destination_sample_rate, |
| 72 | number_of_channels); |
| 73 | |
| 74 | // TODO(yujo): make resampler take an AudioFrame, and add special case |
| 75 | // handling of muted frames. |
| 76 | return resampler->Resample( |
| 77 | frame.data(), frame.samples_per_channel_ * number_of_channels, |
| 78 | destination, number_of_channels * target_number_of_samples_per_channel); |
| 79 | } |
| 80 | } // namespace |
| 81 | |
| 82 | AudioTransportImpl::AudioTransportImpl(AudioMixer* mixer, |
henrika | 649a385 | 2017-12-22 13:58:29 +0100 | [diff] [blame] | 83 | AudioProcessing* audio_processing) |
Yves Gerey | 665174f | 2018-06-19 15:03:05 +0200 | [diff] [blame] | 84 | : audio_processing_(audio_processing), mixer_(mixer) { |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 85 | RTC_DCHECK(mixer); |
| 86 | RTC_DCHECK(audio_processing); |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 87 | } |
| 88 | |
| 89 | AudioTransportImpl::~AudioTransportImpl() {} |
| 90 | |
| 91 | // Not used in Chromium. Process captured audio and distribute to all sending |
| 92 | // streams, and try to do this at the lowest possible sample rate. |
| 93 | int32_t AudioTransportImpl::RecordedDataIsAvailable( |
| 94 | const void* audio_data, |
| 95 | const size_t number_of_frames, |
| 96 | const size_t bytes_per_sample, |
| 97 | const size_t number_of_channels, |
| 98 | const uint32_t sample_rate, |
| 99 | const uint32_t audio_delay_milliseconds, |
| 100 | const int32_t /*clock_drift*/, |
henrika | 649a385 | 2017-12-22 13:58:29 +0100 | [diff] [blame] | 101 | const uint32_t /*volume*/, |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 102 | const bool key_pressed, |
| 103 | uint32_t& /*new_mic_volume*/) { // NOLINT: to avoid changing APIs |
| 104 | RTC_DCHECK(audio_data); |
| 105 | RTC_DCHECK_GE(number_of_channels, 1); |
| 106 | RTC_DCHECK_LE(number_of_channels, 2); |
| 107 | RTC_DCHECK_EQ(2 * number_of_channels, bytes_per_sample); |
| 108 | RTC_DCHECK_GE(sample_rate, AudioProcessing::NativeRate::kSampleRate8kHz); |
| 109 | // 100 = 1 second / data duration (10 ms). |
| 110 | RTC_DCHECK_EQ(number_of_frames * 100, sample_rate); |
| 111 | RTC_DCHECK_LE(bytes_per_sample * number_of_frames * number_of_channels, |
| 112 | AudioFrame::kMaxDataSizeBytes); |
| 113 | |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 114 | int send_sample_rate_hz = 0; |
| 115 | size_t send_num_channels = 0; |
| 116 | bool swap_stereo_channels = false; |
| 117 | { |
| 118 | rtc::CritScope lock(&capture_lock_); |
| 119 | send_sample_rate_hz = send_sample_rate_hz_; |
| 120 | send_num_channels = send_num_channels_; |
| 121 | swap_stereo_channels = swap_stereo_channels_; |
| 122 | } |
| 123 | |
| 124 | std::unique_ptr<AudioFrame> audio_frame(new AudioFrame()); |
Yves Gerey | 665174f | 2018-06-19 15:03:05 +0200 | [diff] [blame] | 125 | InitializeCaptureFrame(sample_rate, send_sample_rate_hz, number_of_channels, |
| 126 | send_num_channels, audio_frame.get()); |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 127 | voe::RemixAndResample(static_cast<const int16_t*>(audio_data), |
| 128 | number_of_frames, number_of_channels, sample_rate, |
| 129 | &capture_resampler_, audio_frame.get()); |
henrika | 649a385 | 2017-12-22 13:58:29 +0100 | [diff] [blame] | 130 | ProcessCaptureFrame(audio_delay_milliseconds, key_pressed, |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 131 | swap_stereo_channels, audio_processing_, |
| 132 | audio_frame.get()); |
| 133 | |
| 134 | // Typing detection (utilizes the APM/VAD decision). We let the VAD determine |
| 135 | // if we're using this feature or not. |
Sam Zackrisson | ba50223 | 2019-01-04 10:36:48 +0100 | [diff] [blame] | 136 | // TODO(solenberg): GetConfig() takes a lock. Work around that. |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 137 | bool typing_detected = false; |
Sam Zackrisson | ba50223 | 2019-01-04 10:36:48 +0100 | [diff] [blame] | 138 | if (audio_processing_->GetConfig().voice_detection.enabled) { |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 139 | if (audio_frame->vad_activity_ != AudioFrame::kVadUnknown) { |
| 140 | bool vad_active = audio_frame->vad_activity_ == AudioFrame::kVadActive; |
| 141 | typing_detected = typing_detection_.Process(key_pressed, vad_active); |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | // Measure audio level of speech after all processing. |
| 146 | double sample_duration = static_cast<double>(number_of_frames) / sample_rate; |
| 147 | audio_level_.ComputeLevel(*audio_frame.get(), sample_duration); |
| 148 | |
| 149 | // Copy frame and push to each sending stream. The copy is required since an |
| 150 | // encoding task will be posted internally to each stream. |
| 151 | { |
| 152 | rtc::CritScope lock(&capture_lock_); |
| 153 | typing_noise_detected_ = typing_detected; |
| 154 | |
| 155 | RTC_DCHECK_GT(audio_frame->samples_per_channel_, 0); |
| 156 | if (!sending_streams_.empty()) { |
| 157 | auto it = sending_streams_.begin(); |
| 158 | while (++it != sending_streams_.end()) { |
| 159 | std::unique_ptr<AudioFrame> audio_frame_copy(new AudioFrame()); |
| 160 | audio_frame_copy->CopyFrom(*audio_frame.get()); |
| 161 | (*it)->SendAudioData(std::move(audio_frame_copy)); |
| 162 | } |
| 163 | // Send the original frame to the first stream w/o copying. |
| 164 | (*sending_streams_.begin())->SendAudioData(std::move(audio_frame)); |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | return 0; |
| 169 | } |
| 170 | |
| 171 | // Mix all received streams, feed the result to the AudioProcessing module, then |
| 172 | // resample the result to the requested output rate. |
| 173 | int32_t AudioTransportImpl::NeedMorePlayData(const size_t nSamples, |
Yves Gerey | 665174f | 2018-06-19 15:03:05 +0200 | [diff] [blame] | 174 | const size_t nBytesPerSample, |
| 175 | const size_t nChannels, |
| 176 | const uint32_t samplesPerSec, |
| 177 | void* audioSamples, |
| 178 | size_t& nSamplesOut, |
| 179 | int64_t* elapsed_time_ms, |
| 180 | int64_t* ntp_time_ms) { |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 181 | RTC_DCHECK_EQ(sizeof(int16_t) * nChannels, nBytesPerSample); |
| 182 | RTC_DCHECK_GE(nChannels, 1); |
| 183 | RTC_DCHECK_LE(nChannels, 2); |
| 184 | RTC_DCHECK_GE( |
| 185 | samplesPerSec, |
| 186 | static_cast<uint32_t>(AudioProcessing::NativeRate::kSampleRate8kHz)); |
| 187 | |
| 188 | // 100 = 1 second / data duration (10 ms). |
| 189 | RTC_DCHECK_EQ(nSamples * 100, samplesPerSec); |
| 190 | RTC_DCHECK_LE(nBytesPerSample * nSamples * nChannels, |
| 191 | AudioFrame::kMaxDataSizeBytes); |
| 192 | |
| 193 | mixer_->Mix(nChannels, &mixed_frame_); |
| 194 | *elapsed_time_ms = mixed_frame_.elapsed_time_ms_; |
| 195 | *ntp_time_ms = mixed_frame_.ntp_time_ms_; |
| 196 | |
| 197 | const auto error = audio_processing_->ProcessReverseStream(&mixed_frame_); |
| 198 | RTC_DCHECK_EQ(error, AudioProcessing::kNoError); |
| 199 | |
| 200 | nSamplesOut = Resample(mixed_frame_, samplesPerSec, &render_resampler_, |
| 201 | static_cast<int16_t*>(audioSamples)); |
| 202 | RTC_DCHECK_EQ(nSamplesOut, nChannels * nSamples); |
| 203 | return 0; |
| 204 | } |
| 205 | |
| 206 | // Used by Chromium - same as NeedMorePlayData() but because Chrome has its |
| 207 | // own APM instance, does not call audio_processing_->ProcessReverseStream(). |
| 208 | void AudioTransportImpl::PullRenderData(int bits_per_sample, |
Yves Gerey | 665174f | 2018-06-19 15:03:05 +0200 | [diff] [blame] | 209 | int sample_rate, |
| 210 | size_t number_of_channels, |
| 211 | size_t number_of_frames, |
| 212 | void* audio_data, |
| 213 | int64_t* elapsed_time_ms, |
| 214 | int64_t* ntp_time_ms) { |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 215 | RTC_DCHECK_EQ(bits_per_sample, 16); |
| 216 | RTC_DCHECK_GE(number_of_channels, 1); |
| 217 | RTC_DCHECK_LE(number_of_channels, 2); |
| 218 | RTC_DCHECK_GE(sample_rate, AudioProcessing::NativeRate::kSampleRate8kHz); |
| 219 | |
| 220 | // 100 = 1 second / data duration (10 ms). |
| 221 | RTC_DCHECK_EQ(number_of_frames * 100, sample_rate); |
| 222 | |
| 223 | // 8 = bits per byte. |
| 224 | RTC_DCHECK_LE(bits_per_sample / 8 * number_of_frames * number_of_channels, |
| 225 | AudioFrame::kMaxDataSizeBytes); |
| 226 | mixer_->Mix(number_of_channels, &mixed_frame_); |
| 227 | *elapsed_time_ms = mixed_frame_.elapsed_time_ms_; |
| 228 | *ntp_time_ms = mixed_frame_.ntp_time_ms_; |
| 229 | |
| 230 | auto output_samples = Resample(mixed_frame_, sample_rate, &render_resampler_, |
| 231 | static_cast<int16_t*>(audio_data)); |
| 232 | RTC_DCHECK_EQ(output_samples, number_of_channels * number_of_frames); |
| 233 | } |
| 234 | |
| 235 | void AudioTransportImpl::UpdateSendingStreams( |
Yves Gerey | 665174f | 2018-06-19 15:03:05 +0200 | [diff] [blame] | 236 | std::vector<AudioSendStream*> streams, |
| 237 | int send_sample_rate_hz, |
Fredrik Solenberg | 2a87797 | 2017-12-15 16:42:15 +0100 | [diff] [blame] | 238 | size_t send_num_channels) { |
| 239 | rtc::CritScope lock(&capture_lock_); |
| 240 | sending_streams_ = std::move(streams); |
| 241 | send_sample_rate_hz_ = send_sample_rate_hz; |
| 242 | send_num_channels_ = send_num_channels; |
| 243 | } |
| 244 | |
| 245 | void AudioTransportImpl::SetStereoChannelSwapping(bool enable) { |
| 246 | rtc::CritScope lock(&capture_lock_); |
| 247 | swap_stereo_channels_ = enable; |
| 248 | } |
| 249 | |
| 250 | bool AudioTransportImpl::typing_noise_detected() const { |
| 251 | rtc::CritScope lock(&capture_lock_); |
| 252 | return typing_noise_detected_; |
| 253 | } |
| 254 | } // namespace webrtc |