peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 1 | /* |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 2 | * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Jonas Olsson | a4d8737 | 2019-07-05 19:08:33 +0200 | [diff] [blame] | 11 | #include "modules/audio_processing/aec3/render_delay_buffer.h" |
| 12 | |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 13 | #include <string.h> |
Jonas Olsson | a4d8737 | 2019-07-05 19:08:33 +0200 | [diff] [blame] | 14 | |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 15 | #include <algorithm> |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 16 | #include <memory> |
Per Åhgren | b6b00dc | 2018-02-20 22:18:27 +0100 | [diff] [blame] | 17 | #include <numeric> |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 18 | #include <vector> |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 19 | |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 20 | #include "absl/types/optional.h" |
| 21 | #include "api/array_view.h" |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 22 | #include "api/audio/echo_canceller3_config.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 23 | #include "modules/audio_processing/aec3/aec3_common.h" |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 24 | #include "modules/audio_processing/aec3/aec3_fft.h" |
Per Åhgren | 38e2d95 | 2017-11-17 14:54:28 +0100 | [diff] [blame] | 25 | #include "modules/audio_processing/aec3/decimator.h" |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 26 | #include "modules/audio_processing/aec3/downsampled_render_buffer.h" |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 27 | #include "modules/audio_processing/aec3/fft_buffer.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 28 | #include "modules/audio_processing/aec3/fft_data.h" |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 29 | #include "modules/audio_processing/aec3/matrix_buffer.h" |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 30 | #include "modules/audio_processing/aec3/render_buffer.h" |
Yves Gerey | 988cc08 | 2018-10-23 12:03:01 +0200 | [diff] [blame] | 31 | #include "modules/audio_processing/aec3/vector_buffer.h" |
| 32 | #include "modules/audio_processing/logging/apm_data_dumper.h" |
Steve Anton | 10542f2 | 2019-01-11 09:11:00 -0800 | [diff] [blame] | 33 | #include "rtc_base/atomic_ops.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 34 | #include "rtc_base/checks.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 35 | #include "rtc_base/logging.h" |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 36 | |
| 37 | namespace webrtc { |
| 38 | namespace { |
| 39 | |
peah | cf02cf1 | 2017-04-05 14:18:07 -0700 | [diff] [blame] | 40 | class RenderDelayBufferImpl final : public RenderDelayBuffer { |
| 41 | public: |
Gustaf Ullberg | cd277b8 | 2019-08-19 12:15:39 +0200 | [diff] [blame^] | 42 | RenderDelayBufferImpl(const EchoCanceller3Config& config, int sample_rate_hz); |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 43 | RenderDelayBufferImpl() = delete; |
peah | cf02cf1 | 2017-04-05 14:18:07 -0700 | [diff] [blame] | 44 | ~RenderDelayBufferImpl() override; |
| 45 | |
| 46 | void Reset() override; |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 47 | BufferingEvent Insert(const std::vector<std::vector<float>>& block) override; |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 48 | BufferingEvent PrepareCaptureProcessing() override; |
Gustaf Ullberg | 8f32b6c | 2019-04-05 16:23:50 +0200 | [diff] [blame] | 49 | bool AlignFromDelay(size_t delay) override; |
| 50 | void AlignFromExternalDelay() override; |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 51 | size_t Delay() const override { return ComputeDelay(); } |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 52 | size_t MaxDelay() const override { |
Per Åhgren | 09a718a | 2017-12-11 22:28:45 +0100 | [diff] [blame] | 53 | return blocks_.buffer.size() - 1 - buffer_headroom_; |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 54 | } |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 55 | RenderBuffer* GetRenderBuffer() override { return &echo_remover_buffer_; } |
peah | cf02cf1 | 2017-04-05 14:18:07 -0700 | [diff] [blame] | 56 | |
| 57 | const DownsampledRenderBuffer& GetDownsampledRenderBuffer() const override { |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 58 | return low_rate_; |
peah | cf02cf1 | 2017-04-05 14:18:07 -0700 | [diff] [blame] | 59 | } |
| 60 | |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 61 | int BufferLatency() const; |
Per Åhgren | d0fa820 | 2018-04-18 09:35:13 +0200 | [diff] [blame] | 62 | void SetAudioBufferDelay(size_t delay_ms) override; |
Gustaf Ullberg | 8f32b6c | 2019-04-05 16:23:50 +0200 | [diff] [blame] | 63 | bool HasReceivedBufferDelay() override; |
Per Åhgren | d0fa820 | 2018-04-18 09:35:13 +0200 | [diff] [blame] | 64 | |
peah | cf02cf1 | 2017-04-05 14:18:07 -0700 | [diff] [blame] | 65 | private: |
Per Åhgren | 38e2d95 | 2017-11-17 14:54:28 +0100 | [diff] [blame] | 66 | static int instance_count_; |
| 67 | std::unique_ptr<ApmDataDumper> data_dumper_; |
peah | cf02cf1 | 2017-04-05 14:18:07 -0700 | [diff] [blame] | 68 | const Aec3Optimization optimization_; |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 69 | const EchoCanceller3Config config_; |
Gustaf Ullberg | 78b1c4a | 2018-05-25 10:12:58 +0200 | [diff] [blame] | 70 | size_t down_sampling_factor_; |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 71 | const int sub_block_size_; |
| 72 | MatrixBuffer blocks_; |
| 73 | VectorBuffer spectra_; |
| 74 | FftBuffer ffts_; |
Danil Chapovalov | db9f7ab | 2018-06-19 10:50:11 +0200 | [diff] [blame] | 75 | absl::optional<size_t> delay_; |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 76 | RenderBuffer echo_remover_buffer_; |
| 77 | DownsampledRenderBuffer low_rate_; |
Per Åhgren | 38e2d95 | 2017-11-17 14:54:28 +0100 | [diff] [blame] | 78 | Decimator render_decimator_; |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 79 | const Aec3Fft fft_; |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 80 | std::vector<float> render_ds_; |
Per Åhgren | 09a718a | 2017-12-11 22:28:45 +0100 | [diff] [blame] | 81 | const int buffer_headroom_; |
Per Åhgren | b4c188d | 2017-12-20 00:25:18 +0100 | [diff] [blame] | 82 | bool last_call_was_render_ = false; |
| 83 | int num_api_calls_in_a_row_ = 0; |
| 84 | int max_observed_jitter_ = 1; |
Gustaf Ullberg | 8f32b6c | 2019-04-05 16:23:50 +0200 | [diff] [blame] | 85 | int64_t capture_call_counter_ = 0; |
| 86 | int64_t render_call_counter_ = 0; |
Per Åhgren | b6b00dc | 2018-02-20 22:18:27 +0100 | [diff] [blame] | 87 | bool render_activity_ = false; |
| 88 | size_t render_activity_counter_ = 0; |
Danil Chapovalov | db9f7ab | 2018-06-19 10:50:11 +0200 | [diff] [blame] | 89 | absl::optional<size_t> external_audio_buffer_delay_; |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 90 | bool external_audio_buffer_delay_verified_after_reset_ = false; |
| 91 | size_t min_latency_blocks_ = 0; |
| 92 | size_t excess_render_detection_counter_ = 0; |
Gustaf Ullberg | cd277b8 | 2019-08-19 12:15:39 +0200 | [diff] [blame^] | 93 | int sample_rate_hz_; |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 94 | |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 95 | int MapDelayToTotalDelay(size_t delay) const; |
| 96 | int ComputeDelay() const; |
| 97 | void ApplyTotalDelay(int delay); |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 98 | void InsertBlock(const std::vector<std::vector<float>>& block, |
| 99 | int previous_write); |
Per Åhgren | b6b00dc | 2018-02-20 22:18:27 +0100 | [diff] [blame] | 100 | bool DetectActiveRender(rtc::ArrayView<const float> x) const; |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 101 | bool DetectExcessRenderBlocks(); |
| 102 | void IncrementWriteIndices(); |
| 103 | void IncrementLowRateReadIndices(); |
| 104 | void IncrementReadIndices(); |
| 105 | bool RenderOverrun(); |
| 106 | bool RenderUnderrun(); |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 107 | }; |
| 108 | |
Per Åhgren | 38e2d95 | 2017-11-17 14:54:28 +0100 | [diff] [blame] | 109 | int RenderDelayBufferImpl::instance_count_ = 0; |
| 110 | |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 111 | RenderDelayBufferImpl::RenderDelayBufferImpl(const EchoCanceller3Config& config, |
Gustaf Ullberg | cd277b8 | 2019-08-19 12:15:39 +0200 | [diff] [blame^] | 112 | int sample_rate_hz) |
Per Åhgren | 38e2d95 | 2017-11-17 14:54:28 +0100 | [diff] [blame] | 113 | : data_dumper_( |
| 114 | new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), |
| 115 | optimization_(DetectOptimization()), |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 116 | config_(config), |
Gustaf Ullberg | 53e2211 | 2018-10-11 15:27:26 +0200 | [diff] [blame] | 117 | down_sampling_factor_(config.delay.down_sampling_factor), |
Gustaf Ullberg | 78b1c4a | 2018-05-25 10:12:58 +0200 | [diff] [blame] | 118 | sub_block_size_(static_cast<int>(down_sampling_factor_ > 0 |
| 119 | ? kBlockSize / down_sampling_factor_ |
| 120 | : kBlockSize)), |
| 121 | blocks_(GetRenderDelayBufferSize(down_sampling_factor_, |
Per Åhgren | 09a718a | 2017-12-11 22:28:45 +0100 | [diff] [blame] | 122 | config.delay.num_filters, |
Per Åhgren | 08ea589 | 2018-01-15 08:07:41 +0100 | [diff] [blame] | 123 | config.filter.main.length_blocks), |
Gustaf Ullberg | cd277b8 | 2019-08-19 12:15:39 +0200 | [diff] [blame^] | 124 | NumBandsForRate(sample_rate_hz), |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 125 | kBlockSize), |
| 126 | spectra_(blocks_.buffer.size(), kFftLengthBy2Plus1), |
| 127 | ffts_(blocks_.buffer.size()), |
Per Åhgren | 5c532d3 | 2018-03-22 00:29:25 +0100 | [diff] [blame] | 128 | delay_(config_.delay.default_delay), |
Per Åhgren | ec22e3f | 2017-12-20 15:20:37 +0100 | [diff] [blame] | 129 | echo_remover_buffer_(&blocks_, &spectra_, &ffts_), |
Gustaf Ullberg | 78b1c4a | 2018-05-25 10:12:58 +0200 | [diff] [blame] | 130 | low_rate_(GetDownSampledBufferSize(down_sampling_factor_, |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 131 | config.delay.num_filters)), |
Gustaf Ullberg | 78b1c4a | 2018-05-25 10:12:58 +0200 | [diff] [blame] | 132 | render_decimator_(down_sampling_factor_), |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 133 | fft_(), |
Per Åhgren | 09a718a | 2017-12-11 22:28:45 +0100 | [diff] [blame] | 134 | render_ds_(sub_block_size_, 0.f), |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 135 | buffer_headroom_(config.filter.main.length_blocks), |
Gustaf Ullberg | cd277b8 | 2019-08-19 12:15:39 +0200 | [diff] [blame^] | 136 | sample_rate_hz_(sample_rate_hz) { |
| 137 | RTC_DCHECK_GE(sample_rate_hz, 8000); |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 138 | RTC_DCHECK_EQ(blocks_.buffer.size(), ffts_.buffer.size()); |
| 139 | RTC_DCHECK_EQ(spectra_.buffer.size(), ffts_.buffer.size()); |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 140 | |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 141 | Reset(); |
peah | cf02cf1 | 2017-04-05 14:18:07 -0700 | [diff] [blame] | 142 | } |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 143 | |
| 144 | RenderDelayBufferImpl::~RenderDelayBufferImpl() = default; |
| 145 | |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 146 | // Resets the buffer delays and clears the reported delays. |
peah | cf02cf1 | 2017-04-05 14:18:07 -0700 | [diff] [blame] | 147 | void RenderDelayBufferImpl::Reset() { |
Per Åhgren | b4c188d | 2017-12-20 00:25:18 +0100 | [diff] [blame] | 148 | last_call_was_render_ = false; |
| 149 | num_api_calls_in_a_row_ = 1; |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 150 | min_latency_blocks_ = 0; |
| 151 | excess_render_detection_counter_ = 0; |
Per Åhgren | b4c188d | 2017-12-20 00:25:18 +0100 | [diff] [blame] | 152 | |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 153 | // Initialize the read index to one sub-block before the write index. |
| 154 | low_rate_.read = low_rate_.OffsetIndex(low_rate_.write, sub_block_size_); |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 155 | |
Per Åhgren | d0fa820 | 2018-04-18 09:35:13 +0200 | [diff] [blame] | 156 | // Check for any external audio buffer delay and whether it is feasible. |
Gustaf Ullberg | a49eacb | 2018-05-04 16:43:56 +0200 | [diff] [blame] | 157 | if (external_audio_buffer_delay_) { |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 158 | const size_t headroom = 2; |
| 159 | size_t audio_buffer_delay_to_set; |
| 160 | // Minimum delay is 1 (like the low-rate render buffer). |
| 161 | if (*external_audio_buffer_delay_ <= headroom) { |
| 162 | audio_buffer_delay_to_set = 1; |
Per Åhgren | d0fa820 | 2018-04-18 09:35:13 +0200 | [diff] [blame] | 163 | } else { |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 164 | audio_buffer_delay_to_set = *external_audio_buffer_delay_ - headroom; |
Per Åhgren | d0fa820 | 2018-04-18 09:35:13 +0200 | [diff] [blame] | 165 | } |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 166 | |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 167 | audio_buffer_delay_to_set = std::min(audio_buffer_delay_to_set, MaxDelay()); |
Per Åhgren | d0fa820 | 2018-04-18 09:35:13 +0200 | [diff] [blame] | 168 | |
| 169 | // When an external delay estimate is available, use that delay as the |
Gustaf Ullberg | 623d281 | 2018-05-07 13:36:07 +0200 | [diff] [blame] | 170 | // initial render buffer delay. |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 171 | ApplyTotalDelay(audio_buffer_delay_to_set); |
| 172 | delay_ = ComputeDelay(); |
Gustaf Ullberg | 623d281 | 2018-05-07 13:36:07 +0200 | [diff] [blame] | 173 | |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 174 | external_audio_buffer_delay_verified_after_reset_ = false; |
Per Åhgren | d0fa820 | 2018-04-18 09:35:13 +0200 | [diff] [blame] | 175 | } else { |
| 176 | // If an external delay estimate is not available, use that delay as the |
| 177 | // initial delay. Set the render buffer delays to the default delay. |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 178 | ApplyTotalDelay(config_.delay.default_delay); |
Per Åhgren | d0fa820 | 2018-04-18 09:35:13 +0200 | [diff] [blame] | 179 | |
Gustaf Ullberg | 8f32b6c | 2019-04-05 16:23:50 +0200 | [diff] [blame] | 180 | // Unset the delays which are set by AlignFromDelay. |
Danil Chapovalov | db9f7ab | 2018-06-19 10:50:11 +0200 | [diff] [blame] | 181 | delay_ = absl::nullopt; |
Per Åhgren | d0fa820 | 2018-04-18 09:35:13 +0200 | [diff] [blame] | 182 | } |
peah | cf02cf1 | 2017-04-05 14:18:07 -0700 | [diff] [blame] | 183 | } |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 184 | |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 185 | // Inserts a new block into the render buffers. |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 186 | RenderDelayBuffer::BufferingEvent RenderDelayBufferImpl::Insert( |
peah | cf02cf1 | 2017-04-05 14:18:07 -0700 | [diff] [blame] | 187 | const std::vector<std::vector<float>>& block) { |
Per Åhgren | b4c188d | 2017-12-20 00:25:18 +0100 | [diff] [blame] | 188 | ++render_call_counter_; |
| 189 | if (delay_) { |
| 190 | if (!last_call_was_render_) { |
| 191 | last_call_was_render_ = true; |
| 192 | num_api_calls_in_a_row_ = 1; |
| 193 | } else { |
| 194 | if (++num_api_calls_in_a_row_ > max_observed_jitter_) { |
| 195 | max_observed_jitter_ = num_api_calls_in_a_row_; |
Gustaf Ullberg | 940c2b5 | 2019-08-08 15:04:41 +0200 | [diff] [blame] | 196 | RTC_LOG(LS_INFO) |
Per Åhgren | b4c188d | 2017-12-20 00:25:18 +0100 | [diff] [blame] | 197 | << "New max number api jitter observed at render block " |
| 198 | << render_call_counter_ << ": " << num_api_calls_in_a_row_ |
| 199 | << " blocks"; |
| 200 | } |
| 201 | } |
| 202 | } |
| 203 | |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 204 | // Increase the write indices to where the new blocks should be written. |
| 205 | const int previous_write = blocks_.write; |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 206 | IncrementWriteIndices(); |
peah | cf02cf1 | 2017-04-05 14:18:07 -0700 | [diff] [blame] | 207 | |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 208 | // Allow overrun and do a reset when render overrun occurrs due to more render |
| 209 | // data being inserted than capture data is received. |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 210 | BufferingEvent event = |
| 211 | RenderOverrun() ? BufferingEvent::kRenderOverrun : BufferingEvent::kNone; |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 212 | |
Per Åhgren | b6b00dc | 2018-02-20 22:18:27 +0100 | [diff] [blame] | 213 | // Detect and update render activity. |
| 214 | if (!render_activity_) { |
| 215 | render_activity_counter_ += DetectActiveRender(block[0]) ? 1 : 0; |
| 216 | render_activity_ = render_activity_counter_ >= 20; |
| 217 | } |
| 218 | |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 219 | // Insert the new render block into the specified position. |
| 220 | InsertBlock(block, previous_write); |
| 221 | |
| 222 | if (event != BufferingEvent::kNone) { |
| 223 | Reset(); |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 224 | } |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 225 | |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 226 | return event; |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 227 | } |
| 228 | |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 229 | // Prepares the render buffers for processing another capture block. |
| 230 | RenderDelayBuffer::BufferingEvent |
| 231 | RenderDelayBufferImpl::PrepareCaptureProcessing() { |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 232 | RenderDelayBuffer::BufferingEvent event = BufferingEvent::kNone; |
Per Åhgren | b4c188d | 2017-12-20 00:25:18 +0100 | [diff] [blame] | 233 | ++capture_call_counter_; |
| 234 | |
| 235 | if (delay_) { |
| 236 | if (last_call_was_render_) { |
| 237 | last_call_was_render_ = false; |
| 238 | num_api_calls_in_a_row_ = 1; |
| 239 | } else { |
| 240 | if (++num_api_calls_in_a_row_ > max_observed_jitter_) { |
| 241 | max_observed_jitter_ = num_api_calls_in_a_row_; |
Gustaf Ullberg | 940c2b5 | 2019-08-08 15:04:41 +0200 | [diff] [blame] | 242 | RTC_LOG(LS_INFO) |
Per Åhgren | b4c188d | 2017-12-20 00:25:18 +0100 | [diff] [blame] | 243 | << "New max number api jitter observed at capture block " |
| 244 | << capture_call_counter_ << ": " << num_api_calls_in_a_row_ |
| 245 | << " blocks"; |
| 246 | } |
| 247 | } |
| 248 | } |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 249 | |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 250 | if (DetectExcessRenderBlocks()) { |
| 251 | // Too many render blocks compared to capture blocks. Risk of delay ending |
| 252 | // up before the filter used by the delay estimator. |
Gustaf Ullberg | 940c2b5 | 2019-08-08 15:04:41 +0200 | [diff] [blame] | 253 | RTC_LOG(LS_INFO) << "Excess render blocks detected at block " |
| 254 | << capture_call_counter_; |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 255 | Reset(); |
| 256 | event = BufferingEvent::kRenderOverrun; |
| 257 | } else if (RenderUnderrun()) { |
| 258 | // Don't increment the read indices of the low rate buffer if there is a |
| 259 | // render underrun. |
Gustaf Ullberg | 940c2b5 | 2019-08-08 15:04:41 +0200 | [diff] [blame] | 260 | RTC_LOG(LS_INFO) << "Render buffer underrun detected at block " |
| 261 | << capture_call_counter_; |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 262 | IncrementReadIndices(); |
| 263 | // Incrementing the buffer index without increasing the low rate buffer |
| 264 | // index means that the delay is reduced by one. |
| 265 | if (delay_ && *delay_ > 0) |
| 266 | delay_ = *delay_ - 1; |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 267 | event = BufferingEvent::kRenderUnderrun; |
peah | debaa44 | 2017-05-03 05:39:09 -0700 | [diff] [blame] | 268 | } else { |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 269 | // Increment the read indices in the render buffers to point to the most |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 270 | // recent block to use in the capture processing. |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 271 | IncrementLowRateReadIndices(); |
| 272 | IncrementReadIndices(); |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 273 | } |
| 274 | |
Per Åhgren | b6b00dc | 2018-02-20 22:18:27 +0100 | [diff] [blame] | 275 | echo_remover_buffer_.SetRenderActivity(render_activity_); |
| 276 | if (render_activity_) { |
| 277 | render_activity_counter_ = 0; |
| 278 | render_activity_ = false; |
| 279 | } |
| 280 | |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 281 | return event; |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 282 | } |
| 283 | |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 284 | // Sets the delay and returns a bool indicating whether the delay was changed. |
Gustaf Ullberg | 8f32b6c | 2019-04-05 16:23:50 +0200 | [diff] [blame] | 285 | bool RenderDelayBufferImpl::AlignFromDelay(size_t delay) { |
| 286 | RTC_DCHECK(!config_.delay.use_external_delay_estimator); |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 287 | if (!external_audio_buffer_delay_verified_after_reset_ && |
| 288 | external_audio_buffer_delay_ && delay_) { |
Gustaf Ullberg | 623d281 | 2018-05-07 13:36:07 +0200 | [diff] [blame] | 289 | int difference = static_cast<int>(delay) - static_cast<int>(*delay_); |
Gustaf Ullberg | 940c2b5 | 2019-08-08 15:04:41 +0200 | [diff] [blame] | 290 | RTC_LOG(LS_INFO) << "Mismatch between first estimated delay after reset " |
| 291 | "and externally reported audio buffer delay: " |
| 292 | << difference << " blocks"; |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 293 | external_audio_buffer_delay_verified_after_reset_ = true; |
Per Åhgren | d0fa820 | 2018-04-18 09:35:13 +0200 | [diff] [blame] | 294 | } |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 295 | if (delay_ && *delay_ == delay) { |
| 296 | return false; |
peah | cf02cf1 | 2017-04-05 14:18:07 -0700 | [diff] [blame] | 297 | } |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 298 | delay_ = delay; |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 299 | |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 300 | // Compute the total delay and limit the delay to the allowed range. |
| 301 | int total_delay = MapDelayToTotalDelay(*delay_); |
| 302 | total_delay = |
| 303 | std::min(MaxDelay(), static_cast<size_t>(std::max(total_delay, 0))); |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 304 | |
| 305 | // Apply the delay to the buffers. |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 306 | ApplyTotalDelay(total_delay); |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 307 | return true; |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 308 | } |
| 309 | |
Per Åhgren | d0fa820 | 2018-04-18 09:35:13 +0200 | [diff] [blame] | 310 | void RenderDelayBufferImpl::SetAudioBufferDelay(size_t delay_ms) { |
Gustaf Ullberg | a49eacb | 2018-05-04 16:43:56 +0200 | [diff] [blame] | 311 | if (!external_audio_buffer_delay_) { |
Gustaf Ullberg | 940c2b5 | 2019-08-08 15:04:41 +0200 | [diff] [blame] | 312 | RTC_LOG(LS_INFO) |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 313 | << "Receiving a first externally reported audio buffer delay of " |
| 314 | << delay_ms << " ms."; |
Per Åhgren | d0fa820 | 2018-04-18 09:35:13 +0200 | [diff] [blame] | 315 | } |
Gustaf Ullberg | 623d281 | 2018-05-07 13:36:07 +0200 | [diff] [blame] | 316 | |
| 317 | // Convert delay from milliseconds to blocks (rounded down). |
Gustaf Ullberg | cd277b8 | 2019-08-19 12:15:39 +0200 | [diff] [blame^] | 318 | external_audio_buffer_delay_ = |
| 319 | delay_ms >> ((sample_rate_hz_ == 8000) ? 1 : 2); |
Per Åhgren | d0fa820 | 2018-04-18 09:35:13 +0200 | [diff] [blame] | 320 | } |
| 321 | |
Gustaf Ullberg | 8f32b6c | 2019-04-05 16:23:50 +0200 | [diff] [blame] | 322 | bool RenderDelayBufferImpl::HasReceivedBufferDelay() { |
| 323 | return external_audio_buffer_delay_.has_value(); |
| 324 | } |
| 325 | |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 326 | // Maps the externally computed delay to the delay used internally. |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 327 | int RenderDelayBufferImpl::MapDelayToTotalDelay( |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 328 | size_t external_delay_blocks) const { |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 329 | const int latency_blocks = BufferLatency(); |
| 330 | return latency_blocks + static_cast<int>(external_delay_blocks); |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 331 | } |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 332 | |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 333 | // Returns the delay (not including call jitter). |
| 334 | int RenderDelayBufferImpl::ComputeDelay() const { |
| 335 | const int latency_blocks = BufferLatency(); |
Per Åhgren | 5c532d3 | 2018-03-22 00:29:25 +0100 | [diff] [blame] | 336 | int internal_delay = spectra_.read >= spectra_.write |
| 337 | ? spectra_.read - spectra_.write |
| 338 | : spectra_.size + spectra_.read - spectra_.write; |
| 339 | |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 340 | return internal_delay - latency_blocks; |
Per Åhgren | 5c532d3 | 2018-03-22 00:29:25 +0100 | [diff] [blame] | 341 | } |
| 342 | |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 343 | // Set the read indices according to the delay. |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 344 | void RenderDelayBufferImpl::ApplyTotalDelay(int delay) { |
Gustaf Ullberg | 940c2b5 | 2019-08-08 15:04:41 +0200 | [diff] [blame] | 345 | RTC_LOG(LS_INFO) << "Applying total delay of " << delay << " blocks."; |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 346 | blocks_.read = blocks_.OffsetIndex(blocks_.write, -delay); |
| 347 | spectra_.read = spectra_.OffsetIndex(spectra_.write, delay); |
| 348 | ffts_.read = ffts_.OffsetIndex(ffts_.write, delay); |
| 349 | } |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 350 | |
Gustaf Ullberg | 8f32b6c | 2019-04-05 16:23:50 +0200 | [diff] [blame] | 351 | void RenderDelayBufferImpl::AlignFromExternalDelay() { |
| 352 | RTC_DCHECK(config_.delay.use_external_delay_estimator); |
| 353 | if (external_audio_buffer_delay_) { |
| 354 | int64_t delay = render_call_counter_ - capture_call_counter_ + |
| 355 | *external_audio_buffer_delay_; |
| 356 | ApplyTotalDelay(delay); |
| 357 | } |
| 358 | } |
| 359 | |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 360 | // Inserts a block into the render buffers. |
| 361 | void RenderDelayBufferImpl::InsertBlock( |
| 362 | const std::vector<std::vector<float>>& block, |
| 363 | int previous_write) { |
| 364 | auto& b = blocks_; |
| 365 | auto& lr = low_rate_; |
| 366 | auto& ds = render_ds_; |
| 367 | auto& f = ffts_; |
| 368 | auto& s = spectra_; |
| 369 | RTC_DCHECK_EQ(block.size(), b.buffer[b.write].size()); |
| 370 | for (size_t k = 0; k < block.size(); ++k) { |
| 371 | RTC_DCHECK_EQ(block[k].size(), b.buffer[b.write][k].size()); |
| 372 | std::copy(block[k].begin(), block[k].end(), b.buffer[b.write][k].begin()); |
| 373 | } |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 374 | |
Gustaf Ullberg | 43c707a | 2018-05-22 15:27:23 +0200 | [diff] [blame] | 375 | data_dumper_->DumpWav("aec3_render_decimator_input", block[0].size(), |
| 376 | block[0].data(), 16000, 1); |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 377 | render_decimator_.Decimate(block[0], ds); |
Gustaf Ullberg | 43c707a | 2018-05-22 15:27:23 +0200 | [diff] [blame] | 378 | data_dumper_->DumpWav("aec3_render_decimator_output", ds.size(), ds.data(), |
Gustaf Ullberg | 78b1c4a | 2018-05-25 10:12:58 +0200 | [diff] [blame] | 379 | 16000 / down_sampling_factor_, 1); |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 380 | std::copy(ds.rbegin(), ds.rend(), lr.buffer.begin() + lr.write); |
| 381 | fft_.PaddedFft(block[0], b.buffer[previous_write][0], &f.buffer[f.write]); |
| 382 | f.buffer[f.write].Spectrum(optimization_, s.buffer[s.write]); |
| 383 | } |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 384 | |
Per Åhgren | b6b00dc | 2018-02-20 22:18:27 +0100 | [diff] [blame] | 385 | bool RenderDelayBufferImpl::DetectActiveRender( |
| 386 | rtc::ArrayView<const float> x) const { |
| 387 | const float x_energy = std::inner_product(x.begin(), x.end(), x.begin(), 0.f); |
| 388 | return x_energy > (config_.render_levels.active_render_limit * |
| 389 | config_.render_levels.active_render_limit) * |
| 390 | kFftLengthBy2; |
| 391 | } |
| 392 | |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 393 | bool RenderDelayBufferImpl::DetectExcessRenderBlocks() { |
| 394 | bool excess_render_detected = false; |
| 395 | const size_t latency_blocks = static_cast<size_t>(BufferLatency()); |
| 396 | // The recently seen minimum latency in blocks. Should be close to 0. |
| 397 | min_latency_blocks_ = std::min(min_latency_blocks_, latency_blocks); |
| 398 | // After processing a configurable number of blocks the minimum latency is |
| 399 | // checked. |
| 400 | if (++excess_render_detection_counter_ >= |
| 401 | config_.buffering.excess_render_detection_interval_blocks) { |
| 402 | // If the minimum latency is not lower than the threshold there have been |
| 403 | // more render than capture frames. |
| 404 | excess_render_detected = min_latency_blocks_ > |
| 405 | config_.buffering.max_allowed_excess_render_blocks; |
| 406 | // Reset the counter and let the minimum latency be the current latency. |
| 407 | min_latency_blocks_ = latency_blocks; |
| 408 | excess_render_detection_counter_ = 0; |
| 409 | } |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 410 | |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 411 | data_dumper_->DumpRaw("aec3_latency_blocks", latency_blocks); |
| 412 | data_dumper_->DumpRaw("aec3_min_latency_blocks", min_latency_blocks_); |
| 413 | data_dumper_->DumpRaw("aec3_excess_render_detected", excess_render_detected); |
| 414 | return excess_render_detected; |
Per Åhgren | c59a576 | 2017-12-11 21:34:19 +0100 | [diff] [blame] | 415 | } |
| 416 | |
Gustaf Ullberg | e47433f | 2019-01-24 16:00:57 +0100 | [diff] [blame] | 417 | // Computes the latency in the buffer (the number of unread sub-blocks). |
| 418 | int RenderDelayBufferImpl::BufferLatency() const { |
| 419 | const DownsampledRenderBuffer& l = low_rate_; |
| 420 | int latency_samples = (l.buffer.size() + l.read - l.write) % l.buffer.size(); |
| 421 | int latency_blocks = latency_samples / sub_block_size_; |
| 422 | return latency_blocks; |
| 423 | } |
| 424 | |
| 425 | // Increments the write indices for the render buffers. |
| 426 | void RenderDelayBufferImpl::IncrementWriteIndices() { |
| 427 | low_rate_.UpdateWriteIndex(-sub_block_size_); |
| 428 | blocks_.IncWriteIndex(); |
| 429 | spectra_.DecWriteIndex(); |
| 430 | ffts_.DecWriteIndex(); |
| 431 | } |
| 432 | |
| 433 | // Increments the read indices of the low rate render buffers. |
| 434 | void RenderDelayBufferImpl::IncrementLowRateReadIndices() { |
| 435 | low_rate_.UpdateReadIndex(-sub_block_size_); |
| 436 | } |
| 437 | |
| 438 | // Increments the read indices for the render buffers. |
| 439 | void RenderDelayBufferImpl::IncrementReadIndices() { |
| 440 | if (blocks_.read != blocks_.write) { |
| 441 | blocks_.IncReadIndex(); |
| 442 | spectra_.DecReadIndex(); |
| 443 | ffts_.DecReadIndex(); |
| 444 | } |
| 445 | } |
| 446 | |
| 447 | // Checks for a render buffer overrun. |
| 448 | bool RenderDelayBufferImpl::RenderOverrun() { |
| 449 | return low_rate_.read == low_rate_.write || blocks_.read == blocks_.write; |
| 450 | } |
| 451 | |
| 452 | // Checks for a render buffer underrun. |
| 453 | bool RenderDelayBufferImpl::RenderUnderrun() { |
| 454 | return low_rate_.read == low_rate_.write; |
| 455 | } |
| 456 | |
| 457 | } // namespace |
| 458 | |
Per Åhgren | 8ba5861 | 2017-12-01 23:01:44 +0100 | [diff] [blame] | 459 | RenderDelayBuffer* RenderDelayBuffer::Create(const EchoCanceller3Config& config, |
Gustaf Ullberg | cd277b8 | 2019-08-19 12:15:39 +0200 | [diff] [blame^] | 460 | int sample_rate_hz) { |
| 461 | return new RenderDelayBufferImpl(config, sample_rate_hz); |
peah | 69221db | 2017-01-27 03:28:19 -0800 | [diff] [blame] | 462 | } |
| 463 | |
| 464 | } // namespace webrtc |