henrik.lundin@webrtc.org | 9a40081 | 2013-01-29 12:09:21 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
| 11 | #include "webrtc/modules/audio_coding/neteq4/merge.h" |
| 12 | |
| 13 | #include <assert.h> |
pbos@webrtc.org | 3f45c2e | 2013-08-05 16:22:53 +0000 | [diff] [blame] | 14 | #include <string.h> // memmove, memcpy, memset, size_t |
henrik.lundin@webrtc.org | 9a40081 | 2013-01-29 12:09:21 +0000 | [diff] [blame] | 15 | |
| 16 | #include <algorithm> // min, max |
henrik.lundin@webrtc.org | 9a40081 | 2013-01-29 12:09:21 +0000 | [diff] [blame] | 17 | |
| 18 | #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h" |
| 19 | #include "webrtc/modules/audio_coding/neteq4/audio_multi_vector.h" |
| 20 | #include "webrtc/modules/audio_coding/neteq4/dsp_helper.h" |
| 21 | #include "webrtc/modules/audio_coding/neteq4/expand.h" |
| 22 | #include "webrtc/modules/audio_coding/neteq4/sync_buffer.h" |
| 23 | |
| 24 | namespace webrtc { |
| 25 | |
turaj@webrtc.org | 045e45e | 2013-09-20 16:25:28 +0000 | [diff] [blame] | 26 | int Merge::Process(int16_t* input, size_t input_length, |
henrik.lundin@webrtc.org | 9a40081 | 2013-01-29 12:09:21 +0000 | [diff] [blame] | 27 | int16_t* external_mute_factor_array, |
henrik.lundin@webrtc.org | 0e9c399 | 2013-09-30 20:38:44 +0000 | [diff] [blame^] | 28 | AudioMultiVector* output) { |
henrik.lundin@webrtc.org | 9a40081 | 2013-01-29 12:09:21 +0000 | [diff] [blame] | 29 | // TODO(hlundin): Change to an enumerator and skip assert. |
| 30 | assert(fs_hz_ == 8000 || fs_hz_ == 16000 || fs_hz_ == 32000 || |
| 31 | fs_hz_ == 48000); |
| 32 | assert(fs_hz_ <= kMaxSampleRate); // Should not be possible. |
| 33 | |
| 34 | int old_length; |
| 35 | int expand_period; |
| 36 | // Get expansion data to overlap and mix with. |
| 37 | int expanded_length = GetExpandedSignal(&old_length, &expand_period); |
| 38 | |
| 39 | // Transfer input signal to an AudioMultiVector. |
henrik.lundin@webrtc.org | 0e9c399 | 2013-09-30 20:38:44 +0000 | [diff] [blame^] | 40 | AudioMultiVector input_vector(num_channels_); |
henrik.lundin@webrtc.org | 9a40081 | 2013-01-29 12:09:21 +0000 | [diff] [blame] | 41 | input_vector.PushBackInterleaved(input, input_length); |
| 42 | size_t input_length_per_channel = input_vector.Size(); |
| 43 | assert(input_length_per_channel == input_length / num_channels_); |
| 44 | |
| 45 | int16_t best_correlation_index = 0; |
| 46 | size_t output_length = 0; |
| 47 | |
| 48 | for (size_t channel = 0; channel < num_channels_; ++channel) { |
| 49 | int16_t* input_channel = &input_vector[channel][0]; |
| 50 | int16_t* expanded_channel = &expanded_[channel][0]; |
| 51 | int16_t expanded_max, input_max; |
turaj@webrtc.org | 045e45e | 2013-09-20 16:25:28 +0000 | [diff] [blame] | 52 | int16_t new_mute_factor = SignalScaling( |
| 53 | input_channel, static_cast<int>(input_length_per_channel), |
| 54 | expanded_channel, &expanded_max, &input_max); |
henrik.lundin@webrtc.org | 9a40081 | 2013-01-29 12:09:21 +0000 | [diff] [blame] | 55 | |
| 56 | // Adjust muting factor (product of "main" muting factor and expand muting |
| 57 | // factor). |
| 58 | int16_t* external_mute_factor = &external_mute_factor_array[channel]; |
| 59 | *external_mute_factor = |
| 60 | (*external_mute_factor * expand_->MuteFactor(channel)) >> 14; |
| 61 | |
| 62 | // Update |external_mute_factor| if it is lower than |new_mute_factor|. |
| 63 | if (new_mute_factor > *external_mute_factor) { |
| 64 | *external_mute_factor = std::min(new_mute_factor, |
| 65 | static_cast<int16_t>(16384)); |
| 66 | } |
| 67 | |
| 68 | if (channel == 0) { |
| 69 | // Downsample, correlate, and find strongest correlation period for the |
| 70 | // master (i.e., first) channel only. |
| 71 | // Downsample to 4kHz sample rate. |
turaj@webrtc.org | 045e45e | 2013-09-20 16:25:28 +0000 | [diff] [blame] | 72 | Downsample(input_channel, static_cast<int>(input_length_per_channel), |
| 73 | expanded_channel, expanded_length); |
henrik.lundin@webrtc.org | 9a40081 | 2013-01-29 12:09:21 +0000 | [diff] [blame] | 74 | |
| 75 | // Calculate the lag of the strongest correlation period. |
turaj@webrtc.org | 045e45e | 2013-09-20 16:25:28 +0000 | [diff] [blame] | 76 | best_correlation_index = CorrelateAndPeakSearch( |
| 77 | expanded_max, input_max, old_length, |
| 78 | static_cast<int>(input_length_per_channel), expand_period); |
henrik.lundin@webrtc.org | 9a40081 | 2013-01-29 12:09:21 +0000 | [diff] [blame] | 79 | } |
| 80 | |
| 81 | static const int kTempDataSize = 3600; |
| 82 | int16_t temp_data[kTempDataSize]; // TODO(hlundin) Remove this. |
| 83 | int16_t* decoded_output = temp_data + best_correlation_index; |
| 84 | |
| 85 | // Mute the new decoded data if needed (and unmute it linearly). |
| 86 | // This is the overlapping part of expanded_signal. |
| 87 | int interpolation_length = std::min( |
| 88 | kMaxCorrelationLength * fs_mult_, |
| 89 | expanded_length - best_correlation_index); |
| 90 | interpolation_length = std::min(interpolation_length, |
| 91 | static_cast<int>(input_length_per_channel)); |
| 92 | if (*external_mute_factor < 16384) { |
| 93 | // Set a suitable muting slope (Q20). 0.004 for NB, 0.002 for WB, |
| 94 | // and so on. |
| 95 | int increment = 4194 / fs_mult_; |
| 96 | *external_mute_factor = DspHelper::RampSignal(input_channel, |
| 97 | interpolation_length, |
| 98 | *external_mute_factor, |
| 99 | increment); |
| 100 | DspHelper::UnmuteSignal(&input_channel[interpolation_length], |
| 101 | input_length_per_channel - interpolation_length, |
| 102 | external_mute_factor, increment, |
| 103 | &decoded_output[interpolation_length]); |
| 104 | } else { |
| 105 | // No muting needed. |
| 106 | memmove( |
| 107 | &decoded_output[interpolation_length], |
| 108 | &input_channel[interpolation_length], |
| 109 | sizeof(int16_t) * (input_length_per_channel - interpolation_length)); |
| 110 | } |
| 111 | |
| 112 | // Do overlap and mix linearly. |
| 113 | int increment = 16384 / (interpolation_length + 1); // In Q14. |
| 114 | int16_t mute_factor = 16384 - increment; |
| 115 | memmove(temp_data, expanded_channel, |
| 116 | sizeof(int16_t) * best_correlation_index); |
| 117 | DspHelper::CrossFade(&expanded_channel[best_correlation_index], |
| 118 | input_channel, interpolation_length, |
| 119 | &mute_factor, increment, decoded_output); |
| 120 | |
| 121 | output_length = best_correlation_index + input_length_per_channel; |
| 122 | if (channel == 0) { |
| 123 | assert(output->Empty()); // Output should be empty at this point. |
| 124 | output->AssertSize(output_length); |
| 125 | } else { |
| 126 | assert(output->Size() == output_length); |
| 127 | } |
| 128 | memcpy(&(*output)[channel][0], temp_data, |
| 129 | sizeof(temp_data[0]) * output_length); |
| 130 | } |
| 131 | |
| 132 | // Copy back the first part of the data to |sync_buffer_| and remove it from |
| 133 | // |output|. |
| 134 | sync_buffer_->ReplaceAtIndex(*output, old_length, sync_buffer_->next_index()); |
| 135 | output->PopFront(old_length); |
| 136 | |
| 137 | // Return new added length. |old_length| samples were borrowed from |
| 138 | // |sync_buffer_|. |
turaj@webrtc.org | 045e45e | 2013-09-20 16:25:28 +0000 | [diff] [blame] | 139 | return static_cast<int>(output_length) - old_length; |
henrik.lundin@webrtc.org | 9a40081 | 2013-01-29 12:09:21 +0000 | [diff] [blame] | 140 | } |
| 141 | |
| 142 | int Merge::GetExpandedSignal(int* old_length, int* expand_period) { |
| 143 | // Check how much data that is left since earlier. |
turaj@webrtc.org | 045e45e | 2013-09-20 16:25:28 +0000 | [diff] [blame] | 144 | *old_length = static_cast<int>(sync_buffer_->FutureLength()); |
henrik.lundin@webrtc.org | 9a40081 | 2013-01-29 12:09:21 +0000 | [diff] [blame] | 145 | // Should never be less than overlap_length. |
| 146 | assert(*old_length >= static_cast<int>(expand_->overlap_length())); |
| 147 | // Generate data to merge the overlap with using expand. |
| 148 | expand_->SetParametersForMergeAfterExpand(); |
| 149 | |
| 150 | if (*old_length >= 210 * kMaxSampleRate / 8000) { |
| 151 | // TODO(hlundin): Write test case for this. |
| 152 | // The number of samples available in the sync buffer is more than what fits |
| 153 | // in expanded_signal. Keep the first 210 * kMaxSampleRate / 8000 samples, |
| 154 | // but shift them towards the end of the buffer. This is ok, since all of |
| 155 | // the buffer will be expand data anyway, so as long as the beginning is |
| 156 | // left untouched, we're fine. |
| 157 | int16_t length_diff = *old_length - 210 * kMaxSampleRate / 8000; |
| 158 | sync_buffer_->InsertZerosAtIndex(length_diff, sync_buffer_->next_index()); |
| 159 | *old_length = 210 * kMaxSampleRate / 8000; |
| 160 | // This is the truncated length. |
| 161 | } |
| 162 | // This assert should always be true thanks to the if statement above. |
| 163 | assert(210 * kMaxSampleRate / 8000 - *old_length >= 0); |
| 164 | |
henrik.lundin@webrtc.org | 0e9c399 | 2013-09-30 20:38:44 +0000 | [diff] [blame^] | 165 | AudioMultiVector expanded_temp(num_channels_); |
henrik.lundin@webrtc.org | 9a40081 | 2013-01-29 12:09:21 +0000 | [diff] [blame] | 166 | expand_->Process(&expanded_temp); |
turaj@webrtc.org | 045e45e | 2013-09-20 16:25:28 +0000 | [diff] [blame] | 167 | *expand_period = static_cast<int>(expanded_temp.Size()); // Samples per |
| 168 | // channel. |
henrik.lundin@webrtc.org | 9a40081 | 2013-01-29 12:09:21 +0000 | [diff] [blame] | 169 | |
| 170 | expanded_.Clear(); |
| 171 | // Copy what is left since earlier into the expanded vector. |
| 172 | expanded_.PushBackFromIndex(*sync_buffer_, sync_buffer_->next_index()); |
| 173 | assert(expanded_.Size() == static_cast<size_t>(*old_length)); |
| 174 | assert(expanded_temp.Size() > 0); |
| 175 | // Do "ugly" copy and paste from the expanded in order to generate more data |
| 176 | // to correlate (but not interpolate) with. |
| 177 | const int required_length = (120 + 80 + 2) * fs_mult_; |
| 178 | if (expanded_.Size() < static_cast<size_t>(required_length)) { |
| 179 | while (expanded_.Size() < static_cast<size_t>(required_length)) { |
| 180 | // Append one more pitch period each time. |
| 181 | expanded_.PushBack(expanded_temp); |
| 182 | } |
| 183 | // Trim the length to exactly |required_length|. |
| 184 | expanded_.PopBack(expanded_.Size() - required_length); |
| 185 | } |
| 186 | assert(expanded_.Size() >= static_cast<size_t>(required_length)); |
| 187 | return required_length; |
| 188 | } |
| 189 | |
| 190 | int16_t Merge::SignalScaling(const int16_t* input, int input_length, |
| 191 | const int16_t* expanded_signal, |
| 192 | int16_t* expanded_max, int16_t* input_max) const { |
| 193 | // Adjust muting factor if new vector is more or less of the BGN energy. |
| 194 | const int mod_input_length = std::min(64 * fs_mult_, input_length); |
| 195 | *expanded_max = WebRtcSpl_MaxAbsValueW16(expanded_signal, mod_input_length); |
| 196 | *input_max = WebRtcSpl_MaxAbsValueW16(input, mod_input_length); |
| 197 | |
| 198 | // Calculate energy of expanded signal. |
| 199 | // |log_fs_mult| is log2(fs_mult_), but is not exact for 48000 Hz. |
| 200 | int log_fs_mult = 30 - WebRtcSpl_NormW32(fs_mult_); |
| 201 | int expanded_shift = 6 + log_fs_mult |
| 202 | - WebRtcSpl_NormW32(*expanded_max * *expanded_max); |
| 203 | expanded_shift = std::max(expanded_shift, 0); |
| 204 | int32_t energy_expanded = WebRtcSpl_DotProductWithScale(expanded_signal, |
| 205 | expanded_signal, |
| 206 | mod_input_length, |
| 207 | expanded_shift); |
| 208 | |
| 209 | // Calculate energy of input signal. |
| 210 | int input_shift = 6 + log_fs_mult - |
| 211 | WebRtcSpl_NormW32(*input_max * *input_max); |
| 212 | input_shift = std::max(input_shift, 0); |
| 213 | int32_t energy_input = WebRtcSpl_DotProductWithScale(input, input, |
| 214 | mod_input_length, |
| 215 | input_shift); |
| 216 | |
| 217 | // Align to the same Q-domain. |
| 218 | if (input_shift > expanded_shift) { |
| 219 | energy_expanded = energy_expanded >> (input_shift - expanded_shift); |
| 220 | } else { |
| 221 | energy_input = energy_input >> (expanded_shift - input_shift); |
| 222 | } |
| 223 | |
| 224 | // Calculate muting factor to use for new frame. |
| 225 | int16_t mute_factor; |
| 226 | if (energy_input > energy_expanded) { |
| 227 | // Normalize |energy_input| to 14 bits. |
| 228 | int16_t temp_shift = WebRtcSpl_NormW32(energy_input) - 17; |
| 229 | energy_input = WEBRTC_SPL_SHIFT_W32(energy_input, temp_shift); |
| 230 | // Put |energy_expanded| in a domain 14 higher, so that |
| 231 | // energy_expanded / energy_input is in Q14. |
| 232 | energy_expanded = WEBRTC_SPL_SHIFT_W32(energy_expanded, temp_shift + 14); |
| 233 | // Calculate sqrt(energy_expanded / energy_input) in Q14. |
| 234 | mute_factor = WebRtcSpl_SqrtFloor((energy_expanded / energy_input) << 14); |
| 235 | } else { |
| 236 | // Set to 1 (in Q14) when |expanded| has higher energy than |input|. |
| 237 | mute_factor = 16384; |
| 238 | } |
| 239 | |
| 240 | return mute_factor; |
| 241 | } |
| 242 | |
| 243 | // TODO(hlundin): There are some parameter values in this method that seem |
| 244 | // strange. Compare with Expand::Correlation. |
| 245 | void Merge::Downsample(const int16_t* input, int input_length, |
| 246 | const int16_t* expanded_signal, int expanded_length) { |
| 247 | const int16_t* filter_coefficients; |
| 248 | int num_coefficients; |
| 249 | int decimation_factor = fs_hz_ / 4000; |
| 250 | static const int kCompensateDelay = 0; |
| 251 | int length_limit = fs_hz_ / 100; |
| 252 | if (fs_hz_ == 8000) { |
| 253 | filter_coefficients = DspHelper::kDownsample8kHzTbl; |
| 254 | num_coefficients = 3; |
| 255 | } else if (fs_hz_ == 16000) { |
| 256 | filter_coefficients = DspHelper::kDownsample16kHzTbl; |
| 257 | num_coefficients = 5; |
| 258 | } else if (fs_hz_ == 32000) { |
| 259 | filter_coefficients = DspHelper::kDownsample32kHzTbl; |
| 260 | num_coefficients = 7; |
| 261 | } else { // fs_hz_ == 48000 |
| 262 | filter_coefficients = DspHelper::kDownsample48kHzTbl; |
| 263 | num_coefficients = 7; |
| 264 | // TODO(hlundin) Why is |length_limit| not 480 (legacy)? |
| 265 | length_limit = 320; |
| 266 | } |
| 267 | int signal_offset = num_coefficients - 1; |
| 268 | WebRtcSpl_DownsampleFast(&expanded_signal[signal_offset], |
| 269 | expanded_length - signal_offset, |
| 270 | expanded_downsampled_, kExpandDownsampLength, |
| 271 | filter_coefficients, num_coefficients, |
| 272 | decimation_factor, kCompensateDelay); |
| 273 | if (input_length <= length_limit) { |
| 274 | // Not quite long enough, so we have to cheat a bit. |
| 275 | int16_t temp_len = input_length - signal_offset; |
| 276 | // TODO(hlundin): Should |downsamp_temp_len| be corrected for round-off |
| 277 | // errors? I.e., (temp_len + decimation_factor - 1) / decimation_factor? |
| 278 | int16_t downsamp_temp_len = temp_len / decimation_factor; |
| 279 | WebRtcSpl_DownsampleFast(&input[signal_offset], temp_len, |
| 280 | input_downsampled_, downsamp_temp_len, |
| 281 | filter_coefficients, num_coefficients, |
| 282 | decimation_factor, kCompensateDelay); |
| 283 | memset(&input_downsampled_[downsamp_temp_len], 0, |
| 284 | sizeof(int16_t) * (kInputDownsampLength - downsamp_temp_len)); |
| 285 | } else { |
| 286 | WebRtcSpl_DownsampleFast(&input[signal_offset], |
| 287 | input_length - signal_offset, input_downsampled_, |
| 288 | kInputDownsampLength, filter_coefficients, |
| 289 | num_coefficients, decimation_factor, |
| 290 | kCompensateDelay); |
| 291 | } |
| 292 | } |
| 293 | |
| 294 | int16_t Merge::CorrelateAndPeakSearch(int16_t expanded_max, int16_t input_max, |
| 295 | int start_position, int input_length, |
| 296 | int expand_period) const { |
| 297 | // Calculate correlation without any normalization. |
| 298 | const int max_corr_length = kMaxCorrelationLength; |
| 299 | int stop_position_downsamp = std::min( |
| 300 | max_corr_length, expand_->max_lag() / (fs_mult_ * 2) + 1); |
| 301 | int16_t correlation_shift = 0; |
| 302 | if (expanded_max * input_max > 26843546) { |
| 303 | correlation_shift = 3; |
| 304 | } |
| 305 | |
| 306 | int32_t correlation[kMaxCorrelationLength]; |
| 307 | WebRtcSpl_CrossCorrelation(correlation, input_downsampled_, |
| 308 | expanded_downsampled_, kInputDownsampLength, |
| 309 | stop_position_downsamp, correlation_shift, 1); |
| 310 | |
| 311 | // Normalize correlation to 14 bits and copy to a 16-bit array. |
| 312 | static const int kPadLength = 4; |
| 313 | int16_t correlation16[kPadLength + kMaxCorrelationLength + kPadLength] = {0}; |
| 314 | int16_t* correlation_ptr = &correlation16[kPadLength]; |
| 315 | int32_t max_correlation = WebRtcSpl_MaxAbsValueW32(correlation, |
| 316 | stop_position_downsamp); |
| 317 | int16_t norm_shift = std::max(0, 17 - WebRtcSpl_NormW32(max_correlation)); |
| 318 | WebRtcSpl_VectorBitShiftW32ToW16(correlation_ptr, stop_position_downsamp, |
| 319 | correlation, norm_shift); |
| 320 | |
| 321 | // Calculate allowed starting point for peak finding. |
| 322 | // The peak location bestIndex must fulfill two criteria: |
| 323 | // (1) w16_bestIndex + input_length < |
| 324 | // timestamps_per_call_ + expand_->overlap_length(); |
| 325 | // (2) w16_bestIndex + input_length < start_position. |
turaj@webrtc.org | 045e45e | 2013-09-20 16:25:28 +0000 | [diff] [blame] | 326 | int start_index = timestamps_per_call_ + |
| 327 | static_cast<int>(expand_->overlap_length()); |
henrik.lundin@webrtc.org | 9a40081 | 2013-01-29 12:09:21 +0000 | [diff] [blame] | 328 | start_index = std::max(start_position, start_index); |
| 329 | start_index = std::max(start_index - input_length, 0); |
| 330 | // Downscale starting index to 4kHz domain. (fs_mult_ * 2 = fs_hz_ / 4000.) |
| 331 | int start_index_downsamp = start_index / (fs_mult_ * 2); |
| 332 | |
| 333 | // Calculate a modified |stop_position_downsamp| to account for the increased |
| 334 | // start index |start_index_downsamp| and the effective array length. |
turaj@webrtc.org | 045e45e | 2013-09-20 16:25:28 +0000 | [diff] [blame] | 335 | int modified_stop_pos = |
henrik.lundin@webrtc.org | 9a40081 | 2013-01-29 12:09:21 +0000 | [diff] [blame] | 336 | std::min(stop_position_downsamp, |
| 337 | kMaxCorrelationLength + kPadLength - start_index_downsamp); |
| 338 | int best_correlation_index; |
| 339 | int16_t best_correlation; |
| 340 | static const int kNumCorrelationCandidates = 1; |
| 341 | DspHelper::PeakDetection(&correlation_ptr[start_index_downsamp], |
| 342 | modified_stop_pos, kNumCorrelationCandidates, |
| 343 | fs_mult_, &best_correlation_index, |
| 344 | &best_correlation); |
| 345 | // Compensate for modified start index. |
| 346 | best_correlation_index += start_index; |
| 347 | |
| 348 | // Ensure that underrun does not occur for 10ms case => we have to get at |
| 349 | // least 10ms + overlap . (This should never happen thanks to the above |
| 350 | // modification of peak-finding starting point.) |
| 351 | while ((best_correlation_index + input_length) < |
| 352 | static_cast<int>(timestamps_per_call_ + expand_->overlap_length()) || |
| 353 | best_correlation_index + input_length < start_position) { |
| 354 | assert(false); // Should never happen. |
| 355 | best_correlation_index += expand_period; // Jump one lag ahead. |
| 356 | } |
| 357 | return best_correlation_index; |
| 358 | } |
| 359 | |
| 360 | } // namespace webrtc |