fix calculation bugs in encode_subframe_(): max_qlp_coeff_precision when qlp_coeff_prec_search; account for qlp coeff overhead in guessing the best lpc order; default to hann instead of rectangular window on assertion failure
diff --git a/src/libFLAC/stream_encoder.c b/src/libFLAC/stream_encoder.c
index 2bfe5d0..21ea026 100644
--- a/src/libFLAC/stream_encoder.c
+++ b/src/libFLAC/stream_encoder.c
@@ -1985,7 +1985,7 @@
default:
FLAC__ASSERT(0);
/* double protection */
- FLAC__window_rectangle(encoder->private_->window[i], new_size);
+ FLAC__window_hann(encoder->private_->window[i], new_size);
break;
}
}
@@ -2519,7 +2519,17 @@
min_lpc_order = 1;
}
else {
- unsigned guess_lpc_order = FLAC__lpc_compute_best_order(lpc_error, max_lpc_order, frame_header->blocksize, subframe_bps);
+ const unsigned guess_lpc_order =
+ FLAC__lpc_compute_best_order(
+ lpc_error,
+ max_lpc_order,
+ frame_header->blocksize,
+ subframe_bps + (
+ encoder->protected_->do_qlp_coeff_prec_search?
+ FLAC__MIN_QLP_COEFF_PRECISION : /* have to guess; use the min possible size to avoid accidentally favoring lower orders */
+ encoder->protected_->qlp_coeff_precision
+ )
+ );
min_lpc_order = max_lpc_order = guess_lpc_order;
}
for(lpc_order = min_lpc_order; lpc_order <= max_lpc_order; lpc_order++) {
@@ -2529,16 +2539,18 @@
rice_parameter = (lpc_residual_bits_per_sample > 0.0)? (unsigned)(lpc_residual_bits_per_sample+0.5) : 0; /* 0.5 is for rounding */
rice_parameter++; /* to account for the signed->unsigned conversion during rice coding */
if(rice_parameter >= FLAC__ENTROPY_CODING_METHOD_PARTITIONED_RICE_ESCAPE_PARAMETER) {
- #ifdef DEBUG_VERBOSE
+#ifdef DEBUG_VERBOSE
fprintf(stderr, "clipping rice_parameter (%u -> %u) @1\n", rice_parameter, FLAC__ENTROPY_CODING_METHOD_PARTITIONED_RICE_ESCAPE_PARAMETER - 1);
- #endif
+#endif
rice_parameter = FLAC__ENTROPY_CODING_METHOD_PARTITIONED_RICE_ESCAPE_PARAMETER - 1;
}
if(encoder->protected_->do_qlp_coeff_prec_search) {
min_qlp_coeff_precision = FLAC__MIN_QLP_COEFF_PRECISION;
- /* ensure a 32-bit datapath throughout for 16bps or less */
- if(subframe_bps <= 16)
+ /* try to ensure a 32-bit datapath throughout for 16bps(+1bps for side channel) or less */
+ if(subframe_bps <= 17) {
max_qlp_coeff_precision = min(32 - subframe_bps - lpc_order, FLAC__MAX_QLP_COEFF_PRECISION);
+ max_qlp_coeff_precision = max(max_qlp_coeff_precision, min_qlp_coeff_precision);
+ }
else
max_qlp_coeff_precision = FLAC__MAX_QLP_COEFF_PRECISION;
}