Commit fe9ba6f4 authored by Josh Coalson's avatar Josh Coalson
Browse files

fix bug where overflow of total_error caused it to go negative

parent f6237c8c
...@@ -34,7 +34,7 @@ ...@@ -34,7 +34,7 @@
#ifdef local_abs #ifdef local_abs
#undef local_abs #undef local_abs
#endif #endif
#define local_abs(x) ((x)<0? -(x) : (x)) #define local_abs(x) ((unsigned)((x)<0? -(x) : (x)))
unsigned FLAC__fixed_compute_best_predictor(const int32 data[], unsigned data_len, real residual_bits_per_sample[FLAC__MAX_FIXED_ORDER+1]) unsigned FLAC__fixed_compute_best_predictor(const int32 data[], unsigned data_len, real residual_bits_per_sample[FLAC__MAX_FIXED_ORDER+1])
{ {
...@@ -43,7 +43,7 @@ unsigned FLAC__fixed_compute_best_predictor(const int32 data[], unsigned data_le ...@@ -43,7 +43,7 @@ unsigned FLAC__fixed_compute_best_predictor(const int32 data[], unsigned data_le
int32 last_error_2 = last_error_1 - (data[-2] - data[-3]); int32 last_error_2 = last_error_1 - (data[-2] - data[-3]);
int32 last_error_3 = last_error_2 - (data[-2] - 2*data[-3] + data[-4]); int32 last_error_3 = last_error_2 - (data[-2] - 2*data[-3] + data[-4]);
int32 error_0, error_1, error_2, error_3, error_4; int32 error_0, error_1, error_2, error_3, error_4;
int32 total_error_0 = 0, total_error_1 = 0, total_error_2 = 0, total_error_3 = 0, total_error_4 = 0; uint32 total_error_0 = 0, total_error_1 = 0, total_error_2 = 0, total_error_3 = 0, total_error_4 = 0;
unsigned i, order; unsigned i, order;
for(i = 0; i < data_len; i++) { for(i = 0; i < data_len; i++) {
...@@ -53,6 +53,12 @@ unsigned FLAC__fixed_compute_best_predictor(const int32 data[], unsigned data_le ...@@ -53,6 +53,12 @@ unsigned FLAC__fixed_compute_best_predictor(const int32 data[], unsigned data_le
error_3 = error_2 - last_error_2; total_error_3 += local_abs(error_3); error_3 = error_2 - last_error_2; total_error_3 += local_abs(error_3);
error_4 = error_3 - last_error_3; total_error_4 += local_abs(error_4); error_4 = error_3 - last_error_3; total_error_4 += local_abs(error_4);
/* WATCHOUT - total_error_* has been know to overflow when encoding
* erratic signals when the bits-per-sample is large. We avoid the
* speed penalty of watching for overflow, and instead rely on the
* encoder's evaluation of the subframe to catch these cases.
*/
last_error_0 = error_0; last_error_0 = error_0;
last_error_1 = error_1; last_error_1 = error_1;
last_error_2 = error_2; last_error_2 = error_2;
...@@ -73,11 +79,11 @@ unsigned FLAC__fixed_compute_best_predictor(const int32 data[], unsigned data_le ...@@ -73,11 +79,11 @@ unsigned FLAC__fixed_compute_best_predictor(const int32 data[], unsigned data_le
/* Estimate the expected number of bits per residual signal sample. */ /* Estimate the expected number of bits per residual signal sample. */
/* 'total_error*' is linearly related to the variance of the residual */ /* 'total_error*' is linearly related to the variance of the residual */
/* signal, so we use it directly to compute E(|x|) */ /* signal, so we use it directly to compute E(|x|) */
residual_bits_per_sample[0] = (real)((total_error_0 > 0 && data_len > 0) ? log(M_LN2 * total_error_0 / (real) data_len) / M_LN2 : 0.0); residual_bits_per_sample[0] = (real)((data_len > 0) ? log(M_LN2 * (real)total_error_0 / (real) data_len) / M_LN2 : 0.0);
residual_bits_per_sample[1] = (real)((total_error_1 > 0 && data_len > 0) ? log(M_LN2 * total_error_1 / (real) data_len) / M_LN2 : 0.0); residual_bits_per_sample[1] = (real)((data_len > 0) ? log(M_LN2 * (real)total_error_1 / (real) data_len) / M_LN2 : 0.0);
residual_bits_per_sample[2] = (real)((total_error_2 > 0 && data_len > 0) ? log(M_LN2 * total_error_2 / (real) data_len) / M_LN2 : 0.0); residual_bits_per_sample[2] = (real)((data_len > 0) ? log(M_LN2 * (real)total_error_2 / (real) data_len) / M_LN2 : 0.0);
residual_bits_per_sample[3] = (real)((total_error_3 > 0 && data_len > 0) ? log(M_LN2 * total_error_3 / (real) data_len) / M_LN2 : 0.0); residual_bits_per_sample[3] = (real)((data_len > 0) ? log(M_LN2 * (real)total_error_3 / (real) data_len) / M_LN2 : 0.0);
residual_bits_per_sample[4] = (real)((total_error_4 > 0 && data_len > 0) ? log(M_LN2 * total_error_4 / (real) data_len) / M_LN2 : 0.0); residual_bits_per_sample[4] = (real)((data_len > 0) ? log(M_LN2 * (real)total_error_4 / (real) data_len) / M_LN2 : 0.0);
return order; return order;
} }
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment