Skip to content

GitLab

Commit 01add699 authored by Cheng Chen's avatar Cheng Chen
Browse files
Options

Fix TX64X64 bug in quantization 

Change-Id: I4b77388fa49e29a8302fe8c3276471e8156ce2d5
parent 41fffae6
Show whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
av1/encoder/av1_quantize.c
View file @ 01add699
...@@ -1570,7 +1570,7 @@ void av1_quantize_fp_64x64_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, ...@@ -1570,7 +1570,7 @@ void av1_quantize_fp_64x64_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
#else #else
if (abs_coeff >= (dequant_ptr[rc != 0] >> 3)) { if (abs_coeff >= (dequant_ptr[rc != 0] >> 3)) {
#endif #endif
abs_coeff += ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1); abs_coeff += ROUND_POWER_OF_TWO(round_ptr[rc != 0], 2);
abs_coeff = clamp(abs_coeff, INT16_MIN, INT16_MAX); abs_coeff = clamp(abs_coeff, INT16_MIN, INT16_MAX);
#if CONFIG_AOM_QM #if CONFIG_AOM_QM
tmp = abs_coeff * wt; tmp = abs_coeff * wt;
...@@ -1578,7 +1578,7 @@ void av1_quantize_fp_64x64_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, ...@@ -1578,7 +1578,7 @@ void av1_quantize_fp_64x64_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign; qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign;
dqcoeff_ptr[rc] = (qcoeff_ptr[rc] * dequant) / 4; dqcoeff_ptr[rc] = (qcoeff_ptr[rc] * dequant) / 4;
#else #else
tmp32 = (abs_coeff * quant_ptr[rc != 0]) >> 15; tmp32 = (abs_coeff * quant_ptr[rc != 0]) >> 14;
qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign; qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign;
dqcoeff_ptr[rc] = (qcoeff_ptr[rc] * dequant_ptr[rc != 0]) / 4; dqcoeff_ptr[rc] = (qcoeff_ptr[rc] * dequant_ptr[rc != 0]) / 4;
#endif #endif
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment