Skip to content

GitLab

Commit bdda9d4e authored by Yaowu Xu's avatar Yaowu Xu
Browse files
Options

convert to int before apply sign 

avoids overflow of unsigned integer.

Change-Id: Ic92974b508bb0cd6fc680203ffa6cff14d644ff7
parent cb637674
Hide whitespace changes
Inline Side-by-side
Showing with 4 additions and 5 deletions
av1/encoder/av1_quantize.c
View file @ bdda9d4e
......@@ -973,7 +973,7 @@ static INLINE void highbd_quantize_dc(
const int coeff_sign = (coeff >> 31);
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
const int64_t tmp = abs_coeff + round_ptr[0];
const uint32_t abs_qcoeff = (uint32_t)((tmp * quant) >> (16 - log_scale));
const int abs_qcoeff = (int)((tmp * quant) >> (16 - log_scale));
qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant_ptr / (1 << log_scale);
if (abs_qcoeff) eob = 0;
......@@ -1564,13 +1564,12 @@ void av1_highbd_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t count,
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
const int64_t tmp = abs_coeff + (round_ptr[rc != 0] >> log_scale);
#if CONFIG_AOM_QM
const uint32_t abs_qcoeff =
(uint32_t)((tmp * quant_ptr[rc != 0] * wt) >> (shift + AOM_QM_BITS));
const int abs_qcoeff =
(int)((tmp * quant_ptr[rc != 0] * wt) >> (shift + AOM_QM_BITS));
qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant / scale;
#else
const uint32_t abs_qcoeff =
(uint32_t)((tmp * quant_ptr[rc != 0]) >> shift);
const int abs_qcoeff = (int)((tmp * quant_ptr[rc != 0]) >> shift);
qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / scale;
#endif
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment