Commit a855cefc authored by Debargha Mukherjee's avatar Debargha Mukherjee
Browse files

A small fix for the new-quant b_quantizer

Change-Id: I3698acb824afc5d0f66c2d4a099403f07a373c86
parent e5267697
...@@ -35,7 +35,7 @@ static INLINE int quantize_coeff_nuq( ...@@ -35,7 +35,7 @@ static INLINE int quantize_coeff_nuq(
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
int q = 0; int q = 0;
int tmp = clamp(abs_coeff, INT16_MIN, INT16_MAX); int tmp = clamp(abs_coeff, INT16_MIN, INT16_MAX);
if (tmp >= zbin) { if (tmp >= AOMMAX(zbin, cuml_bins_ptr[0])) {
tmp -= cuml_bins_ptr[0]; tmp -= cuml_bins_ptr[0];
q = NUQ_KNOTS + (((((tmp * quant) >> 16) + tmp) * quant_shift) >> 16); q = NUQ_KNOTS + (((((tmp * quant) >> 16) + tmp) * quant_shift) >> 16);
...@@ -60,8 +60,10 @@ static INLINE int quantize_coeff_bigtx_nuq( ...@@ -60,8 +60,10 @@ static INLINE int quantize_coeff_bigtx_nuq(
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
int q = 0; int q = 0;
int tmp = clamp(abs_coeff, INT16_MIN, INT16_MAX); int tmp = clamp(abs_coeff, INT16_MIN, INT16_MAX);
if (tmp >= zbin_val) { const int cuml_bins_ptr_val =
tmp -= ROUND_POWER_OF_TWO(cuml_bins_ptr[0], logsizeby16); ROUND_POWER_OF_TWO(cuml_bins_ptr[0], logsizeby16);
if (tmp >= AOMMAX(zbin_val, cuml_bins_ptr_val)) {
tmp -= cuml_bins_ptr_val;
q = NUQ_KNOTS + q = NUQ_KNOTS +
(((((tmp * quant) >> 16) + tmp) * quant_shift) >> (16 - logsizeby16)); (((((tmp * quant) >> 16) + tmp) * quant_shift) >> (16 - logsizeby16));
*dqcoeff_ptr = *dqcoeff_ptr =
...@@ -975,7 +977,7 @@ static INLINE int highbd_quantize_coeff_nuq( ...@@ -975,7 +977,7 @@ static INLINE int highbd_quantize_coeff_nuq(
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
int q = 0; int q = 0;
int64_t tmp = clamp(abs_coeff, INT32_MIN, INT32_MAX); int64_t tmp = clamp(abs_coeff, INT32_MIN, INT32_MAX);
if (tmp >= zbin) { if (tmp >= AOMMAX(zbin, cuml_bins_ptr[0])) {
tmp -= cuml_bins_ptr[0]; tmp -= cuml_bins_ptr[0];
q = NUQ_KNOTS + (int)(((((tmp * quant) >> 16) + tmp) * quant_shift) >> 16); q = NUQ_KNOTS + (int)(((((tmp * quant) >> 16) + tmp) * quant_shift) >> 16);
*dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val, 0); *dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val, 0);
...@@ -1045,8 +1047,10 @@ static INLINE int highbd_quantize_coeff_bigtx_nuq( ...@@ -1045,8 +1047,10 @@ static INLINE int highbd_quantize_coeff_bigtx_nuq(
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
int q = 0; int q = 0;
int64_t tmp = clamp(abs_coeff, INT32_MIN, INT32_MAX); int64_t tmp = clamp(abs_coeff, INT32_MIN, INT32_MAX);
if (tmp >= zbin_val) { const int cuml_bins_ptr_val =
tmp -= ROUND_POWER_OF_TWO(cuml_bins_ptr[0], logsizeby16); ROUND_POWER_OF_TWO(cuml_bins_ptr[0], logsizeby16);
if (tmp >= AOMMAX(zbin_val, cuml_bins_ptr_val)) {
tmp -= cuml_bins_ptr_val;
q = NUQ_KNOTS + (int)(((((tmp * quant) >> 16) + tmp) * quant_shift) >> q = NUQ_KNOTS + (int)(((((tmp * quant) >> 16) + tmp) * quant_shift) >>
(16 - logsizeby16)); (16 - logsizeby16));
*dqcoeff_ptr = *dqcoeff_ptr =
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment