Skip to content

GitLab

A
aom-rav1e
Commit a855cefc authored by Debargha Mukherjee's avatar Debargha Mukherjee
Browse files
Options

A small fix for the new-quant b_quantizer 

Change-Id: I3698acb824afc5d0f66c2d4a099403f07a373c86
parent e5267697
Hide whitespace changes
Inline Side-by-side
Showing with 10 additions and 6 deletions
av1/encoder/av1_quantize.c
View file @ a855cefc
......@@ -35,7 +35,7 @@ static INLINE int quantize_coeff_nuq(
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
int q = 0;
int tmp = clamp(abs_coeff, INT16_MIN, INT16_MAX);
if (tmp >= zbin) {
if (tmp >= AOMMAX(zbin, cuml_bins_ptr[0])) {
tmp -= cuml_bins_ptr[0];
q = NUQ_KNOTS + (((((tmp * quant) >> 16) + tmp) * quant_shift) >> 16);
......@@ -60,8 +60,10 @@ static INLINE int quantize_coeff_bigtx_nuq(
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
int q = 0;
int tmp = clamp(abs_coeff, INT16_MIN, INT16_MAX);
if (tmp >= zbin_val) {
tmp -= ROUND_POWER_OF_TWO(cuml_bins_ptr[0], logsizeby16);
const int cuml_bins_ptr_val =
ROUND_POWER_OF_TWO(cuml_bins_ptr[0], logsizeby16);
if (tmp >= AOMMAX(zbin_val, cuml_bins_ptr_val)) {
tmp -= cuml_bins_ptr_val;
q = NUQ_KNOTS +
(((((tmp * quant) >> 16) + tmp) * quant_shift) >> (16 - logsizeby16));
*dqcoeff_ptr =
......@@ -975,7 +977,7 @@ static INLINE int highbd_quantize_coeff_nuq(
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
int q = 0;
int64_t tmp = clamp(abs_coeff, INT32_MIN, INT32_MAX);
if (tmp >= zbin) {
if (tmp >= AOMMAX(zbin, cuml_bins_ptr[0])) {
tmp -= cuml_bins_ptr[0];
q = NUQ_KNOTS + (int)(((((tmp * quant) >> 16) + tmp) * quant_shift) >> 16);
*dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val, 0);
......@@ -1045,8 +1047,10 @@ static INLINE int highbd_quantize_coeff_bigtx_nuq(
const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
int q = 0;
int64_t tmp = clamp(abs_coeff, INT32_MIN, INT32_MAX);
if (tmp >= zbin_val) {
tmp -= ROUND_POWER_OF_TWO(cuml_bins_ptr[0], logsizeby16);
const int cuml_bins_ptr_val =
ROUND_POWER_OF_TWO(cuml_bins_ptr[0], logsizeby16);
if (tmp >= AOMMAX(zbin_val, cuml_bins_ptr_val)) {
tmp -= cuml_bins_ptr_val;
q = NUQ_KNOTS + (int)(((((tmp * quant) >> 16) + tmp) * quant_shift) >>
(16 - logsizeby16));
*dqcoeff_ptr =
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment