Skip to content

GitLab

A
aom-rav1e
Commit 8a086e05 authored by Sarah Parker's avatar Sarah Parker
Browse files
Options

Precompute shifted dequant values for large tx-sizes 

Change-Id: I87b11ec892fe5048f858869a3b6e68d0fe78ea43
parent 6e42741f
Hide whitespace changes
Inline Side-by-side
Showing with 53 additions and 42 deletions
av1/common/quant_common.c
View file @ 8a086e05
......@@ -192,15 +192,19 @@ void av1_get_dequant_val_nuq(int q, int is_ac_coeff, tran_low_t *dq,
// This is computed as: ((x0 - 64 - d) / 128) * Q
doff = quant_to_doff_fixed(is_ac_coeff, q_profile);
dq[0] = ROUND_POWER_OF_TWO((zbin - 64 - doff) * q, 7);
dq[1] = ROUND_POWER_OF_TWO((zbin - 64 - doff) * q, 8);
dq[2] = ROUND_POWER_OF_TWO((zbin - 64 - doff) * q, 9);
}
tran_low_t av1_dequant_abscoeff_nuq(int v, int q, const tran_low_t *dq) {
tran_low_t av1_dequant_abscoeff_nuq(int v, int q, const tran_low_t *dq,
int shift) {
if (v == 0) return 0;
return (q * v) + *dq;
return ((q * v) >> shift) + dq[shift];
}
tran_low_t av1_dequant_coeff_nuq(int v, int q, const tran_low_t *dq) {
tran_low_t dqmag = av1_dequant_abscoeff_nuq(abs(v), q, dq);
tran_low_t av1_dequant_coeff_nuq(int v, int q, const tran_low_t *dq,
int shift) {
tran_low_t dqmag = av1_dequant_abscoeff_nuq(abs(v), q, dq, shift);
return (v < 0 ? -dqmag : dqmag);
}
#endif // CONFIG_NEW_QUANT
......
av1/common/quant_common.h
View file @ 8a086e05
......@@ -64,12 +64,15 @@ qm_val_t *aom_qmatrix(struct AV1Common *cm, int qindex, int comp,
#define QUANT_RANGES 2
#define NUQ_KNOTS 1
typedef tran_low_t dequant_val_type_nuq[NUQ_KNOTS];
// dequant_val_type_nuq needs space for the 3 possible shift values
// for different tx sizes
typedef tran_low_t dequant_val_type_nuq[NUQ_KNOTS * 3];
typedef tran_low_t cuml_bins_type_nuq[NUQ_KNOTS];
void av1_get_dequant_val_nuq(int q, int is_ac_coeff, tran_low_t *dq,
tran_low_t *cuml_bins, int dq_off_index);
tran_low_t av1_dequant_abscoeff_nuq(int v, int q, const tran_low_t *dq);
tran_low_t av1_dequant_coeff_nuq(int v, int q, const tran_low_t *dq);
tran_low_t av1_dequant_abscoeff_nuq(int v, int q, const tran_low_t *dq,
int shift);
tran_low_t av1_dequant_coeff_nuq(int v, int q, const tran_low_t *dq, int shift);
static INLINE int qindex_to_qrange(int qindex) {
return (qindex < 140 ? 1 : 0);
......
av1/decoder/decodetxb.c
View file @ 8a086e05
......@@ -285,9 +285,10 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
cul_level += level;
#if CONFIG_NEW_QUANT
dqv_val = &dq_val[pos != 0][0];
v = av1_dequant_abscoeff_nuq(level, dequant[!!c], dqv_val);
#if !CONFIG_DAALA_TX
v = shift ? ROUND_POWER_OF_TWO(v, shift) : v;
v = av1_dequant_abscoeff_nuq(level, dequant[!!c], dqv_val, shift);
#else
v = av1_dequant_abscoeff_nuq(level, dequant[!!c], dqv_val, 0);
#endif // !CONFIG_DAALA_TX
#else
v = level * dequant[!!c];
......@@ -326,9 +327,10 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
cul_level += k + 1;
#if CONFIG_NEW_QUANT
dqv_val = &dq_val[pos != 0][0];
v = av1_dequant_abscoeff_nuq(k + 1, dequant[!!c], dqv_val);
#if !CONFIG_DAALA_TX
v = shift ? ROUND_POWER_OF_TWO(v, shift) : v;
v = av1_dequant_abscoeff_nuq(k + 1, dequant[!!c], dqv_val, shift);
#else
v = av1_dequant_abscoeff_nuq(k + 1, dequant[!!c], dqv_val, 0);
#endif // !CONFIG_DAALA_TX
#else
v = (k + 1) * dequant[!!c];
......@@ -410,9 +412,10 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
tran_low_t t;
#if CONFIG_NEW_QUANT
dqv_val = &dq_val[pos != 0][0];
t = av1_dequant_abscoeff_nuq(*level, dequant[!!pos], dqv_val);
#if !CONFIG_DAALA_TX
t = shift ? ROUND_POWER_OF_TWO(t, shift) : t;
t = av1_dequant_abscoeff_nuq(*level, dequant[!!pos], dqv_val, shift);
#else
t = av1_dequant_abscoeff_nuq(*level, dequant[!!pos], dqv_val, 0);
#endif // !CONFIG_DAALA_TX
#else
t = *level * dequant[!!pos];
......@@ -454,9 +457,10 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
tran_high_t t;
#if CONFIG_NEW_QUANT
dqv_val = &dq_val[pos != 0][0];
t = av1_dequant_abscoeff_nuq(*level, dequant[!!pos], dqv_val);
#if !CONFIG_DAALA_TX
t = shift ? ROUND_POWER_OF_TWO(t, shift) : t;
t = av1_dequant_abscoeff_nuq(*level, dequant[!!pos], dqv_val, shift);
#else
t = av1_dequant_abscoeff_nuq(*level, dequant[!!pos], dqv_val, 0);
#endif // !CONFIG_DAALA_TX
#else
t = *level * dequant[!!pos];
......@@ -480,9 +484,10 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
cul_level += (int)t;
#if CONFIG_NEW_QUANT
dqv_val = &dq_val[pos != 0][0];
t = av1_dequant_abscoeff_nuq(t, dequant[!!pos], dqv_val);
#if !CONFIG_DAALA_TX
t = shift ? ROUND_POWER_OF_TWO(t, shift) : t;
t = av1_dequant_abscoeff_nuq(t, dequant[!!pos], dqv_val, shift);
#else
t = av1_dequant_abscoeff_nuq(t, dequant[!!pos], dqv_val, 0);
#endif // !CONFIG_DAALA_TX
#else
t = t * dequant[!!pos];
......
av1/decoder/detokenize.c
View file @ 8a086e05
......@@ -169,9 +169,10 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
#endif
#if CONFIG_NEW_QUANT
v = av1_dequant_abscoeff_nuq(val, dqv, dqv_val);
#if !CONFIG_DAALA_TX
v = dq_shift ? ROUND_POWER_OF_TWO(v, dq_shift) : v;
v = av1_dequant_abscoeff_nuq(val, dqv, dqv_val, dq_shift);
#else
v = av1_dequant_abscoeff_nuq(val, dqv, dqv_val, 0);
#endif
#else
#if CONFIG_AOM_QM
......
av1/encoder/av1_quantize.c
View file @ 8a086e05
......@@ -39,7 +39,7 @@ static INLINE int quantize_coeff_nuq(
tmp -= cuml_bins_ptr[0];
q = NUQ_KNOTS + (((((tmp * quant) >> 16) + tmp) * quant_shift) >> 16);
*dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val);
*dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val, 0);
*qcoeff_ptr = (q ^ coeff_sign) - coeff_sign;
*dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr;
} else {
......@@ -64,10 +64,8 @@ static INLINE int quantize_coeff_bigtx_nuq(
tmp -= ROUND_POWER_OF_TWO(cuml_bins_ptr[0], logsizeby16);
q = NUQ_KNOTS +
(((((tmp * quant) >> 16) + tmp) * quant_shift) >> (16 - logsizeby16));
*dqcoeff_ptr = ROUND_POWER_OF_TWO(
av1_dequant_abscoeff_nuq(q, dequant, dequant_val), logsizeby16);
// *dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val) >>
// (logsizeby16);
*dqcoeff_ptr =
av1_dequant_abscoeff_nuq(q, dequant, dequant_val, logsizeby16);
*qcoeff_ptr = (q ^ coeff_sign) - coeff_sign;
*dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr;
} else {
......@@ -88,7 +86,7 @@ static INLINE int quantize_coeff_fp_nuq(
int tmp = clamp(abs_coeff, INT16_MIN, INT16_MAX);
if (tmp > cuml_bins_ptr[0]) {
q = NUQ_KNOTS + ((((int64_t)tmp - cuml_bins_ptr[0]) * quant) >> 16);
*dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val);
*dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val, 0);
*qcoeff_ptr = (q ^ coeff_sign) - coeff_sign;
*dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr;
} else {
......@@ -112,10 +110,8 @@ static INLINE int quantize_coeff_bigtx_fp_nuq(
((((int64_t)tmp - ROUND_POWER_OF_TWO(cuml_bins_ptr[0], logsizeby16)) *
quant) >>
(16 - logsizeby16));
*dqcoeff_ptr = ROUND_POWER_OF_TWO(
av1_dequant_abscoeff_nuq(q, dequant, dequant_val), logsizeby16);
// *dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val) >>
// (logsizeby16);
*dqcoeff_ptr =
av1_dequant_abscoeff_nuq(q, dequant, dequant_val, logsizeby16);
*qcoeff_ptr = (q ^ coeff_sign) - coeff_sign;
*dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr;
} else {
......@@ -982,7 +978,7 @@ static INLINE int highbd_quantize_coeff_nuq(
if (tmp >= zbin) {
tmp -= cuml_bins_ptr[0];
q = NUQ_KNOTS + (int)(((((tmp * quant) >> 16) + tmp) * quant_shift) >> 16);
*dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val);
*dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val, 0);
*qcoeff_ptr = (q ^ coeff_sign) - coeff_sign;
*dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr;
} else {
......@@ -1003,7 +999,7 @@ static INLINE int highbd_quantize_coeff_fp_nuq(
int64_t tmp = clamp(abs_coeff, INT32_MIN, INT32_MAX);
if (tmp > cuml_bins_ptr[0]) {
q = NUQ_KNOTS + (int)(((tmp - cuml_bins_ptr[0]) * quant) >> 16);
*dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val);
*dqcoeff_ptr = av1_dequant_abscoeff_nuq(q, dequant, dequant_val, 0);
*qcoeff_ptr = (q ^ coeff_sign) - coeff_sign;
*dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr;
} else {
......@@ -1027,8 +1023,8 @@ static INLINE int highbd_quantize_coeff_bigtx_fp_nuq(
(int)(((tmp - ROUND_POWER_OF_TWO(cuml_bins_ptr[0], logsizeby16)) *
quant) >>
(16 - logsizeby16));
*dqcoeff_ptr = ROUND_POWER_OF_TWO(
av1_dequant_abscoeff_nuq(q, dequant, dequant_val), logsizeby16);
*dqcoeff_ptr =
av1_dequant_abscoeff_nuq(q, dequant, dequant_val, logsizeby16);
*qcoeff_ptr = (q ^ coeff_sign) - coeff_sign;
*dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr;
} else {
......@@ -1053,8 +1049,8 @@ static INLINE int highbd_quantize_coeff_bigtx_nuq(
tmp -= ROUND_POWER_OF_TWO(cuml_bins_ptr[0], logsizeby16);
q = NUQ_KNOTS + (int)(((((tmp * quant) >> 16) + tmp) * quant_shift) >>
(16 - logsizeby16));
*dqcoeff_ptr = ROUND_POWER_OF_TWO(
av1_dequant_abscoeff_nuq(q, dequant, dequant_val), logsizeby16);
*dqcoeff_ptr =
av1_dequant_abscoeff_nuq(q, dequant, dequant_val, logsizeby16);
*qcoeff_ptr = (q ^ coeff_sign) - coeff_sign;
*dqcoeff_ptr = *qcoeff_ptr < 0 ? -*dqcoeff_ptr : *dqcoeff_ptr;
} else {
......
av1/encoder/encodemb.c
View file @ 8a086e05
......@@ -255,14 +255,15 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane,
if (x_a != 0) {
#if CONFIG_DAALA_TX
#if CONFIG_NEW_QUANT
dx = av1_dequant_coeff_nuq(x_a, dqv, dequant_val[rc != 0]) - coeff[rc];
dx = av1_dequant_coeff_nuq(x_a, dqv, dequant_val[rc != 0], 0) -
coeff[rc];
#else // CONFIG_NEW_QUANT
dx -= (dqv + sz) ^ sz;
#endif // CONFIG_NEW_QUANT
d2_a = ((int64_t)dx * dx + depth_round) >> depth_shift;
#else // CONFIG_DAALA_TX
#if CONFIG_NEW_QUANT
dx = av1_dequant_coeff_nuq(x_a, dqv, dequant_val[rc != 0]) -
dx = av1_dequant_coeff_nuq(x_a, dqv, dequant_val[rc != 0], 0) -
(coeff[rc] * (1 << shift));
dx >>= xd->bd - 8;
#else // CONFIG_NEW_QUANT
......@@ -344,15 +345,16 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane,
if (x_a != 0) {
#if CONFIG_DAALA_TX
#if CONFIG_NEW_QUANT
dqc_a = av1_dequant_abscoeff_nuq(abs(x_a), dqv, dequant_val[rc != 0]);
dqc_a =
av1_dequant_abscoeff_nuq(abs(x_a), dqv, dequant_val[rc != 0], 0);
if (sz) dqc_a = -dqc_a;
#else
dqc_a = x_a * dqv;
#endif // CONFIG_NEW_QUANT
#else // CONFIG_DAALA_TX
#if CONFIG_NEW_QUANT
dqc_a = av1_dequant_abscoeff_nuq(abs(x_a), dqv, dequant_val[rc != 0]);
dqc_a = shift ? ROUND_POWER_OF_TWO(dqc_a, shift) : dqc_a;
dqc_a = av1_dequant_abscoeff_nuq(abs(x_a), dqv, dequant_val[rc != 0],
shift);
if (sz) dqc_a = -dqc_a;
#else
if (x_a < 0)
......
av1/encoder/encodetxb.c
View file @ 8a086e05
......@@ -143,8 +143,8 @@ static INLINE tran_low_t qcoeff_to_dqcoeff(tran_low_t qc,
int dqv, int shift) {
int sgn = qc < 0 ? -1 : 1;
#if CONFIG_NEW_QUANT
int dqcoeff = av1_dequant_coeff_nuq(abs(qc), dqv, nq_dq);
return sgn * (shift ? ROUND_POWER_OF_TWO(dqcoeff, shift) : dqcoeff);
int dqcoeff = av1_dequant_coeff_nuq(abs(qc), dqv, nq_dq, shift);
return sgn * dqcoeff;
#endif // CONFIG_NEW_QUANT
return sgn * ((abs(qc) * dqv) >> shift);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment