Commit 98c8dd71 authored by Sarah Parker's avatar Sarah Parker Committed by Debargha Mukherjee

Make new-quant compatible with aom-qm

Change-Id: I4d201631e288ca0f5a0d13583707dff483eb5cac
parent 6a0245c5
......@@ -151,18 +151,18 @@ if (aom_config("CONFIG_DAALA_TX") ne "yes") {
}
if (aom_config("CONFIG_NEW_QUANT") eq "yes") {
add_proto qw/void quantize_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void quantize_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, int dq, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr";
add_proto qw/void quantize_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void quantize_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, int dq, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr";
add_proto qw/void quantize_32x32_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void quantize_32x32_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, int dq, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr";
add_proto qw/void quantize_32x32_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void quantize_32x32_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, int dq, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr";
if (aom_config("CONFIG_TX64X64") eq "yes") {
add_proto qw/void quantize_64x64_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void quantize_64x64_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, int dq, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr";
add_proto qw/void quantize_64x64_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void quantize_64x64_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, int dq, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr";
}
}
......@@ -415,18 +415,18 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
# ENCODEMB INVOKE
if (aom_config("CONFIG_NEW_QUANT") eq "yes") {
add_proto qw/void highbd_quantize_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void highbd_quantize_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, int dq, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr";
add_proto qw/void highbd_quantize_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void highbd_quantize_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, int dq, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr";
add_proto qw/void highbd_quantize_32x32_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void highbd_quantize_32x32_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, int dq, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr";
add_proto qw/void highbd_quantize_32x32_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void highbd_quantize_32x32_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, int dq, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr";
if (aom_config("CONFIG_TX64X64") eq "yes") {
add_proto qw/void highbd_quantize_64x64_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void highbd_quantize_64x64_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, const int16_t *dequant_ptr, int dq, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr";
add_proto qw/void highbd_quantize_64x64_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan";
add_proto qw/void highbd_quantize_64x64_fp_nuq/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *quant_ptr, const int16_t *dequant_ptr, int dq, const cuml_bins_type_nuq *cuml_bins_ptr, const dequant_val_type_nuq *dequant_val, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const int16_t *scan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr";
}
}
......
......@@ -232,15 +232,35 @@ void av1_get_dequant_val_nuq(int q, int is_ac_coeff, tran_low_t *dq,
dq[2] = ROUND_POWER_OF_TWO_SIGNED(doff * q, 9);
}
tran_low_t av1_dequant_abscoeff_nuq(int v, int q, const tran_low_t *dq,
tran_low_t av1_dequant_abscoeff_nuq(int v, int q,
#if CONFIG_AOM_QM
int q_profile, int is_ac_coeff,
#else
const tran_low_t *dq,
#endif // CONFIG_AOM_QM
int shift) {
if (v == 0) return 0;
#if CONFIG_AOM_QM
const uint8_t doff = quant_to_doff_fixed(is_ac_coeff, q_profile);
return ((q * v) >> shift) + ROUND_POWER_OF_TWO_SIGNED(doff * q, 7 + shift);
#else
return ((q * v) >> shift) + dq[shift];
#endif
}
tran_low_t av1_dequant_coeff_nuq(int v, int q, const tran_low_t *dq,
tran_low_t av1_dequant_coeff_nuq(int v, int q,
#if CONFIG_AOM_QM
int q_profile, int is_ac_coeff,
#else
const tran_low_t *dq,
#endif // CONFIG_AOM_QM
int shift) {
#if CONFIG_AOM_QM
tran_low_t dqmag =
av1_dequant_abscoeff_nuq(abs(v), q, q_profile, is_ac_coeff, shift);
#else
tran_low_t dqmag = av1_dequant_abscoeff_nuq(abs(v), q, dq, shift);
#endif // CONFIG_AOM_QM
return (v < 0 ? -dqmag : dqmag);
}
#endif // CONFIG_NEW_QUANT
......
......@@ -75,9 +75,20 @@ void av1_get_dequant_val_nuq(int q, int is_ac_coeff, tran_low_t *dq,
int dq_off_index);
void av1_get_cuml_bins_nuq(int q, int is_ac_coeff, tran_low_t *cuml_bins,
int q_profile);
tran_low_t av1_dequant_abscoeff_nuq(int v, int q, const tran_low_t *dq,
tran_low_t av1_dequant_abscoeff_nuq(int v, int q,
#if CONFIG_AOM_QM
int q_profile, int is_ac_coeff,
#else
const tran_low_t *dq,
#endif // CONFIG_AOM_QM
int shift);
tran_low_t av1_dequant_coeff_nuq(int v, int q, const tran_low_t *dq, int shift);
tran_low_t av1_dequant_coeff_nuq(int v, int q,
#if CONFIG_AOM_QM
int q_profile, int is_ac_coeff,
#else
const tran_low_t *dq,
#endif // CONFIG_AOM_QM
int shift);
static INLINE int qindex_to_qrange(int qindex) {
return (qindex < 140 ? 1 : 0);
......
......@@ -57,7 +57,11 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
aom_reader *const r, const int blk_row,
const int blk_col, const int plane,
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
int dq_profile,
#else
dequant_val_type_nuq *dq_val,
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
const TXB_CTX *const txb_ctx, const TX_SIZE tx_size,
int16_t *const max_scan_line, int *const eob) {
......@@ -78,12 +82,20 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
struct macroblockd_plane *const pd = &xd->plane[plane];
const int16_t *const dequant = pd->seg_dequant_QTX[mbmi->segment_id];
tran_low_t *const tcoeffs = pd->dqcoeff;
#if CONFIG_NEW_QUANT
const tran_low_t *dqv_val = &dq_val[0][0];
#endif // CONFIG_NEW_QUANT
#if !CONFIG_DAALA_TX
const int shift = av1_get_tx_scale(tx_size);
#endif
#if CONFIG_NEW_QUANT
#if !CONFIG_AOM_QM
const tran_low_t *dqv_val = &dq_val[0][0];
#endif // !CONFIG_AOM_QM
#if CONFIG_DAALA_TX
const int nq_shift = 0;
#else
const int nq_shift = shift;
#endif // CONFIG_DAALA_TX
#endif // CONFIG_NEW_QUANT && !CONFIG_AOM_QM
const int bwl = get_txb_bwl(tx_size);
const int width = get_txb_wide(tx_size);
const int height = get_txb_high(tx_size);
......@@ -267,12 +279,13 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
if (level < 3) {
cul_level += level;
#if CONFIG_NEW_QUANT
dqv_val = &dq_val[pos != 0][0];
#if !CONFIG_DAALA_TX
v = av1_dequant_abscoeff_nuq(level, dequant[!!c], dqv_val, shift);
#if CONFIG_AOM_QM
v = av1_dequant_abscoeff_nuq(level, dequant[!!c], dq_profile, !!c,
nq_shift);
#else
v = av1_dequant_abscoeff_nuq(level, dequant[!!c], dqv_val, 0);
#endif // !CONFIG_DAALA_TX
dqv_val = &dq_val[pos != 0][0];
v = av1_dequant_abscoeff_nuq(level, dequant[!!c], dqv_val, nq_shift);
#endif // CONFIG_AOM_QM
#else
v = level * dequant[!!c];
#if !CONFIG_DAALA_TX
......@@ -338,12 +351,13 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
cul_level += *level;
tran_low_t t;
#if CONFIG_NEW_QUANT
dqv_val = &dq_val[pos != 0][0];
#if !CONFIG_DAALA_TX
t = av1_dequant_abscoeff_nuq(*level, dequant[!!pos], dqv_val, shift);
#if CONFIG_AOM_QM
t = av1_dequant_abscoeff_nuq(*level, dequant[!!pos], dq_profile, !!pos,
nq_shift);
#else
t = av1_dequant_abscoeff_nuq(*level, dequant[!!pos], dqv_val, 0);
#endif // !CONFIG_DAALA_TX
dqv_val = &dq_val[pos != 0][0];
t = av1_dequant_abscoeff_nuq(*level, dequant[!!pos], dqv_val, nq_shift);
#endif // CONFIG_AOM_QM
#else
t = *level * dequant[!!pos];
#if !CONFIG_DAALA_TX
......@@ -360,12 +374,13 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
tran_low_t t = *level + read_golomb(xd, r, counts);
cul_level += (int)t;
#if CONFIG_NEW_QUANT
dqv_val = &dq_val[pos != 0][0];
#if !CONFIG_DAALA_TX
t = av1_dequant_abscoeff_nuq(t, dequant[!!pos], dqv_val, shift);
#if CONFIG_AOM_QM
t = av1_dequant_abscoeff_nuq(t, dequant[!!pos], dq_profile, !!pos,
nq_shift);
#else
t = av1_dequant_abscoeff_nuq(t, dequant[!!pos], dqv_val, 0);
#endif // !CONFIG_DAALA_TX
dqv_val = &dq_val[pos != 0][0];
t = av1_dequant_abscoeff_nuq(t, dequant[!!pos], dqv_val, nq_shift);
#endif // CONFIG_AOM_QM
#else
t = t * dequant[!!pos];
#if !CONFIG_DAALA_TX
......@@ -408,7 +423,11 @@ uint8_t av1_read_coeffs_txb_facade(const AV1_COMMON *const cm,
uint8_t cul_level =
av1_read_coeffs_txb(cm, xd, r, row, col, plane,
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
dq,
#else
pd->seg_dequant_nuq_QTX[seg_id][dq],
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
&txb_ctx, tx_size, max_scan_line, eob);
av1_set_contexts(xd, pd, plane, tx_size, cul_level, col, row);
......
......@@ -22,7 +22,11 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
aom_reader *const r, const int blk_row,
const int blk_col, const int plane,
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
int dq_profile,
#else
dequant_val_type_nuq *dq_val,
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
const TXB_CTX *const txb_ctx, const TX_SIZE tx_size,
int16_t *const max_scan_line, int *const eob);
......
......@@ -72,18 +72,21 @@ static int token_to_value(FRAME_COUNTS *counts, aom_reader *const r, int token,
static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
TX_SIZE tx_size, TX_TYPE tx_type, const int16_t *dq,
#if CONFIG_NEW_QUANT
dequant_val_type_nuq *dq_val,
#if CONFIG_AOM_QM
int dq_profile,
#else
dequant_val_type_nuq *dq_val,
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
qm_val_t *iqm[TX_SIZES_ALL],
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
int ctx, const int16_t *scan, const int16_t *nb,
int16_t *max_scan_line, aom_reader *r) {
FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
const int max_eob = av1_get_max_eob(tx_size);
const int ref = is_inter_block(&xd->mi[0]->mbmi);
#if CONFIG_AOM_QM && !CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
const qm_val_t *iqmatrix = iqm[tx_size];
#endif // CONFIG_AOM_QM
(void)tx_type;
......@@ -99,14 +102,22 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
const uint8_t *band_translate = get_band_translate(tx_size);
int v, token;
int32_t dqv = dq[0];
#if CONFIG_NEW_QUANT
#if CONFIG_NEW_QUANT && !CONFIG_AOM_QM
const tran_low_t *dqv_val = &dq_val[0][0];
#endif // CONFIG_NEW_QUANT
#endif // CONFIG_NEW_QUANT && !CONFIG_AOM_QM
#if !CONFIG_DAALA_TX
int dq_shift = av1_get_tx_scale(tx_size);
#endif
#if CONFIG_NEW_QUANT
#if CONFIG_DAALA_TX
int nq_shift = 0;
#else
int nq_shift = dq_shift;
#endif // CONFIG_DAALA_TX
#endif // CONFIG_NEW_QUANT
band = *band_translate++;
int more_data = 1;
......@@ -115,9 +126,9 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
int last_pos = (c + 1 == max_eob);
int first_pos = (c == 0);
#if CONFIG_NEW_QUANT
#if CONFIG_NEW_QUANT && !CONFIG_AOM_QM
dqv_val = &dq_val[band != 0][0];
#endif // CONFIG_NEW_QUANT
#endif // CONFIG_NEW_QUANT && !CONFIG_AOM_QM
comb_token = last_pos ? 2 * av1_read_record_bit(xd->counts, r, ACCT_STR) + 2
: av1_read_record_symbol(
......@@ -156,9 +167,9 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
if (token > ONE_TOKEN)
token += av1_read_record_symbol(xd->counts, r, coef_tail_cdfs[band][ctx],
TAIL_TOKENS, ACCT_STR);
#if CONFIG_NEW_QUANT
#if CONFIG_NEW_QUANT && !CONFIG_AOM_QM
dqv_val = &dq_val[band != 0][0];
#endif // CONFIG_NEW_QUANT
#endif // CONFIG_NEW_QUANT && !CONFIG_AOM_QM
*max_scan_line = AOMMAX(*max_scan_line, scan[c]);
token_cache[scan[c]] = av1_pt_energy_class[token];
......@@ -168,19 +179,19 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
av1_record_coeff(xd->counts, val);
#endif
#if CONFIG_NEW_QUANT
#if !CONFIG_DAALA_TX
v = av1_dequant_abscoeff_nuq(val, dqv, dqv_val, dq_shift);
#else
v = av1_dequant_abscoeff_nuq(val, dqv, dqv_val, 0);
#endif
#else
#if CONFIG_AOM_QM
// Apply quant matrix only for 2D transforms
if (IS_2D_TRANSFORM(tx_type) && iqmatrix != NULL)
dqv = ((iqmatrix[scan[c]] * (int)dqv) + (1 << (AOM_QM_BITS - 1))) >>
AOM_QM_BITS;
#endif
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
v = av1_dequant_abscoeff_nuq(val, dqv, dq_profile, band != 0, nq_shift);
#else
v = av1_dequant_abscoeff_nuq(val, dqv, dqv_val, nq_shift);
#endif // CONFIG_AOM_QM
#else
#if !CONFIG_DAALA_TX
v = (int)(((int64_t)val * dqv) >> dq_shift);
#else
......@@ -299,7 +310,11 @@ int av1_decode_block_tokens(AV1_COMMON *cm, MACROBLOCKD *const xd, int plane,
const int eob =
decode_coefs(xd, pd->plane_type, pd->dqcoeff, tx_size, tx_type, dequant,
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
dq, pd->seg_iqmatrix[seg_id],
#else
pd->seg_dequant_nuq_QTX[seg_id][dq],
#endif // CONFIG_AOM_QM
#else
#if CONFIG_AOM_QM
pd->seg_iqmatrix[seg_id],
......
This diff is collapsed.
......@@ -92,14 +92,14 @@ typedef struct {
DECLARE_ALIGNED(16, int16_t, y_dequant_Q3[QINDEX_RANGE][8]); // 8: SIMD width
DECLARE_ALIGNED(16, int16_t, u_dequant_Q3[QINDEX_RANGE][8]); // 8: SIMD width
DECLARE_ALIGNED(16, int16_t, v_dequant_Q3[QINDEX_RANGE][8]); // 8: SIMD width
#if CONFIG_NEW_QUANT
#if CONFIG_NEW_QUANT && !CONFIG_AOM_QM
DECLARE_ALIGNED(16, dequant_val_type_nuq,
y_dequant_val_nuq_QTX[QUANT_PROFILES][QINDEX_RANGE][2]);
DECLARE_ALIGNED(16, dequant_val_type_nuq,
u_dequant_val_nuq_QTX[QUANT_PROFILES][QINDEX_RANGE][2]);
DECLARE_ALIGNED(16, dequant_val_type_nuq,
v_dequant_val_nuq_QTX[QUANT_PROFILES][QINDEX_RANGE][2]);
#endif // CONFIG_NEW_QUANT
#endif // CONFIG_NEW_QUANT && !CONFIG_AOM_QM
} Dequants;
struct AV1_COMP;
......
......@@ -162,7 +162,9 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane,
#endif // CONFIG_AOM_QM
#if CONFIG_NEW_QUANT
int dq = get_dq_profile(cm->dq_type, mb->qindex, ref, plane_type);
#if !CONFIG_AOM_QM
const dequant_val_type_nuq *dequant_val = p->dequant_val_nuq_QTX[dq];
#endif // !CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
int64_t rd_cost0, rd_cost1;
int16_t t0, t1;
......@@ -255,16 +257,25 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane,
if (x_a != 0) {
#if CONFIG_DAALA_TX
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
dx = av1_dequant_coeff_nuq(x_a, dqv, dq, rc != 0, 0) - coeff[rc];
#else
dx = av1_dequant_coeff_nuq(x_a, dqv, dequant_val[rc != 0], 0) -
coeff[rc];
#endif // CONFIG_AOM_QM
#else // CONFIG_NEW_QUANT
dx -= (dqv + sz) ^ sz;
#endif // CONFIG_NEW_QUANT
d2_a = ((int64_t)dx * dx + depth_round) >> depth_shift;
#else // CONFIG_DAALA_TX
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
dx = av1_dequant_coeff_nuq(x_a, dqv, dq, rc != 0, 0) -
(coeff[rc] * (1 << shift));
#else
dx = av1_dequant_coeff_nuq(x_a, dqv, dequant_val[rc != 0], 0) -
(coeff[rc] * (1 << shift));
#endif // CONFIG_AOM_QM
dx >>= xd->bd - 8;
#else // CONFIG_NEW_QUANT
dx -= ((dqv >> (xd->bd - 8)) + sz) ^ sz;
......@@ -345,16 +356,24 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane,
if (x_a != 0) {
#if CONFIG_DAALA_TX
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
dqc_a = av1_dequant_abscoeff_nuq(abs(x_a), dqv, dq, rc != 0, 0);
#else
dqc_a =
av1_dequant_abscoeff_nuq(abs(x_a), dqv, dequant_val[rc != 0], 0);
#endif // CONFIG_AOM_QM
if (sz) dqc_a = -dqc_a;
#else
dqc_a = x_a * dqv;
#endif // CONFIG_NEW_QUANT
#else // CONFIG_DAALA_TX
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
dqc_a = av1_dequant_abscoeff_nuq(abs(x_a), dqv, dq, rc != 0, shift);
#else
dqc_a = av1_dequant_abscoeff_nuq(abs(x_a), dqv, dequant_val[rc != 0],
shift);
#endif // CONFIG_AOM_QM
if (sz) dqc_a = -dqc_a;
#else
if (x_a < 0)
......
......@@ -134,12 +134,20 @@ static INLINE tran_low_t get_lower_coeff(tran_low_t qc) {
static INLINE tran_low_t qcoeff_to_dqcoeff(tran_low_t qc,
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
int dq_idx, int is_ac_coeff,
#else
const tran_low_t *nq_dq,
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
int dqv, int shift) {
int sgn = qc < 0 ? -1 : 1;
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
int dqcoeff = av1_dequant_coeff_nuq(abs(qc), dqv, dq_idx, is_ac_coeff, shift);
#else
int dqcoeff = av1_dequant_coeff_nuq(abs(qc), dqv, nq_dq, shift);
#endif // CONFIG_AOM_QM
return sgn * dqcoeff;
#endif // CONFIG_NEW_QUANT
......@@ -278,7 +286,11 @@ static void get_dist_cost_stats(LevelDownStats *const stats, const int scan_idx,
const tran_low_t tqc = txb_info->tcoeff[coeff_idx];
const int dqv = txb_info->dequant[coeff_idx != 0];
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
const int dq_idx = txb_info->dq_idx;
#else
const tran_low_t *nq_dequant_val = txb_info->nq_dequant_vals[coeff_idx != 0];
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
const int coeff_ctx =
......@@ -293,7 +305,11 @@ static void get_dist_cost_stats(LevelDownStats *const stats, const int scan_idx,
} else {
const tran_low_t dqc = qcoeff_to_dqcoeff(qc,
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
dq_idx, coeff_idx != 0,
#else
nq_dequant_val,
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
dqv, txb_info->shift);
const int64_t dqc_dist = get_coeff_dist(tqc, dqc, txb_info->shift);
......@@ -301,7 +317,11 @@ static void get_dist_cost_stats(LevelDownStats *const stats, const int scan_idx,
// distortion difference when coefficient is quantized to 0
const tran_low_t dqc0 = qcoeff_to_dqcoeff(0,
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
dq_idx, coeff_idx != 0,
#else
nq_dequant_val,
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
dqv, txb_info->shift);
......@@ -322,7 +342,11 @@ static void get_dist_cost_stats(LevelDownStats *const stats, const int scan_idx,
} else {
stats->low_dqc = qcoeff_to_dqcoeff(stats->low_qc,
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
dq_idx, coeff_idx != 0,
#else
nq_dequant_val,
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
dqv, txb_info->shift);
const int64_t low_dqc_dist =
......@@ -366,11 +390,19 @@ static INLINE void update_coeff(const int coeff_idx, const tran_low_t qc,
update_qcoeff(coeff_idx, qc, txb_info);
const int dqv = txb_info->dequant[coeff_idx != 0];
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
const int dq_idx = txb_info->dq_idx;
#else
const tran_low_t *nq_dequant_val = txb_info->nq_dequant_vals[coeff_idx != 0];
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
txb_info->dqcoeff[coeff_idx] = qcoeff_to_dqcoeff(qc,
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
dq_idx, coeff_idx != 0,
#else
nq_dequant_val,
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
dqv, txb_info->shift);
}
......@@ -2024,7 +2056,9 @@ int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
const LV_MAP_COEFF_COST txb_costs = x->coeff_costs[txs_ctx][plane_type];
#if CONFIG_NEW_QUANT
int dq = get_dq_profile(cm->dq_type, x->qindex, is_inter, plane_type);
#if !CONFIG_AOM_QM
const dequant_val_type_nuq *dequant_val = p->dequant_val_nuq_QTX[dq];
#endif // !CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
const int eob_multi_size = txsize_log2_minus4[tx_size];
const LV_MAP_EOB_COST txb_eob_costs =
......@@ -2050,7 +2084,11 @@ int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
tcoeff,
dequant,
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
dq,
#else
dequant_val,
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
shift,
tx_size,
......
......@@ -30,7 +30,11 @@ typedef struct TxbInfo {
const tran_low_t *tcoeff;
const int16_t *dequant;
#if CONFIG_NEW_QUANT
#if CONFIG_AOM_QM
const int dq_idx;
#else
const dequant_val_type_nuq *nq_dequant_vals;
#endif // CONFIG_AOM_QM
#endif // CONFIG_NEW_QUANT
int shift;
TX_SIZE tx_size;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment