Commit 077e4669 authored by Sarah Parker's avatar Sarah Parker
Browse files

Use new-quant lookup table in lv-map deqantization

When lv-map and new-quant are enabled together,
av1_read_coeffs_txb currently does not use the new-quant
lookup table to get the dequantized value, which causes an
encode/decode mismatch. This patch fixes the function to
make lv-map work with new-quant.

Change-Id: I02a5521c3b412ae1c0b960cead7acd14a835c1fe
parent 56d8b95f
......@@ -56,6 +56,9 @@ static INLINE int rec_eob_pos(const int eob_token, const int extra) {
uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
aom_reader *const r, const int blk_row,
const int blk_col, const int plane,
#if CONFIG_NEW_QUANT
dequant_val_type_nuq *dq_val,
#endif // CONFIG_NEW_QUANT
const TXB_CTX *const txb_ctx, const TX_SIZE tx_size,
int16_t *const max_scan_line, int *const eob) {
FRAME_CONTEXT *const ec_ctx = xd->tile_ctx;
......@@ -68,11 +71,14 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
const PLANE_TYPE plane_type = get_plane_type(plane);
MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
const int seg_eob = av1_get_max_eob(tx_size);
int c = 0;
int c = 0, v = 0;
int num_updates = 0;
struct macroblockd_plane *const pd = &xd->plane[plane];
const int16_t *const dequant = pd->seg_dequant_QTX[mbmi->segment_id];
tran_low_t *const tcoeffs = pd->dqcoeff;
#if CONFIG_NEW_QUANT
const tran_low_t *dqv_val = &dq_val[0][0];
#endif // CONFIG_NEW_QUANT
#if !CONFIG_DAALA_TX
const int shift = av1_get_tx_scale(tx_size);
#endif
......@@ -207,11 +213,19 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
*max_scan_line = AOMMAX(*max_scan_line, pos);
if (level < 3) {
cul_level += level;
#if CONFIG_DAALA_TX
tcoeffs[pos] = level * dequant[!!c];
#if CONFIG_NEW_QUANT
dqv_val = &dq_val[pos != 0][0];
v = av1_dequant_abscoeff_nuq(level, dequant[!!c], dqv_val);
#if !CONFIG_DAALA_TX
v = shift ? ROUND_POWER_OF_TWO(v, shift) : v;
#endif // !CONFIG_DAALA_TX
#else
tcoeffs[pos] = (level * dequant[!!c]) >> shift;
#endif
v = level * dequant[!!c];
#if !CONFIG_DAALA_TX
v = v >> shift;
#endif // !CONFIG_DAALA_TX
#endif // CONFIG_NEW_QUANT
tcoeffs[pos] = v;
} else {
update_pos[num_updates++] = pos;
}
......@@ -240,11 +254,19 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
// semantic: is_k = 1 if level > (k+1)
if (is_k == 0) {
cul_level += k + 1;
#if CONFIG_DAALA_TX
tcoeffs[pos] = (k + 1) * dequant[!!c];
#if CONFIG_NEW_QUANT
dqv_val = &dq_val[pos != 0][0];
v = av1_dequant_abscoeff_nuq(k + 1, dequant[!!c], dqv_val);
#if !CONFIG_DAALA_TX
v = shift ? ROUND_POWER_OF_TWO(v, shift) : v;
#endif // !CONFIG_DAALA_TX
#else
tcoeffs[pos] = ((k + 1) * dequant[!!c]) >> shift;
#endif
v = (k + 1) * dequant[!!c];
#if !CONFIG_DAALA_TX
v = v >> shift;
#endif // !CONFIG_DAALA_TX
#endif // CONFIG_NEW_QUANT
tcoeffs[pos] = v;
break;
}
}
......@@ -329,11 +351,19 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
}
if (*level <= NUM_BASE_LEVELS + COEFF_BASE_RANGE) {
cul_level += *level;
#if CONFIG_DAALA_TX
tran_low_t t = *level * dequant[!!pos];
tran_low_t t;
#if CONFIG_NEW_QUANT
dqv_val = &dq_val[pos != 0][0];
t = av1_dequant_abscoeff_nuq(*level, dequant[!!pos], dqv_val);
#if !CONFIG_DAALA_TX
t = shift ? ROUND_POWER_OF_TWO(t, shift) : t;
#endif // !CONFIG_DAALA_TX
#else
tran_low_t t = (*level * dequant[!!pos]) >> shift;
#endif
t = *level * dequant[!!pos];
#if !CONFIG_DAALA_TX
t = t >> shift;
#endif // !CONFIG_DAALA_TX
#endif // CONFIG_NEW_QUANT
if (signs[pos]) t = -t;
tcoeffs[pos] = t;
continue;
......@@ -365,11 +395,19 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
*level = NUM_BASE_LEVELS + 1 + br_base + br_offset;
cul_level += *level;
#if CONFIG_DAALA_TX
tran_low_t t = *level * dequant[!!pos];
tran_low_t t;
#if CONFIG_NEW_QUANT
dqv_val = &dq_val[pos != 0][0];
t = av1_dequant_abscoeff_nuq(*level, dequant[!!pos], dqv_val);
#if !CONFIG_DAALA_TX
t = shift ? ROUND_POWER_OF_TWO(t, shift) : t;
#endif // !CONFIG_DAALA_TX
#else
tran_low_t t = (*level * dequant[!!pos]) >> shift;
#endif
t = *level * dequant[!!pos];
#if !CONFIG_DAALA_TX
t = t >> shift;
#endif // !CONFIG_DAALA_TX
#endif // CONFIG_NEW_QUANT
if (signs[pos]) t = -t;
tcoeffs[pos] = t;
break;
......@@ -384,11 +422,18 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
// Save golomb in tcoeffs because adding it to level may incur overflow
tran_low_t t = *level + read_golomb(xd, r, counts);
cul_level += t;
#if CONFIG_DAALA_TX
t = t * dequant[!!pos];
#if CONFIG_NEW_QUANT
dqv_val = &dq_val[pos != 0][0];
t = av1_dequant_abscoeff_nuq(t, dequant[!!pos], dqv_val);
#if !CONFIG_DAALA_TX
t = shift ? ROUND_POWER_OF_TWO(t, shift) : t;
#endif // !CONFIG_DAALA_TX
#else
t = (t * dequant[!!pos]) >> shift;
#endif
t = t * dequant[!!pos];
#if !CONFIG_DAALA_TX
t = t >> shift;
#endif // !CONFIG_DAALA_TX
#endif // CONFIG_NEW_QUANT
if (signs[pos]) t = -t;
tcoeffs[pos] = t;
}
......@@ -413,12 +458,21 @@ uint8_t av1_read_coeffs_txb_facade(const AV1_COMMON *const cm,
const BLOCK_SIZE bsize = mbmi->sb_type;
const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
#if CONFIG_NEW_QUANT
const int seg_id = mbmi->segment_id;
const int ref = is_inter_block(mbmi);
int dq = get_dq_profile(xd->qindex[seg_id], ref, pd->plane_type);
#endif // CONFIG_NEW_QUANT
TXB_CTX txb_ctx;
get_txb_ctx(plane_bsize, tx_size, plane, pd->above_context + col,
pd->left_context + row, &txb_ctx);
uint8_t cul_level = av1_read_coeffs_txb(cm, xd, r, row, col, plane, &txb_ctx,
tx_size, max_scan_line, eob);
uint8_t cul_level =
av1_read_coeffs_txb(cm, xd, r, row, col, plane,
#if CONFIG_NEW_QUANT
pd->seg_dequant_nuq_QTX[seg_id][dq],
#endif // CONFIG_NEW_QUANT
&txb_ctx, tx_size, max_scan_line, eob);
av1_set_contexts(xd, pd, plane, tx_size, cul_level, col, row);
return cul_level;
}
......@@ -21,6 +21,9 @@
uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
aom_reader *const r, const int blk_row,
const int blk_col, const int plane,
#if CONFIG_NEW_QUANT
dequant_val_type_nuq *dq_val,
#endif // CONFIG_NEW_QUANT
const TXB_CTX *const txb_ctx, const TX_SIZE tx_size,
int16_t *const max_scan_line, int *const eob);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment