Commit be44e512 authored by Thomas Davies's avatar Thomas Davies

AOM_QM: Fix bug with applying QMs to 1D and identity transforms.

Only apply quantisation matrices when the transform is 2D.

Change-Id: Iae9ac910c11199f7944a80d10d334db3b711059d
parent 85b66463
......@@ -237,6 +237,12 @@ typedef enum {
TX_TYPES,
} TX_TYPE;
#if CONFIG_EXT_TX
#define IS_2D_TRANSFORM(tx_type) (tx_type < IDTX)
#else
#define IS_2D_TRANSFORM(tx_type) 1
#endif
typedef enum {
TILE_LEFT_BOUNDARY = 1,
TILE_RIGHT_BOUNDARY = 2,
......
......@@ -120,6 +120,8 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
const int ref = is_inter_block(&xd->mi[0]->mbmi);
#if CONFIG_AOM_QM
const qm_val_t *iqmatrix = iqm[!ref][tx_size];
#else
(void)tx_type;
#endif // CONFIG_AOM_QM
int band, c = 0;
const int tx_size_ctx = txsize_sqr_map[tx_size];
......@@ -142,7 +144,6 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
#if CONFIG_NEW_QUANT
const tran_low_t *dqv_val = &dq_val[0][0];
#endif // CONFIG_NEW_QUANT
(void)tx_type;
if (counts) {
#if !CONFIG_EC_ADAPT
......@@ -226,8 +227,10 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
v = dq_shift ? ROUND_POWER_OF_TWO(v, dq_shift) : v;
#else
#if CONFIG_AOM_QM
dqv = ((iqmatrix[scan[c]] * (int)dqv) + (1 << (AOM_QM_BITS - 1))) >>
AOM_QM_BITS;
// Apply quant matrix only for 2D transforms
if (IS_2D_TRANSFORM(tx_type))
dqv = ((iqmatrix[scan[c]] * (int)dqv) + (1 << (AOM_QM_BITS - 1))) >>
AOM_QM_BITS;
#endif
v = (val * dqv) >> dq_shift;
#endif
......
......@@ -164,7 +164,11 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane,
const int shift = av1_get_tx_scale(tx_size);
#if CONFIG_AOM_QM
int seg_id = xd->mi[0]->mbmi.segment_id;
const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][!ref][tx_size];
// Use a flat matrix (i.e. no weighting) for 1D and Identity transforms
const qm_val_t *iqmatrix =
IS_2D_TRANSFORM(tx_type)
? pd->seg_iqmatrix[seg_id][!ref][tx_size]
: cm->giqmatrix[NUM_QM_LEVELS - 1][0][0][tx_size];
#endif
#if CONFIG_NEW_QUANT
int dq = get_dq_profile_from_ctx(mb->qindex, ctx, ref, plane_type);
......@@ -452,7 +456,6 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane,
mb->plane[plane].eobs[block] = final_eob;
return final_eob;
}
#endif // !CONFIG_LV_MAP
int av1_optimize_b(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, int block,
......@@ -556,8 +559,14 @@ void av1_xform_quant(const AV1_COMMON *cm, MACROBLOCK *x, int plane, int block,
const int diff_stride = block_size_wide[plane_bsize];
#if CONFIG_AOM_QM
int seg_id = mbmi->segment_id;
const qm_val_t *qmatrix = pd->seg_qmatrix[seg_id][!is_inter][tx_size];
const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][!is_inter][tx_size];
// Use a flat matrix (i.e. no weighting) for 1D and Identity transforms
const qm_val_t *qmatrix =
IS_2D_TRANSFORM(tx_type) ? pd->seg_qmatrix[seg_id][!is_inter][tx_size]
: cm->gqmatrix[NUM_QM_LEVELS - 1][0][0][tx_size];
const qm_val_t *iqmatrix =
IS_2D_TRANSFORM(tx_type)
? pd->seg_iqmatrix[seg_id][!is_inter][tx_size]
: cm->giqmatrix[NUM_QM_LEVELS - 1][0][0][tx_size];
#endif
FWD_TXFM_PARAM fwd_txfm_param;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment