diff --git a/av1/common/enums.h b/av1/common/enums.h index d105f943518ed03c3531d7eb16655f04c7e2536d..0437e088464bec29e00b934d331e2867f4242317 100644 --- a/av1/common/enums.h +++ b/av1/common/enums.h @@ -237,6 +237,12 @@ typedef enum { TX_TYPES, } TX_TYPE; +#if CONFIG_EXT_TX +#define IS_2D_TRANSFORM(tx_type) (tx_type < IDTX) +#else +#define IS_2D_TRANSFORM(tx_type) 1 +#endif + typedef enum { TILE_LEFT_BOUNDARY = 1, TILE_RIGHT_BOUNDARY = 2, diff --git a/av1/decoder/detokenize.c b/av1/decoder/detokenize.c index 2e3309c070025472bce94a71591770c18bfe34ea..b48e8b1c7b78cfea584ff729cbaa4e7a28213fbd 100644 --- a/av1/decoder/detokenize.c +++ b/av1/decoder/detokenize.c @@ -120,6 +120,8 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff, const int ref = is_inter_block(&xd->mi[0]->mbmi); #if CONFIG_AOM_QM const qm_val_t *iqmatrix = iqm[!ref][tx_size]; +#else + (void)tx_type; #endif // CONFIG_AOM_QM int band, c = 0; const int tx_size_ctx = txsize_sqr_map[tx_size]; @@ -142,7 +144,6 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff, #if CONFIG_NEW_QUANT const tran_low_t *dqv_val = &dq_val[0][0]; #endif // CONFIG_NEW_QUANT - (void)tx_type; if (counts) { #if !CONFIG_EC_ADAPT @@ -226,8 +227,10 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff, v = dq_shift ? ROUND_POWER_OF_TWO(v, dq_shift) : v; #else #if CONFIG_AOM_QM - dqv = ((iqmatrix[scan[c]] * (int)dqv) + (1 << (AOM_QM_BITS - 1))) >> - AOM_QM_BITS; + // Apply quant matrix only for 2D transforms + if (IS_2D_TRANSFORM(tx_type)) + dqv = ((iqmatrix[scan[c]] * (int)dqv) + (1 << (AOM_QM_BITS - 1))) >> + AOM_QM_BITS; #endif v = (val * dqv) >> dq_shift; #endif diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c index b4624a4e9f3130ccadd9f3674d1e144a6b490695..db630499d5c04870beaea1595a9d80b64da4ad0a 100644 --- a/av1/encoder/encodemb.c +++ b/av1/encoder/encodemb.c @@ -164,7 +164,11 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, const int shift = av1_get_tx_scale(tx_size); #if CONFIG_AOM_QM int seg_id = xd->mi[0]->mbmi.segment_id; - const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][!ref][tx_size]; + // Use a flat matrix (i.e. no weighting) for 1D and Identity transforms + const qm_val_t *iqmatrix = + IS_2D_TRANSFORM(tx_type) + ? pd->seg_iqmatrix[seg_id][!ref][tx_size] + : cm->giqmatrix[NUM_QM_LEVELS - 1][0][0][tx_size]; #endif #if CONFIG_NEW_QUANT int dq = get_dq_profile_from_ctx(mb->qindex, ctx, ref, plane_type); @@ -452,7 +456,6 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, mb->plane[plane].eobs[block] = final_eob; return final_eob; } - #endif // !CONFIG_LV_MAP int av1_optimize_b(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, int block, @@ -556,8 +559,14 @@ void av1_xform_quant(const AV1_COMMON *cm, MACROBLOCK *x, int plane, int block, const int diff_stride = block_size_wide[plane_bsize]; #if CONFIG_AOM_QM int seg_id = mbmi->segment_id; - const qm_val_t *qmatrix = pd->seg_qmatrix[seg_id][!is_inter][tx_size]; - const qm_val_t *iqmatrix = pd->seg_iqmatrix[seg_id][!is_inter][tx_size]; + // Use a flat matrix (i.e. no weighting) for 1D and Identity transforms + const qm_val_t *qmatrix = + IS_2D_TRANSFORM(tx_type) ? pd->seg_qmatrix[seg_id][!is_inter][tx_size] + : cm->gqmatrix[NUM_QM_LEVELS - 1][0][0][tx_size]; + const qm_val_t *iqmatrix = + IS_2D_TRANSFORM(tx_type) + ? pd->seg_iqmatrix[seg_id][!is_inter][tx_size] + : cm->giqmatrix[NUM_QM_LEVELS - 1][0][0][tx_size]; #endif FWD_TXFM_PARAM fwd_txfm_param;