Commit ebecbd3d authored by Urvang Joshi's avatar Urvang Joshi

optimize_b_greedy: Fix for TX64X64.

Change-Id: Iff2780944e954973b9ee9ab3c04d1e6b44a7810d
parent a3eb912b
...@@ -157,11 +157,19 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, ...@@ -157,11 +157,19 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane,
#endif #endif
#if CONFIG_AOM_QM #if CONFIG_AOM_QM
int seg_id = xd->mi[0]->mbmi.segment_id; int seg_id = xd->mi[0]->mbmi.segment_id;
const TX_SIZE qm_tx_size =
#if CONFIG_TX64X64
tx_size == TX_64X64 || tx_size == TX_64X32 || tx_size == TX_32X64
? TX_32X32
:
#endif // CONFIG_TX64X64
tx_size;
// Use a flat matrix (i.e. no weighting) for 1D and Identity transforms // Use a flat matrix (i.e. no weighting) for 1D and Identity transforms
const qm_val_t *iqmatrix = IS_2D_TRANSFORM(tx_type) const qm_val_t *iqmatrix =
? pd->seg_iqmatrix[seg_id][tx_size] IS_2D_TRANSFORM(tx_type)
: cm->giqmatrix[NUM_QM_LEVELS - 1][0][tx_size]; ? pd->seg_iqmatrix[seg_id][qm_tx_size]
#endif : cm->giqmatrix[NUM_QM_LEVELS - 1][0][qm_tx_size];
#endif // CONFIG_AOM_QM
#if CONFIG_NEW_QUANT #if CONFIG_NEW_QUANT
int dq = get_dq_profile_from_ctx(mb->qindex, ctx, ref, plane_type); int dq = get_dq_profile_from_ctx(mb->qindex, ctx, ref, plane_type);
const dequant_val_type_nuq *dequant_val = p->dequant_val_nuq_QTX[dq]; const dequant_val_type_nuq *dequant_val = p->dequant_val_nuq_QTX[dq];
...@@ -233,17 +241,13 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, ...@@ -233,17 +241,13 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane,
const int64_t d0 = (int64_t)dx0 * dx0; const int64_t d0 = (int64_t)dx0 * dx0;
#endif #endif
const int x_a = x - 2 * sz - 1; const int x_a = x - 2 * sz - 1;
int dqv; int dqv = dequant_ptr[rc != 0];
#if CONFIG_AOM_QM #if CONFIG_AOM_QM
int iwt;
dqv = dequant_ptr[rc != 0];
if (iqmatrix != NULL) { if (iqmatrix != NULL) {
iwt = iqmatrix[rc]; const qm_val_t iwt = iqmatrix[rc];
dqv = ((iwt * (int)dqv) + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS; dqv = ((iwt * (int)dqv) + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;
} }
#else #endif // CONFIG_AOM_QM
dqv = dequant_ptr[rc != 0];
#endif
#if CONFIG_DAALA_TX #if CONFIG_DAALA_TX
int dx = dqcoeff[rc] - coeff[rc]; int dx = dqcoeff[rc] - coeff[rc];
...@@ -524,7 +528,7 @@ void av1_xform_quant(const AV1_COMMON *cm, MACROBLOCK *x, int plane, int block, ...@@ -524,7 +528,7 @@ void av1_xform_quant(const AV1_COMMON *cm, MACROBLOCK *x, int plane, int block,
IS_2D_TRANSFORM(tx_type) IS_2D_TRANSFORM(tx_type)
? pd->seg_iqmatrix[seg_id][qm_tx_size] ? pd->seg_iqmatrix[seg_id][qm_tx_size]
: cm->giqmatrix[NUM_QM_LEVELS - 1][0][qm_tx_size]; : cm->giqmatrix[NUM_QM_LEVELS - 1][0][qm_tx_size];
#endif #endif // CONFIG_AOM_QM
TxfmParam txfm_param; TxfmParam txfm_param;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment