Commit d7d78c83 authored by David Barker's avatar David Barker

Fix a bug when combining new-quant + supertx

Previously, we assumed that av1_init_plane_quantizers is always called with
segment_id == xd->mi[0]->mbmi.segment_id (and use the latter to derive the value
of 'qindex' to use in the quantizer). But this is no longer true when supertx
is enabled. This patch instead remembers the value of 'qindex' derived from
the latest call to av1_init_plane_quantizers and uses that directly.

Change-Id: Ifa1c5bf74cad29942ff79b88ca92c231bc07f336
parent d30a563d
......@@ -81,7 +81,7 @@ struct macroblock {
MB_MODE_INFO_EXT *mbmi_ext;
int skip_block;
int select_tx_size;
int q_index;
int qindex;
// The equivalent error at the current rdmult of one whole bit (not one
// bitcost unit).
......
......@@ -852,7 +852,6 @@ static void choose_partitioning(AV1_COMP *const cpi, ThreadData *const td,
if (cyclic_refresh_segment_id_boosted(segment_id)) {
int q = av1_get_qindex(&cm->seg, segment_id, cm->base_qindex);
assert(q == xd->qindex[segment_id]);
set_vbp_thresholds(cpi, thresholds, q);
}
}
......@@ -1606,7 +1605,6 @@ static int set_segment_rdmult(const AV1_COMP *const cpi, MACROBLOCK *const x,
av1_init_plane_quantizers(cpi, x, segment_id);
aom_clear_system_state();
segment_qindex = av1_get_qindex(&cm->seg, segment_id, cm->base_qindex);
assert(segment_qindex == x->e_mbd.qindex[segment_id]);
return av1_compute_rd_mult(cpi, segment_qindex + cm->y_dc_delta_q);
}
......@@ -4675,7 +4673,6 @@ static void encode_frame_internal(AV1_COMP *cpi) {
: cm->base_qindex;
xd->lossless[i] = qindex == 0 && cm->y_dc_delta_q == 0 &&
cm->uv_dc_delta_q == 0 && cm->uv_ac_delta_q == 0;
xd->qindex[i] = qindex;
}
if (!cm->seg.enabled && xd->lossless[0]) x->optimize = 0;
......
......@@ -95,8 +95,7 @@ int av1_optimize_b(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, int block,
#endif
const int shift = get_tx_scale(xd, tx_type, tx_size);
#if CONFIG_NEW_QUANT
int dq = get_dq_profile_from_ctx(xd->qindex[xd->mi[0]->mbmi.segment_id], ctx,
ref, plane_type);
int dq = get_dq_profile_from_ctx(mb->qindex, ctx, ref, plane_type);
const dequant_val_type_nuq *dequant_val = pd->dequant_val_nuq[dq];
#else
const int dq_step[2] = { dequant_ptr[0] >> shift, dequant_ptr[1] >> shift };
......@@ -123,8 +122,7 @@ int av1_optimize_b(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, int block,
int shortcut = 0;
int next_shortcut = 0;
assert((xd->qindex[xd->mi[0]->mbmi.segment_id] == 0) ^
(xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
assert((mb->qindex == 0) ^ (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
token_costs += band;
......@@ -518,8 +516,7 @@ void av1_xform_quant_nuq(const AV1_COMMON *cm, MACROBLOCK *x, int plane,
tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
int dq = get_dq_profile_from_ctx(xd->qindex[xd->mi[0]->mbmi.segment_id], ctx,
is_inter, plane_type);
int dq = get_dq_profile_from_ctx(x->qindex, ctx, is_inter, plane_type);
uint16_t *const eob = &p->eobs[block];
const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
const int16_t *src_diff;
......@@ -527,8 +524,7 @@ void av1_xform_quant_nuq(const AV1_COMMON *cm, MACROBLOCK *x, int plane,
FWD_TXFM_PARAM fwd_txfm_param;
assert((xd->qindex[xd->mi[0]->mbmi.segment_id] == 0) ^
(xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
assert((x->qindex == 0) ^ (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
fwd_txfm_param.tx_type = tx_type;
fwd_txfm_param.tx_size = tx_size;
......@@ -588,8 +584,7 @@ void av1_xform_quant_fp_nuq(const AV1_COMMON *cm, MACROBLOCK *x, int plane,
PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
TX_TYPE tx_type = get_tx_type(plane_type, xd, block, tx_size);
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, is_inter);
int dq = get_dq_profile_from_ctx(xd->qindex[xd->mi[0]->mbmi.segment_id], ctx,
is_inter, plane_type);
int dq = get_dq_profile_from_ctx(x->qindex, ctx, is_inter, plane_type);
tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
......@@ -600,8 +595,7 @@ void av1_xform_quant_fp_nuq(const AV1_COMMON *cm, MACROBLOCK *x, int plane,
FWD_TXFM_PARAM fwd_txfm_param;
assert((xd->qindex[xd->mi[0]->mbmi.segment_id] == 0) ^
(xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
assert((x->qindex == 0) ^ (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
fwd_txfm_param.tx_type = tx_type;
fwd_txfm_param.tx_size = tx_size;
......@@ -665,13 +659,11 @@ void av1_xform_quant_dc_nuq(MACROBLOCK *x, int plane, int block, int blk_row,
const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
const int16_t *src_diff;
const int is_inter = is_inter_block(&xd->mi[0]->mbmi);
int dq = get_dq_profile_from_ctx(xd->qindex[xd->mi[0]->mbmi.segment_id], ctx,
is_inter, plane_type);
int dq = get_dq_profile_from_ctx(x->qindex, ctx, is_inter, plane_type);
FWD_TXFM_PARAM fwd_txfm_param;
assert((xd->qindex[xd->mi[0]->mbmi.segment_id] == 0) ^
(xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
assert((x->qindex == 0) ^ (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
fwd_txfm_param.tx_type = tx_type;
fwd_txfm_param.tx_size = tx_size;
......@@ -730,13 +722,11 @@ void av1_xform_quant_dc_fp_nuq(MACROBLOCK *x, int plane, int block, int blk_row,
const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
const int16_t *src_diff;
const int is_inter = is_inter_block(&xd->mi[0]->mbmi);
int dq = get_dq_profile_from_ctx(xd->qindex[xd->mi[0]->mbmi.segment_id], ctx,
is_inter, plane_type);
int dq = get_dq_profile_from_ctx(x->qindex, ctx, is_inter, plane_type);
FWD_TXFM_PARAM fwd_txfm_param;
assert((xd->qindex[xd->mi[0]->mbmi.segment_id] == 0) ^
(xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
assert((x->qindex == 0) ^ (xd->lossless[xd->mi[0]->mbmi.segment_id] == 0));
fwd_txfm_param.tx_type = tx_type;
fwd_txfm_param.tx_size = tx_size;
......
......@@ -585,7 +585,6 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) {
#if CONFIG_SUPERTX
xd->mi[0]->mbmi.segment_id_supertx = 0;
#endif // CONFIG_SUPERTX
xd->qindex[xd->mi[0]->mbmi.segment_id] = qindex;
xd->lossless[xd->mi[0]->mbmi.segment_id] = (qindex == 0);
xd->mi[0]->mbmi.mode = DC_PRED;
xd->mi[0]->mbmi.tx_size =
......
......@@ -1293,11 +1293,11 @@ void av1_init_plane_quantizers(const AV1_COMP *cpi, MACROBLOCK *x,
}
x->skip_block = segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP);
x->q_index = qindex;
x->qindex = qindex;
set_error_per_bit(x, rdmult);
av1_initialize_me_consts(cpi, x, x->q_index);
av1_initialize_me_consts(cpi, x, qindex);
}
void av1_frame_init_quantizer(AV1_COMP *cpi) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment