Commit 5b8c5fdc authored by Ryan's avatar Ryan Committed by Sarah Parker

fix bug to use b_quantizer when trellis quantization is disabled.

when trellis quantizaiton is disabled, the b_quantizer should be used.
existing code has a lot of hard coded flag not hooked up correctly with
high level trellis quantization enable/disable flag, which causes
fp_quantizer is used. this patch is intended to remove the hard coded
flags and correctly get them from the high level control.

Change-Id: I0142aace03586d07aedea138fce08d3e72d99c32
parent 51fac92a
......@@ -4467,7 +4467,8 @@ static void encode_superblock(const AV1_COMP *const cpi, TileDataEnc *tile_data,
#endif // CONFIG_CFL
mbmi->skip = 1;
for (int plane = 0; plane < num_planes; ++plane) {
av1_encode_intra_block_plane(cpi, x, bsize, plane, 1, mi_row, mi_col);
av1_encode_intra_block_plane(cpi, x, bsize, plane, x->optimize, mi_row,
mi_col);
}
#if CONFIG_CFL
xd->cfl.store_y = 0;
......
......@@ -749,7 +749,8 @@ void av1_encode_sb(const AV1_COMP *const cpi, MACROBLOCK *x, BLOCK_SIZE bsize,
MACROBLOCKD *const xd = &x->e_mbd;
struct optimize_ctx ctx;
MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
struct encode_b_args arg = { cpi, x, &ctx, &mbmi->skip, NULL, NULL, 1 };
struct encode_b_args arg = { cpi, x, &ctx, &mbmi->skip,
NULL, NULL, x->optimize };
int plane;
mbmi->skip = 1;
......
......@@ -1893,7 +1893,8 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
#if !CONFIG_TXK_SEL
// full forward transform and quantization
if (cpi->sf.optimize_coefficients == FINAL_PASS_TRELLIS_OPT) {
if (cpi->sf.optimize_coefficients == FINAL_PASS_TRELLIS_OPT ||
cpi->sf.optimize_coefficients == NO_TRELLIS_OPT) {
av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
AV1_XFORM_QUANT_B);
} else {
......@@ -3537,7 +3538,8 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
NULL, 0, bw, bh);
}
if (cpi->sf.optimize_coefficients == FINAL_PASS_TRELLIS_OPT) {
if (cpi->sf.optimize_coefficients == FINAL_PASS_TRELLIS_OPT ||
cpi->sf.optimize_coefficients == NO_TRELLIS_OPT) {
av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
AV1_XFORM_QUANT_B);
......@@ -8702,8 +8704,8 @@ void av1_rd_pick_intra_mode_sb(const AV1_COMP *cpi, MACROBLOCK *x, int mi_row,
// during luma RDO, so we can store reconstructed luma values
memcpy(x->blk_skip[0], ctx->blk_skip[0],
sizeof(uint8_t) * ctx->num_4x4_blk);
av1_encode_intra_block_plane(cpi, x, bsize, AOM_PLANE_Y, 1, mi_row,
mi_col);
av1_encode_intra_block_plane(cpi, x, bsize, AOM_PLANE_Y, x->optimize,
mi_row, mi_col);
xd->cfl.store_y = 0;
}
#endif // CONFIG_CFL
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment