From 4470af1913f79745cc61c2d7dca783bdadcfe675 Mon Sep 17 00:00:00 2001 From: Jingning Han Date: Mon, 6 Mar 2017 15:34:41 -0800 Subject: [PATCH] Properly force the transform size in aq-mode Make the transform size conform to the segment and frame level constraints. This fixes encoder and decoder mismatch issue in aq-mode 1 and 2. BUG=aomedia:131 Change-Id: I0784aa12aa49541085ce17ca82e77dcff151fd24 --- av1/encoder/encodeframe.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c index a92d39e60..e86a26ffc 100644 --- a/av1/encoder/encodeframe.c +++ b/av1/encoder/encodeframe.c @@ -1038,6 +1038,16 @@ static void update_global_motion_used(PREDICTION_MODE mode, BLOCK_SIZE bsize, } #endif // CONFIG_GLOBAL_MOTION +static void reset_tx_size(MACROBLOCKD *xd, MB_MODE_INFO *mbmi, + const TX_MODE tx_mode) { + if (xd->lossless[mbmi->segment_id]) { + mbmi->tx_size = TX_4X4; + } else if (tx_mode != TX_MODE_SELECT) { + mbmi->tx_size = + tx_size_from_tx_mode(mbmi->sb_type, tx_mode, is_inter_block(mbmi)); + } +} + static void update_state(const AV1_COMP *const cpi, ThreadData *td, PICK_MODE_CONTEXT *ctx, int mi_row, int mi_col, BLOCK_SIZE bsize, RUN_TYPE dry_run) { @@ -1104,12 +1114,14 @@ static void update_state(const AV1_COMP *const cpi, ThreadData *td, const uint8_t *const map = seg->update_map ? cpi->segmentation_map : cm->last_frame_seg_map; mi_addr->mbmi.segment_id = get_segment_id(cm, map, bsize, mi_row, mi_col); + reset_tx_size(xd, &mi_addr->mbmi, cm->tx_mode); } // Else for cyclic refresh mode update the segment map, set the segment id // and then update the quantizer. if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) { av1_cyclic_refresh_update_segment(cpi, &xd->mi[0]->mbmi, mi_row, mi_col, bsize, ctx->rate, ctx->dist, x->skip); + reset_tx_size(xd, &mi_addr->mbmi, cm->tx_mode); } } -- GitLab