Commit 8468a5c1 authored by Jingning Han's avatar Jingning Han

Fix the transform type selection in 4x4 partition

This commit allows proper transform type (DCT/ADST) selection in
the settings of partition 4x4 level.

Change-Id: Iec6f922a46480d777e7ca9142a99e8c131f0077b
parent cd16fe91
......@@ -625,13 +625,11 @@ static TX_TYPE get_tx_type_4x4(const MACROBLOCKD *xd, int ib) {
// is smaller than the prediction size
TX_TYPE tx_type = DCT_DCT;
const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;
const int wb = b_width_log2(sb_type), hb = b_height_log2(sb_type);
const int wb = b_width_log2(sb_type);
#if !USE_ADST_FOR_SB
if (sb_type > BLOCK_SIZE_MB16X16)
return tx_type;
#endif
if (ib >= (1 << (wb + hb))) // no chroma adst
return tx_type;
if (xd->lossless)
return DCT_DCT;
if (xd->mode_info_context->mbmi.mode == I4X4_PRED &&
......
......@@ -2331,8 +2331,7 @@ void vp9_rd_pick_intra_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
*returnrate = rate_y + rate_uv - rate_y_tokenonly - rate_uv_tokenonly +
vp9_cost_bit(vp9_get_pred_prob(cm, xd, PRED_MBSKIP), 1);
*returndist = dist_y + (dist_uv >> 2);
memset(ctx->txfm_rd_diff, 0,
sizeof(x->sb32_context[xd->sb_index].txfm_rd_diff));
memset(ctx->txfm_rd_diff, 0, sizeof(ctx->txfm_rd_diff));
xd->mode_info_context->mbmi.mode = mode;
xd->mode_info_context->mbmi.txfm_size = txfm_size;
#if CONFIG_AB4X4
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment