Skip to content

GitLab

A
aom-rav1e
Commit 9a8a6a1b authored by Debargha Mukherjee's avatar Debargha Mukherjee
Browse files
Options

Some supertx fixes 

Fixes some of the issues introduced by a merge from master.

derflr: -0.893% BDRATE
hevcmr: -1.667% BDRATE

Change-Id: I4c5b40ec63a6f19521191d3c730af87db3c4bc00
parent 968bbc7b
Hide whitespace changes
Inline Side-by-side
Showing with 79 additions and 149 deletions
vp10/common/entropymode.c
View file @ 9a8a6a1b
......@@ -1220,27 +1220,9 @@ default_intra_ext_tx_prob[EXT_TX_SETS_INTRA][EXT_TX_SIZES]
},
},
};
#endif // CONFIG_EXT_TX
#if CONFIG_SUPERTX
static const vpx_prob default_supertx_prob[PARTITION_SUPERTX_CONTEXTS]
[TX_SIZES] = {
{ 1, 160, 160, 170 },
{ 1, 200, 200, 210 },
};
#endif // CONFIG_SUPERTX
// FIXME(someone) need real defaults here
static const struct segmentation_probs default_seg_probs = {
{ 128, 128, 128, 128, 128, 128, 128 },
{ 128, 128, 128 },
};
#if CONFIG_EXT_INTRA
static const vpx_prob default_ext_intra_probs[2] = {230, 230};
#endif // CONFIG_EXT_INTRA
#else
#if !CONFIG_EXT_TX
const vpx_tree_index vp10_ext_tx_tree[TREE_SIZE(TX_TYPES)] = {
-DCT_DCT, 2,
-ADST_ADST, 4,
......@@ -1260,7 +1242,25 @@ static const vpx_prob default_inter_ext_tx_prob[EXT_TX_SIZES]
{176, 85, 128},
{192, 85, 128},
};
#endif
#endif // CONFIG_EXT_TX
#if CONFIG_SUPERTX
static const vpx_prob default_supertx_prob[PARTITION_SUPERTX_CONTEXTS]
[TX_SIZES] = {
{ 1, 160, 160, 170 },
{ 1, 200, 200, 210 },
};
#endif // CONFIG_SUPERTX
// FIXME(someone) need real defaults here
static const struct segmentation_probs default_seg_probs = {
{ 128, 128, 128, 128, 128, 128, 128 },
{ 128, 128, 128 },
};
#if CONFIG_EXT_INTRA
static const vpx_prob default_ext_intra_probs[2] = {230, 230};
#endif // CONFIG_EXT_INTRA
static void init_mode_probs(FRAME_CONTEXT *fc) {
vp10_copy(fc->uv_mode_prob, default_uv_probs);
......
vp10/decoder/decodeframe.c
View file @ 9a8a6a1b
......@@ -1035,9 +1035,7 @@ static void set_offsets_topblock(VP10_COMMON *const cm, MACROBLOCKD *const xd,
static void set_param_topblock(VP10_COMMON *const cm, MACROBLOCKD *const xd,
BLOCK_SIZE bsize, int mi_row, int mi_col,
#if CONFIG_EXT_TX
int txfm,
#endif
int skip) {
const int bw = num_8x8_blocks_wide_lookup[bsize];
const int bh = num_8x8_blocks_high_lookup[bsize];
......@@ -1052,9 +1050,7 @@ static void set_param_topblock(VP10_COMMON *const cm, MACROBLOCKD *const xd,
for (y = 0; y < y_mis; ++y)
for (x = 0; x < x_mis; ++x) {
xd->mi[y * cm->mi_stride + x]->mbmi.skip = skip;
#if CONFIG_EXT_TX
xd->mi[y * cm->mi_stride + x]->mbmi.tx_type = txfm;
#endif
}
#if CONFIG_VAR_TX
xd->above_txfm_context = cm->above_txfm_context + mi_col;
......@@ -1779,9 +1775,7 @@ static void decode_partition(VP10Decoder *const pbi, MACROBLOCKD *const xd,
int skip = 0;
TX_SIZE supertx_size = b_width_log2_lookup[bsize];
const TileInfo *const tile = &xd->tile;
#if CONFIG_EXT_TX
int txfm = DCT_DCT;
#endif // CONFIG_EXT_TX
#endif // CONFIG_SUPERTX
if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols)
......@@ -1813,10 +1807,10 @@ static void decode_partition(VP10Decoder *const pbi, MACROBLOCKD *const xd,
set_skip_context(xd, mi_row, mi_col);
// Here skip is read without using any segment level feature
skip = read_skip_without_seg(cm, xd, r);
if (skip)
if (skip) {
reset_skip_context(xd, bsize);
} else {
#if CONFIG_EXT_TX
if (!skip) {
if (get_ext_tx_types(supertx_size, bsize, 1) > 1) {
int eset = get_ext_tx_set(supertx_size, bsize, 1);
if (eset > 0) {
......@@ -1826,8 +1820,15 @@ static void decode_partition(VP10Decoder *const pbi, MACROBLOCKD *const xd,
++xd->counts->inter_ext_tx[eset][supertx_size][txfm];
}
}
}
#else
if (supertx_size < TX_32X32) {
txfm = vpx_read_tree(r, vp10_ext_tx_tree,
cm->fc->inter_ext_tx_prob[supertx_size]);
if (xd->counts)
++xd->counts->inter_ext_tx[supertx_size][txfm];
}
#endif // CONFIG_EXT_TX
}
}
#endif // CONFIG_SUPERTX
if (!hbs) {
......@@ -1958,9 +1959,7 @@ static void decode_partition(VP10Decoder *const pbi, MACROBLOCKD *const xd,
int eobtotal = 0;
MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
set_offsets_topblock(cm, xd, tile, bsize, mi_row, mi_col);
#if CONFIG_EXT_TX
xd->mi[0]->mbmi.tx_type = txfm;
#endif
for (i = 0; i < MAX_MB_PLANE; ++i) {
const struct macroblockd_plane *const pd = &xd->plane[i];
const int num_4x4_w = pd->n4_w;
......@@ -1991,11 +1990,7 @@ static void decode_partition(VP10Decoder *const pbi, MACROBLOCKD *const xd,
if (!(subsize < BLOCK_8X8) && eobtotal == 0)
skip = 1;
}
set_param_topblock(cm, xd, bsize, mi_row, mi_col,
#if CONFIG_EXT_TX
txfm,
#endif
skip);
set_param_topblock(cm, xd, bsize, mi_row, mi_col, txfm, skip);
}
#endif // CONFIG_SUPERTX
......@@ -3184,7 +3179,9 @@ static void read_ext_tx_probs(FRAME_CONTEXT *fc, vpx_reader *r) {
}
}
}
#else
static void read_ext_tx_probs(FRAME_CONTEXT *fc, vpx_reader *r) {
int i, j, k;
if (vpx_read(r, GROUP_DIFF_UPDATE_PROB)) {
......@@ -3201,7 +3198,6 @@ static void read_ext_tx_probs(FRAME_CONTEXT *fc, vpx_reader *r) {
}
}
}
#endif // CONFIG_EXT_TX
#if CONFIG_SUPERTX
......
vp10/decoder/decodemv.c
View file @ 9a8a6a1b
......@@ -1326,6 +1326,9 @@ static void read_inter_frame_mode_info(VP10Decoder *const pbi,
#else
if (mbmi->tx_size < TX_32X32 &&
cm->base_qindex > 0 && !mbmi->skip &&
#if CONFIG_SUPERTX
!supertx_enabled &&
#endif // CONFIG_SUPERTX
!segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
FRAME_COUNTS *counts = xd->counts;
if (inter_block) {
......
vp10/encoder/bitstream.c
View file @ 9a8a6a1b
......@@ -429,7 +429,9 @@ static void update_ext_tx_probs(VP10_COMMON *cm, vpx_writer *w) {
}
}
}
#else
static void update_ext_tx_probs(VP10_COMMON *cm, vpx_writer *w) {
const int savings_thresh = vp10_cost_one(GROUP_DIFF_UPDATE_PROB) -
vp10_cost_zero(GROUP_DIFF_UPDATE_PROB);
......@@ -1413,7 +1415,8 @@ static void write_modes_sb(VP10_COMP *cpi,
if (supertx_enabled) {
vpx_write(w, xd->mi[0]->mbmi.skip, vp10_get_skip_prob(cm, xd));
#if CONFIG_EXT_TX
if (supertx_size <= TX_16X16 && !xd->mi[0]->mbmi.skip) {
if (get_ext_tx_types(supertx_size, bsize, 1) > 1 &&
!xd->mi[0]->mbmi.skip) {
int eset = get_ext_tx_set(supertx_size, bsize, 1);
if (eset > 0) {
vp10_write_token(
......@@ -1422,6 +1425,13 @@ static void write_modes_sb(VP10_COMP *cpi,
&ext_tx_inter_encodings[eset][xd->mi[0]->mbmi.tx_type]);
}
}
#else
if (supertx_size < TX_32X32 && !xd->mi[0]->mbmi.skip) {
vp10_write_token(
w, vp10_ext_tx_tree,
cm->fc->inter_ext_tx_prob[supertx_size],
&ext_tx_encodings[xd->mi[0]->mbmi.tx_type]);
}
#endif // CONFIG_EXT_TX
}
}
......
vp10/encoder/encodeframe.c
View file @ 9a8a6a1b
......@@ -82,9 +82,7 @@ static void rd_supertx_sb(VP10_COMP *cpi, ThreadData *td,
const TileInfo *const tile,
int mi_row, int mi_col, BLOCK_SIZE bsize,
int *tmp_rate, int64_t *tmp_dist,
#if CONFIG_EXT_TX
TX_TYPE *best_tx,
#endif
PC_TREE *pc_tree);
#endif // CONFIG_SUPERTX
......@@ -1391,9 +1389,7 @@ static void update_state_sb_supertx(VP10_COMP *cpi, ThreadData *td,
static void update_supertx_param(ThreadData *td,
PICK_MODE_CONTEXT *ctx,
#if CONFIG_EXT_TX
int best_tx,
#endif
TX_SIZE supertx_size) {
MACROBLOCK *const x = &td->mb;
......@@ -1401,17 +1397,13 @@ static void update_supertx_param(ThreadData *td,
memcpy(ctx->zcoeff_blk, x->zcoeff_blk[supertx_size],
sizeof(uint8_t) * ctx->num_4x4_blk);
ctx->skip = x->skip;
#if CONFIG_EXT_TX
ctx->mic.mbmi.tx_type = best_tx;
#endif // CONFIG_EXT_TX
}
static void update_supertx_param_sb(VP10_COMP *cpi, ThreadData *td,
int mi_row, int mi_col,
BLOCK_SIZE bsize,
#if CONFIG_EXT_TX
int best_tx,
#endif
TX_SIZE supertx_size, PC_TREE *pc_tree) {
VP10_COMMON *const cm = &cpi->common;
int bsl = b_width_log2_lookup[bsize], hbs = (1 << bsl) / 4;
......@@ -1424,64 +1416,44 @@ static void update_supertx_param_sb(VP10_COMP *cpi, ThreadData *td,
switch (partition) {
case PARTITION_NONE:
update_supertx_param(td, &pc_tree->none,
#if CONFIG_EXT_TX
best_tx,
#endif
supertx_size);
break;
case PARTITION_VERT:
update_supertx_param(td, &pc_tree->vertical[0],
#if CONFIG_EXT_TX
best_tx,
#endif
supertx_size);
if (mi_col + hbs < cm->mi_cols && bsize > BLOCK_8X8)
update_supertx_param(td, &pc_tree->vertical[1],
#if CONFIG_EXT_TX
best_tx,
#endif
supertx_size);
break;
case PARTITION_HORZ:
update_supertx_param(td, &pc_tree->horizontal[0],
#if CONFIG_EXT_TX
best_tx,
#endif
supertx_size);
if (mi_row + hbs < cm->mi_rows && bsize > BLOCK_8X8)
update_supertx_param(td, &pc_tree->horizontal[1],
#if CONFIG_EXT_TX
best_tx,
#endif
supertx_size);
break;
case PARTITION_SPLIT:
if (bsize == BLOCK_8X8) {
update_supertx_param(td, pc_tree->leaf_split[0],
#if CONFIG_EXT_TX
best_tx,
#endif
supertx_size);
} else {
update_supertx_param_sb(cpi, td, mi_row, mi_col, subsize,
#if CONFIG_EXT_TX
best_tx,
#endif
supertx_size, pc_tree->split[0]);
update_supertx_param_sb(cpi, td, mi_row, mi_col + hbs, subsize,
#if CONFIG_EXT_TX
best_tx,
#endif
supertx_size, pc_tree->split[1]);
update_supertx_param_sb(cpi, td, mi_row + hbs, mi_col, subsize,
#if CONFIG_EXT_TX
best_tx,
#endif
supertx_size, pc_tree->split[2]);
update_supertx_param_sb(cpi, td, mi_row + hbs, mi_col + hbs, subsize,
#if CONFIG_EXT_TX
best_tx,
#endif
supertx_size, pc_tree->split[3]);
}
break;
......@@ -2052,6 +2024,11 @@ static void encode_sb(VP10_COMP *cpi, ThreadData *td,
[xd->mi[0]->mbmi.tx_type];
}
}
#else
if (supertx_size < TX_32X32 &&
!xd->mi[0]->mbmi.skip) {
++td->counts->inter_ext_tx[supertx_size][xd->mi[0]->mbmi.tx_type];
}
#endif // CONFIG_EXT_TX
}
if (partition != PARTITION_SPLIT || bsize == BLOCK_8X8)
......@@ -3135,10 +3112,7 @@ static void rd_pick_partition(VP10_COMP *cpi, ThreadData *td,
sum_rdc.rdcost =
RDCOST(x->rdmult, x->rddiv, sum_rdc.rate, sum_rdc.dist);
if (is_inter_mode(pc_tree->leaf_split[0]->mic.mbmi.mode)) {
#if CONFIG_EXT_TX
TX_TYPE best_tx = DCT_DCT;
#endif
tmp_rate = sum_rate_nocoef;
tmp_dist = 0;
#if CONFIG_VAR_TX
......@@ -3151,9 +3125,7 @@ static void rd_pick_partition(VP10_COMP *cpi, ThreadData *td,
#endif // CONFIG_VAR_TX
rd_supertx_sb(cpi, td, tile_info, mi_row, mi_col, bsize,
&tmp_rate, &tmp_dist,
#if CONFIG_EXT_TX
&best_tx,
#endif
pc_tree);
tmp_rate += vp10_cost_bit(
......@@ -3166,9 +3138,7 @@ static void rd_pick_partition(VP10_COMP *cpi, ThreadData *td,
sum_rdc.rate = tmp_rate;
sum_rdc.dist = tmp_dist;
update_supertx_param_sb(cpi, td, mi_row, mi_col, bsize,
#if CONFIG_EXT_TX
best_tx,
#endif
supertx_size, pc_tree);
}
}
......@@ -3235,9 +3205,7 @@ static void rd_pick_partition(VP10_COMP *cpi, ThreadData *td,
RDCOST(x->rdmult, x->rddiv, sum_rdc.rate, sum_rdc.dist);
if (!check_intra_sb(cpi, tile_info, mi_row, mi_col, bsize, pc_tree)) {
#if CONFIG_EXT_TX
TX_TYPE best_tx = DCT_DCT;
#endif
tmp_rate = sum_rate_nocoef;
tmp_dist = 0;
......@@ -3251,9 +3219,7 @@ static void rd_pick_partition(VP10_COMP *cpi, ThreadData *td,
#endif // CONFIG_VAR_TX
rd_supertx_sb(cpi, td, tile_info, mi_row, mi_col, bsize,
&tmp_rate, &tmp_dist,
#if CONFIG_EXT_TX
&best_tx,
#endif
pc_tree);
tmp_rate += vp10_cost_bit(
......@@ -3266,9 +3232,7 @@ static void rd_pick_partition(VP10_COMP *cpi, ThreadData *td,
sum_rdc.rate = tmp_rate;
sum_rdc.dist = tmp_dist;
update_supertx_param_sb(cpi, td, mi_row, mi_col, bsize,
#if CONFIG_EXT_TX
best_tx,
#endif
supertx_size, pc_tree);
}
}
......@@ -3383,10 +3347,7 @@ static void rd_pick_partition(VP10_COMP *cpi, ThreadData *td,
sum_rdc.rdcost = RDCOST(x->rdmult, x->rddiv, sum_rdc.rate, sum_rdc.dist);
if (!check_intra_sb(cpi, tile_info, mi_row, mi_col, bsize, pc_tree)) {
#if CONFIG_EXT_TX
TX_TYPE best_tx = DCT_DCT;
#endif
tmp_rate = sum_rate_nocoef;
tmp_dist = 0;
#if CONFIG_VAR_TX
......@@ -3398,9 +3359,7 @@ static void rd_pick_partition(VP10_COMP *cpi, ThreadData *td,
#endif // CONFIG_VAR_TX
rd_supertx_sb(cpi, td, tile_info, mi_row, mi_col, bsize,
&tmp_rate, &tmp_dist,
#if CONFIG_EXT_TX
&best_tx,
#endif
pc_tree);
tmp_rate += vp10_cost_bit(
......@@ -3413,9 +3372,7 @@ static void rd_pick_partition(VP10_COMP *cpi, ThreadData *td,
sum_rdc.rate = tmp_rate;
sum_rdc.dist = tmp_dist;
update_supertx_param_sb(cpi, td, mi_row, mi_col, bsize,
#if CONFIG_EXT_TX
best_tx,
#endif
supertx_size, pc_tree);
}
}
......@@ -3518,9 +3475,7 @@ static void rd_pick_partition(VP10_COMP *cpi, ThreadData *td,
sum_rdc.rdcost = RDCOST(x->rdmult, x->rddiv, sum_rdc.rate, sum_rdc.dist);
if (!check_intra_sb(cpi, tile_info, mi_row, mi_col, bsize, pc_tree)) {
#if CONFIG_EXT_TX
TX_TYPE best_tx = DCT_DCT;
#endif
tmp_rate = sum_rate_nocoef;
tmp_dist = 0;
......@@ -3533,9 +3488,7 @@ static void rd_pick_partition(VP10_COMP *cpi, ThreadData *td,
#endif // CONFIG_VAR_TX
rd_supertx_sb(cpi, td, tile_info, mi_row, mi_col, bsize,
&tmp_rate, &tmp_dist,
#if CONFIG_EXT_TX
&best_tx,
#endif
pc_tree);
tmp_rate += vp10_cost_bit(
......@@ -3548,9 +3501,7 @@ static void rd_pick_partition(VP10_COMP *cpi, ThreadData *td,
sum_rdc.rate = tmp_rate;
sum_rdc.dist = tmp_dist;
update_supertx_param_sb(cpi, td, mi_row, mi_col, bsize,
#if CONFIG_EXT_TX
best_tx,
#endif
supertx_size, pc_tree);
}
}
......@@ -5107,9 +5058,7 @@ static void rd_supertx_sb(VP10_COMP *cpi, ThreadData *td,
const TileInfo *const tile,
int mi_row, int mi_col, BLOCK_SIZE bsize,
int *tmp_rate, int64_t *tmp_dist,
#if CONFIG_EXT_TX
TX_TYPE *best_tx,
#endif // CONFIG_EXT_TX
PC_TREE *pc_tree) {
VP10_COMMON *const cm = &cpi->common;
MACROBLOCK *const x = &td->mb;
......@@ -5120,14 +5069,14 @@ static void rd_supertx_sb(VP10_COMP *cpi, ThreadData *td,
uint8_t *dst_buf[3];
int dst_stride[3];
TX_SIZE tx_size;
#if CONFIG_EXT_TX
MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
TX_TYPE tx_type, best_tx_nostx = xd->mi[0]->mbmi.tx_type;
#if CONFIG_EXT_TX
int ext_tx_set;
#endif // CONFIG_EXT_TX
int tmp_rate_tx = 0, skip_tx = 0;
int64_t tmp_dist_tx = 0, rd_tx, bestrd_tx = INT64_MAX;
uint8_t tmp_zcoeff_blk = 0;
#endif // CONFIG_EXT_TX
update_state_sb_supertx(cpi, td, tile, mi_row, mi_col, bsize, 0, pc_tree);
vp10_setup_dst_planes(xd->plane, get_frame_new_buffer(cm),
......@@ -5140,9 +5089,7 @@ static void rd_supertx_sb(VP10_COMP *cpi, ThreadData *td,
0, bsize, bsize, dst_buf, dst_stride, pc_tree);
set_offsets(cpi, tile, x, mi_row, mi_col, bsize);
#if CONFIG_EXT_TX
*best_tx = DCT_DCT;
#endif
// chroma
skippable_uv = 1;
......@@ -5171,22 +5118,29 @@ static void rd_supertx_sb(VP10_COMP *cpi, ThreadData *td,
vp10_subtract_plane(x, bsize, 0);
#if CONFIG_EXT_TX
ext_tx_set = get_ext_tx_set(tx_size, bsize, 1);
#endif // CONFIG_EXT_TX
for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
#if CONFIG_EXT_TX
if (!ext_tx_used_inter[ext_tx_set][tx_type])
continue;
mbmi->tx_type = tx_type;
if (ext_tx_set == 1 &&
mbmi->tx_type >= DST_ADST && mbmi->tx_type < IDTX &&
*best_tx == DCT_DCT) {
tx_type >= DST_ADST && tx_type < IDTX && *best_tx == DCT_DCT) {
tx_type = IDTX - 1;
break;
continue;
}
#else
if (tx_size >= TX_32X32 && tx_type != DCT_DCT)
continue;
#endif // CONFIG_EXT_TX
mbmi->tx_type = tx_type;
vp10_txfm_rd_in_plane_supertx(x,
#if CONFIG_VAR_TX
cpi,
#endif
&this_rate, &this_dist, &pnskip,
&pnsse, INT64_MAX, 0, bsize, tx_size, 0);
#if CONFIG_EXT_TX
if (get_ext_tx_types(tx_size, bsize, 1) > 1 &&
!xd->lossless[xd->mi[0]->mbmi.segment_id] &&
this_rate != INT_MAX) {
......@@ -5194,6 +5148,13 @@ static void rd_supertx_sb(VP10_COMP *cpi, ThreadData *td,
this_rate += cpi->inter_tx_type_costs[ext_tx_set]
[mbmi->tx_size][mbmi->tx_type];
}
#else
if (tx_size < TX_32X32 &&
!xd->lossless[xd->mi[0]->mbmi.segment_id] &&
this_rate != INT_MAX) {
this_rate += cpi->inter_tx_type_costs[mbmi->tx_size][mbmi->tx_type];
}
#endif // CONFIG_EXT_TX
*tmp_rate = rate_uv + this_rate;
*tmp_dist = dist_uv + this_dist;
sse = sse_uv + pnsse;
......@@ -5228,34 +5189,5 @@ static void rd_supertx_sb(VP10_COMP *cpi, ThreadData *td,
*tmp_dist = tmp_dist_tx;
x->skip = skip_tx;
xd->mi[0]->mbmi.tx_type = best_tx_nostx;
#else // CONFIG_EXT_TX
vp10_txfm_rd_in_plane_supertx(x,
#if CONFIG_VAR_TX
cpi,
#endif
&this_rate, &this_dist, &pnskip, &pnsse,
INT64_MAX, 0, bsize, tx_size, 0);
*tmp_rate = rate_uv + this_rate;
*tmp_dist = dist_uv + this_dist;
sse = sse_uv + pnsse;
skippable = skippable_uv && pnskip;
if (skippable) {
*tmp_rate = vp10_cost_bit(vp10_get_skip_prob(cm, xd), 1);
x->skip = 1;
} else {
if (RDCOST(x->rdmult, x->rddiv, *tmp_rate, *tmp_dist)
< RDCOST(x->rdmult, x->rddiv, 0, sse)) {
*tmp_rate += vp10_cost_bit(vp10_get_skip_prob(cm, xd), 0);
x->skip = 0;
} else {
*tmp_dist = sse;
*tmp_rate = vp10_cost_bit(vp10_get_skip_prob(cm, xd), 1);
x->skip = 1;
}
}
*tmp_rate += base_rate;
#endif // CONFIG_EXT_TX
}
#endif // CONFIG_SUPERTX
vp10/encoder/rdopt.c
View file @ 9a8a6a1b
......@@ -1761,9 +1761,7 @@ static int rd_pick_ext_intra_sby(VP10_COMP *cpi, MACROBLOCK *x,
EXT_INTRA_MODE mode;
TX_SIZE best_tx_size = TX_4X4;
EXT_INTRA_MODE_INFO ext_intra_mode_info;
#if CONFIG_EXT_TX
TX_TYPE best_tx_type;
#endif // CONFIG_EXT_TX
vp10_zero(ext_intra_mode_info);
mbmi->ext_intra_mode_info.use_ext_intra_mode[0] = 1;
......@@ -1785,9 +1783,7 @@ static int rd_pick_ext_intra_sby(VP10_COMP *cpi, MACROBLOCK *x,
*best_rd = this_rd;
best_tx_size = mic->mbmi.tx_size;
ext_intra_mode_info = mbmi->ext_intra_mode_info;
#if CONFIG_EXT_TX
best_tx_type = mic->mbmi.tx_type;
#endif // CONFIG_EXT_TX
*rate = this_rate;
*rate_tokenonly = this_rate_tokenonly;
*distortion = this_distortion;
......@@ -1803,9 +1799,7 @@ static int rd_pick_ext_intra_sby(VP10_COMP *cpi, MACROBLOCK *x,
ext_intra_mode_info.use_ext_intra_mode[0];
mbmi->ext_intra_mode_info.ext_intra_mode[0] =
ext_intra_mode_info.ext_intra_mode[0];
#if CONFIG_EXT_TX
mbmi->tx_type = best_tx_type;
#endif // CONFIG_EXT_TX
return 1;
} else {
return 0;
......@@ -1825,9 +1819,7 @@ static int64_t rd_pick_intra_angle_sby(VP10_COMP *cpi, MACROBLOCK *x,
const double rd_adjust = 1.2;
int64_t this_distortion, this_rd, sse_dummy;
TX_SIZE best_tx_size = mic->mbmi.tx_size;
#if CONFIG_EXT_TX
TX_TYPE best_tx_type = mbmi->tx_type;
#endif // CONFIG_EXT_TX
if (ANGLE_FAST_SEARCH) {
int deltas_level1[3] = {0, -2, 2};
......@@ -1858,9 +1850,7 @@ static int64_t rd_pick_intra_angle_sby(VP10_COMP *cpi, MACROBLOCK *x,
best_rd = this_rd;
best_angle_delta = mbmi->angle_delta[0];
best_tx_size = mbmi->tx_size;
#if CONFIG_EXT_TX
best_tx_type = mbmi->tx_type;
#endif // CONFIG_EXT_TX
*rate = this_rate;
*rate_tokenonly = this_rate_tokenonly;
*distortion = this_distortion;
......@@ -1881,9 +1871,7 @@ static int64_t rd_pick_intra_angle_sby(VP10_COMP *cpi, MACROBLOCK *x,
best_rd = this_rd;
best_angle_delta = mbmi->angle_delta[0];
best_tx_size = mbmi->tx_size;
#if CONFIG_EXT_TX
best_tx_type = mbmi->tx_type;
#endif // CONFIG_EXT_TX
*rate = this_rate;
*rate_tokenonly = this_rate_tokenonly;
*distortion = this_distortion;
......@@ -1908,9 +1896,7 @@ static int64_t rd_pick_intra_angle_sby(VP10_COMP *cpi, MACROBLOCK *x,
best_rd = this_rd;
best_angle_delta = mbmi->angle_delta[0];
best_tx_size = mbmi->tx_size;
#if CONFIG_EXT_TX
best_tx_type = mbmi->tx_type;
#endif // CONFIG_EXT_TX
*rate = this_rate;
*rate_tokenonly = this_rate_tokenonly;
*distortion = this_distortion;
......@@ -1921,9 +1907,7 @@ static int64_t rd_pick_intra_angle_sby(VP10_COMP *cpi, MACROBLOCK *x,
mbmi->tx_size = best_tx_size;
mbmi->angle_delta[0] = best_angle_delta;
#if CONFIG_EXT_TX
mbmi->tx_type = best_tx_type;
#endif // CONFIG_EXT_TX
if (*rate_tokenonly < INT_MAX) {
txfm_rd_in_plane(x,
......@@ -5440,8 +5424,13 @@ static int64_t handle_inter_mode(VP10_COMP *cpi, MACROBLOCK *x,
vp10_subtract_plane(x, bsize, 0);
#if CONFIG_VAR_TX
if (cm->tx_mode == TX_MODE_SELECT || xd->lossless[mbmi->segment_id]) {
#if CONFIG_EXT_TX
select_tx_type_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
bsize, ref_best_rd);
#else
inter_block_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
bsize, ref_best_rd);
#endif // CONFIG_EXT_TX
} else {
int idx, idy;
super_block_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
......@@ -5453,7 +5442,7 @@ static int64_t handle_inter_mode(VP10_COMP *cpi, MACROBLOCK *x,
#else
super_block_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
bsize, ref_best_rd);
#endif
#endif // CONFIG_VAR_TX
if (*rate_y == INT_MAX) {
*rate2 = INT_MAX;
......@@ -5474,7 +5463,7 @@ static int64_t handle_inter_mode(VP10_COMP *cpi, MACROBLOCK *x,