Commit 9ca3c8b2 authored by Ronald S. Bultje's avatar Ronald S. Bultje
Browse files

Selectable transform size try 2.

Results: derf (vanilla or +hybridtx) +0.2% and (+hybrid16x16
or +tx16x16) +0.7%-0.8%; HD (vanilla or +hybridtx) +0.1-0.2%
and (+hybrid16x16 or +tx16x16) +1.4%, STD/HD (vanilla or +hybridtx)
about even, and (+hybrid16x16 or +tx16x16) +0.8-1.0%.

Change-Id: I03899e2f7a64e725a863f32e55366035ba77aa62
parent 54434bdb
......@@ -229,6 +229,7 @@ EXPERIMENT_LIST="
new_mvref
hybridtransform16x16
newmventropy
tx_select
"
CONFIG_LIST="
external_build
......
......@@ -163,10 +163,16 @@ typedef enum {
NB_PREDICTION_TYPES = 3,
} COMPPREDMODE_TYPE;
/* TODO: allows larger transform */
typedef enum {
ONLY_4X4 = 0,
ALLOW_8X8 = 1
ALLOW_8X8 = 1,
#if CONFIG_TX16X16
ALLOW_16X16 = 2,
#endif
#if CONFIG_TX_SELECT
TX_MODE_SELECT = 2 + CONFIG_TX16X16,
#endif
NB_TXFM_MODES = 2 + CONFIG_TX16X16 + CONFIG_TX_SELECT,
} TXFM_MODE;
typedef struct VP8_COMMON_RTCD {
......@@ -306,6 +312,11 @@ typedef struct VP8Common {
vp8_prob prob_comppred[COMP_PRED_CONTEXTS];
#if CONFIG_TX_SELECT
// FIXME contextualize
vp8_prob prob_tx[TX_SIZE_MAX - 1];
#endif
vp8_prob mbskip_pred_probs[MBSKIP_CONTEXTS];
FRAME_CONTEXT lfc_a; /* last alt ref entropy */
......
......@@ -170,6 +170,27 @@ static void vp8_kfread_modes(VP8D_COMP *pbi,
m->mbmi.second_uv_mode = (MB_PREDICTION_MODE)(DC_PRED - 1);
#endif
#if CONFIG_TX_SELECT
if (cm->txfm_mode == TX_MODE_SELECT && m->mbmi.mb_skip_coeff == 0 &&
m->mbmi.mode <= TM_PRED) {
// FIXME(rbultje) code ternary symbol once all experiments are merged
m->mbmi.txfm_size = vp8_read(bc, cm->prob_tx[0]);
#if CONFIG_TX16X16
if (m->mbmi.txfm_size != TX_4X4)
m->mbmi.txfm_size += vp8_read(bc, cm->prob_tx[1]);
#endif
} else
#endif
#if CONFIG_TX16X16
if (cm->txfm_mode >= ALLOW_16X16 && m->mbmi.mode <= TM_PRED) {
m->mbmi.txfm_size = TX_16X16;
} else
#endif
if (cm->txfm_mode >= ALLOW_8X8 && m->mbmi.mode != B_PRED) {
m->mbmi.txfm_size = TX_8X8;
} else {
m->mbmi.txfm_size = TX_4X4;
}
}
#if CONFIG_NEWMVENTROPY
......@@ -1263,6 +1284,32 @@ static void read_mb_modes_mv(VP8D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi,
#endif
}
#if CONFIG_TX_SELECT
if (cm->txfm_mode == TX_MODE_SELECT && mbmi->mb_skip_coeff == 0 &&
((mbmi->ref_frame == INTRA_FRAME && mbmi->mode <= TM_PRED) ||
(mbmi->ref_frame != INTRA_FRAME && mbmi->mode != SPLITMV))) {
// FIXME(rbultje) code ternary symbol once all experiments are merged
mbmi->txfm_size = vp8_read(bc, cm->prob_tx[0]);
#if CONFIG_TX16X16
if (mbmi->txfm_size != TX_4X4)
mbmi->txfm_size += vp8_read(bc, cm->prob_tx[1]);
#endif
} else
#endif
#if CONFIG_TX16X16
if (cm->txfm_mode >= ALLOW_16X16 &&
((mbmi->ref_frame == INTRA_FRAME && mbmi->mode <= TM_PRED) ||
(mbmi->ref_frame != INTRA_FRAME && mbmi->mode != SPLITMV))) {
mbmi->txfm_size = TX_16X16;
} else
#endif
if (cm->txfm_mode >= ALLOW_8X8 &&
((mbmi->ref_frame == INTRA_FRAME && mbmi->mode != B_PRED) ||
(mbmi->ref_frame != INTRA_FRAME && mbmi->mode != SPLITMV))) {
mbmi->txfm_size = TX_8X8;
} else {
mbmi->txfm_size = TX_4X4;
}
}
void vp8_decode_mode_mvs(VP8D_COMP *pbi) {
......
......@@ -231,39 +231,6 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd,
if (xd->segmentation_enabled)
mb_init_dequantizer(pbi, xd);
if (pbi->common.frame_type == KEY_FRAME) {
#if CONFIG_TX16X16 || CONFIG_HYBRIDTRANSFORM16X16
if (xd->mode_info_context->mbmi.mode <= TM_PRED ||
xd->mode_info_context->mbmi.mode == NEWMV ||
xd->mode_info_context->mbmi.mode == ZEROMV ||
xd->mode_info_context->mbmi.mode == NEARMV ||
xd->mode_info_context->mbmi.mode == NEARESTMV)
xd->mode_info_context->mbmi.txfm_size = TX_16X16;
else
#endif
if (pbi->common.txfm_mode == ALLOW_8X8 &&
xd->mode_info_context->mbmi.mode != B_PRED)
xd->mode_info_context->mbmi.txfm_size = TX_8X8;
else
xd->mode_info_context->mbmi.txfm_size = TX_4X4;
} else {
#if CONFIG_TX16X16 || CONFIG_HYBRIDTRANSFORM16X16
if (xd->mode_info_context->mbmi.mode <= TM_PRED ||
xd->mode_info_context->mbmi.mode == NEWMV ||
xd->mode_info_context->mbmi.mode == ZEROMV ||
xd->mode_info_context->mbmi.mode == NEARMV ||
xd->mode_info_context->mbmi.mode == NEARESTMV)
xd->mode_info_context->mbmi.txfm_size = TX_16X16;
else
#endif
if (pbi->common.txfm_mode == ALLOW_8X8 &&
xd->mode_info_context->mbmi.mode != B_PRED &&
xd->mode_info_context->mbmi.mode != SPLITMV)
xd->mode_info_context->mbmi.txfm_size = TX_8X8;
else
xd->mode_info_context->mbmi.txfm_size = TX_4X4;
}
#if CONFIG_SUPERBLOCKS
if (xd->mode_info_context->mbmi.encoded_as_sb) {
xd->mode_info_context->mbmi.txfm_size = TX_8X8;
......@@ -1006,7 +973,7 @@ static void read_coef_probs(VP8D_COMP *pbi) {
}
#endif
if (pbi->common.txfm_mode == ALLOW_8X8 && vp8_read_bit(bc)) {
if (pbi->common.txfm_mode != ONLY_4X4 && vp8_read_bit(bc)) {
// read coef probability tree
for (i = 0; i < BLOCK_TYPES_8X8; i++)
for (j = !i; j < COEF_BANDS; j++)
......@@ -1025,7 +992,7 @@ static void read_coef_probs(VP8D_COMP *pbi) {
}
}
#if CONFIG_HYBRIDTRANSFORM8X8
if (pbi->common.txfm_mode == ALLOW_8X8 && vp8_read_bit(bc)) {
if (pbi->common.txfm_mode != ONLY_4X4 && vp8_read_bit(bc)) {
// read coef probability tree
for (i = 0; i < BLOCK_TYPES_8X8; i++)
for (j = !i; j < COEF_BANDS; j++)
......@@ -1047,7 +1014,7 @@ static void read_coef_probs(VP8D_COMP *pbi) {
#if CONFIG_TX16X16
// 16x16
if (vp8_read_bit(bc)) {
if (pbi->common.txfm_mode > ALLOW_8X8 && vp8_read_bit(bc)) {
// read coef probability tree
for (i = 0; i < BLOCK_TYPES_16X16; ++i)
for (j = !i; j < COEF_BANDS; ++j)
......@@ -1066,7 +1033,7 @@ static void read_coef_probs(VP8D_COMP *pbi) {
}
}
#if CONFIG_HYBRIDTRANSFORM16X16
if (vp8_read_bit(bc)) {
if (pbi->common.txfm_mode > ALLOW_8X8 && vp8_read_bit(bc)) {
// read coef probability tree
for (i = 0; i < BLOCK_TYPES_16X16; ++i)
for (j = !i; j < COEF_BANDS; ++j)
......@@ -1314,7 +1281,27 @@ int vp8_decode_frame(VP8D_COMP *pbi) {
#endif
/* Read the loop filter level and type */
#if CONFIG_TX_SELECT
#if CONFIG_TX16X16
pc->txfm_mode = vp8_read_literal(bc, 2);
#else
pc->txfm_mode = vp8_read_bit(bc);
if (pc->txfm_mode)
pc->txfm_mode += vp8_read_bit(bc);
#endif
if (pc->txfm_mode == TX_MODE_SELECT) {
pc->prob_tx[0] = vp8_read_literal(bc, 8);
#if CONFIG_TX16X16
pc->prob_tx[1] = vp8_read_literal(bc, 8);
#endif
}
#else
pc->txfm_mode = (TXFM_MODE) vp8_read_bit(bc);
#if CONFIG_TX16X16
if (pc->txfm_mode == ALLOW_8X8)
pc->txfm_mode = ALLOW_16X16;
#endif
#endif
pc->filter_type = (LOOPFILTERTYPE) vp8_read_bit(bc);
pc->filter_level = vp8_read_literal(bc, 6);
......
......@@ -1290,6 +1290,23 @@ static void pack_inter_mode_mvs(VP8_COMP *const cpi) {
}
}
#if CONFIG_TX_SELECT
if (((rf == INTRA_FRAME && mode <= TM_PRED) ||
(rf != INTRA_FRAME && mode != SPLITMV)) &&
pc->txfm_mode == TX_MODE_SELECT &&
!((pc->mb_no_coeff_skip && mi->mb_skip_coeff) ||
(segfeature_active(xd, segment_id, SEG_LVL_EOB) &&
get_segdata(xd, segment_id, SEG_LVL_EOB) == 0))) {
TX_SIZE sz = mi->txfm_size;
// FIXME(rbultje) code ternary symbol once all experiments are merged
vp8_write(w, sz != TX_4X4, pc->prob_tx[0]);
#if CONFIG_TX16X16
if (sz != TX_4X4)
vp8_write(w, sz != TX_8X8, pc->prob_tx[1]);
#endif
}
#endif
#if CONFIG_SUPERBLOCKS
if (m->mbmi.encoded_as_sb) {
assert(!i);
......@@ -1411,6 +1428,7 @@ static void write_kfmodes(VP8_COMP *cpi) {
vp8_encode_bool(bc, skip_coeff,
get_pred_prob(c, xd, PRED_MBSKIP));
}
#if CONFIG_SUPERBLOCKS
if (m->mbmi.encoded_as_sb) {
sb_kfwrite_ymode(bc, ym,
......@@ -1468,6 +1486,21 @@ static void write_kfmodes(VP8_COMP *cpi) {
} else
write_uv_mode(bc, m->mbmi.uv_mode, c->kf_uv_mode_prob[ym]);
#if CONFIG_TX_SELECT
if (ym <= TM_PRED && c->txfm_mode == TX_MODE_SELECT &&
!((c->mb_no_coeff_skip && m->mbmi.mb_skip_coeff) ||
(segfeature_active(xd, segment_id, SEG_LVL_EOB) &&
get_segdata(xd, segment_id, SEG_LVL_EOB) == 0))) {
TX_SIZE sz = m->mbmi.txfm_size;
// FIXME(rbultje) code ternary symbol once all experiments are merged
vp8_write(bc, sz != TX_4X4, c->prob_tx[0]);
#if CONFIG_TX16X16
if (sz != TX_4X4)
vp8_write(bc, sz != TX_8X8, c->prob_tx[1]);
#endif
}
#endif
#if CONFIG_SUPERBLOCKS
if (m->mbmi.encoded_as_sb) {
assert(!i);
......@@ -1564,7 +1597,7 @@ void build_coeff_contexts(VP8_COMP *cpi) {
#endif
if (cpi->common.txfm_mode == ALLOW_8X8) {
if (cpi->common.txfm_mode != ONLY_4X4) {
for (i = 0; i < BLOCK_TYPES_8X8; ++i) {
for (j = 0; j < COEF_BANDS; ++j) {
for (k = 0; k < PREV_COEF_CONTEXTS; ++k) {
......@@ -1618,22 +1651,23 @@ void build_coeff_contexts(VP8_COMP *cpi) {
}
#if CONFIG_TX16X16
//16x16
for (i = 0; i < BLOCK_TYPES_16X16; ++i) {
for (j = 0; j < COEF_BANDS; ++j) {
for (k = 0; k < PREV_COEF_CONTEXTS; ++k) {
if (k >= 3 && ((i == 0 && j == 1) || (i > 0 && j == 0)))
continue;
vp8_tree_probs_from_distribution(
MAX_ENTROPY_TOKENS, vp8_coef_encodings, vp8_coef_tree,
cpi->frame_coef_probs_16x16[i][j][k],
cpi->frame_branch_ct_16x16[i][j][k],
cpi->coef_counts_16x16[i][j][k], 256, 1);
if (cpi->common.txfm_mode > ALLOW_8X8) {
for (i = 0; i < BLOCK_TYPES_16X16; ++i) {
for (j = 0; j < COEF_BANDS; ++j) {
for (k = 0; k < PREV_COEF_CONTEXTS; ++k) {
if (k >= 3 && ((i == 0 && j == 1) || (i > 0 && j == 0)))
continue;
vp8_tree_probs_from_distribution(
MAX_ENTROPY_TOKENS, vp8_coef_encodings, vp8_coef_tree,
cpi->frame_coef_probs_16x16[i][j][k],
cpi->frame_branch_ct_16x16[i][j][k],
cpi->coef_counts_16x16[i][j][k], 256, 1);
#ifdef ENTROPY_STATS
if (!cpi->dummy_packing)
for (t = 0; t < MAX_ENTROPY_TOKENS; ++t)
context_counters_16x16[i][j][k][t] += cpi->coef_counts_16x16[i][j][k][t];
if (!cpi->dummy_packing)
for (t = 0; t < MAX_ENTROPY_TOKENS; ++t)
context_counters_16x16[i][j][k][t] += cpi->coef_counts_16x16[i][j][k][t];
#endif
}
}
}
}
......@@ -1746,8 +1780,7 @@ static void update_coef_probs2(VP8_COMP *cpi) {
}
}
if (cpi->common.txfm_mode != ALLOW_8X8) return;
if (cpi->common.txfm_mode != ONLY_4X4)
for (t = 0; t < ENTROPY_NODES; ++t) {
/* dry run to see if there is any udpate at all needed */
savings = 0;
......@@ -2024,7 +2057,7 @@ static void update_coef_probs(VP8_COMP *cpi) {
#endif
/* do not do this if not even allowed */
if (cpi->common.txfm_mode == ALLOW_8X8) {
if (cpi->common.txfm_mode != ONLY_4X4) {
/* dry run to see if update is necessary */
update[0] = update[1] = 0;
savings = 0;
......@@ -2177,7 +2210,7 @@ static void update_coef_probs(VP8_COMP *cpi) {
}
#if CONFIG_TX16X16
// 16x16
if (cpi->common.txfm_mode > ALLOW_8X8) {
/* dry run to see if update is necessary */
update[0] = update[1] = 0;
savings = 0;
......@@ -2327,6 +2360,7 @@ static void update_coef_probs(VP8_COMP *cpi) {
}
}
#endif
}
#endif
}
......@@ -2616,7 +2650,68 @@ void vp8_pack_bitstream(VP8_COMP *cpi, unsigned char *dest, unsigned long *size)
}
#endif
vp8_write_bit(bc, pc->txfm_mode);
#if CONFIG_TX_SELECT
{
#if CONFIG_TX16X16
int cnt = cpi->txfm_count[0] + cpi->txfm_count[1] + cpi->txfm_count[2];
if (cnt && pc->txfm_mode == TX_MODE_SELECT) {
int prob = (255 * (cpi->txfm_count[1] + cpi->txfm_count[2]) + (cnt >> 1)) / cnt;
if (prob <= 1) {
pc->prob_tx[0] = 1;
} else if (prob >= 255) {
pc->prob_tx[0] = 255;
} else {
pc->prob_tx[0] = prob;
}
pc->prob_tx[0] = 256 - pc->prob_tx[0];
} else {
pc->prob_tx[0] = 128;
}
cnt -= cpi->txfm_count[0];
if (cnt && pc->txfm_mode == TX_MODE_SELECT) {
int prob = (255 * cpi->txfm_count[2] + (cnt >> 1)) / cnt;
if (prob <= 1) {
pc->prob_tx[1] = 1;
} else if (prob >= 255) {
pc->prob_tx[1] = 255;
} else {
pc->prob_tx[1] = prob;
}
pc->prob_tx[1] = 256 - pc->prob_tx[1];
} else {
pc->prob_tx[1] = 128;
}
vp8_write_literal(bc, pc->txfm_mode, 2);
if (pc->txfm_mode == TX_MODE_SELECT) {
vp8_write_literal(bc, pc->prob_tx[0], 8);
vp8_write_literal(bc, pc->prob_tx[1], 8);
}
#else
int cnt = cpi->txfm_count[0] + cpi->txfm_count[1];
if (cnt && pc->txfm_mode == TX_MODE_SELECT) {
int prob = (255 * cpi->txfm_count[1] + (cnt >> 1)) / cnt;
if (prob <= 1) {
pc->prob_tx[0] = 1;
} else if (prob >= 255) {
pc->prob_tx[0] = 255;
} else {
pc->prob_tx[0] = prob;
}
pc->prob_tx[0] = 256 - pc->prob_tx[0];
} else {
pc->prob_tx[0] = 128;
}
vp8_write_bit(bc, pc->txfm_mode != 0);
if (pc->txfm_mode)
vp8_write_bit(bc, pc->txfm_mode - 1);
if (pc->txfm_mode == TX_MODE_SELECT) {
vp8_write_literal(bc, pc->prob_tx[0], 8);
}
#endif
}
#else
vp8_write_bit(bc, !!pc->txfm_mode);
#endif
// Encode the loop filter level and type
vp8_write_bit(bc, pc->filter_type);
......
......@@ -85,6 +85,9 @@ typedef struct {
int hybrid_pred_diff;
int comp_pred_diff;
int single_pred_diff;
#if CONFIG_TX_SELECT
int64_t txfm_rd_diff[NB_TXFM_MODES];
#endif
} PICK_MODE_CONTEXT;
typedef struct {
......
......@@ -417,6 +417,18 @@ static void update_state(VP8_COMP *cpi, MACROBLOCK *x, PICK_MODE_CONTEXT *ctx) {
mbmi->mv[1].as_int = x->partition_info->bmi[15].second_mv.as_int;
}
#if CONFIG_TX_SELECT
{
int segment_id = mbmi->segment_id;
if (!segfeature_active(xd, segment_id, SEG_LVL_EOB) ||
get_segdata(xd, segment_id, SEG_LVL_EOB)) {
for (i = 0; i < NB_TXFM_MODES; i++) {
cpi->rd_tx_select_diff[i] += ctx->txfm_rd_diff[i];
}
}
}
#endif
if (cpi->common.frame_type == KEY_FRAME) {
// Restore the coding modes to that held in the coding context
// if (mb_mode == B_PRED)
......@@ -606,9 +618,6 @@ static void pick_mb_modes(VP8_COMP *cpi,
x->active_ptr = cpi->active_map + map_index;
/* force 4x4 transform for mode selection */
mbmi->txfm_size = TX_4X4; // TODO IS this right??
#if CONFIG_SUPERBLOCKS
xd->mode_info_context->mbmi.encoded_as_sb = 0;
#endif
......@@ -1395,7 +1404,10 @@ static void encode_frame_internal(VP8_COMP *cpi) {
vpx_memset(cpi->rd_comp_pred_diff, 0, sizeof(cpi->rd_comp_pred_diff));
vpx_memset(cpi->single_pred_count, 0, sizeof(cpi->single_pred_count));
vpx_memset(cpi->comp_pred_count, 0, sizeof(cpi->comp_pred_count));
#if CONFIG_TX_SELECT
vpx_memset(cpi->txfm_count, 0, sizeof(cpi->txfm_count));
vpx_memset(cpi->rd_tx_select_diff, 0, sizeof(cpi->rd_tx_select_diff));
#endif
{
struct vpx_usec_timer emr_timer;
vpx_usec_timer_start(&emr_timer);
......@@ -1458,6 +1470,7 @@ static int check_dual_ref_flags(VP8_COMP *cpi) {
void vp8_encode_frame(VP8_COMP *cpi) {
if (cpi->sf.RD) {
int i, frame_type, pred_type;
TXFM_MODE txfm_type;
/*
* This code does a single RD pass over the whole frame assuming
......@@ -1465,9 +1478,8 @@ void vp8_encode_frame(VP8_COMP *cpi) {
* worked best for that type of frame in the past.
* It also predicts whether another coding mode would have worked
* better that this coding mode. If that is the case, it remembers
* that for subsequent frames. If the difference is above a certain
* threshold, it will actually re-encode the current frame using
* that different coding mode.
* that for subsequent frames.
* It does the same analysis for transform size selection also.
*/
if (cpi->common.frame_type == KEY_FRAME)
frame_type = 0;
......@@ -1478,6 +1490,7 @@ void vp8_encode_frame(VP8_COMP *cpi) {
else
frame_type = 2;
/* prediction (compound, single or hybrid) mode selection */
if (frame_type == 3)
pred_type = SINGLE_PREDICTION_ONLY;
else if (cpi->rd_prediction_type_threshes[frame_type][1] >
......@@ -1492,15 +1505,111 @@ void vp8_encode_frame(VP8_COMP *cpi) {
else
pred_type = HYBRID_PREDICTION;
/* transform size (4x4, 8x8, 16x16 or select-per-mb) selection */
#if CONFIG_LOSSLESS
if (cpi->oxcf.lossless) {
txfm_type = ONLY_4X4;
} else
#endif
#if CONFIG_TX_SELECT
/* FIXME (rbultje)
* this is a hack (no really), basically to work around the complete
* nonsense coefficient cost prediction for keyframes. The probabilities
* are reset to defaults, and thus we basically have no idea how expensive
* a 4x4 vs. 8x8 will really be. The result is that any estimate at which
* of the two is better is utterly bogus.
* I'd like to eventually remove this hack, but in order to do that, we
* need to move the frame reset code from the frame encode init to the
* bitstream write code, or alternatively keep a backup of the previous
* keyframe's probabilities as an estimate of what the current keyframe's
* coefficient cost distributions may look like. */
if (frame_type == 0) {
#if CONFIG_TX16X16
txfm_type = ALLOW_16X16;
#else
txfm_type = ALLOW_8X8;
#endif
} else
#if 0
/* FIXME (rbultje)
* this code is disabled for a similar reason as the code above; the
* problem is that each time we "revert" to 4x4 only (or even 8x8 only),
* the coefficient probabilities for 16x16 (and 8x8) start lagging behind,
* thus leading to them lagging further behind and not being chosen for
* subsequent frames either. This is essentially a local minimum problem
* that we can probably fix by estimating real costs more closely within
* a frame, perhaps by re-calculating costs on-the-fly as frame encoding
* progresses. */
if (cpi->rd_tx_select_threshes[frame_type][TX_MODE_SELECT] >
cpi->rd_tx_select_threshes[frame_type][ONLY_4X4] &&
#if CONFIG_TX16X16
cpi->rd_tx_select_threshes[frame_type][TX_MODE_SELECT] >
cpi->rd_tx_select_threshes[frame_type][ALLOW_16X16] &&
#endif
cpi->rd_tx_select_threshes[frame_type][TX_MODE_SELECT] >
cpi->rd_tx_select_threshes[frame_type][ALLOW_8X8]) {
txfm_type = TX_MODE_SELECT;
} else if (cpi->rd_tx_select_threshes[frame_type][ONLY_4X4] >
cpi->rd_tx_select_threshes[frame_type][ALLOW_8X8]
#if CONFIG_TX16X16
&& cpi->rd_tx_select_threshes[frame_type][ONLY_4X4] >
cpi->rd_tx_select_threshes[frame_type][ALLOW_16X16]
#endif
) {
txfm_type = ONLY_4X4;
#if CONFIG_TX16X16
} else if (cpi->rd_tx_select_threshes[frame_type][ALLOW_16X16] >=
cpi->rd_tx_select_threshes[frame_type][ALLOW_8X8]) {
txfm_type = ALLOW_16X16;
#endif
} else
txfm_type = ALLOW_8X8;
#else
#if CONFIG_TX16X16
txfm_type = cpi->rd_tx_select_threshes[frame_type][ALLOW_16X16] >=
cpi->rd_tx_select_threshes[frame_type][TX_MODE_SELECT] ?
ALLOW_16X16 : TX_MODE_SELECT;
#else
txfm_type = cpi->rd_tx_select_threshes[frame_type][ALLOW_8X8] >=
cpi->rd_tx_select_threshes[frame_type][TX_MODE_SELECT] ?
ALLOW_8X8 : TX_MODE_SELECT;
#endif
#endif
#elif CONFIG_TX16X16
txfm_type = ALLOW_16X16;
#else
txfm_type = ALLOW_8X8;
#endif // CONFIG_TX_SELECT
cpi->common.txfm_mode = txfm_type;
#if CONFIG_TX_SELECT
if (txfm_type != TX_MODE_SELECT) {
cpi->common.prob_tx[0] = 128;
#if CONFIG_TX16X16
cpi->common.prob_tx[1] = 128;
#endif
}
#endif
cpi->common.comp_pred_mode = pred_type;
encode_frame_internal(cpi);
for (i = 0; i < NB_PREDICTION_TYPES; ++i) {
int diff = cpi->rd_comp_pred_diff[i] / cpi->common.MBs;
const int diff = cpi->rd_comp_pred_diff[i] / cpi->common.MBs;
cpi->rd_prediction_type_threshes[frame_type][i] += diff;
cpi->rd_prediction_type_threshes[frame_type][i] >>= 1;
}
#if CONFIG_TX_SELECT
for (i = 0; i < NB_TXFM_MODES; ++i) {
int64_t pd = cpi->rd_tx_select_diff[i];
int diff;
if (i == TX_MODE_SELECT)
pd -= RDCOST(cpi->mb.rdmult, cpi->mb.rddiv, 2048 * (TX_SIZE_MAX - 1), 0);
diff = pd / cpi->common.MBs;
cpi->rd_tx_select_threshes[frame_type][i] += diff;
cpi->rd_tx_select_threshes[frame_type][i] /= 2;
}
#endif
if (cpi->common.comp_pred_mode == HYBRID_PREDICTION) {
int single_count_zero = 0;
int comp_count_zero = 0;
......@@ -1516,6 +1625,28 @@ void vp8_encode_frame(VP8_COMP *cpi) {
cpi->common.comp_pred_mode = COMP_PREDICTION_ONLY;
}
}
#if CONFIG_TX_SELECT
if (cpi->common.txfm_mode == TX_MODE_SELECT) {
const int count4x4 = cpi->txfm_count[TX_4X4];
const int count8x8 = cpi->txfm_count[TX_8X8];