Commit 5a3e3c6d authored by Jingning Han's avatar Jingning Han
Browse files

Adaptive txfm size selection depending on residual sse/variance

This commit enables an adaptive transform size selection method
for speed -6. It uses largest transform size when the sse is more
than 4 times of variance, i.e., most energy is compacted in the
DC coefficient. Otherwise, use the default TX_8X8. It improves
the compression efficiency for rtc set of speed -6 by 0.8%, no
speed change observed.

Change-Id: Ie6ed1e728ff7bf88ebe940a60811361cdd19969c
parent 73eeb3be
......@@ -2390,7 +2390,7 @@ static TX_MODE select_tx_mode(const VP9_COMP *cpi) {
rd_opt->tx_select_threshes[frame_type][TX_MODE_SELECT] ?
ALLOW_32X32 : TX_MODE_SELECT;
} else if (cpi->sf.tx_size_search_method == USE_TX_8X8) {
return ALLOW_8X8;
return TX_MODE_SELECT;
} else {
unsigned int total = 0;
int i;
......
......@@ -252,6 +252,17 @@ static void model_rd_for_sb_y(VP9_COMP *cpi, BLOCK_SIZE bsize,
else
x->skip_txfm = 0;
if (cpi->common.tx_mode == TX_MODE_SELECT) {
if (sse > (var << 2))
xd->mi[0]->mbmi.tx_size = MIN(max_txsize_lookup[bsize],
tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
else
xd->mi[0]->mbmi.tx_size = TX_8X8;
} else {
xd->mi[0]->mbmi.tx_size = MIN(max_txsize_lookup[bsize],
tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
}
vp9_model_rd_from_var_lapndz(sse - var, 1 << num_pels_log2_lookup[bsize],
dc_quant >> 3, &rate, &dist);
*out_rate_sum = rate >> 1;
......@@ -293,6 +304,8 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
struct macroblockd_plane *const pd = &xd->plane[0];
PREDICTION_MODE this_mode, best_mode = ZEROMV;
MV_REFERENCE_FRAME ref_frame, best_ref_frame = LAST_FRAME;
TX_SIZE best_tx_size = MIN(max_txsize_lookup[bsize],
tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
INTERP_FILTER best_pred_filter = EIGHTTAP;
int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES];
struct buf_2d yv12_mb[4][MAX_MB_PLANE];
......@@ -486,6 +499,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
int64_t pf_dist[3];
unsigned int pf_var[3];
unsigned int pf_sse[3];
TX_SIZE pf_tx_size[3];
int64_t best_cost = INT64_MAX;
INTERP_FILTER best_filter = SWITCHABLE, filter;
PRED_BUFFER *current_pred = this_mode_pred;
......@@ -499,6 +513,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
cost = RDCOST(x->rdmult, x->rddiv,
vp9_get_switchable_rate(cpi) + pf_rate[filter],
pf_dist[filter]);
pf_tx_size[filter] = mbmi->tx_size;
if (cost < best_cost) {
best_filter = filter;
best_cost = cost;
......@@ -523,6 +538,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
free_pred_buffer(current_pred);
mbmi->interp_filter = best_filter;
mbmi->tx_size = pf_tx_size[mbmi->interp_filter];
rate = pf_rate[mbmi->interp_filter];
dist = pf_dist[mbmi->interp_filter];
var_y = pf_var[mbmi->interp_filter];
......@@ -624,6 +640,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
*returndistortion = dist;
best_mode = this_mode;
best_pred_filter = mbmi->interp_filter;
best_tx_size = mbmi->tx_size;
best_ref_frame = ref_frame;
skip_txfm = x->skip_txfm;
......@@ -657,10 +674,11 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
bw, bh);
}
mbmi->mode = best_mode;
mbmi->mode = best_mode;
mbmi->interp_filter = best_pred_filter;
mbmi->ref_frame[0] = best_ref_frame;
mbmi->mv[0].as_int = frame_mv[best_mode][best_ref_frame].as_int;
mbmi->tx_size = best_tx_size;
mbmi->ref_frame[0] = best_ref_frame;
mbmi->mv[0].as_int = frame_mv[best_mode][best_ref_frame].as_int;
xd->mi[0]->bmi[0].as_mv[0].as_int = mbmi->mv[0].as_int;
x->skip_txfm = skip_txfm;
......@@ -669,7 +687,6 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
if (!x->skip && best_rd > inter_mode_thresh &&
bsize <= cpi->sf.max_intra_bsize) {
int i, j;
const int step = 1 << mbmi->tx_size;
const int width = num_4x4_blocks_wide_lookup[bsize];
const int height = num_4x4_blocks_high_lookup[bsize];
......@@ -679,6 +696,10 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
const int src_stride = p->src.stride;
int block_idx = 0;
TX_SIZE tmp_tx_size = MIN(max_txsize_lookup[bsize],
tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
const int step = 1 << tmp_tx_size;
for (this_mode = DC_PRED; this_mode <= DC_PRED; ++this_mode) {
if (cpi->sf.reuse_inter_pred_sby) {
pd->dst.buf = tmp[0].data;
......@@ -688,7 +709,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
for (j = 0; j < height; j += step) {
for (i = 0; i < width; i += step) {
vp9_predict_intra_block(xd, block_idx, b_width_log2(bsize),
mbmi->tx_size, this_mode,
tmp_tx_size, this_mode,
&p->src.buf[4 * (j * dst_stride + i)],
src_stride,
&pd->dst.buf[4 * (j * dst_stride + i)],
......@@ -715,6 +736,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
*returnrate = rate;
*returndistortion = dist;
mbmi->mode = this_mode;
mbmi->tx_size = tmp_tx_size;
mbmi->ref_frame[0] = INTRA_FRAME;
mbmi->uv_mode = this_mode;
mbmi->mv[0].as_int = INVALID_MV;
......
......@@ -272,7 +272,8 @@ static void set_rt_speed_feature(VP9_COMP *cpi, SPEED_FEATURES *sf,
sf->search_type_check_frequency = 50;
sf->source_var_thresh = 360;
sf->tx_size_search_method = USE_TX_8X8;
sf->tx_size_search_method = (cm->frame_type == KEY_FRAME) ?
USE_LARGESTALL : USE_TX_8X8;
sf->max_intra_bsize = BLOCK_8X8;
// This feature is only enabled when partition search is disabled.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment