Commit e2c1ea94 authored by Debargha Mukherjee's avatar Debargha Mukherjee
Browse files

A variety of fixes for supertx/var-tx rd costing

Change-Id: I8a3d59378abb1dfa4e614b2975c2db05d4224bd5
parent d46c1f23
...@@ -1028,6 +1028,21 @@ static void choose_smallest_tx_size(VP10_COMP *cpi, MACROBLOCK *x, ...@@ -1028,6 +1028,21 @@ static void choose_smallest_tx_size(VP10_COMP *cpi, MACROBLOCK *x,
mbmi->tx_size, cpi->sf.use_fast_coef_costing); mbmi->tx_size, cpi->sf.use_fast_coef_costing);
} }
static INLINE int vp10_cost_tx_size(TX_SIZE tx_size, TX_SIZE max_tx_size,
const vpx_prob *tx_probs) {
int m;
int r_tx_size = 0;
for (m = 0; m <= tx_size - (tx_size == max_tx_size); ++m) {
if (m == tx_size)
r_tx_size += vp10_cost_zero(tx_probs[m]);
else
r_tx_size += vp10_cost_one(tx_probs[m]);
}
return r_tx_size;
}
static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x, static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x,
int *rate, int *rate,
int64_t *distortion, int64_t *distortion,
...@@ -1043,7 +1058,7 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x, ...@@ -1043,7 +1058,7 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x,
int r, s; int r, s;
int64_t d, sse; int64_t d, sse;
int64_t rd = INT64_MAX; int64_t rd = INT64_MAX;
int n, m; int n;
int s0, s1; int s0, s1;
int64_t best_rd = INT64_MAX, last_rd = INT64_MAX; int64_t best_rd = INT64_MAX, last_rd = INT64_MAX;
TX_SIZE best_tx = max_tx_size; TX_SIZE best_tx = max_tx_size;
...@@ -1078,13 +1093,7 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x, ...@@ -1078,13 +1093,7 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x,
for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) { for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
last_rd = INT64_MAX; last_rd = INT64_MAX;
for (n = start_tx; n >= end_tx; --n) { for (n = start_tx; n >= end_tx; --n) {
int r_tx_size = 0; const int r_tx_size = vp10_cost_tx_size(n, max_tx_size, tx_probs);
for (m = 0; m <= n - (n == (int) max_tx_size); ++m) {
if (m == n)
r_tx_size += vp10_cost_zero(tx_probs[m]);
else
r_tx_size += vp10_cost_one(tx_probs[m]);
}
#if CONFIG_EXT_TX #if CONFIG_EXT_TX
ext_tx_set = get_ext_tx_set(n, bs, is_inter); ext_tx_set = get_ext_tx_set(n, bs, is_inter);
...@@ -1214,7 +1223,7 @@ static void super_block_yrd(VP10_COMP *cpi, MACROBLOCK *x, int *rate, ...@@ -1214,7 +1223,7 @@ static void super_block_yrd(VP10_COMP *cpi, MACROBLOCK *x, int *rate,
assert(bs == xd->mi[0]->mbmi.sb_type); assert(bs == xd->mi[0]->mbmi.sb_type);
if (xd->lossless[0]) { if (xd->lossless[xd->mi[0]->mbmi.segment_id]) {
choose_smallest_tx_size(cpi, x, rate, distortion, skip, ret_sse, choose_smallest_tx_size(cpi, x, rate, distortion, skip, ret_sse,
ref_best_rd, bs); ref_best_rd, bs);
} else if (cpi->sf.tx_size_search_method == USE_LARGESTALL) { } else if (cpi->sf.tx_size_search_method == USE_LARGESTALL) {
...@@ -1705,6 +1714,7 @@ static int64_t rd_pick_intra_sub_8x8_y_mode(VP10_COMP *cpi, MACROBLOCK *mb, ...@@ -1705,6 +1714,7 @@ static int64_t rd_pick_intra_sub_8x8_y_mode(VP10_COMP *cpi, MACROBLOCK *mb,
// TODO(any): Add search of the tx_type to improve rd performance at the // TODO(any): Add search of the tx_type to improve rd performance at the
// expense of speed. // expense of speed.
mic->mbmi.tx_type = DCT_DCT; mic->mbmi.tx_type = DCT_DCT;
mic->mbmi.tx_size = TX_4X4;
// Pick modes for each sub-block (of size 4x4, 4x8, or 8x4) in an 8x8 block. // Pick modes for each sub-block (of size 4x4, 4x8, or 8x4) in an 8x8 block.
for (idy = 0; idy < 2; idy += num_4x4_blocks_high) { for (idy = 0; idy < 2; idy += num_4x4_blocks_high) {
...@@ -1741,11 +1751,32 @@ static int64_t rd_pick_intra_sub_8x8_y_mode(VP10_COMP *cpi, MACROBLOCK *mb, ...@@ -1741,11 +1751,32 @@ static int64_t rd_pick_intra_sub_8x8_y_mode(VP10_COMP *cpi, MACROBLOCK *mb,
return INT64_MAX; return INT64_MAX;
} }
} }
mic->mbmi.mode = mic->bmi[3].as_mode;
// Add in the cost of the transform type
if (!xd->lossless[mic->mbmi.segment_id]) {
int rate_tx_type = 0;
#if CONFIG_EXT_TX
if (get_ext_tx_types(TX_4X4, bsize, 0) > 1) {
const int eset = get_ext_tx_set(TX_4X4, bsize, 0);
rate_tx_type =
cpi->intra_tx_type_costs[eset][TX_4X4]
[mic->mbmi.mode][mic->mbmi.tx_type];
}
#else
rate_tx_type =
cpi->intra_tx_type_costs[TX_4X4]
[intra_mode_to_tx_type_context[mic->mbmi.mode]]
[mic->mbmi.tx_type];
#endif
assert(mic->mbmi.tx_size == TX_4X4);
cost += rate_tx_type;
tot_rate_y += rate_tx_type;
}
*rate = cost; *rate = cost;
*rate_y = tot_rate_y; *rate_y = tot_rate_y;
*distortion = total_distortion; *distortion = total_distortion;
mic->mbmi.mode = mic->bmi[3].as_mode;
return RDCOST(mb->rdmult, mb->rddiv, cost, total_distortion); return RDCOST(mb->rdmult, mb->rddiv, cost, total_distortion);
} }
...@@ -2121,6 +2152,9 @@ static int64_t rd_pick_intra_sby_mode(VP10_COMP *cpi, MACROBLOCK *x, ...@@ -2121,6 +2152,9 @@ static int64_t rd_pick_intra_sby_mode(VP10_COMP *cpi, MACROBLOCK *x,
const MODE_INFO *left_mi = xd->left_mi; const MODE_INFO *left_mi = xd->left_mi;
const PREDICTION_MODE A = vp10_above_block_mode(mic, above_mi, 0); const PREDICTION_MODE A = vp10_above_block_mode(mic, above_mi, 0);
const PREDICTION_MODE L = vp10_left_block_mode(mic, left_mi, 0); const PREDICTION_MODE L = vp10_left_block_mode(mic, left_mi, 0);
const TX_SIZE max_tx_size = max_txsize_lookup[bsize];
const vpx_prob *tx_probs = get_tx_probs2(max_tx_size, xd,
&cpi->common.fc->tx_probs);
bmode_costs = cpi->y_mode_costs[A][L]; bmode_costs = cpi->y_mode_costs[A][L];
#if CONFIG_EXT_INTRA #if CONFIG_EXT_INTRA
...@@ -2191,6 +2225,15 @@ static int64_t rd_pick_intra_sby_mode(VP10_COMP *cpi, MACROBLOCK *x, ...@@ -2191,6 +2225,15 @@ static int64_t rd_pick_intra_sby_mode(VP10_COMP *cpi, MACROBLOCK *x,
continue; continue;
this_rate = this_rate_tokenonly + bmode_costs[mode]; this_rate = this_rate_tokenonly + bmode_costs[mode];
if (!xd->lossless[xd->mi[0]->mbmi.segment_id]) {
// super_block_yrd above includes the cost of the tx_size in the
// tokenonly rate, but for intra blocks, tx_size is always coded
// (prediction granularity), so we account for it in the full rate,
// not the tokenonly rate.
this_rate_tokenonly -= vp10_cost_tx_size(mic->mbmi.tx_size, max_tx_size,
tx_probs);
}
if (cpi->common.allow_screen_content_tools && mode == DC_PRED) if (cpi->common.allow_screen_content_tools && mode == DC_PRED)
this_rate += this_rate +=
vp10_cost_bit(vp10_default_palette_y_mode_prob[bsize - BLOCK_8X8] vp10_cost_bit(vp10_default_palette_y_mode_prob[bsize - BLOCK_8X8]
...@@ -5836,6 +5879,8 @@ void vp10_rd_pick_inter_mode_sb(VP10_COMP *cpi, ...@@ -5836,6 +5879,8 @@ void vp10_rd_pick_inter_mode_sb(VP10_COMP *cpi,
const int mode_search_skip_flags = sf->mode_search_skip_flags; const int mode_search_skip_flags = sf->mode_search_skip_flags;
int64_t mask_filter = 0; int64_t mask_filter = 0;
int64_t filter_cache[SWITCHABLE_FILTER_CONTEXTS]; int64_t filter_cache[SWITCHABLE_FILTER_CONTEXTS];
const TX_SIZE max_tx_size = max_txsize_lookup[bsize];
const vpx_prob *tx_probs = get_tx_probs2(max_tx_size, xd, &cm->fc->tx_probs);
vp10_zero(best_mbmode); vp10_zero(best_mbmode);
...@@ -6326,6 +6371,14 @@ void vp10_rd_pick_inter_mode_sb(VP10_COMP *cpi, ...@@ -6326,6 +6371,14 @@ void vp10_rd_pick_inter_mode_sb(VP10_COMP *cpi,
#endif // CONFIG_EXT_INTRA #endif // CONFIG_EXT_INTRA
rate2 = rate_y + intra_mode_cost[mbmi->mode] + rate_uv_intra[uv_tx]; rate2 = rate_y + intra_mode_cost[mbmi->mode] + rate_uv_intra[uv_tx];
if (!xd->lossless[mbmi->segment_id]) {
// super_block_yrd above includes the cost of the tx_size in the
// tokenonly rate, but for intra blocks, tx_size is always coded
// (prediction granularity), so we account for it in the full rate,
// not the tokenonly rate.
rate_y -= vp10_cost_tx_size(mbmi->tx_size, max_tx_size, tx_probs);
}
#if CONFIG_EXT_INTRA #if CONFIG_EXT_INTRA
if (is_directional_mode) { if (is_directional_mode) {
int p_angle; int p_angle;
...@@ -7502,6 +7555,23 @@ void vp10_rd_pick_inter_mode_sub8x8(struct VP10_COMP *cpi, ...@@ -7502,6 +7555,23 @@ void vp10_rd_pick_inter_mode_sub8x8(struct VP10_COMP *cpi,
for (i = 0; i < 4; i++) for (i = 0; i < 4; i++)
xd->mi[0]->bmi[i] = tmp_best_bmodes[i]; xd->mi[0]->bmi[i] = tmp_best_bmodes[i];
} }
// Add in the cost of the transform type
if (!xd->lossless[mbmi->segment_id]) {
int rate_tx_type = 0;
#if CONFIG_EXT_TX
if (get_ext_tx_types(mbmi->tx_size, bsize, 1) > 1) {
const int eset = get_ext_tx_set(mbmi->tx_size, bsize, 1);
rate_tx_type =
cpi->inter_tx_type_costs[eset][mbmi->tx_size][mbmi->tx_type];
}
#else
if (mbmi->tx_size < TX_32X32) {
rate_tx_type = cpi->inter_tx_type_costs[mbmi->tx_size][mbmi->tx_type];
}
#endif
rate += rate_tx_type;
rate_y += rate_tx_type;
}
rate2 += rate; rate2 += rate;
distortion2 += distortion; distortion2 += distortion;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment