Commit 75cfadbd authored by Jingning Han's avatar Jingning Han
Browse files

Fix rd_filter_cache update in sub8x8 RD loop

Properly handle the rd_filter_cache update, when early termination
or skip prediction filter type check is triggered.

Change-Id: Ie7b9a75fed3358f45ffd15817f2b36670c14eb2d
parent 84acc988
...@@ -3876,6 +3876,7 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x, ...@@ -3876,6 +3876,7 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x,
int this_skip2 = 0; int this_skip2 = 0;
int64_t total_sse = INT_MAX; int64_t total_sse = INT_MAX;
int early_term = 0; int early_term = 0;
int64_t mask_rd = 0;
for (i = 0; i < TX_MODES; ++i) for (i = 0; i < TX_MODES; ++i)
tx_cache[i] = INT64_MAX; tx_cache[i] = INT64_MAX;
...@@ -4069,21 +4070,20 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x, ...@@ -4069,21 +4070,20 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x,
cpi->rd_thresh_sub8x8[segment_id][bsize][THR_GOLD] : this_rd_thresh; cpi->rd_thresh_sub8x8[segment_id][bsize][THR_GOLD] : this_rd_thresh;
xd->mi_8x8[0]->mbmi.tx_size = TX_4X4; xd->mi_8x8[0]->mbmi.tx_size = TX_4X4;
cpi->rd_filter_cache[SWITCHABLE_FILTERS] = INT64_MAX; for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i)
cpi->rd_filter_cache[i] = INT64_MAX;
if (cm->mcomp_filter_type != BILINEAR) { if (cm->mcomp_filter_type != BILINEAR) {
tmp_best_filter = EIGHTTAP; tmp_best_filter = EIGHTTAP;
if (x->source_variance < if (x->source_variance <
cpi->sf.disable_filter_search_var_thresh) { cpi->sf.disable_filter_search_var_thresh) {
tmp_best_filter = EIGHTTAP; tmp_best_filter = EIGHTTAP;
vp9_zero(cpi->rd_filter_cache);
} else if (cpi->sf.adaptive_pred_filter_type == 1 && } else if (cpi->sf.adaptive_pred_filter_type == 1 &&
ctx->pred_filter_type < SWITCHABLE) { ctx->pred_filter_type < SWITCHABLE) {
tmp_best_filter = ctx->pred_filter_type; tmp_best_filter = ctx->pred_filter_type;
vp9_zero(cpi->rd_filter_cache);
} else if (cpi->sf.adaptive_pred_filter_type == 2) { } else if (cpi->sf.adaptive_pred_filter_type == 2) {
tmp_best_filter = ctx->pred_filter_type < SWITCHABLE ? tmp_best_filter = ctx->pred_filter_type < SWITCHABLE ?
ctx->pred_filter_type : 0; ctx->pred_filter_type : 0;
vp9_zero(cpi->rd_filter_cache);
} else { } else {
for (switchable_filter_index = 0; for (switchable_filter_index = 0;
switchable_filter_index < SWITCHABLE_FILTERS; switchable_filter_index < SWITCHABLE_FILTERS;
...@@ -4105,7 +4105,6 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x, ...@@ -4105,7 +4105,6 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x,
if (tmp_rd == INT64_MAX) if (tmp_rd == INT64_MAX)
continue; continue;
cpi->rd_filter_cache[switchable_filter_index] = tmp_rd;
rs = get_switchable_rate(x); rs = get_switchable_rate(x);
rs_rd = RDCOST(x->rdmult, x->rddiv, rs, 0); rs_rd = RDCOST(x->rdmult, x->rddiv, rs, 0);
cpi->rd_filter_cache[SWITCHABLE_FILTERS] = cpi->rd_filter_cache[SWITCHABLE_FILTERS] =
...@@ -4114,6 +4113,9 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x, ...@@ -4114,6 +4113,9 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x,
if (cm->mcomp_filter_type == SWITCHABLE) if (cm->mcomp_filter_type == SWITCHABLE)
tmp_rd += rs_rd; tmp_rd += rs_rd;
cpi->rd_filter_cache[switchable_filter_index] = tmp_rd;
mask_rd = MAX(tmp_rd, mask_rd);
newbest = (tmp_rd < tmp_best_rd); newbest = (tmp_rd < tmp_best_rd);
if (newbest) { if (newbest) {
tmp_best_filter = mbmi->interp_filter; tmp_best_filter = mbmi->interp_filter;
...@@ -4361,16 +4363,17 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x, ...@@ -4361,16 +4363,17 @@ int64_t vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x,
cm->mcomp_filter_type != BILINEAR) { cm->mcomp_filter_type != BILINEAR) {
int64_t ref = cpi->rd_filter_cache[cm->mcomp_filter_type == SWITCHABLE ? int64_t ref = cpi->rd_filter_cache[cm->mcomp_filter_type == SWITCHABLE ?
SWITCHABLE_FILTERS : cm->mcomp_filter_type]; SWITCHABLE_FILTERS : cm->mcomp_filter_type];
int64_t adj_rd;
for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; i++) { for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; i++) {
int64_t adj_rd; if (ref == INT64_MAX)
// In cases of poor prediction, filter_cache[] can contain really big adj_rd = 0;
// values, which actually are bigger than this_rd itself. This can else if (cpi->rd_filter_cache[i] == INT64_MAX)
// cause negative best_filter_rd[] values, which is obviously silly. adj_rd = mask_rd - ref + 10;
// Therefore, if filter_cache < ref, we do an adjusted calculation. else
if (cpi->rd_filter_cache[i] >= ref) adj_rd = cpi->rd_filter_cache[i] - ref;
adj_rd = this_rd + cpi->rd_filter_cache[i] - ref;
else // FIXME(rbultje) do this for comppred also adj_rd += this_rd;
adj_rd = this_rd - (ref - cpi->rd_filter_cache[i]) * this_rd / ref;
best_filter_rd[i] = MIN(best_filter_rd[i], adj_rd); best_filter_rd[i] = MIN(best_filter_rd[i], adj_rd);
} }
} }
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment