Commit 2becffae authored by Yaowu Xu's avatar Yaowu Xu

Properly set the estimate of rate_nocoef

This commit fixes the computation of rate_nocoef for situation when
rate_y is uninitialized at INT_MAX for  x->skip is true.

Change-Id: If3dde4e4ee16667f4408067d3bb3084f916272f1
parent dcbbb816
......@@ -1654,6 +1654,7 @@ static void rd_pick_sb_modes(VP10_COMP *cpi,
totalrate_nocoef,
#endif // CONFIG_SUPERTX
bsize, ctx, best_rd);
assert(*totalrate_nocoef >= 0);
}
} else {
vp10_rd_pick_inter_mode_sub8x8(cpi, tile_data, x, mi_row, mi_col, rd_cost,
......@@ -1661,6 +1662,7 @@ static void rd_pick_sb_modes(VP10_COMP *cpi,
totalrate_nocoef,
#endif // CONFIG_SUPERTX
bsize, ctx, best_rd);
assert(*totalrate_nocoef >= 0);
}
}
......
......@@ -8333,11 +8333,12 @@ void vp10_rd_pick_inter_mode_sb(VP10_COMP *cpi,
rd_cost->rate = rate2;
#if CONFIG_SUPERTX
*returnrate_nocoef = rate2 - rate_y - rate_uv;
if (!disable_skip) {
*returnrate_nocoef -= vp10_cost_bit(vp10_get_skip_prob(cm, xd),
skippable || this_skip2);
}
if (x->skip && rate_y == INT_MAX)
*returnrate_nocoef = rate2;
else
*returnrate_nocoef = rate2 - rate_y - rate_uv;
*returnrate_nocoef -= vp10_cost_bit(vp10_get_skip_prob(cm, xd),
disable_skip || skippable || this_skip2);
*returnrate_nocoef -= vp10_cost_bit(vp10_get_intra_inter_prob(cm, xd),
mbmi->ref_frame[0] != INTRA_FRAME);
#if CONFIG_OBMC
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment