Commit bc707450 authored by Jingning Han's avatar Jingning Han

Fix integer overflow issue in rtc coding flow intra mode search

The overflow issue affects a variable that is only used in inter
mode. This commit fixes the ioc warning triggered in the intra
mode. It does not affect the compression performance.

Change-Id: I593d1b5650599de07f3e68176dd1442c6cb7bdbc
parent e820ca69
......@@ -658,7 +658,8 @@ static void block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate, int64_t *dist,
block = 0;
*rate = 0;
*dist = 0;
*sse = (*sse << 6) >> shift;
if (*sse < INT64_MAX)
*sse = (*sse << 6) >> shift;
for (r = 0; r < max_blocks_high; r += block_step) {
for (c = 0; c < num_4x4_w; c += block_step) {
if (c < max_blocks_wide) {
......@@ -912,6 +913,8 @@ static void estimate_block_intra(int plane, int block, BLOCK_SIZE plane_bsize,
// TODO(jingning): This needs further refactoring.
block_yrd(cpi, x, &rate, &dist, &is_skippable, &this_sse, 0,
bsize_tx, MIN(tx_size, TX_16X16));
// this_sse is a dummy variable here. Its value should remain INT64_MAX.
assert(this_sse == INT64_MAX);
x->skip_txfm[0] = is_skippable;
rate += vp9_cost_bit(vp9_get_skip_prob(&cpi->common, xd), is_skippable);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment