Commit 8620dea8 authored by Alex Converse's avatar Alex Converse

variance aq: Fix a variance calculation overflow bug.

This is an actual overflow where the result of the calculation is
materially changed, not just a negative value that is stored in an
unsigned.

Caught with fsanitize=integer on the VP9/AqSegmentTest.TestNoMisMatchAQ2/1 test.

Change-Id: I514b0ef4ae7ad50e3e08c0079aa204d59fa679aa
parent 3c223180
......@@ -167,7 +167,7 @@ static unsigned int block_variance(VP9_COMP *cpi, MACROBLOCK *x,
vp9_64_zeros, 0, bw, bh, &sse, &avg);
#endif // CONFIG_VP9_HIGHBITDEPTH
var = sse - (((int64_t)avg * avg) / (bw * bh));
return (256 * var) / (bw * bh);
return (unsigned int)(((uint64_t)256 * var) / (bw * bh));
} else {
#if CONFIG_VP9_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
......@@ -185,7 +185,7 @@ static unsigned int block_variance(VP9_COMP *cpi, MACROBLOCK *x,
x->plane[0].src.stride,
vp9_64_zeros, 0, &sse);
#endif // CONFIG_VP9_HIGHBITDEPTH
return (256 * var) >> num_pels_log2_lookup[bs];
return (unsigned int)(((uint64_t)256 * var) >> num_pels_log2_lookup[bs]);
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment