Commit 1e90dc19 authored by Frank Galligan's avatar Frank Galligan
Browse files

Fix signed integer overflow in warped_motion.c

BUG=https://bugs.chromium.org/p/aomedia/issues/detail?id=498

Change-Id: I007abe5dd0ad61586de56db3ad2b13f9fb71e5d9
parent 8203514e
......@@ -1558,16 +1558,16 @@ static int find_affine_int(int np, int *pts1, int *pts2, BLOCK_SIZE bsize,
wm->wmmat[2] = (int32_t)(ROUND_POWER_OF_TWO_SIGNED_64(v, shift));
v = Px[1] * (int64_t)iDet;
wm->wmmat[3] = (int32_t)(ROUND_POWER_OF_TWO_SIGNED_64(v, shift));
v = (dux * (1 << WARPEDMODEL_PREC_BITS)) - (int64_t)sux * wm->wmmat[2] -
(int64_t)suy * wm->wmmat[3];
v = ((int64_t)dux * (1 << WARPEDMODEL_PREC_BITS)) -
(int64_t)sux * wm->wmmat[2] - (int64_t)suy * wm->wmmat[3];
wm->wmmat[0] = (int32_t)(ROUND_POWER_OF_TWO_SIGNED(v, 3));
v = Py[0] * (int64_t)iDet;
wm->wmmat[4] = (int32_t)(ROUND_POWER_OF_TWO_SIGNED_64(v, shift));
v = Py[1] * (int64_t)iDet;
wm->wmmat[5] = (int32_t)(ROUND_POWER_OF_TWO_SIGNED_64(v, shift));
v = (duy * (1 << WARPEDMODEL_PREC_BITS)) - (int64_t)sux * wm->wmmat[4] -
(int64_t)suy * wm->wmmat[5];
v = ((int64_t)duy * (1 << WARPEDMODEL_PREC_BITS)) -
(int64_t)sux * wm->wmmat[4] - (int64_t)suy * wm->wmmat[5];
wm->wmmat[1] = (int32_t)(ROUND_POWER_OF_TWO_SIGNED(v, 3));
wm->wmmat[6] = wm->wmmat[7] = 0;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment