Commit b101b1f4 authored by Yaowu Xu's avatar Yaowu Xu

Clarify value ranges

This commit clarifies integer value range for vairables used in
several variance functions, also change to use proper type
conversion to reflect the value ranges.

Ported from libvpx/master: #a564b18d

Change-Id: I94bf12a8567766003ec667b4ebe9b820596d54ca
parent 2c82d97b
......@@ -9,6 +9,8 @@
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <assert.h>
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
#include "aom/aom_integer.h"
......@@ -38,7 +40,9 @@ uint32_t aom_variance_halfpixvar16x16_h_sse2(const unsigned char *src,
&xsum0, &xxsum0);
*sse = xxsum0;
return (xxsum0 - (((uint32_t)xsum0 * xsum0) >> 8));
assert(xsum0 <= 255 * 16 * 16);
assert(xsum0 >= -255 * 16 * 16);
return (xxsum0 - ((uint32_t)((int64_t)xsum0 * xsum0) >> 8));
}
uint32_t aom_variance_halfpixvar16x16_v_sse2(const unsigned char *src,
......@@ -51,7 +55,9 @@ uint32_t aom_variance_halfpixvar16x16_v_sse2(const unsigned char *src,
&xxsum0);
*sse = xxsum0;
return (xxsum0 - (((uint32_t)xsum0 * xsum0) >> 8));
assert(xsum0 <= 255 * 16 * 16);
assert(xsum0 >= -255 * 16 * 16);
return (xxsum0 - ((uint32_t)((int64_t)xsum0 * xsum0) >> 8));
}
uint32_t aom_variance_halfpixvar16x16_hv_sse2(const unsigned char *src,
......@@ -65,5 +71,7 @@ uint32_t aom_variance_halfpixvar16x16_hv_sse2(const unsigned char *src,
&xsum0, &xxsum0);
*sse = xxsum0;
return (xxsum0 - (((uint32_t)xsum0 * xsum0) >> 8));
assert(xsum0 <= 255 * 16 * 16);
assert(xsum0 >= -255 * 16 * 16);
return (xxsum0 - ((uint32_t)((int64_t)xsum0 * xsum0) >> 8));
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment