Commit cc894a18 authored by James Zern's avatar James Zern
Browse files

Fix variance (signed integer) overflow

based on change made in experimental:
  98473443 Fix variance (signed integer) overflow

Change-Id: I36f4ba5700f6f4615057daf7e70868f68a86669f
parent 0ef72720
...@@ -75,7 +75,7 @@ unsigned int vp8_variance16x16_c( ...@@ -75,7 +75,7 @@ unsigned int vp8_variance16x16_c(
variance(src_ptr, source_stride, ref_ptr, recon_stride, 16, 16, &var, &avg); variance(src_ptr, source_stride, ref_ptr, recon_stride, 16, 16, &var, &avg);
*sse = var; *sse = var;
return (var - ((unsigned int)(avg * avg) >> 8)); return (var - (((unsigned int)avg * avg) >> 8));
} }
unsigned int vp8_variance8x16_c( unsigned int vp8_variance8x16_c(
...@@ -91,7 +91,7 @@ unsigned int vp8_variance8x16_c( ...@@ -91,7 +91,7 @@ unsigned int vp8_variance8x16_c(
variance(src_ptr, source_stride, ref_ptr, recon_stride, 8, 16, &var, &avg); variance(src_ptr, source_stride, ref_ptr, recon_stride, 8, 16, &var, &avg);
*sse = var; *sse = var;
return (var - ((unsigned int)(avg * avg) >> 7)); return (var - (((unsigned int)avg * avg) >> 7));
} }
unsigned int vp8_variance16x8_c( unsigned int vp8_variance16x8_c(
...@@ -107,7 +107,7 @@ unsigned int vp8_variance16x8_c( ...@@ -107,7 +107,7 @@ unsigned int vp8_variance16x8_c(
variance(src_ptr, source_stride, ref_ptr, recon_stride, 16, 8, &var, &avg); variance(src_ptr, source_stride, ref_ptr, recon_stride, 16, 8, &var, &avg);
*sse = var; *sse = var;
return (var - ((unsigned int)(avg * avg) >> 7)); return (var - (((unsigned int)avg * avg) >> 7));
} }
...@@ -124,7 +124,7 @@ unsigned int vp8_variance8x8_c( ...@@ -124,7 +124,7 @@ unsigned int vp8_variance8x8_c(
variance(src_ptr, source_stride, ref_ptr, recon_stride, 8, 8, &var, &avg); variance(src_ptr, source_stride, ref_ptr, recon_stride, 8, 8, &var, &avg);
*sse = var; *sse = var;
return (var - ((unsigned int)(avg * avg) >> 6)); return (var - (((unsigned int)avg * avg) >> 6));
} }
unsigned int vp8_variance4x4_c( unsigned int vp8_variance4x4_c(
...@@ -140,7 +140,7 @@ unsigned int vp8_variance4x4_c( ...@@ -140,7 +140,7 @@ unsigned int vp8_variance4x4_c(
variance(src_ptr, source_stride, ref_ptr, recon_stride, 4, 4, &var, &avg); variance(src_ptr, source_stride, ref_ptr, recon_stride, 4, 4, &var, &avg);
*sse = var; *sse = var;
return (var - ((unsigned int)(avg * avg) >> 4)); return (var - (((unsigned int)avg * avg) >> 4));
} }
......
...@@ -91,7 +91,7 @@ unsigned int vp8_variance4x4_mmx( ...@@ -91,7 +91,7 @@ unsigned int vp8_variance4x4_mmx(
vp8_get4x4var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg) ; vp8_get4x4var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg) ;
*sse = var; *sse = var;
return (var - ((unsigned int)(avg * avg) >> 4)); return (var - (((unsigned int)avg * avg) >> 4));
} }
...@@ -108,7 +108,7 @@ unsigned int vp8_variance8x8_mmx( ...@@ -108,7 +108,7 @@ unsigned int vp8_variance8x8_mmx(
vp8_get8x8var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg) ; vp8_get8x8var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg) ;
*sse = var; *sse = var;
return (var - ((unsigned int)(avg * avg) >> 6)); return (var - (((unsigned int)avg * avg) >> 6));
} }
...@@ -153,7 +153,7 @@ unsigned int vp8_variance16x16_mmx( ...@@ -153,7 +153,7 @@ unsigned int vp8_variance16x16_mmx(
var = sse0 + sse1 + sse2 + sse3; var = sse0 + sse1 + sse2 + sse3;
avg = sum0 + sum1 + sum2 + sum3; avg = sum0 + sum1 + sum2 + sum3;
*sse = var; *sse = var;
return (var - ((unsigned int)(avg * avg) >> 8)); return (var - (((unsigned int)avg * avg) >> 8));
} }
unsigned int vp8_variance16x8_mmx( unsigned int vp8_variance16x8_mmx(
...@@ -172,7 +172,7 @@ unsigned int vp8_variance16x8_mmx( ...@@ -172,7 +172,7 @@ unsigned int vp8_variance16x8_mmx(
var = sse0 + sse1; var = sse0 + sse1;
avg = sum0 + sum1; avg = sum0 + sum1;
*sse = var; *sse = var;
return (var - ((unsigned int)(avg * avg) >> 7)); return (var - (((unsigned int)avg * avg) >> 7));
} }
...@@ -194,7 +194,7 @@ unsigned int vp8_variance8x16_mmx( ...@@ -194,7 +194,7 @@ unsigned int vp8_variance8x16_mmx(
avg = sum0 + sum1; avg = sum0 + sum1;
*sse = var; *sse = var;
return (var - ((unsigned int)(avg * avg) >> 7)); return (var - (((unsigned int)avg * avg) >> 7));
} }
...@@ -219,7 +219,7 @@ unsigned int vp8_sub_pixel_variance4x4_mmx ...@@ -219,7 +219,7 @@ unsigned int vp8_sub_pixel_variance4x4_mmx
&xsum, &xxsum &xsum, &xxsum
); );
*sse = xxsum; *sse = xxsum;
return (xxsum - ((unsigned int)(xsum * xsum) >> 4)); return (xxsum - (((unsigned int)xsum * xsum) >> 4));
} }
...@@ -244,7 +244,7 @@ unsigned int vp8_sub_pixel_variance8x8_mmx ...@@ -244,7 +244,7 @@ unsigned int vp8_sub_pixel_variance8x8_mmx
&xsum, &xxsum &xsum, &xxsum
); );
*sse = xxsum; *sse = xxsum;
return (xxsum - ((unsigned int)(xsum * xsum) >> 6)); return (xxsum - (((unsigned int)xsum * xsum) >> 6));
} }
unsigned int vp8_sub_pixel_variance16x16_mmx unsigned int vp8_sub_pixel_variance16x16_mmx
...@@ -282,7 +282,7 @@ unsigned int vp8_sub_pixel_variance16x16_mmx ...@@ -282,7 +282,7 @@ unsigned int vp8_sub_pixel_variance16x16_mmx
xxsum0 += xxsum1; xxsum0 += xxsum1;
*sse = xxsum0; *sse = xxsum0;
return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 8)); return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
} }
...@@ -335,7 +335,7 @@ unsigned int vp8_sub_pixel_variance16x8_mmx ...@@ -335,7 +335,7 @@ unsigned int vp8_sub_pixel_variance16x8_mmx
xxsum0 += xxsum1; xxsum0 += xxsum1;
*sse = xxsum0; *sse = xxsum0;
return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 7)); return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 7));
} }
unsigned int vp8_sub_pixel_variance8x16_mmx unsigned int vp8_sub_pixel_variance8x16_mmx
...@@ -358,7 +358,7 @@ unsigned int vp8_sub_pixel_variance8x16_mmx ...@@ -358,7 +358,7 @@ unsigned int vp8_sub_pixel_variance8x16_mmx
&xsum, &xxsum &xsum, &xxsum
); );
*sse = xxsum; *sse = xxsum;
return (xxsum - ((unsigned int)(xsum * xsum) >> 7)); return (xxsum - (((unsigned int)xsum * xsum) >> 7));
} }
......
...@@ -148,7 +148,7 @@ unsigned int vp8_variance4x4_wmt( ...@@ -148,7 +148,7 @@ unsigned int vp8_variance4x4_wmt(
vp8_get4x4var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg) ; vp8_get4x4var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg) ;
*sse = var; *sse = var;
return (var - ((unsigned int)(avg * avg) >> 4)); return (var - (((unsigned int)avg * avg) >> 4));
} }
...@@ -165,7 +165,7 @@ unsigned int vp8_variance8x8_wmt ...@@ -165,7 +165,7 @@ unsigned int vp8_variance8x8_wmt
vp8_get8x8var_sse2(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg) ; vp8_get8x8var_sse2(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg) ;
*sse = var; *sse = var;
return (var - ((unsigned int)(avg * avg) >> 6)); return (var - (((unsigned int)avg * avg) >> 6));
} }
...@@ -184,7 +184,7 @@ unsigned int vp8_variance16x16_wmt ...@@ -184,7 +184,7 @@ unsigned int vp8_variance16x16_wmt
vp8_get16x16var_sse2(src_ptr, source_stride, ref_ptr, recon_stride, &sse0, &sum0) ; vp8_get16x16var_sse2(src_ptr, source_stride, ref_ptr, recon_stride, &sse0, &sum0) ;
*sse = sse0; *sse = sse0;
return (sse0 - ((unsigned int)(sum0 * sum0) >> 8)); return (sse0 - (((unsigned int)sum0 * sum0) >> 8));
} }
unsigned int vp8_mse16x16_wmt( unsigned int vp8_mse16x16_wmt(
const unsigned char *src_ptr, const unsigned char *src_ptr,
...@@ -220,7 +220,7 @@ unsigned int vp8_variance16x8_wmt ...@@ -220,7 +220,7 @@ unsigned int vp8_variance16x8_wmt
var = sse0 + sse1; var = sse0 + sse1;
avg = sum0 + sum1; avg = sum0 + sum1;
*sse = var; *sse = var;
return (var - ((unsigned int)(avg * avg) >> 7)); return (var - (((unsigned int)avg * avg) >> 7));
} }
...@@ -241,7 +241,7 @@ unsigned int vp8_variance8x16_wmt ...@@ -241,7 +241,7 @@ unsigned int vp8_variance8x16_wmt
var = sse0 + sse1; var = sse0 + sse1;
avg = sum0 + sum1; avg = sum0 + sum1;
*sse = var; *sse = var;
return (var - ((unsigned int)(avg * avg) >> 7)); return (var - (((unsigned int)avg * avg) >> 7));
} }
...@@ -265,7 +265,7 @@ unsigned int vp8_sub_pixel_variance4x4_wmt ...@@ -265,7 +265,7 @@ unsigned int vp8_sub_pixel_variance4x4_wmt
&xsum, &xxsum &xsum, &xxsum
); );
*sse = xxsum; *sse = xxsum;
return (xxsum - ((unsigned int)(xsum * xsum) >> 4)); return (xxsum - (((unsigned int)xsum * xsum) >> 4));
} }
...@@ -314,7 +314,7 @@ unsigned int vp8_sub_pixel_variance8x8_wmt ...@@ -314,7 +314,7 @@ unsigned int vp8_sub_pixel_variance8x8_wmt
} }
*sse = xxsum; *sse = xxsum;
return (xxsum - ((unsigned int)(xsum * xsum) >> 6)); return (xxsum - (((unsigned int)xsum * xsum) >> 6));
} }
unsigned int vp8_sub_pixel_variance16x16_wmt unsigned int vp8_sub_pixel_variance16x16_wmt
...@@ -376,7 +376,7 @@ unsigned int vp8_sub_pixel_variance16x16_wmt ...@@ -376,7 +376,7 @@ unsigned int vp8_sub_pixel_variance16x16_wmt
} }
*sse = xxsum0; *sse = xxsum0;
return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 8)); return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
} }
unsigned int vp8_sub_pixel_mse16x16_wmt( unsigned int vp8_sub_pixel_mse16x16_wmt(
...@@ -447,7 +447,7 @@ unsigned int vp8_sub_pixel_variance16x8_wmt ...@@ -447,7 +447,7 @@ unsigned int vp8_sub_pixel_variance16x8_wmt
} }
*sse = xxsum0; *sse = xxsum0;
return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 7)); return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 7));
} }
unsigned int vp8_sub_pixel_variance8x16_wmt unsigned int vp8_sub_pixel_variance8x16_wmt
...@@ -495,7 +495,7 @@ unsigned int vp8_sub_pixel_variance8x16_wmt ...@@ -495,7 +495,7 @@ unsigned int vp8_sub_pixel_variance8x16_wmt
} }
*sse = xxsum; *sse = xxsum;
return (xxsum - ((unsigned int)(xsum * xsum) >> 7)); return (xxsum - (((unsigned int)xsum * xsum) >> 7));
} }
...@@ -515,7 +515,7 @@ unsigned int vp8_variance_halfpixvar16x16_h_wmt( ...@@ -515,7 +515,7 @@ unsigned int vp8_variance_halfpixvar16x16_h_wmt(
&xsum0, &xxsum0); &xsum0, &xxsum0);
*sse = xxsum0; *sse = xxsum0;
return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 8)); return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
} }
...@@ -534,7 +534,7 @@ unsigned int vp8_variance_halfpixvar16x16_v_wmt( ...@@ -534,7 +534,7 @@ unsigned int vp8_variance_halfpixvar16x16_v_wmt(
&xsum0, &xxsum0); &xsum0, &xxsum0);
*sse = xxsum0; *sse = xxsum0;
return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 8)); return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
} }
...@@ -554,5 +554,5 @@ unsigned int vp8_variance_halfpixvar16x16_hv_wmt( ...@@ -554,5 +554,5 @@ unsigned int vp8_variance_halfpixvar16x16_hv_wmt(
&xsum0, &xxsum0); &xsum0, &xxsum0);
*sse = xxsum0; *sse = xxsum0;
return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 8)); return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
} }
...@@ -113,7 +113,7 @@ unsigned int vp8_sub_pixel_variance16x16_ssse3 ...@@ -113,7 +113,7 @@ unsigned int vp8_sub_pixel_variance16x16_ssse3
} }
*sse = xxsum0; *sse = xxsum0;
return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 8)); return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
} }
unsigned int vp8_sub_pixel_variance16x8_ssse3 unsigned int vp8_sub_pixel_variance16x8_ssse3
...@@ -162,5 +162,5 @@ unsigned int vp8_sub_pixel_variance16x8_ssse3 ...@@ -162,5 +162,5 @@ unsigned int vp8_sub_pixel_variance16x8_ssse3
} }
*sse = xxsum0; *sse = xxsum0;
return (xxsum0 - ((unsigned int)(xsum0 * xsum0) >> 7)); return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 7));
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment