From ee38c4184b46869616e1b0a0d207f34bbed8e7eb Mon Sep 17 00:00:00 2001 From: James Zern Date: Fri, 2 Nov 2012 18:59:52 -0700 Subject: [PATCH] loopfilter: prevent signed integer overflow use unsigned ints to extended filter values in vp9_mbloop_filter_horizontal_edge_c_sse2 Change-Id: I55ec3ac2bcb9baf55626b0384d151b07fc8e087d --- vp9/common/x86/loopfilter_x86.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/vp9/common/x86/loopfilter_x86.c b/vp9/common/x86/loopfilter_x86.c index 6aeddd387..1ce654092 100644 --- a/vp9/common/x86/loopfilter_x86.c +++ b/vp9/common/x86/loopfilter_x86.c @@ -97,13 +97,17 @@ void vp9_mbloop_filter_horizontal_edge_c_sse2(unsigned char *s, DECLARE_ALIGNED(16, unsigned char, flat_oq1[16]); DECLARE_ALIGNED(16, unsigned char, flat_oq0[16]); __m128i mask, hev, flat; - __m128i thresh, limit, blimit; const __m128i zero = _mm_set1_epi16(0); __m128i p4, p3, p2, p1, p0, q0, q1, q2, q3, q4; - - thresh = _mm_shuffle_epi32(_mm_cvtsi32_si128(_thresh[0] * 0x01010101), 0); - limit = _mm_shuffle_epi32(_mm_cvtsi32_si128(_limit[0] * 0x01010101), 0); - blimit = _mm_shuffle_epi32(_mm_cvtsi32_si128(_blimit[0] * 0x01010101), 0); + const unsigned int extended_thresh = _thresh[0] * 0x01010101u; + const unsigned int extended_limit = _limit[0] * 0x01010101u; + const unsigned int extended_blimit = _blimit[0] * 0x01010101u; + const __m128i thresh = + _mm_shuffle_epi32(_mm_cvtsi32_si128((int)extended_thresh), 0); + const __m128i limit = + _mm_shuffle_epi32(_mm_cvtsi32_si128((int)extended_limit), 0); + const __m128i blimit = + _mm_shuffle_epi32(_mm_cvtsi32_si128((int)extended_blimit), 0); p4 = _mm_loadu_si128((__m128i *)(s - 5 * p)); p3 = _mm_loadu_si128((__m128i *)(s - 4 * p)); -- GitLab