Commit 0123bc7d authored by David Barker's avatar David Barker Committed by Steinar Midtskogen

cdef-singlepass: Fix integer overflow

When cdef-singlepass is enabled, it is possible to signal the
CDEF parameters in such a way that we end up with unsigned
integer overflow in constrain().

Fix this by i) using signed instead of unsigned values,
and ii) clamping the result to avoid going on to shift
by a negative amount.

Change-Id: Ib677b2d644e44000c54959f7280e646bf02054da
parent 87b01b5a
......@@ -24,14 +24,12 @@
static INLINE int sign(int i) { return i < 0 ? -1 : 1; }
static INLINE int constrain(int diff, int threshold, unsigned int damping) {
return threshold
? sign(diff) *
AOMMIN(abs(diff),
AOMMAX(0,
threshold - (abs(diff) >>
(damping - get_msb(threshold)))))
: 0;
static INLINE int constrain(int diff, int threshold, int damping) {
if (!threshold) return 0;
const int shift = AOMMAX(0, damping - get_msb(threshold));
return sign(diff) *
AOMMIN(abs(diff), AOMMAX(0, threshold - (abs(diff) >> shift)));
}
#ifdef __cplusplus
......
......@@ -268,8 +268,10 @@ void SIMD_FUNC(cdef_filter_block_4x4_8)(uint8_t *dst, int dstride,
const int *pri_taps = cdef_pri_taps[pri_strength & 1];
const int *sec_taps = cdef_sec_taps[pri_strength & 1];
if (pri_strength) pri_damping -= get_msb(pri_strength);
if (sec_strength) sec_damping -= get_msb(sec_strength);
if (pri_strength)
pri_damping = AOMMAX(0, pri_damping - get_msb(pri_strength));
if (sec_strength)
sec_damping = AOMMAX(0, sec_damping - get_msb(sec_strength));
sum = v256_zero();
row = v256_from_v64(v64_load_aligned(&in[0 * CDEF_BSTRIDE]),
......@@ -504,8 +506,10 @@ void SIMD_FUNC(cdef_filter_block_8x8_8)(uint8_t *dst, int dstride,
const int *pri_taps = cdef_pri_taps[pri_strength & 1];
const int *sec_taps = cdef_sec_taps[pri_strength & 1];
if (pri_strength) pri_damping -= get_msb(pri_strength);
if (sec_strength) sec_damping -= get_msb(sec_strength);
if (pri_strength)
pri_damping = AOMMAX(0, pri_damping - get_msb(pri_strength));
if (sec_strength)
sec_damping = AOMMAX(0, sec_damping - get_msb(sec_strength));
for (i = 0; i < 8; i += 2) {
sum = v256_zero();
row = v256_from_v128(v128_load_aligned(&in[i * CDEF_BSTRIDE]),
......@@ -717,8 +721,10 @@ void SIMD_FUNC(cdef_filter_block_4x4_16)(uint16_t *dst, int dstride,
const int *pri_taps = cdef_pri_taps[pri_strength & 1];
const int *sec_taps = cdef_sec_taps[pri_strength & 1];
if (pri_strength) pri_damping -= get_msb(pri_strength);
if (sec_strength) sec_damping -= get_msb(sec_strength);
if (pri_strength)
pri_damping = AOMMAX(0, pri_damping - get_msb(pri_strength));
if (sec_strength)
sec_damping = AOMMAX(0, sec_damping - get_msb(sec_strength));
for (i = 0; i < 4; i += 2) {
sum = v128_zero();
row = v128_from_v64(v64_load_aligned(&in[i * CDEF_BSTRIDE]),
......@@ -888,8 +894,10 @@ void SIMD_FUNC(cdef_filter_block_8x8_16)(uint16_t *dst, int dstride,
const int *pri_taps = cdef_pri_taps[pri_strength & 1];
const int *sec_taps = cdef_sec_taps[pri_strength & 1];
if (pri_strength) pri_damping -= get_msb(pri_strength);
if (sec_strength) sec_damping -= get_msb(sec_strength);
if (pri_strength)
pri_damping = AOMMAX(0, pri_damping - get_msb(pri_strength));
if (sec_strength)
sec_damping = AOMMAX(0, sec_damping - get_msb(sec_strength));
for (i = 0; i < 8; i++) {
sum = v128_zero();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment