Commit b7e7c044 authored by James Zern's avatar James Zern

Resolve -Wshorten-64-to-32 in highbd obmc/ext-inter variance.

For 8-bit the subtrahend is small enough to fit into uint32_t.
Prevent negative variance in 10/12-bit.
Previously:
51e0c54e Resolve -Wshorten-64-to-32 in highbd variance.

BUG=aomedia:445

Change-Id: Ibb9bfc5e926de354bbab80127012e5e1861348a3
parent e57473ae
......@@ -842,25 +842,29 @@ void highbd_12_masked_variance(const uint8_t *a8, int a_stride,
int sum; \
highbd_masked_variance(a, a_stride, b, b_stride, m, m_stride, W, H, sse, \
&sum); \
return *sse - (((int64_t)sum * sum) / (W * H)); \
return *sse - (unsigned int)(((int64_t)sum * sum) / (W * H)); \
} \
\
unsigned int aom_highbd_10_masked_variance##W##x##H##_c( \
const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, \
const uint8_t *m, int m_stride, unsigned int *sse) { \
int sum; \
int64_t var; \
highbd_10_masked_variance(a, a_stride, b, b_stride, m, m_stride, W, H, \
sse, &sum); \
return *sse - (((int64_t)sum * sum) / (W * H)); \
var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \
return (var >= 0) ? (uint32_t)var : 0; \
} \
\
unsigned int aom_highbd_12_masked_variance##W##x##H##_c( \
const uint8_t *a, int a_stride, const uint8_t *b, int b_stride, \
const uint8_t *m, int m_stride, unsigned int *sse) { \
int sum; \
int64_t var; \
highbd_12_masked_variance(a, a_stride, b, b_stride, m, m_stride, W, H, \
sse, &sum); \
return *sse - (((int64_t)sum * sum) / (W * H)); \
var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \
return (var >= 0) ? (uint32_t)var : 0; \
}
#define HIGHBD_MASK_SUBPIX_VAR(W, H) \
......@@ -1123,23 +1127,27 @@ static INLINE void highbd_12_obmc_variance(const uint8_t *pre8, int pre_stride,
const int32_t *mask, unsigned int *sse) { \
int sum; \
highbd_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \
return *sse - (((int64_t)sum * sum) / (W * H)); \
return *sse - (unsigned int)(((int64_t)sum * sum) / (W * H)); \
} \
\
unsigned int aom_highbd_10_obmc_variance##W##x##H##_c( \
const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
const int32_t *mask, unsigned int *sse) { \
int sum; \
int64_t var; \
highbd_10_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \
return *sse - (((int64_t)sum * sum) / (W * H)); \
var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \
return (var >= 0) ? (uint32_t)var : 0; \
} \
\
unsigned int aom_highbd_12_obmc_variance##W##x##H##_c( \
const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
const int32_t *mask, unsigned int *sse) { \
int sum; \
int64_t var; \
highbd_12_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \
return *sse - (((int64_t)sum * sum) / (W * H)); \
var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \
return (var >= 0) ? (uint32_t)var : 0; \
}
#define HIGHBD_OBMC_SUBPIX_VAR(W, H) \
......
......@@ -311,23 +311,27 @@ static INLINE void highbd_12_obmc_variance(const uint8_t *pre8, int pre_stride,
const int32_t *mask, unsigned int *sse) { \
int sum; \
highbd_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \
return *sse - (((int64_t)sum * sum) / (W * H)); \
return *sse - (unsigned int)(((int64_t)sum * sum) / (W * H)); \
} \
\
unsigned int aom_highbd_10_obmc_variance##W##x##H##_sse4_1( \
const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
const int32_t *mask, unsigned int *sse) { \
int sum; \
int64_t var; \
highbd_10_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \
return *sse - (((int64_t)sum * sum) / (W * H)); \
var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \
return (var >= 0) ? (uint32_t)var : 0; \
} \
\
unsigned int aom_highbd_12_obmc_variance##W##x##H##_sse4_1( \
const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
const int32_t *mask, unsigned int *sse) { \
int sum; \
int64_t var; \
highbd_12_obmc_variance(pre, pre_stride, wsrc, mask, W, H, sse, &sum); \
return *sse - (((int64_t)sum * sum) / (W * H)); \
var = (int64_t)(*sse) - (((int64_t)sum * sum) / (W * H)); \
return (var >= 0) ? (uint32_t)var : 0; \
}
#if CONFIG_EXT_PARTITION
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment