Commit c7a5e883 authored by Debargha Mukherjee's avatar Debargha Mukherjee

Fix a 12-bit convolve bug

Fixes several failing 12-bit tests.

Change-Id: If6da50bbd595adb270cb11820df97b34a22d7b82
parent a840cde6
......@@ -1290,6 +1290,7 @@ void av1_highbd_convolve_2d_facade(const uint8_t *src8, int src_stride,
// 0-bit rounding just to convert from int32 to uint16
const int rbits =
2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
assert(rbits >= 0);
av1_highbd_convolve_rounding(tmp_dst, tmp_dst_stride, dst, dst_stride, w, h,
rbits, bd);
}
......
......@@ -105,11 +105,13 @@ static INLINE ConvolveParams get_conv_params_round(int ref, int do_average,
conv_params.is_compound = 0;
conv_params.dst = NULL;
conv_params.dst_stride = 0;
#if CONFIG_LOWPRECISION_BLEND
const int intbufrange = bd + FILTER_BITS - conv_params.round_0 + 2;
if (bd < 12) assert(intbufrange <= 16);
if (intbufrange > 16) {
conv_params.round_0 += intbufrange - 16;
}
#endif // CONFIG_LOWPRECISION_BLEND
return conv_params;
}
......@@ -127,16 +129,15 @@ static INLINE ConvolveParams get_conv_params_no_round(int ref, int do_average,
#if CONFIG_LOWPRECISION_BLEND
conv_params.round_1 = is_compound ? COMPOUND_ROUND1_BITS
: 2 * FILTER_BITS - conv_params.round_0;
#else
conv_params.round_1 = 0;
#endif
const int intbufrange = bd + FILTER_BITS - conv_params.round_0 + 2;
if (bd < 12) assert(intbufrange <= 16);
if (intbufrange > 16) {
conv_params.round_0 += intbufrange - 16;
if (is_compound && conv_params.round_1 > 0)
conv_params.round_1 -= intbufrange - 16;
conv_params.round_1 -= intbufrange - 16;
}
#else
conv_params.round_1 = 0;
#endif // CONFIG_LOWPRECISION_BLEND
// TODO(yunqing): The following dst should only be valid while
// is_compound = 1;
conv_params.dst = dst;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment