Commit 79b78b7d authored by Sebastien Alaiwan's avatar Sebastien Alaiwan Committed by Frédéric BARBIER

Clamp inverse transform coefficients

When --enable-coefficient-range-checking isn't specificed, clamp the
coefficient at each stage.

This doesn't change the decoder behaviour for existing AV1 streams.
However, some AV1 bitstreams that would have been rejected by the
decoder as illegal (range check failure) are now legal bitstreams.

There is no impact on video quality.

BUG=aomedia:30

Change-Id: Ifa01186bae6bfe5d7712298e33d964c20f88435e
parent b0157aa6
......@@ -80,6 +80,10 @@ static INLINE int clamp(int value, int low, int high) {
return value < low ? low : (value > high ? high : value);
}
static INLINE int64_t clamp_64(int64_t value, int64_t low, int64_t high) {
return value < low ? low : (value > high ? high : (int)value);
}
static INLINE double fclamp(double value, double low, double high) {
return value < low ? low : (value > high ? high : value);
}
......
......@@ -45,6 +45,10 @@ static INLINE tran_high_t check_range(tran_high_t input, int bd) {
assert(int_min <= input);
assert(input <= int_max);
(void)int_min;
#else
const int32_t int_max = (1 << (7 + bd)) - 1;
const int32_t int_min = -int_max - 1;
input = (int32_t)clamp_64(input, int_min, int_max);
#endif // CONFIG_COEFFICIENT_RANGE_CHECKING
(void)bd;
return input;
......
......@@ -21,13 +21,9 @@ void range_check_func(int32_t stage, const int32_t *input, const int32_t *buf,
range_check_func(stage, input, buf, size, bit)
#else
#define range_check(stage, input, buf, size, bit) \
{ \
(void)stage; \
(void)input; \
(void)buf; \
(void)size; \
(void)bit; \
}
clamp_buf((int32_t *)buf, size, bit)
void clamp_buf(const int32_t *buf, int32_t size, int8_t bit);
#endif
// TODO(angiebird): Make 1-d txfm functions static
......
......@@ -43,13 +43,15 @@ void range_check_func(int32_t stage, const int32_t *input, const int32_t *buf,
range_check_func(stage, input, buf, size, bit)
#else
#define range_check(stage, input, buf, size, bit) \
{ \
(void)stage; \
(void)input; \
(void)buf; \
(void)size; \
(void)bit; \
}
clamp_buf((int32_t *)buf, size, bit)
void clamp_buf(int32_t *buf, int32_t size, int8_t bit) {
const int64_t maxValue = (1LL << (bit - 1)) - 1;
const int64_t minValue = -(1LL << (bit - 1));
for (int i = 0; i < size; ++i)
buf[i] = (int32_t)clamp_64(buf[i], minValue, maxValue);
}
#endif
// TODO(angiebird): Make 1-d txfm functions static
......
......@@ -6600,10 +6600,6 @@ static uint32_t *get_non_zero_counts(FRAME_COUNTS *counts, TX_SIZE tx_size,
}
}
static INLINE int clamp_64(int64_t value, int low, int high) {
return value < low ? low : (value > high ? high : (int)value);
}
static void update_scan_prob(AV1_COMMON *cm, TX_SIZE tx_size, TX_TYPE tx_type,
int rate_16) {
FRAME_CONTEXT *pre_fc = cm->pre_fc;
......@@ -6621,7 +6617,7 @@ static void update_scan_prob(AV1_COMMON *cm, TX_SIZE tx_size, TX_TYPE tx_type,
(curr_prob * rate_16 + prev_prob * ((1 << 16) - rate_16)) >> 16;
// TODO(angiebird): reduce the bit usage of probabilities and remove
// clamp_64()
non_zero_prob[i] = clamp_64(pred_prob, 0, UINT16_MAX);
non_zero_prob[i] = (int32_t)clamp_64(pred_prob, 0, UINT16_MAX);
}
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment