Commit 71adf529 authored by Jingning Han's avatar Jingning Han

Revert "Clamp inverse transform coefficients"

This reverts commit 79b78b7d.

The transform coefficient range needs some more tuning.
Before we finalize on that front, directly applying clamping
would cause multiple unit test failure issues. Hence revert
this Cl temporarily.

BUG=aomedia:612

Change-Id: I1dd8680dee17289801c4a209275f05a498355c8e
parent acca889a
......@@ -80,10 +80,6 @@ static INLINE int clamp(int value, int low, int high) {
return value < low ? low : (value > high ? high : value);
}
static INLINE int64_t clamp_64(int64_t value, int64_t low, int64_t high) {
return value < low ? low : (value > high ? high : (int)value);
}
static INLINE double fclamp(double value, double low, double high) {
return value < low ? low : (value > high ? high : value);
}
......
......@@ -45,10 +45,6 @@ static INLINE tran_high_t check_range(tran_high_t input, int bd) {
assert(int_min <= input);
assert(input <= int_max);
(void)int_min;
#else
const int32_t int_max = (1 << (7 + bd)) - 1;
const int32_t int_min = -int_max - 1;
input = (int32_t)clamp_64(input, int_min, int_max);
#endif // CONFIG_COEFFICIENT_RANGE_CHECKING
(void)bd;
return input;
......
......@@ -21,9 +21,13 @@ void range_check_func(int32_t stage, const int32_t *input, const int32_t *buf,
range_check_func(stage, input, buf, size, bit)
#else
#define range_check(stage, input, buf, size, bit) \
clamp_buf((int32_t *)buf, size, bit)
void clamp_buf(const int32_t *buf, int32_t size, int8_t bit);
{ \
(void)stage; \
(void)input; \
(void)buf; \
(void)size; \
(void)bit; \
}
#endif
// TODO(angiebird): Make 1-d txfm functions static
......
......@@ -43,15 +43,13 @@ void range_check_func(int32_t stage, const int32_t *input, const int32_t *buf,
range_check_func(stage, input, buf, size, bit)
#else
#define range_check(stage, input, buf, size, bit) \
clamp_buf((int32_t *)buf, size, bit)
void clamp_buf(int32_t *buf, int32_t size, int8_t bit) {
const int64_t maxValue = (1LL << (bit - 1)) - 1;
const int64_t minValue = -(1LL << (bit - 1));
for (int i = 0; i < size; ++i)
buf[i] = (int32_t)clamp_64(buf[i], minValue, maxValue);
}
{ \
(void)stage; \
(void)input; \
(void)buf; \
(void)size; \
(void)bit; \
}
#endif
// TODO(angiebird): Make 1-d txfm functions static
......
......@@ -6600,6 +6600,10 @@ static uint32_t *get_non_zero_counts(FRAME_COUNTS *counts, TX_SIZE tx_size,
}
}
static INLINE int clamp_64(int64_t value, int low, int high) {
return value < low ? low : (value > high ? high : (int)value);
}
static void update_scan_prob(AV1_COMMON *cm, TX_SIZE tx_size, TX_TYPE tx_type,
int rate_16) {
FRAME_CONTEXT *pre_fc = cm->pre_fc;
......@@ -6617,7 +6621,7 @@ static void update_scan_prob(AV1_COMMON *cm, TX_SIZE tx_size, TX_TYPE tx_type,
(curr_prob * rate_16 + prev_prob * ((1 << 16) - rate_16)) >> 16;
// TODO(angiebird): reduce the bit usage of probabilities and remove
// clamp_64()
non_zero_prob[i] = (int32_t)clamp_64(pred_prob, 0, UINT16_MAX);
non_zero_prob[i] = clamp_64(pred_prob, 0, UINT16_MAX);
}
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment