Commit 92b49a6d authored by David Barker's avatar David Barker Committed by Angie Chiang

[non-normative] Clarify clamping code for inverse transforms

Although the clamping for the inverse transforms is still being
discussed, it is useful to have clear code.

The current code is not ideal - both av1_inv_txfm1d.c and av1_inv_txfm2d.c
have functions called clamp_buf(), but they do different things - one
clamps to the exact number of bits passed in, the other optionally
clamps to 16 bits.

This commit refactors things so that:

* clamp_* always clamps to the exact number of bits passed in
  To compensate, we push the "if bit <= 16: clamp to 16 bits"
  logic into the callers of the clamp_* functions

* We deduplicate some of the code

BUG=aomedia:1352

Change-Id: I1b9d85d6db5d291814c055a8113b7b054690b04c
parent e63a5dbb
......@@ -59,28 +59,12 @@ void range_check_func(int32_t stage, const int32_t *input, const int32_t *buf,
#define apply_range(stage, input, buf, size, bit) \
range_check_func(stage, input, buf, size, bit)
#else
#define apply_range(stage, input, buf, size, bit) \
clamp_buf((int32_t *)buf, size, bit)
static INLINE void clamp_buf(int32_t *buf, int32_t size, int8_t bit) {
if (bit <= 16) {
const int32_t max_value = (1 << 15) - 1;
const int32_t min_value = -(1 << 15);
for (int i = 0; i < size; ++i) buf[i] = clamp(buf[i], min_value, max_value);
}
}
#define apply_range(stage, input, buf, size, bit) \
do { \
if (bit <= 16) clamp_buf((int32_t *)buf, size, 16); \
} while (0)
#endif
static INLINE int32_t clamp_value(int32_t value, int8_t bit) {
if (bit <= 16) {
const int32_t max_value = (1 << 15) - 1;
const int32_t min_value = -(1 << 15);
return clamp(value, min_value, max_value);
}
return value;
}
// TODO(angiebird): Make 1-d txfm functions static
void av1_idct4_new(const int32_t *input, int32_t *output, int8_t cos_bit,
const int8_t *stage_range) {
......@@ -776,10 +760,12 @@ void av1_iadst4_new(const int32_t *input, int32_t *output, int8_t cos_bit,
s4 = range_check_value(sinpi[1] * x2, stage_range[1] + bit);
s5 = range_check_value(sinpi[2] * x3, stage_range[1] + bit);
s6 = range_check_value(sinpi[4] * x3, stage_range[1] + bit);
s7 = clamp_value(x0 - x2, stage_range[1]);
s7 = x0 - x2;
if (stage_range[1] <= 16) s7 = clamp_value(s7, 16);
// stage 2
s7 = clamp_value(s7 + x3, stage_range[2]);
s7 = s7 + x3;
if (stage_range[2] <= 16) s7 = clamp_value(s7, 16);
// stage 3
s0 = range_check_value(s0 + s3, stage_range[3] + bit);
......
......@@ -18,6 +18,16 @@
extern "C" {
#endif
static INLINE int32_t clamp_value(int32_t value, int8_t bit) {
const int64_t max_value = (1LL << (bit - 1)) - 1;
const int64_t min_value = -(1LL << (bit - 1));
return (int32_t)clamp64(value, min_value, max_value);
}
static INLINE void clamp_buf(int32_t *buf, int32_t size, int8_t bit) {
for (int i = 0; i < size; ++i) buf[i] = clamp_value(buf[i], bit);
}
void av1_idct4_new(const int32_t *input, int32_t *output, int8_t cos_bit,
const int8_t *stage_range);
void av1_idct8_new(const int32_t *input, int32_t *output, int8_t cos_bit,
......
......@@ -19,14 +19,6 @@
#define NO_INV_TRANSPOSE 1
static INLINE void clamp_buf(int32_t *buf, int32_t size, int8_t bit) {
const int64_t max_value = (1LL << (bit - 1)) - 1;
const int64_t min_value = -(1LL << (bit - 1));
for (int i = 0; i < size; ++i)
buf[i] = (int32_t)clamp64(buf[i], min_value, max_value);
}
static INLINE TxfmFunc inv_txfm_type_to_func(TXFM_TYPE txfm_type) {
switch (txfm_type) {
case TXFM_TYPE_DCT4: return av1_idct4_new;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment