### [non-normative] Clarify clamping code for inverse transforms

```Although the clamping for the inverse transforms is still being
discussed, it is useful to have clear code.

The current code is not ideal - both av1_inv_txfm1d.c and av1_inv_txfm2d.c
have functions called clamp_buf(), but they do different things - one
clamps to the exact number of bits passed in, the other optionally
clamps to 16 bits.

This commit refactors things so that:

* clamp_* always clamps to the exact number of bits passed in
To compensate, we push the "if bit <= 16: clamp to 16 bits"
logic into the callers of the clamp_* functions

* We deduplicate some of the code

BUG=aomedia:1352

Change-Id: I1b9d85d6db5d291814c055a8113b7b054690b04c```
parent e63a5dbb
 ... ... @@ -59,28 +59,12 @@ void range_check_func(int32_t stage, const int32_t *input, const int32_t *buf, #define apply_range(stage, input, buf, size, bit) \ range_check_func(stage, input, buf, size, bit) #else #define apply_range(stage, input, buf, size, bit) \ clamp_buf((int32_t *)buf, size, bit) static INLINE void clamp_buf(int32_t *buf, int32_t size, int8_t bit) { if (bit <= 16) { const int32_t max_value = (1 << 15) - 1; const int32_t min_value = -(1 << 15); for (int i = 0; i < size; ++i) buf[i] = clamp(buf[i], min_value, max_value); } } #define apply_range(stage, input, buf, size, bit) \ do { \ if (bit <= 16) clamp_buf((int32_t *)buf, size, 16); \ } while (0) #endif static INLINE int32_t clamp_value(int32_t value, int8_t bit) { if (bit <= 16) { const int32_t max_value = (1 << 15) - 1; const int32_t min_value = -(1 << 15); return clamp(value, min_value, max_value); } return value; } // TODO(angiebird): Make 1-d txfm functions static void av1_idct4_new(const int32_t *input, int32_t *output, int8_t cos_bit, const int8_t *stage_range) { ... ... @@ -776,10 +760,12 @@ void av1_iadst4_new(const int32_t *input, int32_t *output, int8_t cos_bit, s4 = range_check_value(sinpi * x2, stage_range + bit); s5 = range_check_value(sinpi * x3, stage_range + bit); s6 = range_check_value(sinpi * x3, stage_range + bit); s7 = clamp_value(x0 - x2, stage_range); s7 = x0 - x2; if (stage_range <= 16) s7 = clamp_value(s7, 16); // stage 2 s7 = clamp_value(s7 + x3, stage_range); s7 = s7 + x3; if (stage_range <= 16) s7 = clamp_value(s7, 16); // stage 3 s0 = range_check_value(s0 + s3, stage_range + bit); ... ...
 ... ... @@ -18,6 +18,16 @@ extern "C" { #endif static INLINE int32_t clamp_value(int32_t value, int8_t bit) { const int64_t max_value = (1LL << (bit - 1)) - 1; const int64_t min_value = -(1LL << (bit - 1)); return (int32_t)clamp64(value, min_value, max_value); } static INLINE void clamp_buf(int32_t *buf, int32_t size, int8_t bit) { for (int i = 0; i < size; ++i) buf[i] = clamp_value(buf[i], bit); } void av1_idct4_new(const int32_t *input, int32_t *output, int8_t cos_bit, const int8_t *stage_range); void av1_idct8_new(const int32_t *input, int32_t *output, int8_t cos_bit, ... ...
 ... ... @@ -19,14 +19,6 @@ #define NO_INV_TRANSPOSE 1 static INLINE void clamp_buf(int32_t *buf, int32_t size, int8_t bit) { const int64_t max_value = (1LL << (bit - 1)) - 1; const int64_t min_value = -(1LL << (bit - 1)); for (int i = 0; i < size; ++i) buf[i] = (int32_t)clamp64(buf[i], min_value, max_value); } static INLINE TxfmFunc inv_txfm_type_to_func(TXFM_TYPE txfm_type) { switch (txfm_type) { case TXFM_TYPE_DCT4: return av1_idct4_new; ... ...
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!