Commit a3457cc2 authored by Sebastien Alaiwan's avatar Sebastien Alaiwan

Clamp inverse transform coeffs after 1st/2nd pass

Add clamping to 16/18/20 bits after first pass of the inverse
transform for 8/10/12 bit source espectively, and clampaing to 16
bits after second stage of the inverse transform.

BUG=aomedia:30

Change-Id: I7a68334c77baecb359a4fb3b801fdfd9d5a62f29
parent df682310
......@@ -19,6 +19,14 @@
#define NO_INV_TRANSPOSE 1
static INLINE void clamp_buf(int32_t *buf, int32_t size, int8_t bit) {
const int64_t maxValue = (1LL << (bit - 1)) - 1;
const int64_t minValue = -(1LL << (bit - 1));
for (int i = 0; i < size; ++i)
buf[i] = (int32_t)clamp64(buf[i], minValue, maxValue);
}
static INLINE TxfmFunc inv_txfm_type_to_func(TXFM_TYPE txfm_type) {
switch (txfm_type) {
case TXFM_TYPE_DCT4: return av1_idct4_new;
......@@ -358,6 +366,7 @@ static INLINE void inv_txfm2d_add_c(const int32_t *input, uint16_t *output,
} else if (rect_type2_shift) {
av1_round_shift_array(buf_ptr, txfm_size_col, -rect_type2_shift);
}
clamp_buf(buf_ptr, txfm_size_col, bd + 8);
input += txfm_size_col;
buf_ptr += txfm_size_col;
}
......@@ -379,6 +388,7 @@ static INLINE void inv_txfm2d_add_c(const int32_t *input, uint16_t *output,
}
}
av1_round_shift_array(temp_out, txfm_size_row, -shift1);
clamp_buf(temp_out, txfm_size_row, bd + 1);
if (cfg->ud_flip == 0) {
for (r = 0; r < txfm_size_row; ++r) {
output[r * stride + c] =
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment