Commit 80b437ce authored by Sebastien Alaiwan's avatar Sebastien Alaiwan Committed by Debargha Mukherjee
Browse files

Clamp inverse transform coefficients

When --enable-coefficient-range-checking isn't specified,
clamp the coefficient at each stage.

The decoder behaviour is unchanged for valid existing AV1 streams.
However, some AV1 bitstreams that would have been rejected by the
decoder as illegal (range check failure) are now legal bitstreams.

There is no impact on video quality for valid bitstreams,
only bitstreams that would have been considered invalid are affected,
and are now considered as valid ones.

BUG=aomedia:30

Change-Id: Ie72ceeb3cb0561e8d3259767b209a4ef3be90438
parent 3fcce0bb
......@@ -56,17 +56,20 @@ void range_check_func(int32_t stage, const int32_t *input, const int32_t *buf,
assert(in_range);
}
#define range_check(stage, input, buf, size, bit) \
#define apply_range(stage, input, buf, size, bit) \
range_check_func(stage, input, buf, size, bit)
#else
#define range_check(stage, input, buf, size, bit) \
{ \
(void)stage; \
(void)input; \
(void)buf; \
(void)size; \
(void)bit; \
#define apply_range(stage, input, buf, size, bit) \
clamp_buf((int32_t *)buf, size, bit)
void clamp_buf(int32_t *buf, int32_t size, int8_t bit) {
if (bit <= 16) {
const int32_t maxValue = (1 << 15) - 1;
const int32_t minValue = -(1 << 15);
for (int i = 0; i < size; ++i) buf[i] = clamp(buf[i], minValue, maxValue);
}
}
#endif
// TODO(angiebird): Make 1-d txfm functions static
......@@ -80,7 +83,7 @@ void av1_idct4_new(const int32_t *input, int32_t *output, const int8_t *cos_bit,
int32_t step[4];
// stage 0;
range_check(stage, input, input, size, stage_range[stage]);
apply_range(stage, input, input, size, stage_range[stage]);
// stage 1;
stage++;
......@@ -90,7 +93,7 @@ void av1_idct4_new(const int32_t *input, int32_t *output, const int8_t *cos_bit,
bf1[1] = input[2];
bf1[2] = input[1];
bf1[3] = input[3];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 2
stage++;
......@@ -101,7 +104,7 @@ void av1_idct4_new(const int32_t *input, int32_t *output, const int8_t *cos_bit,
bf1[1] = half_btf(cospi[32], bf0[0], -cospi[32], bf0[1], cos_bit[stage]);
bf1[2] = half_btf(cospi[48], bf0[2], -cospi[16], bf0[3], cos_bit[stage]);
bf1[3] = half_btf(cospi[16], bf0[2], cospi[48], bf0[3], cos_bit[stage]);
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 3
stage++;
......@@ -111,7 +114,7 @@ void av1_idct4_new(const int32_t *input, int32_t *output, const int8_t *cos_bit,
bf1[1] = bf0[1] + bf0[2];
bf1[2] = bf0[1] - bf0[2];
bf1[3] = bf0[0] - bf0[3];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
}
void av1_idct8_new(const int32_t *input, int32_t *output, const int8_t *cos_bit,
......@@ -124,7 +127,7 @@ void av1_idct8_new(const int32_t *input, int32_t *output, const int8_t *cos_bit,
int32_t step[8];
// stage 0;
range_check(stage, input, input, size, stage_range[stage]);
apply_range(stage, input, input, size, stage_range[stage]);
// stage 1;
stage++;
......@@ -138,7 +141,7 @@ void av1_idct8_new(const int32_t *input, int32_t *output, const int8_t *cos_bit,
bf1[5] = input[5];
bf1[6] = input[3];
bf1[7] = input[7];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 2
stage++;
......@@ -153,7 +156,7 @@ void av1_idct8_new(const int32_t *input, int32_t *output, const int8_t *cos_bit,
bf1[5] = half_btf(cospi[24], bf0[5], -cospi[40], bf0[6], cos_bit[stage]);
bf1[6] = half_btf(cospi[40], bf0[5], cospi[24], bf0[6], cos_bit[stage]);
bf1[7] = half_btf(cospi[8], bf0[4], cospi[56], bf0[7], cos_bit[stage]);
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 3
stage++;
......@@ -168,7 +171,7 @@ void av1_idct8_new(const int32_t *input, int32_t *output, const int8_t *cos_bit,
bf1[5] = bf0[4] - bf0[5];
bf1[6] = -bf0[6] + bf0[7];
bf1[7] = bf0[6] + bf0[7];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 4
stage++;
......@@ -183,7 +186,7 @@ void av1_idct8_new(const int32_t *input, int32_t *output, const int8_t *cos_bit,
bf1[5] = half_btf(-cospi[32], bf0[5], cospi[32], bf0[6], cos_bit[stage]);
bf1[6] = half_btf(cospi[32], bf0[5], cospi[32], bf0[6], cos_bit[stage]);
bf1[7] = bf0[7];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 5
stage++;
......@@ -197,7 +200,7 @@ void av1_idct8_new(const int32_t *input, int32_t *output, const int8_t *cos_bit,
bf1[5] = bf0[2] - bf0[5];
bf1[6] = bf0[1] - bf0[6];
bf1[7] = bf0[0] - bf0[7];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
}
void av1_idct16_new(const int32_t *input, int32_t *output,
......@@ -210,7 +213,7 @@ void av1_idct16_new(const int32_t *input, int32_t *output,
int32_t step[16];
// stage 0;
range_check(stage, input, input, size, stage_range[stage]);
apply_range(stage, input, input, size, stage_range[stage]);
// stage 1;
stage++;
......@@ -232,7 +235,7 @@ void av1_idct16_new(const int32_t *input, int32_t *output,
bf1[13] = input[11];
bf1[14] = input[7];
bf1[15] = input[15];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 2
stage++;
......@@ -255,7 +258,7 @@ void av1_idct16_new(const int32_t *input, int32_t *output,
bf1[13] = half_btf(cospi[20], bf0[10], cospi[44], bf0[13], cos_bit[stage]);
bf1[14] = half_btf(cospi[36], bf0[9], cospi[28], bf0[14], cos_bit[stage]);
bf1[15] = half_btf(cospi[4], bf0[8], cospi[60], bf0[15], cos_bit[stage]);
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 3
stage++;
......@@ -278,7 +281,7 @@ void av1_idct16_new(const int32_t *input, int32_t *output,
bf1[13] = bf0[12] - bf0[13];
bf1[14] = -bf0[14] + bf0[15];
bf1[15] = bf0[14] + bf0[15];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 4
stage++;
......@@ -301,7 +304,7 @@ void av1_idct16_new(const int32_t *input, int32_t *output,
bf1[13] = half_btf(-cospi[16], bf0[10], cospi[48], bf0[13], cos_bit[stage]);
bf1[14] = half_btf(cospi[48], bf0[9], cospi[16], bf0[14], cos_bit[stage]);
bf1[15] = bf0[15];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 5
stage++;
......@@ -324,7 +327,7 @@ void av1_idct16_new(const int32_t *input, int32_t *output,
bf1[13] = -bf0[13] + bf0[14];
bf1[14] = bf0[13] + bf0[14];
bf1[15] = bf0[12] + bf0[15];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 6
stage++;
......@@ -347,7 +350,7 @@ void av1_idct16_new(const int32_t *input, int32_t *output,
bf1[13] = half_btf(cospi[32], bf0[10], cospi[32], bf0[13], cos_bit[stage]);
bf1[14] = bf0[14];
bf1[15] = bf0[15];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 7
stage++;
......@@ -369,7 +372,7 @@ void av1_idct16_new(const int32_t *input, int32_t *output,
bf1[13] = bf0[2] - bf0[13];
bf1[14] = bf0[1] - bf0[14];
bf1[15] = bf0[0] - bf0[15];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
}
void av1_idct32_new(const int32_t *input, int32_t *output,
......@@ -382,7 +385,7 @@ void av1_idct32_new(const int32_t *input, int32_t *output,
int32_t step[32];
// stage 0;
range_check(stage, input, input, size, stage_range[stage]);
apply_range(stage, input, input, size, stage_range[stage]);
// stage 1;
stage++;
......@@ -420,7 +423,7 @@ void av1_idct32_new(const int32_t *input, int32_t *output,
bf1[29] = input[23];
bf1[30] = input[15];
bf1[31] = input[31];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 2
stage++;
......@@ -459,7 +462,7 @@ void av1_idct32_new(const int32_t *input, int32_t *output,
bf1[29] = half_btf(cospi[18], bf0[18], cospi[46], bf0[29], cos_bit[stage]);
bf1[30] = half_btf(cospi[34], bf0[17], cospi[30], bf0[30], cos_bit[stage]);
bf1[31] = half_btf(cospi[2], bf0[16], cospi[62], bf0[31], cos_bit[stage]);
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 3
stage++;
......@@ -498,7 +501,7 @@ void av1_idct32_new(const int32_t *input, int32_t *output,
bf1[29] = bf0[28] - bf0[29];
bf1[30] = -bf0[30] + bf0[31];
bf1[31] = bf0[30] + bf0[31];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 4
stage++;
......@@ -537,7 +540,7 @@ void av1_idct32_new(const int32_t *input, int32_t *output,
bf1[29] = half_btf(-cospi[8], bf0[18], cospi[56], bf0[29], cos_bit[stage]);
bf1[30] = half_btf(cospi[56], bf0[17], cospi[8], bf0[30], cos_bit[stage]);
bf1[31] = bf0[31];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 5
stage++;
......@@ -576,7 +579,7 @@ void av1_idct32_new(const int32_t *input, int32_t *output,
bf1[29] = -bf0[29] + bf0[30];
bf1[30] = bf0[29] + bf0[30];
bf1[31] = bf0[28] + bf0[31];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 6
stage++;
......@@ -615,7 +618,7 @@ void av1_idct32_new(const int32_t *input, int32_t *output,
bf1[29] = half_btf(cospi[48], bf0[18], cospi[16], bf0[29], cos_bit[stage]);
bf1[30] = bf0[30];
bf1[31] = bf0[31];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 7
stage++;
......@@ -654,7 +657,7 @@ void av1_idct32_new(const int32_t *input, int32_t *output,
bf1[29] = bf0[26] + bf0[29];
bf1[30] = bf0[25] + bf0[30];
bf1[31] = bf0[24] + bf0[31];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 8
stage++;
......@@ -693,7 +696,7 @@ void av1_idct32_new(const int32_t *input, int32_t *output,
bf1[29] = bf0[29];
bf1[30] = bf0[30];
bf1[31] = bf0[31];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 9
stage++;
......@@ -731,7 +734,7 @@ void av1_idct32_new(const int32_t *input, int32_t *output,
bf1[29] = bf0[2] - bf0[29];
bf1[30] = bf0[1] - bf0[30];
bf1[31] = bf0[0] - bf0[31];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
}
void av1_iadst4_new(const int32_t *input, int32_t *output,
......@@ -741,7 +744,7 @@ void av1_iadst4_new(const int32_t *input, int32_t *output,
int32_t stage = 0;
int64_t s0, s1, s2, s3, s4, s5, s6, s7;
// stage 0;
range_check(stage, input, input, size, stage_range[stage]);
apply_range(stage, input, input, size, stage_range[stage]);
int32_t x0 = input[0];
int32_t x1 = input[1];
......@@ -776,7 +779,7 @@ void av1_iadst4_new(const int32_t *input, int32_t *output,
output[1] = (int32_t)dct_const_round_shift(s1 + s3);
output[2] = (int32_t)dct_const_round_shift(s2);
output[3] = (int32_t)dct_const_round_shift(s0 + s1 - s3);
range_check(stage, input, output, size, stage_range[stage]);
apply_range(stage, input, output, size, stage_range[stage]);
}
void av1_iadst8_new(const int32_t *input, int32_t *output,
......@@ -789,7 +792,7 @@ void av1_iadst8_new(const int32_t *input, int32_t *output,
int32_t step[8];
// stage 0;
range_check(stage, input, input, size, stage_range[stage]);
apply_range(stage, input, input, size, stage_range[stage]);
// stage 1;
stage++;
......@@ -802,7 +805,7 @@ void av1_iadst8_new(const int32_t *input, int32_t *output,
bf1[5] = input[4];
bf1[6] = input[1];
bf1[7] = input[6];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 2
stage++;
......@@ -817,7 +820,7 @@ void av1_iadst8_new(const int32_t *input, int32_t *output,
bf1[5] = half_btf(-cospi[36], bf0[5], cospi[28], bf0[4], cos_bit[stage]);
bf1[6] = half_btf(cospi[52], bf0[6], cospi[12], bf0[7], cos_bit[stage]);
bf1[7] = half_btf(-cospi[52], bf0[7], cospi[12], bf0[6], cos_bit[stage]);
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 3
stage++;
......@@ -831,7 +834,7 @@ void av1_iadst8_new(const int32_t *input, int32_t *output,
bf1[5] = -bf0[5] + bf0[1];
bf1[6] = -bf0[6] + bf0[2];
bf1[7] = -bf0[7] + bf0[3];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 4
stage++;
......@@ -846,7 +849,7 @@ void av1_iadst8_new(const int32_t *input, int32_t *output,
bf1[5] = half_btf(-cospi[16], bf0[5], cospi[48], bf0[4], cos_bit[stage]);
bf1[6] = half_btf(-cospi[48], bf0[6], cospi[16], bf0[7], cos_bit[stage]);
bf1[7] = half_btf(cospi[48], bf0[7], cospi[16], bf0[6], cos_bit[stage]);
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 5
stage++;
......@@ -860,7 +863,7 @@ void av1_iadst8_new(const int32_t *input, int32_t *output,
bf1[5] = bf0[5] + bf0[7];
bf1[6] = -bf0[6] + bf0[4];
bf1[7] = -bf0[7] + bf0[5];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 6
stage++;
......@@ -875,7 +878,7 @@ void av1_iadst8_new(const int32_t *input, int32_t *output,
bf1[5] = bf0[5];
bf1[6] = half_btf(cospi[32], bf0[6], cospi[32], bf0[7], cos_bit[stage]);
bf1[7] = half_btf(-cospi[32], bf0[7], cospi[32], bf0[6], cos_bit[stage]);
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 7
stage++;
......@@ -889,7 +892,7 @@ void av1_iadst8_new(const int32_t *input, int32_t *output,
bf1[5] = -bf0[7];
bf1[6] = bf0[5];
bf1[7] = -bf0[1];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
}
void av1_iadst16_new(const int32_t *input, int32_t *output,
......@@ -902,7 +905,7 @@ void av1_iadst16_new(const int32_t *input, int32_t *output,
int32_t step[16];
// stage 0;
range_check(stage, input, input, size, stage_range[stage]);
apply_range(stage, input, input, size, stage_range[stage]);
// stage 1;
stage++;
......@@ -923,7 +926,7 @@ void av1_iadst16_new(const int32_t *input, int32_t *output,
bf1[13] = input[12];
bf1[14] = input[1];
bf1[15] = input[14];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 2
stage++;
......@@ -946,7 +949,7 @@ void av1_iadst16_new(const int32_t *input, int32_t *output,
bf1[13] = half_btf(-cospi[50], bf0[13], cospi[14], bf0[12], cos_bit[stage]);
bf1[14] = half_btf(cospi[58], bf0[14], cospi[6], bf0[15], cos_bit[stage]);
bf1[15] = half_btf(-cospi[58], bf0[15], cospi[6], bf0[14], cos_bit[stage]);
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 3
stage++;
......@@ -968,7 +971,7 @@ void av1_iadst16_new(const int32_t *input, int32_t *output,
bf1[13] = -bf0[13] + bf0[5];
bf1[14] = -bf0[14] + bf0[6];
bf1[15] = -bf0[15] + bf0[7];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 4
stage++;
......@@ -991,7 +994,7 @@ void av1_iadst16_new(const int32_t *input, int32_t *output,
bf1[13] = half_btf(cospi[56], bf0[13], cospi[8], bf0[12], cos_bit[stage]);
bf1[14] = half_btf(-cospi[24], bf0[14], cospi[40], bf0[15], cos_bit[stage]);
bf1[15] = half_btf(cospi[24], bf0[15], cospi[40], bf0[14], cos_bit[stage]);
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 5
stage++;
......@@ -1013,7 +1016,7 @@ void av1_iadst16_new(const int32_t *input, int32_t *output,
bf1[13] = -bf0[13] + bf0[9];
bf1[14] = -bf0[14] + bf0[10];
bf1[15] = -bf0[15] + bf0[11];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 6
stage++;
......@@ -1036,7 +1039,7 @@ void av1_iadst16_new(const int32_t *input, int32_t *output,
bf1[13] = half_btf(-cospi[16], bf0[13], cospi[48], bf0[12], cos_bit[stage]);
bf1[14] = half_btf(-cospi[48], bf0[14], cospi[16], bf0[15], cos_bit[stage]);
bf1[15] = half_btf(cospi[48], bf0[15], cospi[16], bf0[14], cos_bit[stage]);
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 7
stage++;
......@@ -1058,7 +1061,7 @@ void av1_iadst16_new(const int32_t *input, int32_t *output,
bf1[13] = bf0[13] + bf0[15];
bf1[14] = -bf0[14] + bf0[12];
bf1[15] = -bf0[15] + bf0[13];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 8
stage++;
......@@ -1081,7 +1084,7 @@ void av1_iadst16_new(const int32_t *input, int32_t *output,
bf1[13] = bf0[13];
bf1[14] = half_btf(cospi[32], bf0[14], cospi[32], bf0[15], cos_bit[stage]);
bf1[15] = half_btf(-cospi[32], bf0[15], cospi[32], bf0[14], cos_bit[stage]);
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 9
stage++;
......@@ -1103,7 +1106,7 @@ void av1_iadst16_new(const int32_t *input, int32_t *output,
bf1[13] = -bf0[13];
bf1[14] = bf0[9];
bf1[15] = -bf0[1];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
}
void av1_iadst32_new(const int32_t *input, int32_t *output,
......@@ -1116,7 +1119,7 @@ void av1_iadst32_new(const int32_t *input, int32_t *output,
int32_t step[32];
// stage 0;
range_check(stage, input, input, size, stage_range[stage]);
apply_range(stage, input, input, size, stage_range[stage]);
// stage 1;
stage++;
......@@ -1154,7 +1157,7 @@ void av1_iadst32_new(const int32_t *input, int32_t *output,
bf1[29] = input[26];
bf1[30] = input[10];
bf1[31] = -input[21];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 2
stage++;
......@@ -1193,7 +1196,7 @@ void av1_iadst32_new(const int32_t *input, int32_t *output,
bf1[29] = bf0[29];
bf1[30] = half_btf(cospi[32], bf0[30], cospi[32], bf0[31], cos_bit[stage]);
bf1[31] = half_btf(cospi[32], bf0[30], -cospi[32], bf0[31], cos_bit[stage]);
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 3
stage++;
......@@ -1231,7 +1234,7 @@ void av1_iadst32_new(const int32_t *input, int32_t *output,
bf1[29] = bf0[29] + bf0[31];
bf1[30] = bf0[28] - bf0[30];
bf1[31] = bf0[29] - bf0[31];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 4
stage++;
......@@ -1270,7 +1273,7 @@ void av1_iadst32_new(const int32_t *input, int32_t *output,
bf1[29] = half_btf(cospi[48], bf0[28], -cospi[16], bf0[29], cos_bit[stage]);
bf1[30] = half_btf(-cospi[48], bf0[30], cospi[16], bf0[31], cos_bit[stage]);
bf1[31] = half_btf(cospi[16], bf0[30], cospi[48], bf0[31], cos_bit[stage]);
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 5
stage++;
......@@ -1308,7 +1311,7 @@ void av1_iadst32_new(const int32_t *input, int32_t *output,
bf1[29] = bf0[25] - bf0[29];
bf1[30] = bf0[26] - bf0[30];
bf1[31] = bf0[27] - bf0[31];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 6
stage++;
......@@ -1347,7 +1350,7 @@ void av1_iadst32_new(const int32_t *input, int32_t *output,
bf1[29] = half_btf(cospi[8], bf0[28], cospi[56], bf0[29], cos_bit[stage]);
bf1[30] = half_btf(-cospi[24], bf0[30], cospi[40], bf0[31], cos_bit[stage]);
bf1[31] = half_btf(cospi[40], bf0[30], cospi[24], bf0[31], cos_bit[stage]);
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 7
stage++;
......@@ -1385,7 +1388,7 @@ void av1_iadst32_new(const int32_t *input, int32_t *output,
bf1[29] = bf0[21] - bf0[29];
bf1[30] = bf0[22] - bf0[30];
bf1[31] = bf0[23] - bf0[31];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 8
stage++;
......@@ -1424,7 +1427,7 @@ void av1_iadst32_new(const int32_t *input, int32_t *output,
bf1[29] = half_btf(cospi[36], bf0[28], cospi[28], bf0[29], cos_bit[stage]);
bf1[30] = half_btf(-cospi[12], bf0[30], cospi[52], bf0[31], cos_bit[stage]);
bf1[31] = half_btf(cospi[52], bf0[30], cospi[12], bf0[31], cos_bit[stage]);
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 9
stage++;
......@@ -1462,7 +1465,7 @@ void av1_iadst32_new(const int32_t *input, int32_t *output,
bf1[29] = bf0[13] - bf0[29];
bf1[30] = bf0[14] - bf0[30];
bf1[31] = bf0[15] - bf0[31];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 10
stage++;
......@@ -1501,7 +1504,7 @@ void av1_iadst32_new(const int32_t *input, int32_t *output,
bf1[29] = half_btf(cospi[7], bf0[28], -cospi[57], bf0[29], cos_bit[stage]);
bf1[30] = half_btf(cospi[61], bf0[30], cospi[3], bf0[31], cos_bit[stage]);
bf1[31] = half_btf(cospi[3], bf0[30], -cospi[61], bf0[31], cos_bit[stage]);
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
// stage 11
stage++;
......@@ -1539,7 +1542,7 @@ void av1_iadst32_new(const int32_t *input, int32_t *output,
bf1[29] = bf0[2];
bf1[30] = bf0[31];
bf1[31] = bf0[0];
range_check(stage, input, bf1, size, stage_range[stage]);
apply_range(stage, input, bf1, size, stage_range[stage]);
}
void av1_iidentity4_c(const int32_t *input, int32_t *output,
......@@ -1547,14 +1550,14 @@ void av1_iidentity4_c(const int32_t *input, int32_t *output,
(void)cos_bit;
for (int i = 0; i < 4; ++i)
output[i] = (int32_t)dct_const_round_shift(Sqrt2 * input[i]);
range_check(0, input, output, 4, stage_range[0]);
apply_range(0, input, output, 4, stage_range[0]);
}
void av1_iidentity8_c(const int32_t *input, int32_t *output,
const int8_t *cos_bit, const int8_t *stage_range) {
(void)cos_bit;
for (int i = 0; i < 8; ++i) output[i] = (int32_t)((int64_t)input[i] * 2);
range_check(0, input, output, 8, stage_range[0]);
apply_range(0, input, output, 8, stage_range[0]);
}
void av1_iidentity16_c(const int32_t *input, int32_t *output,
......@@ -1562,14 +1565,14 @@ void av1_iidentity16_c(const int32_t *input, int32_t *output,
(void)cos_bit;
for (int i = 0; i < 16; ++i)
output[i] = (int32_t)dct_const_round_shift(Sqrt2 * input[i] * 2);
range_check(0, input, output, 16, stage_range[0]);
apply_range(0, input, output, 16, stage_range[0]);
}
void av1_iidentity32_c(const int32_t *input, int32_t *output,
const int8_t *cos_bit, const int8_t *stage_range) {
(void)cos_bit;
for (int i = 0; i < 32; ++i) output[i] = (int32_t)((int64_t)input[i] * 4);
range_check(0, input, output, 32, stage_range[0]);
apply_range(0, input, output, 32, stage_range[0]);
}
#if CONFIG_TX64X64
......@@ -1578,7 +1581,7 @@ void av1_iidentity64_c(const int32_t *input, int32_t *output,
(void)cos_bit;
for (int i = 0; i < 64; ++i)
output[i] = (int32_t)dct_const_round_shift(input[i] * 4 * Sqrt2);
range_check(0, input, output, 64, stage_range[0]);
apply_range(0, input, output, 64, stage_range[0]);
}
#endif // CONFIG_TX64X64
......@@ -1593,7 +1596,7 @@ void av1_idct64_new(const int32_t *input, int32_t *output,
int32_t step[64];
// stage 0;
range_check(stage, input, input, size, stage_range[stage]);
apply_range(stage, input, input, size, stage_range[stage]);
// stage 1;
stage++;
......@@ -1663,7 +1666,7 @@ void av1_idct64_new(const int32_t *input, int32_t *output,
bf1[61] = input[47];
bf1[62] = input[31];