Commit 6a9ed8d2 authored by Jingning Han's avatar Jingning Han
Browse files

Fix forward transform bit range limits

Change-Id: I13c0ecff8c58a0571d9de4bc5fbbebe72533ccdb
parent 8198be11
...@@ -204,7 +204,7 @@ static void fdct4(const tran_low_t *input, tran_low_t *output) { ...@@ -204,7 +204,7 @@ static void fdct4(const tran_low_t *input, tran_low_t *output) {
tran_low_t step[4]; tran_low_t step[4];
// stage 0 // stage 0
range_check(input, 4, 11); range_check(input, 4, 14);
// stage 1 // stage 1
output[0] = input[0] + input[3]; output[0] = input[0] + input[3];
...@@ -212,7 +212,7 @@ static void fdct4(const tran_low_t *input, tran_low_t *output) { ...@@ -212,7 +212,7 @@ static void fdct4(const tran_low_t *input, tran_low_t *output) {
output[2] = input[1] - input[2]; output[2] = input[1] - input[2];
output[3] = input[0] - input[3]; output[3] = input[0] - input[3];
range_check(output, 4, 12); range_check(output, 4, 15);
// stage 2 // stage 2
temp = output[0] * cospi_16_64 + output[1] * cospi_16_64; temp = output[0] * cospi_16_64 + output[1] * cospi_16_64;
...@@ -224,7 +224,7 @@ static void fdct4(const tran_low_t *input, tran_low_t *output) { ...@@ -224,7 +224,7 @@ static void fdct4(const tran_low_t *input, tran_low_t *output) {
temp = output[3] * cospi_24_64 + output[2] * -cospi_8_64; temp = output[3] * cospi_24_64 + output[2] * -cospi_8_64;
step[3] = (tran_low_t)fdct_round_shift(temp); step[3] = (tran_low_t)fdct_round_shift(temp);
range_check(step, 4, 13); range_check(step, 4, 16);
// stage 3 // stage 3
output[0] = step[0]; output[0] = step[0];
...@@ -232,7 +232,7 @@ static void fdct4(const tran_low_t *input, tran_low_t *output) { ...@@ -232,7 +232,7 @@ static void fdct4(const tran_low_t *input, tran_low_t *output) {
output[2] = step[1]; output[2] = step[1];
output[3] = step[3]; output[3] = step[3];
range_check(output, 4, 13); range_check(output, 4, 16);
} }
static void fdct8(const tran_low_t *input, tran_low_t *output) { static void fdct8(const tran_low_t *input, tran_low_t *output) {
...@@ -240,7 +240,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) { ...@@ -240,7 +240,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
tran_low_t step[8]; tran_low_t step[8];
// stage 0 // stage 0
range_check(input, 8, 12); range_check(input, 8, 13);
// stage 1 // stage 1
output[0] = input[0] + input[7]; output[0] = input[0] + input[7];
...@@ -252,7 +252,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) { ...@@ -252,7 +252,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
output[6] = input[1] - input[6]; output[6] = input[1] - input[6];
output[7] = input[0] - input[7]; output[7] = input[0] - input[7];
range_check(output, 8, 13); range_check(output, 8, 14);
// stage 2 // stage 2
step[0] = output[0] + output[3]; step[0] = output[0] + output[3];
...@@ -266,7 +266,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) { ...@@ -266,7 +266,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
step[6] = (tran_low_t)fdct_round_shift(temp); step[6] = (tran_low_t)fdct_round_shift(temp);
step[7] = output[7]; step[7] = output[7];
range_check(step, 8, 14); range_check(step, 8, 15);
// stage 3 // stage 3
temp = step[0] * cospi_16_64 + step[1] * cospi_16_64; temp = step[0] * cospi_16_64 + step[1] * cospi_16_64;
...@@ -282,7 +282,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) { ...@@ -282,7 +282,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
output[6] = step[7] - step[6]; output[6] = step[7] - step[6];
output[7] = step[7] + step[6]; output[7] = step[7] + step[6];
range_check(output, 8, 14); range_check(output, 8, 16);
// stage 4 // stage 4
step[0] = output[0]; step[0] = output[0];
...@@ -298,7 +298,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) { ...@@ -298,7 +298,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
temp = output[7] * cospi_28_64 + output[4] * -cospi_4_64; temp = output[7] * cospi_28_64 + output[4] * -cospi_4_64;
step[7] = (tran_low_t)fdct_round_shift(temp); step[7] = (tran_low_t)fdct_round_shift(temp);
range_check(step, 8, 14); range_check(step, 8, 16);
// stage 5 // stage 5
output[0] = step[0]; output[0] = step[0];
...@@ -310,7 +310,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) { ...@@ -310,7 +310,7 @@ static void fdct8(const tran_low_t *input, tran_low_t *output) {
output[6] = step[3]; output[6] = step[3];
output[7] = step[7]; output[7] = step[7];
range_check(output, 8, 14); range_check(output, 8, 16);
} }
static void fdct16(const tran_low_t *input, tran_low_t *output) { static void fdct16(const tran_low_t *input, tran_low_t *output) {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment