Commit e7f3b89f authored by Debargha Mukherjee's avatar Debargha Mukherjee

Refactoring and clean ups for inv_txfm ranges

Change-Id: Ib37752952832cda36c323e8adcc564b1811b563a
parent 1158bff7
......@@ -15,12 +15,10 @@
// sum of fwd_shift_##
#if CONFIG_TX64X64
static const int8_t fwd_shift_sum[TX_SIZES] = { 2, 1, 0, -2, -4 };
static const int8_t inv_start_range[TX_SIZES_ALL] = {
5, 6, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7
};
#else // CONFIG_TX64X64
static const int8_t fwd_shift_sum[TX_SIZES] = { 2, 1, 0, -2 };
static const int8_t inv_start_range[TX_SIZES_ALL] = { 5, 6, 7, 7, 6, 6, 7,
7, 7, 7, 6, 6, 7, 7 };
#endif // CONFIG_TX64X64
......@@ -30,12 +28,12 @@ static const int8_t inv_start_range[TX_SIZES_ALL] = { 5, 6, 7, 7, 6, 6, 7,
static const int8_t inv_shift_4[2] = { 0, -4 };
// stage range
static const int8_t inv_stage_range_col_dct_4[4] = { 3, 3, 3, 3 };
static const int8_t inv_stage_range_row_dct_4[4] = { 3, 3, 3, 3 };
static const int8_t inv_stage_range_col_adst_4[6] = { 3, 3, 3, 3, 2, 2 };
static const int8_t inv_stage_range_row_adst_4[6] = { 3, 3, 3, 3, 3, 3 };
static const int8_t inv_stage_range_row_idx_4[1] = { 3 };
static const int8_t inv_stage_range_col_idx_4[1] = { 3 };
static const int8_t inv_stage_range_col_dct_4[4] = { 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_dct_4[4] = { 0, 0, 0, 0 };
static const int8_t inv_stage_range_col_adst_4[6] = { 0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_adst_4[6] = { 0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_idx_4[1] = { 0 };
static const int8_t inv_stage_range_col_idx_4[1] = { 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_4[4] = { 13, 13, 13, 13 };
......@@ -47,12 +45,12 @@ static const int8_t inv_cos_bit_row_adst_4[6] = { 13, 13, 13, 13, 13, 13 };
// shift
static const int8_t inv_shift_8[2] = { 0, -5 };
// stage range
static const int8_t inv_stage_range_col_dct_8[6] = { 5, 5, 5, 5, 4, 4 };
static const int8_t inv_stage_range_row_dct_8[6] = { 5, 5, 5, 5, 5, 5 };
static const int8_t inv_stage_range_col_adst_8[8] = { 5, 5, 5, 5, 5, 5, 4, 4 };
static const int8_t inv_stage_range_row_adst_8[8] = { 5, 5, 5, 5, 5, 5, 5, 5 };
static const int8_t inv_stage_range_row_idx_8[1] = { 5 };
static const int8_t inv_stage_range_col_idx_8[1] = { 5 };
static const int8_t inv_stage_range_col_dct_8[6] = { 0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_dct_8[6] = { 0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_col_adst_8[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_adst_8[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_idx_8[1] = { 0 };
static const int8_t inv_stage_range_col_idx_8[1] = { 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_8[6] = { 13, 13, 13, 13, 13, 13 };
......@@ -67,10 +65,10 @@ static const int8_t inv_cos_bit_row_adst_8[8] = {
// ---------------- 4x8 1D constants -----------------------
#define inv_shift_4x8 inv_shift_8
// stage range
static const int8_t inv_stage_range_col_dct_4x8[6] =
ARRAYOFFSET6(-2, 5, 5, 5, 5, 5, 5);
static const int8_t inv_stage_range_col_adst_4x8[8] =
ARRAYOFFSET8(-2, 5, 5, 5, 5, 5, 5, 5, 5);
static const int8_t inv_stage_range_col_dct_4x8[6] = { 0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_col_adst_4x8[8] = {
0, 0, 0, 0, 0, 0, 0, 0
};
// cos bit
static const int8_t inv_cos_bit_col_dct_4x8[6] = { 13, 13, 13, 13, 13, 13 };
static const int8_t inv_cos_bit_col_adst_4x8[8] = { 13, 13, 13, 13,
......@@ -79,10 +77,8 @@ static const int8_t inv_cos_bit_col_adst_4x8[8] = { 13, 13, 13, 13,
// ---------------- 8x4 1D constants -----------------------
#define inv_shift_8x4 inv_shift_8
// stage range
static const int8_t inv_stage_range_col_dct_8x4[4] =
ARRAYOFFSET4(2, 3, 3, 3, 3);
static const int8_t inv_stage_range_col_adst_8x4[6] =
ARRAYOFFSET6(2, 3, 3, 3, 3, 3, 3);
static const int8_t inv_stage_range_col_dct_8x4[4] = { 0, 0, 0, 0 };
static const int8_t inv_stage_range_col_adst_8x4[6] = { 0, 0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_8x4[4] = { 13, 13, 13, 13 };
static const int8_t inv_cos_bit_col_adst_8x4[6] = { 13, 13, 13, 13, 13, 13 };
......@@ -92,14 +88,14 @@ static const int8_t inv_cos_bit_col_adst_8x4[6] = { 13, 13, 13, 13, 13, 13 };
static const int8_t inv_shift_16[2] = { -1, -5 };
// stage range
static const int8_t inv_stage_range_col_dct_16[8] = { 7, 7, 7, 7, 7, 7, 6, 6 };
static const int8_t inv_stage_range_row_dct_16[8] = { 7, 7, 7, 7, 7, 7, 7, 7 };
static const int8_t inv_stage_range_col_adst_16[10] = { 7, 7, 7, 7, 7,
7, 7, 7, 6, 6 };
static const int8_t inv_stage_range_row_adst_16[10] = { 7, 7, 7, 7, 7,
7, 7, 7, 7, 7 };
static const int8_t inv_stage_range_row_idx_16[1] = { 7 };
static const int8_t inv_stage_range_col_idx_16[1] = { 7 };
static const int8_t inv_stage_range_col_dct_16[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_dct_16[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_col_adst_16[10] = { 0, 0, 0, 0, 0,
0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_adst_16[10] = { 0, 0, 0, 0, 0,
0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_idx_16[1] = { 0 };
static const int8_t inv_stage_range_col_idx_16[1] = { 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_16[8] = {
......@@ -116,14 +112,15 @@ static const int8_t inv_cos_bit_row_adst_16[10] = { 12, 12, 12, 12, 12,
// ---------------- 8x16 1D constants -----------------------
#define inv_shift_8x16 inv_shift_16
// stage range
static const int8_t inv_stage_range_row_dct_8x16[6] = { 5, 5, 5, 5, 5, 5 };
static const int8_t inv_stage_range_row_dct_8x16[6] = { 0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_adst_8x16[8] = {
5, 5, 5, 5, 5, 5, 5, 5
0, 0, 0, 0, 0, 0, 0, 0
};
static const int8_t inv_stage_range_col_dct_8x16[8] = {
0, 0, 0, 0, 0, 0, 0, 0
};
static const int8_t inv_stage_range_col_dct_8x16[8] =
ARRAYOFFSET8(-2, 7, 7, 7, 7, 7, 7, 7, 7);
static const int8_t inv_stage_range_col_adst_8x16[10] =
ARRAYOFFSET10(-2, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7);
static const int8_t inv_stage_range_col_adst_8x16[10] = { 0, 0, 0, 0, 0,
0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_row_dct_8x16[6] = { 12, 12, 12, 12, 12, 12 };
static const int8_t inv_cos_bit_row_adst_8x16[8] = { 12, 12, 12, 12,
......@@ -136,10 +133,10 @@ static const int8_t inv_cos_bit_col_adst_8x16[10] = { 13, 13, 13, 13, 13,
// ---------------- 16x8 1D constants -----------------------
#define inv_shift_16x8 inv_shift_16
// stage range
static const int8_t inv_stage_range_col_dct_16x8[6] =
ARRAYOFFSET6(2, 5, 5, 5, 5, 5, 5);
static const int8_t inv_stage_range_col_adst_16x8[8] =
ARRAYOFFSET8(2, 5, 5, 5, 5, 5, 5, 5, 5);
static const int8_t inv_stage_range_col_dct_16x8[6] = { 0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_col_adst_16x8[8] = {
0, 0, 0, 0, 0, 0, 0, 0
};
// cos bit
static const int8_t inv_cos_bit_col_dct_16x8[6] = { 13, 13, 13, 13, 13, 13 };
static const int8_t inv_cos_bit_col_adst_16x8[8] = { 13, 13, 13, 13,
......@@ -150,16 +147,16 @@ static const int8_t inv_cos_bit_col_adst_16x8[8] = { 13, 13, 13, 13,
static const int8_t inv_shift_32[2] = { -1, -5 };
// stage range
static const int8_t inv_stage_range_col_dct_32[10] = { 9, 9, 9, 9, 9,
9, 9, 9, 9, 9 };
static const int8_t inv_stage_range_row_dct_32[10] = { 9, 9, 9, 9, 9,
9, 9, 9, 9, 9 };
static const int8_t inv_stage_range_col_adst_32[12] = { 9, 9, 9, 9, 9, 9,
9, 9, 9, 9, 9, 9 };
static const int8_t inv_stage_range_row_adst_32[12] = { 9, 9, 9, 9, 9, 9,
9, 9, 9, 9, 9, 9 };
static const int8_t inv_stage_range_row_idx_32[1] = { 9 };
static const int8_t inv_stage_range_col_idx_32[1] = { 9 };
static const int8_t inv_stage_range_col_dct_32[10] = { 0, 0, 0, 0, 0,
0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_dct_32[10] = { 0, 0, 0, 0, 0,
0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_col_adst_32[12] = { 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_adst_32[12] = { 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_idx_32[1] = { 0 };
static const int8_t inv_stage_range_col_idx_32[1] = { 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_32[10] = { 13, 13, 13, 13, 13,
......@@ -174,10 +171,10 @@ static const int8_t inv_cos_bit_row_adst_32[12] = { 12, 12, 12, 12, 12, 12,
// ---------------- 16x32 1D constants -----------------------
#define inv_shift_16x32 inv_shift_32
// stage range
static const int8_t inv_stage_range_col_dct_16x32[10] =
ARRAYOFFSET10(-2, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9);
static const int8_t inv_stage_range_col_adst_16x32[12] =
ARRAYOFFSET12(-2, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9);
static const int8_t inv_stage_range_col_dct_16x32[10] = { 0, 0, 0, 0, 0,
0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_col_adst_16x32[12] = { 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_16x32[10] = { 13, 13, 13, 13, 13,
13, 13, 13, 13, 13 };
......@@ -187,10 +184,11 @@ static const int8_t inv_cos_bit_col_adst_16x32[12] = { 13, 13, 13, 13, 13, 13,
// ---------------- 32x16 1D constants -----------------------
#define inv_shift_32x16 inv_shift_32
// stage range
static const int8_t inv_stage_range_col_dct_32x16[8] =
ARRAYOFFSET8(2, 7, 7, 7, 7, 7, 7, 7, 7);
static const int8_t inv_stage_range_col_adst_32x16[10] =
ARRAYOFFSET10(2, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7);
static const int8_t inv_stage_range_col_dct_32x16[8] = {
0, 0, 0, 0, 0, 0, 0, 0
};
static const int8_t inv_stage_range_col_adst_32x16[10] = { 0, 0, 0, 0, 0,
0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_32x16[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
......@@ -202,13 +200,13 @@ static const int8_t inv_cos_bit_col_adst_32x16[10] = { 13, 13, 13, 13, 13,
static const int8_t inv_shift_64[2] = { -1, -5 };
// stage range
static const int8_t inv_stage_range_col_dct_64[12] = { 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11 };
static const int8_t inv_stage_range_row_dct_64[12] = { 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11 };
static const int8_t inv_stage_range_col_dct_64[12] = { 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_dct_64[12] = { 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_idx_64[1] = { 11 };
static const int8_t inv_stage_range_col_idx_64[1] = { 11 };
static const int8_t inv_stage_range_row_idx_64[1] = { 0 };
static const int8_t inv_stage_range_col_idx_64[1] = { 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_64[12] = { 13, 13, 13, 13, 13, 13,
......@@ -219,8 +217,8 @@ static const int8_t inv_cos_bit_row_dct_64[12] = { 12, 12, 12, 12, 12, 12,
// ---------------- 32x64 1D constants -----------------------
#define inv_shift_32x64 inv_shift_64
// stage range
static const int8_t inv_stage_range_col_dct_32x64[12] =
ARRAYOFFSET12(-2, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11);
static const int8_t inv_stage_range_col_dct_32x64[12] = { 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_32x64[12] = { 13, 13, 13, 13, 13, 13,
13, 13, 13, 13, 13, 13 };
......@@ -228,8 +226,8 @@ static const int8_t inv_cos_bit_col_dct_32x64[12] = { 13, 13, 13, 13, 13, 13,
// ---------------- 64x32 1D constants -----------------------
#define inv_shift_64x32 inv_shift_64
// stage range
static const int8_t inv_stage_range_col_dct_64x32[10] =
ARRAYOFFSET10(2, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9);
static const int8_t inv_stage_range_col_dct_64x32[10] = { 0, 0, 0, 0, 0,
0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_64x32[10] = { 13, 13, 13, 13, 13,
13, 13, 13, 13, 13 };
......@@ -237,10 +235,11 @@ static const int8_t inv_cos_bit_col_dct_64x32[10] = { 13, 13, 13, 13, 13,
// ---------------- 4x16 1D constants -----------------------
#define inv_shift_4x16 inv_shift_16
// stage range
static const int8_t inv_stage_range_col_dct_4x16[8] =
ARRAYOFFSET8(-4, 7, 7, 7, 7, 7, 7, 7, 7);
static const int8_t inv_stage_range_col_adst_4x16[10] =
ARRAYOFFSET10(-4, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7);
static const int8_t inv_stage_range_col_dct_4x16[8] = {
0, 0, 0, 0, 0, 0, 0, 0
};
static const int8_t inv_stage_range_col_adst_4x16[10] = { 0, 0, 0, 0, 0,
0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_4x16[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
......@@ -250,10 +249,8 @@ static const int8_t inv_cos_bit_col_adst_4x16[10] = { 13, 13, 13, 13, 13,
// ---------------- 16x4 1D constants -----------------------
#define inv_shift_16x4 inv_shift_16
// stage range
static const int8_t inv_stage_range_col_dct_16x4[4] =
ARRAYOFFSET4(4, 3, 3, 3, 3);
static const int8_t inv_stage_range_col_adst_16x4[6] =
ARRAYOFFSET6(4, 3, 3, 3, 3, 3, 3);
static const int8_t inv_stage_range_col_dct_16x4[4] = { 0, 0, 0, 0 };
static const int8_t inv_stage_range_col_adst_16x4[6] = { 0, 0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_16x4[4] = { 13, 13, 13, 13 };
static const int8_t inv_cos_bit_col_adst_16x4[6] = { 13, 13, 13, 13, 13, 13 };
......@@ -261,14 +258,14 @@ static const int8_t inv_cos_bit_col_adst_16x4[6] = { 13, 13, 13, 13, 13, 13 };
// ---------------- 8x32 1D constants -----------------------
#define inv_shift_8x32 inv_shift_32
// stage range
static const int8_t inv_stage_range_row_dct_8x32[6] = { 5, 5, 5, 5, 5, 5 };
static const int8_t inv_stage_range_row_dct_8x32[6] = { 0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_row_adst_8x32[8] = {
5, 5, 5, 5, 5, 5, 5, 5
0, 0, 0, 0, 0, 0, 0, 0
};
static const int8_t inv_stage_range_col_dct_8x32[10] =
ARRAYOFFSET10(-4, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9);
static const int8_t inv_stage_range_col_adst_8x32[12] =
ARRAYOFFSET12(-4, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9);
static const int8_t inv_stage_range_col_dct_8x32[10] = { 0, 0, 0, 0, 0,
0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_col_adst_8x32[12] = { 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_row_dct_8x32[6] = { 12, 12, 12, 12, 12, 12 };
static const int8_t inv_cos_bit_row_adst_8x32[8] = { 12, 12, 12, 12,
......@@ -281,10 +278,10 @@ static const int8_t inv_cos_bit_col_adst_8x32[12] = { 13, 13, 13, 13, 13, 13,
// ---------------- 32x8 1D constants -----------------------
#define inv_shift_32x8 inv_shift_32
// stage range
static const int8_t inv_stage_range_col_dct_32x8[6] =
ARRAYOFFSET6(4, 5, 5, 5, 5, 5, 5);
static const int8_t inv_stage_range_col_adst_32x8[8] =
ARRAYOFFSET8(4, 5, 5, 5, 5, 5, 5, 5, 5);
static const int8_t inv_stage_range_col_dct_32x8[6] = { 0, 0, 0, 0, 0, 0 };
static const int8_t inv_stage_range_col_adst_32x8[8] = {
0, 0, 0, 0, 0, 0, 0, 0
};
// cos bit
static const int8_t inv_cos_bit_col_dct_32x8[6] = { 13, 13, 13, 13, 13, 13 };
static const int8_t inv_cos_bit_col_adst_32x8[8] = { 13, 13, 13, 13,
......@@ -293,8 +290,8 @@ static const int8_t inv_cos_bit_col_adst_32x8[8] = { 13, 13, 13, 13,
// ---------------- 16x64 1D constants -----------------------
#define inv_shift_16x64 inv_shift_64
// stage range
static const int8_t inv_stage_range_col_dct_16x64[12] =
ARRAYOFFSET12(-4, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11);
static const int8_t inv_stage_range_col_dct_16x64[12] = { 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0 };
// cos bit
static const int8_t inv_cos_bit_col_dct_16x64[12] = { 13, 13, 13, 13, 13, 13,
13, 13, 13, 13, 13, 13 };
......@@ -302,8 +299,9 @@ static const int8_t inv_cos_bit_col_dct_16x64[12] = { 13, 13, 13, 13, 13, 13,
// ---------------- 64x16 1D constants -----------------------
#define inv_shift_64x16 inv_shift_64
// stage range
static const int8_t inv_stage_range_col_dct_64x16[8] =
ARRAYOFFSET8(4, 7, 7, 7, 7, 7, 7, 7, 7);
static const int8_t inv_stage_range_col_dct_64x16[8] = {
0, 0, 0, 0, 0, 0, 0, 0
};
// cos bit
static const int8_t inv_cos_bit_col_dct_64x16[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
......
......@@ -256,7 +256,6 @@ void av1_gen_inv_stage_range(int8_t *stage_range_col, int8_t *stage_range_row,
// rectangular, the number of columns will be the same as the
// txfm_size stored in the row cfg struct. It will make no difference
// for square transforms.
// const int fwd_shift = fwd_shift_sum[txsize_sqr_up_map[tx_size]];
const int fwd_shift = inv_start_range[tx_size];
const int txfm_size_col = cfg->row_cfg->txfm_size;
const int txfm_size_row = cfg->col_cfg->txfm_size;
......@@ -279,14 +278,12 @@ void av1_gen_inv_stage_range(int8_t *stage_range_col, int8_t *stage_range_row,
}
// i < MAX_TXFM_STAGE_NUM will mute above array bounds warning
for (int i = 0; i < cfg->row_cfg->stage_num && i < MAX_TXFM_STAGE_NUM; ++i) {
stage_range_row[i] = cfg->row_cfg->stage_range[i] + fwd_shift + bd + 1 -
cfg->row_cfg->stage_range[0];
stage_range_row[i] = cfg->row_cfg->stage_range[i] + fwd_shift + bd + 1;
}
// i < MAX_TXFM_STAGE_NUM will mute above array bounds warning
for (int i = 0; i < cfg->col_cfg->stage_num && i < MAX_TXFM_STAGE_NUM; ++i) {
stage_range_col[i] = cfg->col_cfg->stage_range[i] + fwd_shift + shift[0] +
bd + 1 - cfg->col_cfg->stage_range[0] +
rect_type2_shift;
bd + 1 + rect_type2_shift;
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment