Commit 901d5148 authored by Debargha Mukherjee's avatar Debargha Mukherjee
Browse files

Use and adjust rectangular txfm stage ranges

Adjusst txfm stage ranges so that Cfg tests pass, however
tests with --enable-coefficient-range-checking are still
failing. Those are to be addressed subsequently.

Change-Id: I1b15406a525a78322211e69a0d3a796823bb645e
parent 6f90fb29
......@@ -29,6 +29,7 @@ void range_check_func(int32_t stage, const int32_t *input, const int32_t *buf,
if (!in_range) {
fprintf(stderr, "Error: coeffs contain out-of-range values\n");
fprintf(stderr, "size: %d\n", size);
fprintf(stderr, "stage: %d\n", stage);
fprintf(stderr, "allowed range: [%" PRId64 ";%" PRId64 "]\n", minValue,
maxValue);
......
......@@ -44,21 +44,21 @@ static INLINE TxfmFunc inv_txfm_type_to_func(TXFM_TYPE txfm_type) {
static const TXFM_1D_CFG *inv_txfm_col_cfg_ls[TX_TYPES_1D][TX_SIZES_ALL] = {
// DCT
{
&inv_txfm_1d_col_cfg_dct_4, &inv_txfm_1d_col_cfg_dct_8,
&inv_txfm_1d_col_cfg_dct_16, &inv_txfm_1d_col_cfg_dct_32,
&inv_txfm_1d_col_cfg_dct_4, &inv_txfm_1d_col_cfg_dct_8,
&inv_txfm_1d_col_cfg_dct_16, &inv_txfm_1d_col_cfg_dct_32,
#if CONFIG_TX64X64
&inv_txfm_1d_col_cfg_dct_64,
#endif // CONFIG_TX64X64
&inv_txfm_1d_col_cfg_dct_8, &inv_txfm_1d_col_cfg_dct_4,
&inv_txfm_1d_col_cfg_dct_16, &inv_txfm_1d_col_cfg_dct_8,
&inv_txfm_1d_col_cfg_dct_32, &inv_txfm_1d_col_cfg_dct_16,
&inv_txfm_1d_col_cfg_dct_4x8, &inv_txfm_1d_col_cfg_dct_8x4,
&inv_txfm_1d_col_cfg_dct_8x16, &inv_txfm_1d_col_cfg_dct_16x8,
&inv_txfm_1d_col_cfg_dct_16x32, &inv_txfm_1d_col_cfg_dct_32x16,
#if CONFIG_TX64X64
&inv_txfm_1d_col_cfg_dct_64, &inv_txfm_1d_col_cfg_dct_32,
&inv_txfm_1d_col_cfg_dct_32x64, &inv_txfm_1d_col_cfg_dct_64x32,
#endif // CONFIG_TX64X64
&inv_txfm_1d_col_cfg_dct_16, &inv_txfm_1d_col_cfg_dct_4,
&inv_txfm_1d_col_cfg_dct_32, &inv_txfm_1d_col_cfg_dct_8,
&inv_txfm_1d_col_cfg_dct_4x16, &inv_txfm_1d_col_cfg_dct_16x4,
&inv_txfm_1d_col_cfg_dct_8x32, &inv_txfm_1d_col_cfg_dct_32x8,
#if CONFIG_TX64X64
&inv_txfm_1d_row_cfg_dct_64, &inv_txfm_1d_row_cfg_dct_16,
&inv_txfm_1d_col_cfg_dct_16x64, &inv_txfm_1d_col_cfg_dct_64x16,
#endif // CONFIG_TX64X64
},
// ADST
......@@ -70,23 +70,23 @@ static const TXFM_1D_CFG *inv_txfm_col_cfg_ls[TX_TYPES_1D][TX_SIZES_ALL] = {
#if CONFIG_TX64X64
NULL,
#endif // CONFIG_TX64X64
&inv_txfm_1d_col_cfg_adst_8,
&inv_txfm_1d_col_cfg_adst_4,
&inv_txfm_1d_col_cfg_adst_16,
&inv_txfm_1d_col_cfg_adst_8,
&inv_txfm_1d_col_cfg_adst_32,
&inv_txfm_1d_col_cfg_adst_16,
&inv_txfm_1d_col_cfg_adst_4x8,
&inv_txfm_1d_col_cfg_adst_8x4,
&inv_txfm_1d_col_cfg_adst_8x16,
&inv_txfm_1d_col_cfg_adst_16x8,
&inv_txfm_1d_col_cfg_adst_16x32,
&inv_txfm_1d_col_cfg_adst_32x16,
#if CONFIG_TX64X64
NULL,
&inv_txfm_1d_col_cfg_adst_32,
NULL,
#endif // CONFIG_TX64X64
&inv_txfm_1d_col_cfg_adst_16,
&inv_txfm_1d_col_cfg_adst_4,
&inv_txfm_1d_col_cfg_adst_32,
&inv_txfm_1d_col_cfg_adst_8,
&inv_txfm_1d_col_cfg_adst_4x16,
&inv_txfm_1d_col_cfg_adst_16x4,
&inv_txfm_1d_col_cfg_adst_8x32,
&inv_txfm_1d_col_cfg_adst_32x8,
#if CONFIG_TX64X64
NULL,
&inv_txfm_1d_row_cfg_adst_16,
NULL,
#endif // CONFIG_TX64X64
},
// FLIPADST
......@@ -98,23 +98,23 @@ static const TXFM_1D_CFG *inv_txfm_col_cfg_ls[TX_TYPES_1D][TX_SIZES_ALL] = {
#if CONFIG_TX64X64
NULL,
#endif // CONFIG_TX64X64
&inv_txfm_1d_col_cfg_adst_8,
&inv_txfm_1d_col_cfg_adst_4,
&inv_txfm_1d_col_cfg_adst_16,
&inv_txfm_1d_col_cfg_adst_8,
&inv_txfm_1d_col_cfg_adst_32,
&inv_txfm_1d_col_cfg_adst_16,
&inv_txfm_1d_col_cfg_adst_4x8,
&inv_txfm_1d_col_cfg_adst_8x4,
&inv_txfm_1d_col_cfg_adst_8x16,
&inv_txfm_1d_col_cfg_adst_16x8,
&inv_txfm_1d_col_cfg_adst_16x32,
&inv_txfm_1d_col_cfg_adst_32x16,
#if CONFIG_TX64X64
NULL,
&inv_txfm_1d_col_cfg_adst_32,
NULL,
#endif // CONFIG_TX64X64
&inv_txfm_1d_col_cfg_adst_16,
&inv_txfm_1d_col_cfg_adst_4,
&inv_txfm_1d_col_cfg_adst_32,
&inv_txfm_1d_col_cfg_adst_8,
&inv_txfm_1d_col_cfg_adst_4x16,
&inv_txfm_1d_col_cfg_adst_16x4,
&inv_txfm_1d_col_cfg_adst_8x32,
&inv_txfm_1d_col_cfg_adst_32x8,
#if CONFIG_TX64X64
NULL,
&inv_txfm_1d_row_cfg_adst_16,
NULL,
#endif // CONFIG_TX64X64
},
// IDENTITY
......
......@@ -145,20 +145,20 @@ static const int8_t fwd_stage_range_row_dct_8x16[6] =
ARRAYOFFSET6(4, 0, 1, 2, 3, 3, 3);
static const int8_t fwd_stage_range_row_adst_8x16[8] =
ARRAYOFFSET8(4, 0, 0, 1, 2, 2, 3, 3, 3);
static const int8_t fwd_cos_bit_row_dct_8x16[6] = { 13, 13, 13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_dct_8x16[6] = { 13, 13, 12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_adst_8x16[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
12, 12, 12, 12 };
// ---------------- 8x16 1D constants -----------------------
// ---------------- 16x8 1D constants -----------------------
#define fwd_shift_16x8 fwd_shift_16
static const int8_t fwd_stage_range_row_dct_16x8[8] =
ARRAYOFFSET8(3, 0, 1, 2, 3, 4, 4, 4, 4);
static const int8_t fwd_stage_range_row_adst_16x8[10] =
ARRAYOFFSET10(3, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4);
static const int8_t fwd_cos_bit_row_dct_16x8[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_adst_16x8[10] = { 13, 13, 13, 13, 13,
13, 13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_dct_16x8[8] = { 13, 13, 13, 12,
12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_adst_16x8[10] = { 13, 13, 13, 13, 12,
12, 12, 12, 12, 12 };
// ---------------- 16x32 1D constants -----------------------
#define fwd_shift_16x32 fwd_shift_32
......@@ -166,8 +166,8 @@ static const int8_t fwd_stage_range_row_dct_16x32[8] =
ARRAYOFFSET8(5, 0, 1, 2, 3, 4, 4, 4, 4);
static const int8_t fwd_stage_range_row_adst_16x32[10] =
ARRAYOFFSET10(5, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4);
static const int8_t fwd_cos_bit_row_dct_16x32[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_dct_16x32[8] = { 13, 13, 13, 12,
12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_adst_16x32[10] = { 12, 12, 12, 12, 12,
12, 12, 12, 12, 12 };
......@@ -186,8 +186,8 @@ static const int8_t fwd_cos_bit_row_adst_32x16[12] = { 12, 12, 12, 12, 12, 12,
#define fwd_shift_32x64 fwd_shift_64
static const int8_t fwd_stage_range_row_dct_32x64[10] =
ARRAYOFFSET10(6, 0, 1, 2, 3, 4, 5, 5, 5, 5, 5);
static const int8_t fwd_cos_bit_row_dct_32x64[10] = { 12, 12, 12, 12, 12,
12, 12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_dct_32x64[10] = { 13, 12, 12, 11, 10,
10, 10, 10, 10, 10 };
// ---------------- 64x32 1D constants -----------------------
#define fwd_shift_64x32 fwd_shift_64
......@@ -241,15 +241,15 @@ static const int8_t fwd_cos_bit_row_adst_32x8[12] = { 12, 12, 12, 12, 12, 12,
#define fwd_shift_16x64 fwd_shift_64
static const int8_t fwd_stage_range_row_dct_16x64[8] =
ARRAYOFFSET8(6, 0, 1, 2, 3, 4, 4, 4, 4);
static const int8_t fwd_cos_bit_row_dct_16x64[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_dct_16x64[8] = { 12, 12, 12, 11,
11, 11, 11, 11 };
// ---------------- 64x16 1D constants -----------------------
#define fwd_shift_64x16 fwd_shift_64
static const int8_t fwd_stage_range_row_dct_64x16[12] =
ARRAYOFFSET12(4, 0, 1, 2, 3, 4, 5, 6, 6, 6, 6, 6, 6);
static const int8_t fwd_cos_bit_row_dct_64x16[12] = { 13, 13, 12, 11, 10, 10,
10, 10, 10, 10, 10, 10 };
static const int8_t fwd_cos_bit_row_dct_64x16[12] = { 13, 13, 12, 11, 11, 11,
11, 11, 11, 11, 11, 11 };
//
// ---------------- row config fwd_dct_4 ----------------
......
......@@ -517,21 +517,21 @@ static const TXFM_1D_CFG *fwd_txfm_col_cfg_ls[TX_TYPES_1D][TX_SIZES_ALL] = {
static const TXFM_1D_CFG *fwd_txfm_row_cfg_ls[TX_TYPES_1D][TX_SIZES_ALL] = {
// DCT
{
&fwd_txfm_1d_row_cfg_dct_4, &fwd_txfm_1d_row_cfg_dct_8,
&fwd_txfm_1d_row_cfg_dct_16, &fwd_txfm_1d_row_cfg_dct_32,
&fwd_txfm_1d_row_cfg_dct_4, &fwd_txfm_1d_row_cfg_dct_8,
&fwd_txfm_1d_row_cfg_dct_16, &fwd_txfm_1d_row_cfg_dct_32,
#if CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_dct_64,
#endif // CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_dct_4, &fwd_txfm_1d_row_cfg_dct_8,
&fwd_txfm_1d_row_cfg_dct_8, &fwd_txfm_1d_row_cfg_dct_16,
&fwd_txfm_1d_row_cfg_dct_16, &fwd_txfm_1d_row_cfg_dct_32,
&fwd_txfm_1d_row_cfg_dct_4x8, &fwd_txfm_1d_row_cfg_dct_8x4,
&fwd_txfm_1d_row_cfg_dct_8x16, &fwd_txfm_1d_row_cfg_dct_16x8,
&fwd_txfm_1d_row_cfg_dct_16x32, &fwd_txfm_1d_row_cfg_dct_32x16,
#if CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_dct_32, &fwd_txfm_1d_row_cfg_dct_64,
&fwd_txfm_1d_row_cfg_dct_32x64, &fwd_txfm_1d_row_cfg_dct_64x32,
#endif // CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_dct_4, &fwd_txfm_1d_row_cfg_dct_16,
&fwd_txfm_1d_row_cfg_dct_8, &fwd_txfm_1d_row_cfg_dct_32,
&fwd_txfm_1d_row_cfg_dct_4x16, &fwd_txfm_1d_row_cfg_dct_16x4,
&fwd_txfm_1d_row_cfg_dct_8x32, &fwd_txfm_1d_row_cfg_dct_32x8,
#if CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_dct_16, &fwd_txfm_1d_row_cfg_dct_64,
&fwd_txfm_1d_row_cfg_dct_16x64, &fwd_txfm_1d_row_cfg_dct_64x16,
#endif // CONFIG_TX64X64
},
// ADST
......@@ -543,22 +543,22 @@ static const TXFM_1D_CFG *fwd_txfm_row_cfg_ls[TX_TYPES_1D][TX_SIZES_ALL] = {
#if CONFIG_TX64X64
NULL,
#endif // CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_adst_4,
&fwd_txfm_1d_row_cfg_adst_8,
&fwd_txfm_1d_row_cfg_adst_8,
&fwd_txfm_1d_row_cfg_adst_16,
&fwd_txfm_1d_row_cfg_adst_16,
&fwd_txfm_1d_row_cfg_adst_32,
&fwd_txfm_1d_row_cfg_adst_4x8,
&fwd_txfm_1d_row_cfg_adst_8x4,
&fwd_txfm_1d_row_cfg_adst_8x16,
&fwd_txfm_1d_row_cfg_adst_16x8,
&fwd_txfm_1d_row_cfg_adst_16x32,
&fwd_txfm_1d_row_cfg_adst_32x16,
#if CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_adst_32,
NULL,
NULL,
#endif // CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_adst_4,
&fwd_txfm_1d_row_cfg_adst_16,
&fwd_txfm_1d_row_cfg_adst_8,
&fwd_txfm_1d_row_cfg_adst_32,
&fwd_txfm_1d_row_cfg_adst_4x16,
&fwd_txfm_1d_row_cfg_adst_16x4,
&fwd_txfm_1d_row_cfg_adst_8x32,
&fwd_txfm_1d_row_cfg_adst_32x8,
#if CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_adst_16,
NULL,
NULL,
#endif // CONFIG_TX64X64
},
......@@ -571,22 +571,22 @@ static const TXFM_1D_CFG *fwd_txfm_row_cfg_ls[TX_TYPES_1D][TX_SIZES_ALL] = {
#if CONFIG_TX64X64
NULL,
#endif // CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_adst_4,
&fwd_txfm_1d_row_cfg_adst_8,
&fwd_txfm_1d_row_cfg_adst_8,
&fwd_txfm_1d_row_cfg_adst_16,
&fwd_txfm_1d_row_cfg_adst_16,
&fwd_txfm_1d_row_cfg_adst_32,
&fwd_txfm_1d_row_cfg_adst_4x8,
&fwd_txfm_1d_row_cfg_adst_8x4,
&fwd_txfm_1d_row_cfg_adst_8x16,
&fwd_txfm_1d_row_cfg_adst_16x8,
&fwd_txfm_1d_row_cfg_adst_16x32,
&fwd_txfm_1d_row_cfg_adst_32x16,
#if CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_adst_32,
NULL,
NULL,
#endif // CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_adst_4,
&fwd_txfm_1d_row_cfg_adst_16,
&fwd_txfm_1d_row_cfg_adst_8,
&fwd_txfm_1d_row_cfg_adst_32,
&fwd_txfm_1d_row_cfg_adst_4x16,
&fwd_txfm_1d_row_cfg_adst_16x4,
&fwd_txfm_1d_row_cfg_adst_8x32,
&fwd_txfm_1d_row_cfg_adst_32x8,
#if CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_adst_16,
NULL,
NULL,
#endif // CONFIG_TX64X64
},
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment