Skip to content

GitLab

Commit 901d5148 authored by Debargha Mukherjee's avatar Debargha Mukherjee
Browse files

Use and adjust rectangular txfm stage ranges 

Adjusst txfm stage ranges so that Cfg tests pass, however
tests with --enable-coefficient-range-checking are still
failing. Those are to be addressed subsequently.

Change-Id: I1b15406a525a78322211e69a0d3a796823bb645e
parent 6f90fb29
Hide whitespace changes
Inline Side-by-side
av1/common/av1_inv_txfm1d.c
View file @ 901d5148
......@@ -29,6 +29,7 @@ void range_check_func(int32_t stage, const int32_t *input, const int32_t *buf,
if (!in_range) {
fprintf(stderr, "Error: coeffs contain out-of-range values\n");
fprintf(stderr, "size: %d\n", size);
fprintf(stderr, "stage: %d\n", stage);
fprintf(stderr, "allowed range: [%" PRId64 ";%" PRId64 "]\n", minValue,
maxValue);
......
av1/common/av1_inv_txfm2d.c
View file @ 901d5148
......@@ -44,21 +44,21 @@ static INLINE TxfmFunc inv_txfm_type_to_func(TXFM_TYPE txfm_type) {
static const TXFM_1D_CFG *inv_txfm_col_cfg_ls[TX_TYPES_1D][TX_SIZES_ALL] = {
// DCT
{
&inv_txfm_1d_col_cfg_dct_4, &inv_txfm_1d_col_cfg_dct_8,
&inv_txfm_1d_col_cfg_dct_16, &inv_txfm_1d_col_cfg_dct_32,
&inv_txfm_1d_col_cfg_dct_4, &inv_txfm_1d_col_cfg_dct_8,
&inv_txfm_1d_col_cfg_dct_16, &inv_txfm_1d_col_cfg_dct_32,
#if CONFIG_TX64X64
&inv_txfm_1d_col_cfg_dct_64,
#endif // CONFIG_TX64X64
&inv_txfm_1d_col_cfg_dct_8, &inv_txfm_1d_col_cfg_dct_4,
&inv_txfm_1d_col_cfg_dct_16, &inv_txfm_1d_col_cfg_dct_8,
&inv_txfm_1d_col_cfg_dct_32, &inv_txfm_1d_col_cfg_dct_16,
&inv_txfm_1d_col_cfg_dct_4x8, &inv_txfm_1d_col_cfg_dct_8x4,
&inv_txfm_1d_col_cfg_dct_8x16, &inv_txfm_1d_col_cfg_dct_16x8,
&inv_txfm_1d_col_cfg_dct_16x32, &inv_txfm_1d_col_cfg_dct_32x16,
#if CONFIG_TX64X64
&inv_txfm_1d_col_cfg_dct_64, &inv_txfm_1d_col_cfg_dct_32,
&inv_txfm_1d_col_cfg_dct_32x64, &inv_txfm_1d_col_cfg_dct_64x32,
#endif // CONFIG_TX64X64
&inv_txfm_1d_col_cfg_dct_16, &inv_txfm_1d_col_cfg_dct_4,
&inv_txfm_1d_col_cfg_dct_32, &inv_txfm_1d_col_cfg_dct_8,
&inv_txfm_1d_col_cfg_dct_4x16, &inv_txfm_1d_col_cfg_dct_16x4,
&inv_txfm_1d_col_cfg_dct_8x32, &inv_txfm_1d_col_cfg_dct_32x8,
#if CONFIG_TX64X64
&inv_txfm_1d_row_cfg_dct_64, &inv_txfm_1d_row_cfg_dct_16,
&inv_txfm_1d_col_cfg_dct_16x64, &inv_txfm_1d_col_cfg_dct_64x16,
#endif // CONFIG_TX64X64
},
// ADST
......@@ -70,23 +70,23 @@ static const TXFM_1D_CFG *inv_txfm_col_cfg_ls[TX_TYPES_1D][TX_SIZES_ALL] = {
#if CONFIG_TX64X64
NULL,
#endif // CONFIG_TX64X64
&inv_txfm_1d_col_cfg_adst_8,
&inv_txfm_1d_col_cfg_adst_4,
&inv_txfm_1d_col_cfg_adst_16,
&inv_txfm_1d_col_cfg_adst_8,
&inv_txfm_1d_col_cfg_adst_32,
&inv_txfm_1d_col_cfg_adst_16,
&inv_txfm_1d_col_cfg_adst_4x8,
&inv_txfm_1d_col_cfg_adst_8x4,
&inv_txfm_1d_col_cfg_adst_8x16,
&inv_txfm_1d_col_cfg_adst_16x8,
&inv_txfm_1d_col_cfg_adst_16x32,
&inv_txfm_1d_col_cfg_adst_32x16,
#if CONFIG_TX64X64
NULL,
&inv_txfm_1d_col_cfg_adst_32,
NULL,
#endif // CONFIG_TX64X64
&inv_txfm_1d_col_cfg_adst_16,
&inv_txfm_1d_col_cfg_adst_4,
&inv_txfm_1d_col_cfg_adst_32,
&inv_txfm_1d_col_cfg_adst_8,
&inv_txfm_1d_col_cfg_adst_4x16,
&inv_txfm_1d_col_cfg_adst_16x4,
&inv_txfm_1d_col_cfg_adst_8x32,
&inv_txfm_1d_col_cfg_adst_32x8,
#if CONFIG_TX64X64
NULL,
&inv_txfm_1d_row_cfg_adst_16,
NULL,
#endif // CONFIG_TX64X64
},
// FLIPADST
......@@ -98,23 +98,23 @@ static const TXFM_1D_CFG *inv_txfm_col_cfg_ls[TX_TYPES_1D][TX_SIZES_ALL] = {
#if CONFIG_TX64X64
NULL,
#endif // CONFIG_TX64X64
&inv_txfm_1d_col_cfg_adst_8,
&inv_txfm_1d_col_cfg_adst_4,
&inv_txfm_1d_col_cfg_adst_16,
&inv_txfm_1d_col_cfg_adst_8,
&inv_txfm_1d_col_cfg_adst_32,
&inv_txfm_1d_col_cfg_adst_16,
&inv_txfm_1d_col_cfg_adst_4x8,
&inv_txfm_1d_col_cfg_adst_8x4,
&inv_txfm_1d_col_cfg_adst_8x16,
&inv_txfm_1d_col_cfg_adst_16x8,
&inv_txfm_1d_col_cfg_adst_16x32,
&inv_txfm_1d_col_cfg_adst_32x16,
#if CONFIG_TX64X64
NULL,
&inv_txfm_1d_col_cfg_adst_32,
NULL,
#endif // CONFIG_TX64X64
&inv_txfm_1d_col_cfg_adst_16,
&inv_txfm_1d_col_cfg_adst_4,
&inv_txfm_1d_col_cfg_adst_32,
&inv_txfm_1d_col_cfg_adst_8,
&inv_txfm_1d_col_cfg_adst_4x16,
&inv_txfm_1d_col_cfg_adst_16x4,
&inv_txfm_1d_col_cfg_adst_8x32,
&inv_txfm_1d_col_cfg_adst_32x8,
#if CONFIG_TX64X64
NULL,
&inv_txfm_1d_row_cfg_adst_16,
NULL,
#endif // CONFIG_TX64X64
},
// IDENTITY
......
av1/encoder/av1_fwd_txfm1d_cfg.h
View file @ 901d5148
......@@ -145,20 +145,20 @@ static const int8_t fwd_stage_range_row_dct_8x16[6] =
ARRAYOFFSET6(4, 0, 1, 2, 3, 3, 3);
static const int8_t fwd_stage_range_row_adst_8x16[8] =
ARRAYOFFSET8(4, 0, 0, 1, 2, 2, 3, 3, 3);
static const int8_t fwd_cos_bit_row_dct_8x16[6] = { 13, 13, 13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_dct_8x16[6] = { 13, 13, 12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_adst_8x16[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
12, 12, 12, 12 };
// ---------------- 8x16 1D constants -----------------------
// ---------------- 16x8 1D constants -----------------------
#define fwd_shift_16x8 fwd_shift_16
static const int8_t fwd_stage_range_row_dct_16x8[8] =
ARRAYOFFSET8(3, 0, 1, 2, 3, 4, 4, 4, 4);
static const int8_t fwd_stage_range_row_adst_16x8[10] =
ARRAYOFFSET10(3, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4);
static const int8_t fwd_cos_bit_row_dct_16x8[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_adst_16x8[10] = { 13, 13, 13, 13, 13,
13, 13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_dct_16x8[8] = { 13, 13, 13, 12,
12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_adst_16x8[10] = { 13, 13, 13, 13, 12,
12, 12, 12, 12, 12 };
// ---------------- 16x32 1D constants -----------------------
#define fwd_shift_16x32 fwd_shift_32
......@@ -166,8 +166,8 @@ static const int8_t fwd_stage_range_row_dct_16x32[8] =
ARRAYOFFSET8(5, 0, 1, 2, 3, 4, 4, 4, 4);
static const int8_t fwd_stage_range_row_adst_16x32[10] =
ARRAYOFFSET10(5, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4);
static const int8_t fwd_cos_bit_row_dct_16x32[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_dct_16x32[8] = { 13, 13, 13, 12,
12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_adst_16x32[10] = { 12, 12, 12, 12, 12,
12, 12, 12, 12, 12 };
......@@ -186,8 +186,8 @@ static const int8_t fwd_cos_bit_row_adst_32x16[12] = { 12, 12, 12, 12, 12, 12,
#define fwd_shift_32x64 fwd_shift_64
static const int8_t fwd_stage_range_row_dct_32x64[10] =
ARRAYOFFSET10(6, 0, 1, 2, 3, 4, 5, 5, 5, 5, 5);
static const int8_t fwd_cos_bit_row_dct_32x64[10] = { 12, 12, 12, 12, 12,
12, 12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_dct_32x64[10] = { 13, 12, 12, 11, 10,
10, 10, 10, 10, 10 };
// ---------------- 64x32 1D constants -----------------------
#define fwd_shift_64x32 fwd_shift_64
......@@ -241,15 +241,15 @@ static const int8_t fwd_cos_bit_row_adst_32x8[12] = { 12, 12, 12, 12, 12, 12,
#define fwd_shift_16x64 fwd_shift_64
static const int8_t fwd_stage_range_row_dct_16x64[8] =
ARRAYOFFSET8(6, 0, 1, 2, 3, 4, 4, 4, 4);
static const int8_t fwd_cos_bit_row_dct_16x64[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_dct_16x64[8] = { 12, 12, 12, 11,
11, 11, 11, 11 };
// ---------------- 64x16 1D constants -----------------------
#define fwd_shift_64x16 fwd_shift_64
static const int8_t fwd_stage_range_row_dct_64x16[12] =
ARRAYOFFSET12(4, 0, 1, 2, 3, 4, 5, 6, 6, 6, 6, 6, 6);
static const int8_t fwd_cos_bit_row_dct_64x16[12] = { 13, 13, 12, 11, 10, 10,
10, 10, 10, 10, 10, 10 };
static const int8_t fwd_cos_bit_row_dct_64x16[12] = { 13, 13, 12, 11, 11, 11,
11, 11, 11, 11, 11, 11 };
//
// ---------------- row config fwd_dct_4 ----------------
......
av1/encoder/av1_fwd_txfm2d.c
View file @ 901d5148
......@@ -517,21 +517,21 @@ static const TXFM_1D_CFG *fwd_txfm_col_cfg_ls[TX_TYPES_1D][TX_SIZES_ALL] = {
static const TXFM_1D_CFG *fwd_txfm_row_cfg_ls[TX_TYPES_1D][TX_SIZES_ALL] = {
// DCT
{
&fwd_txfm_1d_row_cfg_dct_4, &fwd_txfm_1d_row_cfg_dct_8,
&fwd_txfm_1d_row_cfg_dct_16, &fwd_txfm_1d_row_cfg_dct_32,
&fwd_txfm_1d_row_cfg_dct_4, &fwd_txfm_1d_row_cfg_dct_8,
&fwd_txfm_1d_row_cfg_dct_16, &fwd_txfm_1d_row_cfg_dct_32,
#if CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_dct_64,
#endif // CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_dct_4, &fwd_txfm_1d_row_cfg_dct_8,
&fwd_txfm_1d_row_cfg_dct_8, &fwd_txfm_1d_row_cfg_dct_16,
&fwd_txfm_1d_row_cfg_dct_16, &fwd_txfm_1d_row_cfg_dct_32,
&fwd_txfm_1d_row_cfg_dct_4x8, &fwd_txfm_1d_row_cfg_dct_8x4,
&fwd_txfm_1d_row_cfg_dct_8x16, &fwd_txfm_1d_row_cfg_dct_16x8,
&fwd_txfm_1d_row_cfg_dct_16x32, &fwd_txfm_1d_row_cfg_dct_32x16,
#if CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_dct_32, &fwd_txfm_1d_row_cfg_dct_64,
&fwd_txfm_1d_row_cfg_dct_32x64, &fwd_txfm_1d_row_cfg_dct_64x32,
#endif // CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_dct_4, &fwd_txfm_1d_row_cfg_dct_16,
&fwd_txfm_1d_row_cfg_dct_8, &fwd_txfm_1d_row_cfg_dct_32,
&fwd_txfm_1d_row_cfg_dct_4x16, &fwd_txfm_1d_row_cfg_dct_16x4,
&fwd_txfm_1d_row_cfg_dct_8x32, &fwd_txfm_1d_row_cfg_dct_32x8,
#if CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_dct_16, &fwd_txfm_1d_row_cfg_dct_64,
&fwd_txfm_1d_row_cfg_dct_16x64, &fwd_txfm_1d_row_cfg_dct_64x16,
#endif // CONFIG_TX64X64
},
// ADST
......@@ -543,22 +543,22 @@ static const TXFM_1D_CFG *fwd_txfm_row_cfg_ls[TX_TYPES_1D][TX_SIZES_ALL] = {
#if CONFIG_TX64X64
NULL,
#endif // CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_adst_4,
&fwd_txfm_1d_row_cfg_adst_8,
&fwd_txfm_1d_row_cfg_adst_8,
&fwd_txfm_1d_row_cfg_adst_16,
&fwd_txfm_1d_row_cfg_adst_16,
&fwd_txfm_1d_row_cfg_adst_32,
&fwd_txfm_1d_row_cfg_adst_4x8,
&fwd_txfm_1d_row_cfg_adst_8x4,
&fwd_txfm_1d_row_cfg_adst_8x16,
&fwd_txfm_1d_row_cfg_adst_16x8,
&fwd_txfm_1d_row_cfg_adst_16x32,
&fwd_txfm_1d_row_cfg_adst_32x16,
#if CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_adst_32,
NULL,
NULL,
#endif // CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_adst_4,
&fwd_txfm_1d_row_cfg_adst_16,
&fwd_txfm_1d_row_cfg_adst_8,
&fwd_txfm_1d_row_cfg_adst_32,
&fwd_txfm_1d_row_cfg_adst_4x16,
&fwd_txfm_1d_row_cfg_adst_16x4,
&fwd_txfm_1d_row_cfg_adst_8x32,
&fwd_txfm_1d_row_cfg_adst_32x8,
#if CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_adst_16,
NULL,
NULL,
#endif // CONFIG_TX64X64
},
......@@ -571,22 +571,22 @@ static const TXFM_1D_CFG *fwd_txfm_row_cfg_ls[TX_TYPES_1D][TX_SIZES_ALL] = {
#if CONFIG_TX64X64
NULL,
#endif // CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_adst_4,
&fwd_txfm_1d_row_cfg_adst_8,
&fwd_txfm_1d_row_cfg_adst_8,
&fwd_txfm_1d_row_cfg_adst_16,
&fwd_txfm_1d_row_cfg_adst_16,
&fwd_txfm_1d_row_cfg_adst_32,
&fwd_txfm_1d_row_cfg_adst_4x8,
&fwd_txfm_1d_row_cfg_adst_8x4,
&fwd_txfm_1d_row_cfg_adst_8x16,
&fwd_txfm_1d_row_cfg_adst_16x8,
&fwd_txfm_1d_row_cfg_adst_16x32,
&fwd_txfm_1d_row_cfg_adst_32x16,
#if CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_adst_32,
NULL,
NULL,
#endif // CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_adst_4,
&fwd_txfm_1d_row_cfg_adst_16,
&fwd_txfm_1d_row_cfg_adst_8,
&fwd_txfm_1d_row_cfg_adst_32,
&fwd_txfm_1d_row_cfg_adst_4x16,
&fwd_txfm_1d_row_cfg_adst_16x4,
&fwd_txfm_1d_row_cfg_adst_8x32,
&fwd_txfm_1d_row_cfg_adst_32x8,
#if CONFIG_TX64X64
&fwd_txfm_1d_row_cfg_adst_16,
NULL,
NULL,
#endif // CONFIG_TX64X64
},
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment