Commit ddbcde28 authored by Hui Su's avatar Hui Su

Refactor the ext-tx experiment

Use common structure for inter and intra tx type information when
possible.

Change-Id: I1fd3bc86033871ffbcc2b496a31dca00b7d64b31
parent 772cb136
......@@ -933,75 +933,142 @@ static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; }
#if CONFIG_EXT_TX
#define ALLOW_INTRA_EXT_TX 1
typedef enum {
// DCT only
EXT_TX_SET_DCTONLY = 0,
// DCT + Identity only
EXT_TX_SET_DCT_IDTX,
// Number of transform types in each set type
static const int av1_num_ext_tx_set[EXT_TX_SET_TYPES] = {
1, 2,
#if CONFIG_MRC_TX
// DCT + MRC_DCT
EXT_TX_SET_MRC_DCT,
// DCT + MRC_DCT + IDTX
EXT_TX_SET_MRC_DCT_IDTX,
2, 3,
#endif // CONFIG_MRC_TX
// Discrete Trig transforms w/o flip (4) + Identity (1)
EXT_TX_SET_DTT4_IDTX,
// Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2)
EXT_TX_SET_DTT4_IDTX_1DDCT,
// Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver DCT (2)
EXT_TX_SET_DTT9_IDTX_1DDCT,
// Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6)
EXT_TX_SET_ALL16,
EXT_TX_SET_TYPES
} TxSetType;
#if CONFIG_MRC_TX
// Number of transform types in each set type
static const int num_ext_tx_set[EXT_TX_SET_TYPES] = {
1, 2, 2, 3, 5, 7, 12, 16
5, 7, 12, 16,
};
// Maps intra set index to the set type
static const int ext_tx_set_type_intra[EXT_TX_SETS_INTRA] = {
static const int av1_ext_tx_set_type_intra[EXT_TX_SETS_INTRA] = {
EXT_TX_SET_DCTONLY, EXT_TX_SET_DTT4_IDTX_1DDCT, EXT_TX_SET_DTT4_IDTX,
EXT_TX_SET_MRC_DCT
#if CONFIG_MRC_TX
EXT_TX_SET_MRC_DCT,
#endif // CONFIG_MRC_TX
};
// Maps inter set index to the set type
static const int ext_tx_set_type_inter[EXT_TX_SETS_INTER] = {
EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16, EXT_TX_SET_DTT9_IDTX_1DDCT,
EXT_TX_SET_DCT_IDTX, EXT_TX_SET_MRC_DCT_IDTX
static const int av1_ext_tx_set_type_inter[EXT_TX_SETS_INTER] = {
EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16,
EXT_TX_SET_DTT9_IDTX_1DDCT, EXT_TX_SET_DCT_IDTX,
#if CONFIG_MRC_TX
EXT_TX_SET_MRC_DCT_IDTX,
#endif // CONFIG_MRC_TX
};
// Maps set types above to the indices used for intra
static const int ext_tx_set_index_intra[EXT_TX_SET_TYPES] = { 0, -1, 3, -1,
2, 1, -1, -1 };
static const int ext_tx_set_index_intra[EXT_TX_SET_TYPES] = {
0, -1,
#if CONFIG_MRC_TX
3, -1,
#endif // CONFIG_MRC_TX
2, 1, -1, -1,
};
// Maps set types above to the indices used for inter
static const int ext_tx_set_index_inter[EXT_TX_SET_TYPES] = { 0, 3, -1, 4,
-1, -1, 2, 1 };
#else // CONFIG_MRC_TX
// Number of transform types in each set type
static const int num_ext_tx_set[EXT_TX_SET_TYPES] = { 1, 2, 5, 7, 12, 16 };
// Maps intra set index to the set type
static const int ext_tx_set_type_intra[EXT_TX_SETS_INTRA] = {
EXT_TX_SET_DCTONLY, EXT_TX_SET_DTT4_IDTX_1DDCT, EXT_TX_SET_DTT4_IDTX
static const int ext_tx_set_index_inter[EXT_TX_SET_TYPES] = {
0, 3,
#if CONFIG_MRC_TX
-1, 4,
#endif // CONFIG_MRC_TX
-1, -1, 2, 1,
};
// Maps inter set index to the set type
static const int ext_tx_set_type_inter[EXT_TX_SETS_INTER] = {
EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16, EXT_TX_SET_DTT9_IDTX_1DDCT,
EXT_TX_SET_DCT_IDTX
};
static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] =
{
#if CONFIG_CHROMA_2X2
{ 1, 1, 1, 1, 1 }, // unused
{ 0, 1, 1, 0, 0 },
{ 0, 0, 0, 1, 0 },
#if CONFIG_MRC_TX
{ 0, 0, 0, 0, 1 },
#endif // CONFIG_MRC_TX
#else // CONFIG_CHROMA_2X2
{ 1, 1, 1, 1 }, // unused
{ 1, 1, 0, 0 },
{ 0, 0, 1, 0 },
#if CONFIG_MRC_TX
{ 0, 0, 0, 1 },
#endif // CONFIG_MRC_TX
#endif // CONFIG_CHROMA_2X2
};
// Maps set types above to the indices used for intra
static const int ext_tx_set_index_intra[EXT_TX_SET_TYPES] = { 0, -1, 2,
1, -1, -1 };
static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][EXT_TX_SIZES] =
{
#if CONFIG_CHROMA_2X2
{ 1, 1, 1, 1, 1 }, // unused
{ 0, 1, 1, 0, 0 }, { 0, 0, 0, 1, 0 }, { 0, 0, 0, 0, 1 },
#if CONFIG_MRC_TX
{ 0, 0, 0, 0, 1 },
#endif // CONFIG_MRC_TX
#else // CONFIG_CHROMA_2X2
{ 1, 1, 1, 1 }, // unused
{ 1, 1, 0, 0 }, { 0, 0, 1, 0 }, { 0, 0, 0, 1 },
#if CONFIG_MRC_TX
{ 0, 0, 0, 1 },
#endif // CONFIG_MRC_TX
#endif // CONFIG_CHROMA_2X2
};
// Maps set types above to the indices used for inter
static const int ext_tx_set_index_inter[EXT_TX_SET_TYPES] = {
0, 3, -1, -1, 2, 1
// 1D Transforms used in inter set, this needs to be changed if
// ext_tx_used_inter is changed
static const int ext_tx_used_inter_1D[EXT_TX_SETS_INTER][TX_TYPES_1D] = {
{ 1, 0, 0, 0 }, { 1, 1, 1, 1 }, { 1, 1, 1, 1 }, { 1, 0, 0, 1 },
#if CONFIG_MRC_TX
{ 1, 0, 0, 1 },
#endif // CONFIG_MRC_TX
};
#if CONFIG_MRC_TX
static const int av1_ext_tx_used[EXT_TX_SET_TYPES][TX_TYPES] = {
{
1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
},
{
1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,
},
{
1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,
},
{
1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1,
},
{
1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,
},
{
1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0,
},
{
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,
},
{
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0,
},
};
#else // CONFIG_MRC_TX
static const int av1_ext_tx_used[EXT_TX_SET_TYPES][TX_TYPES] = {
{
1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
},
{
1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,
},
{
1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,
},
{
1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0,
},
{
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0,
},
{
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
},
};
#endif // CONFIG_MRC_TX
......@@ -1043,124 +1110,11 @@ static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
: ext_tx_set_index_intra[set_type];
}
#if CONFIG_MRC_TX
static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] =
{
#if CONFIG_CHROMA_2X2
{ 1, 1, 1, 1, 1 }, // unused
{ 0, 1, 1, 0, 0 },
{ 0, 0, 0, 1, 0 },
{ 0, 0, 0, 0, 1 },
#else
{ 1, 1, 1, 1 }, // unused
{ 1, 1, 0, 0 },
{ 0, 0, 1, 0 },
{ 0, 0, 0, 1 },
#endif // CONFIG_CHROMA_2X2
};
static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][EXT_TX_SIZES] =
{
#if CONFIG_CHROMA_2X2
{ 1, 1, 1, 1, 1 }, // unused
{ 0, 1, 1, 0, 0 }, { 0, 0, 0, 1, 0 },
{ 0, 0, 0, 0, 1 }, { 0, 0, 0, 0, 1 },
#else
{ 1, 1, 1, 1 }, // unused
{ 1, 1, 0, 0 }, { 0, 0, 1, 0 }, { 0, 0, 0, 1 }, { 0, 0, 0, 1 },
#endif // CONFIG_CHROMA_2X2
};
// Transform types used in each intra set
static const int ext_tx_used_intra[EXT_TX_SETS_INTRA][TX_TYPES] = {
{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
{ 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0 },
{ 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 },
{ 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1 },
};
// Numbers of transform types used in each intra set
static const int ext_tx_cnt_intra[EXT_TX_SETS_INTRA] = { 1, 7, 5, 2 };
// Transform types used in each inter set
static const int ext_tx_used_inter[EXT_TX_SETS_INTER][TX_TYPES] = {
{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
{ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0 },
{ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 },
{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 },
{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1 },
};
// Numbers of transform types used in each inter set
static const int ext_tx_cnt_inter[EXT_TX_SETS_INTER] = { 1, 16, 12, 2, 3 };
// 1D Transforms used in inter set, this needs to be changed if
// ext_tx_used_inter is changed
static const int ext_tx_used_inter_1D[EXT_TX_SETS_INTER][TX_TYPES_1D] = {
{ 1, 0, 0, 0 }, { 1, 1, 1, 1 }, { 1, 1, 1, 1 }, { 1, 0, 0, 1 }, { 1, 0, 0, 1 }
};
#else // CONFIG_MRC_TX
static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] =
{
#if CONFIG_CHROMA_2X2
{ 1, 1, 1, 1, 1 }, // unused
{ 0, 1, 1, 0, 0 },
{ 0, 0, 0, 1, 0 },
#else
{ 1, 1, 1, 1 }, // unused
{ 1, 1, 0, 0 },
{ 0, 0, 1, 0 },
#endif // CONFIG_CHROMA_2X2
};
static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][EXT_TX_SIZES] =
{
#if CONFIG_CHROMA_2X2
{ 1, 1, 1, 1, 1 }, // unused
{ 0, 1, 1, 0, 0 },
{ 0, 0, 0, 1, 0 },
{ 0, 0, 0, 0, 1 },
#else
{ 1, 1, 1, 1 }, // unused
{ 1, 1, 0, 0 },
{ 0, 0, 1, 0 },
{ 0, 0, 0, 1 },
#endif // CONFIG_CHROMA_2X2
};
// Transform types used in each intra set
static const int ext_tx_used_intra[EXT_TX_SETS_INTRA][TX_TYPES] = {
{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
{ 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0 },
{ 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0 },
};
// Numbers of transform types used in each intra set
static const int ext_tx_cnt_intra[EXT_TX_SETS_INTRA] = { 1, 7, 5 };
// Transform types used in each inter set
static const int ext_tx_used_inter[EXT_TX_SETS_INTER][TX_TYPES] = {
{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
{ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 },
{ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0 },
{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0 },
};
// Numbers of transform types used in each inter set
static const int ext_tx_cnt_inter[EXT_TX_SETS_INTER] = { 1, 16, 12, 2 };
// 1D Transforms used in inter set, this needs to be changed if
// ext_tx_used_inter is changed
static const int ext_tx_used_inter_1D[EXT_TX_SETS_INTER][TX_TYPES_1D] = {
{ 1, 0, 0, 0 }, { 1, 1, 1, 1 }, { 1, 1, 1, 1 }, { 1, 0, 0, 1 },
};
#endif // CONFIG_MRC_TX
static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
int use_reduced_set) {
const int set_type =
get_ext_tx_set_type(tx_size, bs, is_inter, use_reduced_set);
return num_ext_tx_set[set_type];
return av1_num_ext_tx_set[set_type];
}
#if CONFIG_RECT_TX
......
......@@ -1409,10 +1409,8 @@ static const aom_cdf_prob default_delta_lf_cdf[CDF_SIZE(DELTA_LF_PROBS + 1)] = {
#endif
#endif
#if CONFIG_EXT_TX
int av1_ext_tx_intra_ind[EXT_TX_SETS_INTRA][TX_TYPES];
int av1_ext_tx_intra_inv[EXT_TX_SETS_INTRA][TX_TYPES];
int av1_ext_tx_inter_ind[EXT_TX_SETS_INTER][TX_TYPES];
int av1_ext_tx_inter_inv[EXT_TX_SETS_INTER][TX_TYPES];
int av1_ext_tx_ind[EXT_TX_SET_TYPES][TX_TYPES];
int av1_ext_tx_inv[EXT_TX_SET_TYPES][TX_TYPES];
#endif
#if CONFIG_SMOOTH_HV
......@@ -2485,70 +2483,59 @@ static const aom_prob default_switchable_interp_prob[SWITCHABLE_FILTER_CONTEXTS]
#if CONFIG_EXT_TX
/* clang-format off */
const aom_tree_index av1_ext_tx_inter_tree[EXT_TX_SETS_INTER]
[TREE_SIZE(TX_TYPES)] = {
{ // ToDo(yaowu): remove used entry 0.
0
}, {
-IDTX, 2,
4, 14,
6, 8,
-V_DCT, -H_DCT,
10, 12,
-V_ADST, -H_ADST,
-V_FLIPADST, -H_FLIPADST,
-DCT_DCT, 16,
18, 24,
20, 22,
-ADST_DCT, -DCT_ADST,
-FLIPADST_DCT, -DCT_FLIPADST,
26, 28,
-ADST_ADST, -FLIPADST_FLIPADST,
-ADST_FLIPADST, -FLIPADST_ADST
}, {
-IDTX, 2,
4, 6,
-V_DCT, -H_DCT,
-DCT_DCT, 8,
10, 16,
12, 14,
-ADST_DCT, -DCT_ADST,
-FLIPADST_DCT, -DCT_FLIPADST,
18, 20,
-ADST_ADST, -FLIPADST_FLIPADST,
-ADST_FLIPADST, -FLIPADST_ADST
}, {
-IDTX, -DCT_DCT,
},
#if CONFIG_MRC_TX
{
-IDTX, 2, -DCT_DCT, -MRC_DCT,
}
#endif // CONFIG_MRC_TX
};
const aom_tree_index av1_ext_tx_intra_tree[EXT_TX_SETS_INTRA]
[TREE_SIZE(TX_TYPES)] = {
{ // ToDo(yaowu): remove unused entry 0.
0
}, {
-IDTX, 2,
-DCT_DCT, 4,
6, 8,
-V_DCT, -H_DCT,
-ADST_ADST, 10,
-ADST_DCT, -DCT_ADST,
}, {
-IDTX, 2,
-DCT_DCT, 4,
-ADST_ADST, 6,
-ADST_DCT, -DCT_ADST,
},
const aom_tree_index av1_ext_tx_tree[EXT_TX_SET_TYPES][TREE_SIZE(TX_TYPES)] = {
// TODO(yaowu@google.com): remove used entry 0.
{ 0 },
{ -IDTX, -DCT_DCT, },
#if CONFIG_MRC_TX
{
-DCT_DCT, -MRC_DCT,
}
{ -DCT_DCT, -MRC_DCT, },
{ -IDTX, 2,
-DCT_DCT, -MRC_DCT, },
#endif // CONFIG_MRC_TX
{
-IDTX, 2,
-DCT_DCT, 4,
-ADST_ADST, 6,
-ADST_DCT, -DCT_ADST,
},
{
-IDTX, 2,
-DCT_DCT, 4,
6, 8,
-V_DCT, -H_DCT,
-ADST_ADST, 10,
-ADST_DCT, -DCT_ADST,
},
{
-IDTX, 2,
4, 6,
-V_DCT, -H_DCT,
-DCT_DCT, 8,
10, 16,
12, 14,
-ADST_DCT, -DCT_ADST,
-FLIPADST_DCT, -DCT_FLIPADST,
18, 20,
-ADST_ADST, -FLIPADST_FLIPADST,
-ADST_FLIPADST, -FLIPADST_ADST,
},
{
-IDTX, 2,
4, 14,
6, 8,
-V_DCT, -H_DCT,
10, 12,
-V_ADST, -H_ADST,
-V_FLIPADST, -H_FLIPADST,
-DCT_DCT, 16,
18, 24,
20, 22,
-ADST_DCT, -DCT_ADST,
-FLIPADST_DCT, -DCT_FLIPADST,
26, 28,
-ADST_ADST, -FLIPADST_FLIPADST,
-ADST_FLIPADST, -FLIPADST_ADST,
},
};
/* clang-format on */
......@@ -5531,17 +5518,20 @@ void av1_adapt_intra_frame_probs(AV1_COMMON *cm) {
int s;
for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
if (use_inter_ext_tx_for_txsize[s][i]) {
aom_tree_merge_probs(
av1_ext_tx_inter_tree[s], pre_fc->inter_ext_tx_prob[s][i],
counts->inter_ext_tx[s][i], fc->inter_ext_tx_prob[s][i]);
aom_tree_merge_probs(av1_ext_tx_tree[av1_ext_tx_set_type_inter[s]],
pre_fc->inter_ext_tx_prob[s][i],
counts->inter_ext_tx[s][i],
fc->inter_ext_tx_prob[s][i]);
}
}
for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
if (use_intra_ext_tx_for_txsize[s][i]) {
for (j = 0; j < INTRA_MODES; ++j)
aom_tree_merge_probs(
av1_ext_tx_intra_tree[s], pre_fc->intra_ext_tx_prob[s][i][j],
counts->intra_ext_tx[s][i][j], fc->intra_ext_tx_prob[s][i][j]);
for (j = 0; j < INTRA_MODES; ++j) {
aom_tree_merge_probs(av1_ext_tx_tree[av1_ext_tx_set_type_intra[s]],
pre_fc->intra_ext_tx_prob[s][i][j],
counts->intra_ext_tx[s][i][j],
fc->intra_ext_tx_prob[s][i][j]);
}
}
}
}
......
......@@ -551,10 +551,8 @@ extern const aom_prob
extern const int av1_intra_mode_ind[INTRA_MODES];
extern const int av1_intra_mode_inv[INTRA_MODES];
#if CONFIG_EXT_TX
extern int av1_ext_tx_intra_ind[EXT_TX_SETS_INTRA][TX_TYPES];
extern int av1_ext_tx_intra_inv[EXT_TX_SETS_INTRA][TX_TYPES];
extern int av1_ext_tx_inter_ind[EXT_TX_SETS_INTER][TX_TYPES];
extern int av1_ext_tx_inter_inv[EXT_TX_SETS_INTER][TX_TYPES];
extern int av1_ext_tx_ind[EXT_TX_SET_TYPES][TX_TYPES];
extern int av1_ext_tx_inv[EXT_TX_SET_TYPES][TX_TYPES];
#endif
#if CONFIG_EXT_INTER
......@@ -588,6 +586,8 @@ extern const aom_tree_index av1_ext_tx_inter_tree[EXT_TX_SETS_INTER]
[TREE_SIZE(TX_TYPES)];
extern const aom_tree_index av1_ext_tx_intra_tree[EXT_TX_SETS_INTRA]
[TREE_SIZE(TX_TYPES)];
extern const aom_tree_index av1_ext_tx_tree[EXT_TX_SET_TYPES]
[TREE_SIZE(TX_TYPES)];
#else
extern const aom_tree_index av1_ext_tx_tree[TREE_SIZE(TX_TYPES)];
#endif // CONFIG_EXT_TX
......
......@@ -292,6 +292,28 @@ typedef enum {
} TX_TYPE;
#if CONFIG_EXT_TX
typedef enum {
// DCT only
EXT_TX_SET_DCTONLY = 0,
// DCT + Identity only
EXT_TX_SET_DCT_IDTX,
#if CONFIG_MRC_TX
// DCT + MRC_DCT
EXT_TX_SET_MRC_DCT,
// DCT + MRC_DCT + IDTX
EXT_TX_SET_MRC_DCT_IDTX,
#endif // CONFIG_MRC_TX
// Discrete Trig transforms w/o flip (4) + Identity (1)
EXT_TX_SET_DTT4_IDTX,
// Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2)
EXT_TX_SET_DTT4_IDTX_1DDCT,
// Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver DCT (2)
EXT_TX_SET_DTT9_IDTX_1DDCT,
// Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6)
EXT_TX_SET_ALL16,
EXT_TX_SET_TYPES
} TxSetType;
#define IS_2D_TRANSFORM(tx_type) (tx_type < IDTX)
#else
#define IS_2D_TRANSFORM(tx_type) 1
......
......@@ -2468,10 +2468,12 @@ static void decode_partition(AV1Decoder *const pbi, MACROBLOCKD *const xd,
const int eset =
get_ext_tx_set(supertx_size, bsize, 1, cm->reduced_tx_set_used);
if (eset > 0) {
const TxSetType tx_set_type = get_ext_tx_set_type(
supertx_size, bsize, 1, cm->reduced_tx_set_used);
const int packed_sym =
aom_read_symbol(r, ec_ctx->inter_ext_tx_cdf[eset][supertx_size],
ext_tx_cnt_inter[eset], ACCT_STR);
txfm = av1_ext_tx_inter_inv[eset][packed_sym];
av1_num_ext_tx_set[tx_set_type], ACCT_STR);
txfm = av1_ext_tx_inv[tx_set_type][packed_sym];
if (xd->counts) ++xd->counts->inter_ext_tx[eset][supertx_size][txfm];
}
}
......
......@@ -985,6 +985,8 @@ void av1_read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd,
!supertx_enabled &&
#endif // CONFIG_SUPERTX
!segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
const TxSetType tx_set_type = get_ext_tx_set_type(
tx_size, mbmi->sb_type, inter_block, cm->reduced_tx_set_used);
const int eset = get_ext_tx_set(tx_size, mbmi->sb_type, inter_block,
cm->reduced_tx_set_used);
// eset == 0 should correspond to a set with only DCT_DCT and
......@@ -993,14 +995,14 @@ void av1_read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd,
FRAME_COUNTS *counts = xd->counts;
if (inter_block) {
*tx_type = av1_ext_tx_inter_inv[eset][aom_read_symbol(
*tx_type = av1_ext_tx_inv[tx_set_type][aom_read_symbol(
r, ec_ctx->inter_ext_tx_cdf[eset][square_tx_size],
ext_tx_cnt_inter[eset], ACCT_STR)];
av1_num_ext_tx_set[tx_set_type], ACCT_STR)];
if (counts) ++counts->inter_ext_tx[eset][square_tx_size][*tx_type];
} else if (ALLOW_INTRA_EXT_TX) {
*tx_type = av1_ext_tx_intra_inv[eset][aom_read_symbol(
*tx_type = av1_ext_tx_inv[tx_set_type][aom_read_symbol(
r, ec_ctx->intra_ext_tx_cdf[eset][square_tx_size][mbmi->mode],
ext_tx_cnt_intra[eset], ACCT_STR)];
av1_num_ext_tx_set[tx_set_type], ACCT_STR)];
if (counts)
++counts->intra_ext_tx[eset][square_tx_size][mbmi->mode][*tx_type];
}
......
......@@ -55,13 +55,10 @@ static void initialize_dec(void) {
av1_indices_from_tree(av1_switchable_interp_ind, av1_switchable_interp_inv,
av1_switchable_interp_tree);
#if CONFIG_EXT_TX
int s;
for (s = 1; s < EXT_TX_SETS_INTRA; ++s)
av1_indices_from_tree(av1_ext_tx_intra_ind[s], av1_ext_tx_intra_inv[s],
av1_ext_tx_intra_tree[s]);
for (s = 1; s < EXT_TX_SETS_INTER; ++s)
av1_indices_from_tree(av1_ext_tx_inter_ind[s], av1_ext_tx_inter_inv[s],
av1_ext_tx_inter_tree[s]);
for (int s = 1; s < EXT_TX_SET_TYPES; ++s) {
av1_indices_from_tree(av1_ext_tx_ind[s], av1_ext_tx_inv[s],
av1_ext_tx_tree[s]);