Commit ddbcde28 authored by Hui Su's avatar Hui Su

Refactor the ext-tx experiment

Use common structure for inter and intra tx type information when
possible.

Change-Id: I1fd3bc86033871ffbcc2b496a31dca00b7d64b31
parent 772cb136
This diff is collapsed.
......@@ -1409,10 +1409,8 @@ static const aom_cdf_prob default_delta_lf_cdf[CDF_SIZE(DELTA_LF_PROBS + 1)] = {
#endif
#endif
#if CONFIG_EXT_TX
int av1_ext_tx_intra_ind[EXT_TX_SETS_INTRA][TX_TYPES];
int av1_ext_tx_intra_inv[EXT_TX_SETS_INTRA][TX_TYPES];
int av1_ext_tx_inter_ind[EXT_TX_SETS_INTER][TX_TYPES];
int av1_ext_tx_inter_inv[EXT_TX_SETS_INTER][TX_TYPES];
int av1_ext_tx_ind[EXT_TX_SET_TYPES][TX_TYPES];
int av1_ext_tx_inv[EXT_TX_SET_TYPES][TX_TYPES];
#endif
#if CONFIG_SMOOTH_HV
......@@ -2485,70 +2483,59 @@ static const aom_prob default_switchable_interp_prob[SWITCHABLE_FILTER_CONTEXTS]
#if CONFIG_EXT_TX
/* clang-format off */
const aom_tree_index av1_ext_tx_inter_tree[EXT_TX_SETS_INTER]
[TREE_SIZE(TX_TYPES)] = {
{ // ToDo(yaowu): remove used entry 0.
0
}, {
-IDTX, 2,
4, 14,
6, 8,
-V_DCT, -H_DCT,
10, 12,
-V_ADST, -H_ADST,
-V_FLIPADST, -H_FLIPADST,
-DCT_DCT, 16,
18, 24,
20, 22,
-ADST_DCT, -DCT_ADST,
-FLIPADST_DCT, -DCT_FLIPADST,
26, 28,
-ADST_ADST, -FLIPADST_FLIPADST,
-ADST_FLIPADST, -FLIPADST_ADST
}, {
-IDTX, 2,
4, 6,
-V_DCT, -H_DCT,
-DCT_DCT, 8,
10, 16,
12, 14,
-ADST_DCT, -DCT_ADST,
-FLIPADST_DCT, -DCT_FLIPADST,
18, 20,
-ADST_ADST, -FLIPADST_FLIPADST,
-ADST_FLIPADST, -FLIPADST_ADST
}, {
-IDTX, -DCT_DCT,
},
#if CONFIG_MRC_TX
{
-IDTX, 2, -DCT_DCT, -MRC_DCT,
}
#endif // CONFIG_MRC_TX
};
const aom_tree_index av1_ext_tx_intra_tree[EXT_TX_SETS_INTRA]
[TREE_SIZE(TX_TYPES)] = {
{ // ToDo(yaowu): remove unused entry 0.
0
}, {
-IDTX, 2,
-DCT_DCT, 4,
6, 8,
-V_DCT, -H_DCT,
-ADST_ADST, 10,
-ADST_DCT, -DCT_ADST,
}, {
-IDTX, 2,
-DCT_DCT, 4,
-ADST_ADST, 6,
-ADST_DCT, -DCT_ADST,
},
const aom_tree_index av1_ext_tx_tree[EXT_TX_SET_TYPES][TREE_SIZE(TX_TYPES)] = {
// TODO(yaowu@google.com): remove used entry 0.
{ 0 },
{ -IDTX, -DCT_DCT, },
#if CONFIG_MRC_TX
{
-DCT_DCT, -MRC_DCT,
}
{ -DCT_DCT, -MRC_DCT, },
{ -IDTX, 2,
-DCT_DCT, -MRC_DCT, },
#endif // CONFIG_MRC_TX
{
-IDTX, 2,
-DCT_DCT, 4,
-ADST_ADST, 6,
-ADST_DCT, -DCT_ADST,
},
{
-IDTX, 2,
-DCT_DCT, 4,
6, 8,
-V_DCT, -H_DCT,
-ADST_ADST, 10,
-ADST_DCT, -DCT_ADST,
},
{
-IDTX, 2,
4, 6,
-V_DCT, -H_DCT,
-DCT_DCT, 8,
10, 16,
12, 14,
-ADST_DCT, -DCT_ADST,
-FLIPADST_DCT, -DCT_FLIPADST,
18, 20,
-ADST_ADST, -FLIPADST_FLIPADST,
-ADST_FLIPADST, -FLIPADST_ADST,
},
{
-IDTX, 2,
4, 14,
6, 8,
-V_DCT, -H_DCT,
10, 12,
-V_ADST, -H_ADST,
-V_FLIPADST, -H_FLIPADST,
-DCT_DCT, 16,
18, 24,
20, 22,
-ADST_DCT, -DCT_ADST,
-FLIPADST_DCT, -DCT_FLIPADST,
26, 28,
-ADST_ADST, -FLIPADST_FLIPADST,
-ADST_FLIPADST, -FLIPADST_ADST,
},
};
/* clang-format on */
......@@ -5531,17 +5518,20 @@ void av1_adapt_intra_frame_probs(AV1_COMMON *cm) {
int s;
for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
if (use_inter_ext_tx_for_txsize[s][i]) {
aom_tree_merge_probs(
av1_ext_tx_inter_tree[s], pre_fc->inter_ext_tx_prob[s][i],
counts->inter_ext_tx[s][i], fc->inter_ext_tx_prob[s][i]);
aom_tree_merge_probs(av1_ext_tx_tree[av1_ext_tx_set_type_inter[s]],
pre_fc->inter_ext_tx_prob[s][i],
counts->inter_ext_tx[s][i],
fc->inter_ext_tx_prob[s][i]);
}
}
for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
if (use_intra_ext_tx_for_txsize[s][i]) {
for (j = 0; j < INTRA_MODES; ++j)
aom_tree_merge_probs(
av1_ext_tx_intra_tree[s], pre_fc->intra_ext_tx_prob[s][i][j],
counts->intra_ext_tx[s][i][j], fc->intra_ext_tx_prob[s][i][j]);
for (j = 0; j < INTRA_MODES; ++j) {
aom_tree_merge_probs(av1_ext_tx_tree[av1_ext_tx_set_type_intra[s]],
pre_fc->intra_ext_tx_prob[s][i][j],
counts->intra_ext_tx[s][i][j],
fc->intra_ext_tx_prob[s][i][j]);
}
}
}
}
......
......@@ -551,10 +551,8 @@ extern const aom_prob
extern const int av1_intra_mode_ind[INTRA_MODES];
extern const int av1_intra_mode_inv[INTRA_MODES];
#if CONFIG_EXT_TX
extern int av1_ext_tx_intra_ind[EXT_TX_SETS_INTRA][TX_TYPES];
extern int av1_ext_tx_intra_inv[EXT_TX_SETS_INTRA][TX_TYPES];
extern int av1_ext_tx_inter_ind[EXT_TX_SETS_INTER][TX_TYPES];
extern int av1_ext_tx_inter_inv[EXT_TX_SETS_INTER][TX_TYPES];
extern int av1_ext_tx_ind[EXT_TX_SET_TYPES][TX_TYPES];
extern int av1_ext_tx_inv[EXT_TX_SET_TYPES][TX_TYPES];
#endif
#if CONFIG_EXT_INTER
......@@ -588,6 +586,8 @@ extern const aom_tree_index av1_ext_tx_inter_tree[EXT_TX_SETS_INTER]
[TREE_SIZE(TX_TYPES)];
extern const aom_tree_index av1_ext_tx_intra_tree[EXT_TX_SETS_INTRA]
[TREE_SIZE(TX_TYPES)];
extern const aom_tree_index av1_ext_tx_tree[EXT_TX_SET_TYPES]
[TREE_SIZE(TX_TYPES)];
#else
extern const aom_tree_index av1_ext_tx_tree[TREE_SIZE(TX_TYPES)];
#endif // CONFIG_EXT_TX
......
......@@ -292,6 +292,28 @@ typedef enum {
} TX_TYPE;
#if CONFIG_EXT_TX
typedef enum {
// DCT only
EXT_TX_SET_DCTONLY = 0,
// DCT + Identity only
EXT_TX_SET_DCT_IDTX,
#if CONFIG_MRC_TX
// DCT + MRC_DCT
EXT_TX_SET_MRC_DCT,
// DCT + MRC_DCT + IDTX
EXT_TX_SET_MRC_DCT_IDTX,
#endif // CONFIG_MRC_TX
// Discrete Trig transforms w/o flip (4) + Identity (1)
EXT_TX_SET_DTT4_IDTX,
// Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2)
EXT_TX_SET_DTT4_IDTX_1DDCT,
// Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver DCT (2)
EXT_TX_SET_DTT9_IDTX_1DDCT,
// Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6)
EXT_TX_SET_ALL16,
EXT_TX_SET_TYPES
} TxSetType;
#define IS_2D_TRANSFORM(tx_type) (tx_type < IDTX)
#else
#define IS_2D_TRANSFORM(tx_type) 1
......
......@@ -2468,10 +2468,12 @@ static void decode_partition(AV1Decoder *const pbi, MACROBLOCKD *const xd,
const int eset =
get_ext_tx_set(supertx_size, bsize, 1, cm->reduced_tx_set_used);
if (eset > 0) {
const TxSetType tx_set_type = get_ext_tx_set_type(
supertx_size, bsize, 1, cm->reduced_tx_set_used);
const int packed_sym =
aom_read_symbol(r, ec_ctx->inter_ext_tx_cdf[eset][supertx_size],
ext_tx_cnt_inter[eset], ACCT_STR);
txfm = av1_ext_tx_inter_inv[eset][packed_sym];
av1_num_ext_tx_set[tx_set_type], ACCT_STR);
txfm = av1_ext_tx_inv[tx_set_type][packed_sym];
if (xd->counts) ++xd->counts->inter_ext_tx[eset][supertx_size][txfm];
}
}
......
......@@ -985,6 +985,8 @@ void av1_read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd,
!supertx_enabled &&
#endif // CONFIG_SUPERTX
!segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
const TxSetType tx_set_type = get_ext_tx_set_type(
tx_size, mbmi->sb_type, inter_block, cm->reduced_tx_set_used);
const int eset = get_ext_tx_set(tx_size, mbmi->sb_type, inter_block,
cm->reduced_tx_set_used);
// eset == 0 should correspond to a set with only DCT_DCT and
......@@ -993,14 +995,14 @@ void av1_read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd,
FRAME_COUNTS *counts = xd->counts;
if (inter_block) {
*tx_type = av1_ext_tx_inter_inv[eset][aom_read_symbol(
*tx_type = av1_ext_tx_inv[tx_set_type][aom_read_symbol(
r, ec_ctx->inter_ext_tx_cdf[eset][square_tx_size],
ext_tx_cnt_inter[eset], ACCT_STR)];
av1_num_ext_tx_set[tx_set_type], ACCT_STR)];
if (counts) ++counts->inter_ext_tx[eset][square_tx_size][*tx_type];
} else if (ALLOW_INTRA_EXT_TX) {
*tx_type = av1_ext_tx_intra_inv[eset][aom_read_symbol(
*tx_type = av1_ext_tx_inv[tx_set_type][aom_read_symbol(
r, ec_ctx->intra_ext_tx_cdf[eset][square_tx_size][mbmi->mode],
ext_tx_cnt_intra[eset], ACCT_STR)];
av1_num_ext_tx_set[tx_set_type], ACCT_STR)];
if (counts)
++counts->intra_ext_tx[eset][square_tx_size][mbmi->mode][*tx_type];
}
......
......@@ -55,13 +55,10 @@ static void initialize_dec(void) {
av1_indices_from_tree(av1_switchable_interp_ind, av1_switchable_interp_inv,
av1_switchable_interp_tree);
#if CONFIG_EXT_TX
int s;
for (s = 1; s < EXT_TX_SETS_INTRA; ++s)
av1_indices_from_tree(av1_ext_tx_intra_ind[s], av1_ext_tx_intra_inv[s],
av1_ext_tx_intra_tree[s]);
for (s = 1; s < EXT_TX_SETS_INTER; ++s)
av1_indices_from_tree(av1_ext_tx_inter_ind[s], av1_ext_tx_inter_inv[s],
av1_ext_tx_inter_tree[s]);
for (int s = 1; s < EXT_TX_SET_TYPES; ++s) {
av1_indices_from_tree(av1_ext_tx_ind[s], av1_ext_tx_inv[s],
av1_ext_tx_tree[s]);
}
#else
av1_indices_from_tree(av1_ext_tx_ind, av1_ext_tx_inv, av1_ext_tx_tree);
#endif
......
......@@ -121,16 +121,24 @@ static int remux_tiles(const AV1_COMMON *const cm, uint8_t *dst,
void av1_encode_token_init(void) {
#if CONFIG_EXT_TX
int s;
#endif // CONFIG_EXT_TX
#if CONFIG_EXT_TX
for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
av1_tokens_from_tree(ext_tx_inter_encodings[s], av1_ext_tx_inter_tree[s]);
av1_tokens_from_tree(ext_tx_inter_encodings[s],
av1_ext_tx_tree[av1_ext_tx_set_type_inter[s]]);
}
for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
av1_tokens_from_tree(ext_tx_intra_encodings[s], av1_ext_tx_intra_tree[s]);
av1_tokens_from_tree(ext_tx_intra_encodings[s],
av1_ext_tx_tree[av1_ext_tx_set_type_intra[s]]);
}
for (s = 1; s < EXT_TX_SET_TYPES; ++s) {
av1_indices_from_tree(av1_ext_tx_ind[s], av1_ext_tx_inv[s],
av1_ext_tx_tree[s]);
}
#else
av1_tokens_from_tree(ext_tx_encodings, av1_ext_tx_tree);
/* This hack is necessary because the four TX_TYPES are not consecutive,
e.g., 0, 1, 2, 3, when doing an in-order traversal of the av1_ext_tx_tree
structure. */
av1_indices_from_tree(av1_ext_tx_ind, av1_ext_tx_inv, av1_ext_tx_tree);
#endif // CONFIG_EXT_TX
#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP
......@@ -158,19 +166,6 @@ void av1_encode_token_init(void) {
an in-order traversal of the av1_switchable_interp_tree structure. */
av1_indices_from_tree(av1_switchable_interp_ind, av1_switchable_interp_inv,
av1_switchable_interp_tree);
/* This hack is necessary because the four TX_TYPES are not consecutive,
e.g., 0, 1, 2, 3, when doing an in-order traversal of the av1_ext_tx_tree
structure. */
#if CONFIG_EXT_TX
for (s = 1; s < EXT_TX_SETS_INTRA; ++s)
av1_indices_from_tree(av1_ext_tx_intra_ind[s], av1_ext_tx_intra_inv[s],
av1_ext_tx_intra_tree[s]);
for (s = 1; s < EXT_TX_SETS_INTER; ++s)
av1_indices_from_tree(av1_ext_tx_inter_ind[s], av1_ext_tx_inter_inv[s],
av1_ext_tx_inter_tree[s]);
#else
av1_indices_from_tree(av1_ext_tx_ind, av1_ext_tx_inv, av1_ext_tx_tree);
#endif
}
static void write_intra_mode_kf(const AV1_COMMON *cm, FRAME_CONTEXT *frame_ctx,
......@@ -1607,23 +1602,23 @@ void av1_write_tx_type(const AV1_COMMON *const cm, const MACROBLOCKD *xd,
if (tx_type == MRC_DCT)
assert(mbmi->valid_mrc_mask && "Invalid MRC mask");
#endif // CONFIG_MRC_TX
const TxSetType tx_set_type = get_ext_tx_set_type(
tx_size, bsize, is_inter, cm->reduced_tx_set_used);
const int eset =
get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used);
// eset == 0 should correspond to a set with only DCT_DCT and there
// is no need to send the tx_type
assert(eset > 0);
assert(av1_ext_tx_used[tx_set_type][tx_type]);
if (is_inter) {
assert(ext_tx_used_inter[eset][tx_type]);
aom_write_symbol(w, av1_ext_tx_inter_ind[eset][tx_type],
aom_write_symbol(w, av1_ext_tx_ind[tx_set_type][tx_type],
ec_ctx->inter_ext_tx_cdf[eset][square_tx_size],
ext_tx_cnt_inter[eset]);
av1_num_ext_tx_set[tx_set_type]);
} else if (ALLOW_INTRA_EXT_TX) {
assert(ext_tx_used_intra[eset][tx_type]);
aom_write_symbol(
w, av1_ext_tx_intra_ind[eset][tx_type],
w, av1_ext_tx_ind[tx_set_type][tx_type],
ec_ctx->intra_ext_tx_cdf[eset][square_tx_size][mbmi->mode],
ext_tx_cnt_intra[eset]);
av1_num_ext_tx_set[tx_set_type]);
}
}
#else
......@@ -3082,10 +3077,12 @@ static void write_modes_sb(AV1_COMP *const cpi, const TileInfo *const tile,
!skip) {
const int eset =
get_ext_tx_set(supertx_size, bsize, 1, cm->reduced_tx_set_used);
const int tx_set_type =
get_ext_tx_set_type(supertx_size, bsize, 1, cm->reduced_tx_set_used);
if (eset > 0) {
aom_write_symbol(w, av1_ext_tx_inter_ind[eset][mbmi->tx_type],
aom_write_symbol(w, av1_ext_tx_ind[tx_set_type][mbmi->tx_type],
ec_ctx->inter_ext_tx_cdf[eset][supertx_size],
ext_tx_cnt_inter[eset]);
av1_num_ext_tx_set[tx_set_type]);
}
}
#else
......
......@@ -6073,16 +6073,20 @@ void av1_update_tx_type_count(const AV1_COMMON *cm, MACROBLOCKD *xd,
const int eset =
get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used);
if (eset > 0) {
const TxSetType tx_set_type = get_ext_tx_set_type(
tx_size, bsize, is_inter, cm->reduced_tx_set_used);
if (is_inter) {
update_cdf(fc->inter_ext_tx_cdf[eset][txsize_sqr_map[tx_size]],
av1_ext_tx_inter_ind[eset][tx_type], ext_tx_cnt_inter[eset]);
av1_ext_tx_ind[tx_set_type][tx_type],
av1_num_ext_tx_set[tx_set_type]);
++counts->inter_ext_tx[eset][txsize_sqr_map[tx_size]][tx_type];
} else {
++counts->intra_ext_tx[eset][txsize_sqr_map[tx_size]][mbmi->mode]
[tx_type];
update_cdf(
fc->intra_ext_tx_cdf[eset][txsize_sqr_map[tx_size]][mbmi->mode],
av1_ext_tx_intra_ind[eset][tx_type], ext_tx_cnt_intra[eset]);
av1_ext_tx_ind[tx_set_type][tx_type],
av1_num_ext_tx_set[tx_set_type]);
}
}
}
......@@ -7327,9 +7331,6 @@ static void rd_supertx_sb(const AV1_COMP *const cpi, ThreadData *td,
TX_SIZE tx_size;
MB_MODE_INFO *mbmi;
TX_TYPE tx_type, best_tx_nostx;
#if CONFIG_EXT_TX
int ext_tx_set;
#endif // CONFIG_EXT_TX
int tmp_rate_tx = 0, skip_tx = 0;
int64_t tmp_dist_tx = 0, rd_tx, bestrd_tx = INT64_MAX;
......@@ -7399,7 +7400,9 @@ static void rd_supertx_sb(const AV1_COMP *const cpi, ThreadData *td,
tx_size = max_txsize_lookup[bsize];
av1_subtract_plane(x, bsize, 0);
#if CONFIG_EXT_TX
ext_tx_set = get_ext_tx_set(tx_size, bsize, 1, cm->reduced_tx_set_used);
int ext_tx_set = get_ext_tx_set(tx_size, bsize, 1, cm->reduced_tx_set_used);
const TxSetType tx_set_type =
get_ext_tx_set_type(tx_size, bsize, 1, cm->reduced_tx_set_used);
#endif // CONFIG_EXT_TX
for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
#if CONFIG_VAR_TX
......@@ -7410,7 +7413,7 @@ static void rd_supertx_sb(const AV1_COMP *const cpi, ThreadData *td,
#endif // CONFIG_VAR_TX
#if CONFIG_EXT_TX
if (!ext_tx_used_inter[ext_tx_set][tx_type]) continue;
if (!av1_ext_tx_used[tx_set_type][tx_type]) continue;
#else
if (tx_size >= TX_32X32 && tx_type != DCT_DCT) continue;
#endif // CONFIG_EXT_TX
......
......@@ -2044,12 +2044,11 @@ int64_t av1_search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
}
#if CONFIG_EXT_TX
int is_inter = is_inter_block(mbmi);
int ext_tx_set = get_ext_tx_set(get_min_tx_size(tx_size), mbmi->sb_type,
is_inter, cm->reduced_tx_set_used);
if (!(is_inter && ext_tx_used_inter[ext_tx_set][tx_type]) &&
!(!is_inter && ext_tx_used_intra[ext_tx_set][tx_type]))
continue;
const int is_inter = is_inter_block(mbmi);
const TxSetType tx_set_type =
get_ext_tx_set_type(get_min_tx_size(tx_size), mbmi->sb_type, is_inter,
cm->reduced_tx_set_used);
if (!av1_ext_tx_used[tx_set_type][tx_type]) continue;
#endif // CONFIG_EXT_TX
RD_STATS this_rd_stats;
......
......@@ -179,15 +179,16 @@ void av1_fill_mode_rates(AV1_COMMON *const cm, MACROBLOCK *x,
if (use_inter_ext_tx_for_txsize[s][i]) {
av1_cost_tokens_from_cdf(x->inter_tx_type_costs[s][i],
fc->inter_ext_tx_cdf[s][i],
av1_ext_tx_inter_inv[s]);
av1_ext_tx_inv[av1_ext_tx_set_type_inter[s]]);
}
}
for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
if (use_intra_ext_tx_for_txsize[s][i]) {
for (j = 0; j < INTRA_MODES; ++j)
av1_cost_tokens_from_cdf(x->intra_tx_type_costs[s][i][j],
fc->intra_ext_tx_cdf[s][i][j],
av1_ext_tx_intra_inv[s]);
for (j = 0; j < INTRA_MODES; ++j) {
av1_cost_tokens_from_cdf(
x->intra_tx_type_costs[s][i][j], fc->intra_ext_tx_cdf[s][i][j],
av1_ext_tx_inv[av1_ext_tx_set_type_intra[s]]);
}
}
}
}
......
......@@ -2311,7 +2311,7 @@ static int tx_size_cost(const AV1_COMP *const cpi, const MACROBLOCK *const x,
}
}
// #TODO(angiebird): use this function whenever it's possible
// TODO(angiebird): use this function whenever it's possible
int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x,
const MACROBLOCKD *xd, BLOCK_SIZE bsize, int plane,
TX_SIZE tx_size, TX_TYPE tx_type) {
......@@ -2442,10 +2442,10 @@ static int skip_txfm_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bs,
if (max_tx_size >= TX_32X32 && tx_size == TX_4X4) return 1;
#if CONFIG_EXT_TX
const AV1_COMMON *const cm = &cpi->common;
int ext_tx_set =
get_ext_tx_set(tx_size, bs, is_inter, cm->reduced_tx_set_used);
const TxSetType tx_set_type =
get_ext_tx_set_type(tx_size, bs, is_inter, cm->reduced_tx_set_used);
if (!av1_ext_tx_used[tx_set_type][tx_type]) return 1;
if (is_inter) {
if (!ext_tx_used_inter[ext_tx_set][tx_type]) return 1;
if (cpi->sf.tx_type_search.prune_mode > NO_PRUNE) {
if (!do_tx_type_search(tx_type, prune)) return 1;
}
......@@ -2453,7 +2453,6 @@ static int skip_txfm_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bs,
if (!ALLOW_INTRA_EXT_TX && bs >= BLOCK_8X8) {
if (tx_type != intra_mode_to_tx_type_context[mbmi->mode]) return 1;
}
if (!ext_tx_used_intra[ext_tx_set][tx_type]) return 1;
}
#else // CONFIG_EXT_TX
if (tx_size >= TX_32X32 && tx_type != DCT_DCT) return 1;
......@@ -2494,9 +2493,6 @@ static void choose_largest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x,
const int is_inter = is_inter_block(mbmi);
int prune = 0;
const int plane = 0;
#if CONFIG_EXT_TX
int ext_tx_set;
#endif // CONFIG_EXT_TX
av1_invalid_rd_stats(rd_stats);
mbmi->tx_size = tx_size_from_tx_mode(bs, cm->tx_mode, is_inter);
......@@ -2504,8 +2500,10 @@ static void choose_largest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x,
mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
#endif // CONFIG_VAR_TX
#if CONFIG_EXT_TX
ext_tx_set =
int ext_tx_set =
get_ext_tx_set(mbmi->tx_size, bs, is_inter, cm->reduced_tx_set_used);
const TxSetType tx_set_type =
get_ext_tx_set_type(mbmi->tx_size, bs, is_inter, cm->reduced_tx_set_used);
#endif // CONFIG_EXT_TX
if (is_inter && cpi->sf.tx_type_search.prune_mode > NO_PRUNE)
......@@ -2526,12 +2524,12 @@ static void choose_largest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x,
#endif // CONFIG_PVQ
for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
if (!av1_ext_tx_used[tx_set_type][tx_type]) continue;
RD_STATS this_rd_stats;
if (is_inter) {
if (x->use_default_inter_tx_type &&
tx_type != get_default_tx_type(0, xd, 0, mbmi->tx_size))
continue;
if (!ext_tx_used_inter[ext_tx_set][tx_type]) continue;
if (cpi->sf.tx_type_search.prune_mode > NO_PRUNE) {
if (!do_tx_type_search(tx_type, prune)) continue;
}
......@@ -2542,7 +2540,6 @@ static void choose_largest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x,
if (!ALLOW_INTRA_EXT_TX && bs >= BLOCK_8X8) {
if (tx_type != intra_mode_to_tx_type_context[mbmi->mode]) continue;
}
if (!ext_tx_used_intra[ext_tx_set][tx_type]) continue;
}
mbmi->tx_type = tx_type;
......@@ -2696,10 +2693,9 @@ static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi,
if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) continue;
const TX_SIZE rect_tx_size = max_txsize_rect_lookup[bs];
RD_STATS this_rd_stats;
int ext_tx_set =
get_ext_tx_set(rect_tx_size, bs, is_inter, cm->reduced_tx_set_used);
if ((is_inter && ext_tx_used_inter[ext_tx_set][tx_type]) ||
(!is_inter && ext_tx_used_intra[ext_tx_set][tx_type])) {
const TxSetType tx_set_type = get_ext_tx_set_type(
rect_tx_size, bs, is_inter, cm->reduced_tx_set_used);
if (av1_ext_tx_used[tx_set_type][tx_type]) {
rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type,
rect_tx_size);
ref_best_rd = AOMMIN(rd, ref_best_rd);
......@@ -2745,10 +2741,9 @@ static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi,
if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) continue;
const TX_SIZE tx_size = quarter_txsize_lookup[bs];
RD_STATS this_rd_stats;
int ext_tx_set =
get_ext_tx_set(tx_size, bs, is_inter, cm->reduced_tx_set_used);
if ((is_inter && ext_tx_used_inter[ext_tx_set][tx_type]) ||
(!is_inter && ext_tx_used_intra[ext_tx_set][tx_type])) {
const TxSetType tx_set_type =
get_ext_tx_set_type(tx_size, bs, is_inter, cm->reduced_tx_set_used);
if (av1_ext_tx_used[tx_set_type][tx_type]) {
rd =
txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type, tx_size);
if (rd < best_rd) {
......@@ -5271,6 +5266,8 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
RD_STATS rd_stats_stack[4];
#endif // CONFIG_EXT_PARTITION
#if CONFIG_EXT_TX
const TxSetType tx_set_type = get_ext_tx_set_type(
max_tx_size, bsize, is_inter, cm->reduced_tx_set_used);
const int ext_tx_set =
get_ext_tx_set(max_tx_size, bsize, is_inter, cm->reduced_tx_set_used);
#endif // CONFIG_EXT_TX
......@@ -5315,8 +5312,8 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
continue;
#endif // CONFIG_MRC_TX
#if CONFIG_EXT_TX
if (!av1_ext_tx_used[tx_set_type][tx_type]) continue;
if (is_inter) {
if (!ext_tx_used_inter[ext_tx_set][tx_type]) continue;
if (cpi->sf.tx_type_search.prune_mode > NO_PRUNE) {
if (!do_tx_type_search(tx_type, prune)) continue;
}
......@@ -5324,7 +5321,6 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
if (!ALLOW_INTRA_EXT_TX && bsize >= BLOCK_8X8) {
if (tx_type != intra_mode_to_tx_type_context[mbmi->mode]) continue;
}
if (!ext_tx_used_intra[ext_tx_set][tx_type]) continue;
}
#else // CONFIG_EXT_TX
if (is_inter && cpi->sf.tx_type_search.prune_mode > NO_PRUNE &&
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment