Commit 08542b96 authored by Debargha Mukherjee's avatar Debargha Mukherjee

Reduce transform set for 16x32/32x16

Previously the transform type sets used for 16x32/32x16 transform
sizes corresponded to 16x16 transforms. Now they correspond to
32x32 transforms with a reduced available set.

BDRATE impact:
0.05% worse for lowres, 0.13% worse for midres

Change-Id: I01d405d497c13837f5404f9e0caf8c571f3b26cf
parent f24cea31
......@@ -596,21 +596,72 @@ static INLINE int supertx_enabled(const MB_MODE_INFO *mbmi) {
#define ALLOW_INTRA_EXT_TX 1
static const int num_ext_tx_set_inter[EXT_TX_SETS_INTER] = { 1, 16, 12, 2 };
static const int num_ext_tx_set_intra[EXT_TX_SETS_INTRA] = { 1, 7, 5 };
typedef enum {
// DCT only
EXT_TX_SET_DCTONLY = 0,
// DCT + Identity only
EXT_TX_SET_DCT_IDTX = 1,
// Discrete Trig transforms w/o flip (4) + Identity (1)
EXT_TX_SET_DTT4_IDTX = 2,
// Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2)
EXT_TX_SET_DTT4_IDTX_1DDCT = 3,
// Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver DCT (2)
EXT_TX_SET_DTT9_IDTX_1DDCT = 4,
// Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6)
EXT_TX_SET_ALL16 = 5,
EXT_TX_SET_TYPES
} TxSetType;
// Number of transform types in each set type
static const int num_ext_tx_set[EXT_TX_SET_TYPES] = { 1, 2, 5, 7, 12, 16 };
// Maps intra set index to the set type
static const int ext_tx_set_type_intra[EXT_TX_SETS_INTRA] = {
EXT_TX_SET_DCTONLY, EXT_TX_SET_DTT4_IDTX_1DDCT, EXT_TX_SET_DTT4_IDTX
};
static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
int use_default) {
if (use_default) return is_inter ? 3 : 2;
// Maps inter set index to the set type
static const int ext_tx_set_type_inter[EXT_TX_SETS_INTER] = {
EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16, EXT_TX_SET_DTT9_IDTX_1DDCT,
EXT_TX_SET_DCT_IDTX
};
// Maps set types above to the indices used for intra
static const int ext_tx_set_index_intra[EXT_TX_SET_TYPES] = { 0, -1, 2,
1, -1, -1 };
// Maps set types above to the indices used for inter
static const int ext_tx_set_index_inter[EXT_TX_SET_TYPES] = {
0, 3, -1, -1, 2, 1
};
static INLINE TxSetType get_ext_tx_set_type(TX_SIZE tx_size, BLOCK_SIZE bs,
int is_inter, int use_default) {
const TX_SIZE tx_size2 = txsize_sqr_up_map[tx_size];
if (use_default) return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DTT4_IDTX;
tx_size = txsize_sqr_map[tx_size];
#if CONFIG_CB4X4
(void)bs;
if (tx_size > TX_32X32) return 0;
if (tx_size > TX_32X32) return EXT_TX_SET_DCTONLY;
#else
if (tx_size > TX_32X32 || bs < BLOCK_8X8) return 0;
if (tx_size > TX_32X32 || bs < BLOCK_8X8) return EXT_TX_SET_DCTONLY;
#endif
if (tx_size == TX_32X32) return is_inter ? 3 : 0;
return (tx_size == TX_16X16 ? 2 : 1);
if (tx_size2 == TX_32X32)
return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DCTONLY;
if (is_inter)
return (tx_size == TX_16X16 ? EXT_TX_SET_DTT9_IDTX_1DDCT
: EXT_TX_SET_ALL16);
else
return (tx_size == TX_16X16 ? EXT_TX_SET_DTT4_IDTX
: EXT_TX_SET_DTT4_IDTX_1DDCT);
}
static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
int use_default) {
const TxSetType set_type =
get_ext_tx_set_type(tx_size, bs, is_inter, use_default);
return is_inter ? ext_tx_set_index_inter[set_type]
: ext_tx_set_index_intra[set_type];
}
static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] =
......@@ -664,8 +715,8 @@ static const int ext_tx_used_inter_1D[EXT_TX_SETS_INTER][TX_TYPES_1D] = {
static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
int use_default) {
const int set = get_ext_tx_set(tx_size, bs, is_inter, use_default);
return is_inter ? num_ext_tx_set_inter[set] : num_ext_tx_set_intra[set];
const int set_type = get_ext_tx_set_type(tx_size, bs, is_inter, use_default);
return num_ext_tx_set[set_type];
}
#if CONFIG_RECT_TX
......
......@@ -4404,7 +4404,7 @@ static void read_ext_tx_probs(FRAME_CONTEXT *fc, aom_reader *r) {
if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) {
for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
if (!use_inter_ext_tx_for_txsize[s][i]) continue;
for (j = 0; j < num_ext_tx_set_inter[s] - 1; ++j)
for (j = 0; j < num_ext_tx_set[ext_tx_set_type_inter[s]] - 1; ++j)
av1_diff_update_prob(r, &fc->inter_ext_tx_prob[s][i][j], ACCT_STR);
}
}
......@@ -4415,7 +4415,7 @@ static void read_ext_tx_probs(FRAME_CONTEXT *fc, aom_reader *r) {
for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
if (!use_intra_ext_tx_for_txsize[s][i]) continue;
for (j = 0; j < INTRA_MODES; ++j)
for (k = 0; k < num_ext_tx_set_intra[s] - 1; ++k)
for (k = 0; k < num_ext_tx_set[ext_tx_set_type_intra[s]] - 1; ++k)
av1_diff_update_prob(r, &fc->intra_ext_tx_prob[s][i][j][k],
ACCT_STR);
}
......
......@@ -644,16 +644,18 @@ static void update_ext_tx_probs(AV1_COMMON *cm, aom_writer *w) {
if (!use_inter_ext_tx_for_txsize[s][i]) continue;
savings += prob_diff_update_savings(
av1_ext_tx_inter_tree[s], cm->fc->inter_ext_tx_prob[s][i],
cm->counts.inter_ext_tx[s][i], num_ext_tx_set_inter[s], probwt);
cm->counts.inter_ext_tx[s][i],
num_ext_tx_set[ext_tx_set_type_inter[s]], probwt);
}
do_update = savings > savings_thresh;
aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
if (do_update) {
for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
if (!use_inter_ext_tx_for_txsize[s][i]) continue;
prob_diff_update(
av1_ext_tx_inter_tree[s], cm->fc->inter_ext_tx_prob[s][i],
cm->counts.inter_ext_tx[s][i], num_ext_tx_set_inter[s], probwt, w);
prob_diff_update(av1_ext_tx_inter_tree[s],
cm->fc->inter_ext_tx_prob[s][i],
cm->counts.inter_ext_tx[s][i],
num_ext_tx_set[ext_tx_set_type_inter[s]], probwt, w);
}
}
}
......@@ -666,7 +668,8 @@ static void update_ext_tx_probs(AV1_COMMON *cm, aom_writer *w) {
for (j = 0; j < INTRA_MODES; ++j)
savings += prob_diff_update_savings(
av1_ext_tx_intra_tree[s], cm->fc->intra_ext_tx_prob[s][i][j],
cm->counts.intra_ext_tx[s][i][j], num_ext_tx_set_intra[s], probwt);
cm->counts.intra_ext_tx[s][i][j],
num_ext_tx_set[ext_tx_set_type_intra[s]], probwt);
}
do_update = savings > savings_thresh;
aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
......@@ -677,7 +680,7 @@ static void update_ext_tx_probs(AV1_COMMON *cm, aom_writer *w) {
prob_diff_update(av1_ext_tx_intra_tree[s],
cm->fc->intra_ext_tx_prob[s][i][j],
cm->counts.intra_ext_tx[s][i][j],
num_ext_tx_set_intra[s], probwt, w);
num_ext_tx_set[ext_tx_set_type_intra[s]], probwt, w);
}
}
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment