Commit 08542b96 authored by Debargha Mukherjee's avatar Debargha Mukherjee

Reduce transform set for 16x32/32x16

Previously the transform type sets used for 16x32/32x16 transform
sizes corresponded to 16x16 transforms. Now they correspond to
32x32 transforms with a reduced available set.

BDRATE impact:
0.05% worse for lowres, 0.13% worse for midres

Change-Id: I01d405d497c13837f5404f9e0caf8c571f3b26cf
parent f24cea31
...@@ -596,21 +596,72 @@ static INLINE int supertx_enabled(const MB_MODE_INFO *mbmi) { ...@@ -596,21 +596,72 @@ static INLINE int supertx_enabled(const MB_MODE_INFO *mbmi) {
#define ALLOW_INTRA_EXT_TX 1 #define ALLOW_INTRA_EXT_TX 1
static const int num_ext_tx_set_inter[EXT_TX_SETS_INTER] = { 1, 16, 12, 2 }; typedef enum {
static const int num_ext_tx_set_intra[EXT_TX_SETS_INTRA] = { 1, 7, 5 }; // DCT only
EXT_TX_SET_DCTONLY = 0,
// DCT + Identity only
EXT_TX_SET_DCT_IDTX = 1,
// Discrete Trig transforms w/o flip (4) + Identity (1)
EXT_TX_SET_DTT4_IDTX = 2,
// Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2)
EXT_TX_SET_DTT4_IDTX_1DDCT = 3,
// Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver DCT (2)
EXT_TX_SET_DTT9_IDTX_1DDCT = 4,
// Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6)
EXT_TX_SET_ALL16 = 5,
EXT_TX_SET_TYPES
} TxSetType;
// Number of transform types in each set type
static const int num_ext_tx_set[EXT_TX_SET_TYPES] = { 1, 2, 5, 7, 12, 16 };
// Maps intra set index to the set type
static const int ext_tx_set_type_intra[EXT_TX_SETS_INTRA] = {
EXT_TX_SET_DCTONLY, EXT_TX_SET_DTT4_IDTX_1DDCT, EXT_TX_SET_DTT4_IDTX
};
static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter, // Maps inter set index to the set type
int use_default) { static const int ext_tx_set_type_inter[EXT_TX_SETS_INTER] = {
if (use_default) return is_inter ? 3 : 2; EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16, EXT_TX_SET_DTT9_IDTX_1DDCT,
EXT_TX_SET_DCT_IDTX
};
// Maps set types above to the indices used for intra
static const int ext_tx_set_index_intra[EXT_TX_SET_TYPES] = { 0, -1, 2,
1, -1, -1 };
// Maps set types above to the indices used for inter
static const int ext_tx_set_index_inter[EXT_TX_SET_TYPES] = {
0, 3, -1, -1, 2, 1
};
static INLINE TxSetType get_ext_tx_set_type(TX_SIZE tx_size, BLOCK_SIZE bs,
int is_inter, int use_default) {
const TX_SIZE tx_size2 = txsize_sqr_up_map[tx_size];
if (use_default) return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DTT4_IDTX;
tx_size = txsize_sqr_map[tx_size]; tx_size = txsize_sqr_map[tx_size];
#if CONFIG_CB4X4 #if CONFIG_CB4X4
(void)bs; (void)bs;
if (tx_size > TX_32X32) return 0; if (tx_size > TX_32X32) return EXT_TX_SET_DCTONLY;
#else #else
if (tx_size > TX_32X32 || bs < BLOCK_8X8) return 0; if (tx_size > TX_32X32 || bs < BLOCK_8X8) return EXT_TX_SET_DCTONLY;
#endif #endif
if (tx_size == TX_32X32) return is_inter ? 3 : 0; if (tx_size2 == TX_32X32)
return (tx_size == TX_16X16 ? 2 : 1); return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DCTONLY;
if (is_inter)
return (tx_size == TX_16X16 ? EXT_TX_SET_DTT9_IDTX_1DDCT
: EXT_TX_SET_ALL16);
else
return (tx_size == TX_16X16 ? EXT_TX_SET_DTT4_IDTX
: EXT_TX_SET_DTT4_IDTX_1DDCT);
}
static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
int use_default) {
const TxSetType set_type =
get_ext_tx_set_type(tx_size, bs, is_inter, use_default);
return is_inter ? ext_tx_set_index_inter[set_type]
: ext_tx_set_index_intra[set_type];
} }
static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] = static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] =
...@@ -664,8 +715,8 @@ static const int ext_tx_used_inter_1D[EXT_TX_SETS_INTER][TX_TYPES_1D] = { ...@@ -664,8 +715,8 @@ static const int ext_tx_used_inter_1D[EXT_TX_SETS_INTER][TX_TYPES_1D] = {
static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter, static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
int use_default) { int use_default) {
const int set = get_ext_tx_set(tx_size, bs, is_inter, use_default); const int set_type = get_ext_tx_set_type(tx_size, bs, is_inter, use_default);
return is_inter ? num_ext_tx_set_inter[set] : num_ext_tx_set_intra[set]; return num_ext_tx_set[set_type];
} }
#if CONFIG_RECT_TX #if CONFIG_RECT_TX
......
...@@ -4404,7 +4404,7 @@ static void read_ext_tx_probs(FRAME_CONTEXT *fc, aom_reader *r) { ...@@ -4404,7 +4404,7 @@ static void read_ext_tx_probs(FRAME_CONTEXT *fc, aom_reader *r) {
if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) { if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) {
for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
if (!use_inter_ext_tx_for_txsize[s][i]) continue; if (!use_inter_ext_tx_for_txsize[s][i]) continue;
for (j = 0; j < num_ext_tx_set_inter[s] - 1; ++j) for (j = 0; j < num_ext_tx_set[ext_tx_set_type_inter[s]] - 1; ++j)
av1_diff_update_prob(r, &fc->inter_ext_tx_prob[s][i][j], ACCT_STR); av1_diff_update_prob(r, &fc->inter_ext_tx_prob[s][i][j], ACCT_STR);
} }
} }
...@@ -4415,7 +4415,7 @@ static void read_ext_tx_probs(FRAME_CONTEXT *fc, aom_reader *r) { ...@@ -4415,7 +4415,7 @@ static void read_ext_tx_probs(FRAME_CONTEXT *fc, aom_reader *r) {
for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
if (!use_intra_ext_tx_for_txsize[s][i]) continue; if (!use_intra_ext_tx_for_txsize[s][i]) continue;
for (j = 0; j < INTRA_MODES; ++j) for (j = 0; j < INTRA_MODES; ++j)
for (k = 0; k < num_ext_tx_set_intra[s] - 1; ++k) for (k = 0; k < num_ext_tx_set[ext_tx_set_type_intra[s]] - 1; ++k)
av1_diff_update_prob(r, &fc->intra_ext_tx_prob[s][i][j][k], av1_diff_update_prob(r, &fc->intra_ext_tx_prob[s][i][j][k],
ACCT_STR); ACCT_STR);
} }
......
...@@ -644,16 +644,18 @@ static void update_ext_tx_probs(AV1_COMMON *cm, aom_writer *w) { ...@@ -644,16 +644,18 @@ static void update_ext_tx_probs(AV1_COMMON *cm, aom_writer *w) {
if (!use_inter_ext_tx_for_txsize[s][i]) continue; if (!use_inter_ext_tx_for_txsize[s][i]) continue;
savings += prob_diff_update_savings( savings += prob_diff_update_savings(
av1_ext_tx_inter_tree[s], cm->fc->inter_ext_tx_prob[s][i], av1_ext_tx_inter_tree[s], cm->fc->inter_ext_tx_prob[s][i],
cm->counts.inter_ext_tx[s][i], num_ext_tx_set_inter[s], probwt); cm->counts.inter_ext_tx[s][i],
num_ext_tx_set[ext_tx_set_type_inter[s]], probwt);
} }
do_update = savings > savings_thresh; do_update = savings > savings_thresh;
aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB); aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
if (do_update) { if (do_update) {
for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
if (!use_inter_ext_tx_for_txsize[s][i]) continue; if (!use_inter_ext_tx_for_txsize[s][i]) continue;
prob_diff_update( prob_diff_update(av1_ext_tx_inter_tree[s],
av1_ext_tx_inter_tree[s], cm->fc->inter_ext_tx_prob[s][i], cm->fc->inter_ext_tx_prob[s][i],
cm->counts.inter_ext_tx[s][i], num_ext_tx_set_inter[s], probwt, w); cm->counts.inter_ext_tx[s][i],
num_ext_tx_set[ext_tx_set_type_inter[s]], probwt, w);
} }
} }
} }
...@@ -666,7 +668,8 @@ static void update_ext_tx_probs(AV1_COMMON *cm, aom_writer *w) { ...@@ -666,7 +668,8 @@ static void update_ext_tx_probs(AV1_COMMON *cm, aom_writer *w) {
for (j = 0; j < INTRA_MODES; ++j) for (j = 0; j < INTRA_MODES; ++j)
savings += prob_diff_update_savings( savings += prob_diff_update_savings(
av1_ext_tx_intra_tree[s], cm->fc->intra_ext_tx_prob[s][i][j], av1_ext_tx_intra_tree[s], cm->fc->intra_ext_tx_prob[s][i][j],
cm->counts.intra_ext_tx[s][i][j], num_ext_tx_set_intra[s], probwt); cm->counts.intra_ext_tx[s][i][j],
num_ext_tx_set[ext_tx_set_type_intra[s]], probwt);
} }
do_update = savings > savings_thresh; do_update = savings > savings_thresh;
aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB); aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
...@@ -677,7 +680,7 @@ static void update_ext_tx_probs(AV1_COMMON *cm, aom_writer *w) { ...@@ -677,7 +680,7 @@ static void update_ext_tx_probs(AV1_COMMON *cm, aom_writer *w) {
prob_diff_update(av1_ext_tx_intra_tree[s], prob_diff_update(av1_ext_tx_intra_tree[s],
cm->fc->intra_ext_tx_prob[s][i][j], cm->fc->intra_ext_tx_prob[s][i][j],
cm->counts.intra_ext_tx[s][i][j], cm->counts.intra_ext_tx[s][i][j],
num_ext_tx_set_intra[s], probwt, w); num_ext_tx_set[ext_tx_set_type_intra[s]], probwt, w);
} }
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment