Commit da2d4a7a authored by Debargha Mukherjee's avatar Debargha Mukherjee

Hooks to use 32x32 masked transforms for ext-tx

Adds hooks to use 32x32 ext-tx. Also adds scan orders for the masked
transforms for 32x32.
Make macro USE_MSKTX_FOR_32X32 1 in blockd.h to support 32x32 masked
transforms for ext-tx.

Change-Id: Ie6564830266651fcafae2d536c274dafd664ce17
parent aa703adb
......@@ -351,7 +351,9 @@ static INLINE int supertx_enabled(const MB_MODE_INFO *mbmi) {
#endif // CONFIG_SUPERTX
#if CONFIG_EXT_TX
#define ALLOW_INTRA_EXT_TX 1
#define ALLOW_INTRA_EXT_TX 1
// whether masked transforms are used for 32X32
#define USE_MSKTX_FOR_32X32 1
static const int num_ext_tx_set_inter[EXT_TX_SETS_INTER] = {
1, 17, 10, 2
......@@ -360,25 +362,38 @@ static const int num_ext_tx_set_intra[EXT_TX_SETS_INTRA] = {
1, 17, 10
};
#define USE_IDTX_FOR_32X32 0
#if EXT_TX_SIZES == 4
static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs,
int is_inter) {
if (tx_size > TX_32X32 || bs < BLOCK_8X8) return 0;
if (tx_size == TX_32X32)
return is_inter ? 3 - 2 * USE_MSKTX_FOR_32X32 : 0;
return ((is_inter || tx_size < TX_16X16) ? 1 : 2);
}
static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][TX_SIZES] = {
{ 0, 0, 0, 0, }, // unused
{ 1, 1, 0, 0, },
{ 0, 0, 1, 0, },
};
static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][TX_SIZES] = {
{ 0, 0, 0, 0, }, // unused
{ 1, 1, 1, USE_MSKTX_FOR_32X32, },
{ 0, 0, 0, 0, },
{ 0, 0, 0, (!USE_MSKTX_FOR_32X32), },
};
#else // EXT_TX_SIZES == 4
static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs,
int is_inter) {
(void) is_inter;
if (tx_size > TX_32X32 || bs < BLOCK_8X8) return 0;
#if USE_IDTX_FOR_32X32
if (tx_size == TX_32X32) return is_inter ? 3 : 0;
#else
if (tx_size == TX_32X32) return 0;
#endif
return tx_size == TX_16X16 ? 2 : 1;
}
static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs,
int is_inter) {
const int set = get_ext_tx_set(tx_size, bs, is_inter);
return is_inter ? num_ext_tx_set_inter[set] : num_ext_tx_set_intra[set];
}
static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][TX_SIZES] = {
{ 0, 0, 0, 0, }, // unused
{ 1, 1, 0, 0, },
......@@ -389,8 +404,9 @@ static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][TX_SIZES] = {
{ 0, 0, 0, 0, }, // unused
{ 1, 1, 0, 0, },
{ 0, 0, 1, 0, },
{ 0, 0, 0, USE_IDTX_FOR_32X32, },
{ 0, 0, 0, 0, },
};
#endif // EXT_TX_SIZES == 4
// Transform types used in each intra set
static const int ext_tx_used_intra[EXT_TX_SETS_INTRA][TX_TYPES] = {
......@@ -406,6 +422,12 @@ static const int ext_tx_used_inter[EXT_TX_SETS_INTER][TX_TYPES] = {
{ 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, },
{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, },
};
static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs,
int is_inter) {
const int set = get_ext_tx_set(tx_size, bs, is_inter);
return is_inter ? num_ext_tx_set_inter[set] : num_ext_tx_set_intra[set];
}
#endif // CONFIG_EXT_TX
#if CONFIG_EXT_INTRA
......@@ -504,7 +526,7 @@ static INLINE TX_TYPE get_tx_type(PLANE_TYPE plane_type,
#endif // CONFIG_EXT_INTRA
#if CONFIG_EXT_TX
#if USE_IDTX_FOR_32X32
#if EXT_TX_SIZES == 4
if (xd->lossless[mbmi->segment_id] || tx_size > TX_32X32 ||
(tx_size >= TX_32X32 && !is_inter_block(mbmi)))
#else
......
......@@ -951,7 +951,7 @@ default_inter_ext_tx_prob[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES - 1] = {
{ 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
128 },
#if EXT_TX_SIZES == 4
{ 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
{ 12, 160, 16, 144, 160, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
128 },
#endif
}, {
......@@ -959,7 +959,7 @@ default_inter_ext_tx_prob[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES - 1] = {
{ 12, 112, 128, 128, 128, 128, 128, 128, 128 },
{ 12, 112, 128, 128, 128, 128, 128, 128, 128 },
#if EXT_TX_SIZES == 4
{ 12, 112, 128, 128, 128, 128, 128, 128, 128 },
{ 12, 160, 128, 128, 128, 128, 128, 128, 128 },
#endif
}, {
{ 12, },
......@@ -1240,7 +1240,9 @@ default_intra_ext_tx_prob[EXT_TX_SETS_INTRA][EXT_TX_SIZES]
},
},
};
#else
const vpx_tree_index vp10_ext_tx_tree[TREE_SIZE(TX_TYPES)] = {
-DCT_DCT, 2,
-ADST_ADST, 4,
......
......@@ -112,11 +112,13 @@ typedef enum {
TX_TYPES,
} TX_TYPE;
#define EXT_TX_SIZES 3 // number of sizes that use extended transforms
#if CONFIG_EXT_TX
#define EXT_TX_SIZES 4 // number of sizes that use extended transforms
#define EXT_TX_SETS_INTER 4 // Sets of transform selections for INTER
#define EXT_TX_SETS_INTRA 3 // Sets of transform selections for INTRA
#else
#define EXT_TX_SIZES 3 // number of sizes that use extended transforms
#endif // CONFIG_EXT_TX
typedef enum {
......
This diff is collapsed.
......@@ -181,15 +181,27 @@ static void fwd_txfm_32x32(int rd_transform, const int16_t *src_diff,
vpx_fdct32x32_1(src_diff, coeff, diff_stride);
break;
#if CONFIG_EXT_TX
case IDTX:
fwd_idtx_c(src_diff, coeff, diff_stride, 32);
break;
#endif // CONFIG_EXT_TX
case ADST_DCT:
case DCT_ADST:
case ADST_ADST:
assert(0);
case FLIPADST_DCT:
case DCT_FLIPADST:
case FLIPADST_FLIPADST:
case ADST_FLIPADST:
case FLIPADST_ADST:
case DST_DST:
case DCT_DST:
case DST_DCT:
case DST_ADST:
case ADST_DST:
case DST_FLIPADST:
case FLIPADST_DST:
vp10_fht32x32_c(src_diff, coeff, diff_stride, tx_type);
break;
case IDTX:
fwd_idtx_c(src_diff, coeff, diff_stride, 32);
break;
#endif // CONFIG_EXT_TX
default:
assert(0);
break;
......@@ -335,15 +347,27 @@ static void highbd_fwd_txfm_32x32(int rd_transform, const int16_t *src_diff,
vpx_highbd_fdct32x32_1(src_diff, coeff, diff_stride);
break;
#if CONFIG_EXT_TX
case IDTX:
fwd_idtx_c(src_diff, coeff, diff_stride, 32);
break;
#endif // CONFIG_EXT_TX
case ADST_DCT:
case DCT_ADST:
case ADST_ADST:
assert(0);
case FLIPADST_DCT:
case DCT_FLIPADST:
case FLIPADST_FLIPADST:
case ADST_FLIPADST:
case FLIPADST_ADST:
case DST_DST:
case DCT_DST:
case DST_DCT:
case DST_ADST:
case ADST_DST:
case DST_FLIPADST:
case FLIPADST_DST:
vp10_highbd_fht32x32_c(src_diff, coeff, diff_stride, tx_type);
break;
case IDTX:
fwd_idtx_c(src_diff, coeff, diff_stride, 32);
break;
#endif // CONFIG_EXT_TX
default:
assert(0);
break;
......
......@@ -82,7 +82,7 @@
#define NEW_MV_DISCOUNT_FACTOR 8
#if CONFIG_EXT_TX
const double ext_tx_th = 0.98;
const double ext_tx_th = 0.99;
#else
const double ext_tx_th = 0.99;
#endif
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment