Commit 5a488a62 authored by Debargha Mukherjee's avatar Debargha Mukherjee

Some fixes related to 64x64 tx, ext-tx, supertx

Change-Id: I95455ed7e44f3a163c751bd8a89f6119c0055c71
parent 838a53d6
......@@ -512,6 +512,7 @@ static INLINE int supertx_enabled(const MB_MODE_INFO *mbmi) {
#endif // CONFIG_SUPERTX
#if CONFIG_EXT_TX
#define ALLOW_INTRA_EXT_TX 1
static const int num_ext_tx_set_inter[EXT_TX_SETS_INTER] = { 1, 16, 12, 2 };
......@@ -524,18 +525,20 @@ static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter) {
return (tx_size == TX_16X16 ? 2 : 1);
}
static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][TX_SIZES] = {
{ 0, 0, 0, 0 }, // unused
{ 1, 1, 0, 0 },
{ 0, 0, 1, 0 },
};
static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA]
[EXT_TX_SIZES] = {
{ 1, 1, 1, 1 }, // unused
{ 1, 1, 0, 0 },
{ 0, 0, 1, 0 },
};
static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][TX_SIZES] = {
{ 0, 0, 0, 0 }, // unused
{ 1, 1, 0, 0 },
{ 0, 0, 1, 0 },
{ 0, 0, 0, 1 },
};
static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER]
[EXT_TX_SIZES] = {
{ 1, 1, 1, 1 }, // unused
{ 1, 1, 0, 0 },
{ 0, 0, 1, 0 },
{ 0, 0, 0, 1 },
};
// Transform types used in each intra set
static const int ext_tx_used_intra[EXT_TX_SETS_INTRA][TX_TYPES] = {
......@@ -757,7 +760,7 @@ static INLINE TX_TYPE get_tx_type(PLANE_TYPE plane_type, const MACROBLOCKD *xd,
}
if (is_inter_block(mbmi))
// UV Inter only
return (mbmi->tx_type == IDTX && txsize_sqr_map[tx_size] == TX_32X32)
return (mbmi->tx_type == IDTX && txsize_sqr_map[tx_size] >= TX_32X32)
? DCT_DCT
: mbmi->tx_type;
}
......
......@@ -1438,8 +1438,13 @@ static const aom_prob default_filter_intra_probs[2] = { 230, 230 };
#if CONFIG_SUPERTX
static const aom_prob default_supertx_prob[PARTITION_SUPERTX_CONTEXTS]
[TX_SIZES] = {
#if CONFIG_TX64X64
{ 1, 160, 160, 170, 180 },
{ 1, 200, 200, 210, 220 },
#else
{ 1, 160, 160, 170 },
{ 1, 200, 200, 210 },
#endif // CONFIG_TX64X64
};
#endif // CONFIG_SUPERTX
......
......@@ -901,7 +901,7 @@ void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
case BLOCK_64X64: build_masks(lfi_n, mip[0], 0, 0, lfm); break;
case BLOCK_64X32: build_masks(lfi_n, mip[0], 0, 0, lfm);
#if CONFIG_SUPERTX && CONFIG_TX64X64
if (supertx_enabled(&mip->mbmi)) break;
if (supertx_enabled(&mip[0]->mbmi)) break;
#endif // CONFIG_SUPERTX && CONFIG_TX64X64
mip2 = mip + mode_info_stride * 4;
if (4 >= max_rows) break;
......@@ -909,7 +909,7 @@ void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
break;
case BLOCK_32X64: build_masks(lfi_n, mip[0], 0, 0, lfm);
#if CONFIG_SUPERTX && CONFIG_TX64X64
if (supertx_enabled(&mip->mbmi)) break;
if (supertx_enabled(&mip[0]->mbmi)) break;
#endif // CONFIG_SUPERTX && CONFIG_TX64X64
mip2 = mip + 4;
if (4 >= max_cols) break;
......@@ -917,8 +917,8 @@ void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
break;
default:
#if CONFIG_SUPERTX && CONFIG_TX64X64
if (mip->mbmi.tx_size == TX_64X64) {
build_masks(cm, mip, 0, 0, lfm);
if (mip[0]->mbmi.tx_size == TX_64X64) {
build_masks(lfi_n, mip[0], 0, 0, lfm);
} else {
#endif // CONFIG_SUPERTX && CONFIG_TX64X64
for (idx_32 = 0; idx_32 < 4; mip += offset_32[idx_32], ++idx_32) {
......
......@@ -5108,7 +5108,7 @@ void av1_encode_frame(AV1_COMP *cpi) {
count16x16_lp += counts->tx_size_implied[4][TX_16X16];
count32x32_32x32p += counts->tx_size_implied[3][TX_32X32];
count32x32_lp += counts->tx_size_implied[4][TX_32X32];
count64x64_64x64p += counts->tx_size[4][TX_64X64];
count64x64_64x64p += counts->tx_size_implied[4][TX_64X64];
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
if (count4x4 == 0 && count16x16_lp == 0 && count16x16_16x16p == 0 &&
count32x32_lp == 0 && count32x32_32x32p == 0 &&
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment