Skip to content

GitLab

Commit 428bbb2b authored by Debargha Mukherjee's avatar Debargha Mukherjee
Browse files

Fix config flags among var-tx/ext-tx/rect-tx 

BUG=aomedia:396

Change-Id: I1b7f6dddb06eac9e341518a95a0d71465e613e9e
parent c16e044c
Hide whitespace changes
Inline Side-by-side
av1/common/blockd.h
View file @ 428bbb2b
......@@ -602,7 +602,6 @@ static INLINE int supertx_enabled(const MB_MODE_INFO *mbmi) {
#define USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4 1
#if CONFIG_RECT_TX
static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; }
#endif // CONFIG_RECT_TX
......@@ -776,13 +775,13 @@ static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
static INLINE TX_SIZE tx_size_from_tx_mode(BLOCK_SIZE bsize, TX_MODE tx_mode,
int is_inter) {
const TX_SIZE largest_tx_size = tx_mode_to_biggest_tx_size[tx_mode];
#if CONFIG_VAR_TX || (CONFIG_EXT_TX && CONFIG_RECT_TX)
#if (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
const TX_SIZE max_rect_tx_size = max_txsize_rect_lookup[bsize];
#else
const TX_SIZE max_tx_size = max_txsize_lookup[bsize];
#endif // CONFIG_VAR_TX || (CONFIG_EXT_TX && CONFIG_RECT_TX)
#endif // (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
(void)is_inter;
#if CONFIG_VAR_TX
#if CONFIG_VAR_TX && CONFIG_RECT_TX
#if CONFIG_CB4X4
if (bsize == BLOCK_4X4)
return AOMMIN(max_txsize_lookup[bsize], largest_tx_size);
......@@ -802,7 +801,7 @@ static INLINE TX_SIZE tx_size_from_tx_mode(BLOCK_SIZE bsize, TX_MODE tx_mode,
}
#else
return AOMMIN(max_tx_size, largest_tx_size);
#endif // CONFIG_VAR_TX
#endif // CONFIG_VAR_TX && CONFIG_RECT_TX
}
#if CONFIG_EXT_INTRA
......
av1/common/common_data.h
View file @ 428bbb2b
......@@ -455,7 +455,7 @@ static const TX_SIZE max_txsize_lookup[BLOCK_SIZES] = {
#endif // CONFIG_TX64X64
};
#if CONFIG_RECT_TX
#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
static const TX_SIZE max_txsize_rect_lookup[BLOCK_SIZES] = {
#if CONFIG_CB4X4
// 2X2, 2X4, 4X2,
......@@ -489,9 +489,9 @@ static const TX_SIZE max_txsize_rect_lookup[BLOCK_SIZES] = {
};
#else
#define max_txsize_rect_lookup max_txsize_lookup
#endif // CONFIG_RECT_TX
#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
#if (CONFIG_VAR_TX || CONFIG_RECT_TX)
#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
// Same as "max_txsize_lookup[bsize] - TX_8X8", except for rectangular
// block which may use a rectangular transform, in which case it is
// "(max_txsize_lookup[bsize] + 1) - TX_8X8", invalid for bsize < 8X8
......@@ -564,7 +564,7 @@ static const int32_t intra_tx_size_cat_lookup[BLOCK_SIZES] = {
#endif // CONFIG_EXT_PARTITION
#endif // CONFIG_TX64X64
};
#endif // CONFIG_VAR_TX && CONFIG_RECT_TX
#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
#define inter_tx_size_cat_lookup intra_tx_size_cat_lookup
......
av1/common/reconintra.c
View file @ 428bbb2b
......@@ -2262,7 +2262,7 @@ void av1_predict_intra_block(const MACROBLOCKD *xd, int wpx, int hpx,
predict_square_intra_block(xd, wpx, hpx, tx_size, mode, ref, ref_stride,
dst, dst_stride, col_off, row_off, plane);
} else {
#if (CONFIG_VAR_TX || CONFIG_RECT_TX) || \
#if (CONFIG_RECT_TX && (CONFIG_VAR_TX || CONFIG_EXT_TX)) || \
(CONFIG_EXT_INTER && USE_RECT_INTERINTRA)
#if CONFIG_AOM_HIGHBITDEPTH
uint16_t tmp16[MAX_SB_SIZE];
......@@ -2384,7 +2384,8 @@ void av1_predict_intra_block(const MACROBLOCKD *xd, int wpx, int hpx,
}
#else
assert(0);
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#endif // (CONFIG_RECT_TX && (CONFIG_VAR_TX || CONFIG_EXT_TX)) ||
// (CONFIG_EXT_INTER && USE_RECT_INTERINTRA)
}
}
......
av1/common/scan.c
View file @ 428bbb2b
......@@ -6500,14 +6500,14 @@ static uint32_t *get_non_zero_prob(FRAME_CONTEXT *fc, TX_SIZE tx_size,
case TX_8X8: return fc->non_zero_prob_8X8[tx_type];
case TX_16X16: return fc->non_zero_prob_16X16[tx_type];
case TX_32X32: return fc->non_zero_prob_32X32[tx_type];
#if CONFIG_VAR_TX || CONFIG_RECT_TX
#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
case TX_4X8: return fc->non_zero_prob_4X8[tx_type];
case TX_8X4: return fc->non_zero_prob_8X4[tx_type];
case TX_8X16: return fc->non_zero_prob_8X16[tx_type];
case TX_16X8: return fc->non_zero_prob_16X8[tx_type];
case TX_16X32: return fc->non_zero_prob_16X32[tx_type];
case TX_32X16: return fc->non_zero_prob_32X16[tx_type];
#endif
#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
default: assert(0); return NULL;
}
}
......@@ -6522,14 +6522,14 @@ static int16_t *get_adapt_scan(FRAME_CONTEXT *fc, TX_SIZE tx_size,
case TX_8X8: return fc->scan_8X8[tx_type];
case TX_16X16: return fc->scan_16X16[tx_type];
case TX_32X32: return fc->scan_32X32[tx_type];
#if CONFIG_VAR_TX || CONFIG_RECT_TX
#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
case TX_4X8: return fc->scan_4X8[tx_type];
case TX_8X4: return fc->scan_8X4[tx_type];
case TX_8X16: return fc->scan_8X16[tx_type];
case TX_16X8: return fc->scan_16X8[tx_type];
case TX_16X32: return fc->scan_16X32[tx_type];
case TX_32X16: return fc->scan_32X16[tx_type];
#endif
#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
default: assert(0); return NULL;
}
}
......@@ -6544,14 +6544,14 @@ static int16_t *get_adapt_iscan(FRAME_CONTEXT *fc, TX_SIZE tx_size,
case TX_8X8: return fc->iscan_8X8[tx_type];
case TX_16X16: return fc->iscan_16X16[tx_type];
case TX_32X32: return fc->iscan_32X32[tx_type];
#if CONFIG_VAR_TX || CONFIG_RECT_TX
#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
case TX_4X8: return fc->iscan_4X8[tx_type];
case TX_8X4: return fc->iscan_8X4[tx_type];
case TX_8X16: return fc->iscan_8X16[tx_type];
case TX_16X8: return fc->iscan_16X8[tx_type];
case TX_16X32: return fc->iscan_16X32[tx_type];
case TX_32X16: return fc->iscan_32X16[tx_type];
#endif
#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
default: assert(0); return NULL;
}
}
......@@ -6566,14 +6566,14 @@ static int16_t *get_adapt_nb(FRAME_CONTEXT *fc, TX_SIZE tx_size,
case TX_8X8: return fc->nb_8X8[tx_type];
case TX_16X16: return fc->nb_16X16[tx_type];
case TX_32X32: return fc->nb_32X32[tx_type];
#if CONFIG_VAR_TX || CONFIG_RECT_TX
#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
case TX_4X8: return fc->nb_4X8[tx_type];
case TX_8X4: return fc->nb_8X4[tx_type];
case TX_8X16: return fc->nb_8X16[tx_type];
case TX_16X8: return fc->nb_16X8[tx_type];
case TX_16X32: return fc->nb_16X32[tx_type];
case TX_32X16: return fc->nb_32X16[tx_type];
#endif
#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
default: assert(0); return NULL;
}
}
......@@ -6588,14 +6588,14 @@ static uint32_t *get_non_zero_counts(FRAME_COUNTS *counts, TX_SIZE tx_size,
case TX_8X8: return counts->non_zero_count_8X8[tx_type];
case TX_16X16: return counts->non_zero_count_16X16[tx_type];
case TX_32X32: return counts->non_zero_count_32X32[tx_type];
#if CONFIG_VAR_TX || CONFIG_RECT_TX
#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
case TX_4X8: return counts->non_zero_count_4x8[tx_type];
case TX_8X4: return counts->non_zero_count_8x4[tx_type];
case TX_8X16: return counts->non_zero_count_8x16[tx_type];
case TX_16X8: return counts->non_zero_count_16x8[tx_type];
case TX_16X32: return counts->non_zero_count_16x32[tx_type];
case TX_32X16: return counts->non_zero_count_32x16[tx_type];
#endif
#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
default: assert(0); return NULL;
}
}
......@@ -6774,11 +6774,11 @@ void av1_init_scan_order(AV1_COMMON *cm) {
TX_SIZE tx_size;
TX_TYPE tx_type;
for (tx_size = 0; tx_size < TX_SIZES_ALL; ++tx_size) {
#if !(CONFIG_VAR_TX || CONFIG_RECT_TX)
if (tx_size >= TX_SIZES) continue;
#else
#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
if (tx_size > TX_32X16) continue;
#endif
#else
if (tx_size >= TX_SIZES) continue;
#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
uint32_t *non_zero_prob = get_non_zero_prob(cm->fc, tx_size, tx_type);
const int tx2d_size = tx_size_2d[tx_size];
......@@ -6798,11 +6798,11 @@ void av1_init_scan_order(AV1_COMMON *cm) {
void av1_adapt_scan_order(AV1_COMMON *cm) {
TX_SIZE tx_size;
for (tx_size = 0; tx_size < TX_SIZES_ALL; ++tx_size) {
#if !(CONFIG_VAR_TX || CONFIG_RECT_TX)
if (tx_size >= TX_SIZES) continue;
#else
#if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
if (tx_size > TX_32X16) continue;
#endif // !(CONFIG_VAR_TX || CONFIG_RECT_TX)
#else
if (tx_size >= TX_SIZES) continue;
#endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
TX_TYPE tx_type;
for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
update_scan_prob(cm, tx_size, tx_type, ADAPT_SCAN_UPDATE_RATE_16);
......
av1/decoder/decodemv.c
View file @ 428bbb2b
......@@ -448,12 +448,8 @@ static TX_SIZE read_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd, int is_inter,
const TX_MODE tx_mode = cm->tx_mode;
const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type;
if (xd->lossless[xd->mi[0]->mbmi.segment_id]) return TX_4X4;
#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
#if CONFIG_RECT_TX
#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
if (bsize > BLOCK_4X4) {
#else
if (bsize >= BLOCK_8X8 || (bsize > BLOCK_4X4 && is_inter)) {
#endif // CONFIG_RECT_TX
#else
if (bsize >= BLOCK_8X8) {
#endif // CONFIG_CB4X4 && CONFIG_VAR_TX
......
av1/encoder/bitstream.c
View file @ 428bbb2b
......@@ -456,12 +456,8 @@ static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
FRAME_CONTEXT *ec_ctx = cm->fc;
#endif
// For sub8x8 blocks the tx_size symbol does not need to be sent
#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
#if CONFIG_RECT_TX
#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
if (bsize > BLOCK_4X4) {
#else
if (bsize >= BLOCK_8X8 || (bsize > BLOCK_4X4 && is_inter_block(mbmi))) {
#endif // CONFIG_RECT_TX
#else
if (bsize >= BLOCK_8X8) {
#endif
......
av1/encoder/encodeframe.c
View file @ 428bbb2b
......@@ -5771,13 +5771,8 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td,
TX_SIZE tx_size = mbmi->tx_size;
#endif
if (cm->tx_mode == TX_MODE_SELECT && !xd->lossless[mbmi->segment_id] &&
#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
#if CONFIG_RECT_TX
#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
mbmi->sb_type > BLOCK_4X4 &&
#else
(mbmi->sb_type >= BLOCK_8X8 ||
(mbmi->sb_type > BLOCK_4X4 && is_inter)) &&
#endif
#else
mbmi->sb_type >= BLOCK_8X8 &&
#endif
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment