Commit 1a5ab9f2 authored by Yue Chen's avatar Yue Chen
Browse files

Avoid trying filter_intra + >32x32 tx

Filter intra mode is disabled if any dimention of tx_size is
greater than 32. In this patch, the max qualified tx_size is
found and set for testing filter_intra in intra_model_yrd().

BUG=aomedia:1058

Change-Id: I105aa68e722fce74b8828860a40040764cf20118
parent b971c866
......@@ -195,7 +195,6 @@ typedef struct {
} PALETTE_MODE_INFO;
#if CONFIG_FILTER_INTRA
#define USE_3TAP_INTRA_FILTER 1 // 0: 4-tap; 1: 3-tap
typedef struct {
// 1: an ext intra mode is used; 0: otherwise.
uint8_t use_filter_intra_mode[PLANE_TYPES];
......@@ -205,27 +204,6 @@ typedef struct {
static const PREDICTION_MODE fimode_to_intradir[FILTER_INTRA_MODES] = {
DC_PRED, V_PRED, H_PRED, D153_PRED, DC_PRED
};
#define DISABLE_SUB8X8_FILTER_INTRA 0
static INLINE int av1_filter_intra_allowed_bsize(BLOCK_SIZE bs) {
(void)bs;
#if DISABLE_SUB8X8_FILTER_INTRA
return block_size_wide[bs] >= 8 && block_size_high[bs] >= 8;
#else
return 1;
#endif
}
static INLINE int av1_filter_intra_allowed_txsize(TX_SIZE tx) {
(void)tx;
#if DISABLE_SUB8X8_FILTER_INTRA
return tx_size_wide[tx] >= 8 && tx_size_high[tx] >= 8 &&
tx_size_wide[tx] <= 32 && tx_size_high[tx] <= 32;
#else
return tx_size_wide[tx] <= 32 && tx_size_high[tx] <= 32;
#endif
}
#endif // CONFIG_FILTER_INTRA
#if CONFIG_RD_DEBUG
......@@ -982,6 +960,47 @@ static const uint8_t mode_to_angle_map[] = {
};
#endif // CONFIG_EXT_INTRA
#if CONFIG_FILTER_INTRA
#define DISABLE_SUB8X8_FILTER_INTRA 0
static INLINE int av1_filter_intra_allowed_bsize(BLOCK_SIZE bs) {
(void)bs;
#if DISABLE_SUB8X8_FILTER_INTRA
return block_size_wide[bs] >= 8 && block_size_high[bs] >= 8;
#else
return 1;
#endif
}
static INLINE int av1_filter_intra_allowed_txsize(TX_SIZE tx) {
(void)tx;
if (tx == TX_INVALID) return 0;
#if DISABLE_SUB8X8_FILTER_INTRA
return tx_size_wide[tx] >= 8 && tx_size_high[tx] >= 8 &&
tx_size_wide[tx] <= 32 && tx_size_high[tx] <= 32;
#else
return tx_size_wide[tx] <= 32 && tx_size_high[tx] <= 32;
#endif
}
static INLINE TX_SIZE av1_max_tx_size_for_filter_intra(BLOCK_SIZE bsize,
TX_MODE tx_mode) {
const TX_SIZE max_tx_size = tx_size_from_tx_mode(bsize, tx_mode, 0);
if (tx_mode != TX_MODE_SELECT) return max_tx_size;
int depth = 0;
TX_SIZE tx_size = max_tx_size;
while (depth < MAX_TX_DEPTH && tx_size != TX_4X4) {
if (av1_filter_intra_allowed_txsize(tx_size)) return tx_size;
depth++;
tx_size = sub_tx_size_map[0][tx_size];
}
return TX_INVALID;
}
#endif
#if CONFIG_DCT_ONLY
#define FIXED_TX_TYPE 1
#else
......
......@@ -2721,7 +2721,13 @@ static int64_t intra_model_yrd(const AV1_COMP *const cpi, MACROBLOCK *const x,
RD_STATS this_rd_stats;
int row, col;
int64_t temp_sse, this_rd;
const TX_SIZE tx_size = tx_size_from_tx_mode(bsize, cpi->common.tx_mode, 0);
TX_SIZE tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode, 0);
#if CONFIG_FILTER_INTRA
if (mbmi->filter_intra_mode_info.use_filter_intra_mode[0]) {
tx_size = av1_max_tx_size_for_filter_intra(bsize, cm->tx_mode);
if (!av1_filter_intra_allowed_txsize(tx_size)) return INT64_MAX;
}
#endif
const int stepr = tx_size_high_unit[tx_size];
const int stepc = tx_size_wide_unit[tx_size];
const int max_blocks_wide = max_block_wide(xd, bsize, 0);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment