Commit 35a4db38 authored by Debargha Mukherjee's avatar Debargha Mukherjee

Replace RECT_TX_EXT experiment

Remove the previous experiment and now use the same name for a
simpler experiment that only enables 4:1 transforms for 4:1
partitions when ext_partition_types is on, and that which was
previously enabled with the USE_RECT_TX_EXT macro.

Change-Id: Iccc35744bd292abf3c187da6f23b787692d50296
parent d467bba2
......@@ -413,7 +413,7 @@ void av1_inv_txfm2d_add_32x64_c(const int32_t *input, uint16_t *output,
}
#endif // CONFIG_TX64X64
#if CONFIG_RECT_TX_EXT || (CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT)
#if CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
void av1_inv_txfm2d_add_4x16_c(const int32_t *input, uint16_t *output,
int stride, TX_TYPE tx_type, int bd) {
int txfm_buf[4 * 16 + 16 + 16];
......@@ -471,4 +471,4 @@ void av1_inv_txfm2d_add_32x8_c(const int32_t *input, uint16_t *output,
inv_txfm2d_add_facade(input, output, stride, txfm_buf, tx_type, TX_32X8, bd);
#endif // CONFIG_TXMG
}
#endif
#endif // CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
......@@ -1000,53 +1000,6 @@ static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
!xd->lossless[mbmi->segment_id];
}
#if CONFIG_RECT_TX_EXT
static INLINE int is_quarter_tx_allowed_bsize(BLOCK_SIZE bsize) {
static const char LUT_QTTX[BLOCK_SIZES_ALL] = {
0, // BLOCK_2X2
0, // BLOCK_2X4
0, // BLOCK_4X2
0, // BLOCK_4X4
0, // BLOCK_4X8
0, // BLOCK_8X4
0, // BLOCK_8X8
1, // BLOCK_8X16
1, // BLOCK_16X8
0, // BLOCK_16X16
0, // BLOCK_16X32
0, // BLOCK_32X16
0, // BLOCK_32X32
0, // BLOCK_32X64
0, // BLOCK_64X32
0, // BLOCK_64X64
#if CONFIG_EXT_PARTITION
0, // BLOCK_64X128
0, // BLOCK_128X64
0, // BLOCK_128X128
#endif // CONFIG_EXT_PARTITION
0, // BLOCK_4X16
0, // BLOCK_16X4
0, // BLOCK_8X32
0, // BLOCK_32X8
0, // BLOCK_16X64
0, // BLOCK_64X16
#if CONFIG_EXT_PARTITION
0, // BLOCK_32X128
0, // BLOCK_128X32
#endif // CONFIG_EXT_PARTITION
};
return LUT_QTTX[bsize];
}
static INLINE int is_quarter_tx_allowed(const MACROBLOCKD *xd,
const MB_MODE_INFO *mbmi,
int is_inter) {
return is_quarter_tx_allowed_bsize(mbmi->sb_type) && is_inter &&
!xd->lossless[mbmi->segment_id];
}
#endif
static INLINE TX_SIZE get_max_rect_tx_size(BLOCK_SIZE bsize, int is_inter) {
if (is_inter)
return max_txsize_rect_lookup[bsize];
......
......@@ -717,7 +717,7 @@ static const TX_SIZE max_txsize_rect_lookup[BLOCK_SIZES_ALL] = {
TX_32X32, TX_32X32, TX_32X32,
#endif // CONFIG_EXT_PARTITION
#endif // CONFIG_TX64X64
#if CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT
#if CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
// 4x16, 16x4, 8x32
TX_4X16, TX_16X4, TX_8X16,
// 32x8
......@@ -727,7 +727,7 @@ static const TX_SIZE max_txsize_rect_lookup[BLOCK_SIZES_ALL] = {
TX_4X8, TX_8X4, TX_8X16,
// 32x8
TX_16X8,
#endif // CONFIG_RECT_TX_EXT
#endif // CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
// 16x64, 64x16
TX_16X32, TX_32X16,
#if CONFIG_EXT_PARTITION
......@@ -736,35 +736,6 @@ static const TX_SIZE max_txsize_rect_lookup[BLOCK_SIZES_ALL] = {
#endif // CONFIG_EXT_PARTITION
};
#if CONFIG_RECT_TX_EXT
static const TX_SIZE quarter_txsize_lookup[BLOCK_SIZES_ALL] = {
// 2X2, 2X4, 4X2,
TX_INVALID, TX_INVALID, TX_INVALID,
// 4x4,
TX_INVALID,
// 4x8, 8x4, 8x8,
TX_INVALID, TX_INVALID, TX_INVALID,
// 8x16, 16x8, 16x16,
TX_4X16, TX_16X4, TX_INVALID,
// 16x32, 32x16, 32x32,
TX_8X32, TX_32X8, TX_INVALID,
// 32x64, 64x32, 64x64
TX_INVALID, TX_INVALID, TX_INVALID,
#if CONFIG_EXT_PARTITION
// 64x128, 128x64, 128x128
TX_INVALID, TX_INVALID, TX_INVALID,
#endif
// 4x16, 16x4, 8x32
TX_4X16, TX_16X4, TX_8X32,
// 32x8 16x64 64x16
TX_32X8, TX_INVALID, TX_INVALID,
#if CONFIG_EXT_PARTITION
// 32x128 128x32
TX_INVALID, TX_INVALID
#endif // CONFIG_EXT_PARTITION
};
#endif // CONFIG_RECT_TX_EXT
static const TX_TYPE_1D vtx_tab[TX_TYPES] = {
DCT_1D, ADST_1D, DCT_1D, ADST_1D,
FLIPADST_1D, DCT_1D, FLIPADST_1D, ADST_1D, FLIPADST_1D, IDTX_1D,
......
......@@ -1749,9 +1749,6 @@ void av1_average_tile_intra_cdfs(FRAME_CONTEXT *fc, FRAME_CONTEXT *ec_ctxs[],
#if CONFIG_NEW_MULTISYMBOL
AVERAGE_TILE_CDFS(palette_y_mode_cdf)
AVERAGE_TILE_CDFS(palette_uv_mode_cdf)
#if CONFIG_RECT_TX_EXT
AVERAGE_TILE_CDFS(quarter_tx_size_cdf)
#endif
#endif
#if CONFIG_LPF_SB
AVERAGE_TILE_CDFS(lpf_reuse_cdf);
......
......@@ -342,7 +342,7 @@ static INLINE int get_entropy_context(TX_SIZE tx_size, const ENTROPY_CONTEXT *a,
left_ec = !!*(const uint64_t *)l;
break;
#endif // CONFIG_TX64X64
#if CONFIG_RECT_TX_EXT || (CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT)
#if CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
case TX_4X16:
above_ec = a[0] != 0;
left_ec = !!*(const uint32_t *)l;
......@@ -359,7 +359,7 @@ static INLINE int get_entropy_context(TX_SIZE tx_size, const ENTROPY_CONTEXT *a,
above_ec = !!*(const uint64_t *)a;
left_ec = !!*(const uint16_t *)l;
break;
#endif
#endif // CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
default: assert(0 && "Invalid transform size."); break;
}
return combine_entropy_contexts(above_ec, left_ec);
......
......@@ -1566,14 +1566,6 @@ static const int palette_color_index_context_lookup[MAX_COLOR_CONTEXT_HASH +
1] = { -1, -1, 0, -1, -1,
4, 3, 2, 1 };
#if CONFIG_RECT_TX_EXT
static const aom_prob default_quarter_tx_size_prob = 192;
#if CONFIG_NEW_MULTISYMBOL
static const aom_cdf_prob default_quarter_tx_size_cdf[CDF_SIZE(2)] = { AOM_CDF2(
192 * 128) };
#endif
#endif
#if CONFIG_LOOP_RESTORATION
static const aom_cdf_prob default_switchable_restore_cdf[CDF_SIZE(
RESTORE_SWITCHABLE_TYPES)] = { AOM_CDF3(32 * 128, 144 * 128) };
......@@ -3254,12 +3246,6 @@ static void init_mode_probs(FRAME_CONTEXT *fc) {
#if CONFIG_COMPOUND_SINGLEREF
av1_copy(fc->comp_inter_mode_prob, default_comp_inter_mode_p);
#endif // CONFIG_COMPOUND_SINGLEREF
#if CONFIG_RECT_TX_EXT
fc->quarter_tx_size_prob = default_quarter_tx_size_prob;
#if CONFIG_NEW_MULTISYMBOL
av1_copy(fc->quarter_tx_size_cdf, default_quarter_tx_size_cdf);
#endif // CONFIG_NEW_MULTISYMBOL
#endif
av1_copy(fc->txfm_partition_prob, default_txfm_partition_probs);
#if CONFIG_NEW_MULTISYMBOL
av1_copy(fc->txfm_partition_cdf, default_txfm_partition_cdf);
......@@ -3485,13 +3471,6 @@ void av1_adapt_intra_frame_probs(AV1_COMMON *cm) {
const FRAME_CONTEXT *pre_fc = cm->pre_fc;
const FRAME_COUNTS *counts = &cm->counts;
if (cm->tx_mode == TX_MODE_SELECT) {
#if CONFIG_RECT_TX_EXT
fc->quarter_tx_size_prob = av1_mode_mv_merge_probs(
pre_fc->quarter_tx_size_prob, counts->quarter_tx_size);
#endif
}
if (cm->tx_mode == TX_MODE_SELECT) {
for (i = 0; i < TXFM_PARTITION_CONTEXTS; ++i)
fc->txfm_partition_prob[i] = av1_mode_mv_merge_probs(
......
......@@ -266,12 +266,6 @@ typedef struct frame_contexts {
#if CONFIG_COMPOUND_SINGLEREF
aom_prob comp_inter_mode_prob[COMP_INTER_MODE_CONTEXTS];
#endif // CONFIG_COMPOUND_SINGLEREF
#if CONFIG_RECT_TX_EXT
aom_prob quarter_tx_size_prob;
#if CONFIG_NEW_MULTISYMBOL
aom_cdf_prob quarter_tx_size_cdf[CDF_SIZE(2)];
#endif
#endif
aom_prob txfm_partition_prob[TXFM_PARTITION_CONTEXTS];
#if CONFIG_NEW_MULTISYMBOL
aom_cdf_prob txfm_partition_cdf[TXFM_PARTITION_CONTEXTS][CDF_SIZE(2)];
......@@ -459,9 +453,6 @@ typedef struct FRAME_COUNTS {
#if CONFIG_COMPOUND_SINGLEREF
unsigned int comp_inter_mode[COMP_INTER_MODE_CONTEXTS][2];
#endif // CONFIG_COMPOUND_SINGLEREF
#if CONFIG_RECT_TX_EXT
unsigned int quarter_tx_size[2];
#endif
unsigned int txfm_partition[TXFM_PARTITION_CONTEXTS][2];
#if CONFIG_EXT_SKIP
unsigned int skip_mode[SKIP_MODE_CONTEXTS][2];
......
......@@ -37,10 +37,6 @@ extern "C" {
#define MAX_SB_SIZE (1 << MAX_SB_SIZE_LOG2)
#define MAX_SB_SQUARE (MAX_SB_SIZE * MAX_SB_SIZE)
#if CONFIG_EXT_PARTITION_TYPES
#define USE_RECT_TX_EXT 1
#endif
// Min superblock size
#define MIN_SB_SIZE_LOG2 6
......
......@@ -1873,7 +1873,7 @@ static void inv_txfm_add_8x4(const tran_low_t *input, uint8_t *dest, int stride,
}
// These will be used by the masked-tx experiment in the future.
#if CONFIG_RECT_TX_EXT || (CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT)
#if CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
static void inv_txfm_add_4x16(const tran_low_t *input, uint8_t *dest,
int stride, const TxfmParam *txfm_param) {
av1_iht4x16_64_add(input, dest, stride, txfm_param);
......@@ -1893,7 +1893,7 @@ static void inv_txfm_add_32x8(const tran_low_t *input, uint8_t *dest,
int stride, const TxfmParam *txfm_param) {
av1_iht32x8_256_add(input, dest, stride, txfm_param);
}
#endif
#endif // CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
static void inv_txfm_add_8x16(const tran_low_t *input, uint8_t *dest,
int stride, const TxfmParam *txfm_param) {
......@@ -2191,7 +2191,7 @@ static void highbd_inv_txfm_add_32x16(const tran_low_t *input, uint8_t *dest,
txfm_param->tx_type, txfm_param->bd);
}
#if CONFIG_RECT_TX_EXT || (CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT)
#if CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
static void highbd_inv_txfm_add_16x4(const tran_low_t *input, uint8_t *dest,
int stride, const TxfmParam *txfm_param) {
const int32_t *src = cast_to_int32(input);
......@@ -2219,7 +2219,7 @@ static void highbd_inv_txfm_add_8x32(const tran_low_t *input, uint8_t *dest,
av1_inv_txfm2d_add_8x32_c(src, CONVERT_TO_SHORTPTR(dest), stride,
txfm_param->tx_type, txfm_param->bd);
}
#endif
#endif // CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
#if CONFIG_TX64X64
static void highbd_inv_txfm_add_32x64(const tran_low_t *input, uint8_t *dest,
......@@ -2419,12 +2419,12 @@ void av1_inv_txfm_add(const tran_low_t *input, uint8_t *dest, int stride,
// case.
inv_txfm_add_4x4(input, dest, stride, txfm_param);
break;
#if CONFIG_RECT_TX_EXT || (CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT)
#if CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
case TX_32X8: inv_txfm_add_32x8(input, dest, stride, txfm_param); break;
case TX_8X32: inv_txfm_add_8x32(input, dest, stride, txfm_param); break;
case TX_16X4: inv_txfm_add_16x4(input, dest, stride, txfm_param); break;
case TX_4X16: inv_txfm_add_4x16(input, dest, stride, txfm_param); break;
#endif
#endif // CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
default: assert(0 && "Invalid transform size"); break;
}
#endif
......@@ -2570,7 +2570,7 @@ void av1_highbd_inv_txfm_add(const tran_low_t *input, uint8_t *dest, int stride,
// case.
av1_highbd_inv_txfm_add_4x4(input, dest, stride, txfm_param);
break;
#if CONFIG_RECT_TX_EXT || (CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT)
#if CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
case TX_16X4:
highbd_inv_txfm_add_16x4(input, dest, stride, txfm_param);
break;
......@@ -2583,8 +2583,8 @@ void av1_highbd_inv_txfm_add(const tran_low_t *input, uint8_t *dest, int stride,
case TX_32X8:
highbd_inv_txfm_add_32x8(input, dest, stride, txfm_param);
break;
#endif
#endif // CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
default: assert(0 && "Invalid transform size"); break;
}
#endif
#endif // CONFIG_DAALA_TX
}
......@@ -341,15 +341,9 @@ static void decode_reconstruct_tx(AV1_COMMON *cm, MACROBLOCKD *const xd,
pd->dst.stride, max_scan_line, eob);
*eob_total += eob;
} else {
#if CONFIG_RECT_TX_EXT
int is_qttx = plane_tx_size == quarter_txsize_lookup[plane_bsize];
const TX_SIZE sub_txs = is_qttx ? plane_tx_size : sub_tx_size_map[tx_size];
if (is_qttx) assert(blk_row == 0 && blk_col == 0 && block == 0);
#else
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
assert(IMPLIES(tx_size <= TX_4X4, sub_txs == tx_size));
assert(IMPLIES(tx_size > TX_4X4, sub_txs < tx_size));
#endif
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
const int sub_step = bsw * bsh;
......@@ -3371,11 +3365,6 @@ static int read_compressed_header(AV1Decoder *pbi, const uint8_t *data,
aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
"Failed to allocate bool decoder 0");
#if CONFIG_RECT_TX_EXT
if (cm->tx_mode == TX_MODE_SELECT)
av1_diff_update_prob(&r, &fc->quarter_tx_size_prob, ACCT_STR);
#endif
if (cm->tx_mode == TX_MODE_SELECT)
for (int i = 0; i < TXFM_PARTITION_CONTEXTS; ++i)
av1_diff_update_prob(&r, &fc->txfm_partition_prob[i], ACCT_STR);
......
......@@ -539,27 +539,6 @@ static TX_SIZE read_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd, int is_inter,
read_selected_tx_size(cm, xd, tx_size_cat, r);
if (coded_tx_size > max_txsize_lookup[bsize]) {
assert(coded_tx_size == max_txsize_lookup[bsize] + 1);
#if CONFIG_RECT_TX_EXT
if (is_quarter_tx_allowed(xd, &xd->mi[0]->mbmi, is_inter)) {
int quarter_tx;
if (quarter_txsize_lookup[bsize] != max_txsize_lookup[bsize]) {
#if CONFIG_NEW_MULTISYMBOL
quarter_tx =
aom_read_symbol(r, cm->fc->quarter_tx_size_cdf, 2, ACCT_STR);
#else
quarter_tx = aom_read(r, cm->fc->quarter_tx_size_prob, ACCT_STR);
FRAME_COUNTS *counts = xd->counts;
if (counts) ++counts->quarter_tx_size[quarter_tx];
#endif
} else {
quarter_tx = 1;
}
return quarter_tx ? quarter_txsize_lookup[bsize]
: get_max_rect_tx_size(bsize, is_inter);
}
#endif // CONFIG_RECT_TX_EXT
return get_max_rect_tx_size(bsize, is_inter);
}
return coded_tx_size;
......@@ -2714,31 +2693,6 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi,
for (idx = 0; idx < width; idx += bw)
read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size, 0, idy, idx,
r);
#if CONFIG_RECT_TX_EXT
if (is_quarter_tx_allowed(xd, mbmi, inter_block) &&
mbmi->tx_size == max_tx_size) {
int quarter_tx;
if (quarter_txsize_lookup[bsize] != max_tx_size) {
#if CONFIG_NEW_MULTISYMBOL
quarter_tx =
aom_read_symbol(r, cm->fc->quarter_tx_size_cdf, 2, ACCT_STR);
#else
quarter_tx = aom_read(r, cm->fc->quarter_tx_size_prob, ACCT_STR);
if (xd->counts) ++xd->counts->quarter_tx_size[quarter_tx];
#endif
} else {
quarter_tx = 1;
}
if (quarter_tx) {
mbmi->tx_size = quarter_txsize_lookup[bsize];
for (idy = 0; idy < tx_size_high_unit[max_tx_size] / 2; ++idy)
for (idx = 0; idx < tx_size_wide_unit[max_tx_size] / 2; ++idx)
mbmi->inter_tx_size[idy][idx] = mbmi->tx_size;
mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
}
}
#endif
} else {
mbmi->tx_size = read_tx_size(cm, xd, inter_block, !mbmi->skip, r);
......
......@@ -261,12 +261,7 @@ static void write_tx_size_vartx(const AV1_COMMON *cm, MACROBLOCKD *xd,
}
const int write_txfm_partition =
#if CONFIG_RECT_TX_EXT
tx_size == mbmi->inter_tx_size[tx_row][tx_col] ||
mbmi->tx_size == quarter_txsize_lookup[mbmi->sb_type];
#else
tx_size == mbmi->inter_tx_size[tx_row][tx_col];
#endif
if (write_txfm_partition) {
#if CONFIG_NEW_MULTISYMBOL
aom_write_symbol(w, 0, ec_ctx->txfm_partition_cdf[ctx], 2);
......@@ -337,16 +332,6 @@ static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
aom_write_symbol(w, depth, ec_ctx->tx_size_cdf[tx_size_cat][tx_size_ctx],
max_depths + 1);
#if CONFIG_RECT_TX_EXT
if (is_quarter_tx_allowed(xd, mbmi, 0) && tx_size != coded_tx_size)
#if CONFIG_NEW_MULTISYMBOL
aom_write_symbol(w, tx_size == quarter_txsize_lookup[bsize],
cm->fc->quarter_tx_size_cdf, 2);
#else
aom_write(w, tx_size == quarter_txsize_lookup[bsize],
cm->fc->quarter_tx_size_prob);
#endif
#endif
}
}
......@@ -714,12 +699,7 @@ static void pack_txb_tokens(aom_writer *w, const TOKENEXTRA **tp,
token_stats->cost += tmp_token_stats.cost;
#endif
} else {
#if CONFIG_RECT_TX_EXT
int is_qttx = plane_tx_size == quarter_txsize_lookup[plane_bsize];
const TX_SIZE sub_txs = is_qttx ? plane_tx_size : sub_tx_size_map[tx_size];
#else
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
#endif
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
......@@ -1451,20 +1431,6 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const int mi_row,
for (idy = 0; idy < height; idy += bh)
for (idx = 0; idx < width; idx += bw)
write_tx_size_vartx(cm, xd, mbmi, max_tx_size, 0, idy, idx, w);
#if CONFIG_RECT_TX_EXT
if (is_quarter_tx_allowed(xd, mbmi, is_inter_block(mbmi)) &&
quarter_txsize_lookup[bsize] != max_tx_size &&
(mbmi->tx_size == quarter_txsize_lookup[bsize] ||
mbmi->tx_size == max_tx_size)) {
#if CONFIG_NEW_MULTISYMBOL
aom_write_symbol(w, mbmi->tx_size != max_tx_size,
cm->fc->quarter_tx_size_cdf, 2);
#else
aom_write(w, mbmi->tx_size != max_tx_size,
cm->fc->quarter_tx_size_prob);
#endif
}
#endif
} else {
set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, skip, xd);
write_selected_tx_size(cm, xd, w);
......@@ -4506,12 +4472,6 @@ static uint32_t write_compressed_header(AV1_COMP *cpi, uint8_t *data) {
#endif
aom_start_encode(header_bc, data);
#if CONFIG_RECT_TX_EXT
if (cm->tx_mode == TX_MODE_SELECT)
av1_cond_prob_diff_update(header_bc, &cm->fc->quarter_tx_size_prob,
cm->counts.quarter_tx_size, probwt);
#endif
#if !CONFIG_NEW_MULTISYMBOL
if (cm->tx_mode == TX_MODE_SELECT)
update_txfm_partition_probs(cm, header_bc, counts, probwt);
......
......@@ -266,9 +266,6 @@ struct macroblock {
int cfl_cost[CFL_JOINT_SIGNS][CFL_PRED_PLANES][CFL_ALPHABET_SIZE];
#endif // CONFIG_CFL
int tx_size_cost[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES];
#if CONFIG_RECT_TX_EXT
int quarter_tx_size_cost[2];
#endif
int txfm_partition_cost[TXFM_PARTITION_CONTEXTS][2];
int inter_tx_type_costs[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES];
int intra_tx_type_costs[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES]
......
......@@ -4433,12 +4433,7 @@ void av1_encode_frame(AV1_COMP *cpi) {
}
make_consistent_compound_tools(cm);
#if CONFIG_RECT_TX_EXT
if (cm->tx_mode == TX_MODE_SELECT && cpi->td.mb.txb_split_count == 0 &&
counts->quarter_tx_size[1] == 0)
#else
if (cm->tx_mode == TX_MODE_SELECT && cpi->td.mb.txb_split_count == 0)
#endif
#if CONFIG_SIMPLIFY_TX_MODE
cm->tx_mode = TX_MODE_LARGEST;
#else
......@@ -4596,32 +4591,20 @@ static void update_txfm_count(MACROBLOCK *x, MACROBLOCKD *xd,
assert(tx_size > TX_4X4);
if (depth == MAX_VARTX_DEPTH) {
// Don't add to counts in this case
#if CONFIG_RECT_TX_EXT
if (tx_size == plane_tx_size)
#endif
mbmi->tx_size = tx_size;
// Don't add to counts in this case
mbmi->tx_size = tx_size;
txfm_partition_update(xd->above_txfm_context + blk_col,
xd->left_txfm_context + blk_row, tx_size, tx_size);
return;
}
#if CONFIG_RECT_TX_EXT
if (tx_size == plane_tx_size ||
mbmi->tx_size == quarter_txsize_lookup[mbmi->sb_type])
#else
if (tx_size == plane_tx_size)
#endif
{
if (tx_size == plane_tx_size) {
++counts->txfm_partition[ctx][0];
#if CONFIG_NEW_MULTISYMBOL
if (allow_update_cdf)
update_cdf(xd->tile_ctx->txfm_partition_cdf[ctx], 0, 2);
#endif
#if CONFIG_RECT_TX_EXT
if (tx_size == plane_tx_size)
#endif
mbmi->tx_size = tx_size;
mbmi->tx_size = tx_size;
txfm_partition_update(xd->above_txfm_context + blk_col,
xd->left_txfm_context + blk_row, tx_size, tx_size);
} else {
......@@ -4941,21 +4924,6 @@ static void encode_superblock(const AV1_COMP *const cpi, TileDataEnc *tile_data,
} else {
if (tx_size != get_max_rect_tx_size(bsize, 0)) ++x->txb_split_count;
}
#if CONFIG_RECT_TX_EXT
if (is_quarter_tx_allowed(xd, mbmi, is_inter) &&
quarter_txsize_lookup[bsize] !=
get_max_rect_tx_size(bsize, is_inter) &&
(mbmi->tx_size == quarter_txsize_lookup[bsize] ||
mbmi->tx_size == get_max_rect_tx_size(bsize, is_inter))) {
const int use_qttx = mbmi->tx_size == quarter_txsize_lookup[bsize];
++td->counts->quarter_tx_size[use_qttx];
#if CONFIG_NEW_MULTISYMBOL
if (tile_data->allow_update_cdf)
update_cdf(xd->tile_ctx->quarter_tx_size_cdf, use_qttx, 2);
#endif
}
#endif
assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
} else {
int i, j;
......
......@@ -663,15 +663,9 @@ static void encode_block_inter(int plane, int block, int blk_row, int blk_col,
encode_block(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg);
} else {
assert(tx_size < TX_SIZES_ALL);
#if CONFIG_RECT_TX_EXT
int is_qttx = plane_tx_size == quarter_txsize_lookup[plane_bsize];
const TX_SIZE sub_txs = is_qttx ? plane_tx_size : sub_tx_size_map[tx_size];
if (is_qttx) assert(blk_row == 0 && blk_col == 0 && block == 0);
#else
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
assert(IMPLIES(tx_size <= TX_4X4, sub_txs == tx_size));
assert(IMPLIES(tx_size > TX_4X4, sub_txs < tx_size));
#endif
// This is the square transform block partition entry point.
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
......
......@@ -145,7 +145,7 @@ static void fwd_txfm_64x32(const int16_t *src_diff, tran_low_t *coeff,
}
#endif // CONFIG_TX64X64
#if CONFIG_RECT_TX_EXT || (CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT)
#if CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
static void fwd_txfm_16x4(const int16_t *src_diff, tran_low_t *coeff,
int diff_stride, TxfmParam *txfm_param) {
av1_fht16x4(src_diff, coeff, diff_stride, txfm_param);
......@@ -165,7 +165,7 @@ static void fwd_txfm_8x32(const int16_t *src_diff, tran_low_t *coeff,
int diff_stride, TxfmParam *txfm_param) {
av1_fht8x32(src_diff, coeff, diff_stride, txfm_param);
}
#endif
#endif // CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
static void highbd_fwd_txfm_4x4(const int16_t *src_diff, tran_low_t *coeff,
int diff_stride, TxfmParam *txfm_param) {
......@@ -247,7 +247,7 @@ static void highbd_fwd_txfm_32x16(const int16_t *src_diff, tran_low_t *coeff,
txfm_param->bd);
}
#if CONFIG_RECT_TX_EXT || (CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT)
#if CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
static void highbd_fwd_txfm_16x4(const int16_t *src_diff, tran_low_t *coeff,
int diff_stride, TxfmParam *txfm_param) {
av1_fht16x4(src_diff, coeff, diff_stride, txfm_param);
......@@ -267,7 +267,7 @@ static void highbd_fwd_txfm_8x32(const int16_t *src_diff, tran_low_t *coeff,
int diff_stride, TxfmParam *txfm_param) {
av1_fht8x32(src_diff, coeff, diff_stride, txfm_param);
}
#endif
#endif // CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
static void highbd_fwd_txfm_8x8(const int16_t *src_diff, tran_low_t *coeff,
int diff_stride, TxfmParam *txfm_param) {
......@@ -525,7 +525,7 @@ void av1_fwd_txfm(const int16_t *src_diff, tran_low_t *coeff, int diff_stride,
fwd_txfm_32x16(src_diff, coeff, diff_stride, txfm_param);
break;
case TX_4X4: fwd_txfm_4x4(src_diff, coeff, diff_stride, txfm_param); break;
#if CONFIG_RECT_TX_EXT || (CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT)
#if CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
case TX_4X16:
fwd_txfm_4x16(src_diff, coeff, diff_stride, txfm_param);
break;
......@@ -538,7 +538,7 @@ void av1_fwd_txfm(const int16_t *src_diff, tran_low_t *coeff, int diff_stride,
case TX_32X8:
fwd_txfm_32x8(src_diff, coeff, diff_stride, txfm_param);
break;
#endif
#endif // CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
default: assert(0); break;
}
#endif
......@@ -593,7 +593,7 @@ void av1_highbd_fwd_txfm(const int16_t *src_diff, tran_low_t *coeff,
case TX_4X4:
highbd_fwd_txfm_4x4(src_diff, coeff, diff_stride, txfm_param);
break;
#if CONFIG_RECT_TX_EXT || (CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT)
#if CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
case TX_4X16:
highbd_fwd_txfm_4x16(src_diff, coeff, diff_stride, txfm_param);
break;
......@@ -606,7 +606,7 @@ void av1_highbd_fwd_txfm(const int16_t *src_diff, tran_low_t *coeff,
case TX_32X8:
highbd_fwd_txfm_32x8(src_diff, coeff, diff_stride, txfm_param);
break;
#endif
#endif // CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
default: assert(0); break;
}
#endif
......
......@@ -205,16 +205,6 @@ void av1_fill_mode_rates(AV1_COMMON *const cm, MACROBLOCK *x,
av1_cost_tokens_from_cdf(x->tx_size_cost[i][j], fc->tx_size_cdf[i][j],
NULL);
#if CONFIG_RECT_TX_EXT
#if CONFIG_NEW_MULTISYMBOL
av1_cost_tokens_from_cdf(x->quarter_tx_size_cost, fc->quarter_tx_size_cdf,
NULL);
#else
x->quarter_tx_size_cost[0] = av1_cost_bit(fc->quarter_tx_size_prob, 0);
x->quarter_tx_size_cost[1] = av1_cost_bit(fc->quarter_tx_size_prob, 1);
#endif
#endif
for (i = 0; i < TXFM_PARTITION_CONTEXTS; ++i) {
#if CONFIG_NEW_MULTISYMBOL
av1_cost_tokens_from_cdf(x->txfm_partition_cost[i],
......@@ -925,7 +915,7 @@ static void get_entropy_contexts_plane(
for (i = 0; i < num_4x4_h; i += 4)
t_left[i] = !!*(const uint32_t *)&left[i];
break;
#if CONFIG_RECT_TX_EXT || (CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT)
#if CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
case TX_4X16:
memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
for (i = 0; i < num_4x4_h; i += 4)
......@@ -948,7 +938,7 @@ static void get_entropy_contexts_plane(
for (i = 0; i < num_4x4_h; i += 2)
t_left[i] = !!*(const uint16_t *)&left[i];
break;
#endif
#endif // CONFIG_EXT_PARTITION_TYPES && CONFIG_RECT_TX_EXT
default: assert(0 && "Invalid transform size."); break;
}
}
......
This diff is collapsed.
......@@ -605,13 +605,8 @@ void tokenize_vartx(ThreadData *td, TOKENEXTRA **t, RUN_TYPE dry_run,
cost_coeffs_b(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg);
#endif
} else {
#if CONFIG_RECT_TX_EXT
int is_qttx = plane_tx_size == quarter_txsize_lookup[plane_bsize];
const TX_SIZE sub_txs = is_qttx ? plane_tx_size : sub_tx_size_map[tx_size];
#else
// Half the block size in transform block unit.
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
#endif
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
const int step = bsw * bsh;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment