Skip to content

GitLab

Commit 0797a208 authored by Yue Chen's avatar Yue Chen Committed by Debargha Mukherjee
Browse files

Add the option of using 1:4/4:1 tx_size+sb_type 

Change-Id: I96e5ff72caee8935efb7535afa3a534175bc425c
parent b09e55cf
Hide whitespace changes
Inline Side-by-side
Showing with 205 additions and 169 deletions
av1/common/blockd.h
View file @ 0797a208
......@@ -1047,10 +1047,17 @@ static INLINE int is_quarter_tx_allowed(const MACROBLOCKD *xd,
}
#endif
static INLINE TX_SIZE get_max_rect_tx_size(BLOCK_SIZE bsize, int is_inter) {
if (is_inter)
return max_txsize_rect_lookup[bsize];
else
return max_txsize_rect_intra_lookup[bsize];
}
static INLINE TX_SIZE tx_size_from_tx_mode(BLOCK_SIZE bsize, TX_MODE tx_mode,
int is_inter) {
const TX_SIZE largest_tx_size = tx_mode_to_biggest_tx_size[tx_mode];
const TX_SIZE max_rect_tx_size = max_txsize_rect_lookup[bsize];
const TX_SIZE max_rect_tx_size = get_max_rect_tx_size(bsize, is_inter);
(void)is_inter;
if (bsize == BLOCK_4X4)
return AOMMIN(max_txsize_lookup[bsize], largest_tx_size);
......@@ -1293,7 +1300,7 @@ static INLINE int is_interintra_pred(const MB_MODE_INFO *mbmi) {
static INLINE int get_vartx_max_txsize(const MB_MODE_INFO *const mbmi,
BLOCK_SIZE bsize, int subsampled) {
(void)mbmi;
TX_SIZE max_txsize = max_txsize_rect_lookup[bsize];
TX_SIZE max_txsize = get_max_rect_tx_size(bsize, is_inter_block(mbmi));
#if CONFIG_EXT_PARTITION && CONFIG_TX64X64
// The decoder is designed so that it can process 64x64 luma pixels at a
......
av1/common/common_data.h
View file @ 0797a208
......@@ -647,6 +647,46 @@ static const TX_SIZE max_txsize_lookup[BLOCK_SIZES_ALL] = {
#endif // CONFIG_EXT_PARTITION
};
static const TX_SIZE max_txsize_rect_intra_lookup[BLOCK_SIZES_ALL] = {
// 2X2, 2X4, 4X2,
TX_4X4, TX_4X4, TX_4X4,
// 4X4
TX_4X4,
// 4X8, 8X4, 8X8
TX_4X8, TX_8X4, TX_8X8,
// 8X16, 16X8, 16X16
TX_8X16, TX_16X8, TX_16X16,
// 16X32, 32X16, 32X32
TX_16X32, TX_32X16, TX_32X32,
#if CONFIG_TX64X64
// 32X64, 64X32,
TX_32X64, TX_64X32,
// 64X64
TX_64X64,
#if CONFIG_EXT_PARTITION
// 64x128, 128x64, 128x128
TX_64X64, TX_64X64, TX_64X64,
#endif // CONFIG_EXT_PARTITION
#else
// 32X64, 64X32,
TX_32X32, TX_32X32,
// 64X64
TX_32X32,
#if CONFIG_EXT_PARTITION
// 64x128, 128x64, 128x128
TX_32X32, TX_32X32, TX_32X32,
#endif // CONFIG_EXT_PARTITION
#endif // CONFIG_TX64X64
// 4x16, 16x4, 8x32
TX_4X8, TX_8X4, TX_8X16,
// 32x8 16x64, 64x16
TX_16X8, TX_16X32, TX_32X16,
#if CONFIG_EXT_PARTITION
// 32x128 128x32
TX_32X32, TX_32X32
#endif // CONFIG_EXT_PARTITION
};
static const TX_SIZE max_txsize_rect_lookup[BLOCK_SIZES_ALL] = {
// 2X2, 2X4, 4X2,
TX_4X4, TX_4X4, TX_4X4,
......@@ -677,11 +717,11 @@ static const TX_SIZE max_txsize_rect_lookup[BLOCK_SIZES_ALL] = {
TX_32X32, TX_32X32, TX_32X32,
#endif // CONFIG_EXT_PARTITION
#endif // CONFIG_TX64X64
#if CONFIG_RECT_TX_EXT
#if CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT
// 4x16, 16x4, 8x32
TX_4X16, TX_16X4, TX_8X32,
TX_4X16, TX_16X4, TX_8X16,
// 32x8
TX_32X8,
TX_16X8,
#else
// 4x16, 16x4, 8x32
TX_4X8, TX_8X4, TX_8X16,
......
av1/common/entropy.h
View file @ 0797a208
......@@ -330,7 +330,7 @@ static INLINE int get_entropy_context(TX_SIZE tx_size, const ENTROPY_CONTEXT *a,
left_ec = !!*(const uint64_t *)l;
break;
#endif // CONFIG_TX64X64
#if CONFIG_RECT_TX_EXT
#if CONFIG_RECT_TX_EXT || (CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT)
case TX_4X16:
above_ec = a[0] != 0;
left_ec = !!*(const uint32_t *)l;
......
av1/common/enums.h
View file @ 0797a208
......@@ -37,6 +37,10 @@ extern "C" {
#define MAX_SB_SIZE (1 << MAX_SB_SIZE_LOG2)
#define MAX_SB_SQUARE (MAX_SB_SIZE * MAX_SB_SIZE)
#if CONFIG_EXT_PARTITION_TYPES
#define USE_RECT_TX_EXT 0
#endif
// Min superblock size
#define MIN_SB_SIZE_LOG2 6
......
av1/common/idct.c
View file @ 0797a208
......@@ -1866,7 +1866,7 @@ static void inv_txfm_add_8x4(const tran_low_t *input, uint8_t *dest, int stride,
}
// These will be used by the masked-tx experiment in the future.
#if CONFIG_RECT_TX_EXT
#if CONFIG_RECT_TX_EXT || (CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT)
static void inv_txfm_add_4x16(const tran_low_t *input, uint8_t *dest,
int stride, const TxfmParam *txfm_param) {
av1_iht4x16_64_add(input, dest, stride, txfm_param);
......@@ -2375,7 +2375,7 @@ void av1_inv_txfm_add(const tran_low_t *input, uint8_t *dest, int stride,
// case.
inv_txfm_add_4x4(input, dest, stride, txfm_param);
break;
#if CONFIG_RECT_TX_EXT
#if CONFIG_RECT_TX_EXT || (CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT)
case TX_32X8: inv_txfm_add_32x8(input, dest, stride, txfm_param); break;
case TX_8X32: inv_txfm_add_8x32(input, dest, stride, txfm_param); break;
case TX_16X4: inv_txfm_add_16x4(input, dest, stride, txfm_param); break;
......
av1/common/reconintra.c
View file @ 0797a208
......@@ -2429,7 +2429,7 @@ void av1_predict_intra_block(const AV1_COMMON *cm, const MACROBLOCKD *xd,
const int block_width = block_size_wide[bsize];
const int block_height = block_size_high[bsize];
#if INTRA_USES_RECT_TRANSFORMS
const TX_SIZE tx_size = max_txsize_rect_lookup[bsize];
const TX_SIZE tx_size = get_max_rect_tx_size(bsize, 0);
assert(tx_size < TX_SIZES_ALL);
#else
const TX_SIZE tx_size = max_txsize_lookup[bsize];
......
av1/decoder/decodeframe.c
View file @ 0797a208
......@@ -350,31 +350,23 @@ static void decode_reconstruct_tx(AV1_COMMON *cm, MACROBLOCKD *const xd,
assert(IMPLIES(tx_size <= TX_4X4, sub_txs == tx_size));
assert(IMPLIES(tx_size > TX_4X4, sub_txs < tx_size));
#endif
const int bsl = tx_size_wide_unit[sub_txs];
int sub_step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];
int i;
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
const int sub_step = bsw * bsh;
assert(bsl > 0);
assert(bsw > 0 && bsh > 0);
for (i = 0; i < 4; ++i) {
#if CONFIG_RECT_TX_EXT
int is_wide_tx = tx_size_wide_unit[sub_txs] > tx_size_high_unit[sub_txs];
const int offsetr =
is_qttx ? (is_wide_tx ? i * tx_size_high_unit[sub_txs] : 0)
: blk_row + ((i >> 1) * bsl);
const int offsetc =
is_qttx ? (is_wide_tx ? 0 : i * tx_size_wide_unit[sub_txs])
: blk_col + (i & 0x01) * bsl;
#else
const int offsetr = blk_row + (i >> 1) * bsl;
const int offsetc = blk_col + (i & 0x01) * bsl;
#endif
for (int row = 0; row < tx_size_high_unit[tx_size]; row += bsh) {
for (int col = 0; col < tx_size_wide_unit[tx_size]; col += bsw) {
const int offsetr = blk_row + row;
const int offsetc = blk_col + col;
if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
decode_reconstruct_tx(cm, xd, r, mbmi, plane, plane_bsize, offsetr,
offsetc, block, sub_txs, eob_total);
block += sub_step;
decode_reconstruct_tx(cm, xd, r, mbmi, plane, plane_bsize, offsetr,
offsetc, block, sub_txs, eob_total);
block += sub_step;
}
}
}
}
......
av1/decoder/decodemv.c
View file @ 0797a208
......@@ -469,8 +469,8 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
if (is_split) {
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const int bsl = tx_size_wide_unit[sub_txs];
int i;
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
if (counts) ++counts->txfm_partition[ctx][1];
......@@ -487,12 +487,14 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
return;
}
assert(bsl > 0);
for (i = 0; i < 4; ++i) {
int offsetr = blk_row + (i >> 1) * bsl;
int offsetc = blk_col + (i & 0x01) * bsl;
read_tx_size_vartx(cm, xd, mbmi, counts, sub_txs, depth + 1, offsetr,
offsetc, r);
assert(bsw > 0 && bsh > 0);
for (int row = 0; row < tx_size_high_unit[tx_size]; row += bsh) {
for (int col = 0; col < tx_size_wide_unit[tx_size]; col += bsw) {
int offsetr = blk_row + row;
int offsetc = blk_col + col;
read_tx_size_vartx(cm, xd, mbmi, counts, sub_txs, depth + 1, offsetr,
offsetc, r);
}
}
} else {
int idx, idy;
......@@ -554,11 +556,11 @@ static TX_SIZE read_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd, int is_inter,
quarter_tx = 1;
}
return quarter_tx ? quarter_txsize_lookup[bsize]
: max_txsize_rect_lookup[bsize];
: get_max_rect_tx_size(bsize, is_inter);
}
#endif // CONFIG_RECT_TX_EXT
return max_txsize_rect_lookup[bsize];
return get_max_rect_tx_size(bsize, is_inter);
}
return coded_tx_size;
} else {
......@@ -566,7 +568,7 @@ static TX_SIZE read_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd, int is_inter,
}
} else {
assert(IMPLIES(tx_mode == ONLY_4X4, bsize == BLOCK_4X4));
return max_txsize_rect_lookup[bsize];
return get_max_rect_tx_size(bsize, is_inter);
}
}
......@@ -1086,7 +1088,7 @@ static void read_intrabc_info(AV1_COMMON *const cm, MACROBLOCKD *const xd,
int idx, idy;
if ((cm->tx_mode == TX_MODE_SELECT && block_signals_txsize(bsize) &&
!xd->lossless[mbmi->segment_id] && !mbmi->skip)) {
const TX_SIZE max_tx_size = max_txsize_rect_lookup[bsize];
const TX_SIZE max_tx_size = get_max_rect_tx_size(bsize, 0);
const int bh = tx_size_high_unit[max_tx_size];
const int bw = tx_size_wide_unit[max_tx_size];
mbmi->min_tx_size = TX_SIZES_ALL;
......@@ -2690,7 +2692,7 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi,
if (cm->tx_mode == TX_MODE_SELECT && block_signals_txsize(bsize) &&
!mbmi->skip && inter_block && !xd->lossless[mbmi->segment_id]) {
const TX_SIZE max_tx_size = max_txsize_rect_lookup[bsize];
const TX_SIZE max_tx_size = get_max_rect_tx_size(bsize, inter_block);
const int bh = tx_size_high_unit[max_tx_size];
const int bw = tx_size_wide_unit[max_tx_size];
const int width = block_size_wide[bsize] >> tx_size_wide_log2[0];
......
av1/encoder/bitstream.c
View file @ 0797a208
......@@ -279,8 +279,8 @@ static void write_tx_size_vartx(const AV1_COMMON *cm, MACROBLOCKD *xd,
// TODO(yuec): set correct txfm partition update for qttx
} else {
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const int bsl = tx_size_wide_unit[sub_txs];
int i;
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
#if CONFIG_NEW_MULTISYMBOL
aom_write_symbol(w, 1, ec_ctx->txfm_partition_cdf[ctx], 2);
......@@ -294,13 +294,14 @@ static void write_tx_size_vartx(const AV1_COMMON *cm, MACROBLOCKD *xd,
return;
}
assert(bsl > 0);
for (i = 0; i < 4; ++i) {
int offsetr = blk_row + (i >> 1) * bsl;
int offsetc = blk_col + (i & 0x01) * bsl;
write_tx_size_vartx(cm, xd, mbmi, sub_txs, depth + 1, offsetr, offsetc,
w);
}
assert(bsw > 0 && bsh > 0);
for (int row = 0; row < tx_size_high_unit[tx_size]; row += bsh)
for (int col = 0; col < tx_size_wide_unit[tx_size]; col += bsw) {
int offsetr = blk_row + row;
int offsetc = blk_col + col;
write_tx_size_vartx(cm, xd, mbmi, sub_txs, depth + 1, offsetr, offsetc,
w);
}
}
}
......@@ -654,21 +655,24 @@ static void pack_txb_tokens(aom_writer *w, AV1_COMMON *cm, MACROBLOCK *const x,
#endif
} else {
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const int bsl = tx_size_wide_unit[sub_txs];
int i;
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
assert(bsl > 0);
assert(bsw > 0 && bsh > 0);
for (i = 0; i < 4; ++i) {
const int offsetr = blk_row + (i >> 1) * bsl;
const int offsetc = blk_col + (i & 0x01) * bsl;
const int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];
for (int r = 0; r < tx_size_high_unit[tx_size]; r += bsh) {
for (int c = 0; c < tx_size_wide_unit[tx_size]; c += bsw) {
const int offsetr = blk_row + r;
const int offsetc = blk_col + c;
const int step = bsh * bsw;
if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
pack_txb_tokens(w, cm, x, tp, tok_end, xd, mbmi, plane, plane_bsize,
bit_depth, block, offsetr, offsetc, sub_txs, token_stats);
block += step;
pack_txb_tokens(w, cm, x, tp, tok_end, xd, mbmi, plane, plane_bsize,
bit_depth, block, offsetr, offsetc, sub_txs,
token_stats);
block += step;
}
}
}
}
......@@ -716,31 +720,23 @@ static void pack_txb_tokens(aom_writer *w, const TOKENEXTRA **tp,
#else
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
#endif
const int bsl = tx_size_wide_unit[sub_txs];
int i;
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
assert(bsl > 0);
assert(bsw > 0 && bsh > 0);
for (i = 0; i < 4; ++i) {
#if CONFIG_RECT_TX_EXT
int is_wide_tx = tx_size_wide_unit[sub_txs] > tx_size_high_unit[sub_txs];
const int offsetr =
is_qttx ? (is_wide_tx ? i * tx_size_high_unit[sub_txs] : 0)
: blk_row + (i >> 1) * bsl;
const int offsetc =
is_qttx ? (is_wide_tx ? 0 : i * tx_size_wide_unit[sub_txs])
: blk_col + (i & 0x01) * bsl;
#else
const int offsetr = blk_row + (i >> 1) * bsl;
const int offsetc = blk_col + (i & 0x01) * bsl;
#endif
const int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];
for (int r = 0; r < tx_size_high_unit[tx_size]; r += bsh) {
for (int c = 0; c < tx_size_wide_unit[tx_size]; c += bsw) {
const int offsetr = blk_row + r;
const int offsetc = blk_col + c;
const int step = bsh * bsw;
if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
pack_txb_tokens(w, tp, tok_end, xd, mbmi, plane, plane_bsize, bit_depth,
block, offsetr, offsetc, sub_txs, token_stats);
block += step;
pack_txb_tokens(w, tp, tok_end, xd, mbmi, plane, plane_bsize, bit_depth,
block, offsetr, offsetc, sub_txs, token_stats);
block += step;
}
}
}
}
......
av1/encoder/encodeframe.c
View file @ 0797a208
......@@ -4626,8 +4626,8 @@ static void update_txfm_count(MACROBLOCK *x, MACROBLOCKD *xd,
xd->left_txfm_context + blk_row, tx_size, tx_size);
} else {
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const int bs = tx_size_wide_unit[sub_txs];
int i;
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
++counts->txfm_partition[ctx][1];
#if CONFIG_NEW_MULTISYMBOL
......@@ -4644,11 +4644,14 @@ static void update_txfm_count(MACROBLOCK *x, MACROBLOCKD *xd,
return;
}
for (i = 0; i < 4; ++i) {
int offsetr = (i >> 1) * bs;
int offsetc = (i & 0x01) * bs;
update_txfm_count(x, xd, counts, sub_txs, depth + 1, blk_row + offsetr,
blk_col + offsetc, allow_update_cdf);
for (int row = 0; row < tx_size_high_unit[tx_size]; row += bsh) {
for (int col = 0; col < tx_size_wide_unit[tx_size]; col += bsw) {
int offsetr = row;
int offsetc = col;
update_txfm_count(x, xd, counts, sub_txs, depth + 1, blk_row + offsetr,
blk_col + offsetc, allow_update_cdf);
}
}
}
}
......@@ -4936,14 +4939,15 @@ static void encode_superblock(const AV1_COMP *const cpi, TileDataEnc *tile_data,
tx_partition_count_update(cm, x, bsize, mi_row, mi_col, td->counts,
tile_data->allow_update_cdf);
} else {
if (tx_size != max_txsize_rect_lookup[bsize]) ++x->txb_split_count;
if (tx_size != get_max_rect_tx_size(bsize, 0)) ++x->txb_split_count;
}
#if CONFIG_RECT_TX_EXT
if (is_quarter_tx_allowed(xd, mbmi, is_inter) &&
quarter_txsize_lookup[bsize] != max_txsize_rect_lookup[bsize] &&
quarter_txsize_lookup[bsize] !=
get_max_rect_tx_size(bsize, is_inter) &&
(mbmi->tx_size == quarter_txsize_lookup[bsize] ||
mbmi->tx_size == max_txsize_rect_lookup[bsize])) {
mbmi->tx_size == get_max_rect_tx_size(bsize, is_inter))) {
const int use_qttx = mbmi->tx_size == quarter_txsize_lookup[bsize];
++td->counts->quarter_tx_size[use_qttx];
#if CONFIG_NEW_MULTISYMBOL
......@@ -4975,7 +4979,8 @@ static void encode_superblock(const AV1_COMP *const cpi, TileDataEnc *tile_data,
mi_8x8[mis * j + i]->mbmi.tx_size = intra_tx_size;
mbmi->min_tx_size = get_min_tx_size(intra_tx_size);
if (intra_tx_size != max_txsize_rect_lookup[bsize]) ++x->txb_split_count;
if (intra_tx_size != get_max_rect_tx_size(bsize, is_inter))
++x->txb_split_count;
}
#if !CONFIG_TXK_SEL
......
av1/encoder/encodemb.c
View file @ 0797a208
......@@ -670,30 +670,22 @@ static void encode_block_inter(int plane, int block, int blk_row, int blk_col,
assert(IMPLIES(tx_size > TX_4X4, sub_txs < tx_size));
#endif
// This is the square transform block partition entry point.
int bsl = tx_size_wide_unit[sub_txs];
int i;
assert(bsl > 0);
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
const int step = bsh * bsw;
assert(bsw > 0 && bsh > 0);
for (i = 0; i < 4; ++i) {
#if CONFIG_RECT_TX_EXT
int is_wide_tx = tx_size_wide_unit[sub_txs] > tx_size_high_unit[sub_txs];
const int offsetr =
is_qttx ? (is_wide_tx ? i * tx_size_high_unit[sub_txs] : 0)
: blk_row + ((i >> 1) * bsl);
const int offsetc =
is_qttx ? (is_wide_tx ? 0 : i * tx_size_wide_unit[sub_txs])
: blk_col + ((i & 0x01) * bsl);
#else
const int offsetr = blk_row + ((i >> 1) * bsl);
const int offsetc = blk_col + ((i & 0x01) * bsl);
#endif
int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];
for (int row = 0; row < tx_size_high_unit[tx_size]; row += bsh) {
for (int col = 0; col < tx_size_wide_unit[tx_size]; col += bsw) {
const int offsetr = blk_row + row;
const int offsetc = blk_col + col;
if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
encode_block_inter(plane, block, offsetr, offsetc, plane_bsize, sub_txs,
arg);
block += step;
encode_block_inter(plane, block, offsetr, offsetc, plane_bsize, sub_txs,
arg);
block += step;
}
}
}
}
......
av1/encoder/hybrid_fwd_txfm.c
View file @ 0797a208
......@@ -145,7 +145,7 @@ static void fwd_txfm_64x32(const int16_t *src_diff, tran_low_t *coeff,
}
#endif // CONFIG_TX64X64
#if CONFIG_RECT_TX_EXT
#if CONFIG_RECT_TX_EXT || (CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT)
static void fwd_txfm_16x4(const int16_t *src_diff, tran_low_t *coeff,
int diff_stride, TxfmParam *txfm_param) {
av1_fht16x4(src_diff, coeff, diff_stride, txfm_param);
......@@ -503,7 +503,7 @@ void av1_fwd_txfm(const int16_t *src_diff, tran_low_t *coeff, int diff_stride,
fwd_txfm_32x16(src_diff, coeff, diff_stride, txfm_param);
break;
case TX_4X4: fwd_txfm_4x4(src_diff, coeff, diff_stride, txfm_param); break;
#if CONFIG_RECT_TX_EXT
#if CONFIG_RECT_TX_EXT || (CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT)
case TX_4X16:
fwd_txfm_4x16(src_diff, coeff, diff_stride, txfm_param);
break;
......
av1/encoder/rd.c
View file @ 0797a208
......@@ -897,7 +897,7 @@ static void get_entropy_contexts_plane(
for (i = 0; i < num_4x4_h; i += 4)
t_left[i] = !!*(const uint32_t *)&left[i];
break;
#if CONFIG_RECT_TX_EXT
#if CONFIG_RECT_TX_EXT || (CONFIG_EXT_PARTITION_TYPES && USE_RECT_TX_EXT)
case TX_4X16:
memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
for (i = 0; i < num_4x4_h; i += 4)
......
av1/encoder/rdopt.c
View file @ 0797a208
......@@ -2663,7 +2663,7 @@ static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi,
TX_TYPE tx_type;
for (tx_type = tx_start; tx_type < tx_end; ++tx_type) {
if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) continue;
const TX_SIZE rect_tx_size = max_txsize_rect_lookup[bs];
const TX_SIZE rect_tx_size = get_max_rect_tx_size(bs, is_inter);
RD_STATS this_rd_stats;
const TxSetType tx_set_type = get_ext_tx_set_type(
rect_tx_size, bs, is_inter, cm->reduced_tx_set_used);
......@@ -3890,7 +3890,6 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
int64_t this_rd = INT64_MAX;
ENTROPY_CONTEXT *pta = ta + blk_col;
ENTROPY_CONTEXT *ptl = tl + blk_row;
int i;
int ctx = txfm_partition_context(tx_above + blk_col, tx_left + blk_row,
mbmi->sb_type, tx_size);
int64_t sum_rd = INT64_MAX;
......@@ -3904,7 +3903,7 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
#if CONFIG_RECT_TX_EXT
TX_SIZE quarter_txsize = quarter_txsize_lookup[mbmi->sb_type];
int check_qttx = is_quarter_tx_allowed(xd, mbmi, is_inter_block(mbmi)) &&
tx_size == max_txsize_rect_lookup[mbmi->sb_type] &&
tx_size == get_max_rect_tx_size(mbmi->sb_type, 1) &&
quarter_txsize != tx_size;
int is_qttx_picked = 0;
int eobs_qttx[2] = { 0, 0 };
......@@ -4102,13 +4101,14 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
#endif // CONFIG_MRC_TX
) {
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const int bsl = tx_size_wide_unit[sub_txs];
int sub_step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
int sub_step = bsw * bsh;
RD_STATS this_rd_stats;
int this_cost_valid = 1;
int64_t tmp_rd = 0;
#if CONFIG_DIST_8X8
int sub8x8_eob[4];
int sub8x8_eob[4] = { 0, 0, 0, 0 };
#endif
sum_rd_stats.rate = x->txfm_partition_cost[ctx][1];
......@@ -4116,29 +4116,35 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
ref_best_rd = AOMMIN(this_rd, ref_best_rd);
for (i = 0; i < 4 && this_cost_valid; ++i) {
int offsetr = blk_row + (i >> 1) * bsl;
int offsetc = blk_col + (i & 0x01) * bsl;
for (int r = 0; r < tx_size_high_unit[tx_size]; r += bsh) {
for (int c = 0; c < tx_size_wide_unit[tx_size]; c += bsw) {
int offsetr = blk_row + r;
int offsetc = blk_col + c;
if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
select_tx_block(cpi, x, offsetr, offsetc, plane, block, sub_txs,
depth + 1, plane_bsize, ta, tl, tx_above, tx_left,
&this_rd_stats, ref_best_rd - tmp_rd, &this_cost_valid,
fast, 0);
select_tx_block(cpi, x, offsetr, offsetc, plane, block, sub_txs,
depth + 1, plane_bsize, ta, tl, tx_above, tx_left,
&this_rd_stats, ref_best_rd - tmp_rd, &this_cost_valid,
fast, 0);
#if CONFIG_DIST_8X8
if (x->using_dist_8x8 && plane == 0 && tx_size == TX_8X8) {
sub8x8_eob[i] = p->eobs[block];
}
if (!x->using_dist_8x8)
#endif
if (!this_cost_valid) break;
#if CONFIG_DIST_8X8
if (x->using_dist_8x8 && plane == 0 && tx_size == TX_8X8) {
sub8x8_eob[2 * (r / bsh) + (c / bsw)] = p->eobs[block];
}
#endif // CONFIG_DIST_8X8
av1_merge_rd_stats(&sum_rd_stats, &this_rd_stats);
av1_merge_rd_stats(&sum_rd_stats, &this_rd_stats);
tmp_rd = RDCOST(x->rdmult, sum_rd_stats.rate, sum_rd_stats.dist);
tmp_rd = RDCOST(x->rdmult, sum_rd_stats.rate, sum_rd_stats.dist);
#if CONFIG_DIST_8X8
if (!x->using_dist_8x8)
if (!x->using_dist_8x8)
#endif
if (this_rd < tmp_rd) break;
block += sub_step;
if (this_rd < tmp_rd) break;
block += sub_step;
}
}
#if CONFIG_DIST_8X8
if (x->using_dist_8x8 && this_cost_valid && plane == 0 &&
......@@ -4158,7 +4164,7 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
const int pred_idx = (blk_row * pred_stride + blk_col)
<< tx_size_wide_log2[0];
int16_t *pred = &pd->pred[pred_idx];
int j;
int i, j;