Commit d2cfbefb authored by Debargha Mukherjee's avatar Debargha Mukherjee

Fixes to make 4:1 rectangular intra work correctly

This patch fixes and enables rectangular intra transform
sizes for 4:1 partitions (that were turned off before).
4:1 partitions can now use rectangular intra predictions with
2:1 rectangular transform sizes.
BDRATE lowres (single keyframe): -0.612%

Change-Id: I6f062f7c08aae8eeb0a55d31e792c8f7e3f302a2
parent f03f543d
......@@ -900,15 +900,15 @@ static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
0, // BLOCK_128X64
0, // BLOCK_128X128
#endif // CONFIG_EXT_PARTITION
0, // BLOCK_4X16
0, // BLOCK_16X4
0, // BLOCK_8X32
0, // BLOCK_32X8
0, // BLOCK_16X64
0, // BLOCK_64X16
1, // BLOCK_4X16
1, // BLOCK_16X4
1, // BLOCK_8X32
1, // BLOCK_32X8
1, // BLOCK_16X64
1, // BLOCK_64X16
#if CONFIG_EXT_PARTITION
0, // BLOCK_32X128
0, // BLOCK_128X32
1, // BLOCK_32X128
1, // BLOCK_128X32
#endif // CONFIG_EXT_PARTITION
};
......@@ -1075,15 +1075,21 @@ static INLINE TX_TYPE av1_get_tx_type(PLANE_TYPE plane_type,
void av1_setup_block_planes(MACROBLOCKD *xd, int ss_x, int ss_y);
static INLINE int tx_size_cat_to_max_depth(int tx_size_cat) {
static INLINE int bsize_to_max_depth(BLOCK_SIZE bsize) {
const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize];
return AOMMIN(tx_size_cat + 1, MAX_TX_DEPTH);
}
static INLINE int tx_size_to_depth(TX_SIZE tx_size, int tx_size_cat) {
return (int)(tx_size_cat + 1 - (int)tx_size);
static INLINE int tx_size_to_depth(TX_SIZE tx_size, BLOCK_SIZE bsize) {
if (tx_size == max_txsize_rect_intra_lookup[bsize]) return 0;
const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize];
const TX_SIZE coded_tx_size = txsize_sqr_map[tx_size];
return (int)(tx_size_cat + 1 - (int)coded_tx_size);
}
static INLINE TX_SIZE depth_to_tx_size(int depth, int tx_size_cat) {
static INLINE TX_SIZE depth_to_tx_size(int depth, BLOCK_SIZE bsize) {
if (depth == 0) return max_txsize_rect_intra_lookup[bsize];
const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize];
assert(tx_size_cat + 1 - depth >= 0 && tx_size_cat + 1 - depth < TX_SIZES);
return (TX_SIZE)(tx_size_cat + 1 - depth);
}
......
......@@ -803,12 +803,17 @@ static const int32_t intra_tx_size_cat_lookup[BLOCK_SIZES_ALL] = {
// TODO(david.barker): Change these if we support rectangular transforms
// for 4:1 shaped partitions
// 4x16, 16x4, 8x32
TX_8X8 - TX_8X8, TX_8X8 - TX_8X8, TX_8X8 - TX_8X8,
TX_8X8 - TX_8X8, TX_8X8 - TX_8X8, TX_16X16 - TX_8X8,
// 32x8, 16x64, 64x16
TX_8X8 - TX_8X8, TX_16X16 - TX_8X8, TX_16X16 - TX_8X8,
TX_16X16 - TX_8X8, TX_32X32 - TX_8X8, TX_32X32 - TX_8X8,
#if CONFIG_EXT_PARTITION
#if CONFIG_TX64X64
// 32x128, 128x32
TX_64X64 - TX_8X8, TX_64X64 - TX_8X8
#else
// 32x128, 128x32
TX_32X32 - TX_8X8, TX_32X32 - TX_8X8
#endif // CONFIG_TX64X64
#endif // CONFIG_EXT_PARTITION
};
......
......@@ -5396,10 +5396,8 @@ const SCAN_ORDER av1_intra_scan_orders[TX_SIZES_ALL][TX_TYPES] = {
// TX_4X16
{ default_scan_4x16, av1_default_iscan_4x16,
default_scan_4x16_neighbors },
{ default_scan_4x16, av1_default_iscan_4x16,
default_scan_4x16_neighbors },
{ default_scan_4x16, av1_default_iscan_4x16,
default_scan_4x16_neighbors },
{ mrow_scan_4x16, av1_mrow_iscan_4x16, mrow_scan_4x16_neighbors },
{ mcol_scan_4x16, av1_mcol_iscan_4x16, mcol_scan_4x16_neighbors },
{ default_scan_4x16, av1_default_iscan_4x16,
default_scan_4x16_neighbors },
{ default_scan_4x16, av1_default_iscan_4x16,
......@@ -5424,10 +5422,8 @@ const SCAN_ORDER av1_intra_scan_orders[TX_SIZES_ALL][TX_TYPES] = {
// TX_16X4
{ default_scan_16x4, av1_default_iscan_16x4,
default_scan_16x4_neighbors },
{ default_scan_16x4, av1_default_iscan_16x4,
default_scan_16x4_neighbors },
{ default_scan_16x4, av1_default_iscan_16x4,
default_scan_16x4_neighbors },
{ mrow_scan_16x4, av1_mrow_iscan_16x4, mrow_scan_16x4_neighbors },
{ mcol_scan_16x4, av1_mcol_iscan_16x4, mcol_scan_16x4_neighbors },
{ default_scan_16x4, av1_default_iscan_16x4,
default_scan_16x4_neighbors },
{ default_scan_16x4, av1_default_iscan_16x4,
......@@ -5452,10 +5448,8 @@ const SCAN_ORDER av1_intra_scan_orders[TX_SIZES_ALL][TX_TYPES] = {
// TX_8X32
{ default_scan_8x32, av1_default_iscan_8x32,
default_scan_8x32_neighbors },
{ default_scan_8x32, av1_default_iscan_8x32,
default_scan_8x32_neighbors },
{ default_scan_8x32, av1_default_iscan_8x32,
default_scan_8x32_neighbors },
{ mrow_scan_8x32, av1_mrow_iscan_8x32, mrow_scan_8x32_neighbors },
{ mcol_scan_8x32, av1_mcol_iscan_8x32, mcol_scan_8x32_neighbors },
{ default_scan_8x32, av1_default_iscan_8x32,
default_scan_8x32_neighbors },
{ default_scan_8x32, av1_default_iscan_8x32,
......@@ -5480,10 +5474,8 @@ const SCAN_ORDER av1_intra_scan_orders[TX_SIZES_ALL][TX_TYPES] = {
// TX_32X8
{ default_scan_32x8, av1_default_iscan_32x8,
default_scan_32x8_neighbors },
{ default_scan_32x8, av1_default_iscan_32x8,
default_scan_32x8_neighbors },
{ default_scan_32x8, av1_default_iscan_32x8,
default_scan_32x8_neighbors },
{ mrow_scan_32x8, av1_mrow_iscan_32x8, mrow_scan_32x8_neighbors },
{ mcol_scan_32x8, av1_mcol_iscan_32x8, mcol_scan_32x8_neighbors },
{ default_scan_32x8, av1_default_iscan_32x8,
default_scan_32x8_neighbors },
{ default_scan_32x8, av1_default_iscan_32x8,
......
......@@ -498,8 +498,13 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
}
static TX_SIZE read_selected_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd,
int32_t tx_size_cat, aom_reader *r) {
const int max_depths = tx_size_cat_to_max_depth(tx_size_cat);
int is_inter, aom_reader *r) {
// TODO(debargha): Clean up the logic here. This function should only
// be called for intra.
const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type;
const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
: intra_tx_size_cat_lookup[bsize];
const int max_depths = bsize_to_max_depth(bsize);
const int ctx = get_tx_size_context(xd);
FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
(void)cm;
......@@ -507,8 +512,7 @@ static TX_SIZE read_selected_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd,
const int depth = aom_read_symbol(r, ec_ctx->tx_size_cdf[tx_size_cat][ctx],
max_depths + 1, ACCT_STR);
assert(depth >= 0 && depth <= max_depths);
const TX_SIZE tx_size = depth_to_tx_size(depth, tx_size_cat);
assert(!is_rect_tx(tx_size));
const TX_SIZE tx_size = depth_to_tx_size(depth, bsize);
return tx_size;
}
......@@ -520,14 +524,7 @@ static TX_SIZE read_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd, int is_inter,
if (block_signals_txsize(bsize)) {
if ((!is_inter || allow_select_inter) && tx_mode == TX_MODE_SELECT) {
const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
: intra_tx_size_cat_lookup[bsize];
const TX_SIZE coded_tx_size =
read_selected_tx_size(cm, xd, tx_size_cat, r);
if (coded_tx_size > max_txsize_lookup[bsize]) {
assert(coded_tx_size == max_txsize_lookup[bsize] + 1);
return get_max_rect_tx_size(bsize, is_inter);
}
const TX_SIZE coded_tx_size = read_selected_tx_size(cm, xd, is_inter, r);
return coded_tx_size;
} else {
return tx_size_from_tx_mode(bsize, tx_mode, is_inter);
......
......@@ -257,13 +257,10 @@ static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
const TX_SIZE tx_size = mbmi->tx_size;
const int tx_size_ctx = get_tx_size_context(xd);
const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize];
const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
const int depth = tx_size_to_depth(coded_tx_size, tx_size_cat);
const int max_depths = tx_size_cat_to_max_depth(tx_size_cat);
const int depth = tx_size_to_depth(tx_size, bsize);
const int max_depths = bsize_to_max_depth(bsize);
assert(coded_tx_size <= tx_size_cat + 1);
assert(depth >= 0 && depth <= max_depths);
assert(!is_inter_block(mbmi));
assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
......
......@@ -4506,9 +4506,8 @@ static void sum_intra_stats(FRAME_COUNTS *counts, MACROBLOCKD *xd,
const TX_SIZE tx_size = mbmi->tx_size;
const int tx_size_ctx = get_tx_size_context(xd);
const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize];
const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
const int depth = tx_size_to_depth(coded_tx_size, tx_size_cat);
const int max_depths = tx_size_cat_to_max_depth(tx_size_cat);
const int depth = tx_size_to_depth(tx_size, bsize);
const int max_depths = bsize_to_max_depth(bsize);
update_cdf(fc->tx_size_cdf[tx_size_cat][tx_size_ctx], depth,
max_depths + 1);
}
......
......@@ -2153,6 +2153,7 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
const PLANE_TYPE plane_type = get_plane_type(plane);
const TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, block, tx_size);
const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, mbmi);
this_rd_stats.rate =
av1_cost_coeffs(cpi, x, plane, blk_row, blk_col, block, tx_size,
......@@ -2326,8 +2327,7 @@ static int tx_size_cost(const AV1_COMMON *const cm, const MACROBLOCK *const x,
const int is_inter = is_inter_block(mbmi);
const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
: intra_tx_size_cat_lookup[bsize];
const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
const int depth = tx_size_to_depth(coded_tx_size, tx_size_cat);
const int depth = tx_size_to_depth(tx_size, bsize);
const int tx_size_ctx = get_tx_size_context(xd);
int r_tx_size = x->tx_size_cost[tx_size_cat][tx_size_ctx][depth];
return r_tx_size;
......@@ -3627,7 +3627,8 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
tx_size, a, l, 0, rd_stats);
return;
#endif
// This function is used only for inter
assert(is_inter_block(&xd->mi[0]->mbmi));
int64_t tmp;
tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
PLANE_TYPE plane_type = get_plane_type(plane);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment