Skip to content
GitLab
Commit 3ebb0d00 authored by Debargha Mukherjee's avatar Debargha Mukherjee
Browse files

Correct/Cleanup min_tx_size computation 

Change-Id: I17fd041e9e1cead6b2b6339f0efa55c1ee9938af
parent e33fcbd9
Hide whitespace changes
Inline Side-by-side
av1/common/blockd.h
View file @ 3ebb0d00
......@@ -925,6 +925,9 @@ static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
return LUT[bsize];
}
#define TXSIZEMAX(t1, t2) (tx_size_2d[(t1)] >= tx_size_2d[(t2)] ? (t1) : (t2))
#define TXSIZEMIN(t1, t2) (tx_size_2d[(t1)] <= tx_size_2d[(t2)] ? (t1) : (t2))
static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
const MB_MODE_INFO *mbmi) {
return is_rect_tx_allowed_bsize(mbmi->sb_type) &&
......
av1/common/common_data.h
View file @ 3ebb0d00
......@@ -845,20 +845,20 @@ static const int tx_size_high_log2[TX_SIZES_ALL] = {
#define TX_UNIT_WIDE_LOG2 (MI_SIZE_LOG2 - tx_size_wide_log2[0])
#define TX_UNIT_HIGH_LOG2 (MI_SIZE_LOG2 - tx_size_high_log2[0])
static const int tx_size_2d[TX_SIZES_ALL] = {
16, 64, 256, 1024,
static const int tx_size_2d[TX_SIZES_ALL + 1] = { 16, 64, 256, 1024,
#if CONFIG_TX64X64
4096,
4096,
#endif // CONFIG_TX64X64
32, 32, 128, 128, 512, 512,
32, 32, 128, 128,
512, 512,
#if CONFIG_TX64X64
2048, 2048,
2048, 2048,
#endif // CONFIG_TX64X64
64, 64, 256, 256,
64, 64, 256, 256,
#if CONFIG_TX64X64
1024, 1024,
1024, 1024,
#endif // CONFIG_TX64X64
};
32767 };
static const BLOCK_SIZE txsize_to_bsize[TX_SIZES_ALL] = {
BLOCK_4X4, // TX_4X4
......
av1/common/onyxc_int.h
View file @ 3ebb0d00
......@@ -1162,10 +1162,7 @@ static INLINE void av1_zero_left_context(MACROBLOCKD *const xd) {
#if defined(__GNUC__) && __GNUC__ >= 4
#pragma GCC diagnostic ignored "-Warray-bounds"
#endif
static INLINE TX_SIZE get_min_tx_size(TX_SIZE tx_size) {
assert(tx_size < TX_SIZES_ALL);
return txsize_sqr_map[tx_size];
}
#if defined(__GNUC__) && __GNUC__ >= 4
#pragma GCC diagnostic warning "-Warray-bounds"
#endif
......
av1/decoder/decodemv.c
View file @ 3ebb0d00
......@@ -436,7 +436,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
for (idx = 0; idx < AOMMAX(1, tx_size_wide_unit[tx_size] / 2); ++idx)
inter_tx_size[idy][idx] = tx_size;
mbmi->tx_size = tx_size;
mbmi->min_tx_size = AOMMIN(mbmi->min_tx_size, get_min_tx_size(tx_size));
mbmi->min_tx_size = TXSIZEMIN(mbmi->min_tx_size, tx_size);
txfm_partition_update(xd->above_txfm_context + blk_col,
xd->left_txfm_context + blk_row, tx_size, tx_size);
return;
......@@ -458,7 +458,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
for (idx = 0; idx < AOMMAX(1, tx_size_wide_unit[tx_size] / 2); ++idx)
inter_tx_size[idy][idx] = inter_tx_size[0][0];
mbmi->tx_size = sub_txs;
mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
mbmi->min_tx_size = mbmi->tx_size;
txfm_partition_update(xd->above_txfm_context + blk_col,
xd->left_txfm_context + blk_row, sub_txs, tx_size);
return;
......@@ -480,7 +480,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
for (idx = 0; idx < AOMMAX(1, tx_size_wide_unit[tx_size] / 2); ++idx)
inter_tx_size[idy][idx] = tx_size;
mbmi->tx_size = tx_size;
mbmi->min_tx_size = AOMMIN(mbmi->min_tx_size, get_min_tx_size(tx_size));
mbmi->min_tx_size = TXSIZEMIN(mbmi->min_tx_size, tx_size);
if (counts) ++counts->txfm_partition[ctx][0];
txfm_partition_update(xd->above_txfm_context + blk_col,
xd->left_txfm_context + blk_row, tx_size, tx_size);
......@@ -1045,7 +1045,7 @@ static void read_intrabc_info(AV1_COMMON *const cm, MACROBLOCKD *const xd,
for (int idy = 0; idy < height; ++idy)
for (int idx = 0; idx < width; ++idx)
mbmi->inter_tx_size[idy >> 1][idx >> 1] = mbmi->tx_size;
mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
mbmi->min_tx_size = mbmi->tx_size;
set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, mbmi->skip, xd);
}
mbmi->mode = mbmi->uv_mode = UV_DC_PRED;
......@@ -2419,7 +2419,7 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi,
for (int idx = 0; idx < width; ++idx)
mbmi->inter_tx_size[idy >> 1][idx >> 1] = mbmi->tx_size;
}
mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
mbmi->min_tx_size = mbmi->tx_size;
set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, mbmi->skip, xd);
}
......
av1/decoder/decodetxb.c
View file @ 3ebb0d00
......@@ -109,8 +109,7 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
(void)blk_row;
(void)blk_col;
#if CONFIG_TXK_SEL
av1_read_tx_type(cm, xd, blk_row, blk_col, plane, get_min_tx_size(tx_size),
r);
av1_read_tx_type(cm, xd, blk_row, blk_col, plane, tx_size, r);
#endif
const TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
......
av1/encoder/bitstream.c
View file @ 3ebb0d00
......@@ -1080,7 +1080,7 @@ void av1_write_tx_type(const AV1_COMMON *const cm, const MACROBLOCKD *xd,
const TX_SIZE mtx_size =
get_max_rect_tx_size(xd->mi[0]->mbmi.sb_type, is_inter);
const TX_SIZE tx_size =
is_inter ? AOMMAX(sub_tx_size_map[1][mtx_size], mbmi->min_tx_size)
is_inter ? TXSIZEMAX(sub_tx_size_map[1][mtx_size], mbmi->min_tx_size)
: mbmi->tx_size;
#endif // !CONFIG_TXK_SEL
FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
......@@ -1095,7 +1095,7 @@ void av1_write_tx_type(const AV1_COMMON *const cm, const MACROBLOCKD *xd,
#endif
if (!FIXED_TX_TYPE) {
const TX_SIZE square_tx_size = get_min_tx_size(tx_size);
const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
const BLOCK_SIZE bsize = mbmi->sb_type;
if (get_ext_tx_types(tx_size, bsize, is_inter, cm->reduced_tx_set_used) >
1 &&
......
av1/encoder/encodeframe.c
View file @ 3ebb0d00
......@@ -393,7 +393,7 @@ static void reset_tx_size(MACROBLOCKD *xd, MB_MODE_INFO *mbmi,
BLOCK_SIZE bsize = mbmi->sb_type;
TX_SIZE min_tx_size =
depth_to_tx_size(MAX_TX_DEPTH, bsize, is_inter_block(mbmi));
mbmi->tx_size = (TX_SIZE)AOMMAX(mbmi->tx_size, min_tx_size);
mbmi->tx_size = (TX_SIZE)TXSIZEMAX(mbmi->tx_size, min_tx_size);
}
}
......@@ -4525,7 +4525,7 @@ static void encode_superblock(const AV1_COMP *const cpi, TileDataEnc *tile_data,
}
}
mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
mbmi->min_tx_size = mbmi->tx_size;
#if CONFIG_LV_MAP
av1_update_txb_context(cpi, td, dry_run, block_size, rate, mi_row, mi_col,
tile_data->allow_update_cdf);
......@@ -4572,7 +4572,7 @@ static void encode_superblock(const AV1_COMP *const cpi, TileDataEnc *tile_data,
#endif
av1_encode_sb((AV1_COMMON *)cm, x, block_size, mi_row, mi_col, dry_run);
if (mbmi->skip) mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
if (mbmi->skip) mbmi->min_tx_size = mbmi->tx_size;
av1_tokenize_sb_vartx(cpi, td, t, dry_run, mi_row, mi_col, block_size, rate,
tile_data->allow_update_cdf);
}
......@@ -4612,7 +4612,7 @@ static void encode_superblock(const AV1_COMP *const cpi, TileDataEnc *tile_data,
if (mi_col + i < cm->mi_cols && mi_row + j < cm->mi_rows)
mi_8x8[mis * j + i]->mbmi.tx_size = intra_tx_size;
mbmi->min_tx_size = get_min_tx_size(intra_tx_size);
mbmi->min_tx_size = intra_tx_size;
if (intra_tx_size != get_max_rect_tx_size(bsize, is_inter))
++x->txb_split_count;
}
......
av1/encoder/encodetxb.c
View file @ 3ebb0d00
......@@ -350,8 +350,7 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd,
av1_txb_init_levels(tcoeff, width, height, levels);
#if CONFIG_TXK_SEL
av1_write_tx_type(cm, xd, blk_row, blk_col, plane, get_min_tx_size(tx_size),
w);
av1_write_tx_type(cm, xd, blk_row, blk_col, plane, tx_size, w);
#endif
int eob_extra, dummy;
......@@ -2149,8 +2148,7 @@ void av1_update_and_record_txb_context(int plane, int block, int blk_row,
#if CONFIG_TXK_SEL
av1_update_tx_type_count(cm, xd, blk_row, blk_col, plane, mbmi->sb_type,
get_min_tx_size(tx_size), td->counts,
allow_update_cdf);
tx_size, td->counts, allow_update_cdf);
#endif
unsigned int(*nz_map_count)[SIG_COEF_CONTEXTS][2] =
......
av1/encoder/rdopt.c
View file @ 3ebb0d00
......@@ -2347,7 +2347,7 @@ int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x,
TX_SIZE tx_size, TX_TYPE tx_type) {
if (plane > 0) return 0;
const TX_SIZE square_tx_size = get_min_tx_size(tx_size);
const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
const int is_inter = is_inter_block(mbmi);
......@@ -2500,7 +2500,7 @@ static void choose_largest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x,
av1_invalid_rd_stats(rd_stats);
mbmi->tx_size = tx_size_from_tx_mode(bs, cm->tx_mode, is_inter);
mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
mbmi->min_tx_size = mbmi->tx_size;
const TxSetType tx_set_type =
get_ext_tx_set_type(mbmi->tx_size, bs, is_inter, cm->reduced_tx_set_used);
......@@ -2539,13 +2539,15 @@ static void choose_largest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x,
mbmi->tx_size, cpi->sf.use_fast_coef_costing);
if (this_rd_stats.rate == INT_MAX) continue;
av1_tx_type_cost(cm, x, xd, bs, AOM_PLANE_Y, mbmi->tx_size, tx_type);
if (this_rd_stats.skip)
if (this_rd_stats.skip) {
this_rd = RDCOST(x->rdmult, s1, this_rd_stats.sse);
else
} else {
this_rd_stats.rate += av1_tx_type_cost(cm, x, xd, bs, AOM_PLANE_Y,
mbmi->tx_size, tx_type);
this_rd =
RDCOST(x->rdmult, this_rd_stats.rate + s0, this_rd_stats.dist);
}
if (is_inter_block(mbmi) && !xd->lossless[mbmi->segment_id] &&
!this_rd_stats.skip)
this_rd = AOMMIN(this_rd, RDCOST(x->rdmult, s1, this_rd_stats.sse));
......@@ -2556,7 +2558,6 @@ static void choose_largest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x,
*rd_stats = this_rd_stats;
}
}
} else {
mbmi->tx_type = DCT_DCT;
txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, AOM_PLANE_Y, bs,
......@@ -2573,7 +2574,7 @@ static void choose_smallest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x,
mbmi->tx_size = TX_4X4;
mbmi->tx_type = DCT_DCT;
mbmi->min_tx_size = get_min_tx_size(TX_4X4);
mbmi->min_tx_size = TX_4X4;
txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, 0, bs, mbmi->tx_size,
cpi->sf.use_fast_coef_costing);
......@@ -2673,7 +2674,7 @@ static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi,
(TX_SIZE_W_MIN * TX_SIZE_H_MIN));
#endif
mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
mbmi->min_tx_size = mbmi->tx_size;
}
static void super_block_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
......@@ -4235,11 +4236,11 @@ static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x,
tx_split_prune_flag, rd_info_tree);
if (rd_stats->rate == INT_MAX) return INT64_MAX;
mbmi->min_tx_size = get_min_tx_size(mbmi->inter_tx_size[0][0]);
mbmi->min_tx_size = mbmi->inter_tx_size[0][0];
for (row = 0; row < max_blocks_high / 2; ++row)
for (col = 0; col < max_blocks_wide / 2; ++col)
mbmi->min_tx_size = AOMMIN(
mbmi->min_tx_size, get_min_tx_size(mbmi->inter_tx_size[row][col]));
mbmi->min_tx_size =
TXSIZEMIN(mbmi->min_tx_size, mbmi->inter_tx_size[row][col]);
if (fast) {
// Do a better (non-fast) search with tx sizes already decided.
......@@ -4779,7 +4780,7 @@ static void set_skip_flag(const AV1_COMP *cpi, MACROBLOCK *x,
for (int idx = 0; idx < xd->n8_w; ++idx)
mbmi->inter_tx_size[idy][idx] = tx_size;
mbmi->tx_size = tx_size;
mbmi->min_tx_size = get_min_tx_size(tx_size);
mbmi->min_tx_size = tx_size;
memset(x->blk_skip[0], 1, sizeof(uint8_t) * n4);
rd_stats->skip = 1;
......@@ -4807,8 +4808,8 @@ static void set_skip_flag(const AV1_COMP *cpi, MACROBLOCK *x,
}
#if !CONFIG_TXK_SEL
const AV1_COMMON *cm = &cpi->common;
const int ext_tx_set = get_ext_tx_set(max_txsize_lookup[bsize], bsize, 1,
cm->reduced_tx_set_used);
const int ext_tx_set =
get_ext_tx_set(mbmi->min_tx_size, bsize, 1, cm->reduced_tx_set_used);
if (get_ext_tx_types(mbmi->min_tx_size, bsize, 1, cm->reduced_tx_set_used) >
1 &&
!xd->lossless[xd->mi[0]->mbmi.segment_id]) {
......@@ -10695,7 +10696,7 @@ PALETTE_EXIT:
for (int idx = 0; idx < width; ++idx)
best_mbmode.inter_tx_size[idy >> 1][idx >> 1] = best_mbmode.tx_size;
}
best_mbmode.min_tx_size = get_min_tx_size(best_mbmode.tx_size);
best_mbmode.min_tx_size = best_mbmode.tx_size;
set_txfm_ctxs(best_mbmode.tx_size, xd->n8_w, xd->n8_h, best_mbmode.skip,
xd);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment