Commit fcff0b25 authored by Rupert Swarbrick's avatar Rupert Swarbrick

Define block_signals_txsize function

This returns true if a block signals tx_size in the stream and uses it
in the bitstream writing code and the decoder.

Note that we can't quite use it in pack_inter_mode_mvs when
CONFIG_VAR_TX && !CONFIG_RECT_TX but I've switched the code to using
it the rest of the time since rect-tx is adopted and eventually the
other code path should be deleted.

Also use the helper function in tx_size_cost in rdopt.c, where the
test was wrong and caused underestimates of block
costs. (Specifically, the code that subtracts tx_size_cost from
this_rate_tokenonly in rd_pick_intra_sby_mode ended up subtracting
zero for a 4x8 block).

The behaviour of the decoder should be unchanged. The only change in
the encoder's behaviour should be in tx_size_cost where it should now
match the rest of the code.

Change-Id: I97236c9ce444993afe01ac5c6f4a0bb9e5049217
parent a3c5b9da
......@@ -892,6 +892,14 @@ static INLINE int supertx_enabled(const MB_MODE_INFO *mbmi) {
static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; }
#endif // CONFIG_RECT_TX
static INLINE int block_signals_txsize(BLOCK_SIZE bsize) {
#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
return bsize > BLOCK_4X4;
#else
return bsize >= BLOCK_8X8;
#endif
}
#if CONFIG_MRC_TX
#define USE_MRC_INTRA 0
#define USE_MRC_INTER 1
......
......@@ -520,12 +520,8 @@ static TX_SIZE read_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd, int is_inter,
const TX_MODE tx_mode = cm->tx_mode;
const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type;
if (xd->lossless[xd->mi[0]->mbmi.segment_id]) return TX_4X4;
#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
if (bsize > BLOCK_4X4)
#else
if (bsize >= BLOCK_8X8)
#endif // CONFIG_CB4X4 && CONFIG_VAR_TX
{
if (block_signals_txsize(bsize)) {
if ((!is_inter || allow_select_inter) && tx_mode == TX_MODE_SELECT) {
const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
: intra_tx_size_cat_lookup[bsize];
......
......@@ -348,12 +348,7 @@ static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
const BLOCK_SIZE bsize = mbmi->sb_type;
FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
(void)cm;
// For sub8x8 blocks the tx_size symbol does not need to be sent
#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
if (bsize > BLOCK_4X4) {
#else
if (bsize >= BLOCK_8X8) {
#endif
if (block_signals_txsize(bsize)) {
const TX_SIZE tx_size = mbmi->tx_size;
const int is_inter = is_inter_block(mbmi);
const TX_SIZE tx_size_ctx = get_tx_size_context(xd);
......@@ -1754,14 +1749,10 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const int mi_row,
write_is_inter(cm, xd, mbmi->segment_id, w, is_inter);
if (cm->tx_mode == TX_MODE_SELECT &&
#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
#if CONFIG_RECT_TX
bsize > BLOCK_4X4 &&
#else
#if CONFIG_CB4X4 && CONFIG_VAR_TX && !CONFIG_RECT_TX
(bsize >= BLOCK_8X8 || (bsize > BLOCK_4X4 && is_inter)) &&
#endif // CONFIG_RECT_TX
#else
bsize >= BLOCK_8X8 &&
block_signals_txsize(bsize) &&
#endif
#if CONFIG_SUPERTX
!supertx_enabled &&
......@@ -2157,15 +2148,7 @@ static void write_mb_modes_kf(AV1_COMMON *cm, MACROBLOCKD *xd,
}
int enable_tx_size = cm->tx_mode == TX_MODE_SELECT &&
#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_RECT_TX)
#if CONFIG_RECT_TX
bsize > BLOCK_4X4 &&
#else
bsize >= BLOCK_8X8 &&
#endif // CONFIG_RECT_TX
#else
bsize >= BLOCK_8X8 &&
#endif
block_signals_txsize(bsize) &&
!xd->lossless[mbmi->segment_id];
#if CONFIG_INTRABC
......
......@@ -2219,16 +2219,7 @@ static TX_SIZE tx_size_cost(const AV1_COMP *const cpi,
const MACROBLOCKD *const xd = &x->e_mbd;
const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
const int tx_select = cm->tx_mode == TX_MODE_SELECT &&
#if CONFIG_EXT_PARTITION_TYPES
// Currently these block shapes can only use 4x4
// transforms
mbmi->sb_type != BLOCK_4X16 &&
mbmi->sb_type != BLOCK_16X4 &&
#endif
mbmi->sb_type >= BLOCK_8X8;
if (tx_select) {
if (cm->tx_mode == TX_MODE_SELECT && block_signals_txsize(mbmi->sb_type)) {
const int is_inter = is_inter_block(mbmi);
const TX_SIZE tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
: intra_tx_size_cat_lookup[bsize];
......@@ -3060,7 +3051,8 @@ static int rd_pick_palette_intra_sby(const AV1_COMP *const cpi, MACROBLOCK *x,
if (tokenonly_rd_stats.rate == INT_MAX) continue;
this_rate = tokenonly_rd_stats.rate + palette_mode_cost;
this_rd = RDCOST(x->rdmult, this_rate, tokenonly_rd_stats.dist);
if (!xd->lossless[mbmi->segment_id] && mbmi->sb_type >= BLOCK_8X8) {
if (!xd->lossless[mbmi->segment_id] &&
block_signals_txsize(mbmi->sb_type)) {
tokenonly_rd_stats.rate -= tx_size_cost(cpi, x, bsize, mbmi->tx_size);
}
if (this_rd < *best_rd) {
......@@ -4212,7 +4204,8 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
this_rate = this_rate_tokenonly + bmode_costs[mbmi->mode];
if (!xd->lossless[mbmi->segment_id] && mbmi->sb_type >= BLOCK_8X8) {
if (!xd->lossless[mbmi->segment_id] &&
block_signals_txsize(mbmi->sb_type)) {
// super_block_yrd above includes the cost of the tx_size in the
// tokenonly rate, but for intra blocks, tx_size is always coded
// (prediction granularity), so we account for it in the full rate,
......@@ -10867,7 +10860,7 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
av1_default_palette_y_mode_prob[bsize - BLOCK_8X8][palette_ctx], 0);
}
if (!xd->lossless[mbmi->segment_id] && bsize >= BLOCK_8X8) {
if (!xd->lossless[mbmi->segment_id] && block_signals_txsize(bsize)) {
// super_block_yrd above includes the cost of the tx_size in the
// tokenonly rate, but for intra blocks, tx_size is always coded
// (prediction granularity), so we account for it in the full rate,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment