Commit a1e48dcc authored by Yue Chen's avatar Yue Chen

Make RECT_TX(>=8x8) work with VAR_TX

Bitstream syntax:
For a rectangular inter block, 'rect_tx' flag is sent to indicate if
the biggest rect tx is used. If no, continue to decode regular
recursive tx partition.

Change-Id: I127e35cc619b65acb5e9a0717f399cdcdb73fbf0
parent 5ebdf40d
......@@ -857,6 +857,12 @@ static const aom_prob
},
};
#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_VAR_TX
// the probability of (0) using recursive square tx partition vs.
// (1) biggest rect tx for 4X8-8X4/8X16-16X8/16X32-32X16 blocks
static const aom_prob default_rect_tx_prob[TX_SIZES - 1] = { 192, 192, 192 };
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_VAR_TX
int av1_get_palette_color_context(const uint8_t *color_map, int cols, int r,
int c, int n, int *color_order) {
int i, j, max, max_idx, temp;
......@@ -1298,6 +1304,9 @@ static void init_mode_probs(FRAME_CONTEXT *fc) {
av1_copy(fc->tx_size_probs, default_tx_size_prob);
#if CONFIG_VAR_TX
av1_copy(fc->txfm_partition_prob, default_txfm_partition_probs);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
av1_copy(fc->rect_tx_prob, default_rect_tx_prob);
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#endif
av1_copy(fc->skip_probs, default_skip_probs);
#if CONFIG_REF_MV
......@@ -1451,6 +1460,15 @@ void av1_adapt_inter_frame_probs(AV1_COMMON *cm) {
}
#endif // CONFIG_EXT_INTER
#if CONFIG_VAR_TX && CONFIG_EXT_TX && CONFIG_RECT_TX
if (cm->tx_mode == TX_MODE_SELECT) {
for (i = 0; i < TX_SIZES - 1; ++i) {
fc->rect_tx_prob[i] =
av1_mode_mv_merge_probs(pre_fc->rect_tx_prob[i], counts->rect_tx[i]);
}
}
#endif // CONFIG_VAR_TX && CONFIG_EXT_TX && CONFIG_RECT_TX
for (i = 0; i < BLOCK_SIZE_GROUPS; i++)
aom_tree_merge_probs(av1_intra_mode_tree, pre_fc->y_mode_prob[i],
counts->y_mode[i], fc->y_mode_prob[i]);
......@@ -1478,10 +1496,11 @@ void av1_adapt_intra_frame_probs(AV1_COMMON *cm) {
}
#if CONFIG_VAR_TX
if (cm->tx_mode == TX_MODE_SELECT)
if (cm->tx_mode == TX_MODE_SELECT) {
for (i = 0; i < TXFM_PARTITION_CONTEXTS; ++i)
fc->txfm_partition_prob[i] = av1_mode_mv_merge_probs(
pre_fc->txfm_partition_prob[i], counts->txfm_partition[i]);
}
#endif
for (i = 0; i < SKIP_CONTEXTS; ++i)
......
......@@ -96,6 +96,10 @@ typedef struct frame_contexts {
aom_prob tx_size_probs[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES - 1];
#if CONFIG_VAR_TX
aom_prob txfm_partition_prob[TXFM_PARTITION_CONTEXTS];
#if CONFIG_EXT_TX && CONFIG_RECT_TX
// TODO(yuec) make this flag harmonize with the original syntax
aom_prob rect_tx_prob[TX_SIZES - 1];
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#endif
aom_prob skip_probs[SKIP_CONTEXTS];
#if CONFIG_REF_MV
......@@ -179,6 +183,9 @@ typedef struct FRAME_COUNTS {
unsigned int tx_size[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES];
#if CONFIG_VAR_TX
unsigned int txfm_partition[TXFM_PARTITION_CONTEXTS][2];
#if CONFIG_EXT_TX && CONFIG_RECT_TX
unsigned int rect_tx[TX_SIZES - 1][2];
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#endif
unsigned int skip[SKIP_CONTEXTS][2];
#if CONFIG_REF_MV
......
......@@ -1246,12 +1246,18 @@ void av1_filter_block_plane_non420(AV1_COMMON *cm,
tx_size_mask = 0;
#if CONFIG_VAR_TX
if (is_inter_block(mbmi) && !mbmi->skip)
tx_size =
(plane->plane_type == PLANE_TYPE_UV)
? uv_txsize_lookup[sb_type][mbmi->inter_tx_size
[blk_row][blk_col]][ss_x][ss_y]
: mbmi->inter_tx_size[blk_row][blk_col];
if (is_inter_block(mbmi) && !mbmi->skip) {
#if CONFIG_EXT_TX && CONFIG_RECT_TX
TX_SIZE mb_tx_size = is_rect_tx(mbmi->tx_size)
? mbmi->tx_size
: mbmi->inter_tx_size[blk_row][blk_col];
#else
TX_SIZE mb_tx_size = mbmi->inter_tx_size[blk_row][blk_col];
#endif
tx_size = (plane->plane_type == PLANE_TYPE_UV)
? uv_txsize_lookup[sb_type][mb_tx_size][ss_x][ss_y]
: mb_tx_size;
}
#if CONFIG_EXT_TX && CONFIG_RECT_TX
tx_size_r =
......
......@@ -1308,10 +1308,9 @@ static void decode_block(AV1Decoder *const pbi, MACROBLOCKD *const xd,
const int step = num_4x4_blocks_txsize_lookup[max_tx_size];
int block = 0;
#if CONFIG_EXT_TX && CONFIG_RECT_TX
const TX_SIZE tx_size =
plane ? get_uv_tx_size(mbmi, pd) : mbmi->tx_size;
if (tx_size >= TX_SIZES) { // rect txsize is used
if (is_rect_tx(mbmi->tx_size)) {
const TX_SIZE tx_size =
plane ? get_uv_tx_size(mbmi, pd) : mbmi->tx_size;
const int stepr = num_4x4_blocks_high_txsize_lookup[tx_size];
const int stepc = num_4x4_blocks_wide_txsize_lookup[tx_size];
const int max_blocks_wide =
......@@ -3491,6 +3490,12 @@ static int read_compressed_header(AV1Decoder *pbi, const uint8_t *data,
#if CONFIG_VAR_TX
for (k = 0; k < TXFM_PARTITION_CONTEXTS; ++k)
av1_diff_update_prob(&r, &fc->txfm_partition_prob[k]);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (cm->tx_mode == TX_MODE_SELECT) {
for (i = 1; i < TX_SIZES - 1; ++i)
av1_diff_update_prob(&r, &fc->rect_tx_prob[i]);
}
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#endif
for (k = 0; k < SKIP_CONTEXTS; ++k)
......
......@@ -1632,13 +1632,34 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi,
const int width = num_4x4_blocks_wide_lookup[bsize];
const int height = num_4x4_blocks_high_lookup[bsize];
int idx, idy;
for (idy = 0; idy < height; idy += bs)
for (idx = 0; idx < width; idx += bs)
read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size, idy, idx,
r);
int tx_size_cat = inter_tx_size_cat_lookup[bsize];
#if CONFIG_EXT_TX && CONFIG_RECT_TX
int is_rect_tx_allowed = inter_block && is_rect_tx_allowed_bsize(bsize);
int use_rect_tx = 0;
if (is_rect_tx_allowed) {
use_rect_tx = aom_read(r, cm->fc->rect_tx_prob[tx_size_cat]);
if (xd->counts) {
++xd->counts->rect_tx[tx_size_cat][use_rect_tx];
}
}
if (use_rect_tx) {
mbmi->tx_size = max_txsize_rect_lookup[bsize];
set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
} else {
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
for (idy = 0; idy < height; idy += bs)
for (idx = 0; idx < width; idx += bs)
read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size, idy, idx,
r);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
}
#endif
if (xd->counts) {
const int ctx = get_tx_size_context(xd);
++xd->counts->tx_size[max_tx_size - TX_8X8][ctx][mbmi->tx_size];
++xd->counts
->tx_size[tx_size_cat][ctx][txsize_sqr_up_map[mbmi->tx_size]];
}
} else {
if (inter_block)
......
......@@ -1121,9 +1121,25 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const MODE_INFO *mi,
const int width = num_4x4_blocks_wide_lookup[bsize];
const int height = num_4x4_blocks_high_lookup[bsize];
int idx, idy;
for (idy = 0; idy < height; idy += bs)
for (idx = 0; idx < width; idx += bs)
write_tx_size_vartx(cm, xd, mbmi, max_tx_size, idy, idx, w);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_rect_tx_allowed(mbmi)) {
int tx_size_cat = inter_tx_size_cat_lookup[bsize];
aom_write(w, is_rect_tx(mbmi->tx_size),
cm->fc->rect_tx_prob[tx_size_cat]);
}
if (is_rect_tx(mbmi->tx_size)) {
set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
} else {
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
for (idy = 0; idy < height; idy += bs)
for (idx = 0; idx < width; idx += bs)
write_tx_size_vartx(cm, xd, mbmi, max_tx_size, idy, idx, w);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
}
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
} else {
set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
write_selected_tx_size(cm, xd, w);
......@@ -1650,7 +1666,7 @@ static void write_modes_b(AV1_COMP *cpi, const TileInfo *const tile,
TX_SIZE tx_size =
plane ? get_uv_tx_size(mbmi, &xd->plane[plane]) : mbmi->tx_size;
if (is_inter_block(mbmi) && tx_size < TX_SIZES) {
if (is_inter_block(mbmi) && !is_rect_tx(tx_size)) {
#else
if (is_inter_block(mbmi)) {
#endif
......@@ -3271,6 +3287,13 @@ static uint32_t write_compressed_header(AV1_COMP *cpi, uint8_t *data) {
#if CONFIG_VAR_TX
update_txfm_partition_probs(cm, header_bc, counts);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (cm->tx_mode == TX_MODE_SELECT) {
for (i = 1; i < TX_SIZES - 1; ++i)
av1_cond_prob_diff_update(header_bc, &fc->rect_tx_prob[i],
counts->rect_tx[i]);
}
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#endif
update_skip_probs(cm, header_bc, counts);
......
......@@ -5031,7 +5031,7 @@ static void encode_superblock(AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t,
av1_encode_sb(x, AOMMAX(bsize, BLOCK_8X8));
#if CONFIG_VAR_TX
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (mbmi->tx_size >= TX_SIZES)
if (is_rect_tx(mbmi->tx_size))
av1_tokenize_sb(cpi, td, t, !output_enabled, AOMMAX(bsize, BLOCK_8X8));
else
#endif
......@@ -5054,8 +5054,17 @@ static void encode_superblock(AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t,
assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(mbmi)));
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#if CONFIG_VAR_TX
if (is_inter)
tx_partition_count_update(cm, xd, bsize, mi_row, mi_col, td->counts);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_rect_tx_allowed(mbmi)) {
td->counts->rect_tx[tx_size_cat][is_rect_tx(mbmi->tx_size)]++;
}
if (!is_rect_tx_allowed(mbmi) || !is_rect_tx(mbmi->tx_size)) {
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_inter)
tx_partition_count_update(cm, xd, bsize, mi_row, mi_col, td->counts);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
}
#endif
#endif
++td->counts->tx_size[tx_size_cat][tx_size_ctx][coded_tx_size];
} else {
......@@ -5118,6 +5127,11 @@ static void encode_superblock(AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t,
is_inter_block(mbmi) && !(mbmi->skip || seg_skip)) {
if (!output_enabled)
tx_partition_set_contexts(cm, xd, bsize, mi_row, mi_col);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_rect_tx(mbmi->tx_size)) {
set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
}
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
} else {
TX_SIZE tx_size;
// The new intra coding scheme requires no change of transform size
......
......@@ -791,7 +791,7 @@ static void encode_block(int plane, int block, int blk_row, int blk_col,
#endif
#if CONFIG_VAR_TX
// Assert not magic number (uninitialised).
// Assert not magic number (uninitialized).
assert(x->blk_skip[plane][(blk_row << bwl) + blk_col] != 234);
if (x->blk_skip[plane][(blk_row << bwl) + blk_col] == 0) {
......@@ -976,9 +976,6 @@ void av1_encode_sb(MACROBLOCK *x, BLOCK_SIZE bsize) {
int idx, idy;
int block = 0;
int step = num_4x4_blocks_txsize_lookup[max_tx_size];
#if CONFIG_EXT_TX && CONFIG_RECT_TX
const TX_SIZE tx_size = plane ? get_uv_tx_size(mbmi, pd) : mbmi->tx_size;
#endif
av1_get_entropy_contexts(bsize, TX_4X4, pd, ctx.ta[plane], ctx.tl[plane]);
#else
const struct macroblockd_plane *const pd = &xd->plane[plane];
......@@ -991,7 +988,7 @@ void av1_encode_sb(MACROBLOCK *x, BLOCK_SIZE bsize) {
#if CONFIG_VAR_TX
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (tx_size >= TX_SIZES) {
if (is_rect_tx(mbmi->tx_size)) {
av1_foreach_transformed_block_in_plane(xd, bsize, plane, encode_block,
&arg);
} else {
......
......@@ -2814,6 +2814,61 @@ static int64_t rd_pick_intra_sby_mode(AV1_COMP *cpi, MACROBLOCK *x, int *rate,
return best_rd;
}
// Return value 0: early termination triggered, no valid rd cost available;
// 1: rd cost values are valid.
static int super_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
int64_t *distortion, int *skippable, int64_t *sse,
BLOCK_SIZE bsize, int64_t ref_best_rd) {
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
const TX_SIZE uv_tx_size = get_uv_tx_size(mbmi, &xd->plane[1]);
int plane;
int pnrate = 0, pnskip = 1;
int64_t pndist = 0, pnsse = 0;
int is_cost_valid = 1;
if (ref_best_rd < 0) is_cost_valid = 0;
if (is_inter_block(mbmi) && is_cost_valid) {
int plane;
for (plane = 1; plane < MAX_MB_PLANE; ++plane)
av1_subtract_plane(x, bsize, plane);
}
*rate = 0;
*distortion = 0;
*sse = 0;
*skippable = 1;
for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
txfm_rd_in_plane(x, cpi, &pnrate, &pndist, &pnskip, &pnsse, ref_best_rd,
plane, bsize, uv_tx_size, cpi->sf.use_fast_coef_costing);
if (pnrate == INT_MAX) {
is_cost_valid = 0;
break;
}
*rate += pnrate;
*distortion += pndist;
*sse += pnsse;
*skippable &= pnskip;
if (RDCOST(x->rdmult, x->rddiv, *rate, *distortion) > ref_best_rd &&
RDCOST(x->rdmult, x->rddiv, 0, *sse) > ref_best_rd) {
is_cost_valid = 0;
break;
}
}
if (!is_cost_valid) {
// reset cost value
*rate = INT_MAX;
*distortion = INT64_MAX;
*sse = INT64_MAX;
*skippable = 0;
}
return is_cost_valid;
}
#if CONFIG_VAR_TX
void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
int blk_row, int blk_col, int plane, int block,
......@@ -3196,6 +3251,61 @@ static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x,
mbmi->tx_type = tx_type;
inter_block_yrd(cpi, x, rate, dist, skippable, sse, bsize, ref_best_rd);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_rect_tx_allowed(mbmi)) {
int rate_rect_tx, skippable_rect_tx = 0;
int64_t dist_rect_tx, sse_rect_tx, rd, rd_rect_tx;
int tx_size_cat = inter_tx_size_cat_lookup[bsize];
TX_SIZE tx_size = max_txsize_rect_lookup[bsize];
TX_SIZE var_tx_size = mbmi->tx_size;
txfm_rd_in_plane(x, cpi, &rate_rect_tx, &dist_rect_tx, &skippable_rect_tx,
&sse_rect_tx, ref_best_rd, 0, bsize, tx_size,
cpi->sf.use_fast_coef_costing);
if (*rate != INT_MAX) {
*rate += av1_cost_bit(cm->fc->rect_tx_prob[tx_size_cat], 0);
if (*skippable) {
rd = RDCOST(x->rdmult, x->rddiv, s1, *sse);
} else {
rd = RDCOST(x->rdmult, x->rddiv, *rate + s0, *dist);
if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] &&
!(*skippable))
rd = AOMMIN(rd, RDCOST(x->rdmult, x->rddiv, s1, *sse));
}
} else {
rd = INT64_MAX;
}
if (rate_rect_tx != INT_MAX) {
rate_rect_tx += av1_cost_bit(cm->fc->rect_tx_prob[tx_size_cat], 1);
if (skippable_rect_tx) {
rd_rect_tx = RDCOST(x->rdmult, x->rddiv, s1, sse_rect_tx);
} else {
rd_rect_tx =
RDCOST(x->rdmult, x->rddiv, rate_rect_tx + s0, dist_rect_tx);
if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] &&
!(skippable_rect_tx))
rd_rect_tx =
AOMMIN(rd_rect_tx, RDCOST(x->rdmult, x->rddiv, s1, sse_rect_tx));
}
} else {
rd_rect_tx = INT64_MAX;
}
if (rd_rect_tx < rd) {
*rate = rate_rect_tx;
*dist = dist_rect_tx;
*sse = sse_rect_tx;
*skippable = skippable_rect_tx;
if (!xd->lossless[mbmi->segment_id]) x->blk_skip[0][0] = *skippable;
mbmi->tx_size = tx_size;
mbmi->inter_tx_size[0][0] = mbmi->tx_size;
} else {
mbmi->tx_size = var_tx_size;
}
}
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
if (*rate == INT_MAX) return INT64_MAX;
......@@ -3409,17 +3519,24 @@ static int inter_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
if (ref_best_rd < 0) is_cost_valid = 0;
*rate = 0;
*distortion = 0;
*sse = 0;
*skippable = 1;
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_rect_tx(mbmi->tx_size)) {
return super_block_uvrd(cpi, x, rate, distortion, skippable, sse, bsize,
ref_best_rd);
}
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_inter_block(mbmi) && is_cost_valid) {
int plane;
for (plane = 1; plane < MAX_MB_PLANE; ++plane)
av1_subtract_plane(x, bsize, plane);
}
*rate = 0;
*distortion = 0;
*sse = 0;
*skippable = 1;
for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
const struct macroblockd_plane *const pd = &xd->plane[plane];
const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
......@@ -3477,61 +3594,6 @@ static int inter_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
}
#endif // CONFIG_VAR_TX
// Return value 0: early termination triggered, no valid rd cost available;
// 1: rd cost values are valid.
static int super_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
int64_t *distortion, int *skippable, int64_t *sse,
BLOCK_SIZE bsize, int64_t ref_best_rd) {
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
const TX_SIZE uv_tx_size = get_uv_tx_size(mbmi, &xd->plane[1]);
int plane;
int pnrate = 0, pnskip = 1;
int64_t pndist = 0, pnsse = 0;
int is_cost_valid = 1;
if (ref_best_rd < 0) is_cost_valid = 0;
if (is_inter_block(mbmi) && is_cost_valid) {
int plane;
for (plane = 1; plane < MAX_MB_PLANE; ++plane)
av1_subtract_plane(x, bsize, plane);
}
*rate = 0;
*distortion = 0;
*sse = 0;
*skippable = 1;
for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
txfm_rd_in_plane(x, cpi, &pnrate, &pndist, &pnskip, &pnsse, ref_best_rd,
plane, bsize, uv_tx_size, cpi->sf.use_fast_coef_costing);
if (pnrate == INT_MAX) {
is_cost_valid = 0;
break;
}
*rate += pnrate;
*distortion += pndist;
*sse += pnsse;
*skippable &= pnskip;
if (RDCOST(x->rdmult, x->rddiv, *rate, *distortion) > ref_best_rd &&
RDCOST(x->rdmult, x->rddiv, 0, *sse) > ref_best_rd) {
is_cost_valid = 0;
break;
}
}
if (!is_cost_valid) {
// reset cost value
*rate = INT_MAX;
*distortion = INT64_MAX;
*sse = INT64_MAX;
*skippable = 0;
}
return is_cost_valid;
}
static void rd_pick_palette_intra_sbuv(
AV1_COMP *cpi, MACROBLOCK *x, int dc_mode_cost,
PALETTE_MODE_INFO *palette_mode_info, uint8_t *best_palette_color_map,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment