Commit 199502f2 authored by Jingning Han's avatar Jingning Han Committed by Gerrit Code Review

Merge "Support potential 2x2 transform block unit" into nextgenv2

parents f6f2cfca 607fa6a6
This diff is collapsed.
......@@ -58,6 +58,9 @@ const aom_prob av1_cat6_prob_high12[] = { 255, 255, 255, 255, 254, 254,
#endif
const uint16_t band_count_table[TX_SIZES_ALL][8] = {
#if CONFIG_CB4X4
{ 1, 2, 2, 3, 0, 0, 0 },
#endif
{ 1, 2, 3, 4, 3, 16 - 13, 0 }, { 1, 2, 3, 4, 11, 64 - 21, 0 },
{ 1, 2, 3, 4, 11, 256 - 21, 0 }, { 1, 2, 3, 4, 11, 1024 - 21, 0 },
#if CONFIG_EXT_TX
......@@ -68,6 +71,9 @@ const uint16_t band_count_table[TX_SIZES_ALL][8] = {
};
const uint16_t band_cum_count_table[TX_SIZES_ALL][8] = {
#if CONFIG_CB4X4
{ 0, 1, 3, 6, 10, 13, 16, 0 },
#endif
{ 0, 1, 3, 6, 10, 13, 16, 0 }, { 0, 1, 3, 6, 10, 21, 64, 0 },
{ 0, 1, 3, 6, 10, 21, 256, 0 }, { 0, 1, 3, 6, 10, 21, 1024, 0 },
#if CONFIG_EXT_TX
......@@ -2833,6 +2839,9 @@ void av1_default_coef_probs(AV1_COMMON *cm) {
ROUND_POWER_OF_TWO(cm->base_qindex, 8 - QCTX_BIN_BITS), QCTX_BINS - 1);
av1_copy(cm->fc->coef_probs, default_qctx_coef_probs[index]);
#else
#if CONFIG_CB4X4
av1_copy(cm->fc->coef_probs[TX_2X2], default_coef_probs_4x4);
#endif
av1_copy(cm->fc->coef_probs[TX_4X4], default_coef_probs_4x4);
av1_copy(cm->fc->coef_probs[TX_8X8], default_coef_probs_8x8);
av1_copy(cm->fc->coef_probs[TX_16X16], default_coef_probs_16x16);
......
......@@ -851,18 +851,18 @@ static const int palette_color_context_lookup[PALETTE_COLOR_CONTEXTS] = {
};
#endif // CONFIG_PALETTE
const aom_tree_index av1_tx_size_tree[TX_SIZES - 1][TREE_SIZE(TX_SIZES)] = {
const aom_tree_index av1_tx_size_tree[MAX_TX_DEPTH][TREE_SIZE(TX_SIZES)] = {
{
// Max tx_size is 8X8
-TX_4X4, -TX_8X8,
-0, -1,
},
{
// Max tx_size is 16X16
-TX_4X4, 2, -TX_8X8, -TX_16X16,
-0, 2, -1, -2,
},
{
// Max tx_size is 32X32
-TX_4X4, 2, -TX_8X8, 4, -TX_16X16, -TX_32X32,
-0, 2, -1, 4, -2, -3,
},
};
......@@ -1309,13 +1309,21 @@ int av1_ext_tx_inv[TX_TYPES];
static const aom_prob
default_intra_ext_tx_prob[EXT_TX_SIZES][TX_TYPES][TX_TYPES - 1] = {
#if CONFIG_CB4X4
{ { 240, 85, 128 }, { 4, 1, 248 }, { 4, 1, 8 }, { 4, 248, 128 } },
#endif
{ { 240, 85, 128 }, { 4, 1, 248 }, { 4, 1, 8 }, { 4, 248, 128 } },
{ { 244, 85, 128 }, { 8, 2, 248 }, { 8, 2, 8 }, { 8, 248, 128 } },
{ { 248, 85, 128 }, { 16, 4, 248 }, { 16, 4, 8 }, { 16, 248, 128 } },
};
static const aom_prob default_inter_ext_tx_prob[EXT_TX_SIZES][TX_TYPES - 1] = {
{ 160, 85, 128 }, { 176, 85, 128 }, { 192, 85, 128 },
#if CONFIG_CB4X4
{ 160, 85, 128 },
#endif
{ 160, 85, 128 },
{ 176, 85, 128 },
{ 192, 85, 128 },
};
#endif // CONFIG_EXT_TX
......@@ -1560,7 +1568,7 @@ void av1_adapt_inter_frame_probs(AV1_COMMON *cm) {
#if CONFIG_VAR_TX && CONFIG_EXT_TX && CONFIG_RECT_TX
if (cm->tx_mode == TX_MODE_SELECT) {
for (i = 0; i < TX_SIZES - 1; ++i) {
for (i = 0; i < MAX_TX_DEPTH; ++i) {
fc->rect_tx_prob[i] =
av1_mode_mv_merge_probs(pre_fc->rect_tx_prob[i], counts->rect_tx[i]);
}
......
......@@ -324,7 +324,7 @@ extern const aom_tree_index av1_palette_size_tree[TREE_SIZE(PALETTE_SIZES)];
extern const aom_tree_index av1_palette_color_tree[PALETTE_MAX_SIZE - 1]
[TREE_SIZE(PALETTE_COLORS)];
#endif // CONFIG_PALETTE
extern const aom_tree_index av1_tx_size_tree[TX_SIZES - 1][TREE_SIZE(TX_SIZES)];
extern const aom_tree_index av1_tx_size_tree[MAX_TX_DEPTH][TREE_SIZE(TX_SIZES)];
#if CONFIG_EXT_INTRA
extern const aom_tree_index av1_intra_filter_tree[TREE_SIZE(INTRA_FILTERS)];
#endif // CONFIG_EXT_INTRA
......
......@@ -131,6 +131,9 @@ typedef char PARTITION_CONTEXT;
// block transform size
typedef enum ATTRIBUTE_PACKED {
#if CONFIG_CB4X4
TX_2X2, // 2x2 transform
#endif
TX_4X4, // 4x4 transform
TX_8X8, // 8x8 transform
TX_16X16, // 16x16 transform
......@@ -208,8 +211,12 @@ typedef enum {
#define EXT_TX_SETS_INTER 4 // Sets of transform selections for INTER
#define EXT_TX_SETS_INTRA 3 // Sets of transform selections for INTRA
#else
#if CONFIG_CB4X4
#define EXT_TX_SIZES 4 // number of sizes that use extended transforms
#else
#define EXT_TX_SIZES 3 // number of sizes that use extended transforms
#endif // CONFIG_EXT_TX
#endif
#endif // CONFIG_EXT_TX
typedef enum {
AOM_LAST_FLAG = 1 << 0,
......
......@@ -40,6 +40,9 @@
//
// A loopfilter should be applied to every other 8x8 horizontally.
static const uint64_t left_64x64_txform_mask[TX_SIZES] = {
#if CONFIG_CB4X4
0xffffffffffffffffULL, // TX_2X2
#endif
0xffffffffffffffffULL, // TX_4X4
0xffffffffffffffffULL, // TX_8x8
0x5555555555555555ULL, // TX_16x16
......@@ -64,6 +67,9 @@ static const uint64_t left_64x64_txform_mask[TX_SIZES] = {
//
// A loopfilter should be applied to every other 4 the row vertically.
static const uint64_t above_64x64_txform_mask[TX_SIZES] = {
#if CONFIG_CB4X4
0xffffffffffffffffULL, // TX_4X4
#endif
0xffffffffffffffffULL, // TX_4X4
0xffffffffffffffffULL, // TX_8x8
0x00ff00ff00ff00ffULL, // TX_16x16
......@@ -142,6 +148,9 @@ static const uint64_t above_border = 0x000000ff000000ffULL;
// 16 bit masks for uv transform sizes.
static const uint16_t left_64x64_txform_mask_uv[TX_SIZES] = {
#if CONFIG_CB4X4
0xffff, // TX_2X2
#endif
0xffff, // TX_4X4
0xffff, // TX_8x8
0x5555, // TX_16x16
......@@ -149,6 +158,9 @@ static const uint16_t left_64x64_txform_mask_uv[TX_SIZES] = {
};
static const uint16_t above_64x64_txform_mask_uv[TX_SIZES] = {
#if CONFIG_CB4X4
0xffff, // TX_2X2
#endif
0xffff, // TX_4X4
0xffff, // TX_8x8
0x0f0f, // TX_16x16
......
......@@ -733,7 +733,8 @@ static INLINE int txfm_partition_context(TXFM_CONTEXT *above_ctx,
TX_SIZE tx_size) {
int above = *above_ctx < tx_size;
int left = *left_ctx < tx_size;
return (tx_size - 1) * 3 + above + left;
return (tx_size - TX_8X8) * 3 + above + left;
}
#endif
......
......@@ -186,8 +186,13 @@ static INLINE int get_tx_size_context(const MACROBLOCKD *xd) {
if (!has_left) left_ctx = above_ctx;
if (!has_above) above_ctx = left_ctx;
#if CONFIG_CB4X4
// TODO(jingning): Temporary setup. Will rework this after the cb4x4
// framework is up running.
return (above_ctx + left_ctx) > max_tx_size + 1;
#else
return (above_ctx + left_ctx) > max_tx_size;
#endif
}
#if CONFIG_VAR_TX
......
......@@ -3801,6 +3801,9 @@ DECLARE_ALIGNED(16, static const int16_t, av1_qtr_iscan_32x32[1024]) = {
#endif // CONFIG_EXT_TX
const SCAN_ORDER av1_default_scan_orders[TX_SIZES] = {
#if CONFIG_CB4X4
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
#endif
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ default_scan_8x8, av1_default_iscan_8x8, default_scan_8x8_neighbors },
{ default_scan_16x16, av1_default_iscan_16x16, default_scan_16x16_neighbors },
......@@ -3809,6 +3812,27 @@ const SCAN_ORDER av1_default_scan_orders[TX_SIZES] = {
#if CONFIG_EXT_TX
const SCAN_ORDER av1_intra_scan_orders[TX_SIZES][TX_TYPES] = {
#if CONFIG_CB4X4
{
// TX_2X2
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ row_scan_4x4, av1_row_iscan_4x4, row_scan_4x4_neighbors },
{ col_scan_4x4, av1_col_iscan_4x4, col_scan_4x4_neighbors },
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ mrow_scan_4x4, av1_mrow_iscan_4x4, mrow_scan_4x4_neighbors },
{ row_scan_4x4, av1_row_iscan_4x4, row_scan_4x4_neighbors },
{ col_scan_4x4, av1_col_iscan_4x4, col_scan_4x4_neighbors },
{ row_scan_4x4, av1_row_iscan_4x4, row_scan_4x4_neighbors },
{ col_scan_4x4, av1_col_iscan_4x4, col_scan_4x4_neighbors },
{ row_scan_4x4, av1_row_iscan_4x4, row_scan_4x4_neighbors },
{ col_scan_4x4, av1_col_iscan_4x4, col_scan_4x4_neighbors },
},
#endif
{
// TX_4X4
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
......@@ -3896,6 +3920,27 @@ const SCAN_ORDER av1_intra_scan_orders[TX_SIZES][TX_TYPES] = {
};
const SCAN_ORDER av1_inter_scan_orders[TX_SIZES_ALL][TX_TYPES] = {
#if CONFIG_CB4X4
{
// TX_2X2
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ mrow_scan_4x4, av1_mrow_iscan_4x4, mrow_scan_4x4_neighbors },
{ mrow_scan_4x4, av1_mrow_iscan_4x4, mrow_scan_4x4_neighbors },
{ mcol_scan_4x4, av1_mcol_iscan_4x4, mcol_scan_4x4_neighbors },
{ mrow_scan_4x4, av1_mrow_iscan_4x4, mrow_scan_4x4_neighbors },
{ mcol_scan_4x4, av1_mcol_iscan_4x4, mcol_scan_4x4_neighbors },
{ mrow_scan_4x4, av1_mrow_iscan_4x4, mrow_scan_4x4_neighbors },
{ mcol_scan_4x4, av1_mcol_iscan_4x4, mcol_scan_4x4_neighbors },
},
#endif
{
// TX_4X4
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
......@@ -4134,9 +4179,16 @@ const SCAN_ORDER av1_inter_scan_orders[TX_SIZES_ALL][TX_TYPES] = {
}
};
#else // CONFIG_EXT_TX
#else // CONFIG_EXT_TX
const SCAN_ORDER av1_intra_scan_orders[TX_SIZES][TX_TYPES] = {
#if CONFIG_CB4X4
{ // TX_2X2
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ row_scan_4x4, av1_row_iscan_4x4, row_scan_4x4_neighbors },
{ col_scan_4x4, av1_col_iscan_4x4, col_scan_4x4_neighbors },
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors } },
#endif
{ // TX_4X4
{ default_scan_4x4, av1_default_iscan_4x4, default_scan_4x4_neighbors },
{ row_scan_4x4, av1_row_iscan_4x4, row_scan_4x4_neighbors },
......
......@@ -3674,7 +3674,7 @@ static int read_compressed_header(AV1Decoder *pbi, const uint8_t *data,
av1_diff_update_prob(&r, &fc->txfm_partition_prob[k], ACCT_STR);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (cm->tx_mode == TX_MODE_SELECT) {
for (i = 1; i < TX_SIZES - 1; ++i)
for (i = 1; i < MAX_TX_DEPTH; ++i)
av1_diff_update_prob(&r, &fc->rect_tx_prob[i], ACCT_STR);
}
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
......
......@@ -3222,7 +3222,8 @@ static void inter_block_yrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
int bh = num_4x4_blocks_wide_lookup[txb_size];
int idx, idy;
int block = 0;
int step = 1 << (max_txsize_lookup[plane_bsize] * 2);
int step = tx_size_wide_unit[max_txsize_lookup[plane_bsize]] *
tx_size_high_unit[max_txsize_lookup[plane_bsize]];
ENTROPY_CONTEXT ctxa[2 * MAX_MIB_SIZE];
ENTROPY_CONTEXT ctxl[2 * MAX_MIB_SIZE];
TXFM_CONTEXT tx_above[MAX_MIB_SIZE];
......@@ -3519,6 +3520,7 @@ static void tx_block_rd(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
coeff_ctx = combine_entropy_contexts(ta[0], tl[0]);
av1_tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block,
plane_bsize, coeff_ctx, rate, dist, bsse, skip);
for (i = 0; i < num_4x4_blocks_wide_txsize_lookup[tx_size]; ++i)
ta[i] = !(p->eobs[block] == 0);
for (i = 0; i < num_4x4_blocks_high_txsize_lookup[tx_size]; ++i)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment