Commit 0db7d0eb authored by Dake He's avatar Dake He

[lv_map_multi] multisymbol coding of eob

1. code the range of eob in a transform block as a single symbol.
2. initialize cdfs directly.
3. initial cdfs are trained by using laplace prob estimates as in base
level coding (CL 38638)

Change-Id: I65c88d6dd0da3381feeec837928ac5bd81be4296
parent 298b072d
......@@ -944,6 +944,36 @@ static const TX_SIZE txsize_sqr_up_map[TX_SIZES_ALL] = {
#endif // CONFIG_TX64X64
};
#if CONFIG_LV_MAP_MULTI
static const int8_t txsize_log2_minus4[TX_SIZES_ALL] = {
0, // TX_4X4
2, // TX_8X8
4, // TX_16X16
6, // TX_32X32
#if CONFIG_TX64X64
6, // TX_64X64
#endif // CONFIG_TX64X64
1, // TX_4X8
1, // TX_8X4
3, // TX_8X16
3, // TX_16X8
5, // TX_16X32
5, // TX_32X16
#if CONFIG_TX64X64
6, // TX_32X64
6, // TX_64X32
#endif // CONFIG_TX64X64
2, // TX_4X16
2, // TX_16X4
4, // TX_8X32
4, // TX_32X8
#if CONFIG_TX64X64
5, // TX_16X64
5, // TX_64X16
#endif // CONFIG_TX64X64
};
#endif
/* clang-format off */
#if CONFIG_SIMPLIFY_TX_MODE
static const TX_SIZE tx_mode_to_biggest_tx_size[TX_MODES] = {
......
......@@ -931,12 +931,19 @@ void av1_average_tile_coef_cdfs(FRAME_CONTEXT *fc, FRAME_CONTEXT *ec_ctxs[],
AVERAGE_TILE_CDFS(txb_skip_cdf)
#if !CONFIG_LV_MAP_MULTI
AVERAGE_TILE_CDFS(nz_map_cdf)
#endif
AVERAGE_TILE_CDFS(eob_flag_cdf)
#endif
AVERAGE_TILE_CDFS(eob_extra_cdf)
AVERAGE_TILE_CDFS(dc_sign_cdf)
AVERAGE_TILE_CDFS(coeff_base_cdf)
#if CONFIG_LV_MAP_MULTI
AVERAGE_TILE_CDFS(eob_flag_cdf16)
AVERAGE_TILE_CDFS(eob_flag_cdf32)
AVERAGE_TILE_CDFS(eob_flag_cdf64)
AVERAGE_TILE_CDFS(eob_flag_cdf128)
AVERAGE_TILE_CDFS(eob_flag_cdf256)
AVERAGE_TILE_CDFS(eob_flag_cdf512)
AVERAGE_TILE_CDFS(eob_flag_cdf1024)
AVERAGE_TILE_CDFS(coeff_base_eob_cdf)
#else // CONFIG_LV_MAP_MULTI
AVERAGE_TILE_CDFS(coeff_lps_cdf)
......
This diff is collapsed.
......@@ -86,11 +86,12 @@ typedef struct frame_contexts {
#if CONFIG_LV_MAP
aom_prob txb_skip[TX_SIZES][TXB_SKIP_CONTEXTS];
aom_prob eob_flag[TX_SIZES][PLANE_TYPES][EOB_COEF_CONTEXTS];
aom_prob eob_extra[TX_SIZES][PLANE_TYPES][EOB_COEF_CONTEXTS];
aom_prob dc_sign[PLANE_TYPES][DC_SIGN_CONTEXTS];
#if !CONFIG_LV_MAP_MULTI
aom_prob eob_flag[TX_SIZES][PLANE_TYPES][EOB_COEF_CONTEXTS];
aom_prob nz_map[TX_SIZES][PLANE_TYPES][SIG_COEF_CONTEXTS];
aom_prob coeff_base[TX_SIZES][PLANE_TYPES][NUM_BASE_LEVELS]
[COEFF_BASE_CONTEXTS];
......@@ -104,13 +105,20 @@ typedef struct frame_contexts {
#if !CONFIG_LV_MAP_MULTI
aom_cdf_prob nz_map_cdf[TX_SIZES][PLANE_TYPES][SIG_COEF_CONTEXTS]
[CDF_SIZE(2)];
#endif
aom_cdf_prob eob_flag_cdf[TX_SIZES][PLANE_TYPES][EOB_COEF_CONTEXTS]
[CDF_SIZE(2)];
#endif
aom_cdf_prob eob_extra_cdf[TX_SIZES][PLANE_TYPES][EOB_COEF_CONTEXTS]
[CDF_SIZE(2)];
aom_cdf_prob dc_sign_cdf[PLANE_TYPES][DC_SIGN_CONTEXTS][CDF_SIZE(2)];
#if CONFIG_LV_MAP_MULTI
aom_cdf_prob eob_flag_cdf16[PLANE_TYPES][2][CDF_SIZE(5)];
aom_cdf_prob eob_flag_cdf32[PLANE_TYPES][2][CDF_SIZE(6)];
aom_cdf_prob eob_flag_cdf64[PLANE_TYPES][2][CDF_SIZE(7)];
aom_cdf_prob eob_flag_cdf128[PLANE_TYPES][2][CDF_SIZE(8)];
aom_cdf_prob eob_flag_cdf256[PLANE_TYPES][2][CDF_SIZE(9)];
aom_cdf_prob eob_flag_cdf512[PLANE_TYPES][2][CDF_SIZE(10)];
aom_cdf_prob eob_flag_cdf1024[PLANE_TYPES][2][CDF_SIZE(11)];
aom_cdf_prob coeff_base_eob_cdf[TX_SIZES][PLANE_TYPES][SIG_COEF_CONTEXTS_EOB]
[CDF_SIZE(3)];
aom_cdf_prob coeff_base_cdf[TX_SIZES][PLANE_TYPES][SIG_COEF_CONTEXTS]
......@@ -276,6 +284,14 @@ typedef struct FRAME_COUNTS {
[2];
unsigned int coeff_lps[TX_SIZES][PLANE_TYPES][LEVEL_CONTEXTS][2];
#else
unsigned int eob_multi16[PLANE_TYPES][2][5];
unsigned int eob_multi32[PLANE_TYPES][2][6];
unsigned int eob_multi64[PLANE_TYPES][2][7];
unsigned int eob_multi128[PLANE_TYPES][2][8];
unsigned int eob_multi256[PLANE_TYPES][2][9];
unsigned int eob_multi512[PLANE_TYPES][2][10];
unsigned int eob_multi1024[PLANE_TYPES][2][11];
unsigned int coeff_lps[TX_SIZES][PLANE_TYPES][BR_CDF_SIZE - 1][LEVEL_CONTEXTS]
[2];
unsigned int coeff_lps_multi[TX_SIZES][PLANE_TYPES][LEVEL_CONTEXTS]
......
......@@ -294,13 +294,13 @@ void av1_init_txb_probs(FRAME_CONTEXT *fc) {
for (tx_size = 0; tx_size < TX_SIZES; ++tx_size) {
for (plane = 0; plane < PLANE_TYPES; ++plane) {
for (ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx) {
#if !CONFIG_LV_MAP_MULTI
for (ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx) {
fc->nz_map_cdf[tx_size][plane][ctx][0] =
AOM_ICDF(128 * (aom_cdf_prob)fc->nz_map[tx_size][plane][ctx]);
fc->nz_map_cdf[tx_size][plane][ctx][1] = AOM_ICDF(32768);
fc->nz_map_cdf[tx_size][plane][ctx][2] = 0;
#endif
}
for (ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx) {
fc->eob_flag_cdf[tx_size][plane][ctx][0] =
......@@ -308,6 +308,7 @@ void av1_init_txb_probs(FRAME_CONTEXT *fc) {
fc->eob_flag_cdf[tx_size][plane][ctx][1] = AOM_ICDF(32768);
fc->eob_flag_cdf[tx_size][plane][ctx][2] = 0;
}
#endif
for (ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx) {
fc->eob_extra_cdf[tx_size][plane][ctx][0] =
......
......@@ -126,6 +126,86 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
int eob_extra = 0;
int eob_pt = 1;
#if CONFIG_LV_MAP_MULTI
(void)max_eob_pt;
const int eob_multi_size = txsize_log2_minus4[tx_size];
const int eob_multi_ctx = (tx_type_to_class[tx_type] == TX_CLASS_2D) ? 0 : 1;
switch (eob_multi_size) {
case 0:
eob_pt = av1_read_record_symbol(
counts, r, ec_ctx->eob_flag_cdf16[plane_type][eob_multi_ctx],
5, ACCT_STR) +
1;
break;
case 1:
eob_pt = av1_read_record_symbol(
counts, r, ec_ctx->eob_flag_cdf32[plane_type][eob_multi_ctx],
6, ACCT_STR) +
1;
break;
case 2:
eob_pt = av1_read_record_symbol(
counts, r, ec_ctx->eob_flag_cdf64[plane_type][eob_multi_ctx],
7, ACCT_STR) +
1;
break;
case 3:
eob_pt =
av1_read_record_symbol(
counts, r, ec_ctx->eob_flag_cdf128[plane_type][eob_multi_ctx], 8,
ACCT_STR) +
1;
break;
case 4:
eob_pt =
av1_read_record_symbol(
counts, r, ec_ctx->eob_flag_cdf256[plane_type][eob_multi_ctx], 9,
ACCT_STR) +
1;
break;
case 5:
eob_pt =
av1_read_record_symbol(
counts, r, ec_ctx->eob_flag_cdf512[plane_type][eob_multi_ctx], 10,
ACCT_STR) +
1;
break;
case 6:
default:
eob_pt =
av1_read_record_symbol(
counts, r, ec_ctx->eob_flag_cdf1024[plane_type][eob_multi_ctx],
11, ACCT_STR) +
1;
break;
}
if (counts) {
switch (eob_multi_size) {
case 0:
++counts->eob_multi16[plane_type][eob_multi_ctx][eob_pt - 1];
break;
case 1:
++counts->eob_multi32[plane_type][eob_multi_ctx][eob_pt - 1];
break;
case 2:
++counts->eob_multi64[plane_type][eob_multi_ctx][eob_pt - 1];
break;
case 3:
++counts->eob_multi128[plane_type][eob_multi_ctx][eob_pt - 1];
break;
case 4:
++counts->eob_multi256[plane_type][eob_multi_ctx][eob_pt - 1];
break;
case 5:
++counts->eob_multi512[plane_type][eob_multi_ctx][eob_pt - 1];
break;
case 6:
default:
++counts->eob_multi1024[plane_type][eob_multi_ctx][eob_pt - 1];
break;
}
}
#else
for (eob_pt = 1; eob_pt < max_eob_pt; eob_pt++) {
const int eob_pos_ctx = av1_get_eob_pos_ctx(tx_type, eob_pt);
const int is_equal = av1_read_record_bin(
......@@ -141,6 +221,7 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
break;
}
}
#endif
// printf("Dec: ");
if (k_eob_offset_bits[eob_pt] > 0) {
......
......@@ -67,9 +67,9 @@ typedef struct {
int base_eob_cost[SIG_COEF_CONTEXTS_EOB][3];
int base_cost[SIG_COEF_CONTEXTS][4];
#else
int eob_cost[EOB_COEF_CONTEXTS][2];
int nz_map_cost[SIG_COEF_CONTEXTS][2];
#endif
int eob_cost[EOB_COEF_CONTEXTS][2];
int eob_extra_cost[EOB_COEF_CONTEXTS][2];
int dc_sign_cost[DC_SIGN_CONTEXTS][2];
#if !CONFIG_LV_MAP_MULTI
......@@ -81,6 +81,10 @@ typedef struct {
#endif
} LV_MAP_COEFF_COST;
#if CONFIG_LV_MAP_MULTI
typedef struct { int eob_cost[2][11]; } LV_MAP_EOB_COST;
#endif
typedef struct {
tran_low_t tcoeff[MAX_MB_PLANE][MAX_SB_SQUARE];
uint16_t eobs[MAX_MB_PLANE][MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
......@@ -261,6 +265,9 @@ struct macroblock {
#if CONFIG_LV_MAP
LV_MAP_COEFF_COST coeff_costs[TX_SIZES][PLANE_TYPES];
#if CONFIG_LV_MAP_MULTI
LV_MAP_EOB_COST eob_costs[7][2];
#endif
uint16_t cb_offset;
#endif
......
......@@ -161,6 +161,54 @@ void av1_update_eob_context(int eob, int seg_eob, TX_SIZE tx_size,
const int max_eob_pt = get_eob_pos_token(seg_eob, &dummy);
TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
#if CONFIG_LV_MAP_MULTI
(void)max_eob_pt;
const int eob_multi_size = txsize_log2_minus4[tx_size];
const int eob_multi_ctx = (tx_type_to_class[tx_type] == TX_CLASS_2D) ? 0 : 1;
switch (eob_multi_size) {
case 0:
++counts->eob_multi16[plane][eob_multi_ctx][eob_pt - 1];
if (allow_update_cdf)
update_cdf(ec_ctx->eob_flag_cdf16[plane][eob_multi_ctx], eob_pt - 1, 5);
break;
case 1:
++counts->eob_multi32[plane][eob_multi_ctx][eob_pt - 1];
if (allow_update_cdf)
update_cdf(ec_ctx->eob_flag_cdf32[plane][eob_multi_ctx], eob_pt - 1, 6);
break;
case 2:
++counts->eob_multi64[plane][eob_multi_ctx][eob_pt - 1];
if (allow_update_cdf)
update_cdf(ec_ctx->eob_flag_cdf64[plane][eob_multi_ctx], eob_pt - 1, 7);
break;
case 3:
++counts->eob_multi128[plane][eob_multi_ctx][eob_pt - 1];
if (allow_update_cdf)
update_cdf(ec_ctx->eob_flag_cdf128[plane][eob_multi_ctx], eob_pt - 1,
8);
break;
case 4:
++counts->eob_multi256[plane][eob_multi_ctx][eob_pt - 1];
if (allow_update_cdf)
update_cdf(ec_ctx->eob_flag_cdf256[plane][eob_multi_ctx], eob_pt - 1,
9);
break;
case 5:
++counts->eob_multi512[plane][eob_multi_ctx][eob_pt - 1];
if (allow_update_cdf)
update_cdf(ec_ctx->eob_flag_cdf512[plane][eob_multi_ctx], eob_pt - 1,
10);
break;
case 6:
default:
++counts->eob_multi1024[plane][eob_multi_ctx][eob_pt - 1];
if (allow_update_cdf)
update_cdf(ec_ctx->eob_flag_cdf1024[plane][eob_multi_ctx], eob_pt - 1,
11);
break;
}
#else
for (int i = 1; i < max_eob_pt; i++) {
int eob_pos_ctx = av1_get_eob_pos_ctx(tx_type, i);
counts->eob_flag[txs_ctx][plane][eob_pos_ctx][eob_pt == i]++;
......@@ -171,6 +219,7 @@ void av1_update_eob_context(int eob, int seg_eob, TX_SIZE tx_size,
break;
}
}
#endif
if (k_eob_offset_bits[eob_pt] > 0) {
int eob_shift = k_eob_offset_bits[eob_pt] - 1;
......@@ -182,12 +231,22 @@ void av1_update_eob_context(int eob, int seg_eob, TX_SIZE tx_size,
}
static int get_eob_cost(int eob, int seg_eob,
const LV_MAP_COEFF_COST *txb_costs, TX_TYPE tx_type) {
#if CONFIG_LV_MAP_MULTI
const LV_MAP_EOB_COST *txb_eob_costs,
const LV_MAP_COEFF_COST *txb_costs,
#else
const LV_MAP_COEFF_COST *txb_costs,
#endif
TX_TYPE tx_type) {
int eob_extra, dummy;
const int eob_pt = get_eob_pos_token(eob, &eob_extra);
const int max_eob_pt = get_eob_pos_token(seg_eob, &dummy);
int eob_cost = 0;
#if CONFIG_LV_MAP_MULTI
(void)max_eob_pt;
const int eob_multi_ctx = (tx_type_to_class[tx_type] == TX_CLASS_2D) ? 0 : 1;
eob_cost = txb_eob_costs->eob_cost[eob_multi_ctx][eob_pt - 1];
#else
for (int i = 1; i < max_eob_pt; i++) {
int eob_pos_ctx = av1_get_eob_pos_ctx(tx_type, i);
eob_cost += txb_costs->eob_cost[eob_pos_ctx][eob_pt == i];
......@@ -195,6 +254,8 @@ static int get_eob_cost(int eob, int seg_eob,
break;
}
}
#endif
if (k_eob_offset_bits[eob_pt] > 0) {
int eob_shift = k_eob_offset_bits[eob_pt] - 1;
int bit = (eob_extra & (1 << eob_shift)) ? 1 : 0;
......@@ -423,6 +484,42 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd,
const int eob_pt = get_eob_pos_token(eob, &eob_extra);
const int max_eob_pt = get_eob_pos_token(seg_eob, &dummy);
#if CONFIG_LV_MAP_MULTI
(void)max_eob_pt;
const int eob_multi_size = txsize_log2_minus4[tx_size];
const int eob_multi_ctx = (tx_type_to_class[tx_type] == TX_CLASS_2D) ? 0 : 1;
switch (eob_multi_size) {
case 0:
aom_write_symbol(w, eob_pt - 1,
ec_ctx->eob_flag_cdf16[plane_type][eob_multi_ctx], 5);
break;
case 1:
aom_write_symbol(w, eob_pt - 1,
ec_ctx->eob_flag_cdf32[plane_type][eob_multi_ctx], 6);
break;
case 2:
aom_write_symbol(w, eob_pt - 1,
ec_ctx->eob_flag_cdf64[plane_type][eob_multi_ctx], 7);
break;
case 3:
aom_write_symbol(w, eob_pt - 1,
ec_ctx->eob_flag_cdf128[plane_type][eob_multi_ctx], 8);
break;
case 4:
aom_write_symbol(w, eob_pt - 1,
ec_ctx->eob_flag_cdf256[plane_type][eob_multi_ctx], 9);
break;
case 5:
aom_write_symbol(w, eob_pt - 1,
ec_ctx->eob_flag_cdf512[plane_type][eob_multi_ctx], 10);
break;
default:
aom_write_symbol(w, eob_pt - 1,
ec_ctx->eob_flag_cdf1024[plane_type][eob_multi_ctx], 11);
break;
}
#else
for (int i = 1; i < max_eob_pt; i++) {
int eob_pos_ctx = av1_get_eob_pos_ctx(tx_type, i);
......@@ -434,6 +531,7 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd,
break;
}
}
#endif
if (k_eob_offset_bits[eob_pt] > 0) {
int eob_shift = k_eob_offset_bits[eob_pt] - 1;
......@@ -668,6 +766,18 @@ int av1_cost_coeffs_txb(const AV1_COMMON *const cm, const MACROBLOCK *x,
const LV_MAP_COEFF_COST *const coeff_costs =
&x->coeff_costs[txs_ctx][plane_type];
#if CONFIG_LV_MAP_MULTI
const int eob_multi_size = txsize_log2_minus4[tx_size];
const LV_MAP_EOB_COST *const eob_costs =
&x->eob_costs[eob_multi_size][plane_type];
#endif
cost = 0;
if (eob == 0) {
cost = coeff_costs->txb_skip_cost[txb_skip_ctx][1];
return cost;
}
cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0];
av1_txb_init_levels(qcoeff, width, height, levels);
......@@ -677,7 +787,13 @@ int av1_cost_coeffs_txb(const AV1_COMMON *const cm, const MACROBLOCK *x,
#endif
const int seg_eob = av1_get_max_eob(tx_size);
int eob_cost = get_eob_cost(eob, seg_eob, coeff_costs, tx_type);
int eob_cost = get_eob_cost(eob, seg_eob,
#if CONFIG_LV_MAP_MULTI
eob_costs, coeff_costs,
#else
coeff_costs,
#endif
tx_type);
#if !CONFIG_LV_MAP_MULTI
av1_get_br_level_counts(levels, width, height, level_counts);
#endif
......@@ -1284,6 +1400,7 @@ int try_level_down(int coeff_idx, const TxbCache *txb_cache,
return accu_cost_diff;
}
#if !CONFIG_LV_MAP_MULTI
static int get_low_coeff_cost(int coeff_idx, const TxbCache *txb_cache,
const LV_MAP_COEFF_COST *txb_costs,
const TxbInfo *txb_info) {
......@@ -1317,6 +1434,7 @@ static int get_low_coeff_cost(int coeff_idx, const TxbCache *txb_cache,
#endif
return cost;
}
#endif
static INLINE void set_eob(TxbInfo *txb_info, int eob) {
txb_info->eob = eob;
......@@ -1340,6 +1458,7 @@ static INLINE int get_eob_ctx(const int coeff_idx, // raster order
return 0;
}
#if !CONFIG_LV_MAP_MULTI
// TODO(angiebird): add static to this function once it's called
int try_change_eob(int *new_eob, int coeff_idx, const TxbCache *txb_cache,
const LV_MAP_COEFF_COST *txb_costs, TxbInfo *txb_info,
......@@ -1386,6 +1505,7 @@ int try_change_eob(int *new_eob, int coeff_idx, const TxbCache *txb_cache,
}
return cost_diff;
}
#endif
// TODO(angiebird): add static to this function it's called
void update_level_down(const int coeff_idx, TxbCache *const txb_cache,
......@@ -1704,6 +1824,7 @@ void test_try_change_eob(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs,
}
#endif
#if !CONFIG_LV_MAP_MULTI
void try_level_down_facade(LevelDownStats *stats, int scan_idx,
const TxbCache *txb_cache,
const LV_MAP_COEFF_COST *txb_costs,
......@@ -1761,9 +1882,13 @@ void try_level_down_facade(LevelDownStats *stats, int scan_idx,
if (stats->rd_diff < 0) stats->update = 1;
return;
}
#endif
#if 1
static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs,
#if CONFIG_LV_MAP_MULTI
const LV_MAP_EOB_COST *txb_eob_costs,
#endif
TxbCache *txb_cache, int dry_run, int fast_mode) {
(void)fast_mode;
(void)txb_cache;
......@@ -1808,8 +1933,13 @@ static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs,
// forward optimize the nz_map`
const int init_eob = txb_info->eob;
const int seg_eob = txb_info->seg_eob;
const int eob_cost =
get_eob_cost(init_eob, seg_eob, txb_costs, txb_info->tx_type);
const int eob_cost = get_eob_cost(init_eob, seg_eob,
#if CONFIG_LV_MAP_MULTI
txb_eob_costs, txb_costs,
#else
txb_costs,
#endif
txb_info->tx_type);
// backward optimize the level-k map
int64_t accu_rate = eob_cost;
......@@ -1834,8 +1964,13 @@ static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs,
} else {
if (has_nz_tail < 2) {
// check if it is better to make this the last significant coefficient
int cur_eob_rate =
get_eob_cost(si + 1, seg_eob, txb_costs, txb_info->tx_type);
int cur_eob_rate = get_eob_cost(si + 1, seg_eob,
#if CONFIG_LV_MAP_MULTI
txb_eob_costs, txb_costs,
#else
txb_costs,
#endif
txb_info->tx_type);
cur_eob_rd_cost = RDCOST(txb_info->rdmult, cur_eob_rate, 0);
prev_eob_rd_cost =
#if CONFIG_LV_MAP_MULTI
......@@ -2083,6 +2218,11 @@ int av1_optimize_txb(const AV1_COMMON *cm, MACROBLOCK *x, int plane,
int dq = get_dq_profile(x->qindex, is_inter, plane_type);
const dequant_val_type_nuq *dequant_val = p->dequant_val_nuq_QTX[dq];
#endif // CONFIG_NEW_QUANT
#if CONFIG_LV_MAP_MULTI
const int eob_multi_size = txsize_log2_minus4[tx_size];
const LV_MAP_EOB_COST txb_eob_costs =
x->eob_costs[eob_multi_size][plane_type];
#endif
#if CONFIG_DAALA_TX
const int shift = 0;
......@@ -2123,7 +2263,11 @@ int av1_optimize_txb(const AV1_COMMON *cm, MACROBLOCK *x, int plane,
av1_txb_init_levels(qcoeff, width, height, levels);
const int update = optimize_txb(&txb_info, &txb_costs, NULL, 0, fast_mode);
const int update = optimize_txb(&txb_info, &txb_costs,
#if CONFIG_LV_MAP_MULTI
&txb_eob_costs,
#endif
NULL, 0, fast_mode);
if (update) {
p->eobs[block] = txb_info.eob;
......
......@@ -417,6 +417,35 @@ void av1_set_mvcost(MACROBLOCK *x, MV_REFERENCE_FRAME ref_frame, int ref,
#if CONFIG_LV_MAP
void av1_fill_coeff_costs(MACROBLOCK *x, FRAME_CONTEXT *fc) {
#if CONFIG_LV_MAP_MULTI
for (int eob_multi_size = 0; eob_multi_size < 7; ++eob_multi_size) {
for (int plane = 0; plane < PLANE_TYPES; ++plane) {
LV_MAP_EOB_COST *pcost = &x->eob_costs[eob_multi_size][plane];
for (int ctx = 0; ctx < 2; ++ctx) {
aom_cdf_prob *pcdf;
switch (eob_multi_size) {
case 0: pcdf = fc->eob_flag_cdf16[plane][ctx]; break;
case 1: pcdf = fc->eob_flag_cdf32[plane][ctx]; break;
case 2: pcdf = fc->eob_flag_cdf64[plane][ctx]; break;
case 3: pcdf = fc->eob_flag_cdf128[plane][ctx]; break;
case 4: pcdf = fc->eob_flag_cdf256[plane][ctx]; break;
case 5: pcdf = fc->eob_flag_cdf512[plane][ctx]; break;
case 6:
default: pcdf = fc->eob_flag_cdf1024[plane][ctx]; break;
}
av1_cost_tokens_from_cdf(pcost->eob_cost[ctx], pcdf, NULL);
#if 0
printf("[%d, %d, %d]: ", eob_multi_size, plane, ctx);
for (int i = 0; i < 5; i++) {
printf("(%d, %d, %d) ", i, pcost->eob_cost[ctx][i], pcdf[i]);
}
printf("\n");
#endif
}
}
}
#endif
for (int tx_size = 0; tx_size < TX_SIZES; ++tx_size) {
for (int plane = 0; plane < PLANE_TYPES; ++plane) {
LV_MAP_COEFF_COST *pcost = &x->coeff_costs[tx_size][plane];
......@@ -437,10 +466,10 @@ void av1_fill_coeff_costs(MACROBLOCK *x, FRAME_CONTEXT *fc) {
for (int ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx)
av1_cost_tokens_from_cdf(pcost->nz_map_cost[ctx],
fc->nz_map_cdf[tx_size][plane][ctx], NULL);
#endif
for (int ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx)
av1_cost_tokens_from_cdf(pcost->eob_cost[ctx],
fc->eob_flag_cdf[tx_size][plane][ctx], NULL);
#endif
for (int ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx)
av1_cost_tokens_from_cdf(pcost->eob_extra_cost[ctx],
......
......@@ -593,10 +593,6 @@ int main(int argc, const char **argv) {
&fc.eob_flag[0][0][0][0], probsfile, 4, cts_each_dim, NULL, 1,
"static const aom_prob "
"default_eob_flag[TX_SIZES][PLANE_TYPES][EOB_COEF_CONTEXTS]");
optimize_cdf_table(&fc.eob_flag[0][0][0][0], probsfile, 4, cts_each_dim,
"static const aom_cdf_prob "
"default_eob_flag_cdf[TX_SIZES][PLANE_TYPES][EOB_COEF_"
"CONTEXTS][CDF_SIZE(2)]");
cts_each_dim[0] = TX_SIZES;
cts_each_dim[1] = PLANE_TYPES;
......@@ -608,6 +604,54 @@ int main(int argc, const char **argv) {
"default_eob_extra[TX_SIZES][PLANE_TYPES][EOB_COEF_CONTEXTS]");
#if CONFIG_LV_MAP_MULTI
cts_each_dim[0] = PLANE_TYPES;
cts_each_dim[1] = 2;
cts_each_dim[2] = 5;