Commit 1015a347 authored by Linfeng Zhang's avatar Linfeng Zhang

Add levels array in av1_optimize_txb()

av1_txb_init_levels() has prefix 'av1' because it will be optimized
and not static later.

Change-Id: I988da817335f122522a76c4412207c3a6fdd6b71
parent 2edfc37b
...@@ -12,9 +12,6 @@ ...@@ -12,9 +12,6 @@
#ifndef AV1_COMMON_TXB_COMMON_H_ #ifndef AV1_COMMON_TXB_COMMON_H_
#define AV1_COMMON_TXB_COMMON_H_ #define AV1_COMMON_TXB_COMMON_H_
#define REDUCE_CONTEXT_DEPENDENCY 0
#define MIN_SCAN_IDX_REDUCE_CONTEXT_DEPENDENCY 0
extern const int16_t k_eob_group_start[12]; extern const int16_t k_eob_group_start[12];
extern const int16_t k_eob_offset_bits[12]; extern const int16_t k_eob_offset_bits[12];
int16_t get_eob_pos_token(int eob, int16_t *extra); int16_t get_eob_pos_token(int eob, int16_t *extra);
...@@ -336,9 +333,8 @@ static const int sig_ref_offset_horiz[SIG_REF_OFFSET_NUM][2] = { ...@@ -336,9 +333,8 @@ static const int sig_ref_offset_horiz[SIG_REF_OFFSET_NUM][2] = {
}; };
#if USE_CAUSAL_BASE_CTX #if USE_CAUSAL_BASE_CTX
static INLINE int get_nz_count_mag(const tran_low_t *tcoeffs, int bwl, static INLINE int get_nz_count_mag(const void *tcoeffs, int bwl, int height,
int height, int row, int col, int row, int col, TX_CLASS tx_class,
TX_CLASS tx_class,
const int coeff_is_byte_flag, int *mag) { const int coeff_is_byte_flag, int *mag) {
int count = 0; int count = 0;
*mag = 0; *mag = 0;
...@@ -371,7 +367,7 @@ static INLINE int get_nz_count_mag(const tran_low_t *tcoeffs, int bwl, ...@@ -371,7 +367,7 @@ static INLINE int get_nz_count_mag(const tran_low_t *tcoeffs, int bwl,
return count; return count;
} }
#endif #endif
static INLINE int get_nz_count(const tran_low_t *tcoeffs, int bwl, int height, static INLINE int get_nz_count(const void *tcoeffs, int bwl, int height,
int row, int col, TX_CLASS tx_class, int row, int col, TX_CLASS tx_class,
const int coeff_is_byte_flag) { const int coeff_is_byte_flag) {
int count = 0; int count = 0;
......
...@@ -70,8 +70,8 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd, ...@@ -70,8 +70,8 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd,
const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2; const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2;
const int height = tx_size_high[tx_size]; const int height = tx_size_high[tx_size];
int cul_level = 0; int cul_level = 0;
uint8_t levels[64 * 64]; uint8_t levels[MAX_TX_SQUARE];
int8_t signs[64 * 64]; int8_t signs[MAX_TX_SQUARE];
memset(tcoeffs, 0, sizeof(*tcoeffs) * seg_eob); memset(tcoeffs, 0, sizeof(*tcoeffs) * seg_eob);
......
...@@ -189,8 +189,9 @@ static int get_eob_cost(int eob, int seg_eob, ...@@ -189,8 +189,9 @@ static int get_eob_cost(int eob, int seg_eob,
return eob_cost; return eob_cost;
} }
extern int get_coeff_cost(tran_low_t qc, int scan_idx, TxbInfo *txb_info, static int get_coeff_cost(const tran_low_t qc, const int scan_idx,
const LV_MAP_COEFF_COST *txb_costs); const TxbInfo *const txb_info,
const LV_MAP_COEFF_COST *const txb_costs);
void get_dist_cost_stats(LevelDownStats *stats, int scan_idx, void get_dist_cost_stats(LevelDownStats *stats, int scan_idx,
const LV_MAP_COEFF_COST *txb_costs, const LV_MAP_COEFF_COST *txb_costs,
...@@ -198,6 +199,7 @@ void get_dist_cost_stats(LevelDownStats *stats, int scan_idx, ...@@ -198,6 +199,7 @@ void get_dist_cost_stats(LevelDownStats *stats, int scan_idx,
const int16_t *scan = txb_info->scan_order->scan; const int16_t *scan = txb_info->scan_order->scan;
const int coeff_idx = scan[scan_idx]; const int coeff_idx = scan[scan_idx];
const tran_low_t qc = txb_info->qcoeff[coeff_idx]; const tran_low_t qc = txb_info->qcoeff[coeff_idx];
const uint8_t *const levels = txb_info->levels;
stats->new_eob = -1; stats->new_eob = -1;
stats->update = 0; stats->update = 0;
...@@ -230,9 +232,8 @@ void get_dist_cost_stats(LevelDownStats *stats, int scan_idx, ...@@ -230,9 +232,8 @@ void get_dist_cost_stats(LevelDownStats *stats, int scan_idx,
stats->rate_low = low_qc_cost; stats->rate_low = low_qc_cost;
stats->rd_low = RDCOST(txb_info->rdmult, stats->rate_low, stats->dist_low); stats->rd_low = RDCOST(txb_info->rdmult, stats->rate_low, stats->dist_low);
int coeff_ctx = int coeff_ctx = get_nz_map_ctx(levels, scan_idx, scan, txb_info->bwl,
get_nz_map_ctx(txb_info->qcoeff, scan_idx, scan, txb_info->bwl, txb_info->height, txb_info->tx_type, 1);
txb_info->height, txb_info->tx_type, 0);
if ((stats->rd_low < stats->rd) && (stats->low_qc == 0)) { if ((stats->rd_low < stats->rd) && (stats->low_qc == 0)) {
stats->nz_rate = txb_costs->nz_map_cost[coeff_ctx][0]; stats->nz_rate = txb_costs->nz_map_cost[coeff_ctx][0];
...@@ -241,12 +242,26 @@ void get_dist_cost_stats(LevelDownStats *stats, int scan_idx, ...@@ -241,12 +242,26 @@ void get_dist_cost_stats(LevelDownStats *stats, int scan_idx,
} }
} }
static void update_coeff(int coeff_idx, tran_low_t qc, TxbInfo *txb_info) { static INLINE void update_qcoeff(const int coeff_idx, const tran_low_t qc,
const TxbInfo *const txb_info) {
txb_info->qcoeff[coeff_idx] = qc; txb_info->qcoeff[coeff_idx] = qc;
txb_info->levels[coeff_idx] = (uint8_t)clamp(abs(qc), 0, UINT8_MAX);
}
static INLINE void update_coeff(const int coeff_idx, const tran_low_t qc,
const TxbInfo *const txb_info) {
update_qcoeff(coeff_idx, qc, txb_info);
const int dqv = txb_info->dequant[coeff_idx != 0]; const int dqv = txb_info->dequant[coeff_idx != 0];
txb_info->dqcoeff[coeff_idx] = qcoeff_to_dqcoeff(qc, dqv, txb_info->shift); txb_info->dqcoeff[coeff_idx] = qcoeff_to_dqcoeff(qc, dqv, txb_info->shift);
} }
static INLINE void av1_txb_init_levels(const tran_low_t *const coeff,
const int size, uint8_t *const levels) {
for (int i = 0; i < size; i++) {
levels[i] = (uint8_t)clamp(abs(coeff[i]), 0, UINT8_MAX);
}
}
void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd, void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd,
aom_writer *w, int blk_row, int blk_col, int block, aom_writer *w, int blk_row, int blk_col, int block,
int plane, TX_SIZE tx_size, const tran_low_t *tcoeff, int plane, TX_SIZE tx_size, const tran_low_t *tcoeff,
...@@ -264,8 +279,7 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd, ...@@ -264,8 +279,7 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd,
const int height = tx_size_high[tx_size]; const int height = tx_size_high[tx_size];
uint16_t update_eob = 0; uint16_t update_eob = 0;
FRAME_CONTEXT *ec_ctx = xd->tile_ctx; FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
uint8_t levels[64 * 64]; uint8_t levels[MAX_TX_SQUARE];
int8_t signs[64 * 64];
(void)blk_row; (void)blk_row;
(void)blk_col; (void)blk_col;
...@@ -275,10 +289,7 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd, ...@@ -275,10 +289,7 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd,
if (eob == 0) return; if (eob == 0) return;
for (int i = 0; i < seg_eob; i++) { av1_txb_init_levels(tcoeff, seg_eob, levels);
levels[i] = (uint8_t)clamp(abs(tcoeff[i]), 0, UINT8_MAX);
signs[i] = (int8_t)(tcoeff[i] < 0);
}
#if CONFIG_TXK_SEL #if CONFIG_TXK_SEL
av1_write_tx_type(cm, xd, blk_row, blk_col, block, plane, av1_write_tx_type(cm, xd, blk_row, blk_col, block, plane,
...@@ -345,7 +356,7 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd, ...@@ -345,7 +356,7 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd,
#else #else
for (int i = 1; i < eob; ++i) { for (int i = 1; i < eob; ++i) {
c = eob - 1 - i; c = eob - 1 - i;
int coeff_ctx = get_nz_map_ctx(tcoeff, c, scan, bwl, height, tx_type, 0); int coeff_ctx = get_nz_map_ctx(levels, c, scan, bwl, height, tx_type, 1);
tran_low_t v = tcoeff[scan[c]]; tran_low_t v = tcoeff[scan[c]];
int is_nz = (v != 0); int is_nz = (v != 0);
...@@ -357,7 +368,7 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd, ...@@ -357,7 +368,7 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd,
for (int i = 0; i < NUM_BASE_LEVELS; ++i) { for (int i = 0; i < NUM_BASE_LEVELS; ++i) {
update_eob = 0; update_eob = 0;
for (c = eob - 1; c >= 0; --c) { for (c = eob - 1; c >= 0; --c) {
const int level = levels[scan[c]]; const tran_low_t level = abs(tcoeff[scan[c]]);
int ctx; int ctx;
if (level <= i) continue; if (level <= i) continue;
...@@ -380,8 +391,9 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd, ...@@ -380,8 +391,9 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd,
// Loop to code all signs in the transform block, // Loop to code all signs in the transform block,
// starting with the sign of DC (if applicable) // starting with the sign of DC (if applicable)
for (c = 0; c < eob; ++c) { for (c = 0; c < eob; ++c) {
const int level = levels[scan[c]]; const tran_low_t v = tcoeff[scan[c]];
const int sign = signs[scan[c]]; const tran_low_t level = abs(v);
const int sign = (v < 0) ? 1 : 0;
if (level == 0) continue; if (level == 0) continue;
if (c == 0) { if (c == 0) {
...@@ -397,7 +409,7 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd, ...@@ -397,7 +409,7 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd,
} }
for (c = update_eob; c >= 0; --c) { for (c = update_eob; c >= 0; --c) {
const int level = levels[scan[c]]; const tran_low_t level = abs(tcoeff[scan[c]]);
int idx; int idx;
int ctx; int ctx;
...@@ -561,6 +573,7 @@ int av1_cost_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCK *x, int plane, ...@@ -561,6 +573,7 @@ int av1_cost_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCK *x, int plane,
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi); const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
const int16_t *scan = scan_order->scan; const int16_t *scan = scan_order->scan;
uint8_t levels[MAX_TX_SQUARE];
LV_MAP_COEFF_COST *coeff_costs = &x->coeff_costs[txs_ctx][plane_type]; LV_MAP_COEFF_COST *coeff_costs = &x->coeff_costs[txs_ctx][plane_type];
...@@ -572,6 +585,8 @@ int av1_cost_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCK *x, int plane, ...@@ -572,6 +585,8 @@ int av1_cost_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCK *x, int plane,
} }
cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0]; cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0];
av1_txb_init_levels(qcoeff, tx_size_2d[tx_size], levels);
#if CONFIG_TXK_SEL #if CONFIG_TXK_SEL
cost += av1_tx_type_cost(cm, x, xd, mbmi->sb_type, plane, tx_size, tx_type); cost += av1_tx_type_cost(cm, x, xd, mbmi->sb_type, plane, tx_size, tx_type);
#endif #endif
...@@ -593,7 +608,7 @@ int av1_cost_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCK *x, int plane, ...@@ -593,7 +608,7 @@ int av1_cost_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCK *x, int plane,
if (c < eob - 1) { if (c < eob - 1) {
#if !USE_CAUSAL_BASE_CTX #if !USE_CAUSAL_BASE_CTX
int coeff_ctx = get_nz_map_ctx(qcoeff, c, scan, bwl, height, tx_type, 0); int coeff_ctx = get_nz_map_ctx(levels, c, scan, bwl, height, tx_type, 1);
#endif #endif
cost += coeff_costs->nz_map_cost[coeff_ctx][is_nz]; cost += coeff_costs->nz_map_cost[coeff_ctx][is_nz];
} }
...@@ -925,14 +940,14 @@ static int try_neighbor_level_down_nz(int coeff_idx, int nb_coeff_idx, ...@@ -925,14 +940,14 @@ static int try_neighbor_level_down_nz(int coeff_idx, int nb_coeff_idx,
if (nb_scan_idx < scan_idx) { if (nb_scan_idx < scan_idx) {
const int count = txb_cache->nz_count_arr[coeff_idx]; const int count = txb_cache->nz_count_arr[coeff_idx];
assert(count > 0); assert(count > 0);
txb_info->qcoeff[nb_coeff_idx] = get_lower_coeff(nb_coeff); update_qcoeff(nb_coeff_idx, get_lower_coeff(nb_coeff), txb_info);
const int new_ctx = get_nz_map_ctx_from_count( const int new_ctx = get_nz_map_ctx_from_count(
count - 1, coeff_idx, txb_info->bwl, txb_info->height, count - 1, coeff_idx, txb_info->bwl, txb_info->height,
#if USE_CAUSAL_BASE_CTX #if USE_CAUSAL_BASE_CTX
0, 0,
#endif #endif
txb_info->tx_type); txb_info->tx_type);
txb_info->qcoeff[nb_coeff_idx] = nb_coeff; update_qcoeff(nb_coeff_idx, nb_coeff, txb_info);
const int ctx = txb_cache->nz_ctx_arr[coeff_idx]; const int ctx = txb_cache->nz_ctx_arr[coeff_idx];
const int is_nz = abs_qc > 0; const int is_nz = abs_qc > 0;
const int org_cost = txb_costs->nz_map_cost[ctx][is_nz]; const int org_cost = txb_costs->nz_map_cost[ctx][is_nz];
...@@ -1261,15 +1276,13 @@ int try_change_eob(int *new_eob, int coeff_idx, const TxbCache *txb_cache, ...@@ -1261,15 +1276,13 @@ int try_change_eob(int *new_eob, int coeff_idx, const TxbCache *txb_cache,
} }
// TODO(angiebird): add static to this function it's called // TODO(angiebird): add static to this function it's called
void update_level_down(int coeff_idx, TxbCache *txb_cache, TxbInfo *txb_info) { void update_level_down(const int coeff_idx, TxbCache *const txb_cache,
TxbInfo *const txb_info) {
const tran_low_t qc = txb_info->qcoeff[coeff_idx]; const tran_low_t qc = txb_info->qcoeff[coeff_idx];
const int abs_qc = abs(qc); const int abs_qc = abs(qc);
if (qc == 0) return; if (qc == 0) return;
const tran_low_t low_coeff = get_lower_coeff(qc); const tran_low_t low_coeff = get_lower_coeff(qc);
txb_info->qcoeff[coeff_idx] = low_coeff; update_coeff(coeff_idx, low_coeff, txb_info);
const int dqv = txb_info->dequant[coeff_idx != 0];
txb_info->dqcoeff[coeff_idx] =
qcoeff_to_dqcoeff(low_coeff, dqv, txb_info->shift);
const int row = coeff_idx >> txb_info->bwl; const int row = coeff_idx >> txb_info->bwl;
const int col = coeff_idx - (row << txb_info->bwl); const int col = coeff_idx - (row << txb_info->bwl);
...@@ -1300,8 +1313,8 @@ void update_level_down(int coeff_idx, TxbCache *txb_cache, TxbInfo *txb_info) { ...@@ -1300,8 +1313,8 @@ void update_level_down(int coeff_idx, TxbCache *txb_cache, TxbInfo *txb_info) {
0, 0,
#endif #endif
txb_info->tx_type); txb_info->tx_type);
// int ref_ctx = get_nz_map_ctx(txb_info->qcoeff, nb_coeff_idx, // int ref_ctx = get_nz_map_ctx(txb_info->levels, nb_coeff_idx,
// txb_info->bwl, tx_type, 0); // txb_info->bwl, tx_type, 1);
// if (ref_ctx != txb_cache->nz_ctx_arr[nb_coeff_idx]) // if (ref_ctx != txb_cache->nz_ctx_arr[nb_coeff_idx])
// printf("nz ctx %d ref_ctx %d\n", // printf("nz ctx %d ref_ctx %d\n",
// txb_cache->nz_ctx_arr[nb_coeff_idx], ref_ctx); // txb_cache->nz_ctx_arr[nb_coeff_idx], ref_ctx);
...@@ -1382,8 +1395,9 @@ void update_level_down(int coeff_idx, TxbCache *txb_cache, TxbInfo *txb_info) { ...@@ -1382,8 +1395,9 @@ void update_level_down(int coeff_idx, TxbCache *txb_cache, TxbInfo *txb_info) {
} }
} }
int get_coeff_cost(tran_low_t qc, int scan_idx, TxbInfo *txb_info, static int get_coeff_cost(const tran_low_t qc, const int scan_idx,
const LV_MAP_COEFF_COST *txb_costs) { const TxbInfo *const txb_info,
const LV_MAP_COEFF_COST *const txb_costs) {
const TXB_CTX *txb_ctx = txb_info->txb_ctx; const TXB_CTX *txb_ctx = txb_info->txb_ctx;
const int is_nz = (qc != 0); const int is_nz = (qc != 0);
const tran_low_t abs_qc = abs(qc); const tran_low_t abs_qc = abs(qc);
...@@ -1399,8 +1413,8 @@ int get_coeff_cost(tran_low_t qc, int scan_idx, TxbInfo *txb_info, ...@@ -1399,8 +1413,8 @@ int get_coeff_cost(tran_low_t qc, int scan_idx, TxbInfo *txb_info,
if (scan_idx < txb_info->eob - 1) { if (scan_idx < txb_info->eob - 1) {
#if !USE_CAUSAL_BASE_CTX #if !USE_CAUSAL_BASE_CTX
int coeff_ctx = int coeff_ctx =
get_nz_map_ctx(txb_info->qcoeff, scan_idx, scan, txb_info->bwl, get_nz_map_ctx(txb_info->levels, scan_idx, scan, txb_info->bwl,
txb_info->height, txb_info->tx_type, 0); txb_info->height, txb_info->tx_type, 1);
#endif #endif
cost += txb_costs->nz_map_cost[coeff_ctx][is_nz]; cost += txb_costs->nz_map_cost[coeff_ctx][is_nz];
} }
...@@ -1474,7 +1488,7 @@ static int try_level_down_ref(int coeff_idx, const LV_MAP_COEFF_COST *txb_costs, ...@@ -1474,7 +1488,7 @@ static int try_level_down_ref(int coeff_idx, const LV_MAP_COEFF_COST *txb_costs,
org_cost += cost; org_cost += cost;
} }
} }
txb_info->qcoeff[coeff_idx] = get_lower_coeff(qc); update_qcoeff(coeff_idx, get_lower_coeff(qc), txb_info);
int new_cost = 0; int new_cost = 0;
for (int i = 0; i < ALL_REF_OFFSET_NUM; ++i) { for (int i = 0; i < ALL_REF_OFFSET_NUM; ++i) {
int nb_row = row - all_ref_offset[i][0]; int nb_row = row - all_ref_offset[i][0];
...@@ -1491,7 +1505,7 @@ static int try_level_down_ref(int coeff_idx, const LV_MAP_COEFF_COST *txb_costs, ...@@ -1491,7 +1505,7 @@ static int try_level_down_ref(int coeff_idx, const LV_MAP_COEFF_COST *txb_costs,
new_cost += cost; new_cost += cost;
} }
} }
txb_info->qcoeff[coeff_idx] = qc; update_qcoeff(coeff_idx, qc, txb_info);
return new_cost - org_cost; return new_cost - org_cost;
} }
...@@ -1551,11 +1565,11 @@ void test_try_change_eob(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs, ...@@ -1551,11 +1565,11 @@ void test_try_change_eob(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs,
int org_eob = txb_info->eob; int org_eob = txb_info->eob;
int cost = get_txb_cost(txb_info, txb_costs); int cost = get_txb_cost(txb_info, txb_costs);
txb_info->qcoeff[last_ci] = get_lower_coeff(last_coeff); update_qcoeff(last_ci, get_lower_coeff(last_coeff), txb_info);
set_eob(txb_info, new_eob); set_eob(txb_info, new_eob);
int new_cost = get_txb_cost(txb_info, txb_costs); int new_cost = get_txb_cost(txb_info, txb_costs);
set_eob(txb_info, org_eob); set_eob(txb_info, org_eob);
txb_info->qcoeff[last_ci] = last_coeff; update_qcoeff(last_ci, last_coeff, txb_info);
int ref_cost_diff = -cost + new_cost; int ref_cost_diff = -cost + new_cost;
if (cost_diff != ref_cost_diff) if (cost_diff != ref_cost_diff)
...@@ -1632,15 +1646,19 @@ static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs, ...@@ -1632,15 +1646,19 @@ static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs,
tran_low_t *org_qcoeff = txb_info->qcoeff; tran_low_t *org_qcoeff = txb_info->qcoeff;
tran_low_t *org_dqcoeff = txb_info->dqcoeff; tran_low_t *org_dqcoeff = txb_info->dqcoeff;
uint8_t *const org_levels = txb_info->levels;
tran_low_t tmp_qcoeff[MAX_TX_SQUARE]; tran_low_t tmp_qcoeff[MAX_TX_SQUARE];
tran_low_t tmp_dqcoeff[MAX_TX_SQUARE]; tran_low_t tmp_dqcoeff[MAX_TX_SQUARE];
uint8_t tmp_levels[MAX_TX_SQUARE];
const int org_eob = txb_info->eob; const int org_eob = txb_info->eob;
if (dry_run) { if (dry_run) {
memcpy(tmp_qcoeff, org_qcoeff, sizeof(org_qcoeff[0]) * max_eob); memcpy(tmp_qcoeff, org_qcoeff, sizeof(org_qcoeff[0]) * max_eob);
memcpy(tmp_dqcoeff, org_dqcoeff, sizeof(org_dqcoeff[0]) * max_eob); memcpy(tmp_dqcoeff, org_dqcoeff, sizeof(org_dqcoeff[0]) * max_eob);
memcpy(tmp_levels, org_levels, sizeof(org_levels[0]) * max_eob);
txb_info->qcoeff = tmp_qcoeff; txb_info->qcoeff = tmp_qcoeff;
txb_info->dqcoeff = tmp_dqcoeff; txb_info->dqcoeff = tmp_dqcoeff;
txb_info->levels = tmp_levels;
} }
const int16_t *scan = txb_info->scan_order->scan; const int16_t *scan = txb_info->scan_order->scan;
...@@ -1741,6 +1759,7 @@ static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs, ...@@ -1741,6 +1759,7 @@ static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs,
if (dry_run) { if (dry_run) {
txb_info->qcoeff = org_qcoeff; txb_info->qcoeff = org_qcoeff;
txb_info->dqcoeff = org_dqcoeff; txb_info->dqcoeff = org_dqcoeff;
txb_info->levels = org_levels;
set_eob(txb_info, org_eob); set_eob(txb_info, org_eob);
} }
return update; return update;
...@@ -1766,15 +1785,19 @@ static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs, ...@@ -1766,15 +1785,19 @@ static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs,
tran_low_t *org_qcoeff = txb_info->qcoeff; tran_low_t *org_qcoeff = txb_info->qcoeff;
tran_low_t *org_dqcoeff = txb_info->dqcoeff; tran_low_t *org_dqcoeff = txb_info->dqcoeff;
uint8_t *const org_levels = txb_info->levels;
tran_low_t tmp_qcoeff[MAX_TX_SQUARE]; tran_low_t tmp_qcoeff[MAX_TX_SQUARE];
tran_low_t tmp_dqcoeff[MAX_TX_SQUARE]; tran_low_t tmp_dqcoeff[MAX_TX_SQUARE];
uint8_t tmp_levels[MAX_TX_SQUARE];
const int org_eob = txb_info->eob; const int org_eob = txb_info->eob;
if (dry_run) { if (dry_run) {
memcpy(tmp_qcoeff, org_qcoeff, sizeof(org_qcoeff[0]) * max_eob); memcpy(tmp_qcoeff, org_qcoeff, sizeof(org_qcoeff[0]) * max_eob);
memcpy(tmp_dqcoeff, org_dqcoeff, sizeof(org_dqcoeff[0]) * max_eob); memcpy(tmp_dqcoeff, org_dqcoeff, sizeof(org_dqcoeff[0]) * max_eob);
memcpy(tmp_levels, org_levels, sizeof(org_levels[0]) * max_eob);
txb_info->qcoeff = tmp_qcoeff; txb_info->qcoeff = tmp_qcoeff;
txb_info->dqcoeff = tmp_dqcoeff; txb_info->dqcoeff = tmp_dqcoeff;
txb_info->levels = tmp_levels;
} }
const int16_t *scan = txb_info->scan_order->scan; const int16_t *scan = txb_info->scan_order->scan;
...@@ -1844,6 +1867,7 @@ static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs, ...@@ -1844,6 +1867,7 @@ static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs,
if (dry_run) { if (dry_run) {
txb_info->qcoeff = org_qcoeff; txb_info->qcoeff = org_qcoeff;
txb_info->dqcoeff = org_dqcoeff; txb_info->dqcoeff = org_dqcoeff;
txb_info->levels = org_levels;
set_eob(txb_info, org_eob); set_eob(txb_info, org_eob);
} }
return update; return update;
...@@ -1882,24 +1906,15 @@ int av1_optimize_txb(const AV1_COMMON *cm, MACROBLOCK *x, int plane, ...@@ -1882,24 +1906,15 @@ int av1_optimize_txb(const AV1_COMMON *cm, MACROBLOCK *x, int plane,
const int shift = av1_get_tx_scale(tx_size); const int shift = av1_get_tx_scale(tx_size);
const int64_t rdmult = const int64_t rdmult =
(x->rdmult * plane_rd_mult[is_inter][plane_type] + 2) >> 2; (x->rdmult * plane_rd_mult[is_inter][plane_type] + 2) >> 2;
uint8_t levels[MAX_TX_SQUARE];
TxbInfo txb_info = {
qcoeff, levels, dqcoeff, tcoeff, dequant, shift,
tx_size, txs_ctx, tx_type, bwl, stride, height,
eob, seg_eob, scan_order, txb_ctx, rdmult, &cm->coeff_ctx_table
};
TxbInfo txb_info = { qcoeff, av1_txb_init_levels(qcoeff, tx_size_2d[tx_size], levels);
dqcoeff,
tcoeff,
dequant,
shift,
tx_size,
txs_ctx,
tx_type,
bwl,
stride,
height,
eob,
seg_eob,
scan_order,
txb_ctx,
rdmult,
&cm->coeff_ctx_table };
const int update = optimize_txb(&txb_info, &txb_costs, NULL, 0, fast_mode); const int update = optimize_txb(&txb_info, &txb_costs, NULL, 0, fast_mode);
...@@ -1969,14 +1984,13 @@ void av1_update_and_record_txb_context(int plane, int block, int blk_row, ...@@ -1969,14 +1984,13 @@ void av1_update_and_record_txb_context(int plane, int block, int blk_row,
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi); const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
const int16_t *scan = scan_order->scan; const int16_t *scan = scan_order->scan;
const int seg_eob = av1_get_tx_eob(&cpi->common.seg, segment_id, tx_size); const int seg_eob = av1_get_tx_eob(&cpi->common.seg, segment_id, tx_size);
int c, i; int c;
TXB_CTX txb_ctx; TXB_CTX txb_ctx;
get_txb_ctx(plane_bsize, tx_size, plane, pd->above_context + blk_col, get_txb_ctx(plane_bsize, tx_size, plane, pd->above_context + blk_col,
pd->left_context + blk_row, &txb_ctx); pd->left_context + blk_row, &txb_ctx);
const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2; const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2;
const int height = tx_size_high[tx_size]; const int height = tx_size_high[tx_size];
uint8_t levels[64 * 64]; uint8_t levels[MAX_TX_SQUARE];
int8_t signs[64 * 64];
const uint8_t allow_update_cdf = args->allow_update_cdf; const uint8_t allow_update_cdf = args->allow_update_cdf;
TX_SIZE txsize_ctx = get_txsize_context(tx_size); TX_SIZE txsize_ctx = get_txsize_context(tx_size);
...@@ -1997,10 +2011,7 @@ void av1_update_and_record_txb_context(int plane, int block, int blk_row, ...@@ -1997,10 +2011,7 @@ void av1_update_and_record_txb_context(int plane, int block, int blk_row,
return; return;
} }
for (i = 0; i < seg_eob; i++) { av1_txb_init_levels(tcoeff, tx_size_2d[tx_size], levels);
levels[i] = (uint8_t)clamp(abs(tcoeff[i]), 0, UINT8_MAX);
signs[i] = (int8_t)(tcoeff[i] < 0);
}
#if CONFIG_TXK_SEL #if CONFIG_TXK_SEL
av1_update_tx_type_count(cm, xd, blk_row, blk_col, block, plane, av1_update_tx_type_count(cm, xd, blk_row, blk_col, block, plane,
...@@ -2043,7 +2054,7 @@ void av1_update_and_record_txb_context(int plane, int block, int blk_row, ...@@ -2043,7 +2054,7 @@ void av1_update_and_record_txb_context(int plane, int block, int blk_row,
} }
} }
#else #else
int coeff_ctx = get_nz_map_ctx(tcoeff, c, scan, bwl, height, tx_type, 0); int coeff_ctx = get_nz_map_ctx(levels, c, scan, bwl, height, tx_type, 1);
if (c == eob - 1) continue; if (c == eob - 1) continue;
...@@ -2056,10 +2067,10 @@ void av1_update_and_record_txb_context(int plane, int block, int blk_row, ...@@ -2056,10 +2067,10 @@ void av1_update_and_record_txb_context(int plane, int block, int blk_row,
#if !USE_CAUSAL_BASE_CTX #if !USE_CAUSAL_BASE_CTX
// Reverse process order to handle coefficient level and sign. // Reverse process order to handle coefficient level and sign.
for (i = 0; i < NUM_BASE_LEVELS; ++i) { for (int i = 0; i < NUM_BASE_LEVELS; ++i) {
update_eob = 0; update_eob = 0;