Commit 72e0b3fa authored by Linfeng Zhang's avatar Linfeng Zhang

Clean av1_read_coeffs_txb()

Let read_nz_map() etc. output to uint8_t*.

Temporarily introduced coeff_is_byte_flag as an argument of
get_nz_count() and get_nz_map_ctx() to handle different types of input
coefficients. It helps to not duplicate too many functions.
If possible, unify types to uint8_t* later.

Change-Id: Idbbe67b7ec563f8c9299daa0fa7d05e15c8295e6
parent edb76fe5
......@@ -39,6 +39,9 @@ static const int base_ref_offset[BASE_CONTEXT_POSITION_NUM][2] = {
/* clang-format on*/
};
// TODO(linfengz): Some functions have coeff_is_byte_flag to handle different
// types of input coefficients. If possible, unify types to uint8_t* later.
static INLINE int get_level_count(const tran_low_t *tcoeffs, int bwl,
int height, int row, int col, int level,
int (*nb_offset)[2], int nb_num) {
......@@ -306,8 +309,10 @@ static int sig_ref_offset[SIG_REF_OFFSET_NUM][2] = {
};
#if REDUCE_CONTEXT_DEPENDENCY
static INLINE int get_nz_count(const tran_low_t *tcoeffs, int bwl, int height,
int row, int col, int prev_row, int prev_col) {
static INLINE int get_nz_count(const void *const tcoeffs, const int bwl,
const int height, const int row, const int col,
const int coeff_is_byte_flag, const int prev_row,
const int prev_col) {
int count = 0;
for (int idx = 0; idx < SIG_REF_OFFSET_NUM; ++idx) {
const int ref_row = row + sig_ref_offset[idx][0];
......@@ -316,13 +321,16 @@ static INLINE int get_nz_count(const tran_low_t *tcoeffs, int bwl, int height,
ref_col >= (1 << bwl) || (prev_row == ref_row && prev_col == ref_col))
continue;
const int nb_pos = (ref_row << bwl) + ref_col;
count += (tcoeffs[nb_pos] != 0);
count +=
((coeff_is_byte_flag ? ((const uint8_t *)tcoeffs)[nb_pos]
: ((const tran_low_t *)tcoeffs)[nb_pos]) != 0);
}
return count;
}
#else
static INLINE int get_nz_count(const tran_low_t *tcoeffs, int bwl, int height,
int row, int col) {
static INLINE int get_nz_count(const void *const tcoeffs, const int bwl,
const int height, const int row, const int col,
const int coeff_is_byte_flag) {
int count = 0;
for (int idx = 0; idx < SIG_REF_OFFSET_NUM; ++idx) {
const int ref_row = row + sig_ref_offset[idx][0];
......@@ -331,7 +339,9 @@ static INLINE int get_nz_count(const tran_low_t *tcoeffs, int bwl, int height,
ref_col >= (1 << bwl))
continue;
const int nb_pos = (ref_row << bwl) + ref_col;
count += (tcoeffs[nb_pos] != 0);
count +=
((coeff_is_byte_flag ? ((const uint8_t *)tcoeffs)[nb_pos]
: ((const tran_low_t *)tcoeffs)[nb_pos]) != 0);
}
return count;
}
......@@ -408,9 +418,10 @@ static INLINE int get_nz_map_ctx_from_count(int count,
return offset + 12 + ctx;
}
static INLINE int get_nz_map_ctx(const tran_low_t *tcoeffs, const int scan_idx,
const int16_t *scan, const int bwl,
const int height, TX_TYPE tx_type) {
static INLINE int get_nz_map_ctx(const void *const tcoeffs, const int scan_idx,
const int16_t *const scan, const int bwl,
const int height, const TX_TYPE tx_type,
const int coeff_is_byte_flag) {
const int coeff_idx = scan[scan_idx];
const int row = coeff_idx >> bwl;
const int col = coeff_idx - (row << bwl);
......@@ -427,9 +438,11 @@ static INLINE int get_nz_map_ctx(const tran_low_t *tcoeffs, const int scan_idx,
prev_row = -1;
prev_col = -1;
}
int count = get_nz_count(tcoeffs, bwl, height, row, col, prev_row, prev_col);
const int count = get_nz_count(tcoeffs, bwl, height, row, col,
coeff_is_byte_flag, prev_row, prev_col);
#else
int count = get_nz_count(tcoeffs, bwl, height, row, col);
const int count =
get_nz_count(tcoeffs, bwl, height, row, col, coeff_is_byte_flag);
#endif
return get_nz_map_ctx_from_count(count, coeff_idx, bwl, tx_type);
}
......
......@@ -45,11 +45,12 @@ static int read_golomb(MACROBLOCKD *xd, aom_reader *r, FRAME_COUNTS *counts) {
return x - 1;
}
static INLINE int read_nz_map(aom_reader *r, tran_low_t *tcoeffs, int plane,
const int16_t *scan, TX_SIZE tx_size,
TX_TYPE tx_type, FRAME_CONTEXT *fc,
FRAME_COUNTS *counts) {
TX_SIZE txs_ctx = get_txsize_context(tx_size);
static INLINE int read_nz_map(aom_reader *const r, uint8_t *const levels,
const int plane, const int16_t *const scan,
const TX_SIZE tx_size, const TX_TYPE tx_type,
FRAME_CONTEXT *const fc,
FRAME_COUNTS *const counts) {
const TX_SIZE txs_ctx = get_txsize_context(tx_size);
const int bwl = b_width_log2_lookup[txsize_to_bsize[tx_size]] + 2;
const int height = tx_size_high[tx_size];
#if CONFIG_CTX1D
......@@ -71,7 +72,7 @@ static INLINE int read_nz_map(aom_reader *r, tran_low_t *tcoeffs, int plane,
int c;
for (c = 0; c < seg_eob; ++c) {
int is_nz;
int coeff_ctx = get_nz_map_ctx(tcoeffs, c, scan, bwl, height, tx_type);
int coeff_ctx = get_nz_map_ctx(levels, c, scan, bwl, height, tx_type, 1);
int eob_ctx = get_eob_ctx(scan[c], txs_ctx, tx_type);
if (c < seg_eob - 1) {
......@@ -87,7 +88,7 @@ static INLINE int read_nz_map(aom_reader *r, tran_low_t *tcoeffs, int plane,
}
// set non-zero coefficient map.
tcoeffs[scan[c]] = is_nz;
levels[scan[c]] = is_nz;
if (c == seg_eob - 1) {
++c;
......@@ -112,11 +113,10 @@ static INLINE int read_nz_map(aom_reader *r, tran_low_t *tcoeffs, int plane,
}
#if CONFIG_CTX1D
static INLINE int read_nz_map_vert(aom_reader *r, tran_low_t *tcoeffs,
int plane, const int16_t *scan,
const int16_t *iscan, TX_SIZE tx_size,
TX_TYPE tx_type, FRAME_CONTEXT *fc,
FRAME_COUNTS *counts) {
static INLINE int read_nz_map_vert(aom_reader *r, uint8_t *levels, int plane,
const int16_t *scan, const int16_t *iscan,
TX_SIZE tx_size, TX_TYPE tx_type,
FRAME_CONTEXT *fc, FRAME_COUNTS *counts) {
const TX_SIZE txs_ctx = get_txsize_context(tx_size);
const PLANE_TYPE plane_type = get_plane_type(plane);
const TX_CLASS tx_class = get_tx_class(tx_type);
......@@ -147,7 +147,7 @@ static INLINE int read_nz_map_vert(aom_reader *r, tran_low_t *tcoeffs,
int coeff_idx = row * width + col;
int scan_idx = iscan[coeff_idx];
int coeff_ctx =
get_nz_map_ctx(tcoeffs, scan_idx, scan, bwl, height, tx_type);
get_nz_map_ctx(levels, scan_idx, scan, bwl, height, tx_type, 1);
#if LV_MAP_PROB
int is_nz = av1_read_record_bin(
counts, r, fc->nz_map_cdf[txs_ctx][plane_type][coeff_ctx], 2,
......@@ -156,7 +156,7 @@ static INLINE int read_nz_map_vert(aom_reader *r, tran_low_t *tcoeffs,
int is_nz = aom_read(r, nz_map[coeff_ctx], ACCT_STR);
#endif
if (counts) ++counts->nz_map[txs_ctx][plane_type][coeff_ctx][is_nz];
tcoeffs[coeff_idx] = is_nz;
levels[coeff_idx] = is_nz;
if (is_nz) {
eob = AOMMAX(eob, iscan[coeff_idx] + 1);
if (row + 1 != height) {
......@@ -179,7 +179,7 @@ static INLINE int read_nz_map_vert(aom_reader *r, tran_low_t *tcoeffs,
}
} else {
int coeff_idx = row * width + col;
tcoeffs[coeff_idx] = 1;
levels[coeff_idx] = 1;
eob = AOMMAX(eob, iscan[coeff_idx] + 1);
}
}
......@@ -191,11 +191,10 @@ static INLINE int read_nz_map_vert(aom_reader *r, tran_low_t *tcoeffs,
return eob;
}
static INLINE int read_nz_map_horiz(aom_reader *r, tran_low_t *tcoeffs,
int plane, const int16_t *scan,
const int16_t *iscan, TX_SIZE tx_size,
TX_TYPE tx_type, FRAME_CONTEXT *fc,
FRAME_COUNTS *counts) {
static INLINE int read_nz_map_horiz(aom_reader *r, uint8_t *levels, int plane,
const int16_t *scan, const int16_t *iscan,
TX_SIZE tx_size, TX_TYPE tx_type,
FRAME_CONTEXT *fc, FRAME_COUNTS *counts) {
const TX_SIZE txs_ctx = get_txsize_context(tx_size);
const PLANE_TYPE plane_type = get_plane_type(plane);
const TX_CLASS tx_class = get_tx_class(tx_type);
......@@ -226,7 +225,7 @@ static INLINE int read_nz_map_horiz(aom_reader *r, tran_low_t *tcoeffs,
int coeff_idx = row * width + col;
int scan_idx = iscan[coeff_idx];
int coeff_ctx =
get_nz_map_ctx(tcoeffs, scan_idx, scan, bwl, height, tx_type);
get_nz_map_ctx(levels, scan_idx, scan, bwl, height, tx_type, 1);
#if LV_MAP_PROB
int is_nz = av1_read_record_bin(
counts, r, fc->nz_map_cdf[txs_ctx][plane_type][coeff_ctx], 2,
......@@ -235,7 +234,7 @@ static INLINE int read_nz_map_horiz(aom_reader *r, tran_low_t *tcoeffs,
int is_nz = aom_read(r, nz_map[coeff_ctx], ACCT_STR);
#endif
if (counts) ++counts->nz_map[txs_ctx][plane_type][coeff_ctx][is_nz];
tcoeffs[coeff_idx] = is_nz;
levels[coeff_idx] = is_nz;
if (is_nz) {
eob = AOMMAX(eob, iscan[coeff_idx] + 1);
int eob_ctx = get_hv_eob_ctx(row, col, eob_ls);
......@@ -255,7 +254,7 @@ static INLINE int read_nz_map_horiz(aom_reader *r, tran_low_t *tcoeffs,
}
} else {
int coeff_idx = row * width + col;
tcoeffs[coeff_idx] = 1;
levels[coeff_idx] = 1;
eob = AOMMAX(eob, iscan[coeff_idx] + 1);
}
}
......@@ -314,6 +313,7 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd,
return 0;
}
memset(levels, 0, sizeof(levels[0]) * seg_eob);
memset(signs, 0, sizeof(signs[0]) * seg_eob);
(void)blk_row;
......@@ -332,7 +332,7 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd,
TX_CLASS tx_class = get_tx_class(tx_type);
if (tx_class == TX_CLASS_2D) {
*eob =
read_nz_map(r, tcoeffs, plane, scan, tx_size, tx_type, ec_ctx, counts);
read_nz_map(r, levels, plane, scan, tx_size, tx_type, ec_ctx, counts);
} else {
#if LV_MAP_PROB
const int eob_mode = av1_read_record_bin(
......@@ -344,28 +344,24 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd,
#endif
if (counts) ++counts->eob_mode[txs_ctx][plane_type][tx_class][eob_mode];
if (eob_mode == 0) {
*eob = read_nz_map(r, tcoeffs, plane, scan, tx_size, tx_type, ec_ctx,
counts);
*eob =
read_nz_map(r, levels, plane, scan, tx_size, tx_type, ec_ctx, counts);
} else {
assert(tx_class == TX_CLASS_VERT || tx_class == TX_CLASS_HORIZ);
if (tx_class == TX_CLASS_VERT)
*eob = read_nz_map_vert(r, tcoeffs, plane, scan, iscan, tx_size,
tx_type, ec_ctx, counts);
*eob = read_nz_map_vert(r, levels, plane, scan, iscan, tx_size, tx_type,
ec_ctx, counts);
else
*eob = read_nz_map_horiz(r, tcoeffs, plane, scan, iscan, tx_size,
*eob = read_nz_map_horiz(r, levels, plane, scan, iscan, tx_size,
tx_type, ec_ctx, counts);
}
}
#else
*eob = read_nz_map(r, tcoeffs, plane, scan, tx_size, tx_type, ec_ctx, counts);
*eob = read_nz_map(r, levels, plane, scan, tx_size, tx_type, ec_ctx, counts);
#endif
*max_scan_line = *eob;
int i;
for (i = 0; i < seg_eob; i++) {
levels[i] = (uint8_t)tcoeffs[i];
}
for (i = 0; i < NUM_BASE_LEVELS; ++i) {
#if !LV_MAP_PROB
aom_prob *coeff_base = ec_ctx->coeff_base[txs_ctx][plane_type][i];
......
......@@ -116,7 +116,7 @@ static INLINE void write_nz_map(aom_writer *w, const tran_low_t *tcoeff,
#endif
for (int c = 0; c < eob; ++c) {
int coeff_ctx = get_nz_map_ctx(tcoeff, c, scan, bwl, height, tx_type);
int coeff_ctx = get_nz_map_ctx(tcoeff, c, scan, bwl, height, tx_type, 0);
int eob_ctx = get_eob_ctx(scan[c], txs_ctx, tx_type);
tran_low_t v = tcoeff[scan[c]];
......@@ -177,7 +177,7 @@ static INLINE void write_nz_map_vert(aom_writer *w, const tran_low_t *tcoeff,
int scan_idx = iscan[coeff_idx];
int is_nz = tcoeff[coeff_idx] != 0;
int coeff_ctx =
get_nz_map_ctx(tcoeff, scan_idx, scan, bwl, height, tx_type);
get_nz_map_ctx(tcoeff, scan_idx, scan, bwl, height, tx_type, 0);
#if LV_MAP_PROB
aom_write_bin(w, is_nz,
fc->nz_map_cdf[txs_ctx][plane_type][coeff_ctx], 2);
......@@ -236,7 +236,7 @@ static INLINE void write_nz_map_horiz(aom_writer *w, const tran_low_t *tcoeff,
int scan_idx = iscan[coeff_idx];
int is_nz = tcoeff[coeff_idx] != 0;
int coeff_ctx =
get_nz_map_ctx(tcoeff, scan_idx, scan, bwl, height, tx_type);
get_nz_map_ctx(tcoeff, scan_idx, scan, bwl, height, tx_type, 0);
#if LV_MAP_PROB
aom_write_bin(w, is_nz,
fc->nz_map_cdf[txs_ctx][plane_type][coeff_ctx], 2);
......@@ -583,7 +583,7 @@ int get_nz_eob_map_cost(const LV_MAP_COEFF_COST *coeff_costs,
tran_low_t v = qcoeff[scan[c]];
int is_nz = (v != 0);
if (c + 1 != seg_eob) {
int coeff_ctx = get_nz_map_ctx(qcoeff, c, scan, bwl, height, tx_type);
int coeff_ctx = get_nz_map_ctx(qcoeff, c, scan, bwl, height, tx_type, 0);
cost += coeff_costs->nz_map_cost[coeff_ctx][is_nz];
if (is_nz) {
int eob_ctx = get_eob_ctx(scan[c], txs_ctx, tx_type);
......@@ -624,7 +624,7 @@ static INLINE int get_nz_eob_map_cost_vert(const LV_MAP_COEFF_COST *coeff_costs,
int scan_idx = iscan[coeff_idx];
int is_nz = qcoeff[coeff_idx] != 0;
int coeff_ctx =
get_nz_map_ctx(qcoeff, scan_idx, scan, bwl, height, tx_type);
get_nz_map_ctx(qcoeff, scan_idx, scan, bwl, height, tx_type, 0);
cost += coeff_costs->nz_map_cost[coeff_ctx][is_nz];
if (is_nz) {
int eob_ctx = get_hv_eob_ctx(c, r, eob_ls);
......@@ -664,7 +664,7 @@ static INLINE int get_nz_eob_map_cost_horiz(
int scan_idx = iscan[coeff_idx];
int is_nz = qcoeff[coeff_idx] != 0;
int coeff_ctx =
get_nz_map_ctx(qcoeff, scan_idx, scan, bwl, height, tx_type);
get_nz_map_ctx(qcoeff, scan_idx, scan, bwl, height, tx_type, 0);
cost += coeff_costs->nz_map_cost[coeff_ctx][is_nz];
if (is_nz) {
int eob_ctx = get_hv_eob_ctx(r, c, eob_ls);
......@@ -858,7 +858,7 @@ void gen_txb_cache(TxbCache *txb_cache, TxbInfo *txb_info) {
const int16_t *scan = txb_info->scan_order->scan;
const int bwl = txb_info->bwl;
const int height = txb_info->height;
tran_low_t *qcoeff = txb_info->qcoeff;
const tran_low_t *const qcoeff = txb_info->qcoeff;
const BASE_CTX_TABLE *base_ctx_table =
txb_info->coeff_ctx_table->base_ctx_table;
for (int c = 0; c < txb_info->eob; ++c) {
......@@ -879,10 +879,10 @@ void gen_txb_cache(TxbCache *txb_cache, TxbInfo *txb_info) {
prev_col = -1;
}
txb_cache->nz_count_arr[coeff_idx] =
get_nz_count(qcoeff, bwl, height, row, col, prev_row, prev_col);
get_nz_count(qcoeff, bwl, height, row, col, 0, prev_row, prev_col);
#else
txb_cache->nz_count_arr[coeff_idx] =
get_nz_count(qcoeff, bwl, height, row, col);
get_nz_count(qcoeff, bwl, height, row, col, 0);
#endif
const int nz_count = txb_cache->nz_count_arr[coeff_idx];
txb_cache->nz_ctx_arr[coeff_idx] =
......@@ -1487,7 +1487,7 @@ void update_level_down(int coeff_idx, TxbCache *txb_cache, TxbInfo *txb_info) {
txb_cache->nz_ctx_arr[nb_coeff_idx] = get_nz_map_ctx_from_count(
count, nb_coeff_idx, txb_info->bwl, txb_info->tx_type);
// int ref_ctx = get_nz_map_ctx(txb_info->qcoeff, nb_coeff_idx,
// txb_info->bwl, tx_type);
// txb_info->bwl, tx_type, 0);
// if (ref_ctx != txb_cache->nz_ctx_arr[nb_coeff_idx])
// printf("nz ctx %d ref_ctx %d\n",
// txb_cache->nz_ctx_arr[nb_coeff_idx], ref_ctx);
......@@ -1579,7 +1579,7 @@ static int get_coeff_cost(tran_low_t qc, int scan_idx, TxbInfo *txb_info,
if (scan_idx < txb_info->seg_eob) {
int coeff_ctx =
get_nz_map_ctx(txb_info->qcoeff, scan_idx, scan, txb_info->bwl,
txb_info->height, txb_info->tx_type);
txb_info->height, txb_info->tx_type, 0);
cost += txb_costs->nz_map_cost[coeff_ctx][is_nz];
}
......@@ -2027,7 +2027,7 @@ static INLINE void av1_update_nz_eob_counts(FRAME_CONTEXT *fc,
for (int c = 0; c < eob; ++c) {
tran_low_t v = tcoeff[scan[c]];
int is_nz = (v != 0);
int coeff_ctx = get_nz_map_ctx(tcoeff, c, scan, bwl, height, tx_type);
int coeff_ctx = get_nz_map_ctx(tcoeff, c, scan, bwl, height, tx_type, 0);
int eob_ctx = get_eob_ctx(scan[c], txsize_ctx, tx_type);
if (c == seg_eob - 1) break;
......@@ -2079,7 +2079,7 @@ static INLINE void av1_update_nz_eob_counts_vert(
int scan_idx = iscan[coeff_idx];
int is_nz = tcoeff[coeff_idx] != 0;
int coeff_ctx =
get_nz_map_ctx(tcoeff, scan_idx, scan, bwl, height, tx_type);
get_nz_map_ctx(tcoeff, scan_idx, scan, bwl, height, tx_type, 0);
++(*nz_map_count)[coeff_ctx][is_nz];
#if LV_MAP_PROB
update_bin(fc->nz_map_cdf[txs_ctx][plane_type][coeff_ctx], is_nz, 2);
......@@ -2130,7 +2130,7 @@ static INLINE void av1_update_nz_eob_counts_horiz(
int scan_idx = iscan[coeff_idx];
int is_nz = tcoeff[coeff_idx] != 0;
int coeff_ctx =
get_nz_map_ctx(tcoeff, scan_idx, scan, bwl, height, tx_type);
get_nz_map_ctx(tcoeff, scan_idx, scan, bwl, height, tx_type, 0);
++(*nz_map_count)[coeff_ctx][is_nz];
#if LV_MAP_PROB
update_bin(fc->nz_map_cdf[txs_ctx][plane_type][coeff_ctx], is_nz, 2);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment