Commit 7c71cc03 authored by Sarah Parker's avatar Sarah Parker

Use reduced_tx_set_used for all av1_get_tx_type calls

BUG=aomedia:1232

Change-Id: Ib2578cdb41128997f7513bc11f86edf43a799959
parent bc50abba
......@@ -916,13 +916,14 @@ get_plane_block_size(BLOCK_SIZE bsize, const struct macroblockd_plane *pd) {
static INLINE TX_TYPE av1_get_tx_type(PLANE_TYPE plane_type,
const MACROBLOCKD *xd, int blk_row,
int blk_col, TX_SIZE tx_size) {
int blk_col, TX_SIZE tx_size,
int reduced_tx_set) {
const MODE_INFO *const mi = xd->mi[0];
const MB_MODE_INFO *const mbmi = &mi->mbmi;
const struct macroblockd_plane *const pd = &xd->plane[plane_type];
const BLOCK_SIZE plane_bsize = get_plane_block_size(mbmi->sb_type, pd);
const TxSetType tx_set_type =
get_ext_tx_set_type(tx_size, plane_bsize, is_inter_block(mbmi), 0);
const TxSetType tx_set_type = get_ext_tx_set_type(
tx_size, plane_bsize, is_inter_block(mbmi), reduced_tx_set);
#if CONFIG_TXK_SEL
TX_TYPE tx_type;
......
......@@ -156,9 +156,11 @@ static void predict_and_reconstruct_intra_block(
av1_read_coeffs_txb_facade(cm, xd, r, row, col, plane, tx_size,
&max_scan_line, &eob);
// tx_type will be read out in av1_read_coeffs_txb_facade
const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, row, col, tx_size);
const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, row, col, tx_size,
cm->reduced_tx_set_used);
#else // CONFIG_LV_MAP
const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, row, col, tx_size);
const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, row, col, tx_size,
cm->reduced_tx_set_used);
const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, mbmi);
int16_t max_scan_line = 0;
const int eob =
......@@ -218,11 +220,11 @@ static void decode_reconstruct_tx(AV1_COMMON *cm, MACROBLOCKD *const xd,
av1_read_coeffs_txb_facade(cm, xd, r, blk_row, blk_col, plane, tx_size,
&max_scan_line, &eob);
// tx_type will be read out in av1_read_coeffs_txb_facade
const TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
tx_size, cm->reduced_tx_set_used);
#else // CONFIG_LV_MAP
const TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
tx_size, cm->reduced_tx_set_used);
const SCAN_ORDER *sc = get_scan(cm, tx_size, tx_type, mbmi);
int16_t max_scan_line = 0;
const int eob =
......
......@@ -122,8 +122,8 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
#if CONFIG_TXK_SEL
av1_read_tx_type(cm, xd, blk_row, blk_col, plane, tx_size, r);
#endif
const TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
tx_size, cm->reduced_tx_set_used);
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
const int16_t *const scan = scan_order->scan;
int dummy;
......
......@@ -1031,7 +1031,8 @@ void av1_write_tx_type(const AV1_COMMON *const cm, const MACROBLOCKD *xd,
// Only y plane's tx_type is transmitted
if (plane > 0) return;
PLANE_TYPE plane_type = get_plane_type(plane);
TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size,
cm->reduced_tx_set_used);
#endif
const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
......
......@@ -4528,8 +4528,8 @@ void av1_update_tx_type_count(const AV1_COMMON *cm, MACROBLOCKD *xd,
#else
// Only y plane's tx_type is updated
if (plane > 0) return;
TX_TYPE tx_type =
av1_get_tx_type(PLANE_TYPE_Y, xd, blk_row, blk_col, tx_size);
TX_TYPE tx_type = av1_get_tx_type(PLANE_TYPE_Y, xd, blk_row, blk_col, tx_size,
cm->reduced_tx_set_used);
#endif
if (get_ext_tx_types(tx_size, bsize, is_inter, cm->reduced_tx_set_used) > 1 &&
cm->base_qindex > 0 && !mbmi->skip &&
......
......@@ -134,8 +134,8 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane,
tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
const int16_t *const dequant_ptr = p->dequant_QTX;
const uint8_t *const band_translate = get_band_translate(tx_size);
const TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
tx_size, cm->reduced_tx_set_used);
const SCAN_ORDER *const scan_order =
get_scan(cm, tx_size, tx_type, &xd->mi[0]->mbmi);
const int16_t *const scan = scan_order->scan;
......@@ -446,7 +446,8 @@ void av1_xform_quant(const AV1_COMMON *cm, MACROBLOCK *x, int plane, int block,
const struct macroblock_plane *const p = &x->plane[plane];
const struct macroblockd_plane *const pd = &xd->plane[plane];
PLANE_TYPE plane_type = get_plane_type(plane);
TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size,
cm->reduced_tx_set_used);
#if CONFIG_NEW_QUANT
const int is_inter = is_inter_block(mbmi);
......@@ -565,8 +566,8 @@ static void encode_block(int plane, int block, int blk_row, int blk_col,
if (p->eobs[block]) {
*(args->skip) = 0;
TX_TYPE tx_type =
av1_get_tx_type(pd->plane_type, xd, blk_row, blk_col, tx_size);
TX_TYPE tx_type = av1_get_tx_type(pd->plane_type, xd, blk_row, blk_col,
tx_size, cm->reduced_tx_set_used);
av1_inverse_transform_block(xd, dqcoeff, plane, tx_type, tx_size, dst,
pd->dst.stride, p->eobs[block],
cm->reduced_tx_set_used);
......@@ -818,8 +819,8 @@ void av1_encode_block_intra(int plane, int block, int blk_row, int blk_col,
struct macroblockd_plane *const pd = &xd->plane[plane];
tran_low_t *dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
PLANE_TYPE plane_type = get_plane_type(plane);
const TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
tx_size, cm->reduced_tx_set_used);
uint16_t *eob = &p->eobs[block];
const int dst_stride = pd->dst.stride;
uint8_t *dst =
......
......@@ -444,8 +444,8 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd,
MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
const PLANE_TYPE plane_type = get_plane_type(plane);
const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
const TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
tx_size, cm->reduced_tx_set_used);
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
const int16_t *const scan = scan_order->scan;
const int seg_eob = av1_get_max_eob(tx_size);
......@@ -684,8 +684,8 @@ int av1_cost_coeffs_txb(const AV1_COMMON *const cm, const MACROBLOCK *x,
const MACROBLOCKD *const xd = &x->e_mbd;
const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
const PLANE_TYPE plane_type = get_plane_type(plane);
const TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
tx_size, cm->reduced_tx_set_used);
const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
const struct macroblock_plane *p = &x->plane[plane];
const int eob = p->eobs[block];
......@@ -2031,8 +2031,8 @@ int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
MACROBLOCKD *const xd = &x->e_mbd;
const PLANE_TYPE plane_type = get_plane_type(plane);
const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
const TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
tx_size, cm->reduced_tx_set_used);
const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
const struct macroblock_plane *p = &x->plane[plane];
struct macroblockd_plane *pd = &xd->plane[plane];
......@@ -2148,8 +2148,8 @@ void av1_update_txb_context_b(int plane, int block, int blk_row, int blk_col,
const uint16_t eob = p->eobs[block];
const tran_low_t *qcoeff = BLOCK_OFFSET(p->qcoeff, block);
const PLANE_TYPE plane_type = pd->plane_type;
const TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
tx_size, cm->reduced_tx_set_used);
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
(void)plane_bsize;
......@@ -2174,8 +2174,8 @@ void av1_update_and_record_txb_context(int plane, int block, int blk_row,
const tran_low_t *qcoeff = BLOCK_OFFSET(p->qcoeff, block);
tran_low_t *tcoeff = BLOCK_OFFSET(x->mbmi_ext->tcoeff[plane], block);
const int segment_id = mbmi->segment_id;
const TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
tx_size, cm->reduced_tx_set_used);
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
const int16_t *const scan = scan_order->scan;
const int seg_eob = av1_get_tx_eob(&cpi->common.seg, segment_id, tx_size);
......
......@@ -1660,8 +1660,8 @@ void av1_inverse_transform_block_facade(MACROBLOCKD *xd, int plane, int block,
tran_low_t *dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
const PLANE_TYPE plane_type = get_plane_type(plane);
const TX_SIZE tx_size = av1_get_tx_size(plane, xd);
const TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
tx_size, reduced_tx_set);
const int dst_stride = pd->dst.stride;
uint8_t *dst =
&pd->dst.buf[(blk_row * dst_stride + blk_col) << tx_size_wide_log2[0]];
......@@ -1764,7 +1764,8 @@ void av1_dist_block(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
const PLANE_TYPE plane_type = get_plane_type(plane);
TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size,
cpi->common.reduced_tx_set_used);
av1_inverse_transform_block(xd, dqcoeff, plane, tx_type, tx_size, recon,
MAX_TX_SIZE, eob,
cpi->common.reduced_tx_set_used);
......@@ -1827,7 +1828,8 @@ static int64_t search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
if (plane == 0)
mbmi->txk_type[(blk_row << MAX_MIB_SIZE_LOG2) + blk_col] = tx_type;
TX_TYPE ref_tx_type =
av1_get_tx_type(get_plane_type(plane), xd, blk_row, blk_col, tx_size);
av1_get_tx_type(get_plane_type(plane), xd, blk_row, blk_col, tx_size,
cm->reduced_tx_set_used);
if (tx_type != ref_tx_type) {
// use av1_get_tx_type() to check if the tx_type is valid for the current
// mode if it's not, we skip it here.
......@@ -1946,8 +1948,8 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
#if !CONFIG_TXK_SEL
const PLANE_TYPE plane_type = get_plane_type(plane);
const TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
tx_size, cm->reduced_tx_set_used);
const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, mbmi);
int rate_cost = 0;
......@@ -3612,7 +3614,8 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
int64_t tmp;
tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
PLANE_TYPE plane_type = get_plane_type(plane);
TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size,
cm->reduced_tx_set_used);
const SCAN_ORDER *const scan_order =
get_scan(cm, tx_size, tx_type, &xd->mi[0]->mbmi);
BLOCK_SIZE txm_bsize = txsize_to_bsize[tx_size];
......@@ -4653,7 +4656,8 @@ static const uint32_t skip_pred_threshold[3][BLOCK_SIZES_ALL] = {
// Uses simple features on top of DCT coefficients to quickly predict
// whether optimal RD decision is to skip encoding the residual.
// The sse value is stored in dist.
static int predict_skip_flag(MACROBLOCK *x, BLOCK_SIZE bsize, int64_t *dist) {
static int predict_skip_flag(MACROBLOCK *x, BLOCK_SIZE bsize, int64_t *dist,
int reduced_tx_set) {
int max_tx_size =
get_max_rect_tx_size(bsize, is_inter_block(&x->e_mbd.mi[0]->mbmi));
if (tx_size_high[max_tx_size] > 16 || tx_size_wide[max_tx_size] > 16)
......@@ -4684,11 +4688,9 @@ static int predict_skip_flag(MACROBLOCK *x, BLOCK_SIZE bsize, int64_t *dist) {
const struct macroblockd_plane *const pd = &xd->plane[0];
const BLOCK_SIZE plane_bsize =
get_plane_block_size(xd->mi[0]->mbmi.sb_type, pd);
// TODO(sarahparker) This assumes reduced_tx_set_used == 0. I will do a
// follow up refactor to make the actual value of reduced_tx_set_used
// within this function.
param.tx_set_type = get_ext_tx_set_type(param.tx_size, plane_bsize,
is_inter_block(&xd->mi[0]->mbmi), 0);
param.tx_set_type =
get_ext_tx_set_type(param.tx_size, plane_bsize,
is_inter_block(&xd->mi[0]->mbmi), reduced_tx_set);
const uint32_t ac_q = (uint32_t)av1_ac_quant_QTX(x->qindex, 0, xd->bd);
uint32_t max_quantized_coef = 0;
const int bd_idx = (xd->bd == 8) ? 0 : ((xd->bd == 10) ? 1 : 2);
......@@ -4831,7 +4833,7 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
// context and terminate early.
int64_t dist;
if (is_inter && cpi->sf.tx_type_search.use_skip_flag_prediction &&
predict_skip_flag(x, bsize, &dist)) {
predict_skip_flag(x, bsize, &dist, cm->reduced_tx_set_used)) {
set_skip_flag(cpi, x, rd_stats, bsize, dist);
// Save the RD search results into tx_rd_record.
if (within_border) save_tx_rd_info(n4, hash, x, rd_stats, tx_rd_record);
......
......@@ -240,7 +240,8 @@ static void cost_coeffs_b(int plane, int block, int blk_row, int blk_col,
struct macroblock_plane *p = &x->plane[plane];
struct macroblockd_plane *pd = &xd->plane[plane];
const PLANE_TYPE type = pd->plane_type;
const TX_TYPE tx_type = av1_get_tx_type(type, xd, blk_row, blk_col, tx_size);
const TX_TYPE tx_type = av1_get_tx_type(type, xd, blk_row, blk_col, tx_size,
cm->reduced_tx_set_used);
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
const int rate = av1_cost_coeffs(
cpi, x, plane, blk_row, blk_col, block, tx_size, scan_order,
......@@ -400,7 +401,8 @@ static void tokenize_b(int plane, int block, int blk_row, int blk_col,
const tran_low_t *qcoeff = BLOCK_OFFSET(p->qcoeff, block);
const int segment_id = mbmi->segment_id;
const int16_t *scan, *nb;
const TX_TYPE tx_type = av1_get_tx_type(type, xd, blk_row, blk_col, tx_size);
const TX_TYPE tx_type = av1_get_tx_type(type, xd, blk_row, blk_col, tx_size,
cm->reduced_tx_set_used);
const SCAN_ORDER *const scan_order = get_scan(cm, tx_size, tx_type, mbmi);
const int ref = is_inter_block(mbmi);
FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment