Commit 439ba638 authored by Luc Trudeau's avatar Luc Trudeau
Browse files

[CFL] Code cleaning and Fix for TXK_SEL

Small code cleaning and partially resolves issue 1171. I can't confirm
that it's a complete fix because the code asserts further a long

aomenc: ../av1/common/blockd.h:1064: av1_get_tx_type:
  Assertion `tx_type < TX_TYPES' failed.

BUG=aomedia:1171

Change-Id: I3f1c91d751e937dd8e07f7e2f786a75eba4b6891
parent 3d75209a
......@@ -4366,8 +4366,6 @@ void av1_update_tx_type_count(const AV1_COMMON *cm, MACROBLOCKD *xd,
#if !CONFIG_TXK_SEL
TX_TYPE tx_type = mbmi->tx_type;
#else
(void)blk_row;
(void)blk_col;
// Only y plane's tx_type is updated
if (plane > 0) return;
TX_TYPE tx_type =
......
......@@ -386,8 +386,6 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *xd,
DECLARE_ALIGNED(16, uint8_t, level_counts[MAX_TX_SQUARE]);
DECLARE_ALIGNED(16, int8_t, coeff_contexts[MAX_TX_SQUARE]);
(void)blk_row;
(void)blk_col;
aom_write_bin(w, eob == 0,
ec_ctx->txb_skip_cdf[txs_ctx][txb_ctx->txb_skip_ctx], 2);
#if CONFIG_TXK_SEL
......
......@@ -2087,7 +2087,7 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
int bw = block_size_wide[plane_bsize];
int bh = block_size_high[plane_bsize];
int disable_early_skip =
x->using_dist_8x8 && plane == 0 && bw >= 8 && bh >= 8 &&
x->using_dist_8x8 && plane == AOM_PLANE_Y && bw >= 8 && bh >= 8 &&
(tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4) &&
x->tune_metric != AOM_TUNE_PSNR;
#endif // CONFIG_DIST_8X8
......@@ -2166,12 +2166,6 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
args->exit_early = 1;
return;
}
#if CONFIG_CFL
if (plane == AOM_PLANE_Y && xd->cfl.store_y && is_cfl_allowed(mbmi)) {
assert(!is_inter_block(mbmi) || plane_bsize < BLOCK_8X8);
cfl_store_tx(xd, blk_row, blk_col, tx_size, plane_bsize);
}
#endif // CONFIG_CFL
const PLANE_TYPE plane_type = get_plane_type(plane);
const TX_TYPE tx_type =
av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
......@@ -2186,6 +2180,13 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
&this_rd_stats);
#endif // !CONFIG_TXK_SEL
#if CONFIG_CFL
if (plane == AOM_PLANE_Y && xd->cfl.store_y && is_cfl_allowed(mbmi)) {
assert(!is_inter_block(mbmi) || plane_bsize < BLOCK_8X8);
cfl_store_tx(xd, blk_row, blk_col, tx_size, plane_bsize);
}
#endif // CONFIG_CFL
#if CONFIG_RD_DEBUG
av1_update_txb_coeff_cost(&this_rd_stats, plane, tx_size, blk_row, blk_col,
this_rd_stats.rate);
......@@ -2446,12 +2447,12 @@ static int64_t txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
return INT64_MAX;
}
#endif
txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, 0, bs, tx_size,
txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, AOM_PLANE_Y, bs, tx_size,
cpi->sf.use_fast_coef_costing);
if (rd_stats->rate == INT_MAX) return INT64_MAX;
#if !CONFIG_TXK_SEL
int plane = 0;
rd_stats->rate += av1_tx_type_cost(cm, x, xd, bs, plane, tx_size, tx_type);
rd_stats->rate +=
av1_tx_type_cost(cm, x, xd, bs, AOM_PLANE_Y, tx_size, tx_type);
#endif
if (rd_stats->skip) {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment