Commit a4817a6b authored by Yushin Cho's avatar Yushin Cho

Fix dist_8x8 broken with 3bce7547

The commit 3bce7547 has introduced an another early-exit based on MSE distortion
in transform domain, which enables skipping trellis coding and
calling av1_dist_block() in block_rd_txfm() and skipping trellis coding in av1_tx_block_rd_b().

However, with dist-8x8, the early-exit for sub8x8 tx block in a partition >= 8x8 in plane 0
is disabled because that the reference distortion metric
(which would be non-MSE and applied to 8x8 or larger) can not be compared to
MSE distortions of sub8x8 tx blocks.

Change-Id: I46ada7c90a869d23fc0f0166a01dfdc5392af311
parent 6e1cd787
...@@ -1869,6 +1869,11 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col, ...@@ -1869,6 +1869,11 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
#endif #endif
int64_t rd1, rd2, rd; int64_t rd1, rd2, rd;
RD_STATS this_rd_stats; RD_STATS this_rd_stats;
#if CONFIG_DIST_8X8
int sub8x8tx_in_gte8x8blk_in_plane0 =
plane == 0 && plane_bsize >= BLOCK_8X8 &&
(tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4);
#endif // CONFIG_DIST_8X8
#if !CONFIG_SUPERTX && !CONFIG_VAR_TX #if !CONFIG_SUPERTX && !CONFIG_VAR_TX
assert(tx_size == av1_get_tx_size(plane, xd)); assert(tx_size == av1_get_tx_size(plane, xd));
...@@ -1927,7 +1932,11 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col, ...@@ -1927,7 +1932,11 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
#endif #endif
tmp_dist = av1_block_error(coeff, dqcoeff, buffer_length, &tmp) >> shift; tmp_dist = av1_block_error(coeff, dqcoeff, buffer_length, &tmp) >> shift;
if (RDCOST(x->rdmult, 0, tmp_dist) + args->this_rd < args->best_rd) { if (
#if CONFIG_DIST_8X8
sub8x8tx_in_gte8x8blk_in_plane0 ||
#endif
RDCOST(x->rdmult, 0, tmp_dist) + args->this_rd < args->best_rd) {
av1_optimize_b(cm, x, plane, blk_row, blk_col, block, plane_bsize, tx_size, av1_optimize_b(cm, x, plane, blk_row, blk_col, block, plane_bsize, tx_size,
a, l); a, l);
} else { } else {
...@@ -2008,8 +2017,7 @@ CALCULATE_RD : {} ...@@ -2008,8 +2017,7 @@ CALCULATE_RD : {}
args->this_rd += rd; args->this_rd += rd;
#if CONFIG_DIST_8X8 #if CONFIG_DIST_8X8
if (!(plane == 0 && plane_bsize >= BLOCK_8X8 && if (!sub8x8tx_in_gte8x8blk_in_plane0) {
(tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4))) {
#endif #endif
if (args->this_rd > args->best_rd) { if (args->this_rd > args->best_rd) {
args->exit_early = 1; args->exit_early = 1;
...@@ -4341,6 +4349,11 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, ...@@ -4341,6 +4349,11 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
const int16_t *diff = const int16_t *diff =
&p->src_diff[(blk_row * diff_stride + blk_col) << tx_size_wide_log2[0]]; &p->src_diff[(blk_row * diff_stride + blk_col) << tx_size_wide_log2[0]];
int txb_coeff_cost; int txb_coeff_cost;
#if CONFIG_DIST_8X8
int sub8x8tx_in_gte8x8blk_in_plane0 =
plane == 0 && plane_bsize >= BLOCK_8X8 &&
(tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4);
#endif // CONFIG_DIST_8X8
assert(tx_size < TX_SIZES_ALL); assert(tx_size < TX_SIZES_ALL);
...@@ -4400,7 +4413,11 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, ...@@ -4400,7 +4413,11 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
tmp_dist = tmp_dist =
av1_block_error(coeff, dqcoeff, buffer_length, &tmp_sse) >> shift; av1_block_error(coeff, dqcoeff, buffer_length, &tmp_sse) >> shift;
if (RDCOST(x->rdmult, 0, tmp_dist) < rd_stats->ref_rdcost) { if (
#if CONFIG_DIST_8X8
sub8x8tx_in_gte8x8blk_in_plane0 ||
#endif
RDCOST(x->rdmult, 0, tmp_dist) < rd_stats->ref_rdcost) {
av1_optimize_b(cm, x, plane, blk_row, blk_col, block, plane_bsize, tx_size, av1_optimize_b(cm, x, plane, blk_row, blk_col, block, plane_bsize, tx_size,
a, l); a, l);
} else { } else {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment