Commit c7ea761f authored by Jingning Han's avatar Jingning Han

Make rd_debug aligned to var-tx

Fix the corner case and use the right rate cost udpate for rd_debug.
This would make the var-tx pass rd_debug test.

Change-Id: Ib0fbd2d73030c0d150222c6b7c2dfffc0c6af085
parent 0c70a80f
......@@ -2946,7 +2946,7 @@ static void update_coef_probs(AV1_COMP *cpi, aom_writer *w) {
for (tx_size = 0; tx_size <= max_tx_size; ++tx_size) {
av1_coeff_stats frame_branch_ct[PLANE_TYPES];
av1_coeff_probs_model frame_coef_probs[PLANE_TYPES];
if (cpi->td.counts->tx_size_totals[tx_size] <= 20 ||
if (cpi->td.counts->tx_size_totals[tx_size] <= 20 || CONFIG_RD_DEBUG ||
(tx_size >= TX_16X16 && cpi->sf.tx_size_search_method == USE_TX_8X8)) {
aom_write_bit(w, 0);
} else {
......
......@@ -3344,6 +3344,10 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
RDCOST(x->rdmult, x->rddiv, zero_blk_rate, rd_stats->sse) ||
rd_stats->skip == 1) &&
!xd->lossless[mbmi->segment_id]) {
#if CONFIG_RD_DEBUG
av1_update_txb_coeff_cost(rd_stats, plane, tx_size, blk_row, blk_col,
zero_blk_rate - rd_stats->rate);
#endif
rd_stats->rate = zero_blk_rate;
rd_stats->dist = rd_stats->sse;
rd_stats->skip = 1;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment