Commit 0474912c authored by Yushin Cho's avatar Yushin Cho

Fix daala-dist for var-tx

The var-tx has its own suite of tx size/type RD search functions,
which recursively split the partition into square tx blocks.

The Daala-dist requires access to 8x8 pixels (both decoded and predicted)
since it measures the distortion for multiple of a 8x8 pixels.
Thus, if tx block is smaller than 8x8, it waits until all of sub8x8 blocks
are RD searched (with MSE) then replaces the MSE of 8x8 pixels with
daala-dist's calculated distortion for 8x8 pixels.

It is also applied to luma pixels only.

Change-Id: Ic4891e89b4ef05cf880aa26781d2d06ccf3142de
parent ad8290b8
......@@ -3660,6 +3660,9 @@ static void rd_pick_partition(const AV1_COMP *const cpi, ThreadData *td,
int use_activity_masking = 0;
int64_t daala_dist;
const int src_stride = x->plane[0].src.stride;
#if CONFIG_PVQ
use_activity_masking = x->daala_enc.use_activity_masking;
#endif
daala_dist = av1_daala_dist(x->plane[0].src.buf - 4 * src_stride - 4,
src_stride, x->decoded_8x8, 8, 8, 8, 8, 8,
1, use_activity_masking, x->qindex)
......@@ -3836,6 +3839,9 @@ static void rd_pick_partition(const AV1_COMP *const cpi, ThreadData *td,
int use_activity_masking = 0;
int64_t daala_dist;
const int src_stride = x->plane[0].src.stride;
#if CONFIG_PVQ
use_activity_masking = x->daala_enc.use_activity_masking;
#endif
daala_dist = av1_daala_dist(x->plane[0].src.buf - 4 * src_stride,
src_stride, x->decoded_8x8, 8, 8, 8, 8, 8,
1, use_activity_masking, x->qindex)
......@@ -4008,6 +4014,9 @@ static void rd_pick_partition(const AV1_COMP *const cpi, ThreadData *td,
int use_activity_masking = 0;
int64_t daala_dist;
const int src_stride = x->plane[0].src.stride;
#if CONFIG_PVQ
use_activity_masking = x->daala_enc.use_activity_masking;
#endif
daala_dist =
av1_daala_dist(x->plane[0].src.buf - 4, src_stride, x->decoded_8x8,
8, 8, 8, 8, 8, 1, use_activity_masking, x->qindex)
......
......@@ -1821,9 +1821,6 @@ static void daala_dist_sub8x8_txfm_rd(MACROBLOCK *x, BLOCK_SIZE bsize,
const struct macroblockd_plane *const pd = &xd->plane[0];
const struct macroblock_plane *const p = &x->plane[0];
MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
#if CONFIG_PVQ
use_activity_masking = x->daala_enc.use_activity_masking;
#endif // CONFIG_PVQ
const int src_stride = p->src.stride;
const int dst_stride = pd->dst.stride;
const uint8_t *src = &p->src.buf[0];
......@@ -1838,6 +1835,9 @@ static void daala_dist_sub8x8_txfm_rd(MACROBLOCK *x, BLOCK_SIZE bsize,
int use_activity_masking = 0;
unsigned int tmp1, tmp2;
int qindex = x->qindex;
#if CONFIG_PVQ
use_activity_masking = x->daala_enc.use_activity_masking;
#endif
assert((bw & 0x07) == 0);
assert((bh & 0x07) == 0);
......@@ -4081,7 +4081,6 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
BLOCK_SIZE txm_bsize = txsize_to_bsize[tx_size];
int bh = block_size_high[txm_bsize];
int bw = block_size_wide[txm_bsize];
int src_stride = p->src.stride;
uint8_t *src =
&p->src.buf[(blk_row * src_stride + blk_col) << tx_size_wide_log2[0]];
......@@ -4137,6 +4136,23 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
av1_inverse_transform_block(xd, dqcoeff, tx_type, tx_size, rec_buffer,
MAX_TX_SIZE, eob);
if (eob > 0) {
#if CONFIG_DAALA_DIST
if (plane == 0 && (bw < 8 && bh < 8)) {
// Save sub8x8 luma decoded pixels
// since 8x8 luma decoded pixels are not available for daala-dist
// after recursive split of BLOCK_8x8 is done.
const int pred_stride = block_size_wide[plane_bsize];
const int pred_idx = (blk_row * pred_stride + blk_col)
<< tx_size_wide_log2[0];
int16_t *decoded = &pd->pred[pred_idx];
int i, j;
// TODO(yushin): HBD support
for (j = 0; j < bh; j++)
for (i = 0; i < bw; i++)
decoded[j * pred_stride + i] = rec_buffer[j * MAX_TX_SIZE + i];
}
#endif // CONFIG_DAALA_DIST
tmp = pixel_dist(cpi, x, plane, src, src_stride, rec_buffer, MAX_TX_SIZE,
blk_row, blk_col, plane_bsize, txm_bsize);
}
......@@ -4249,7 +4265,9 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
RD_STATS this_rd_stats;
int this_cost_valid = 1;
int64_t tmp_rd = 0;
#if CONFIG_DAALA_DIST
int sub8x8_eob[4];
#endif
sum_rd_stats.rate =
av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 1);
......@@ -4265,13 +4283,76 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
depth + 1, plane_bsize, ta, tl, tx_above, tx_left,
&this_rd_stats, ref_best_rd - tmp_rd, &this_cost_valid,
rd_stats_stack);
#if CONFIG_DAALA_DIST
if (plane == 0 && tx_size == TX_8X8) {
sub8x8_eob[i] = p->eobs[block];
}
#endif // CONFIG_DAALA_DIST
av1_merge_rd_stats(&sum_rd_stats, &this_rd_stats);
tmp_rd = RDCOST(x->rdmult, sum_rd_stats.rate, sum_rd_stats.dist);
#if !CONFIG_DAALA_DIST
if (this_rd < tmp_rd) break;
#endif
block += sub_step;
}
#if CONFIG_DAALA_DIST
if (this_cost_valid && plane == 0 && tx_size == TX_8X8) {
const int src_stride = p->src.stride;
const int dst_stride = pd->dst.stride;
const uint8_t *src =
&p->src.buf[(blk_row * src_stride + blk_col) << tx_size_wide_log2[0]];
const uint8_t *dst =
&pd->dst
.buf[(blk_row * dst_stride + blk_col) << tx_size_wide_log2[0]];
int64_t daala_dist;
int qindex = x->qindex;
const int pred_stride = block_size_wide[plane_bsize];
const int pred_idx = (blk_row * pred_stride + blk_col)
<< tx_size_wide_log2[0];
int16_t *pred = &pd->pred[pred_idx];
int j;
int qm = OD_HVS_QM;
int use_activity_masking = 0;
int row, col;
DECLARE_ALIGNED(16, uint8_t, pred8[8 * 8]);
#if CONFIG_PVQ
use_activity_masking = x->daala_enc.use_activity_masking;
#endif
daala_dist = av1_daala_dist(src, src_stride, dst, dst_stride, 8, 8, 8, 8,
qm, use_activity_masking, qindex) *
16;
sum_rd_stats.sse = daala_dist;
for (row = 0; row < 2; ++row) {
for (col = 0; col < 2; ++col) {
int idx = row * 2 + col;
int eob = sub8x8_eob[idx];
if (eob > 0) {
for (j = 0; j < 4; j++)
for (i = 0; i < 4; i++)
pred8[(row * 4 + j) * 8 + 4 * col + i] =
pred[(row * 4 + j) * pred_stride + 4 * col + i];
} else {
for (j = 0; j < 4; j++)
for (i = 0; i < 4; i++)
pred8[(row * 4 + j) * 8 + 4 * col + i] =
dst[(row * 4 + j) * dst_stride + 4 * col + i];
}
}
}
daala_dist = av1_daala_dist(src, src_stride, pred8, 8, 8, 8, 8, 8, qm,
use_activity_masking, qindex) *
16;
sum_rd_stats.dist = daala_dist;
tmp_rd = RDCOST(x->rdmult, sum_rd_stats.rate, sum_rd_stats.dist);
}
#endif // CONFIG_DAALA_DIST
if (this_cost_valid) sum_rd = tmp_rd;
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment