Commit f986af1c authored by Yushin Cho's avatar Yushin Cho

[dist-8x8] Enable assert for 8 bit input only

Larger than 8 bit input seems to introduce different rounding methods in
different distortion calculation functions, which causes assertion failed
for dist-8x8's sanity check.

BUG=aomedia:1027

Change-Id: I1f607f32d4017d8f9abbbe0b09d9dbb01a63fe9c
parent 19619886
......@@ -2819,7 +2819,8 @@ static void rd_pick_partition(const AV1_COMP *const cpi, ThreadData *td,
const int src_stride = x->plane[0].src.stride;
int64_t dist_8x8;
dist_8x8 = dist_8x8_yuv(cpi, x, x->plane[0].src.buf - 4 * src_stride - 4);
if (x->tune_metric == AOM_TUNE_PSNR) assert(sum_rdc.dist == dist_8x8);
if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8)
assert(sum_rdc.dist == dist_8x8);
sum_rdc.dist = dist_8x8;
sum_rdc.rdcost = RDCOST(x->rdmult, sum_rdc.rate, sum_rdc.dist);
}
......@@ -2907,7 +2908,8 @@ static void rd_pick_partition(const AV1_COMP *const cpi, ThreadData *td,
const int src_stride = x->plane[0].src.stride;
int64_t dist_8x8;
dist_8x8 = dist_8x8_yuv(cpi, x, x->plane[0].src.buf - 4 * src_stride);
if (x->tune_metric == AOM_TUNE_PSNR) assert(sum_rdc.dist == dist_8x8);
if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8)
assert(sum_rdc.dist == dist_8x8);
sum_rdc.dist = dist_8x8;
sum_rdc.rdcost = RDCOST(x->rdmult, sum_rdc.rate, sum_rdc.dist);
}
......@@ -2991,7 +2993,8 @@ static void rd_pick_partition(const AV1_COMP *const cpi, ThreadData *td,
bsize == BLOCK_8X8) {
int64_t dist_8x8;
dist_8x8 = dist_8x8_yuv(cpi, x, x->plane[0].src.buf - 4);
if (x->tune_metric == AOM_TUNE_PSNR) assert(sum_rdc.dist == dist_8x8);
if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8)
assert(sum_rdc.dist == dist_8x8);
sum_rdc.dist = dist_8x8;
sum_rdc.rdcost = RDCOST(x->rdmult, sum_rdc.rate, sum_rdc.dist);
}
......
......@@ -2288,7 +2288,7 @@ static void dist_8x8_sub8x8_txfm_rd(const AV1_COMP *const cpi, MACROBLOCK *x,
bw, bh, visible_w, visible_h, qindex);
if (!is_inter_block(mbmi)) {
if (x->tune_metric == AOM_TUNE_PSNR) {
if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8) {
assert(args->rd_stats.sse == tmp1 * 16);
assert(args->rd_stats.dist == tmp2 * 16);
}
......@@ -2297,7 +2297,7 @@ static void dist_8x8_sub8x8_txfm_rd(const AV1_COMP *const cpi, MACROBLOCK *x,
} else {
// For inter mode, the decoded pixels are provided in pd->pred,
// while the predicted pixels are in dst.
if (x->tune_metric == AOM_TUNE_PSNR) {
if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8) {
assert(args->rd_stats.sse == tmp2 * 16);
assert(args->rd_stats.dist == tmp1 * 16);
}
......@@ -4052,7 +4052,8 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
dist_8x8 = av1_dist_8x8(cpi, x, src, src_stride, dst, dst_stride,
BLOCK_8X8, 8, 8, 8, 8, qindex) *
16;
if (x->tune_metric == AOM_TUNE_PSNR) assert(sum_rd_stats.sse == dist_8x8);
if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8)
assert(sum_rd_stats.sse == dist_8x8);
sum_rd_stats.sse = dist_8x8;
#if CONFIG_HIGHBITDEPTH
......@@ -4110,7 +4111,7 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
dist_8x8 = av1_dist_8x8(cpi, x, src, src_stride, pred8, 8, BLOCK_8X8, 8,
8, 8, 8, qindex) *
16;
if (x->tune_metric == AOM_TUNE_PSNR)
if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8)
assert(sum_rd_stats.dist == dist_8x8);
sum_rd_stats.dist = dist_8x8;
tmp_rd = RDCOST(x->rdmult, sum_rd_stats.rate, sum_rd_stats.dist);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment