[dist-8x8] Enable assert for 8 bit input only

Larger than 8 bit input seems to introduce different rounding methods in
different distortion calculation functions, which causes assertion failed
for dist-8x8's sanity check.

BUG=aomedia:1027

Change-Id: I1f607f32d4017d8f9abbbe0b09d9dbb01a63fe9c

av1/encoder/encodeframe.c

@@ -2819,7 +2819,8 @@ static void rd_pick_partition(const AV1_COMP *const cpi, ThreadData *td,
       const int src_stride = x->plane[0].src.stride;
       int64_t dist_8x8;
       dist_8x8 = dist_8x8_yuv(cpi, x, x->plane[0].src.buf - 4 * src_stride - 4);
-      if (x->tune_metric == AOM_TUNE_PSNR) assert(sum_rdc.dist == dist_8x8);
+      if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8)
+        assert(sum_rdc.dist == dist_8x8);
       sum_rdc.dist = dist_8x8;
       sum_rdc.rdcost = RDCOST(x->rdmult, sum_rdc.rate, sum_rdc.dist);
     }
@@ -2907,7 +2908,8 @@ static void rd_pick_partition(const AV1_COMP *const cpi, ThreadData *td,
         const int src_stride = x->plane[0].src.stride;
         int64_t dist_8x8;
         dist_8x8 = dist_8x8_yuv(cpi, x, x->plane[0].src.buf - 4 * src_stride);
-        if (x->tune_metric == AOM_TUNE_PSNR) assert(sum_rdc.dist == dist_8x8);
+        if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8)
+          assert(sum_rdc.dist == dist_8x8);
         sum_rdc.dist = dist_8x8;
         sum_rdc.rdcost = RDCOST(x->rdmult, sum_rdc.rate, sum_rdc.dist);
       }
@@ -2991,7 +2993,8 @@ static void rd_pick_partition(const AV1_COMP *const cpi, ThreadData *td,
           bsize == BLOCK_8X8) {
         int64_t dist_8x8;
         dist_8x8 = dist_8x8_yuv(cpi, x, x->plane[0].src.buf - 4);
-        if (x->tune_metric == AOM_TUNE_PSNR) assert(sum_rdc.dist == dist_8x8);
+        if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8)
+          assert(sum_rdc.dist == dist_8x8);
         sum_rdc.dist = dist_8x8;
         sum_rdc.rdcost = RDCOST(x->rdmult, sum_rdc.rate, sum_rdc.dist);
       }

av1/encoder/rdopt.c

@@ -2288,7 +2288,7 @@ static void dist_8x8_sub8x8_txfm_rd(const AV1_COMP *const cpi, MACROBLOCK *x,
                                 bw, bh, visible_w, visible_h, qindex);
 
   if (!is_inter_block(mbmi)) {
-    if (x->tune_metric == AOM_TUNE_PSNR) {
+    if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8) {
       assert(args->rd_stats.sse == tmp1 * 16);
       assert(args->rd_stats.dist == tmp2 * 16);
     }
@@ -2297,7 +2297,7 @@ static void dist_8x8_sub8x8_txfm_rd(const AV1_COMP *const cpi, MACROBLOCK *x,
   } else {
     // For inter mode, the decoded pixels are provided in pd->pred,
     // while the predicted pixels are in dst.
-    if (x->tune_metric == AOM_TUNE_PSNR) {
+    if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8) {
       assert(args->rd_stats.sse == tmp2 * 16);
       assert(args->rd_stats.dist == tmp1 * 16);
     }
@@ -4052,7 +4052,8 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
       dist_8x8 = av1_dist_8x8(cpi, x, src, src_stride, dst, dst_stride,
                               BLOCK_8X8, 8, 8, 8, 8, qindex) *
                  16;
-      if (x->tune_metric == AOM_TUNE_PSNR) assert(sum_rd_stats.sse == dist_8x8);
+      if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8)
+        assert(sum_rd_stats.sse == dist_8x8);
       sum_rd_stats.sse = dist_8x8;
 
 #if CONFIG_HIGHBITDEPTH
@@ -4110,7 +4111,7 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
       dist_8x8 = av1_dist_8x8(cpi, x, src, src_stride, pred8, 8, BLOCK_8X8, 8,
                               8, 8, 8, qindex) *
                  16;
-      if (x->tune_metric == AOM_TUNE_PSNR)
+      if (x->tune_metric == AOM_TUNE_PSNR && xd->bd == 8)
         assert(sum_rd_stats.dist == dist_8x8);
       sum_rd_stats.dist = dist_8x8;
       tmp_rd = RDCOST(x->rdmult, sum_rd_stats.rate, sum_rd_stats.dist);