Merge "Use vp10_[fwd/inv]_txfm2d_add_32x32 for bd 10" into nextgenv2

vp10/common/common.h

@@ -67,6 +67,7 @@ static INLINE int get_unsigned_bits(unsigned int num_values) {
 
 #define VP9_FRAME_MARKER 0x2
 
+#define BITDEPTH_10 10
 
 #ifdef __cplusplus
 }  // extern "C"

vp10/common/idct.c

@@ -1289,7 +1289,7 @@ void vp10_highbd_inv_txfm_add_4x4(const tran_low_t *input, uint8_t *dest,
 
   switch (tx_type) {
     case DCT_DCT:
-      if (bd == 10)
+      if (bd == BITDEPTH_10)
         vp10_inv_txfm2d_add_4x4(input, CONVERT_TO_SHORTPTR(dest), stride,
                                 &inv_txfm_2d_cfg_dct_dct_4, bd);
       else
@@ -1332,7 +1332,7 @@ void vp10_highbd_inv_txfm_add_8x8(const tran_low_t *input, uint8_t *dest,
                                   TX_TYPE tx_type) {
   switch (tx_type) {
     case DCT_DCT:
-      if (bd == 10)
+      if (bd == BITDEPTH_10)
         vp10_inv_txfm2d_add_8x8(input, CONVERT_TO_SHORTPTR(dest), stride,
                                 &inv_txfm_2d_cfg_dct_dct_8, bd);
       else
@@ -1375,7 +1375,7 @@ void vp10_highbd_inv_txfm_add_16x16(const tran_low_t *input, uint8_t *dest,
                                     TX_TYPE tx_type) {
   switch (tx_type) {
     case DCT_DCT:
-      if (bd == 10)
+      if (bd == BITDEPTH_10)
         vp10_inv_txfm2d_add_16x16(input, CONVERT_TO_SHORTPTR(dest), stride,
                                   &inv_txfm_2d_cfg_dct_dct_16, bd);
       else
@@ -1418,7 +1418,11 @@ void vp10_highbd_inv_txfm_add_32x32(const tran_low_t *input, uint8_t *dest,
                                     TX_TYPE tx_type) {
   switch (tx_type) {
     case DCT_DCT:
-      vp10_highbd_idct32x32_add(input, dest, stride, eob, bd);
+      if (bd == BITDEPTH_10)
+        vp10_inv_txfm2d_add_32x32(input, CONVERT_TO_SHORTPTR(dest), stride,
+                                  &inv_txfm_2d_cfg_dct_dct_32, bd);
+      else
+        vp10_highbd_idct32x32_add(input, dest, stride, eob, bd);
       break;
 #if CONFIG_EXT_TX
     case ADST_DCT:

vp10/decoder/detokenize.c

@@ -64,7 +64,7 @@ static int decode_coefs(const MACROBLOCKD *xd,
   unsigned int (*eob_branch_count)[COEFF_CONTEXTS];
   uint8_t token_cache[32 * 32];
   const uint8_t *band_translate = get_band_translate(tx_size);
-  const int dq_shift = (tx_size == TX_32X32);
+  int dq_shift;
   int v, token;
   int16_t dqv = dq[0];
   const uint8_t *cat1_prob;
@@ -113,6 +113,16 @@ static int decode_coefs(const MACROBLOCKD *xd,
   cat6_prob = vp10_cat6_prob;
 #endif
 
+#if CONFIG_VP9_HIGHBITDEPTH
+  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH && xd->bd == BITDEPTH_10) {
+    dq_shift = 0;
+  } else {
+    dq_shift = (tx_size == TX_32X32);
+  }
+#else
+  dq_shift = (tx_size == TX_32X32);
+#endif
+
   while (c < max_eob) {
     int val = -1;
     band = *band_translate++;
@@ -237,7 +247,7 @@ static int decode_coefs_ans(const MACROBLOCKD *const xd,
   unsigned int (*eob_branch_count)[COEFF_CONTEXTS];
   uint8_t token_cache[32 * 32];
   const uint8_t *band_translate = get_band_translate(tx_size);
-  const int dq_shift = (tx_size == TX_32X32);
+  int dq_shift;
   int v, token;
   int16_t dqv = dq[0];
   const uint8_t *cat1_prob;
@@ -247,6 +257,16 @@ static int decode_coefs_ans(const MACROBLOCKD *const xd,
   const uint8_t *cat5_prob;
   const uint8_t *cat6_prob;
 
+#if CONFIG_VP9_HIGHBITDEPTH
+  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH && xd->bd == BITDEPTH_10) {
+    dq_shift = 0;
+  } else {
+    dq_shift = (tx_size == TX_32X32);
+  }
+#else
+  dq_shift = (tx_size == TX_32X32);
+#endif
+
   if (counts) {
     coef_counts = counts->coef[tx_size][type][ref];
     eob_branch_count = counts->eob_branch[tx_size][type][ref];

vp10/encoder/encodemb.c

@@ -105,7 +105,7 @@ static int optimize_b(MACROBLOCK *mb, int plane, int block,
   const int eob = p->eobs[block];
   const PLANE_TYPE type = pd->plane_type;
   const int default_eob = 16 << (tx_size << 1);
-  const int mul = 1 + (tx_size == TX_32X32);
+  int mul;
   const int16_t *dequant_ptr = pd->dequant;
   const uint8_t *const band_translate = get_band_translate(tx_size);
   TX_TYPE tx_type = get_tx_type(type, xd, block, tx_size);
@@ -129,6 +129,16 @@ static int optimize_b(MACROBLOCK *mb, int plane, int block,
   assert((!type && !plane) || (type && plane));
   assert(eob <= default_eob);
 
+#if CONFIG_VP9_HIGHBITDEPTH
+  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH && xd->bd == BITDEPTH_10) {
+    mul = 1;
+  } else {
+    mul = 1 + (tx_size == TX_32X32);
+  }
+#else
+  mul = 1 + (tx_size == TX_32X32);
+#endif
+
   /* Now set up a Viterbi trellis to evaluate alternative roundings. */
   if (!ref)
     rdmult = (rdmult * 9) >> 4;
@@ -384,7 +394,7 @@ void vp10_xform_quant(MACROBLOCK *x, int plane, int block, int blk_row,
       if (x->skip_block) {
         vp10_quantize_skip(tx2d_size, qcoeff, dqcoeff, eob);
       } else {
-        if (tx_size == TX_32X32)
+        if (tx_size == TX_32X32 && xd->bd != 10)
           quant_func_list[xform_quant_idx][QUANT_FUNC_HIGHBD_32](
               coeff, tx2d_size, p, qcoeff, pd, dqcoeff, eob, scan_order);
         else

vp10/encoder/hybrid_fwd_txfm.c

@@ -195,7 +195,7 @@ void vp10_highbd_fwd_txfm_4x4(const int16_t *src_diff, tran_low_t *coeff,
 
   switch (tx_type) {
     case DCT_DCT:
-      if (bd == 10) {
+      if (bd == BITDEPTH_10) {
         vp10_fwd_txfm2d_4x4(src_diff, coeff, diff_stride,
                             &fwd_txfm_2d_cfg_dct_dct_4, bd);
       } else {
@@ -239,7 +239,7 @@ static void highbd_fwd_txfm_8x8(const int16_t *src_diff, tran_low_t *coeff,
   (void)fwd_txfm_opt;
   switch (tx_type) {
     case DCT_DCT:
-      if (bd == 10) {
+      if (bd == BITDEPTH_10) {
         vp10_fwd_txfm2d_8x8(src_diff, coeff, diff_stride,
                             &fwd_txfm_2d_cfg_dct_dct_8, bd);
         break;
@@ -285,7 +285,7 @@ static void highbd_fwd_txfm_16x16(const int16_t *src_diff, tran_low_t *coeff,
   (void)fwd_txfm_opt;
   switch (tx_type) {
     case DCT_DCT:
-      if (bd == 10) {
+      if (bd == BITDEPTH_10) {
         vp10_fwd_txfm2d_16x16(src_diff, coeff, diff_stride,
                               &fwd_txfm_2d_cfg_dct_dct_16, bd);
         break;
@@ -332,10 +332,15 @@ static void highbd_fwd_txfm_32x32(int rd_transform, const int16_t *src_diff,
   (void)bd;
   switch (tx_type) {
     case DCT_DCT:
-      if (fwd_txfm_opt == FWD_TXFM_OPT_NORMAL)
-        highbd_fdct32x32(rd_transform, src_diff, coeff, diff_stride);
-      else  // FWD_TXFM_OPT_DC
-        vpx_highbd_fdct32x32_1(src_diff, coeff, diff_stride);
+      if (bd == BITDEPTH_10) {
+        vp10_fwd_txfm2d_32x32(src_diff, coeff, diff_stride,
+                              &fwd_txfm_2d_cfg_dct_dct_32, bd);
+      } else {
+        if (fwd_txfm_opt == FWD_TXFM_OPT_NORMAL)
+          highbd_fdct32x32(rd_transform, src_diff, coeff, diff_stride);
+        else  // FWD_TXFM_OPT_DC
+          vpx_highbd_fdct32x32_1(src_diff, coeff, diff_stride);
+      }
       break;
 #if CONFIG_EXT_TX
     case ADST_DCT:

vp10/encoder/rdopt.c

@@ -1008,6 +1008,9 @@ static void dist_block(const VP10_COMP *cpi, MACROBLOCK *x, int plane,
     tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
 #if CONFIG_VP9_HIGHBITDEPTH
     const int bd = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd : 8;
+    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH && xd->bd == BITDEPTH_10) {
+      shift = 2;
+    }
     *out_dist = vp10_highbd_block_error(coeff, dqcoeff, 16 << ss_txfrm_size,
                                         &this_sse, bd) >> shift;
 #else
@@ -1176,9 +1179,17 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
         int64_t dc_correct = orig_sse - resd_sse * resd_sse;
 #if CONFIG_VP9_HIGHBITDEPTH
         dc_correct >>= ((xd->bd - 8) * 2);
-#endif
+        if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH &&
+            xd->bd == BITDEPTH_10) {
+          dc_correct >>= 2;
+        } else {
+          if (tx_size != TX_32X32)
+            dc_correct >>= 2;
+        }
+#else
         if (tx_size != TX_32X32)
           dc_correct >>= 2;
+#endif
 
         dist = VPXMAX(0, sse - dc_correct);
       }