Harmonize quantizers for new-quant and regular

Change-Id: Ia8ccf6f270c06d14ecd37882860ce6a5a50434e5

av1/decoder/detokenize.c

@@ -52,6 +52,9 @@ static INLINE int read_coeff(const aom_prob *probs, int n, aom_reader *r) {
 #if CONFIG_AOM_QM
 static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
                         TX_SIZE tx_size, TX_TYPE tx_type, const int16_t *dq,
+#if CONFIG_NEW_QUANT
+                        dequant_val_type_nuq *dq_val,
+#endif  // CONFIG_NEW_QUANT
                         int ctx, const int16_t *scan, const int16_t *nb,
                         int16_t *max_scan_line, aom_reader *r,
                         const qm_val_t *iqm[2][TX_SIZES])
@@ -63,7 +66,7 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
 #endif  // CONFIG_NEW_QUANT
                         int ctx, const int16_t *scan, const int16_t *nb,
                         int16_t *max_scan_line, aom_reader *r)
-#endif
+#endif  // CONFIG_AOM_QM
 {
   FRAME_COUNTS *counts = xd->counts;
   FRAME_CONTEXT *const fc = xd->fc;
@@ -71,7 +74,7 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
   const int ref = is_inter_block(&xd->mi[0]->mbmi);
 #if CONFIG_AOM_QM
   const qm_val_t *iqmatrix = iqm[!ref][tx_size];
-#endif
+#endif  // CONFIG_AOM_QM
   int band, c = 0;
   const int tx_size_ctx = txsize_sqr_map[tx_size];
   aom_prob(*coef_probs)[COEFF_CONTEXTS][UNCONSTRAINED_NODES] =
@@ -99,6 +102,9 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
   const uint8_t *cat5_prob;
   const uint8_t *cat6_prob;
   (void)tx_type;
+#if CONFIG_AOM_QM
+  (void)iqmatrix;
+#endif  // CONFIG_AOM_QM
 
   if (counts) {
     coef_counts = counts->coef[tx_size_ctx][type][ref];
@@ -345,9 +351,12 @@ int av1_decode_block_tokens(MACROBLOCKD *const xd, int plane,
 #endif  //  CONFIG_NEW_QUANT
 
 #if CONFIG_AOM_QM
-  const int eob = decode_coefs(xd, pd->plane_type, pd->dqcoeff, tx_size,
-                               tx_type, dequant, ctx, sc->scan, sc->neighbors,
-                               max_scan_line, r, pd->seg_iqmatrix[seg_id]);
+  const int eob = decode_coefs(
+      xd, pd->plane_type, pd->dqcoeff, tx_size, tx_type, dequant,
+#if CONFIG_NEW_QUANT
+      pd->seg_dequant_nuq[seg_id][dq],
+#endif  // CONFIG_NEW_QUANT
+      ctx, sc->scan, sc->neighbors, max_scan_line, r, pd->seg_iqmatrix[seg_id]);
 #else
   const int eob =
       decode_coefs(xd, pd->plane_type, pd->dqcoeff, tx_size, tx_type, dequant,

av1/encoder/encodemb.h

@@ -39,8 +39,13 @@ typedef enum AV1_XFORM_QUANT {
   AV1_XFORM_QUANT_FP = 0,
   AV1_XFORM_QUANT_B = 1,
   AV1_XFORM_QUANT_DC = 2,
-  AV1_XFORM_QUANT_SKIP_QUANT = 3,
-  AV1_XFORM_QUANT_LAST = 4
+#if CONFIG_NEW_QUANT
+  AV1_XFORM_QUANT_FP_NUQ = 3,
+  AV1_XFORM_QUANT_B_NUQ = 4,
+  AV1_XFORM_QUANT_DC_NUQ = 5,
+#endif  // CONFIG_NEW_QUANT
+  AV1_XFORM_QUANT_SKIP_QUANT,
+  AV1_XFORM_QUANT_TYPES,
 } AV1_XFORM_QUANT;
 
 void av1_encode_sb(AV1_COMMON *cm, MACROBLOCK *x, BLOCK_SIZE bsize);
@@ -50,21 +55,7 @@ void av1_encode_sb_supertx(AV1_COMMON *cm, MACROBLOCK *x, BLOCK_SIZE bsize);
 void av1_encode_sby_pass1(AV1_COMMON *cm, MACROBLOCK *x, BLOCK_SIZE bsize);
 void av1_xform_quant(const AV1_COMMON *cm, MACROBLOCK *x, int plane, int block,
                      int blk_row, int blk_col, BLOCK_SIZE plane_bsize,
-                     TX_SIZE tx_size, AV1_XFORM_QUANT xform_quant_idx);
-#if CONFIG_NEW_QUANT
-void av1_xform_quant_nuq(const AV1_COMMON *cm, MACROBLOCK *x, int plane,
-                         int block, int blk_row, int blk_col,
-                         BLOCK_SIZE plane_bsize, TX_SIZE tx_size, int ctx);
-void av1_xform_quant_dc_nuq(MACROBLOCK *x, int plane, int block, int blk_row,
-                            int blk_col, BLOCK_SIZE plane_bsize,
-                            TX_SIZE tx_size, int ctx);
-void av1_xform_quant_fp_nuq(const AV1_COMMON *cm, MACROBLOCK *x, int plane,
-                            int block, int blk_row, int blk_col,
-                            BLOCK_SIZE plane_bsize, TX_SIZE tx_size, int ctx);
-void av1_xform_quant_dc_fp_nuq(MACROBLOCK *x, int plane, int block, int blk_row,
-                               int blk_col, BLOCK_SIZE plane_bsize,
-                               TX_SIZE tx_size, int ctx);
-#endif
+                     TX_SIZE tx_size, int ctx, AV1_XFORM_QUANT xform_quant_idx);
 
 int av1_optimize_b(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, int block,
                    TX_SIZE tx_size, int ctx);

av1/encoder/quantize.c

@@ -442,14 +442,13 @@ void av1_quantize_fp_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
                             const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr,
                             const MACROBLOCKD_PLANE *pd,
                             tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
-                            const SCAN_ORDER *sc, const QUANT_PARAM *qparam
-#if CONFIG_AOM_QM
-                            ,
-                            const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr
-#endif
-                            ) {
+                            const SCAN_ORDER *sc, const QUANT_PARAM *qparam) {
   // obsolete skip_block
   const int skip_block = 0;
+#if CONFIG_AOM_QM
+  const qm_val_t *qm_ptr = qparam->qmatrix;
+  const qm_val_t *iqm_ptr = qparam->iqmatrix;
+#endif  // CONFIG_AOM_QM
 
   switch (qparam->log_scale) {
     case 0:
@@ -494,14 +493,13 @@ void av1_quantize_b_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
                            const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr,
                            const MACROBLOCKD_PLANE *pd, tran_low_t *dqcoeff_ptr,
                            uint16_t *eob_ptr, const SCAN_ORDER *sc,
-                           const QUANT_PARAM *qparam
-#if CONFIG_AOM_QM
-                           ,
-                           const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr
-#endif
-                           ) {
+                           const QUANT_PARAM *qparam) {
   // obsolete skip_block
   const int skip_block = 0;
+#if CONFIG_AOM_QM
+  const qm_val_t *qm_ptr = qparam->qmatrix;
+  const qm_val_t *iqm_ptr = qparam->iqmatrix;
+#endif  // CONFIG_AOM_QM
 
   switch (qparam->log_scale) {
     case 0:
@@ -544,14 +542,14 @@ void av1_quantize_dc_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
                             const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr,
                             const MACROBLOCKD_PLANE *pd,
                             tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
-                            const SCAN_ORDER *sc, const QUANT_PARAM *qparam
-#if CONFIG_AOM_QM
-                            ,
-                            const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr
-#endif
-                            ) {
+                            const SCAN_ORDER *sc, const QUANT_PARAM *qparam) {
   // obsolete skip_block
   const int skip_block = 0;
+#if CONFIG_AOM_QM
+  const qm_val_t *qm_ptr = qparam->qmatrix;
+  const qm_val_t *iqm_ptr = qparam->iqmatrix;
+#endif  // CONFIG_AOM_QM
+
   (void)sc;
 
   switch (qparam->log_scale) {
@@ -588,19 +586,138 @@ void av1_quantize_dc_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
   }
 }
 
+#if CONFIG_NEW_QUANT
+void av1_quantize_b_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
+                               const MACROBLOCK_PLANE *p,
+                               tran_low_t *qcoeff_ptr,
+                               const MACROBLOCKD_PLANE *pd,
+                               tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
+                               const SCAN_ORDER *sc,
+                               const QUANT_PARAM *qparam) {
+  // obsolete skip_block
+  const int skip_block = 0;
+  const uint8_t *band = get_band_translate(qparam->tx_size);
+  int dq = qparam->dq;
+
+  switch (qparam->log_scale) {
+    case 0:
+      quantize_nuq(coeff_ptr, n_coeffs, skip_block, p->quant, p->quant_shift,
+                   pd->dequant,
+                   (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
+                   (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq],
+                   qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan, band);
+      break;
+    case 1:
+      quantize_32x32_nuq(coeff_ptr, n_coeffs, skip_block, p->quant,
+                         p->quant_shift, pd->dequant,
+                         (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
+                         (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq],
+                         qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan, band);
+      break;
+#if CONFIG_TX64X64
+    case 2:
+      quantize_64x64_nuq(coeff_ptr, n_coeffs, skip_block, p->quant,
+                         p->quant_shift, pd->dequant,
+                         (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
+                         (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq],
+                         qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan, band);
+      break;
+#endif  // CONFIG_TX64X64
+    default: assert(0);
+  }
+}
+
+void av1_quantize_fp_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
+                                const MACROBLOCK_PLANE *p,
+                                tran_low_t *qcoeff_ptr,
+                                const MACROBLOCKD_PLANE *pd,
+                                tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
+                                const SCAN_ORDER *sc,
+                                const QUANT_PARAM *qparam) {
+  // obsolete skip_block
+  const int skip_block = 0;
+  const uint8_t *band = get_band_translate(qparam->tx_size);
+  int dq = qparam->dq;
+
+  switch (qparam->log_scale) {
+    case 0:
+      quantize_fp_nuq(coeff_ptr, n_coeffs, skip_block, p->quant_fp, pd->dequant,
+                      (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
+                      (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq],
+                      qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan, band);
+      break;
+    case 1:
+      quantize_32x32_fp_nuq(
+          coeff_ptr, n_coeffs, skip_block, p->quant_fp, pd->dequant,
+          (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
+          (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], qcoeff_ptr,
+          dqcoeff_ptr, eob_ptr, sc->scan, band);
+      break;
+#if CONFIG_TX64X64
+    case 2:
+      quantize_64x64_fp_nuq(
+          coeff_ptr, n_coeffs, skip_block, p->quant_fp, pd->dequant,
+          (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
+          (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], qcoeff_ptr,
+          dqcoeff_ptr, eob_ptr, sc->scan, band);
+      break;
+#endif  // CONFIG_TX64X64
+    default: assert(0);
+  }
+}
+
+void av1_quantize_dc_nuq_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
+                                const MACROBLOCK_PLANE *p,
+                                tran_low_t *qcoeff_ptr,
+                                const MACROBLOCKD_PLANE *pd,
+                                tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
+                                const SCAN_ORDER *sc,
+                                const QUANT_PARAM *qparam) {
+  // obsolete skip_block
+  const int skip_block = 0;
+  int dq = qparam->dq;
+  (void)sc;
+
+  switch (qparam->log_scale) {
+    case 0:
+      quantize_dc_fp_nuq(coeff_ptr, n_coeffs, skip_block, p->quant_fp[0],
+                         pd->dequant[0], p->cuml_bins_nuq[dq][0],
+                         pd->dequant_val_nuq[dq][0], qcoeff_ptr, dqcoeff_ptr,
+                         eob_ptr);
+      break;
+    case 1:
+      quantize_dc_32x32_fp_nuq(coeff_ptr, n_coeffs, skip_block, p->quant_fp[0],
+                               pd->dequant[0], p->cuml_bins_nuq[dq][0],
+                               pd->dequant_val_nuq[dq][0], qcoeff_ptr,
+                               dqcoeff_ptr, eob_ptr);
+      break;
+#if CONFIG_TX64X64
+    case 2:
+      quantize_dc_64x64_fp_nuq(coeff_ptr, n_coeffs, skip_block, p->quant_fp[0],
+                               pd->dequant[0], p->cuml_bins_nuq[dq][0],
+                               pd->dequant_val_nuq[dq][0], qcoeff_ptr,
+                               dqcoeff_ptr, eob_ptr);
+      break;
+#endif  // CONFIG_TX64X64
+    default: assert(0);
+  }
+}
+#endif  // CONFIG_NEW_QUANT
+
 #if CONFIG_AOM_HIGHBITDEPTH
-void av1_highbd_quantize_fp_facade(
-    const tran_low_t *coeff_ptr, intptr_t n_coeffs, const MACROBLOCK_PLANE *p,
-    tran_low_t *qcoeff_ptr, const MACROBLOCKD_PLANE *pd,
-    tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const SCAN_ORDER *sc,
-    const QUANT_PARAM *qparam
-#if CONFIG_AOM_QM
-    ,
-    const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr
-#endif
-    ) {
+void av1_highbd_quantize_fp_facade(const tran_low_t *coeff_ptr,
+                                   intptr_t n_coeffs, const MACROBLOCK_PLANE *p,
+                                   tran_low_t *qcoeff_ptr,
+                                   const MACROBLOCKD_PLANE *pd,
+                                   tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
+                                   const SCAN_ORDER *sc,
+                                   const QUANT_PARAM *qparam) {
   // obsolete skip_block
   const int skip_block = 0;
+#if CONFIG_AOM_QM
+  const qm_val_t *qm_ptr = qparam->qmatrix;
+  const qm_val_t *iqm_ptr = qparam->iqmatrix;
+#endif  // CONFIG_AOM_QM
 
   av1_highbd_quantize_fp(coeff_ptr, n_coeffs, skip_block, p->zbin, p->round_fp,
                          p->quant_fp, p->quant_shift, qcoeff_ptr, dqcoeff_ptr,
@@ -611,18 +728,19 @@ void av1_highbd_quantize_fp_facade(
                          qparam->log_scale);
 }
 
-void av1_highbd_quantize_b_facade(
-    const tran_low_t *coeff_ptr, intptr_t n_coeffs, const MACROBLOCK_PLANE *p,
-    tran_low_t *qcoeff_ptr, const MACROBLOCKD_PLANE *pd,
-    tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const SCAN_ORDER *sc,
-    const QUANT_PARAM *qparam
-#if CONFIG_AOM_QM
-    ,
-    const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr
-#endif
-    ) {
+void av1_highbd_quantize_b_facade(const tran_low_t *coeff_ptr,
+                                  intptr_t n_coeffs, const MACROBLOCK_PLANE *p,
+                                  tran_low_t *qcoeff_ptr,
+                                  const MACROBLOCKD_PLANE *pd,
+                                  tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
+                                  const SCAN_ORDER *sc,
+                                  const QUANT_PARAM *qparam) {
   // obsolete skip_block
   const int skip_block = 0;
+#if CONFIG_AOM_QM
+  const qm_val_t *qm_ptr = qparam->qmatrix;
+  const qm_val_t *iqm_ptr = qparam->iqmatrix;
+#endif  // CONFIG_AOM_QM
 
   av1_highbd_quantize_b(coeff_ptr, n_coeffs, skip_block, p->zbin, p->round,
                         p->quant, p->quant_shift, qcoeff_ptr, dqcoeff_ptr,
@@ -633,28 +751,60 @@ void av1_highbd_quantize_b_facade(
                         qparam->log_scale);
 }
 
-void av1_highbd_quantize_dc_facade(
-    const tran_low_t *coeff_ptr, intptr_t n_coeffs, const MACROBLOCK_PLANE *p,
-    tran_low_t *qcoeff_ptr, const MACROBLOCKD_PLANE *pd,
-    tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const SCAN_ORDER *sc,
-    const QUANT_PARAM *qparam
+#if CONFIG_AOM_HIGHBITDEPTH
+static INLINE void highbd_quantize_dc(
+    const tran_low_t *coeff_ptr, int n_coeffs, int skip_block,
+    const int16_t *round_ptr, const int16_t quant, tran_low_t *qcoeff_ptr,
+    tran_low_t *dqcoeff_ptr, const int16_t dequant_ptr, uint16_t *eob_ptr,
 #if CONFIG_AOM_QM
-    ,
-    const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr
+    const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
 #endif
-    ) {
+    const int log_scale) {
+  int eob = -1;
+
+  memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
+  memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
+#if CONFIG_AOM_QM
+  (void)qm_ptr;
+  (void)iqm_ptr;
+#endif
+  if (!skip_block) {
+    const int coeff = coeff_ptr[0];
+    const int coeff_sign = (coeff >> 31);
+    const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
+    const int64_t tmp = abs_coeff + round_ptr[0];
+    const uint32_t abs_qcoeff = (uint32_t)((tmp * quant) >> (16 - log_scale));
+    qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
+    dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant_ptr / (1 << log_scale);
+    if (abs_qcoeff) eob = 0;
+  }
+  *eob_ptr = eob + 1;
+}
+#endif  // CONFIG_AOM_HIGHBITDEPTH
+
+void av1_highbd_quantize_dc_facade(const tran_low_t *coeff_ptr,
+                                   intptr_t n_coeffs, const MACROBLOCK_PLANE *p,
+                                   tran_low_t *qcoeff_ptr,
+                                   const MACROBLOCKD_PLANE *pd,
+                                   tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
+                                   const SCAN_ORDER *sc,
+                                   const QUANT_PARAM *qparam) {
   // obsolete skip_block
   const int skip_block = 0;
+#if CONFIG_AOM_QM
+  const qm_val_t *qm_ptr = qparam->qmatrix;
+  const qm_val_t *iqm_ptr = qparam->iqmatrix;
+#endif  // CONFIG_AOM_QM
 
   (void)sc;
 
-  av1_highbd_quantize_dc(coeff_ptr, (int)n_coeffs, skip_block, p->round,
-                         p->quant_fp[0], qcoeff_ptr, dqcoeff_ptr,
-                         pd->dequant[0], eob_ptr,
+  highbd_quantize_dc(coeff_ptr, (int)n_coeffs, skip_block, p->round,
+                     p->quant_fp[0], qcoeff_ptr, dqcoeff_ptr, pd->dequant[0],
+                     eob_ptr,
 #if CONFIG_AOM_QM
-                         qm_ptr, iqm_ptr,
+                     qm_ptr, iqm_ptr,
 #endif
-                         qparam->log_scale);
+                     qparam->log_scale);
 }
 
 #if CONFIG_NEW_QUANT
@@ -1057,6 +1207,117 @@ void highbd_quantize_dc_64x64_fp_nuq(
   *eob_ptr = eob + 1;
 }
 #endif  // CONFIG_TX64X64
+
+void av1_highbd_quantize_b_nuq_facade(
+    const tran_low_t *coeff_ptr, intptr_t n_coeffs, const MACROBLOCK_PLANE *p,
+    tran_low_t *qcoeff_ptr, const MACROBLOCKD_PLANE *pd,
+    tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const SCAN_ORDER *sc,
+    const QUANT_PARAM *qparam) {
+  // obsolete skip_block
+  const int skip_block = 0;
+  const uint8_t *band = get_band_translate(qparam->tx_size);
+  const int dq = qparam->dq;
+
+  switch (qparam->log_scale) {
+    case 0:
+      highbd_quantize_nuq(coeff_ptr, n_coeffs, skip_block, p->quant,
+                          p->quant_shift, pd->dequant,
+                          (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
+                          (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq],
+                          qcoeff_ptr, dqcoeff_ptr, eob_ptr, sc->scan, band);
+      break;
+    case 1:
+      highbd_quantize_32x32_nuq(
+          coeff_ptr, n_coeffs, skip_block, p->quant, p->quant_shift,
+          pd->dequant, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
+          (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], qcoeff_ptr,
+          dqcoeff_ptr, eob_ptr, sc->scan, band);
+      break;
+#if CONFIG_TX64X64
+    case 2:
+      highbd_quantize_64x64_nuq(
+          coeff_ptr, n_coeffs, skip_block, p->quant, p->quant_shift,
+          pd->dequant, (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
+          (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], qcoeff_ptr,
+          dqcoeff_ptr, eob_ptr, sc->scan, band);
+      break;
+#endif  // CONFIG_TX64X64
+    default: assert(0);
+  }
+}
+
+void av1_highbd_quantize_fp_nuq_facade(
+    const tran_low_t *coeff_ptr, intptr_t n_coeffs, const MACROBLOCK_PLANE *p,
+    tran_low_t *qcoeff_ptr, const MACROBLOCKD_PLANE *pd,
+    tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const SCAN_ORDER *sc,
+    const QUANT_PARAM *qparam) {
+  // obsolete skip_block
+  const int skip_block = 0;
+  const uint8_t *band = get_band_translate(qparam->tx_size);
+  const int dq = qparam->dq;
+
+  switch (qparam->log_scale) {
+    case 0:
+      highbd_quantize_fp_nuq(
+          coeff_ptr, n_coeffs, skip_block, p->quant_fp, pd->dequant,
+          (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
+          (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], qcoeff_ptr,
+          dqcoeff_ptr, eob_ptr, sc->scan, band);
+      break;
+    case 1:
+      highbd_quantize_32x32_fp_nuq(
+          coeff_ptr, n_coeffs, skip_block, p->quant_fp, pd->dequant,
+          (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
+          (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], qcoeff_ptr,
+          dqcoeff_ptr, eob_ptr, sc->scan, band);
+      break;
+#if CONFIG_TX64X64
+    case 2:
+      highbd_quantize_64x64_fp_nuq(
+          coeff_ptr, n_coeffs, skip_block, p->quant_fp, pd->dequant,
+          (const cuml_bins_type_nuq *)p->cuml_bins_nuq[dq],
+          (const dequant_val_type_nuq *)pd->dequant_val_nuq[dq], qcoeff_ptr,
+          dqcoeff_ptr, eob_ptr, sc->scan, band);
+      break;
+#endif  // CONFIG_TX64X64
+    default: assert(0);
+  }
+}
+
+void av1_highbd_quantize_dc_nuq_facade(
+    const tran_low_t *coeff_ptr, intptr_t n_coeffs, const MACROBLOCK_PLANE *p,
+    tran_low_t *qcoeff_ptr, const MACROBLOCKD_PLANE *pd,
+    tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr, const SCAN_ORDER *sc,
+    const QUANT_PARAM *qparam) {
+  // obsolete skip_block
+  const int skip_block = 0;
+  const int dq = qparam->dq;
+  (void)sc;
+
+  switch (qparam->log_scale) {
+    case 0:
+      highbd_quantize_dc_fp_nuq(coeff_ptr, n_coeffs, skip_block, p->quant_fp[0],
+                                pd->dequant[0], p->cuml_bins_nuq[dq][0],
+                                pd->dequant_val_nuq[dq][0], qcoeff_ptr,
+                                dqcoeff_ptr, eob_ptr);
+      break;
+    case 1:
+      highbd_quantize_dc_32x32_fp_nuq(
+          coeff_ptr, n_coeffs, skip_block, p->quant_fp[0], pd->dequant[0],
+          p->cuml_bins_nuq[dq][0], pd->dequant_val_nuq[dq][0], qcoeff_ptr,
+          dqcoeff_ptr, eob_ptr);
+      break;
+#if CONFIG_TX64X64
+    case 2:
+      highbd_quantize_dc_64x64_fp_nuq(
+          coeff_ptr, n_coeffs, skip_block, p->quant_fp[0], pd->dequant[0],
+          p->cuml_bins_nuq[dq][0], pd->dequant_val_nuq[dq][0], qcoeff_ptr,
+          dqcoeff_ptr, eob_ptr);
+      break;
+#endif  // CONFIG_TX64X64
+    default: assert(0);
+  }
+}
 #endif  // CONFIG_NEW_QUANT
 #endif  // CONFIG_AOM_HIGHBITDEPTH
 
@@ -1476,38 +1737,6 @@ void av1_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
 }
 #endif
 
-#if CONFIG_AOM_HIGHBITDEPTH
-void av1_highbd_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs,
-                            int skip_block, const int16_t *round_ptr,
-                            const int16_t quant, tran_low_t *qcoeff_ptr,
-                            tran_low_t *dqcoeff_ptr, const int16_t dequant_ptr,
-                            uint16_t *eob_ptr,
-#if CONFIG_AOM_QM
-                            const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
-#endif
-                            const int log_scale) {
-  int eob = -1;
-
-  memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
-  memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
-#if CONFIG_AOM_QM
-  (void)qm_ptr;
-  (void)iqm_ptr;
-#endif
-  if (!skip_block) {
-    const int coeff = coeff_ptr[0];
-    const int coeff_sign = (coeff >> 31);
-    const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
-    const int64_t tmp = abs_coeff + round_ptr[0];
-    const uint32_t abs_qcoeff = (uint32_t)((tmp * quant) >> (16 - log_scale));
-    qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
-    dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant_ptr / (1 << log_scale);
-    if (abs_qcoeff) eob = 0;
-  }
-  *eob_ptr = eob + 1;
-}
-#endif
-
 static void invert_quant(int16_t *quant, int16_t *shift, int d) {
   uint32_t t;
   int l, m;

av1/encoder/quantize.h

@@ -21,19 +21,25 @@
 extern "C" {
 #endif
 
-typedef struct QUANT_PARAM { int log_scale; } QUANT_PARAM;
+typedef struct QUANT_PARAM {
+  int log_scale;
+#if CONFIG_NEW_QUANT
+  TX_SIZE tx_size;
+  int dq;
+#endif  // CONFIG_NEW_QUANT
+#if CONFIG_AOM_QM
+  const qm_val_t *qmatrix;
+  const qm_val_t *iqmatrix;
+#endif  // CONFIG_AOM_QM
+} QUANT_PARAM;
 
 typedef void (*AV1_QUANT_FACADE)(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
                                  const MACROBLOCK_PLANE *p,
                                  tran_low_t *qcoeff_ptr,
                                  const MACROBLOCKD_PLANE *pd,
                                  tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
-                                 const SCAN_ORDER *sc, const QUANT_PARAM *qparam
-#if CONFIG_AOM_QM
-                                 ,
-                                 const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr
-#endif
-                                 );
+                                 const SCAN_ORDER *sc,
+                                 const QUANT_PARAM *qparam);
 
 typedef struct {
 #if CONFIG_NEW_QUANT
@@ -84,163 +90,88 @@ void av1_quantize_fp_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
                             const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr,
                             const MACROBLOCKD_PLANE *pd,
                             tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
-                            const SCAN_ORDER *sc, const QUANT_PARAM *qparam
-#if CONFIG_AOM_QM
-                            ,