Add option to use fp quant when new-quant is used

Change-Id: I861ccc09454606ce4172db02a52550f9b7a7800c

av1/common/blockd.h

@@ -31,6 +31,12 @@
 extern "C" {
 #endif
 
+#if CONFIG_NEW_QUANT
+#define USE_B_QUANT_NO_TRELLIS 0
+#else
+#define USE_B_QUANT_NO_TRELLIS 1
+#endif  // CONFIG_NEW_QUANT
+
 #define MAX_MB_PLANE 3
 
 // Set COMPOUND_SEGMENT_TYPE to one of the three

av1/encoder/encodemb.c

@@ -585,8 +585,9 @@ static void encode_block(int plane, int block, int blk_row, int blk_col,
       av1_optimize_b(args->cpi, x, plane, blk_row, blk_col, block, plane_bsize,
                      tx_size, a, l, CONFIG_LV_MAP);
     } else {
-      av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
-                      tx_size, AV1_XFORM_QUANT_B);
+      av1_xform_quant(
+          cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+          USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
     }
   } else {
     p->eobs[block] = 0;
@@ -901,8 +902,9 @@ void av1_encode_block_intra(int plane, int block, int blk_row, int blk_col,
     }
 #endif  // CONFIG_TXK_SEL
   } else {
-    av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
-                    AV1_XFORM_QUANT_B);
+    av1_xform_quant(
+        cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+        USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
   }
 
   av1_inverse_transform_block(xd, dqcoeff, plane, tx_type, tx_size, dst,

av1/encoder/encodetxb.c

@@ -2337,8 +2337,9 @@ int64_t av1_search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
     RD_STATS this_rd_stats;
     av1_invalid_rd_stats(&this_rd_stats);
     if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) {
-      av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
-                      tx_size, AV1_XFORM_QUANT_B);
+      av1_xform_quant(
+          cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+          USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
     } else {
       av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
                       tx_size, AV1_XFORM_QUANT_FP);
@@ -2376,8 +2377,9 @@ int64_t av1_search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
     // intra mode needs decoded result such that the next transform block
     // can use it for prediction.
     if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) {
-      av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
-                      tx_size, AV1_XFORM_QUANT_B);
+      av1_xform_quant(
+          cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+          USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
     } else {
       av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
                       tx_size, AV1_XFORM_QUANT_FP);

av1/encoder/rdopt.c

@@ -1858,10 +1858,10 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
 
 #if !CONFIG_TXK_SEL
   // full forward transform and quantization
-  if (cpi->sf.optimize_coefficients == FINAL_PASS_TRELLIS_OPT ||
-      cpi->sf.optimize_coefficients == NO_TRELLIS_OPT) {
-    av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
-                    AV1_XFORM_QUANT_B);
+  if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) {
+    av1_xform_quant(
+        cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+        USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
   } else {
     av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
                     AV1_XFORM_QUANT_FP);
@@ -3510,10 +3510,10 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
                       NULL, 0, bw, bh);
   }
 
-  if (cpi->sf.optimize_coefficients == FINAL_PASS_TRELLIS_OPT ||
-      cpi->sf.optimize_coefficients == NO_TRELLIS_OPT) {
-    av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
-                    AV1_XFORM_QUANT_B);
+  if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) {
+    av1_xform_quant(
+        cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+        USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
 
   } else {
     av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,