quantize.c: convert to int before apply sign

This change makes the conversions similar to those in av1_quantize.c,
and fix ubsan warnings shown in nightly tests.

Change-Id: I90851a80dcb9f052a32bf22199fd9ef8ff927725

aom_dsp/quantize.c

@@ -716,8 +716,7 @@ void aom_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
       if (abs_coeff >= zbins[rc != 0]) {
         const int64_t tmp1 = abs_coeff + round_ptr[rc != 0];
         const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1;
-        const uint32_t abs_qcoeff =
-            (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 16);
+        const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 16);
         qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
         dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
         if (abs_qcoeff) eob = i;
@@ -767,8 +766,7 @@ void aom_highbd_quantize_b_32x32_c(
       const int64_t tmp1 =
           abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1);
       const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1;
-      const uint32_t abs_qcoeff =
-          (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 15);
+      const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 15);
       qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
       dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;
       if (abs_qcoeff) eob = idx_arr[i];
@@ -818,8 +816,7 @@ void aom_highbd_quantize_b_64x64_c(
       const int64_t tmp1 =
           abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 2);
       const int64_t tmp2 = ((tmp1 * quant_ptr[rc != 0]) >> 16) + tmp1;
-      const uint32_t abs_qcoeff =
-          (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> 14);
+      const int abs_qcoeff = (int)((tmp2 * quant_shift_ptr[rc != 0]) >> 14);
       qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
       dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 4;
       if (abs_qcoeff) eob = idx_arr[i];