Simplify coefficient range checking

Deduplicate implementations of check_range, and deduplicate the call
to aom_read_bit.

Change-Id: I63b023758248717125e4df6d1c382d4c517bae84

aom_dsp/inv_txfm.h

@@ -22,7 +22,12 @@
 extern "C" {
 #endif
 
-static INLINE tran_high_t check_range(tran_high_t input) {
+static INLINE tran_high_t dct_const_round_shift(tran_high_t input) {
+  tran_high_t rv = ROUND_POWER_OF_TWO(input, DCT_CONST_BITS);
+  return rv;
+}
+
+static INLINE tran_high_t check_range(tran_high_t input, int bd) {
 #if CONFIG_COEFFICIENT_RANGE_CHECKING
   // For valid AV1 input streams, intermediate stage coefficients should always
   // stay within the range of a signed 16 bit integer. Coefficients can go out
@@ -30,20 +35,6 @@ static INLINE tran_high_t check_range(tran_high_t input) {
   // this range for every intermediate coefficient can burdensome for a decoder,
   // therefore the following assertion is only enabled when configured with
   // --enable-coefficient-range-checking.
-  assert(INT16_MIN <= input);
-  assert(input <= INT16_MAX);
-#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
-  return input;
-}
-
-static INLINE tran_high_t dct_const_round_shift(tran_high_t input) {
-  tran_high_t rv = ROUND_POWER_OF_TWO(input, DCT_CONST_BITS);
-  return rv;
-}
-
-#if CONFIG_HIGHBITDEPTH
-static INLINE tran_high_t highbd_check_range(tran_high_t input, int bd) {
-#if CONFIG_COEFFICIENT_RANGE_CHECKING
   // For valid highbitdepth AV1 streams, intermediate stage coefficients will
   // stay within the ranges:
   // - 8 bit: signed 16 bit integer
@@ -59,8 +50,6 @@ static INLINE tran_high_t highbd_check_range(tran_high_t input, int bd) {
   return input;
 }
 
-#endif  // CONFIG_HIGHBITDEPTH
-
 #if CONFIG_EMULATE_HARDWARE
 // When CONFIG_EMULATE_HARDWARE is 1 the transform performs a
 // non-normative method to handle overflows. A stream that causes
@@ -79,17 +68,17 @@ static INLINE tran_high_t highbd_check_range(tran_high_t input, int bd) {
 // bd of 12 uses trans_low with 20bits, need to remove 12bits
 // bd of x uses trans_low with 8+x bits, need to remove 24-x bits
 
-#define WRAPLOW(x) ((((int32_t)check_range(x)) << 16) >> 16)
+#define WRAPLOW(x) ((((int32_t)check_range(x, 8)) << 16) >> 16)
 #if CONFIG_HIGHBITDEPTH
 #define HIGHBD_WRAPLOW(x, bd) \
-  ((((int32_t)highbd_check_range((x), bd)) << (24 - bd)) >> (24 - bd))
+  ((((int32_t)check_range((x), bd)) << (24 - bd)) >> (24 - bd))
 #endif  // CONFIG_HIGHBITDEPTH
 
 #else  // CONFIG_EMULATE_HARDWARE
 
-#define WRAPLOW(x) ((int32_t)check_range(x))
+#define WRAPLOW(x) ((int32_t)check_range(x, 8))
 #if CONFIG_HIGHBITDEPTH
-#define HIGHBD_WRAPLOW(x, bd) ((int32_t)highbd_check_range((x), bd))
+#define HIGHBD_WRAPLOW(x, bd) ((int32_t)check_range((x), bd))
 #endif  // CONFIG_HIGHBITDEPTH
 #endif  // CONFIG_EMULATE_HARDWARE
 

av1/decoder/detokenize.c

@@ -227,18 +227,21 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
 #endif
     v = (val * dqv) >> dq_shift;
 #endif
+
+    if (v) {
+      v = aom_read_bit(r, ACCT_STR) ? -v : v;
+
 #if CONFIG_COEFFICIENT_RANGE_CHECKING
 #if CONFIG_HIGHBITDEPTH
-    if (v)
-      dqcoeff[scan[c]] =
-          highbd_check_range((aom_read_bit(r, ACCT_STR) ? -v : v), xd->bd);
+      check_range(v, xd->bd);
 #else
-    if (v) dqcoeff[scan[c]] = check_range(aom_read_bit(r, ACCT_STR) ? -v : v);
+      check_range(v, 8);
 #endif  // CONFIG_HIGHBITDEPTH
-#else
-    if (v) dqcoeff[scan[c]] = aom_read_bit(r, ACCT_STR) ? -v : v;
 #endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
 
+      dqcoeff[scan[c]] = v;
+    }
+
     ++c;
     more_data &= (c < max_eob);
     if (!more_data) break;
@@ -373,7 +376,7 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
     dqcoeff[scan[c]] =
         highbd_check_range((aom_read_bit(r, ACCT_STR) ? -v : v), xd->bd);
 #else
-    dqcoeff[scan[c]] = check_range(aom_read_bit(r, ACCT_STR) ? -v : v);
+    dqcoeff[scan[c]] = check_range(aom_read_bit(r, ACCT_STR) ? -v : v, 8);
 #endif  // CONFIG_HIGHBITDEPTH
 #else
     dqcoeff[scan[c]] = aom_read_bit(r, ACCT_STR) ? -v : v;