Add range check in inverse ADST 16x16

Bit-stream clarification related to Issue 868.

Change-Id: I92a7bc5b7782c9ea5c3f6cceec761742183c9514

test/dct16x16_test.cc

@@ -443,7 +443,7 @@ class Trans16x16TestBase {
 
   void RunQuantCheck(int dc_thred, int ac_thred) {
     ACMRandom rnd(ACMRandom::DeterministicSeed());
-    const int count_test_block = 1000;
+    const int count_test_block = 100000;
     DECLARE_ALIGNED_ARRAY(16, int16_t, input_block, kNumCoeffs);
     DECLARE_ALIGNED_ARRAY(16, int16_t, input_extreme_block, kNumCoeffs);
     DECLARE_ALIGNED_ARRAY(16, tran_low_t, output_ref_block, kNumCoeffs);
@@ -700,7 +700,7 @@ TEST_P(Trans16x16HT, MemCheck) {
 TEST_P(Trans16x16HT, QuantCheck) {
   // The encoder skips any non-DC intra prediction modes,
   // when the quantization step size goes beyond 988.
-  RunQuantCheck(549, 988);
+  RunQuantCheck(429, 729);
 }
 
 using std::tr1::make_tuple;

vp9/common/vp9_idct.c

@@ -794,18 +794,18 @@ static void iadst16(const tran_low_t *input, tran_low_t *output) {
   s14 = - x14 * cospi_24_64 + x15 * cospi_8_64;
   s15 =   x14 * cospi_8_64  + x15 * cospi_24_64;
 
-  x0 = WRAPLOW(s0 + s2, 8);
-  x1 = WRAPLOW(s1 + s3, 8);
-  x2 = WRAPLOW(s0 - s2, 8);
-  x3 = WRAPLOW(s1 - s3, 8);
+  x0 = WRAPLOW(check_range(s0 + s2), 8);
+  x1 = WRAPLOW(check_range(s1 + s3), 8);
+  x2 = WRAPLOW(check_range(s0 - s2), 8);
+  x3 = WRAPLOW(check_range(s1 - s3), 8);
   x4 = WRAPLOW(dct_const_round_shift(s4 + s6), 8);
   x5 = WRAPLOW(dct_const_round_shift(s5 + s7), 8);
   x6 = WRAPLOW(dct_const_round_shift(s4 - s6), 8);
   x7 = WRAPLOW(dct_const_round_shift(s5 - s7), 8);
-  x8 = WRAPLOW(s8 + s10, 8);
-  x9 = WRAPLOW(s9 + s11, 8);
-  x10 = WRAPLOW(s8 - s10, 8);
-  x11 = WRAPLOW(s9 - s11, 8);
+  x8 = WRAPLOW(check_range(s8 + s10), 8);
+  x9 = WRAPLOW(check_range(s9 + s11), 8);
+  x10 = WRAPLOW(check_range(s8 - s10), 8);
+  x11 = WRAPLOW(check_range(s9 - s11), 8);
   x12 = WRAPLOW(dct_const_round_shift(s12 + s14), 8);
   x13 = WRAPLOW(dct_const_round_shift(s13 + s15), 8);
   x14 = WRAPLOW(dct_const_round_shift(s12 - s14), 8);

vp9/common/vp9_idct.h

@@ -77,8 +77,7 @@ static const tran_high_t sinpi_2_9 = 9929;
 static const tran_high_t sinpi_3_9 = 13377;
 static const tran_high_t sinpi_4_9 = 15212;
 
-static INLINE tran_low_t dct_const_round_shift(tran_high_t input) {
-  tran_high_t rv = ROUND_POWER_OF_TWO(input, DCT_CONST_BITS);
+static INLINE tran_low_t check_range(tran_high_t input) {
 #if CONFIG_VP9_HIGHBITDEPTH
   // For valid highbitdepth VP9 streams, intermediate stage coefficients will
   // stay within the ranges:
@@ -92,10 +91,15 @@ static INLINE tran_low_t dct_const_round_shift(tran_high_t input) {
   // this range for every intermediate coefficient can burdensome for a decoder,
   // therefore the following assertion is only enabled when configured with
   // --enable-coefficient-range-checking.
-  assert(INT16_MIN <= rv);
-  assert(rv <= INT16_MAX);
+  assert(INT16_MIN <= input);
+  assert(input <= INT16_MAX);
 #endif
-  return (tran_low_t)rv;
+  return (tran_low_t)input;
+}
+
+static INLINE tran_low_t dct_const_round_shift(tran_high_t input) {
+  tran_high_t rv = ROUND_POWER_OF_TWO(input, DCT_CONST_BITS);
+  return check_range(rv);
 }
 
 typedef void (*transform_1d)(const tran_low_t*, tran_low_t*);