Merge "Switch rANS to 15 bit precision, and adjust L_BASE." into nextgenv2

aom_dsp/ans.h

@@ -26,10 +26,12 @@ extern "C" {
 typedef uint8_t AnsP8;
 #define ANS_P8_PRECISION 256u
 #define ANS_P8_SHIFT 8
-#define RANS_PRECISION 1024u
-#define RANS_PROB_BITS 10
+#define RANS_PROB_BITS 15
+#define RANS_PRECISION (1u << RANS_PROB_BITS)
 
-#define L_BASE (RANS_PRECISION * 4)  // L_BASE % precision must be 0
+// L_BASE % PRECISION must be 0. Increasing L_BASE beyond 2**15 will cause uabs
+// to overflow.
+#define L_BASE (RANS_PRECISION)
 #define IO_BASE 256
 // Range I = { L_BASE, L_BASE + 1, ..., L_BASE * IO_BASE - 1 }
 

aom_dsp/ansreader.h

@@ -111,8 +111,12 @@ static INLINE int ans_read_init(struct AnsDecoder *const ans,
     if (offset < 3) return 1;
     ans->buf_offset = offset - 3;
     ans->state = mem_get_le24(buf + offset - 3) & 0x3FFFFF;
+  } else if ((buf[offset - 1] & 0xE0) == 0xE0) {
+    if (offset < 4) return 1;
+    ans->buf_offset = offset - 4;
+    ans->state = mem_get_le32(buf + offset - 4) & 0x1FFFFFFF;
   } else {
-    // x == 3 implies this byte is a superframe marker
+    // 110xxxxx implies this byte is a superframe marker
     return 1;
   }
   ans->state += L_BASE;

aom_dsp/answriter.h

@@ -54,6 +54,9 @@ static INLINE int ans_write_end(struct AnsCoder *const ans) {
   } else if (state < (1 << 22)) {
     mem_put_le24(ans->buf + ans->buf_offset, (0x02 << 22) + state);
     return ans->buf_offset + 3;
+  } else if (state < (1 << 29)) {
+    mem_put_le32(ans->buf + ans->buf_offset, (0x07 << 29) + state);
+    return ans->buf_offset + 4;
   } else {
     assert(0 && "State is too large to be serialized");
     return ans->buf_offset;

test/ans_test.cc

@@ -74,7 +74,8 @@ bool check_uabs(const PvVec &pv_vec, uint8_t *buf) {
   return ans_read_end(&d);
 }
 
-const aom_cdf_prob spareto65[] = { 260, 188, 138, 102, 133, 122, 64, 15, 1, 1 };
+const aom_cdf_prob spareto65[] = { 8320, 6018, 4402, 3254, 4259,
+                                   3919, 2057, 492,  45,   2 };
 
 const int kRansSymbols =
     static_cast<int>(sizeof(spareto65) / sizeof(spareto65[0]));
@@ -94,7 +95,8 @@ std::vector<int> ans_encode_build_vals(rans_sym *const tab, int iters) {
   std::vector<int> ret;
   libaom_test::ACMRandom gen(18543637);
   for (int i = 0; i < iters; ++i) {
-    int sym = p_to_sym[gen.Rand8() * 4];
+    int sym =
+        p_to_sym[((gen.Rand8() << 8) + gen.Rand8()) & (RANS_PRECISION - 1)];
     ret.push_back(sym);
   }
   return ret;