Commit 62a94a64 authored by Alex Converse's avatar Alex Converse
Browse files

Switch rANS to 15 bit precision, and adjust L_BASE.

This causes rANS to operate at the same precision as the Daala EC.

aom/master stats: rans10uabs8lbase12 → rans15uabs8lbase15

objective-1-fast
PSNR YCbCr:      0.01%      0.01%      0.01%
   PSNRHVS:      0.01%
      SSIM:      0.01%
    MSSSIM:      0.01%
 CIEDE2000:      0.01%

subset1
PSNR YCbCr:     -0.01%     -0.00%     -0.00%
   PSNRHVS:     -0.01%
      SSIM:     -0.01%
    MSSSIM:     -0.01%
 CIEDE2000:     -0.01%

(cherry picked from aom/master commit ddbc2e2a)

Change-Id: I6ef0a4f6198784b3712a61af9f105d560a22eaea
parent fbabcad6
...@@ -26,10 +26,12 @@ extern "C" { ...@@ -26,10 +26,12 @@ extern "C" {
typedef uint8_t AnsP8; typedef uint8_t AnsP8;
#define ANS_P8_PRECISION 256u #define ANS_P8_PRECISION 256u
#define ANS_P8_SHIFT 8 #define ANS_P8_SHIFT 8
#define RANS_PRECISION 1024u #define RANS_PROB_BITS 15
#define RANS_PROB_BITS 10 #define RANS_PRECISION (1u << RANS_PROB_BITS)
#define L_BASE (RANS_PRECISION * 4) // L_BASE % precision must be 0 // L_BASE % PRECISION must be 0. Increasing L_BASE beyond 2**15 will cause uabs
// to overflow.
#define L_BASE (RANS_PRECISION)
#define IO_BASE 256 #define IO_BASE 256
// Range I = { L_BASE, L_BASE + 1, ..., L_BASE * IO_BASE - 1 } // Range I = { L_BASE, L_BASE + 1, ..., L_BASE * IO_BASE - 1 }
......
...@@ -111,8 +111,12 @@ static INLINE int ans_read_init(struct AnsDecoder *const ans, ...@@ -111,8 +111,12 @@ static INLINE int ans_read_init(struct AnsDecoder *const ans,
if (offset < 3) return 1; if (offset < 3) return 1;
ans->buf_offset = offset - 3; ans->buf_offset = offset - 3;
ans->state = mem_get_le24(buf + offset - 3) & 0x3FFFFF; ans->state = mem_get_le24(buf + offset - 3) & 0x3FFFFF;
} else if ((buf[offset - 1] & 0xE0) == 0xE0) {
if (offset < 4) return 1;
ans->buf_offset = offset - 4;
ans->state = mem_get_le32(buf + offset - 4) & 0x1FFFFFFF;
} else { } else {
// x == 3 implies this byte is a superframe marker // 110xxxxx implies this byte is a superframe marker
return 1; return 1;
} }
ans->state += L_BASE; ans->state += L_BASE;
......
...@@ -54,6 +54,9 @@ static INLINE int ans_write_end(struct AnsCoder *const ans) { ...@@ -54,6 +54,9 @@ static INLINE int ans_write_end(struct AnsCoder *const ans) {
} else if (state < (1 << 22)) { } else if (state < (1 << 22)) {
mem_put_le24(ans->buf + ans->buf_offset, (0x02 << 22) + state); mem_put_le24(ans->buf + ans->buf_offset, (0x02 << 22) + state);
return ans->buf_offset + 3; return ans->buf_offset + 3;
} else if (state < (1 << 29)) {
mem_put_le32(ans->buf + ans->buf_offset, (0x07 << 29) + state);
return ans->buf_offset + 4;
} else { } else {
assert(0 && "State is too large to be serialized"); assert(0 && "State is too large to be serialized");
return ans->buf_offset; return ans->buf_offset;
......
This diff is collapsed.
...@@ -74,7 +74,8 @@ bool check_uabs(const PvVec &pv_vec, uint8_t *buf) { ...@@ -74,7 +74,8 @@ bool check_uabs(const PvVec &pv_vec, uint8_t *buf) {
return ans_read_end(&d); return ans_read_end(&d);
} }
const aom_cdf_prob spareto65[] = { 260, 188, 138, 102, 133, 122, 64, 15, 1, 1 }; const aom_cdf_prob spareto65[] = { 8320, 6018, 4402, 3254, 4259,
3919, 2057, 492, 45, 2 };
const int kRansSymbols = const int kRansSymbols =
static_cast<int>(sizeof(spareto65) / sizeof(spareto65[0])); static_cast<int>(sizeof(spareto65) / sizeof(spareto65[0]));
...@@ -94,7 +95,8 @@ std::vector<int> ans_encode_build_vals(rans_sym *const tab, int iters) { ...@@ -94,7 +95,8 @@ std::vector<int> ans_encode_build_vals(rans_sym *const tab, int iters) {
std::vector<int> ret; std::vector<int> ret;
libaom_test::ACMRandom gen(18543637); libaom_test::ACMRandom gen(18543637);
for (int i = 0; i < iters; ++i) { for (int i = 0; i < iters; ++i) {
int sym = p_to_sym[gen.Rand8() * 4]; int sym =
p_to_sym[((gen.Rand8() << 8) + gen.Rand8()) & (RANS_PRECISION - 1)];
ret.push_back(sym); ret.push_back(sym);
} }
return ret; return ret;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment