Commit 62a94a64 authored by Alex Converse's avatar Alex Converse

Switch rANS to 15 bit precision, and adjust L_BASE.

This causes rANS to operate at the same precision as the Daala EC.

aom/master stats: rans10uabs8lbase12 → rans15uabs8lbase15

objective-1-fast
PSNR YCbCr:      0.01%      0.01%      0.01%
   PSNRHVS:      0.01%
      SSIM:      0.01%
    MSSSIM:      0.01%
 CIEDE2000:      0.01%

subset1
PSNR YCbCr:     -0.01%     -0.00%     -0.00%
   PSNRHVS:     -0.01%
      SSIM:     -0.01%
    MSSSIM:     -0.01%
 CIEDE2000:     -0.01%

(cherry picked from aom/master commit ddbc2e2a)

Change-Id: I6ef0a4f6198784b3712a61af9f105d560a22eaea
parent fbabcad6
......@@ -26,10 +26,12 @@ extern "C" {
typedef uint8_t AnsP8;
#define ANS_P8_PRECISION 256u
#define ANS_P8_SHIFT 8
#define RANS_PRECISION 1024u
#define RANS_PROB_BITS 10
#define RANS_PROB_BITS 15
#define RANS_PRECISION (1u << RANS_PROB_BITS)
#define L_BASE (RANS_PRECISION * 4) // L_BASE % precision must be 0
// L_BASE % PRECISION must be 0. Increasing L_BASE beyond 2**15 will cause uabs
// to overflow.
#define L_BASE (RANS_PRECISION)
#define IO_BASE 256
// Range I = { L_BASE, L_BASE + 1, ..., L_BASE * IO_BASE - 1 }
......
......@@ -111,8 +111,12 @@ static INLINE int ans_read_init(struct AnsDecoder *const ans,
if (offset < 3) return 1;
ans->buf_offset = offset - 3;
ans->state = mem_get_le24(buf + offset - 3) & 0x3FFFFF;
} else if ((buf[offset - 1] & 0xE0) == 0xE0) {
if (offset < 4) return 1;
ans->buf_offset = offset - 4;
ans->state = mem_get_le32(buf + offset - 4) & 0x1FFFFFFF;
} else {
// x == 3 implies this byte is a superframe marker
// 110xxxxx implies this byte is a superframe marker
return 1;
}
ans->state += L_BASE;
......
......@@ -54,6 +54,9 @@ static INLINE int ans_write_end(struct AnsCoder *const ans) {
} else if (state < (1 << 22)) {
mem_put_le24(ans->buf + ans->buf_offset, (0x02 << 22) + state);
return ans->buf_offset + 3;
} else if (state < (1 << 29)) {
mem_put_le32(ans->buf + ans->buf_offset, (0x07 << 29) + state);
return ans->buf_offset + 4;
} else {
assert(0 && "State is too large to be serialized");
return ans->buf_offset;
......
This diff is collapsed.
......@@ -74,7 +74,8 @@ bool check_uabs(const PvVec &pv_vec, uint8_t *buf) {
return ans_read_end(&d);
}
const aom_cdf_prob spareto65[] = { 260, 188, 138, 102, 133, 122, 64, 15, 1, 1 };
const aom_cdf_prob spareto65[] = { 8320, 6018, 4402, 3254, 4259,
3919, 2057, 492, 45, 2 };
const int kRansSymbols =
static_cast<int>(sizeof(spareto65) / sizeof(spareto65[0]));
......@@ -94,7 +95,8 @@ std::vector<int> ans_encode_build_vals(rans_sym *const tab, int iters) {
std::vector<int> ret;
libaom_test::ACMRandom gen(18543637);
for (int i = 0; i < iters; ++i) {
int sym = p_to_sym[gen.Rand8() * 4];
int sym =
p_to_sym[((gen.Rand8() << 8) + gen.Rand8()) & (RANS_PRECISION - 1)];
ret.push_back(sym);
}
return ret;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment