Commit fc4980ed authored by Alex Converse's avatar Alex Converse Committed by Gerrit Code Review
Browse files

Merge changes Ic74d9d88,Ie93b474e,I544989ea,Ic273f7d9,Idfd2d2b3, ... into nextgenv2

* changes:
  Remove custom rans types
  Remove add_token_no_extra.
  Remove unused aom_rans_build_cdf_from_pdf
  Add the tool used to generate the constrained tokenset.
  Remove the starting zero from ANS CDFs.
  Import the aom_read/write_symbol abstractions from aom/master
parents cad8283e 9ed1a2ff
......@@ -15,16 +15,7 @@
#include "aom_dsp/ans.h"
#include "aom_dsp/prob.h"
void aom_rans_build_cdf_from_pdf(const AnsP10 token_probs[], rans_lut cdf_tab) {
int i;
cdf_tab[0] = 0;
for (i = 1; cdf_tab[i - 1] < RANS_PRECISION; ++i) {
cdf_tab[i] = cdf_tab[i - 1] + token_probs[i - 1];
}
assert(cdf_tab[i - 1] == RANS_PRECISION);
}
static int find_largest(const AnsP10 *const pdf_tab, int num_syms) {
static int find_largest(const aom_cdf_prob *const pdf_tab, int num_syms) {
int largest_idx = -1;
int largest_p = -1;
int i;
......@@ -38,8 +29,9 @@ static int find_largest(const AnsP10 *const pdf_tab, int num_syms) {
return largest_idx;
}
void aom_rans_merge_prob8_pdf(AnsP10 *const out_pdf, const AnsP8 node_prob,
const AnsP10 *const src_pdf, int in_syms) {
void aom_rans_merge_prob8_pdf(aom_cdf_prob *const out_pdf,
const AnsP8 node_prob,
const aom_cdf_prob *const src_pdf, int in_syms) {
int i;
int adjustment = RANS_PRECISION;
const int round_fact = ANS_P8_PRECISION >> 1;
......
......@@ -26,24 +26,16 @@ extern "C" {
typedef uint8_t AnsP8;
#define ANS_P8_PRECISION 256u
#define ANS_P8_SHIFT 8
typedef uint16_t AnsP10;
#define ANS_P10_PRECISION 1024u
#define RANS_PRECISION 1024u
#define RANS_PROB_BITS 10
#define RANS_PRECISION ANS_P10_PRECISION
#define L_BASE (ANS_P10_PRECISION * 4) // L_BASE % precision must be 0
#define L_BASE (RANS_PRECISION * 4) // L_BASE % precision must be 0
#define IO_BASE 256
// Range I = { L_BASE, L_BASE + 1, ..., L_BASE * IO_BASE - 1 }
// This is now just a boring cdf. It starts with an explicit zero.
// TODO(aconverse): Remove starting zero.
typedef uint16_t rans_lut[16];
void aom_rans_build_cdf_from_pdf(const AnsP10 token_probs[], rans_lut cdf_tab);
void aom_rans_merge_prob8_pdf(AnsP10 *const out_pdf, const AnsP8 node_prob,
const AnsP10 *const src_pdf, int in_syms);
void aom_rans_merge_prob8_pdf(aom_cdf_prob *const out_pdf,
const AnsP8 node_prob,
const aom_cdf_prob *const src_pdf, int in_syms);
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
......
......@@ -62,24 +62,25 @@ static INLINE int uabs_read_bit(struct AnsDecoder *ans) {
struct rans_dec_sym {
uint8_t val;
AnsP10 prob;
AnsP10 cum_prob; // not-inclusive
aom_cdf_prob prob;
aom_cdf_prob cum_prob; // not-inclusive
};
static INLINE void fetch_sym(struct rans_dec_sym *out, const rans_lut cdf,
AnsP10 rem) {
int i = 0;
static INLINE void fetch_sym(struct rans_dec_sym *out, const aom_cdf_prob *cdf,
aom_cdf_prob rem) {
int i;
aom_cdf_prob cum_prob = 0, top_prob;
// TODO(skal): if critical, could be a binary search.
// Or, better, an O(1) alias-table.
while (rem >= cdf[i]) {
++i;
for (i = 0; rem >= (top_prob = cdf[i]); ++i) {
cum_prob = top_prob;
}
out->val = i - 1;
out->prob = (AnsP10)(cdf[i] - cdf[i - 1]);
out->cum_prob = (AnsP10)cdf[i - 1];
out->val = i;
out->prob = top_prob - cum_prob;
out->cum_prob = cum_prob;
}
static INLINE int rans_read(struct AnsDecoder *ans, const rans_lut tab) {
static INLINE int rans_read(struct AnsDecoder *ans, const aom_cdf_prob *tab) {
unsigned rem;
unsigned quo;
struct rans_dec_sym sym;
......
......@@ -75,8 +75,8 @@ static INLINE void uabs_write(struct AnsCoder *ans, int val, AnsP8 p0) {
}
struct rans_sym {
AnsP10 prob;
AnsP10 cum_prob; // not-inclusive
aom_cdf_prob prob;
aom_cdf_prob cum_prob; // not-inclusive
};
// rANS with normalization
......@@ -84,7 +84,7 @@ struct rans_sym {
// ANS_P10_PRECISION is m
static INLINE void rans_write(struct AnsCoder *ans,
const struct rans_sym *const sym) {
const AnsP10 p = sym->prob;
const aom_cdf_prob p = sym->prob;
while (ans->state >= L_BASE / RANS_PRECISION * IO_BASE * p) {
ans->buf[ans->buf_offset++] = ans->state % IO_BASE;
ans->state /= IO_BASE;
......
......@@ -104,6 +104,20 @@ static INLINE int aom_read_tree(aom_reader *r, const aom_tree_index *tree,
return aom_read_tree_bits(r, tree, probs);
}
static INLINE int aom_read_symbol(aom_reader *r, const aom_cdf_prob *cdf,
int nsymbs) {
#if CONFIG_ANS
(void)nsymbs;
return rans_read(r, cdf);
#else
(void)r;
(void)cdf;
(void)nsymbs;
assert(0 && "Unsupported bitreader operation");
return -1;
#endif
}
#ifdef __cplusplus
} // extern "C"
#endif
......
......@@ -86,6 +86,24 @@ static INLINE void aom_write_tree(aom_writer *w, const aom_tree_index *tree,
aom_write_tree_bits(w, tree, probs, bits, len, i);
}
static INLINE void aom_write_symbol(aom_writer *w, int symb,
const aom_cdf_prob *cdf, int nsymbs) {
#if CONFIG_ANS
struct rans_sym s;
(void)nsymbs;
assert(cdf);
s.cum_prob = symb > 0 ? cdf[symb - 1] : 0;
s.prob = cdf[symb] - s.cum_prob;
buf_rans_write(w, &s);
#else
(void)w;
(void)symb;
(void)cdf;
(void)nsymbs;
assert(0 && "Unsupported bitwriter operation");
#endif
}
#ifdef __cplusplus
} // extern "C"
#endif
......
......@@ -23,6 +23,9 @@ extern "C" {
typedef uint8_t aom_prob;
// TODO(negge): Rename this aom_prob once we remove vpxbool.
typedef uint16_t aom_cdf_prob;
#define MAX_PROB 255
#define aom_prob_half ((aom_prob)128)
......
This diff is collapsed.
......@@ -191,10 +191,10 @@ static INLINE const uint8_t *get_band_translate(TX_SIZE tx_size) {
extern const aom_tree_index av1_coef_con_tree[TREE_SIZE(ENTROPY_TOKENS)];
extern const aom_prob av1_pareto8_full[COEFF_PROB_MODELS][MODEL_NODES];
#if CONFIG_ANS
extern const AnsP10 av1_pareto8_token_probs[COEFF_PROB_MODELS]
[ENTROPY_TOKENS - 2];
typedef rans_lut coeff_cdf_model[REF_TYPES][COEF_BANDS][COEFF_CONTEXTS];
typedef aom_cdf_prob coeff_cdf_model[REF_TYPES][COEF_BANDS][COEFF_CONTEXTS]
[ENTROPY_TOKENS];
extern const aom_cdf_prob av1_pareto8_token_probs[COEFF_PROB_MODELS]
[ENTROPY_TOKENS - 2];
#endif // CONFIG_ANS
typedef aom_prob av1_coeff_probs_model[REF_TYPES][COEF_BANDS][COEFF_CONTEXTS]
......
......@@ -75,9 +75,9 @@ static int decode_coefs(const MACROBLOCKD *xd, PLANE_TYPE type,
fc->coef_probs[tx_size_ctx][type][ref];
const aom_prob *prob;
#if CONFIG_ANS
const rans_lut(*coef_cdfs)[COEFF_CONTEXTS] =
const aom_cdf_prob(*const coef_cdfs)[COEFF_CONTEXTS][ENTROPY_TOKENS] =
fc->coef_cdfs[tx_size_ctx][type][ref];
const rans_lut *cdf;
const aom_cdf_prob(*cdf)[ENTROPY_TOKENS];
#endif // CONFIG_ANS
unsigned int(*coef_counts)[COEFF_CONTEXTS][UNCONSTRAINED_NODES + 1];
unsigned int(*eob_branch_count)[COEFF_CONTEXTS];
......@@ -166,7 +166,8 @@ static int decode_coefs(const MACROBLOCKD *xd, PLANE_TYPE type,
}
#if CONFIG_ANS
cdf = &coef_cdfs[band][ctx];
token = ONE_TOKEN + rans_read(r, *cdf);
token =
ONE_TOKEN + aom_read_symbol(r, *cdf, CATEGORY6_TOKEN - ONE_TOKEN + 1);
INCREMENT_COUNT(ONE_TOKEN + (token > ONE_TOKEN));
switch (token) {
case ONE_TOKEN:
......
......@@ -657,11 +657,8 @@ static void pack_mb_tokens(aom_writer *w, const TOKENEXTRA **tp,
aom_write(w, t != ZERO_TOKEN, p->context_tree[1]);
if (t != ZERO_TOKEN) {
struct rans_sym s;
const rans_lut *token_cdf = p->token_cdf;
s.cum_prob = (*token_cdf)[t - ONE_TOKEN];
s.prob = (*token_cdf)[t - ONE_TOKEN + 1] - s.cum_prob;
buf_rans_write(w, &s);
aom_write_symbol(w, t - ONE_TOKEN, *p->token_cdf,
CATEGORY6_TOKEN - ONE_TOKEN + 1);
}
}
#else
......
......@@ -387,7 +387,7 @@ static void set_entropy_context_b(int plane, int block, int blk_row,
static INLINE void add_token(TOKENEXTRA **t, const aom_prob *context_tree,
#if CONFIG_ANS
const rans_lut *token_cdf,
const aom_cdf_prob (*token_cdf)[ENTROPY_TOKENS],
#endif // CONFIG_ANS
int32_t extra, uint8_t token,
uint8_t skip_eob_node, unsigned int *counts) {
......@@ -402,17 +402,6 @@ static INLINE void add_token(TOKENEXTRA **t, const aom_prob *context_tree,
++counts[token];
}
static INLINE void add_token_no_extra(TOKENEXTRA **t,
const aom_prob *context_tree,
uint8_t token, uint8_t skip_eob_node,
unsigned int *counts) {
(*t)->token = token;
(*t)->context_tree = context_tree;
(*t)->skip_eob_node = skip_eob_node;
(*t)++;
++counts[token];
}
static INLINE int get_tx_eob(const struct segmentation *seg, int segment_id,
TX_SIZE tx_size) {
const int eob_max = num_4x4_blocks_txsize_lookup[tx_size] << 4;
......@@ -498,8 +487,8 @@ static void tokenize_b(int plane, int block, int blk_row, int blk_col,
cpi->common.fc->coef_probs[txsize_sqr_map[tx_size]][type][ref];
#endif // CONFIG_ENTROPY
#if CONFIG_ANS
rans_lut(*const coef_cdfs)[COEFF_CONTEXTS] =
cpi->common.fc->coef_cdfs[txsize_sqr_map[tx_size]][type][ref];
aom_cdf_prob(*const coef_cdfs)[COEFF_CONTEXTS][ENTROPY_TOKENS] =
cpi->common.fc->coef_cdfs[tx_size][type][ref];
#endif // CONFIG_ANS
unsigned int(*const eob_branch)[COEFF_CONTEXTS] =
td->counts->eob_branch[txsize_sqr_map[tx_size]][type][ref];
......@@ -522,7 +511,7 @@ static void tokenize_b(int plane, int block, int blk_row, int blk_col,
add_token(&t, coef_probs[band[c]][pt],
#if CONFIG_ANS
(const rans_lut *)&coef_cdfs[band[c]][pt],
(const aom_cdf_prob(*)[ENTROPY_TOKENS]) & coef_cdfs[band[c]][pt],
#endif // CONFIG_ANS
extra, (uint8_t)token, (uint8_t)skip_eob, counts[band[c]][pt]);
......@@ -532,8 +521,11 @@ static void tokenize_b(int plane, int block, int blk_row, int blk_col,
skip_eob = (token == ZERO_TOKEN);
}
if (c < seg_eob) {
add_token_no_extra(&t, coef_probs[band[c]][pt], EOB_TOKEN, 0,
counts[band[c]][pt]);
add_token(&t, coef_probs[band[c]][pt],
#if CONFIG_ANS || CONFIG_DAALA_EC
NULL,
#endif
0, EOB_TOKEN, 0, counts[band[c]][pt]);
++eob_branch[band[c]][pt];
}
......
......@@ -37,7 +37,7 @@ typedef struct {
typedef struct {
const aom_prob *context_tree;
#if CONFIG_ANS
const rans_lut *token_cdf;
const aom_cdf_prob (*token_cdf)[ENTROPY_TOKENS];
#endif // CONFIG_ANS
EXTRABIT extra;
uint8_t token;
......
......@@ -79,13 +79,13 @@ bool check_uabs(const PvVec &pv_vec, uint8_t *buf) {
const rans_sym rans_sym_tab[] = {
{ 67, 0 }, { 99, 67 }, { 575, 166 }, { 283, 741 },
};
const int kRansSymbols =
static_cast<int>(sizeof(rans_sym_tab) / sizeof(rans_sym_tab[0]));
std::vector<int> ans_encode_build_vals(const rans_sym *tab, int iters) {
std::vector<int> p_to_sym;
int i = 0;
while (p_to_sym.size() < RANS_PRECISION) {
for (int i = 0; i < kRansSymbols; ++i) {
p_to_sym.insert(p_to_sym.end(), tab[i].prob, i);
++i;
}
assert(p_to_sym.size() == RANS_PRECISION);
std::vector<int> ret;
......@@ -97,10 +97,11 @@ std::vector<int> ans_encode_build_vals(const rans_sym *tab, int iters) {
return ret;
}
void rans_build_dec_tab(const struct rans_sym sym_tab[], rans_lut dec_tab) {
dec_tab[0] = 0;
for (int i = 1; dec_tab[i - 1] < RANS_PRECISION; ++i) {
dec_tab[i] = dec_tab[i - 1] + sym_tab[i - 1].prob;
void rans_build_dec_tab(const struct rans_sym sym_tab[],
aom_cdf_prob *dec_tab) {
unsigned int sum = 0;
for (int i = 0; sum < RANS_PRECISION; ++i) {
dec_tab[i] = sum += sym_tab[i].prob;
}
}
......@@ -108,7 +109,7 @@ bool check_rans(const std::vector<int> &sym_vec, const rans_sym *const tab,
uint8_t *buf) {
AnsCoder a;
ans_write_init(&a, buf);
rans_lut dec_tab;
aom_cdf_prob dec_tab[kRansSymbols];
rans_build_dec_tab(tab, dec_tab);
std::clock_t start = std::clock();
......
#!/usr/bin/python
##
## Copyright (c) 2016, Alliance for Open Media. All rights reserved
##
## This source code is subject to the terms of the BSD 2 Clause License and
## the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
## was not distributed with this source code in the LICENSE file, you can
## obtain it at www.aomedia.org/license/software. If the Alliance for Open
## Media Patent License 1.0 was not distributed with this source code in the
## PATENTS file, you can obtain it at www.aomedia.org/license/patent.
##
"""Generate the probability model for the constrained token set.
Model obtained from a 2-sided zero-centered distribution derived
from a Pareto distribution. The cdf of the distribution is:
cdf(x) = 0.5 + 0.5 * sgn(x) * [1 - {alpha/(alpha + |x|)} ^ beta]
For a given beta and a given probability of the 1-node, the alpha
is first solved, and then the {alpha, beta} pair is used to generate
the probabilities for the rest of the nodes.
"""
import heapq
import sys
import numpy as np
import scipy.optimize
import scipy.stats
def cdf_spareto(x, xm, beta):
p = 1 - (xm / (np.abs(x) + xm))**beta
p = 0.5 + 0.5 * np.sign(x) * p
return p
def get_spareto(p, beta):
cdf = cdf_spareto
def func(x):
return ((cdf(1.5, x, beta) - cdf(0.5, x, beta)) /
(1 - cdf(0.5, x, beta)) - p)**2
alpha = scipy.optimize.fminbound(func, 1e-12, 10000, xtol=1e-12)
parray = np.zeros(11)
parray[0] = 2 * (cdf(0.5, alpha, beta) - 0.5)
parray[1] = (2 * (cdf(1.5, alpha, beta) - cdf(0.5, alpha, beta)))
parray[2] = (2 * (cdf(2.5, alpha, beta) - cdf(1.5, alpha, beta)))
parray[3] = (2 * (cdf(3.5, alpha, beta) - cdf(2.5, alpha, beta)))
parray[4] = (2 * (cdf(4.5, alpha, beta) - cdf(3.5, alpha, beta)))
parray[5] = (2 * (cdf(6.5, alpha, beta) - cdf(4.5, alpha, beta)))
parray[6] = (2 * (cdf(10.5, alpha, beta) - cdf(6.5, alpha, beta)))
parray[7] = (2 * (cdf(18.5, alpha, beta) - cdf(10.5, alpha, beta)))
parray[8] = (2 * (cdf(34.5, alpha, beta) - cdf(18.5, alpha, beta)))
parray[9] = (2 * (cdf(66.5, alpha, beta) - cdf(34.5, alpha, beta)))
parray[10] = 2 * (1. - cdf(66.5, alpha, beta))
return parray
def quantize_probs(p, save_first_bin, bits):
"""Quantize probability precisely.
Quantize probabilities minimizing dH (Kullback-Leibler divergence)
approximated by: sum (p_i-q_i)^2/p_i.
References:
https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence
https://github.com/JarekDuda/AsymmetricNumeralSystemsToolkit
"""
num_sym = p.size
p = np.clip(p, 1e-16, 1)
L = 2**bits
pL = p * L
ip = 1. / p # inverse probability
q = np.clip(np.round(pL), 1, L + 1 - num_sym)
quant_err = (pL - q)**2 * ip
sgn = np.sign(L - q.sum()) # direction of correction
if sgn != 0: # correction is needed
v = [] # heap of adjustment results (adjustment err, index) of each symbol
for i in range(1 if save_first_bin else 0, num_sym):
q_adj = q[i] + sgn
if q_adj > 0 and q_adj < L:
adj_err = (pL[i] - q_adj)**2 * ip[i] - quant_err[i]
heapq.heappush(v, (adj_err, i))
while q.sum() != L:
# apply lowest error adjustment
(adj_err, i) = heapq.heappop(v)
quant_err[i] += adj_err
q[i] += sgn
# calculate the cost of adjusting this symbol again
q_adj = q[i] + sgn
if q_adj > 0 and q_adj < L:
adj_err = (pL[i] - q_adj)**2 * ip[i] - quant_err[i]
heapq.heappush(v, (adj_err, i))
return q
def get_quantized_spareto(p, beta, bits):
parray = get_spareto(p, beta)
parray = parray[1:] / (1 - parray[0])
qarray = quantize_probs(parray, True, bits)
return qarray.astype(np.int)
def main(bits=8):
beta = 8
for q in range(1, 256):
parray = get_quantized_spareto(q / 256., beta, bits)
assert parray.sum() == 2**bits
print '{', ', '.join('%d' % i for i in parray), '},'
if __name__ == '__main__':
if len(sys.argv) > 1:
main(int(sys.argv[1]))
else:
main()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment