Commit da3d94fe authored by Alex Converse's avatar Alex Converse

Account for elided extrabits during rate cost calculation.

Fixes some rd-debug mismatches coding cat6 tokens with tx size < 32x32.
For these tokens the high extrabits are elided during tokenization and
detokenization, but the rd cost was computed with the old tables from
VP9 where these high extrabits are always coded.

Change-Id: I4a9a6ea822ff821e1932c351d43a57bdb4d6d466
parent e063e2dd
......@@ -17,6 +17,7 @@
#include "aom_dsp/prob.h"
#include "av1/common/common.h"
#include "av1/common/common_data.h"
#include "av1/common/enums.h"
#ifdef __cplusplus
......@@ -111,6 +112,18 @@ typedef struct {
// indexed by token value
extern const av1_extra_bit av1_extra_bits[ENTROPY_TOKENS];
static INLINE int av1_get_cat6_extrabits_size(TX_SIZE tx_size,
aom_bit_depth_t bit_depth) {
tx_size = txsize_sqr_up_map[tx_size];
#if CONFIG_TX64X64
// TODO(debargha): Does TX_64X64 require an additional extrabit?
if (tx_size > TX_32X32) tx_size = TX_32X32;
#endif
int bits = (int)bit_depth + 3 + (int)tx_size;
assert(bits < (int)sizeof(av1_cat6_prob));
return bits;
}
#define DCT_MAX_VALUE 16384
#if CONFIG_AOM_HIGHBITDEPTH
#define DCT_MAX_VALUE_HIGH10 65536
......
......@@ -194,11 +194,11 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
break;
case CATEGORY6_TOKEN: {
#if CONFIG_AOM_HIGHBITDEPTH
const int skip_bits =
TX_SIZES - 1 - txsize_sqr_up_map[tx_size] + (12 - xd->bd);
const int skip_bits = (int)sizeof(av1_cat6_prob) -
av1_get_cat6_extrabits_size(tx_size, xd->bd);
#else
const int skip_bits =
TX_SIZES - 1 - txsize_sqr_up_map[tx_size] + (12 - 8);
const int skip_bits = (int)sizeof(av1_cat6_prob) -
av1_get_cat6_extrabits_size(tx_size, 8);
#endif
val = CAT6_MIN_VAL +
read_coeff(av1_cat6_prob + skip_bits, 18 - skip_bits, r);
......@@ -292,11 +292,11 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
break;
case CATEGORY6_TOKEN: {
#if CONFIG_AOM_HIGHBITDEPTH
const int skip_bits =
TX_SIZES - 1 - txsize_sqr_up_map[tx_size] + (12 - xd->bd);
const int skip_bits = (int)sizeof(av1_cat6_prob) -
av1_get_cat6_extrabits_size(tx_size, xd->bd);
#else
const int skip_bits =
TX_SIZES - 1 - txsize_sqr_up_map[tx_size] + (12 - 8);
const int skip_bits = (int)sizeof(av1_cat6_prob) -
av1_get_cat6_extrabits_size(tx_size, 8);
#endif
val = CAT6_MIN_VAL +
read_coeff(av1_cat6_prob + skip_bits, 18 - skip_bits, r);
......@@ -332,11 +332,11 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
break;
case CATEGORY6_TOKEN: {
#if CONFIG_AOM_HIGHBITDEPTH
const int skip_bits =
TX_SIZES - 1 - txsize_sqr_up_map[tx_size] + (12 - xd->bd);
const int skip_bits = (int)sizeof(av1_cat6_prob) -
av1_get_cat6_extrabits_size(tx_size, xd->bd);
#else
const int skip_bits =
TX_SIZES - 1 - txsize_sqr_up_map[tx_size] + (12 - 8);
const int skip_bits = (int)sizeof(av1_cat6_prob) -
av1_get_cat6_extrabits_size(tx_size, 8);
#endif
val = CAT6_MIN_VAL +
read_coeff(av1_cat6_prob + skip_bits, 18 - skip_bits, r);
......
......@@ -846,10 +846,10 @@ static void pack_mb_tokens(aom_writer *w, const TOKENEXTRA **tp,
const int bit_string_length = extra_bits->len; // Length of extra bits to
// be written excluding
// the sign bit.
int skip_bits =
(extra_bits->base_val == CAT6_MIN_VAL)
? TX_SIZES - 1 - txsize_sqr_up_map[tx_size] + (12 - bit_depth)
: 0;
int skip_bits = (extra_bits->base_val == CAT6_MIN_VAL)
? (int)sizeof(av1_cat6_prob) -
av1_get_cat6_extrabits_size(tx_size, bit_depth)
: 0;
if (bit_string_length > 0) {
const unsigned char *pb = extra_bits->prob;
......@@ -943,10 +943,10 @@ static void pack_mb_tokens(aom_writer *w, const TOKENEXTRA **tp,
const int bit_string_length = extra_bits->len; // Length of extra bits to
// be written excluding
// the sign bit.
int skip_bits =
(extra_bits->base_val == CAT6_MIN_VAL)
? TX_SIZES - 1 - txsize_sqr_up_map[tx_size] + (12 - bit_depth)
: 0;
int skip_bits = (extra_bits->base_val == CAT6_MIN_VAL)
? (int)sizeof(av1_cat6_prob) -
av1_get_cat6_extrabits_size(tx_size, bit_depth)
: 0;
if (bit_string_length > 0) {
const unsigned char *pb = extra_bits->prob;
const int value = bit_string >> 1;
......
......@@ -153,9 +153,9 @@ int av1_optimize_b(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, int block,
: band_translate[eob - 1];
int pt, i, final_eob;
#if CONFIG_AOM_HIGHBITDEPTH
const int *cat6_high_cost = av1_get_high_cost_table(xd->bd);
const int cat6_bits = av1_get_cat6_extrabits_size(tx_size, xd->bd);
#else
const int *cat6_high_cost = av1_get_high_cost_table(8);
const int cat6_bits = av1_get_cat6_extrabits_size(tx_size, 8);
#endif
unsigned int(*token_costs)[2][COEFF_CONTEXTS][ENTROPY_TOKENS] =
mb->token_costs[txsize_sqr_map[tx_size]][plane_type][ref];
......@@ -182,7 +182,7 @@ int av1_optimize_b(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, int block,
for (i = 0; i < eob; i++) {
const int rc = scan[i];
tokens[i][0].rate = av1_get_token_cost(qcoeff[rc], &t0, cat6_high_cost);
tokens[i][0].rate = av1_get_token_cost(qcoeff[rc], &t0, cat6_bits);
tokens[i][0].token = t0;
token_cache[rc] = av1_pt_energy_class[t0];
}
......@@ -302,7 +302,7 @@ int av1_optimize_b(const AV1_COMMON *cm, MACROBLOCK *mb, int plane, int block,
t1 = tokens[next][1].token == EOB_TOKEN ? EOB_TOKEN : ZERO_TOKEN;
base_bits = 0;
} else {
base_bits = av1_get_token_cost(x, &t0, cat6_high_cost);
base_bits = av1_get_token_cost(x, &t0, cat6_bits);
t1 = t0;
}
......
......@@ -1120,9 +1120,9 @@ int av1_cost_coeffs(const AV1_COMMON *const cm, MACROBLOCK *x, int plane,
#endif // CONFIG_NEW_TOKENSET
#if CONFIG_AOM_HIGHBITDEPTH
const int *cat6_high_cost = av1_get_high_cost_table(xd->bd);
const int cat6_bits = av1_get_cat6_extrabits_size(tx_size, xd->bd);
#else
const int *cat6_high_cost = av1_get_high_cost_table(8);
const int cat6_bits = av1_get_cat6_extrabits_size(tx_size, 8);
#endif // CONFIG_AOM_HIGHBITDEPTH
#if !CONFIG_VAR_TX && !CONFIG_SUPERTX
......@@ -1145,7 +1145,7 @@ int av1_cost_coeffs(const AV1_COMMON *const cm, MACROBLOCK *x, int plane,
// dc token
int v = qcoeff[0];
int16_t prev_t;
cost = av1_get_token_cost(v, &prev_t, cat6_high_cost);
cost = av1_get_token_cost(v, &prev_t, cat6_bits);
#if CONFIG_NEW_TOKENSET
cost += (*token_costs)[!prev_t][pt][prev_t];
#else
......@@ -1161,7 +1161,7 @@ int av1_cost_coeffs(const AV1_COMMON *const cm, MACROBLOCK *x, int plane,
int16_t t;
v = qcoeff[rc];
cost += av1_get_token_cost(v, &t, cat6_high_cost);
cost += av1_get_token_cost(v, &t, cat6_bits);
#if CONFIG_NEW_TOKENSET
cost += (*token_costs)[!t][!prev_t][t];
#else
......@@ -1187,7 +1187,7 @@ int av1_cost_coeffs(const AV1_COMMON *const cm, MACROBLOCK *x, int plane,
#if !CONFIG_NEW_TOKENSET
unsigned int(*tok_cost_ptr)[COEFF_CONTEXTS][ENTROPY_TOKENS];
#endif
cost = av1_get_token_cost(v, &tok, cat6_high_cost);
cost = av1_get_token_cost(v, &tok, cat6_bits);
#if CONFIG_NEW_TOKENSET
cost += (*token_costs)[!tok][pt][tok];
#else
......@@ -1206,7 +1206,7 @@ int av1_cost_coeffs(const AV1_COMMON *const cm, MACROBLOCK *x, int plane,
const int rc = scan[c];
v = qcoeff[rc];
cost += av1_get_token_cost(v, &tok, cat6_high_cost);
cost += av1_get_token_cost(v, &tok, cat6_bits);
pt = get_coef_context(nb, token_cache, c);
#if CONFIG_NEW_TOKENSET
cost += (*token_costs)[!tok][pt][tok];
......
This diff is collapsed.
......@@ -98,29 +98,13 @@ extern const TOKENVALUE *av1_dct_value_tokens_ptr;
extern const TOKENVALUE *av1_dct_cat_lt_10_value_tokens;
extern const int *av1_dct_cat_lt_10_value_cost;
extern const int16_t av1_cat6_low_cost[256];
extern const int av1_cat6_high_cost[64];
extern const int av1_cat6_high10_high_cost[256];
extern const int av1_cat6_high12_high_cost[1024];
static INLINE int av1_get_cost(int16_t token, EXTRABIT extrabits,
const int *cat6_high_table) {
if (token != CATEGORY6_TOKEN)
return av1_extra_bits[token].cost[extrabits >> 1];
return av1_cat6_low_cost[(extrabits >> 1) & 0xff] +
cat6_high_table[extrabits >> 9];
}
#if CONFIG_AOM_HIGHBITDEPTH
static INLINE const int *av1_get_high_cost_table(int bit_depth) {
return bit_depth == 8 ? av1_cat6_high_cost
: (bit_depth == 10 ? av1_cat6_high10_high_cost
: av1_cat6_high12_high_cost);
}
#define CAT6_HIGH_COST_ENTRIES 1024
#else
static INLINE const int *av1_get_high_cost_table(int bit_depth) {
(void)bit_depth;
return av1_cat6_high_cost;
}
#endif // CONFIG_AOM_HIGHBITDEPTH
#define CAT6_HIGH_COST_ENTRIES 64
#endif
extern const int av1_cat6_high_cost[CAT6_HIGH_COST_ENTRIES];
extern const uint8_t av1_cat6_skipped_bits_discount[8];
static INLINE void av1_get_token_extra(int v, int16_t *token, EXTRABIT *extra) {
if (v >= CAT6_MIN_VAL || v <= -CAT6_MIN_VAL) {
......@@ -139,14 +123,14 @@ static INLINE int16_t av1_get_token(int v) {
return av1_dct_cat_lt_10_value_tokens[v].token;
}
static INLINE int av1_get_token_cost(int v, int16_t *token,
const int *cat6_high_table) {
static INLINE int av1_get_token_cost(int v, int16_t *token, int cat6_bits) {
if (v >= CAT6_MIN_VAL || v <= -CAT6_MIN_VAL) {
EXTRABIT extrabits;
*token = CATEGORY6_TOKEN;
extrabits = abs(v) - CAT6_MIN_VAL;
return av1_cat6_low_cost[extrabits & 0xff] +
cat6_high_table[extrabits >> 8];
av1_cat6_high_cost[extrabits >> 8] -
av1_cat6_skipped_bits_discount[18 - cat6_bits];
}
*token = av1_dct_cat_lt_10_value_tokens[v].token;
return av1_dct_cat_lt_10_value_cost[v];
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment