NEW_TOKENSET: Make zero block value uncodeable for AC coeffs.

Change-Id: Icd7528d1ec8e117205e5e35ad9a11cfa83b08253

av1/common/entropy.h

@@ -53,6 +53,8 @@ extern "C" {
 #define EARLY_EOB 1        // End of block before the last position
 #define LAST_EOB 2         // End of block in the last position (implicit)
 #define BLOCK_Z_TOKEN 255  // block zero
+#define HEAD_TOKENS 5
+#define TAIL_TOKENS 9
 #define ONE_TOKEN_EOB 1
 #define ONE_TOKEN_NEOB 2
 #define TWO_TOKEN_PLUS_EOB 3

av1/decoder/detokenize.c

@@ -170,6 +170,7 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
     int more_data;
     int comb_token;
     int last_pos = (c + 1 == max_eob);
+    int first_pos = (c == 0);
 
 #if CONFIG_NEW_QUANT
     dqv_val = &dq_val[band][0];
@@ -177,9 +178,11 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
 
     cdf_head = &coef_head_cdfs[band][ctx];
     cdf_tail = &coef_tail_cdfs[band][ctx];
-    comb_token = last_pos ? (aom_read_bit(r, ACCT_STR) + 1) * 2
-                          : aom_read_symbol(r, *cdf_head, 6, ACCT_STR);
-    if (c == 0) {
+    comb_token = last_pos ? 2 * aom_read_bit(r, ACCT_STR) + 2
+                          : aom_read_symbol(r, *cdf_head,
+                                            HEAD_TOKENS + first_pos, ACCT_STR) +
+                                !first_pos;
+    if (first_pos) {
 #if !CONFIG_EC_ADAPT
       if (counts) ++blockz_count[comb_token != 0];
 #endif
@@ -189,7 +192,7 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff,
     more_data = !token || ((comb_token & 1) == 1);
 
     if (token > ONE_TOKEN)
-      token += aom_read_symbol(r, *cdf_tail, CATEGORY6_TOKEN + 1 - 2, ACCT_STR);
+      token += aom_read_symbol(r, *cdf_tail, TAIL_TOKENS, ACCT_STR);
 #if !CONFIG_EC_ADAPT
     if (!last_pos)
       INCREMENT_COUNT(ZERO_TOKEN + (token > ZERO_TOKEN) + (token > ONE_TOKEN));

av1/encoder/bitstream.c

@@ -835,7 +835,7 @@ static void pack_mb_tokens(aom_writer *w, const TOKENEXTRA **tp,
     const av1_extra_bit *const extra_bits = &extra_bits_table[token];
 
     if (token == BLOCK_Z_TOKEN) {
-      aom_write_symbol(w, 0, *p->head_cdf, 6);
+      aom_write_symbol(w, 0, *p->head_cdf, HEAD_TOKENS + 1);
       p++;
       continue;
     }
@@ -843,12 +843,11 @@ static void pack_mb_tokens(aom_writer *w, const TOKENEXTRA **tp,
       // Just code a flag indicating whether the value is >1 or 1.
       aom_write_bit(w, token != ONE_TOKEN);
     } else {
-      int comb_symb = 2 * AOMMIN(token, TWO_TOKEN) - p->eob_val + 1;
-      aom_write_symbol(w, comb_symb, *p->head_cdf, 6);
+      int comb_symb = 2 * AOMMIN(token, TWO_TOKEN) - p->eob_val + p->first_val;
+      aom_write_symbol(w, comb_symb, *p->head_cdf, HEAD_TOKENS + p->first_val);
     }
     if (token > ONE_TOKEN) {
-      aom_write_symbol(w, token - TWO_TOKEN, *p->tail_cdf,
-                       CATEGORY6_TOKEN + 1 - 2);
+      aom_write_symbol(w, token - TWO_TOKEN, *p->tail_cdf, TAIL_TOKENS);
     }
 
     if (extra_bits->base_val) {

av1/encoder/tokenize.c

@@ -364,12 +364,14 @@ static void set_entropy_context_b(int plane, int block, int blk_row,
 static INLINE void add_token(TOKENEXTRA **t,
                              aom_cdf_prob (*tail_cdf)[CDF_SIZE(ENTROPY_TOKENS)],
                              aom_cdf_prob (*head_cdf)[CDF_SIZE(ENTROPY_TOKENS)],
-                             int eob_val, int32_t extra, uint8_t token) {
+                             int eob_val, int first_val, int32_t extra,
+                             uint8_t token) {
   (*t)->token = token;
   (*t)->extra = extra;
   (*t)->tail_cdf = tail_cdf;
   (*t)->head_cdf = head_cdf;
   (*t)->eob_val = eob_val;
+  (*t)->first_val = first_val;
   (*t)++;
 }
 
@@ -547,6 +549,7 @@ static void tokenize_b(int plane, int block, int blk_row, int blk_col,
   unsigned int(*const blockz_count)[2] =
       td->counts->blockz_count[txsize_sqr_map[tx_size]][type][ref];
   int eob_val;
+  int first_val = 1;
 #else
 #if CONFIG_EC_MULTISYMBOL
   aom_cdf_prob(*const coef_cdfs)[COEFF_CONTEXTS][CDF_SIZE(ENTROPY_TOKENS)] =
@@ -569,17 +572,18 @@ static void tokenize_b(int plane, int block, int blk_row, int blk_col,
 
 #if CONFIG_NEW_TOKENSET
   if (eob == 0)
-    add_token(&t, &coef_tail_cdfs[band[c]][pt], &coef_head_cdfs[band[c]][pt], 0,
-              0, BLOCK_Z_TOKEN);
+    add_token(&t, &coef_tail_cdfs[band[c]][pt], &coef_head_cdfs[band[c]][pt], 1,
+              1, 0, BLOCK_Z_TOKEN);
 
   ++blockz_count[pt][eob != 0];
 
   while (c < eob) {
     int v = qcoeff[scan[c]];
+    first_val = (c == 0);
 
     if (!v) {
       add_token(&t, &coef_tail_cdfs[band[c]][pt], &coef_head_cdfs[band[c]][pt],
-                0, 0, ZERO_TOKEN);
+                0, first_val, 0, ZERO_TOKEN);
       ++counts[band[c]][pt][ZERO_TOKEN];
       token_cache[scan[c]] = 0;
     } else {
@@ -589,7 +593,7 @@ static void tokenize_b(int plane, int block, int blk_row, int blk_col,
       av1_get_token_extra(v, &token, &extra);
 
       add_token(&t, &coef_tail_cdfs[band[c]][pt], &coef_head_cdfs[band[c]][pt],
-                eob_val, extra, (uint8_t)token);
+                eob_val, first_val, extra, (uint8_t)token);
 
       if (eob_val != LAST_EOB) {
         ++counts[band[c]][pt][token];

av1/encoder/tokenize.h

@@ -39,6 +39,7 @@ typedef struct {
   aom_cdf_prob (*tail_cdf)[CDF_SIZE(ENTROPY_TOKENS)];
   aom_cdf_prob (*head_cdf)[CDF_SIZE(ENTROPY_TOKENS)];
   int eob_val;
+  int first_val;
 #elif CONFIG_EC_MULTISYMBOL
   aom_cdf_prob (*token_cdf)[CDF_SIZE(ENTROPY_TOKENS)];
 #endif