filter_intra: make fi mode index entropy coded

Make fi mode index entropy coded instead of using raw bits. Mode
cost estimation in key-frame RDO is updated as well. Modification
to inter frame RDO is not included in this patch.
Also key-frame y mode cdf table is re-trained since fi modes are
attached to DC_PRED

Key frame BDRate:
-0.399% lowres, -0.339% midres

Change-Id: I9ccf478b0a2e48fb1870fe8451e45e2c858a5f63

av1/common/entropymode.h

@@ -345,6 +345,7 @@ typedef struct frame_contexts {
 #endif  // CONFIG_EXT_INTRA
 #if CONFIG_FILTER_INTRA
   aom_prob filter_intra_probs[PLANE_TYPES];
+  aom_cdf_prob filter_intra_mode_cdf[PLANE_TYPES][CDF_SIZE(FILTER_INTRA_MODES)];
 #endif  // CONFIG_FILTER_INTRA
 #if CONFIG_LOOP_RESTORATION
   aom_prob switchable_restore_prob[RESTORE_SWITCHABLE_TYPES - 1];
@@ -568,7 +569,7 @@ typedef struct FRAME_COUNTS {
 #endif  // CONFIG_EXT_INTRA
 #if CONFIG_FILTER_INTRA
   unsigned int filter_intra[PLANE_TYPES][2];
-  unsigned int filter_intra_mode[PLANE_TYPES][10];
+  unsigned int filter_intra_mode[PLANE_TYPES][FILTER_INTRA_MODES];
 #endif  // CONFIG_FILTER_INTRA
 #if CONFIG_LPF_SB
   unsigned int lpf_reuse[LPF_REUSE_CONTEXT][2];

av1/decoder/decodemv.c

@@ -890,7 +890,8 @@ static void read_filter_intra_mode_info(AV1_COMMON *const cm,
         aom_read(r, cm->fc->filter_intra_probs[0], ACCT_STR);
     if (filter_intra_mode_info->use_filter_intra_mode[0]) {
       filter_intra_mode_info->filter_intra_mode[0] =
-          av1_read_uniform(r, FILTER_INTRA_MODES);
+          aom_read_symbol(r, xd->tile_ctx->filter_intra_mode_cdf[0],
+                          FILTER_INTRA_MODES, ACCT_STR);
     }
     if (counts) {
       ++counts
@@ -914,7 +915,8 @@ static void read_filter_intra_mode_info(AV1_COMMON *const cm,
         aom_read(r, cm->fc->filter_intra_probs[1], ACCT_STR);
     if (filter_intra_mode_info->use_filter_intra_mode[1]) {
       filter_intra_mode_info->filter_intra_mode[1] =
-          av1_read_uniform(r, FILTER_INTRA_MODES);
+          aom_read_symbol(r, xd->tile_ctx->filter_intra_mode_cdf[1],
+                          FILTER_INTRA_MODES, ACCT_STR);
     }
     if (counts) {
       ++counts

av1/encoder/bitstream.c

@@ -1235,7 +1235,8 @@ static void write_filter_intra_mode_info(const AV1_COMMON *const cm,
     if (mbmi->filter_intra_mode_info.use_filter_intra_mode[0]) {
       const FILTER_INTRA_MODE mode =
           mbmi->filter_intra_mode_info.filter_intra_mode[0];
-      write_uniform(w, FILTER_INTRA_MODES, mode);
+      aom_write_symbol(w, mode, xd->tile_ctx->filter_intra_mode_cdf[0],
+                       FILTER_INTRA_MODES);
     }
   }
 
@@ -1257,7 +1258,8 @@ static void write_filter_intra_mode_info(const AV1_COMMON *const cm,
     if (mbmi->filter_intra_mode_info.use_filter_intra_mode[1]) {
       const FILTER_INTRA_MODE mode =
           mbmi->filter_intra_mode_info.filter_intra_mode[1];
-      write_uniform(w, FILTER_INTRA_MODES, mode);
+      aom_write_symbol(w, mode, xd->tile_ctx->filter_intra_mode_cdf[1],
+                       FILTER_INTRA_MODES);
     }
   }
 }

av1/encoder/block.h

@@ -257,6 +257,9 @@ struct macroblock {
 #endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
   int intra_uv_mode_cost[INTRA_MODES][UV_INTRA_MODES];
   int y_mode_costs[INTRA_MODES][INTRA_MODES][INTRA_MODES];
+#if CONFIG_FILTER_INTRA
+  int filter_intra_mode_cost[PLANE_TYPES][FILTER_INTRA_MODES];
+#endif
   int switchable_interp_costs[SWITCHABLE_FILTER_CONTEXTS][SWITCHABLE_FILTERS];
 #if CONFIG_EXT_PARTITION_TYPES
   int partition_cost[PARTITION_CONTEXTS + CONFIG_UNPOISON_PARTITION_CTX]

av1/encoder/encodeframe.c

@@ -5978,6 +5978,9 @@ static void sum_intra_stats(FRAME_COUNTS *counts, MACROBLOCKD *xd,
       ++counts->filter_intra[0][use_filter_intra_mode];
       ++counts->filter_intra_mode[0][mbmi->filter_intra_mode_info
                                          .filter_intra_mode[0]];
+      update_cdf(fc->filter_intra_mode_cdf[0],
+                 mbmi->filter_intra_mode_info.filter_intra_mode[0],
+                 FILTER_INTRA_MODES);
     }
     if (mbmi->uv_mode == UV_DC_PRED
 #if CONFIG_CB4X4
@@ -5991,6 +5994,9 @@ static void sum_intra_stats(FRAME_COUNTS *counts, MACROBLOCKD *xd,
       ++counts->filter_intra[1][use_filter_intra_mode];
       ++counts->filter_intra_mode[1][mbmi->filter_intra_mode_info
                                          .filter_intra_mode[1]];
+      update_cdf(fc->filter_intra_mode_cdf[1],
+                 mbmi->filter_intra_mode_info.filter_intra_mode[1],
+                 FILTER_INTRA_MODES);
     }
 #endif  // CONFIG_FILTER_INTRA
 #if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP

av1/encoder/rd.c

@@ -153,6 +153,13 @@ void av1_fill_mode_rates(AV1_COMMON *const cm, MACROBLOCK *x,
     av1_cost_tokens_from_cdf(x->intra_uv_mode_cost[i], fc->uv_mode_cdf[i],
                              NULL);
 
+#if CONFIG_FILTER_INTRA
+  for (i = 0; i < 2; ++i) {
+    av1_cost_tokens_from_cdf(x->filter_intra_mode_cost[i],
+                             fc->filter_intra_mode_cdf[i], NULL);
+  }
+#endif
+
   for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i)
     av1_cost_tokens_from_cdf(x->switchable_interp_costs[i],
                              fc->switchable_interp_cdf[i], NULL);

av1/encoder/rdopt.c

@@ -3881,7 +3881,7 @@ static int rd_pick_filter_intra_sby(const AV1_COMP *const cpi, MACROBLOCK *x,
     if (tokenonly_rd_stats.rate == INT_MAX) continue;
     this_rate = tokenonly_rd_stats.rate +
                 av1_cost_bit(cpi->common.fc->filter_intra_probs[0], 1) +
-                write_uniform_cost(FILTER_INTRA_MODES, mode) + mode_cost;
+                x->filter_intra_mode_cost[0][mode] + mode_cost;
     this_rd = RDCOST(x->rdmult, this_rate, tokenonly_rd_stats.dist);
 
     if (this_rd < *best_rd) {
@@ -5965,7 +5965,7 @@ static int rd_pick_filter_intra_sbuv(const AV1_COMP *const cpi, MACROBLOCK *x,
     this_rate = tokenonly_rd_stats.rate +
                 av1_cost_bit(cpi->common.fc->filter_intra_probs[1], 1) +
                 x->intra_uv_mode_cost[mbmi->mode][mbmi->uv_mode] +
-                write_uniform_cost(FILTER_INTRA_MODES, mode);
+                x->filter_intra_mode_cost[1][mode];
     this_rd = RDCOST(x->rdmult, this_rate, tokenonly_rd_stats.dist);
     if (this_rd < *best_rd) {
       *best_rd = this_rd;
@@ -10306,7 +10306,7 @@ static void pick_filter_intra_interframe(
                                 MAX_ANGLE_DELTA + mbmi->angle_delta[1]);
   }
 #endif  // CONFIG_EXT_INTRA
-  if (mbmi->mode == DC_PRED) {
+  if (mbmi->uv_mode == DC_PRED) {
     rate2 +=
         av1_cost_bit(cpi->common.fc->filter_intra_probs[1],
                      mbmi->filter_intra_mode_info.use_filter_intra_mode[1]);

tools/aom_entropy_optimizer.c

@@ -779,10 +779,11 @@ int main(int argc, const char **argv) {
       &fc.filter_intra[0][0], probsfile, 2, cts_each_dim,
       "static const aom_cdf_prob default_filter_intra_cdf[2][CDF_SIZE(2)]");
   cts_each_dim[0] = PLANE_TYPES;
-  cts_each_dim[1] = 10;
-  optimize_cdf_table(&fc.filter_intra_mode[0][0], probsfile, 2, cts_each_dim,
-                     "static const aom_cdf_prob "
-                     "default_filter_intra_mode_cdf[2][CDF_SIZE(10)]");
+  cts_each_dim[1] = FILTER_INTRA_MODES;
+  optimize_cdf_table(
+      &fc.filter_intra_mode[0][0], probsfile, 2, cts_each_dim,
+      "static const aom_cdf_prob "
+      "default_filter_intra_mode_cdf[2][CDF_SIZE(FILTER_INTRA_MODES)]");
 #endif
 
 #if CONFIG_LV_MAP