New filter_intra implementation + entropy coding

Use 4x2 processing unit.
Reduce # of modes from 6 to 5.

Change-Id: I3c12e18084636de0e279c9102a8b212342faf4c7

av1/common/blockd.h

@@ -195,7 +195,7 @@ typedef struct {
 } FILTER_INTRA_MODE_INFO;
 
 static const PREDICTION_MODE fimode_to_intradir[FILTER_INTRA_MODES] = {
-  DC_PRED, V_PRED, H_PRED, D117_PRED, D153_PRED, DC_PRED
+  DC_PRED, V_PRED, H_PRED, D153_PRED, DC_PRED
 };
 
 #define DISABLE_SUB8X8_FILTER_INTRA 0

av1/common/entropymode.c

@@ -1560,8 +1560,8 @@ static const aom_prob default_compound_idx_probs[COMP_INDEX_CONTEXTS] = {
 
 #if CONFIG_FILTER_INTRA
 static const aom_cdf_prob default_filter_intra_mode_cdf[2][CDF_SIZE(
-    FILTER_INTRA_MODES)] = { { AOM_CDF6(22207, 23158, 24144, 24278, 30434) },
-                             { AOM_CDF6(32768, 32768, 32768, 32768, 32768) } };
+    FILTER_INTRA_MODES)] = { { AOM_CDF5(14259, 17304, 20463, 29377) },
+                             { AOM_CDF5(32768, 32768, 32768, 32768) } };
 
 static const aom_cdf_prob default_filter_intra_cdfs[TX_SIZES_ALL][CDF_SIZE(2)] =
     { { AOM_CDF2(10985) }, { AOM_CDF2(10985) }, { AOM_CDF2(16645) },

av1/common/entropymode.h

@@ -428,6 +428,8 @@ typedef struct FRAME_COUNTS {
 #if CONFIG_FILTER_INTRA
   unsigned int filter_intra_mode[PLANE_TYPES][FILTER_INTRA_MODES];
   unsigned int filter_intra_tx[TX_SIZES_ALL][2];
+  unsigned int filter_intra_mode_ctx[KF_MODE_CONTEXTS][KF_MODE_CONTEXTS]
+                                    [FILTER_INTRA_MODES];
 #endif  // CONFIG_FILTER_INTRA
 #if CONFIG_LPF_SB
   unsigned int lpf_reuse[LPF_REUSE_CONTEXT][2];

av1/common/enums.h

@@ -567,7 +567,6 @@ typedef enum ATTRIBUTE_PACKED {
   FILTER_DC_PRED,
   FILTER_V_PRED,
   FILTER_H_PRED,
-  FILTER_D117_PRED,
   FILTER_D153_PRED,
   FILTER_PAETH_PRED,
   FILTER_INTRA_MODES,

av1/common/reconintra.h

@@ -42,8 +42,10 @@ static const INTERINTRA_MODE intra_to_interintra_mode[INTRA_MODES] = {
 };
 
 #if CONFIG_FILTER_INTRA
-#define FILTER_INTRA_PROC_UNIT_SIZE 2
-#if FILTER_INTRA_PROC_UNIT_SIZE == 4
+#define FILTER_INTRA_PROC_UNIT_SIZE 42  // ij means a i(cols)xj(rows) unit
+#if FILTER_INTRA_PROC_UNIT_SIZE == 44
+#define FILTER_INTRA_SCALE_BITS 5
+#elif FILTER_INTRA_PROC_UNIT_SIZE == 42
 #define FILTER_INTRA_SCALE_BITS 4
 #else
 #define FILTER_INTRA_SCALE_BITS 3

av1/encoder/encodeframe.c

@@ -4518,14 +4518,26 @@ static void sum_intra_stats(FRAME_COUNTS *counts, MACROBLOCKD *xd,
     const int use_filter_intra_mode =
         mbmi->filter_intra_mode_info.use_filter_intra_mode[0];
 #if CONFIG_ENTROPY_STATS
-    ++counts->filter_intra_mode[0][mbmi->filter_intra_mode_info
-                                       .filter_intra_mode[0]];
+    if (use_filter_intra_mode) {
+      const PREDICTION_MODE above = av1_above_block_mode(mi, above_mi, 0);
+      const PREDICTION_MODE left = av1_left_block_mode(mi, left_mi, 0);
+#if CONFIG_KF_CTX
+      int above_ctx = intra_mode_context[above];
+      int left_ctx = intra_mode_context[left];
+#endif
+      ++counts->filter_intra_mode[0][mbmi->filter_intra_mode_info
+                                         .filter_intra_mode[0]];
+      ++counts->filter_intra_mode_ctx[above_ctx][left_ctx]
+                                     [mbmi->filter_intra_mode_info
+                                          .filter_intra_mode[0]];
+    }
     ++counts->filter_intra_tx[mbmi->tx_size][use_filter_intra_mode];
 #endif  // CONFIG_ENTROPY_STATS
     if (allow_update_cdf) {
-      update_cdf(fc->filter_intra_mode_cdf[0],
-                 mbmi->filter_intra_mode_info.filter_intra_mode[0],
-                 FILTER_INTRA_MODES);
+      if (use_filter_intra_mode)
+        update_cdf(fc->filter_intra_mode_cdf[0],
+                   mbmi->filter_intra_mode_info.filter_intra_mode[0],
+                   FILTER_INTRA_MODES);
       update_cdf(fc->filter_intra_cdfs[mbmi->tx_size], use_filter_intra_mode,
                  2);
     }

av1/encoder/rdopt.c

@@ -3056,8 +3056,7 @@ static int rd_pick_filter_intra_sby(const AV1_COMP *const cpi, MACROBLOCK *x,
                                     int *rate, int *rate_tokenonly,
                                     int64_t *distortion, int *skippable,
                                     BLOCK_SIZE bsize, int mode_cost,
-                                    int64_t *best_rd, int64_t *best_model_rd,
-                                    uint16_t skip_mask) {
+                                    int64_t *best_rd, int64_t *best_model_rd) {
   MACROBLOCKD *const xd = &x->e_mbd;
   MODE_INFO *const mic = xd->mi[0];
   MB_MODE_INFO *mbmi = &mic->mbmi;
@@ -3076,7 +3075,6 @@ static int rd_pick_filter_intra_sby(const AV1_COMP *const cpi, MACROBLOCK *x,
     int this_rate;
     int64_t this_rd, this_model_rd;
     RD_STATS tokenonly_rd_stats;
-    if (skip_mask & (1 << fimode_to_intradir[mode])) continue;
     mbmi->filter_intra_mode_info.filter_intra_mode[0] = mode;
     this_model_rd = intra_model_yrd(cpi, x, bsize, mode_cost);
     if (*best_model_rd != INT64_MAX &&
@@ -3388,7 +3386,6 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
 #endif  // CONFIG_EXT_INTRA
 #if CONFIG_FILTER_INTRA
   int beat_best_rd = 0;
-  uint16_t filter_intra_mode_skip_mask = (1 << FILTER_INTRA_MODES) - 1;
 #endif  // CONFIG_FILTER_INTRA
   const int *bmode_costs;
   PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
@@ -3521,11 +3518,6 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
       this_rate += x->intrabc_cost[0];
 #endif  // CONFIG_INTRABC
     this_rd = RDCOST(x->rdmult, this_rate, this_distortion);
-#if CONFIG_FILTER_INTRA
-    if (best_rd == INT64_MAX || this_rd - best_rd < (best_rd >> 4)) {
-      filter_intra_mode_skip_mask ^= (1 << mbmi->mode);
-    }
-#endif  // CONFIG_FILTER_INTRA
 
     if (this_rd < best_rd) {
       best_mbmi = *mbmi;
@@ -3552,8 +3544,7 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
       !xd->lossless[mbmi->segment_id]) {
     if (rd_pick_filter_intra_sby(cpi, x, rate, rate_tokenonly, distortion,
                                  skippable, bsize, bmode_costs[DC_PRED],
-                                 &best_rd, &best_model_rd,
-                                 filter_intra_mode_skip_mask)) {
+                                 &best_rd, &best_model_rd)) {
       best_mbmi = *mbmi;
     }
   }

tools/aom_entropy_optimizer.c

@@ -662,6 +662,15 @@ int main(int argc, const char **argv) {
   optimize_cdf_table(&fc.filter_intra_tx[0][0], probsfile, 2, cts_each_dim,
                      "static const aom_cdf_prob "
                      "default_filter_intra_cdfs[TX_SIZES_ALL][CDF_SIZE(2)]");
+
+  cts_each_dim[0] = KF_MODE_CONTEXTS;
+  cts_each_dim[1] = KF_MODE_CONTEXTS;
+  cts_each_dim[2] = FILTER_INTRA_MODES;
+  optimize_cdf_table(&fc.filter_intra_mode_ctx[0][0][0], probsfile, 3,
+                     cts_each_dim,
+                     "static const aom_cdf_prob "
+                     "default_filter_intra_mode_cdf[KF_MODE_CONTEXTS][KF_MODE_"
+                     "CONTEXTS][CDF_SIZE(FILTER_INTRA_MODES)]");
 #endif
 
 #if CONFIG_LV_MAP