Make LOOPFILTER_LVL support segment

Experiment LOOPFILTER_LVL has 4 deblocking filter levels for:
y plane horizontal, y plane vertical, u plane, and v plane.

The patch supports segment updates for each of the four filter levels.

Change-Id: I494e8d9ab67e37c5f6234b8d9db8e3e3abf70d52

av1/common/av1_loopfilter.c

@@ -21,6 +21,14 @@
 #include "av1/common/reconinter.h"
 #include "av1/common/seg_common.h"
 
+#if CONFIG_LOOPFILTER_LEVEL
+static const SEG_LVL_FEATURES seg_lvl_lf_lut[MAX_MB_PLANE][2] = {
+  { SEG_LVL_ALT_LF_Y_V, SEG_LVL_ALT_LF_Y_H },
+  { SEG_LVL_ALT_LF_U, SEG_LVL_ALT_LF_U },
+  { SEG_LVL_ALT_LF_V, SEG_LVL_ALT_LF_V }
+};
+#endif  // CONFIG_LOOPFILTER_LEVEL
+
 #if CONFIG_LPF_DIRECT
 static void pick_filter_pixel_left(uint8_t *const src, uint8_t *const line,
                                    int *const orig_pos, int length, int row,
@@ -618,7 +626,7 @@ static void update_sharpness(loop_filter_info_n *lfi, int sharpness_lvl) {
 static uint8_t get_filter_level(const AV1_COMMON *cm,
                                 const loop_filter_info_n *lfi_n,
 #if CONFIG_LOOPFILTER_LEVEL
-                                const int dir_idx,
+                                const int dir_idx, int plane,
 #endif
 #if CONFIG_LPF_SB
                                 int mi_row, int mi_col,
@@ -647,12 +655,23 @@ static uint8_t get_filter_level(const AV1_COMMON *cm,
                         0, MAX_LOOP_FILTER);
 #endif
     const int scale = 1 << (lvl_seg >> 5);
+#if CONFIG_LOOPFILTER_LEVEL
+    assert(plane >= 0 && plane <= 2);
+    const int seg_lf_feature_id = seg_lvl_lf_lut[plane][dir_idx];
+    if (segfeature_active(&cm->seg, segment_id, seg_lf_feature_id)) {
+      const int data = get_segdata(&cm->seg, segment_id, seg_lf_feature_id);
+      lvl_seg =
+          clamp(cm->seg.abs_delta == SEGMENT_ABSDATA ? data : lvl_seg + data, 0,
+                MAX_LOOP_FILTER);
+    }
+#else
     if (segfeature_active(&cm->seg, segment_id, SEG_LVL_ALT_LF)) {
       const int data = get_segdata(&cm->seg, segment_id, SEG_LVL_ALT_LF);
       lvl_seg =
           clamp(cm->seg.abs_delta == SEGMENT_ABSDATA ? data : lvl_seg + data, 0,
                 MAX_LOOP_FILTER);
     }
+#endif  // CONFIG_LOOPFILTER_LEVEL
 
     if (cm->lf.mode_ref_delta_enabled) {
       lvl_seg += cm->lf.ref_deltas[mbmi->ref_frame[0]] * scale;
@@ -723,7 +742,12 @@ void av1_loop_filter_sb_level_init(AV1_COMMON *cm, int mi_row, int mi_col,
 #endif  // CONFIG_LPF_SB
 
 void av1_loop_filter_frame_init(AV1_COMMON *cm, int default_filt_lvl,
-                                int default_filt_lvl_r) {
+                                int default_filt_lvl_r
+#if CONFIG_LOOPFILTER_LEVEL
+                                ,
+                                int plane
+#endif
+                                ) {
   int seg_id;
   // n_shift is the multiplier for lf_deltas
   // the multiplier is 1 for when filter_lvl is between 0 and 31;
@@ -740,23 +764,37 @@ void av1_loop_filter_frame_init(AV1_COMMON *cm, int default_filt_lvl,
   }
 
   for (seg_id = 0; seg_id < MAX_SEGMENTS; seg_id++) {
-    int lvl_seg = default_filt_lvl;
-    if (segfeature_active(seg, seg_id, SEG_LVL_ALT_LF)) {
-      const int data = get_segdata(seg, seg_id, SEG_LVL_ALT_LF);
-      lvl_seg = clamp(
-          seg->abs_delta == SEGMENT_ABSDATA ? data : default_filt_lvl + data, 0,
-          MAX_LOOP_FILTER);
-    }
+    for (int dir = 0; dir < 2; ++dir) {
+      int lvl_seg = (dir == 0) ? default_filt_lvl : default_filt_lvl_r;
+#if CONFIG_LOOPFILTER_LEVEL
+      assert(plane >= 0 && plane <= 2);
+      const int seg_lf_feature_id = seg_lvl_lf_lut[plane][dir];
+      if (segfeature_active(seg, seg_id, seg_lf_feature_id)) {
+        const int data = get_segdata(&cm->seg, seg_id, seg_lf_feature_id);
+        lvl_seg = clamp(
+            seg->abs_delta == SEGMENT_ABSDATA ? data : default_filt_lvl + data,
+            0, MAX_LOOP_FILTER);
+      }
+#else
+      if (segfeature_active(seg, seg_id, SEG_LVL_ALT_LF)) {
+        const int data = get_segdata(seg, seg_id, SEG_LVL_ALT_LF);
+        lvl_seg = clamp(
+            seg->abs_delta == SEGMENT_ABSDATA ? data : default_filt_lvl + data,
+            0, MAX_LOOP_FILTER);
+      }
+#endif  // CONFIG_LOOPFILTER_LEVEL
 
-    if (!lf->mode_ref_delta_enabled) {
-      // we could get rid of this if we assume that deltas are set to
-      // zero when not in use; encoder always uses deltas
-      memset(lfi->lvl[seg_id], lvl_seg, sizeof(lfi->lvl[seg_id]));
-    } else {
-      int ref, mode;
+      if (!lf->mode_ref_delta_enabled) {
+// we could get rid of this if we assume that deltas are set to
+// zero when not in use; encoder always uses deltas
+#if CONFIG_LOOPFILTER_LEVEL
+        memset(lfi->lvl[seg_id][dir], lvl_seg, sizeof(lfi->lvl[seg_id][dir]));
+#else
+        memset(lfi->lvl[seg_id], lvl_seg, sizeof(lfi->lvl[seg_id]));
+#endif  // CONFIG_LOOPFILTER_LEVEL
+      } else {
+        int ref, mode;
 #if CONFIG_LOOPFILTER_LEVEL
-      for (int dir = 0; dir < 2; ++dir) {
-        lvl_seg = (dir == 0) ? default_filt_lvl : default_filt_lvl_r;
         scale = 1 << (lvl_seg >> 5);
 
         const int intra_lvl = lvl_seg + lf->ref_deltas[INTRA_FRAME] * scale;
@@ -771,20 +809,20 @@ void av1_loop_filter_frame_init(AV1_COMMON *cm, int default_filt_lvl,
                 clamp(inter_lvl, 0, MAX_LOOP_FILTER);
           }
         }
-      }
 #else
-      (void)default_filt_lvl_r;
-      const int intra_lvl = lvl_seg + lf->ref_deltas[INTRA_FRAME] * scale;
-      lfi->lvl[seg_id][INTRA_FRAME][0] = clamp(intra_lvl, 0, MAX_LOOP_FILTER);
-
-      for (ref = LAST_FRAME; ref < TOTAL_REFS_PER_FRAME; ++ref) {
-        for (mode = 0; mode < MAX_MODE_LF_DELTAS; ++mode) {
-          const int inter_lvl = lvl_seg + lf->ref_deltas[ref] * scale +
-                                lf->mode_deltas[mode] * scale;
-          lfi->lvl[seg_id][ref][mode] = clamp(inter_lvl, 0, MAX_LOOP_FILTER);
+        (void)default_filt_lvl_r;
+        const int intra_lvl = lvl_seg + lf->ref_deltas[INTRA_FRAME] * scale;
+        lfi->lvl[seg_id][INTRA_FRAME][0] = clamp(intra_lvl, 0, MAX_LOOP_FILTER);
+
+        for (ref = LAST_FRAME; ref < TOTAL_REFS_PER_FRAME; ++ref) {
+          for (mode = 0; mode < MAX_MODE_LF_DELTAS; ++mode) {
+            const int inter_lvl = lvl_seg + lf->ref_deltas[ref] * scale +
+                                  lf->mode_deltas[mode] * scale;
+            lfi->lvl[seg_id][ref][mode] = clamp(inter_lvl, 0, MAX_LOOP_FILTER);
+          }
         }
-      }
 #endif
+      }
     }
   }
 }
@@ -1481,7 +1519,7 @@ static void build_masks(AV1_COMMON *const cm,
       txsize_vert_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
 #if CONFIG_EXT_DELTA_Q
 #if CONFIG_LOOPFILTER_LEVEL
-  const int filter_level = get_filter_level(cm, lfi_n, 0, mbmi);
+  const int filter_level = get_filter_level(cm, lfi_n, 0, 0, mbmi);
 #else
 #if CONFIG_LPF_SB
   const int filter_level = get_filter_level(cm, lfi_n, 0, 0, mbmi);
@@ -1583,7 +1621,7 @@ static void build_y_mask(AV1_COMMON *const cm,
 #endif
 #if CONFIG_EXT_DELTA_Q
 #if CONFIG_LOOPFILTER_LEVEL
-  const int filter_level = get_filter_level(cm, lfi_n, 0, mbmi);
+  const int filter_level = get_filter_level(cm, lfi_n, 0, 0, mbmi);
 #else
 #if CONFIG_LPF_SB
   const int filter_level = get_filter_level(cm, lfi_n, 0, 0, mbmi);
@@ -2196,7 +2234,7 @@ static void get_filter_level_and_masks_non420(
 // Filter level can vary per MI
 #if CONFIG_EXT_DELTA_Q
 #if CONFIG_LOOPFILTER_LEVEL
-    if (!(lfl_r[c_step] = get_filter_level(cm, &cm->lf_info, 0, mbmi)))
+    if (!(lfl_r[c_step] = get_filter_level(cm, &cm->lf_info, 0, 0, mbmi)))
       continue;
 #else
 #if CONFIG_LPF_SB
@@ -2901,7 +2939,7 @@ static void set_lpf_parameters(
 #if CONFIG_EXT_DELTA_Q
 #if CONFIG_LOOPFILTER_LEVEL
     const uint32_t curr_level =
-        get_filter_level(cm, &cm->lf_info, edge_dir, mbmi);
+        get_filter_level(cm, &cm->lf_info, edge_dir, plane, mbmi);
 #else
 #if CONFIG_LPF_SB
     const uint32_t curr_level =
@@ -2942,8 +2980,8 @@ static void set_lpf_parameters(
 
 #if CONFIG_EXT_DELTA_Q
 #if CONFIG_LOOPFILTER_LEVEL
-          const uint32_t pv_lvl =
-              get_filter_level(cm, &cm->lf_info, edge_dir, &mi_prev->mbmi);
+          const uint32_t pv_lvl = get_filter_level(cm, &cm->lf_info, edge_dir,
+                                                   plane, &mi_prev->mbmi);
 #else
 #if CONFIG_LPF_SB
           const uint32_t pv_lvl = get_filter_level(cm, &cm->lf_info, pv_row,
@@ -3675,7 +3713,10 @@ void av1_loop_filter_frame(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm,
   }
   end_mi_row = start_mi_row + mi_rows_to_filter;
 #if CONFIG_LOOPFILTER_LEVEL
-  av1_loop_filter_frame_init(cm, frame_filter_level, frame_filter_level_r);
+  // TODO(chengchen): refactor the code such that y_only has its matching
+  // meaning. Now it means the plane to be filtered in this experiment.
+  av1_loop_filter_frame_init(cm, frame_filter_level, frame_filter_level_r,
+                             y_only);
 #else
   av1_loop_filter_frame_init(cm, frame_filter_level, frame_filter_level);
 #endif

av1/common/av1_loopfilter.h

@@ -138,7 +138,12 @@ void av1_loop_filter_init(struct AV1Common *cm);
 // av1_loop_filter_frame()
 // calls this function directly.
 void av1_loop_filter_frame_init(struct AV1Common *cm, int default_filt_lvl,
-                                int default_filt_lvl_r);
+                                int default_filt_lvl_r
+#if CONFIG_LOOPFILTER_LEVEL
+                                ,
+                                int plane
+#endif
+                                );
 
 #if CONFIG_LPF_SB
 void av1_loop_filter_frame(YV12_BUFFER_CONFIG *frame, struct AV1Common *cm,

av1/common/seg_common.c

@@ -16,10 +16,18 @@
 #include "av1/common/seg_common.h"
 #include "av1/common/quant_common.h"
 
+#if CONFIG_LOOPFILTER_LEVEL
+static const int seg_feature_data_signed[SEG_LVL_MAX] = { 1, 1, 1, 1, 0, 0 };
+
+static const int seg_feature_data_max[SEG_LVL_MAX] = {
+  MAXQ, MAX_LOOP_FILTER, MAX_LOOP_FILTER, MAX_LOOP_FILTER, 0
+};
+#else
 static const int seg_feature_data_signed[SEG_LVL_MAX] = { 1, 1, 0, 0 };
 
 static const int seg_feature_data_max[SEG_LVL_MAX] = { MAXQ, MAX_LOOP_FILTER, 3,
                                                        0 };
+#endif  // CONFIG_LOOPFILTER_LEVEL
 
 // These functions provide access to new segment level features.
 // Eventually these function may be "optimized out" but for the moment,

av1/common/seg_common.h

@@ -26,12 +26,29 @@ extern "C" {
 
 #define PREDICTION_PROBS 3
 
+#if CONFIG_LOOPFILTER_LEVEL
+typedef enum {
+  SEG_LVL_ALT_Q,       // Use alternate Quantizer ....
+  SEG_LVL_ALT_LF_Y_V,  // Use alternate loop filter value on y plane vertical
+  SEG_LVL_ALT_LF_Y_H,  // Use alternate loop filter value on y plane horizontal
+  SEG_LVL_ALT_LF_U,    // Use alternate loop filter value on u plane
+  SEG_LVL_ALT_LF_V,    // Use alternate loop filter value on v plane
+  SEG_LVL_REF_FRAME,   // Optional Segment reference frame
+  SEG_LVL_SKIP,        // Optional Segment (0,0) + skip mode
+#if CONFIG_SEGMENT_ZEROMV
+  SEG_LVL_ZEROMV,
+  SEG_LVL_MAX
+#else
+  SEG_LVL_MAX
+#endif
+} SEG_LVL_FEATURES;
+#else  // CONFIG_LOOPFILTER_LEVEL
 // Segment level features.
 typedef enum {
   SEG_LVL_ALT_Q = 0,      // Use alternate Quantizer ....
   SEG_LVL_ALT_LF = 1,     // Use alternate loop filter value...
   SEG_LVL_REF_FRAME = 2,  // Optional Segment reference frame
-  SEG_LVL_SKIP = 3,       // Optional Segment (0,0) + skip mode
+  SEG_LVL_SKIP = 3,  // Optional Segment (0,0) + skip mode
 #if CONFIG_SEGMENT_ZEROMV
   SEG_LVL_ZEROMV = 4,
   SEG_LVL_MAX = 5
@@ -39,6 +56,7 @@ typedef enum {
   SEG_LVL_MAX = 4
 #endif
 } SEG_LVL_FEATURES;
+#endif  // CONFIG_LOOPFILTER_LEVEL
 
 struct segmentation {
   uint8_t enabled;

av1/common/thread_common.c

@@ -435,7 +435,8 @@ void av1_loop_filter_frame_mt(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm,
   }
   end_mi_row = start_mi_row + mi_rows_to_filter;
 #if CONFIG_LOOPFILTER_LEVEL
-  av1_loop_filter_frame_init(cm, frame_filter_level, frame_filter_level_r);
+  av1_loop_filter_frame_init(cm, frame_filter_level, frame_filter_level_r,
+                             y_only);
 #else
   av1_loop_filter_frame_init(cm, frame_filter_level, frame_filter_level);
 #endif  // CONFIG_LOOPFILTER_LEVEL

av1/decoder/decodeframe.c

@@ -5515,13 +5515,7 @@ void av1_decode_frame(AV1Decoder *pbi, const uint8_t *data,
   }
 #endif
 
-#if CONFIG_LOOPFILTER_LEVEL
-  if ((cm->lf.filter_level[0] || cm->lf.filter_level[1]) &&
-      !cm->skip_loop_filter) {
-    av1_loop_filter_frame_init(cm, cm->lf.filter_level[0],
-                               cm->lf.filter_level[1]);
-  }
-#else
+#if !CONFIG_LOOPFILTER_LEVEL
   if (cm->lf.filter_level && !cm->skip_loop_filter) {
     av1_loop_filter_frame_init(cm, cm->lf.filter_level, cm->lf.filter_level);
   }
@@ -5828,13 +5822,7 @@ void av1_decode_tg_tiles_and_wrapup(AV1Decoder *pbi, const uint8_t *data,
   }
 #endif
 
-#if CONFIG_LOOPFILTER_LEVEL
-  if ((cm->lf.filter_level[0] || cm->lf.filter_level[1]) &&
-      !cm->skip_loop_filter) {
-    av1_loop_filter_frame_init(cm, cm->lf.filter_level[0],
-                               cm->lf.filter_level[1]);
-  }
-#else
+#if !CONFIG_LOOPFILTER_LEVEL
   if (cm->lf.filter_level && !cm->skip_loop_filter) {
     av1_loop_filter_frame_init(cm, cm->lf.filter_level, cm->lf.filter_level);
   }

av1/encoder/encoder.c

@@ -180,14 +180,37 @@ static void apply_active_map(AV1_COMP *cpi) {
         if (seg_map[i] == AM_SEGMENT_ID_ACTIVE) seg_map[i] = active_map[i];
       av1_enable_segmentation(seg);
       av1_enable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_SKIP);
+#if CONFIG_LOOPFILTER_LEVEL
+      av1_enable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF_Y_H);
+      av1_enable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF_Y_V);
+      av1_enable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF_U);
+      av1_enable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF_V);
+
+      av1_set_segdata(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF_Y_H,
+                      -MAX_LOOP_FILTER);
+      av1_set_segdata(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF_Y_V,
+                      -MAX_LOOP_FILTER);
+      av1_set_segdata(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF_U,
+                      -MAX_LOOP_FILTER);
+      av1_set_segdata(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF_V,
+                      -MAX_LOOP_FILTER);
+#else
       av1_enable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF);
       // Setting the data to -MAX_LOOP_FILTER will result in the computed loop
       // filter level being zero regardless of the value of seg->abs_delta.
       av1_set_segdata(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF,
                       -MAX_LOOP_FILTER);
+#endif  // CONFIG_LOOPFILTER_LEVEL
     } else {
       av1_disable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_SKIP);
+#if CONFIG_LOOPFILTER_LEVEL
+      av1_disable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF_Y_H);
+      av1_disable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF_Y_V);
+      av1_disable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF_U);
+      av1_disable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF_V);
+#else
       av1_disable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF);
+#endif  // CONFIG_LOOPFILTER_LEVEL
       if (seg->enabled) {
         seg->update_data = 1;
         seg->update_map = 1;
@@ -649,10 +672,22 @@ static void configure_static_seg_features(AV1_COMP *cpi) {
       qi_delta =
           av1_compute_qdelta(rc, rc->avg_q, rc->avg_q * 0.875, cm->bit_depth);
       av1_set_segdata(seg, 1, SEG_LVL_ALT_Q, qi_delta - 2);
+#if CONFIG_LOOPFILTER_LEVEL
+      av1_set_segdata(seg, 1, SEG_LVL_ALT_LF_Y_H, -2);
+      av1_set_segdata(seg, 1, SEG_LVL_ALT_LF_Y_V, -2);
+      av1_set_segdata(seg, 1, SEG_LVL_ALT_LF_U, -2);
+      av1_set_segdata(seg, 1, SEG_LVL_ALT_LF_V, -2);
+
+      av1_enable_segfeature(seg, 1, SEG_LVL_ALT_LF_Y_H);
+      av1_enable_segfeature(seg, 1, SEG_LVL_ALT_LF_Y_V);
+      av1_enable_segfeature(seg, 1, SEG_LVL_ALT_LF_U);
+      av1_enable_segfeature(seg, 1, SEG_LVL_ALT_LF_V);
+#else
       av1_set_segdata(seg, 1, SEG_LVL_ALT_LF, -2);
+      av1_enable_segfeature(seg, 1, SEG_LVL_ALT_LF);
+#endif  // CONFIG_LOOPFILTER_LEVEL
 
       av1_enable_segfeature(seg, 1, SEG_LVL_ALT_Q);
-      av1_enable_segfeature(seg, 1, SEG_LVL_ALT_LF);
 
       // Where relevant assume segment data is delta data
       seg->abs_delta = SEGMENT_DELTADATA;
@@ -673,8 +708,20 @@ static void configure_static_seg_features(AV1_COMP *cpi) {
         av1_set_segdata(seg, 1, SEG_LVL_ALT_Q, qi_delta + 2);
         av1_enable_segfeature(seg, 1, SEG_LVL_ALT_Q);
 
+#if CONFIG_LOOPFILTER_LEVEL
+        av1_set_segdata(seg, 1, SEG_LVL_ALT_LF_Y_H, -2);
+        av1_set_segdata(seg, 1, SEG_LVL_ALT_LF_Y_V, -2);
+        av1_set_segdata(seg, 1, SEG_LVL_ALT_LF_U, -2);
+        av1_set_segdata(seg, 1, SEG_LVL_ALT_LF_V, -2);
+
+        av1_enable_segfeature(seg, 1, SEG_LVL_ALT_LF_Y_H);
+        av1_enable_segfeature(seg, 1, SEG_LVL_ALT_LF_Y_V);
+        av1_enable_segfeature(seg, 1, SEG_LVL_ALT_LF_U);
+        av1_enable_segfeature(seg, 1, SEG_LVL_ALT_LF_V);
+#else
         av1_set_segdata(seg, 1, SEG_LVL_ALT_LF, -2);
         av1_enable_segfeature(seg, 1, SEG_LVL_ALT_LF);
+#endif  // CONFIG_LOOPFILTER_LEVEL
 
         // Segment coding disabled for compred testing
         if (high_q || (cpi->static_mb_pct == 100)) {