vp9_loopfilter.c 27.6 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10
 */

Attila Nagy's avatar
Attila Nagy committed
11
#include "vpx_config.h"
12 13
#include "vp9/common/vp9_loopfilter.h"
#include "vp9/common/vp9_onyxc_int.h"
14
#include "vp9/common/vp9_reconinter.h"
Attila Nagy's avatar
Attila Nagy committed
15
#include "vpx_mem/vpx_mem.h"
John Koleszar's avatar
John Koleszar committed
16

17
#include "vp9/common/vp9_seg_common.h"
18

John Koleszar's avatar
John Koleszar committed
19 20 21 22 23 24 25 26 27 28 29 30 31 32 33
static void lf_init_lut(loop_filter_info_n *lfi) {
  lfi->mode_lf_lut[DC_PRED] = 1;
  lfi->mode_lf_lut[D45_PRED] = 1;
  lfi->mode_lf_lut[D135_PRED] = 1;
  lfi->mode_lf_lut[D117_PRED] = 1;
  lfi->mode_lf_lut[D153_PRED] = 1;
  lfi->mode_lf_lut[D27_PRED] = 1;
  lfi->mode_lf_lut[D63_PRED] = 1;
  lfi->mode_lf_lut[V_PRED] = 1;
  lfi->mode_lf_lut[H_PRED] = 1;
  lfi->mode_lf_lut[TM_PRED] = 1;
  lfi->mode_lf_lut[ZEROMV]  = 1;
  lfi->mode_lf_lut[NEARESTMV] = 2;
  lfi->mode_lf_lut[NEARMV] = 2;
  lfi->mode_lf_lut[NEWMV] = 2;
Attila Nagy's avatar
Attila Nagy committed
34 35
}

36
void vp9_loop_filter_update_sharpness(loop_filter_info_n *lfi,
John Koleszar's avatar
John Koleszar committed
37 38 39 40 41 42 43
                                      int sharpness_lvl) {
  int i;

  /* For each possible value for the loop filter fill out limits */
  for (i = 0; i <= MAX_LOOP_FILTER; i++) {
    int filt_lvl = i;
    int block_inside_limit = 0;
John Koleszar's avatar
John Koleszar committed
44

John Koleszar's avatar
John Koleszar committed
45 46 47
    /* Set loop filter paramaeters that control sharpness. */
    block_inside_limit = filt_lvl >> (sharpness_lvl > 0);
    block_inside_limit = block_inside_limit >> (sharpness_lvl > 4);
John Koleszar's avatar
John Koleszar committed
48

John Koleszar's avatar
John Koleszar committed
49 50 51
    if (sharpness_lvl > 0) {
      if (block_inside_limit > (9 - sharpness_lvl))
        block_inside_limit = (9 - sharpness_lvl);
John Koleszar's avatar
John Koleszar committed
52
    }
John Koleszar's avatar
John Koleszar committed
53 54 55 56 57 58 59 60 61 62

    if (block_inside_limit < 1)
      block_inside_limit = 1;

    vpx_memset(lfi->lim[i], block_inside_limit, SIMD_WIDTH);
    vpx_memset(lfi->blim[i], (2 * filt_lvl + block_inside_limit),
               SIMD_WIDTH);
    vpx_memset(lfi->mblim[i], (2 * (filt_lvl + 2) + block_inside_limit),
               SIMD_WIDTH);
  }
Attila Nagy's avatar
Attila Nagy committed
63
}
John Koleszar's avatar
John Koleszar committed
64

65
void vp9_loop_filter_init(VP9_COMMON *cm) {
John Koleszar's avatar
John Koleszar committed
66 67
  loop_filter_info_n *lfi = &cm->lf_info;
  int i;
Attila Nagy's avatar
Attila Nagy committed
68

69
  // init limits for given sharpness
70
  vp9_loop_filter_update_sharpness(lfi, cm->sharpness_level);
John Koleszar's avatar
John Koleszar committed
71
  cm->last_sharpness_level = cm->sharpness_level;
Attila Nagy's avatar
Attila Nagy committed
72

73
  // init LUT for lvl  and hev thr picking
John Koleszar's avatar
John Koleszar committed
74
  lf_init_lut(lfi);
Attila Nagy's avatar
Attila Nagy committed
75

76 77
  // init hev threshold const vectors
  for (i = 0; i < 4; i++)
John Koleszar's avatar
John Koleszar committed
78
    vpx_memset(lfi->hev_thr[i], i, SIMD_WIDTH);
John Koleszar's avatar
John Koleszar committed
79 80
}

81
void vp9_loop_filter_frame_init(VP9_COMMON *cm,
82
                                MACROBLOCKD *xd,
John Koleszar's avatar
John Koleszar committed
83
                                int default_filt_lvl) {
84 85 86 87 88 89 90
  int seg,    // segment number
      ref,    // index in ref_lf_deltas
      mode;   // index in mode_lf_deltas
  // n_shift is the a multiplier for lf_deltas
  // the multiplier is 1 for when filter_lvl is between 0 and 31;
  // 2 when filter_lvl is between 32 and 63
  int n_shift = default_filt_lvl >> 5;
John Koleszar's avatar
John Koleszar committed
91

John Koleszar's avatar
John Koleszar committed
92
  loop_filter_info_n *lfi = &cm->lf_info;
John Koleszar's avatar
John Koleszar committed
93

John Koleszar's avatar
John Koleszar committed
94
  /* update limits if sharpness has changed */
95 96 97
  // printf("vp9_loop_filter_frame_init %d\n", default_filt_lvl);
  // printf("sharpness level: %d [%d]\n",
  //        cm->sharpness_level, cm->last_sharpness_level);
John Koleszar's avatar
John Koleszar committed
98
  if (cm->last_sharpness_level != cm->sharpness_level) {
99
    vp9_loop_filter_update_sharpness(lfi, cm->sharpness_level);
John Koleszar's avatar
John Koleszar committed
100 101 102 103 104 105 106 107 108
    cm->last_sharpness_level = cm->sharpness_level;
  }

  for (seg = 0; seg < MAX_MB_SEGMENTS; seg++) {
    int lvl_seg = default_filt_lvl;
    int lvl_ref, lvl_mode;


    // Set the baseline filter values for each segment
109
    if (vp9_segfeature_active(xd, seg, SEG_LVL_ALT_LF)) {
John Koleszar's avatar
John Koleszar committed
110 111
      /* Abs value */
      if (xd->mb_segment_abs_delta == SEGMENT_ABSDATA) {
112
        lvl_seg = vp9_get_segdata(xd, seg, SEG_LVL_ALT_LF);
John Koleszar's avatar
John Koleszar committed
113
      } else { /* Delta Value */
114
        lvl_seg += vp9_get_segdata(xd, seg, SEG_LVL_ALT_LF);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
115
        lvl_seg = clamp(lvl_seg, 0, 63);
John Koleszar's avatar
John Koleszar committed
116 117
      }
    }
John Koleszar's avatar
John Koleszar committed
118

John Koleszar's avatar
John Koleszar committed
119 120 121 122 123 124 125
    if (!xd->mode_ref_lf_delta_enabled) {
      /* we could get rid of this if we assume that deltas are set to
       * zero when not in use; encoder always uses deltas
       */
      vpx_memset(lfi->lvl[seg][0], lvl_seg, 4 * 4);
      continue;
    }
John Koleszar's avatar
John Koleszar committed
126

John Koleszar's avatar
John Koleszar committed
127
    lvl_ref = lvl_seg;
John Koleszar's avatar
John Koleszar committed
128

John Koleszar's avatar
John Koleszar committed
129 130
    /* INTRA_FRAME */
    ref = INTRA_FRAME;
John Koleszar's avatar
John Koleszar committed
131

John Koleszar's avatar
John Koleszar committed
132
    /* Apply delta for reference frame */
133
    lvl_ref += xd->ref_lf_deltas[ref] << n_shift;
Attila Nagy's avatar
Attila Nagy committed
134

John Koleszar's avatar
John Koleszar committed
135
    /* Apply delta for Intra modes */
Yaowu Xu's avatar
Yaowu Xu committed
136 137
    mode = 0; /* I4X4_PRED */
    /* Only the split mode I4X4_PRED has a further special case */
138 139
    lvl_mode = lvl_ref + (xd->mode_lf_deltas[mode] << n_shift);
    lfi->lvl[seg][ref][mode] = clamp(lvl_mode, 0, 63);
Attila Nagy's avatar
Attila Nagy committed
140

John Koleszar's avatar
John Koleszar committed
141
    mode = 1; /* all the rest of Intra modes */
142 143
    lvl_mode = lvl_ref;
    lfi->lvl[seg][ref][mode] = clamp(lvl_mode, 0, 63);
John Koleszar's avatar
John Koleszar committed
144

John Koleszar's avatar
John Koleszar committed
145 146 147
    /* LAST, GOLDEN, ALT */
    for (ref = 1; ref < MAX_REF_FRAMES; ref++) {
      int lvl_ref = lvl_seg;
John Koleszar's avatar
John Koleszar committed
148

John Koleszar's avatar
John Koleszar committed
149
      /* Apply delta for reference frame */
150
      lvl_ref += xd->ref_lf_deltas[ref] << n_shift;
John Koleszar's avatar
John Koleszar committed
151

John Koleszar's avatar
John Koleszar committed
152 153
      /* Apply delta for Inter modes */
      for (mode = 1; mode < 4; mode++) {
154 155
        lvl_mode = lvl_ref + (xd->mode_lf_deltas[mode] << n_shift);
        lfi->lvl[seg][ref][mode] = clamp(lvl_mode, 0, 63);
John Koleszar's avatar
John Koleszar committed
156
      }
John Koleszar's avatar
John Koleszar committed
157
    }
John Koleszar's avatar
John Koleszar committed
158
  }
John Koleszar's avatar
John Koleszar committed
159 160
}

161
#if !CONFIG_NEW_LOOPFILTER
Yaowu Xu's avatar
Yaowu Xu committed
162 163 164 165
// Determine if we should skip inner-MB loop filtering within a MB
// The current condition is that the loop filtering is skipped only
// the MB uses a prediction size of 16x16 and either 16x16 transform
// is used or there is no residue at all.
166 167 168
static int mb_lf_skip(const MB_MODE_INFO *const mbmi) {
  const int skip_coef = mbmi->mb_skip_coeff;
  const int tx_size = mbmi->txfm_size;
169
  return mbmi->sb_type >= BLOCK_SIZE_MB16X16 &&
Yaowu Xu's avatar
Yaowu Xu committed
170
         (tx_size >= TX_16X16 || skip_coef);
171
}
Yaowu Xu's avatar
Yaowu Xu committed
172 173 174 175 176

// Determine if we should skip MB loop filtering on a MB edge within
// a superblock, the current condition is that MB loop filtering is
// skipped only when both MBs do not use inner MB loop filtering, and
// same motion vector with same reference frame
177 178
static int sb_mb_lf_skip(const MODE_INFO *const mip0,
                         const MODE_INFO *const mip1) {
Yaowu Xu's avatar
Yaowu Xu committed
179
  const MB_MODE_INFO *mbmi0 = &mip0->mbmi;
Yaowu Xu's avatar
Yaowu Xu committed
180
  const MB_MODE_INFO *mbmi1 = &mip1->mbmi;
Yaowu Xu's avatar
Yaowu Xu committed
181
  return mb_lf_skip(mbmi0) && mb_lf_skip(mbmi1) &&
Ronald S. Bultje's avatar
Ronald S. Bultje committed
182 183
         mbmi0->ref_frame[0] != INTRA_FRAME &&
         mbmi1->ref_frame[0] != INTRA_FRAME;
184
}
185

Yaowu Xu's avatar
Yaowu Xu committed
186 187 188 189
static void lpf_mb(VP9_COMMON *cm, const MODE_INFO *mi,
                   int do_left_mb_v, int do_above_mb_h,
                   int do_left_mbuv_v, int do_above_mbuv_h,
                   uint8_t *y_ptr, uint8_t *u_ptr, uint8_t *v_ptr,
Paul Wilkins's avatar
Paul Wilkins committed
190
                   int y_stride, int uv_stride) {
Yaowu Xu's avatar
Yaowu Xu committed
191 192 193 194 195
  loop_filter_info_n *lfi_n = &cm->lf_info;
  struct loop_filter_info lfi;
  int mode = mi->mbmi.mode;
  int mode_index = lfi_n->mode_lf_lut[mode];
  int seg = mi->mbmi.segment_id;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
196
  MV_REFERENCE_FRAME ref_frame = mi->mbmi.ref_frame[0];
Yaowu Xu's avatar
Yaowu Xu committed
197 198 199 200 201
  int filter_level = lfi_n->lvl[seg][ref_frame][mode_index];

  if (filter_level) {
    const int skip_lf = mb_lf_skip(&mi->mbmi);
    const int tx_size = mi->mbmi.txfm_size;
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
    const int hev_index = filter_level >> 4;
    lfi.mblim = lfi_n->mblim[filter_level];
    lfi.blim = lfi_n->blim[filter_level];
    lfi.lim = lfi_n->lim[filter_level];
    lfi.hev_thr = lfi_n->hev_thr[hev_index];

    if (do_above_mb_h) {
      if (tx_size >= TX_16X16)
        vp9_lpf_mbh_w(y_ptr,
                      do_above_mbuv_h ? u_ptr : NULL,
                      do_above_mbuv_h ? v_ptr : NULL,
                      y_stride, uv_stride, &lfi);
      else
        vp9_loop_filter_mbh(y_ptr, u_ptr, v_ptr, y_stride, uv_stride, &lfi);
    }

    if (!skip_lf) {
      if (tx_size >= TX_8X8) {
        if (tx_size == TX_8X8 &&
            mi->mbmi.sb_type < BLOCK_SIZE_MB16X16)
          vp9_loop_filter_bh8x8(y_ptr, u_ptr, v_ptr,
                                y_stride, uv_stride, &lfi);
Yaowu Xu's avatar
Yaowu Xu committed
224
        else
225 226 227 228 229
          vp9_loop_filter_bh8x8(y_ptr, NULL, NULL,
                                y_stride, uv_stride, &lfi);
      } else {
        vp9_loop_filter_bh(y_ptr, u_ptr, v_ptr,
                           y_stride, uv_stride, &lfi);
Yaowu Xu's avatar
Yaowu Xu committed
230
      }
231
    }
Yaowu Xu's avatar
Yaowu Xu committed
232

233 234 235 236 237 238 239 240 241
    if (do_left_mb_v) {
      if (tx_size >= TX_16X16)
        vp9_lpf_mbv_w(y_ptr,
                      do_left_mbuv_v ? u_ptr : NULL,
                      do_left_mbuv_v ? v_ptr : NULL,
                      y_stride, uv_stride, &lfi);
      else
        vp9_loop_filter_mbv(y_ptr, u_ptr, v_ptr, y_stride, uv_stride, &lfi);
    }
Yaowu Xu's avatar
Yaowu Xu committed
242

243 244 245 246 247 248
    if (!skip_lf) {
      if (tx_size >= TX_8X8) {
        if (tx_size == TX_8X8 &&
            mi->mbmi.sb_type < BLOCK_SIZE_MB16X16)
          vp9_loop_filter_bv8x8(y_ptr, u_ptr, v_ptr,
                                y_stride, uv_stride, &lfi);
Yaowu Xu's avatar
Yaowu Xu committed
249
        else
250 251 252 253 254
          vp9_loop_filter_bv8x8(y_ptr, NULL, NULL,
                                y_stride, uv_stride, &lfi);
      } else {
        vp9_loop_filter_bv(y_ptr, u_ptr, v_ptr,
                           y_stride, uv_stride, &lfi);
Yaowu Xu's avatar
Yaowu Xu committed
255
      }
256
    }
Yaowu Xu's avatar
Yaowu Xu committed
257 258 259 260 261 262 263
  }
}

static void lpf_sb32(VP9_COMMON *cm, const MODE_INFO *mode_info_context,
                     int mb_row, int mb_col,
                     uint8_t *y_ptr, uint8_t *u_ptr, uint8_t *v_ptr,
                     int y_stride, int uv_stride,
Paul Wilkins's avatar
Paul Wilkins committed
264
                     int y_only) {
265 266 267 268 269 270 271 272 273 274 275 276 277
  BLOCK_SIZE_TYPE sb_type = mode_info_context->mbmi.sb_type;
  const int wbl = b_width_log2(sb_type), hbl = b_height_log2(sb_type);
  TX_SIZE tx_size = mode_info_context->mbmi.txfm_size;
  int do_left_v, do_above_h;
  int do_left_v_mbuv, do_above_h_mbuv;
  int mis = cm->mode_info_stride;
  const MODE_INFO *mi;

  // process 1st MB top-left
  mi = mode_info_context;
  do_left_v = (mb_col > 0);
  do_above_h = (mb_row > 0);
  do_left_v_mbuv = !(sb_type >= BLOCK_SIZE_SB64X64 &&
Yaowu Xu's avatar
Yaowu Xu committed
278
      tx_size >= TX_32X32 && (mb_col & 2));
279
  do_above_h_mbuv = !(sb_type >= BLOCK_SIZE_SB64X64 &&
Yaowu Xu's avatar
Yaowu Xu committed
280
      tx_size >= TX_32X32 && (mb_row & 2));
281
  lpf_mb(cm, mi, do_left_v, do_above_h,
Yaowu Xu's avatar
Yaowu Xu committed
282 283 284 285
      do_left_v_mbuv, do_above_h_mbuv,
      y_ptr,
      y_only? 0 : u_ptr,
      y_only? 0 : v_ptr,
Paul Wilkins's avatar
Paul Wilkins committed
286
      y_stride, uv_stride);
287
  // process 2nd MB top-right
Jingning Han's avatar
Jingning Han committed
288
  mi = mode_info_context + 2;
289
  do_left_v = !(wbl >= 3 /* 32x16 or >=32x32 */ && (tx_size >= TX_32X32 ||
Yaowu Xu's avatar
Yaowu Xu committed
290
      sb_mb_lf_skip(mode_info_context, mi)));
291
  do_above_h = (mb_row > 0);
292 293
  do_left_v_mbuv = !(wbl >= 3 /* 32x16 or >=32x32 */ && (tx_size >= TX_16X16 ||
      sb_mb_lf_skip(mode_info_context, mi)));
294
  do_above_h_mbuv = !(sb_type >= BLOCK_SIZE_SB64X64 &&
Yaowu Xu's avatar
Yaowu Xu committed
295
      tx_size >= TX_32X32 && (mb_row & 2));
296
  lpf_mb(cm, mi, do_left_v, do_above_h,
Yaowu Xu's avatar
Yaowu Xu committed
297 298 299 300
      do_left_v_mbuv, do_above_h_mbuv,
      y_ptr + 16,
      y_only ? 0 : (u_ptr + 8),
      y_only ? 0 : (v_ptr + 8),
Paul Wilkins's avatar
Paul Wilkins committed
301
      y_stride, uv_stride);
Yaowu Xu's avatar
Yaowu Xu committed
302

303
  // process 3rd MB bottom-left
Jingning Han's avatar
Jingning Han committed
304
  mi = mode_info_context + (mis << 1);
305 306
  do_left_v = (mb_col > 0);
  do_above_h = !(hbl >= 3 /* 16x32 or >=32x32 */ && (tx_size >= TX_32X32 ||
Yaowu Xu's avatar
Yaowu Xu committed
307
      sb_mb_lf_skip(mode_info_context, mi)));
308
  do_left_v_mbuv = !(sb_type >= BLOCK_SIZE_SB64X64 &&
Yaowu Xu's avatar
Yaowu Xu committed
309
      tx_size >= TX_32X32 && (mb_col & 2));
310 311
  do_above_h_mbuv = !(hbl >= 3 /* 16x32 or >=32x32 */ && (tx_size >= TX_16X16 ||
      sb_mb_lf_skip(mode_info_context, mi)));
312
  lpf_mb(cm, mi, do_left_v, do_above_h,
Yaowu Xu's avatar
Yaowu Xu committed
313 314 315 316
      do_left_v_mbuv, do_above_h_mbuv,
      y_ptr + 16 * y_stride,
      y_only ? 0 : (u_ptr + 8 * uv_stride),
      y_only ? 0 : (v_ptr + 8 * uv_stride),
Paul Wilkins's avatar
Paul Wilkins committed
317
      y_stride, uv_stride);
Yaowu Xu's avatar
Yaowu Xu committed
318

319
  // process 4th MB bottom right
Jingning Han's avatar
Jingning Han committed
320
  mi = mode_info_context + ((mis + 1) << 1);
321
  do_left_v = !(wbl >= 3 /* 32x16 or >=32x32 */ && (tx_size >= TX_32X32 ||
Jingning Han's avatar
Jingning Han committed
322
      sb_mb_lf_skip(mi - 2, mi)));
323
  do_above_h = !(hbl >= 3 /* 16x32 or >=32x32 */ && (tx_size >= TX_32X32 ||
Jingning Han's avatar
Jingning Han committed
324
      sb_mb_lf_skip(mode_info_context + 2, mi)));
325
  do_left_v_mbuv = (wbl >= 3 /* 32x16 or >=32x32 */ && (tx_size >= TX_16X16 ||
Jingning Han's avatar
Jingning Han committed
326
      sb_mb_lf_skip(mi - 2, mi)));
327
  do_above_h_mbuv = !(hbl >= 3 /* 16x32 or >=32x32 */ && (tx_size >= TX_16X16 ||
Jingning Han's avatar
Jingning Han committed
328
      sb_mb_lf_skip(mode_info_context + 2, mi)));
329
  lpf_mb(cm, mi, do_left_v, do_above_h,
Yaowu Xu's avatar
Yaowu Xu committed
330 331 332 333
      do_left_v_mbuv, do_above_h_mbuv,
      y_ptr + 16 * y_stride + 16,
      y_only ? 0 : (u_ptr + 8 * uv_stride + 8),
      y_only ? 0 : (v_ptr + 8 * uv_stride + 8),
Paul Wilkins's avatar
Paul Wilkins committed
334
      y_stride, uv_stride);
Yaowu Xu's avatar
Yaowu Xu committed
335 336 337 338 339 340
}

static void lpf_sb64(VP9_COMMON *cm, const MODE_INFO *mode_info_context,
                     int mb_row, int mb_col,
                     uint8_t *y_ptr, uint8_t *u_ptr, uint8_t *v_ptr,
                     int y_stride, int uv_stride,
Paul Wilkins's avatar
Paul Wilkins committed
341
                     int y_only) {
342
  lpf_sb32(cm, mode_info_context, mb_row, mb_col,
Yaowu Xu's avatar
Yaowu Xu committed
343
      y_ptr, u_ptr, v_ptr,
Paul Wilkins's avatar
Paul Wilkins committed
344
      y_stride, uv_stride, y_only);
Jingning Han's avatar
Jingning Han committed
345
  lpf_sb32(cm, mode_info_context + 4, mb_row, mb_col + 2,
Yaowu Xu's avatar
Yaowu Xu committed
346
      y_ptr + 32, u_ptr + 16, v_ptr + 16,
Paul Wilkins's avatar
Paul Wilkins committed
347
      y_stride, uv_stride, y_only);
Jingning Han's avatar
Jingning Han committed
348
  lpf_sb32(cm, mode_info_context + cm->mode_info_stride * 4,
Yaowu Xu's avatar
Yaowu Xu committed
349 350 351 352
      mb_row + 2, mb_col,
      y_ptr + 32 * y_stride,
      u_ptr + 16 * uv_stride,
      v_ptr + 16 * uv_stride,
Paul Wilkins's avatar
Paul Wilkins committed
353
      y_stride, uv_stride, y_only);
Jingning Han's avatar
Jingning Han committed
354
  lpf_sb32(cm, mode_info_context + cm->mode_info_stride * 4 + 4,
Yaowu Xu's avatar
Yaowu Xu committed
355 356 357 358
      mb_row + 2, mb_col + 2,
      y_ptr + 32 * y_stride + 32,
      u_ptr + 16 * uv_stride + 16,
      v_ptr + 16 * uv_stride + 16,
Paul Wilkins's avatar
Paul Wilkins committed
359
      y_stride, uv_stride, y_only);
Yaowu Xu's avatar
Yaowu Xu committed
360
}
361

362 363 364
void vp9_loop_filter_frame(VP9_COMMON *cm,
                           MACROBLOCKD *xd,
                           int frame_filter_level,
Paul Wilkins's avatar
Paul Wilkins committed
365
                           int y_only) {
John Koleszar's avatar
John Koleszar committed
366
  YV12_BUFFER_CONFIG *post = cm->frame_to_show;
367
  int mb_row, mb_col;
Yaowu Xu's avatar
Yaowu Xu committed
368 369 370 371 372 373
  const int sb64_rows = cm->mb_rows / 4;
  const int sb64_cols = cm->mb_cols / 4;
  const int extra_sb32_row = (cm->mb_rows & 2) != 0;
  const int extra_sb32_col = (cm->mb_cols & 2) != 0;
  const int extra_mb_col = cm->mb_cols & 1;
  const int extra_mb_row = cm->mb_rows & 1;
374 375 376 377 378 379
  // Set up the buffer pointers
  uint8_t *y_ptr = post->y_buffer;
  uint8_t *u_ptr = y_only ? 0 : post->u_buffer;
  uint8_t *v_ptr = y_only ? 0 : post->v_buffer;

  // Point at base of Mb MODE_INFO list
John Koleszar's avatar
John Koleszar committed
380
  const MODE_INFO *mode_info_context = cm->mi;
Yaowu Xu's avatar
Yaowu Xu committed
381
  const MODE_INFO *mi;
382
  const int mis = cm->mode_info_stride;
Yaowu Xu's avatar
Yaowu Xu committed
383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
  const int y_stride = post->y_stride;
  const int uv_stride = post->uv_stride;
  // These two flags signal if MB left edge and above edge
  // should be filtered using MB edge filter. Currently, MB
  // edge filtering is not applied on MB edge internal to a
  // 32x32 superblock if:
  // 1) SB32 is using 32x32 prediction and 32x32 transform
  // 2) SB32 is using 32x32 prediction and 16x16 transform
  //    but all coefficients are zero.
  // MB edges are on 32x32 superblock boundary are always
  // filtered except on image frame boundary.
  int do_left_v, do_above_h;
  // These two flags signal if MB UV left edge and above edge
  // should be filtered using MB edge filter. Currently, MB
  // edge filtering is not applied for MB edges internal to
  // a 32x32 superblock if:
  // 1) SB32 is using 32x32 prediction and 32x32 transform
  // 2) SB32 is using 32x32 prediction and 16x16 transform
  //    but all coefficients are zero.
  // 3) SB32 UV edges internal to a SB64 and 32x32 transform
  //    is used, i.e. UV is doing 32x32 transform hence no
  //    transform boundary exists inside the SB64 for UV
  int do_left_v_mbuv, do_above_h_mbuv;
John Koleszar's avatar
John Koleszar committed
406

407
  // Initialize the loop filter for this frame.
408
  vp9_loop_filter_frame_init(cm, xd, frame_filter_level);
John Koleszar's avatar
John Koleszar committed
409

Yaowu Xu's avatar
Yaowu Xu committed
410 411 412 413 414 415 416 417 418
  // vp9_filter each 64x64 SB
  // For each SB64: the 4 SB32 are filtered in raster scan order
  // For each SB32: the 4 MBs are filtered in raster scan order
  // For each MB:   the left and above MB edges as well as the
  //                internal block edges are processed together
  for (mb_row = 0; mb_row < sb64_rows * 4; mb_row += 4) {
    for (mb_col = 0; mb_col < sb64_cols * 4; mb_col += 4) {
      lpf_sb64(cm, mode_info_context, mb_row, mb_col,
               y_ptr, u_ptr, v_ptr,
Paul Wilkins's avatar
Paul Wilkins committed
419
               y_stride, uv_stride, y_only);
Yaowu Xu's avatar
Yaowu Xu committed
420 421 422
      y_ptr += 64;
      u_ptr = y_only? 0 : u_ptr + 32;
      v_ptr = y_only? 0 : v_ptr + 32;
Jingning Han's avatar
Jingning Han committed
423
      mode_info_context += 8;       // step to next SB64
Yaowu Xu's avatar
Yaowu Xu committed
424 425 426 427 428
    }
    if (extra_sb32_col) {
      // process 2 SB32s in the extra SB32 col
      lpf_sb32(cm, mode_info_context, mb_row, mb_col,
               y_ptr, u_ptr, v_ptr,
Paul Wilkins's avatar
Paul Wilkins committed
429
               y_stride, uv_stride, y_only);
Jingning Han's avatar
Jingning Han committed
430
      lpf_sb32(cm, mode_info_context + mis * 4,
Yaowu Xu's avatar
Yaowu Xu committed
431 432 433 434
               mb_row + 2, mb_col,
               y_ptr + 32 * y_stride,
               u_ptr + 16 * uv_stride,
               v_ptr + 16 * uv_stride,
Paul Wilkins's avatar
Paul Wilkins committed
435
               y_stride, uv_stride, y_only);
Yaowu Xu's avatar
Yaowu Xu committed
436 437 438
      y_ptr += 32;
      u_ptr = y_only? 0 : u_ptr + 16;
      v_ptr = y_only? 0 : v_ptr + 16;
Jingning Han's avatar
Jingning Han committed
439
      mode_info_context += 4;       // step to next SB32
Yaowu Xu's avatar
Yaowu Xu committed
440 441 442 443
      mb_col += 2;
    }
    if (extra_mb_col) {
      // process 4 MB in the extra MB col
444 445 446 447 448 449 450 451 452 453 454 455
      int k;
      for (k = 0; k < 4; ++k) {
        mi = mode_info_context + (mis << 1) * k;
        do_left_v = (mb_col > 0);
        do_above_h = k == 0 ? mb_row > 0 : 1;
        do_left_v_mbuv =  1;
        do_above_h_mbuv = 1;
        lpf_mb(cm, mi, do_left_v, do_above_h,
               do_left_v_mbuv, do_above_h_mbuv,
               y_ptr + (k * 16) * y_stride,
               y_only ? 0 : (u_ptr + (k * 8) * uv_stride),
               y_only ? 0 : (v_ptr + (k * 8) * uv_stride),
Paul Wilkins's avatar
Paul Wilkins committed
456
               y_stride, uv_stride);
457 458
      }

John Koleszar's avatar
John Koleszar committed
459
      y_ptr += 16;
Yaowu Xu's avatar
Yaowu Xu committed
460 461
      u_ptr = y_only? 0 : u_ptr + 8;
      v_ptr = y_only? 0 : v_ptr + 8;
Jingning Han's avatar
Jingning Han committed
462
      mode_info_context += 2;       // step to next MB
John Koleszar's avatar
John Koleszar committed
463
    }
Yaowu Xu's avatar
Yaowu Xu committed
464 465
    // move pointers to the begining of next sb64 row
    y_ptr += y_stride  * 64 - post->y_width;
466
    if (!y_only) {
Yaowu Xu's avatar
Yaowu Xu committed
467 468 469 470
      u_ptr += uv_stride *  32 - post->uv_width;
      v_ptr += uv_stride *  32 - post->uv_width;
    }
    /* skip to next SB64 row */
Jingning Han's avatar
Jingning Han committed
471
    mode_info_context += mis * 8 - cm->mi_cols;
Yaowu Xu's avatar
Yaowu Xu committed
472 473 474 475 476 477
  }
  if (extra_sb32_row) {
    const int sb32_cols = sb64_cols * 2 + extra_sb32_col;
    for (mb_col = 0; mb_col < sb32_cols * 2; mb_col += 2) {
      lpf_sb32(cm, mode_info_context, mb_row, mb_col,
               y_ptr, u_ptr, v_ptr,
Paul Wilkins's avatar
Paul Wilkins committed
478
               y_stride, uv_stride, y_only);
Yaowu Xu's avatar
Yaowu Xu committed
479 480 481
      y_ptr += 32;
      u_ptr = y_only? 0 : u_ptr + 16;
      v_ptr = y_only? 0 : v_ptr + 16;
Jingning Han's avatar
Jingning Han committed
482
      mode_info_context += 4;       // step to next SB32
Yaowu Xu's avatar
Yaowu Xu committed
483 484 485 486 487 488 489 490 491 492 493 494 495
    }
    if (extra_mb_col) {
      // process 1st MB
      mi = mode_info_context;
      do_left_v = (mb_col > 0);
      do_above_h = (mb_row > 0);
      do_left_v_mbuv =  1;
      do_above_h_mbuv = 1;
      lpf_mb(cm, mi, do_left_v, do_above_h,
             do_left_v_mbuv, do_above_h_mbuv,
             y_ptr,
             y_only? NULL : u_ptr,
             y_only? NULL : v_ptr,
Paul Wilkins's avatar
Paul Wilkins committed
496
             y_stride, uv_stride);
Yaowu Xu's avatar
Yaowu Xu committed
497
      // process 2nd MB
Jingning Han's avatar
Jingning Han committed
498
      mi = mode_info_context + (mis << 1);
Yaowu Xu's avatar
Yaowu Xu committed
499 500 501 502 503 504 505 506 507
      do_left_v = (mb_col > 0);
      do_above_h = 1;
      do_left_v_mbuv =  1;
      do_above_h_mbuv = 1;
      lpf_mb(cm, mi, do_left_v, do_above_h,
             do_left_v_mbuv, do_above_h_mbuv,
             y_ptr + 16 * y_stride,
             y_only ? NULL : (u_ptr + 8 * uv_stride),
             y_only ? NULL : (v_ptr + 8 * uv_stride),
Paul Wilkins's avatar
Paul Wilkins committed
508
             y_stride, uv_stride);
Yaowu Xu's avatar
Yaowu Xu committed
509 510 511
      y_ptr += 16;
      u_ptr = y_only? 0 : u_ptr + 8;
      v_ptr = y_only? 0 : v_ptr + 8;
Jingning Han's avatar
Jingning Han committed
512
      mode_info_context += 2;       /* step to next MB */
Yaowu Xu's avatar
Yaowu Xu committed
513 514 515 516 517 518
    }
    // move pointers to the beginning of next sb64 row
    y_ptr += y_stride * 32 - post->y_width;
    u_ptr += y_only? 0 : uv_stride *  16 - post->uv_width;
    v_ptr += y_only? 0 : uv_stride *  16 - post->uv_width;
    // skip to next MB row if exist
Jingning Han's avatar
Jingning Han committed
519
    mode_info_context += mis * 4 - cm->mi_cols;
Yaowu Xu's avatar
Yaowu Xu committed
520 521 522 523 524 525 526 527 528 529 530 531 532 533
    mb_row += 2;
  }
  if (extra_mb_row) {
    for (mb_col = 0; mb_col < cm->mb_cols; mb_col++) {
      const MODE_INFO *mi = mode_info_context;
      do_left_v =  (mb_col > 0);
      do_above_h = (mb_row > 0);
      do_left_v_mbuv = 1;
      do_above_h_mbuv = 1;
      lpf_mb(cm, mi, do_left_v, do_above_h,
             do_left_v_mbuv, do_above_h_mbuv,
             y_ptr,
             y_only? 0 : u_ptr,
             y_only? 0 : v_ptr,
Paul Wilkins's avatar
Paul Wilkins committed
534
             y_stride, uv_stride);
Yaowu Xu's avatar
Yaowu Xu committed
535 536 537
      y_ptr += 16;
      u_ptr = y_only? 0 : u_ptr + 8;
      v_ptr = y_only? 0 : v_ptr + 8;
Jingning Han's avatar
Jingning Han committed
538
      mode_info_context += 2;     // step to next MB
539
    }
John Koleszar's avatar
John Koleszar committed
540
  }
John Koleszar's avatar
John Koleszar committed
541
}
542 543 544 545 546 547 548 549

#else
static int build_lfi(const VP9_COMMON *cm, const MB_MODE_INFO *mbmi,
                      struct loop_filter_info *lfi) {
  const loop_filter_info_n *lfi_n = &cm->lf_info;
  int mode = mbmi->mode;
  int mode_index = lfi_n->mode_lf_lut[mode];
  int seg = mbmi->segment_id;
Yaowu Xu's avatar
Yaowu Xu committed
550
  int ref_frame = mbmi->ref_frame[0];
551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773
  int filter_level = lfi_n->lvl[seg][ref_frame][mode_index];

  if (filter_level) {
    const int hev_index = filter_level >> 4;
    lfi->mblim = lfi_n->mblim[filter_level];
    lfi->blim = lfi_n->blim[filter_level];
    lfi->lim = lfi_n->lim[filter_level];
    lfi->hev_thr = lfi_n->hev_thr[hev_index];
    return 1;
  }
  return 0;
}

static void filter_selectively_vert(uint8_t *s, int pitch,
                                    unsigned int mask_16x16,
                                    unsigned int mask_8x8,
                                    unsigned int mask_4x4,
                                    unsigned int mask_4x4_1,
                                    const struct loop_filter_info *lfi) {
  unsigned int mask;

  for (mask = mask_16x16 | mask_8x8 | mask_4x4; mask; mask >>= 1) {
    if (mask & 1) {
      if (mask_16x16 & 1) {
        vp9_mb_lpf_vertical_edge_w(s, pitch, lfi->mblim, lfi->lim,
                                   lfi->hev_thr, 1);
        assert(!(mask_8x8 & 1));
        assert(!(mask_4x4 & 1));
        assert(!(mask_4x4_1 & 1));
      } else if (mask_8x8 & 1) {
        vp9_mbloop_filter_vertical_edge(s, pitch, lfi->mblim, lfi->lim,
                                        lfi->hev_thr, 1);
        assert(!(mask_16x16 & 1));
        assert(!(mask_4x4 & 1));
      } else if (mask_4x4 & 1) {
        vp9_loop_filter_vertical_edge(s, pitch, lfi->mblim, lfi->lim,
                                      lfi->hev_thr, 1);
        assert(!(mask_16x16 & 1));
        assert(!(mask_8x8 & 1));
      } else {
        assert(0);
      }

      if (mask_4x4_1 & 1)
        vp9_loop_filter_vertical_edge(s + 4, pitch, lfi->mblim, lfi->lim,
                                      lfi->hev_thr, 1);
    }
    s += 8;
    lfi++;
    mask_16x16 >>= 1;
    mask_8x8 >>= 1;
    mask_4x4 >>= 1;
    mask_4x4_1 >>= 1;
  }
}

static void filter_selectively_horiz(uint8_t *s, int pitch,
                                     unsigned int mask_16x16,
                                     unsigned int mask_8x8,
                                     unsigned int mask_4x4,
                                     unsigned int mask_4x4_1,
                                     int only_4x4_1,
                                     const struct loop_filter_info *lfi) {
  unsigned int mask;

  for (mask = mask_16x16 | mask_8x8 | mask_4x4; mask; mask >>= 1) {
    if (mask & 1) {
      if (!only_4x4_1) {
        if (mask_16x16 & 1) {
          vp9_mb_lpf_horizontal_edge_w(s, pitch, lfi->mblim, lfi->lim,
                                       lfi->hev_thr, 1);
          assert(!(mask_8x8 & 1));
          assert(!(mask_4x4 & 1));
          assert(!(mask_4x4_1 & 1));
        } else if (mask_8x8 & 1) {
          vp9_mbloop_filter_horizontal_edge(s, pitch, lfi->mblim, lfi->lim,
                                            lfi->hev_thr, 1);
          assert(!(mask_16x16 & 1));
          assert(!(mask_4x4 & 1));
        } else if (mask_4x4 & 1) {
          vp9_loop_filter_horizontal_edge(s, pitch, lfi->mblim, lfi->lim,
                                          lfi->hev_thr, 1);
          assert(!(mask_16x16 & 1));
          assert(!(mask_8x8 & 1));
        } else {
          assert(0);
        }
      }

      if (mask_4x4_1 & 1)
        vp9_loop_filter_horizontal_edge(s + 4 * pitch, pitch, lfi->mblim,
                                        lfi->lim, lfi->hev_thr, 1);
    }
    s += 8;
    lfi++;
    mask_16x16 >>= 1;
    mask_8x8 >>= 1;
    mask_4x4 >>= 1;
    mask_4x4_1 >>= 1;
  }
}

static void filter_block_plane(VP9_COMMON *cm, MACROBLOCKD *xd,
                               int plane, int mi_row, int mi_col) {
  const int ss_x = xd->plane[plane].subsampling_x;
  const int row_step = 1 << xd->plane[plane].subsampling_y;
  const int col_step = 1 << xd->plane[plane].subsampling_x;
  struct buf_2d * const dst = &xd->plane[plane].dst;
  uint8_t* const dst0 = dst->buf;
  MODE_INFO* const mi0 = xd->mode_info_context;
  unsigned int mask_16x16[64 / MI_SIZE] = {0};
  unsigned int mask_8x8[64 / MI_SIZE] = {0};
  unsigned int mask_4x4[64 / MI_SIZE] = {0};
  unsigned int mask_4x4_1[64 / MI_SIZE] = {0};
  struct loop_filter_info lfi[64 / MI_SIZE][64 / MI_SIZE];
  int r, c;

  for (r = 0; r < 64 / MI_SIZE && mi_row + r < cm->mi_rows; r += row_step) {
    unsigned int mask_16x16_c = 0;
    unsigned int mask_8x8_c = 0;
    unsigned int mask_4x4_c = 0;
    unsigned int border_mask;

    // Determine the vertical edges that need filtering
    for (c = 0; c < 64 / MI_SIZE && mi_col + c < cm->mi_cols; c += col_step) {
      const MODE_INFO const *mi = xd->mode_info_context;
      const MODE_INFO const *mi_above = xd->mode_info_context -
          cm->mode_info_stride;
      const int skip_above =
          (r + mi_row > 0) ? mi_above[c].mbmi.mb_skip_coeff : 0;
      const int skip_left =
          (c + mi_col > 0) ? mi[c - 1].mbmi.mb_skip_coeff : 0;
      const int skip_this = mi[c].mbmi.mb_skip_coeff;
      const int skip_this_c = skip_this && skip_left;
      const int skip_this_r = skip_this && skip_above;
      const TX_SIZE tx_size = plane ? get_uv_tx_size(xd) : mi[c].mbmi.txfm_size;

      // Filter level can vary per MI
      if (!build_lfi(cm, &mi[c].mbmi,
                     lfi[r] + (c >> xd->plane[plane].subsampling_x)))
        continue;

      // Build masks based on the transform size of each block
      if (tx_size == TX_32X32) {
        if (!skip_this_c && (c & 3) == 0)
          mask_16x16_c |= 1 << (c >> ss_x);
        if (!skip_this_r && (r & 3) == 0)
          mask_16x16[r] |= 1 << (c >> ss_x);
      } else if (tx_size == TX_16X16) {
        if (!skip_this_c && (c & 1) == 0)
          mask_16x16_c |= 1 << (c >> ss_x);
        if (!skip_this_r && (r & 1) == 0)
          mask_16x16[r] |= 1 << (c >> ss_x);
      } else {
        // force 8x8 filtering on 32x32 boundaries
        if (!skip_this_c) {
          if (tx_size == TX_8X8 || (c & 3) == 0)
            mask_8x8_c |= 1 << (c >> ss_x);
          else
            mask_4x4_c |= 1 << (c >> ss_x);
        }

        if (!skip_this_r) {
          if (tx_size == TX_8X8 || (r & 3) == 0)
            mask_8x8[r] |= 1 << (c >> ss_x);
          else
            mask_4x4[r] |= 1 << (c >> ss_x);
        }

        if (!skip_this && tx_size < TX_8X8)
          mask_4x4_1[r] |= 1 << (c >> ss_x);
      }
    }

    // Disable filtering on the leftmost column
    border_mask = ~(mi_col == 0);
    filter_selectively_vert(dst->buf, dst->stride,
                            mask_16x16_c & border_mask,
                            mask_8x8_c & border_mask,
                            mask_4x4_c & border_mask,
                            mask_4x4_1[r], lfi[r]);
    dst->buf += 8 * dst->stride;
    xd->mode_info_context += cm->mode_info_stride * row_step;
  }

  // Now do horizontal pass
  dst->buf = dst0;
  xd->mode_info_context = mi0;
  for (r = 0; r < 64 / MI_SIZE && mi_row + r < cm->mi_rows; r += row_step) {
    filter_selectively_horiz(dst->buf, dst->stride,
                             mask_16x16[r],
                             mask_8x8[r],
                             mask_4x4[r],
                             mask_4x4_1[r], mi_row + r == 0, lfi[r]);
    dst->buf += 8 * dst->stride;
    xd->mode_info_context += cm->mode_info_stride * row_step;
  }
}

void vp9_loop_filter_frame(VP9_COMMON *cm,
                           MACROBLOCKD *xd,
                           int frame_filter_level,
                           int y_only) {
  int mi_row, mi_col;

  // Initialize the loop filter for this frame.
  vp9_loop_filter_frame_init(cm, xd, frame_filter_level);

  for (mi_row = 0; mi_row < cm->mi_rows; mi_row += 64 / MI_SIZE) {
    MODE_INFO* const mi = cm->mi + mi_row * cm->mode_info_stride;

    for (mi_col = 0; mi_col < cm->mi_cols; mi_col += 64 / MI_SIZE) {
      int plane;

      setup_dst_planes(xd, cm->frame_to_show, mi_row, mi_col);
      for (plane = 0; plane < (y_only ? 1 : MAX_MB_PLANE); plane++) {
        xd->mode_info_context = mi + mi_col;
        filter_block_plane(cm, xd, plane, mi_row, mi_col);
      }
    }
  }
}
#endif