vp9_loopfilter.c 27.9 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
 */

Attila Nagy's avatar
Attila Nagy committed
11
#include "vpx_config.h"
12
13
#include "vp9/common/vp9_loopfilter.h"
#include "vp9/common/vp9_onyxc_int.h"
14
#include "vp9/common/vp9_reconinter.h"
Attila Nagy's avatar
Attila Nagy committed
15
#include "vpx_mem/vpx_mem.h"
John Koleszar's avatar
John Koleszar committed
16

17
#include "vp9/common/vp9_seg_common.h"
18

John Koleszar's avatar
John Koleszar committed
19
static void lf_init_lut(loop_filter_info_n *lfi) {
20
21
22
23
24
25
26
27
28
29
30
31
32
33
  lfi->mode_lf_lut[DC_PRED] = 0;
  lfi->mode_lf_lut[D45_PRED] = 0;
  lfi->mode_lf_lut[D135_PRED] = 0;
  lfi->mode_lf_lut[D117_PRED] = 0;
  lfi->mode_lf_lut[D153_PRED] = 0;
  lfi->mode_lf_lut[D27_PRED] = 0;
  lfi->mode_lf_lut[D63_PRED] = 0;
  lfi->mode_lf_lut[V_PRED] = 0;
  lfi->mode_lf_lut[H_PRED] = 0;
  lfi->mode_lf_lut[TM_PRED] = 0;
  lfi->mode_lf_lut[ZEROMV]  = 0;
  lfi->mode_lf_lut[NEARESTMV] = 1;
  lfi->mode_lf_lut[NEARMV] = 1;
  lfi->mode_lf_lut[NEWMV] = 1;
Attila Nagy's avatar
Attila Nagy committed
34
35
}

36
void vp9_loop_filter_update_sharpness(loop_filter_info_n *lfi,
John Koleszar's avatar
John Koleszar committed
37
38
39
40
41
42
43
                                      int sharpness_lvl) {
  int i;

  /* For each possible value for the loop filter fill out limits */
  for (i = 0; i <= MAX_LOOP_FILTER; i++) {
    int filt_lvl = i;
    int block_inside_limit = 0;
John Koleszar's avatar
John Koleszar committed
44

John Koleszar's avatar
John Koleszar committed
45
46
47
    /* Set loop filter paramaeters that control sharpness. */
    block_inside_limit = filt_lvl >> (sharpness_lvl > 0);
    block_inside_limit = block_inside_limit >> (sharpness_lvl > 4);
John Koleszar's avatar
John Koleszar committed
48

John Koleszar's avatar
John Koleszar committed
49
50
51
    if (sharpness_lvl > 0) {
      if (block_inside_limit > (9 - sharpness_lvl))
        block_inside_limit = (9 - sharpness_lvl);
John Koleszar's avatar
John Koleszar committed
52
    }
John Koleszar's avatar
John Koleszar committed
53
54
55
56
57
58
59
60
61
62

    if (block_inside_limit < 1)
      block_inside_limit = 1;

    vpx_memset(lfi->lim[i], block_inside_limit, SIMD_WIDTH);
    vpx_memset(lfi->blim[i], (2 * filt_lvl + block_inside_limit),
               SIMD_WIDTH);
    vpx_memset(lfi->mblim[i], (2 * (filt_lvl + 2) + block_inside_limit),
               SIMD_WIDTH);
  }
Attila Nagy's avatar
Attila Nagy committed
63
}
John Koleszar's avatar
John Koleszar committed
64

65
void vp9_loop_filter_init(VP9_COMMON *cm) {
John Koleszar's avatar
John Koleszar committed
66
67
  loop_filter_info_n *lfi = &cm->lf_info;
  int i;
Attila Nagy's avatar
Attila Nagy committed
68

69
  // init limits for given sharpness
70
  vp9_loop_filter_update_sharpness(lfi, cm->sharpness_level);
John Koleszar's avatar
John Koleszar committed
71
  cm->last_sharpness_level = cm->sharpness_level;
Attila Nagy's avatar
Attila Nagy committed
72

73
  // init LUT for lvl  and hev thr picking
John Koleszar's avatar
John Koleszar committed
74
  lf_init_lut(lfi);
Attila Nagy's avatar
Attila Nagy committed
75

76
77
  // init hev threshold const vectors
  for (i = 0; i < 4; i++)
John Koleszar's avatar
John Koleszar committed
78
    vpx_memset(lfi->hev_thr[i], i, SIMD_WIDTH);
John Koleszar's avatar
John Koleszar committed
79
80
}

81
void vp9_loop_filter_frame_init(VP9_COMMON *cm,
82
                                MACROBLOCKD *xd,
John Koleszar's avatar
John Koleszar committed
83
                                int default_filt_lvl) {
84
85
86
87
88
89
90
  int seg,    // segment number
      ref,    // index in ref_lf_deltas
      mode;   // index in mode_lf_deltas
  // n_shift is the a multiplier for lf_deltas
  // the multiplier is 1 for when filter_lvl is between 0 and 31;
  // 2 when filter_lvl is between 32 and 63
  int n_shift = default_filt_lvl >> 5;
John Koleszar's avatar
John Koleszar committed
91

John Koleszar's avatar
John Koleszar committed
92
  loop_filter_info_n *lfi = &cm->lf_info;
John Koleszar's avatar
John Koleszar committed
93

John Koleszar's avatar
John Koleszar committed
94
  /* update limits if sharpness has changed */
95
96
97
  // printf("vp9_loop_filter_frame_init %d\n", default_filt_lvl);
  // printf("sharpness level: %d [%d]\n",
  //        cm->sharpness_level, cm->last_sharpness_level);
John Koleszar's avatar
John Koleszar committed
98
  if (cm->last_sharpness_level != cm->sharpness_level) {
99
    vp9_loop_filter_update_sharpness(lfi, cm->sharpness_level);
John Koleszar's avatar
John Koleszar committed
100
101
102
103
104
105
106
107
108
    cm->last_sharpness_level = cm->sharpness_level;
  }

  for (seg = 0; seg < MAX_MB_SEGMENTS; seg++) {
    int lvl_seg = default_filt_lvl;
    int lvl_ref, lvl_mode;


    // Set the baseline filter values for each segment
109
    if (vp9_segfeature_active(xd, seg, SEG_LVL_ALT_LF)) {
John Koleszar's avatar
John Koleszar committed
110
111
      /* Abs value */
      if (xd->mb_segment_abs_delta == SEGMENT_ABSDATA) {
112
        lvl_seg = vp9_get_segdata(xd, seg, SEG_LVL_ALT_LF);
John Koleszar's avatar
John Koleszar committed
113
      } else { /* Delta Value */
114
        lvl_seg += vp9_get_segdata(xd, seg, SEG_LVL_ALT_LF);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
115
        lvl_seg = clamp(lvl_seg, 0, 63);
John Koleszar's avatar
John Koleszar committed
116
117
      }
    }
John Koleszar's avatar
John Koleszar committed
118

John Koleszar's avatar
John Koleszar committed
119
120
121
122
123
124
125
    if (!xd->mode_ref_lf_delta_enabled) {
      /* we could get rid of this if we assume that deltas are set to
       * zero when not in use; encoder always uses deltas
       */
      vpx_memset(lfi->lvl[seg][0], lvl_seg, 4 * 4);
      continue;
    }
John Koleszar's avatar
John Koleszar committed
126

John Koleszar's avatar
John Koleszar committed
127
    lvl_ref = lvl_seg;
John Koleszar's avatar
John Koleszar committed
128

John Koleszar's avatar
John Koleszar committed
129
130
    /* INTRA_FRAME */
    ref = INTRA_FRAME;
John Koleszar's avatar
John Koleszar committed
131

John Koleszar's avatar
John Koleszar committed
132
    /* Apply delta for reference frame */
133
    lvl_ref += xd->ref_lf_deltas[ref] << n_shift;
Attila Nagy's avatar
Attila Nagy committed
134

135
    mode = 0; /* all the rest of Intra modes */
136
137
    lvl_mode = lvl_ref;
    lfi->lvl[seg][ref][mode] = clamp(lvl_mode, 0, 63);
John Koleszar's avatar
John Koleszar committed
138

John Koleszar's avatar
John Koleszar committed
139
140
141
    /* LAST, GOLDEN, ALT */
    for (ref = 1; ref < MAX_REF_FRAMES; ref++) {
      int lvl_ref = lvl_seg;
John Koleszar's avatar
John Koleszar committed
142

John Koleszar's avatar
John Koleszar committed
143
      /* Apply delta for reference frame */
144
      lvl_ref += xd->ref_lf_deltas[ref] << n_shift;
John Koleszar's avatar
John Koleszar committed
145

John Koleszar's avatar
John Koleszar committed
146
      /* Apply delta for Inter modes */
147
      for (mode = 0; mode < MAX_MODE_LF_DELTAS; mode++) {
148
149
        lvl_mode = lvl_ref + (xd->mode_lf_deltas[mode] << n_shift);
        lfi->lvl[seg][ref][mode] = clamp(lvl_mode, 0, 63);
John Koleszar's avatar
John Koleszar committed
150
      }
John Koleszar's avatar
John Koleszar committed
151
    }
John Koleszar's avatar
John Koleszar committed
152
  }
John Koleszar's avatar
John Koleszar committed
153
154
}

155
#if !CONFIG_NEW_LOOPFILTER
Yaowu Xu's avatar
Yaowu Xu committed
156
157
158
159
// Determine if we should skip inner-MB loop filtering within a MB
// The current condition is that the loop filtering is skipped only
// the MB uses a prediction size of 16x16 and either 16x16 transform
// is used or there is no residue at all.
160
161
162
static int mb_lf_skip(const MB_MODE_INFO *const mbmi) {
  const int skip_coef = mbmi->mb_skip_coeff;
  const int tx_size = mbmi->txfm_size;
163
  return mbmi->sb_type >= BLOCK_SIZE_MB16X16 &&
Yaowu Xu's avatar
Yaowu Xu committed
164
         (tx_size >= TX_16X16 || skip_coef);
165
}
Yaowu Xu's avatar
Yaowu Xu committed
166
167
168
169
170

// Determine if we should skip MB loop filtering on a MB edge within
// a superblock, the current condition is that MB loop filtering is
// skipped only when both MBs do not use inner MB loop filtering, and
// same motion vector with same reference frame
171
172
static int sb_mb_lf_skip(const MODE_INFO *const mip0,
                         const MODE_INFO *const mip1) {
Yaowu Xu's avatar
Yaowu Xu committed
173
  const MB_MODE_INFO *mbmi0 = &mip0->mbmi;
Yaowu Xu's avatar
Yaowu Xu committed
174
  const MB_MODE_INFO *mbmi1 = &mip1->mbmi;
Yaowu Xu's avatar
Yaowu Xu committed
175
  return mb_lf_skip(mbmi0) && mb_lf_skip(mbmi1) &&
Ronald S. Bultje's avatar
Ronald S. Bultje committed
176
177
         mbmi0->ref_frame[0] != INTRA_FRAME &&
         mbmi1->ref_frame[0] != INTRA_FRAME;
178
}
179

Yaowu Xu's avatar
Yaowu Xu committed
180
181
182
static void lpf_mb(VP9_COMMON *cm, const MODE_INFO *mi,
                   int do_left_mb_v, int do_above_mb_h,
                   int do_left_mbuv_v, int do_above_mbuv_h,
183
                   int mb_row, int mb_col,
Yaowu Xu's avatar
Yaowu Xu committed
184
                   uint8_t *y_ptr, uint8_t *u_ptr, uint8_t *v_ptr,
Paul Wilkins's avatar
Paul Wilkins committed
185
                   int y_stride, int uv_stride) {
Yaowu Xu's avatar
Yaowu Xu committed
186
187
188
189
190
  loop_filter_info_n *lfi_n = &cm->lf_info;
  struct loop_filter_info lfi;
  int mode = mi->mbmi.mode;
  int mode_index = lfi_n->mode_lf_lut[mode];
  int seg = mi->mbmi.segment_id;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
191
  MV_REFERENCE_FRAME ref_frame = mi->mbmi.ref_frame[0];
Yaowu Xu's avatar
Yaowu Xu committed
192
193
194
195
196
  int filter_level = lfi_n->lvl[seg][ref_frame][mode_index];

  if (filter_level) {
    const int skip_lf = mb_lf_skip(&mi->mbmi);
    const int tx_size = mi->mbmi.txfm_size;
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
    const int hev_index = filter_level >> 4;
    lfi.mblim = lfi_n->mblim[filter_level];
    lfi.blim = lfi_n->blim[filter_level];
    lfi.lim = lfi_n->lim[filter_level];
    lfi.hev_thr = lfi_n->hev_thr[hev_index];

    if (do_above_mb_h) {
      if (tx_size >= TX_16X16)
        vp9_lpf_mbh_w(y_ptr,
                      do_above_mbuv_h ? u_ptr : NULL,
                      do_above_mbuv_h ? v_ptr : NULL,
                      y_stride, uv_stride, &lfi);
      else
        vp9_loop_filter_mbh(y_ptr, u_ptr, v_ptr, y_stride, uv_stride, &lfi);
    }

213
    if (!skip_lf && mb_row * 2 + 1 < cm->mi_rows) {
214
215
216
217
218
      if (tx_size >= TX_8X8) {
        if (tx_size == TX_8X8 &&
            mi->mbmi.sb_type < BLOCK_SIZE_MB16X16)
          vp9_loop_filter_bh8x8(y_ptr, u_ptr, v_ptr,
                                y_stride, uv_stride, &lfi);
Yaowu Xu's avatar
Yaowu Xu committed
219
        else
220
221
222
223
224
          vp9_loop_filter_bh8x8(y_ptr, NULL, NULL,
                                y_stride, uv_stride, &lfi);
      } else {
        vp9_loop_filter_bh(y_ptr, u_ptr, v_ptr,
                           y_stride, uv_stride, &lfi);
Yaowu Xu's avatar
Yaowu Xu committed
225
      }
226
    }
Yaowu Xu's avatar
Yaowu Xu committed
227

228
229
230
231
232
233
234
235
236
    if (do_left_mb_v) {
      if (tx_size >= TX_16X16)
        vp9_lpf_mbv_w(y_ptr,
                      do_left_mbuv_v ? u_ptr : NULL,
                      do_left_mbuv_v ? v_ptr : NULL,
                      y_stride, uv_stride, &lfi);
      else
        vp9_loop_filter_mbv(y_ptr, u_ptr, v_ptr, y_stride, uv_stride, &lfi);
    }
Yaowu Xu's avatar
Yaowu Xu committed
237

238
    if (!skip_lf && mb_col * 2 + 1 < cm->mi_cols) {
239
240
241
242
243
      if (tx_size >= TX_8X8) {
        if (tx_size == TX_8X8 &&
            mi->mbmi.sb_type < BLOCK_SIZE_MB16X16)
          vp9_loop_filter_bv8x8(y_ptr, u_ptr, v_ptr,
                                y_stride, uv_stride, &lfi);
Yaowu Xu's avatar
Yaowu Xu committed
244
        else
245
246
247
248
249
          vp9_loop_filter_bv8x8(y_ptr, NULL, NULL,
                                y_stride, uv_stride, &lfi);
      } else {
        vp9_loop_filter_bv(y_ptr, u_ptr, v_ptr,
                           y_stride, uv_stride, &lfi);
Yaowu Xu's avatar
Yaowu Xu committed
250
      }
251
    }
Yaowu Xu's avatar
Yaowu Xu committed
252
253
254
255
256
257
258
  }
}

static void lpf_sb32(VP9_COMMON *cm, const MODE_INFO *mode_info_context,
                     int mb_row, int mb_col,
                     uint8_t *y_ptr, uint8_t *u_ptr, uint8_t *v_ptr,
                     int y_stride, int uv_stride,
Paul Wilkins's avatar
Paul Wilkins committed
259
                     int y_only) {
260
261
262
263
264
265
266
267
268
269
270
271
272
  BLOCK_SIZE_TYPE sb_type = mode_info_context->mbmi.sb_type;
  const int wbl = b_width_log2(sb_type), hbl = b_height_log2(sb_type);
  TX_SIZE tx_size = mode_info_context->mbmi.txfm_size;
  int do_left_v, do_above_h;
  int do_left_v_mbuv, do_above_h_mbuv;
  int mis = cm->mode_info_stride;
  const MODE_INFO *mi;

  // process 1st MB top-left
  mi = mode_info_context;
  do_left_v = (mb_col > 0);
  do_above_h = (mb_row > 0);
  do_left_v_mbuv = !(sb_type >= BLOCK_SIZE_SB64X64 &&
Yaowu Xu's avatar
Yaowu Xu committed
273
      tx_size >= TX_32X32 && (mb_col & 2));
274
  do_above_h_mbuv = !(sb_type >= BLOCK_SIZE_SB64X64 &&
Yaowu Xu's avatar
Yaowu Xu committed
275
      tx_size >= TX_32X32 && (mb_row & 2));
276
  lpf_mb(cm, mi, do_left_v, do_above_h,
Yaowu Xu's avatar
Yaowu Xu committed
277
      do_left_v_mbuv, do_above_h_mbuv,
278
      mb_row, mb_col,
Yaowu Xu's avatar
Yaowu Xu committed
279
280
281
      y_ptr,
      y_only? 0 : u_ptr,
      y_only? 0 : v_ptr,
Paul Wilkins's avatar
Paul Wilkins committed
282
      y_stride, uv_stride);
283
  // process 2nd MB top-right
Jingning Han's avatar
Jingning Han committed
284
  mi = mode_info_context + 2;
285
  do_left_v = !(wbl >= 3 /* 32x16 or >=32x32 */ && (tx_size >= TX_32X32 ||
Yaowu Xu's avatar
Yaowu Xu committed
286
      sb_mb_lf_skip(mode_info_context, mi)));
287
  do_above_h = (mb_row > 0);
288
289
  do_left_v_mbuv = !(wbl >= 3 /* 32x16 or >=32x32 */ && (tx_size >= TX_16X16 ||
      sb_mb_lf_skip(mode_info_context, mi)));
290
  do_above_h_mbuv = !(sb_type >= BLOCK_SIZE_SB64X64 &&
Yaowu Xu's avatar
Yaowu Xu committed
291
      tx_size >= TX_32X32 && (mb_row & 2));
292
  lpf_mb(cm, mi, do_left_v, do_above_h,
Yaowu Xu's avatar
Yaowu Xu committed
293
      do_left_v_mbuv, do_above_h_mbuv,
294
      mb_row, mb_col + 1,
Yaowu Xu's avatar
Yaowu Xu committed
295
296
297
      y_ptr + 16,
      y_only ? 0 : (u_ptr + 8),
      y_only ? 0 : (v_ptr + 8),
Paul Wilkins's avatar
Paul Wilkins committed
298
      y_stride, uv_stride);
Yaowu Xu's avatar
Yaowu Xu committed
299

300
  // process 3rd MB bottom-left
Jingning Han's avatar
Jingning Han committed
301
  mi = mode_info_context + (mis << 1);
302
303
  do_left_v = (mb_col > 0);
  do_above_h = !(hbl >= 3 /* 16x32 or >=32x32 */ && (tx_size >= TX_32X32 ||
Yaowu Xu's avatar
Yaowu Xu committed
304
      sb_mb_lf_skip(mode_info_context, mi)));
305
  do_left_v_mbuv = !(sb_type >= BLOCK_SIZE_SB64X64 &&
Yaowu Xu's avatar
Yaowu Xu committed
306
      tx_size >= TX_32X32 && (mb_col & 2));
307
308
  do_above_h_mbuv = !(hbl >= 3 /* 16x32 or >=32x32 */ && (tx_size >= TX_16X16 ||
      sb_mb_lf_skip(mode_info_context, mi)));
309
  lpf_mb(cm, mi, do_left_v, do_above_h,
Yaowu Xu's avatar
Yaowu Xu committed
310
      do_left_v_mbuv, do_above_h_mbuv,
311
      mb_row + 1, mb_col,
Yaowu Xu's avatar
Yaowu Xu committed
312
313
314
      y_ptr + 16 * y_stride,
      y_only ? 0 : (u_ptr + 8 * uv_stride),
      y_only ? 0 : (v_ptr + 8 * uv_stride),
Paul Wilkins's avatar
Paul Wilkins committed
315
      y_stride, uv_stride);
Yaowu Xu's avatar
Yaowu Xu committed
316

317
  // process 4th MB bottom right
Jingning Han's avatar
Jingning Han committed
318
  mi = mode_info_context + ((mis + 1) << 1);
319
  do_left_v = !(wbl >= 3 /* 32x16 or >=32x32 */ && (tx_size >= TX_32X32 ||
Jingning Han's avatar
Jingning Han committed
320
      sb_mb_lf_skip(mi - 2, mi)));
321
  do_above_h = !(hbl >= 3 /* 16x32 or >=32x32 */ && (tx_size >= TX_32X32 ||
Jingning Han's avatar
Jingning Han committed
322
      sb_mb_lf_skip(mode_info_context + 2, mi)));
323
  do_left_v_mbuv = (wbl >= 3 /* 32x16 or >=32x32 */ && (tx_size >= TX_16X16 ||
Jingning Han's avatar
Jingning Han committed
324
      sb_mb_lf_skip(mi - 2, mi)));
325
  do_above_h_mbuv = !(hbl >= 3 /* 16x32 or >=32x32 */ && (tx_size >= TX_16X16 ||
Jingning Han's avatar
Jingning Han committed
326
      sb_mb_lf_skip(mode_info_context + 2, mi)));
327
  lpf_mb(cm, mi, do_left_v, do_above_h,
Yaowu Xu's avatar
Yaowu Xu committed
328
      do_left_v_mbuv, do_above_h_mbuv,
329
      mb_row + 1, mb_col + 1,
Yaowu Xu's avatar
Yaowu Xu committed
330
331
332
      y_ptr + 16 * y_stride + 16,
      y_only ? 0 : (u_ptr + 8 * uv_stride + 8),
      y_only ? 0 : (v_ptr + 8 * uv_stride + 8),
Paul Wilkins's avatar
Paul Wilkins committed
333
      y_stride, uv_stride);
Yaowu Xu's avatar
Yaowu Xu committed
334
335
336
337
338
339
}

static void lpf_sb64(VP9_COMMON *cm, const MODE_INFO *mode_info_context,
                     int mb_row, int mb_col,
                     uint8_t *y_ptr, uint8_t *u_ptr, uint8_t *v_ptr,
                     int y_stride, int uv_stride,
Paul Wilkins's avatar
Paul Wilkins committed
340
                     int y_only) {
341
  lpf_sb32(cm, mode_info_context, mb_row, mb_col,
Yaowu Xu's avatar
Yaowu Xu committed
342
      y_ptr, u_ptr, v_ptr,
Paul Wilkins's avatar
Paul Wilkins committed
343
      y_stride, uv_stride, y_only);
Jingning Han's avatar
Jingning Han committed
344
  lpf_sb32(cm, mode_info_context + 4, mb_row, mb_col + 2,
Yaowu Xu's avatar
Yaowu Xu committed
345
      y_ptr + 32, u_ptr + 16, v_ptr + 16,
Paul Wilkins's avatar
Paul Wilkins committed
346
      y_stride, uv_stride, y_only);
Jingning Han's avatar
Jingning Han committed
347
  lpf_sb32(cm, mode_info_context + cm->mode_info_stride * 4,
Yaowu Xu's avatar
Yaowu Xu committed
348
349
350
351
      mb_row + 2, mb_col,
      y_ptr + 32 * y_stride,
      u_ptr + 16 * uv_stride,
      v_ptr + 16 * uv_stride,
Paul Wilkins's avatar
Paul Wilkins committed
352
      y_stride, uv_stride, y_only);
Jingning Han's avatar
Jingning Han committed
353
  lpf_sb32(cm, mode_info_context + cm->mode_info_stride * 4 + 4,
Yaowu Xu's avatar
Yaowu Xu committed
354
355
356
357
      mb_row + 2, mb_col + 2,
      y_ptr + 32 * y_stride + 32,
      u_ptr + 16 * uv_stride + 16,
      v_ptr + 16 * uv_stride + 16,
Paul Wilkins's avatar
Paul Wilkins committed
358
      y_stride, uv_stride, y_only);
Yaowu Xu's avatar
Yaowu Xu committed
359
}
360

361
362
363
void vp9_loop_filter_frame(VP9_COMMON *cm,
                           MACROBLOCKD *xd,
                           int frame_filter_level,
Paul Wilkins's avatar
Paul Wilkins committed
364
                           int y_only) {
John Koleszar's avatar
John Koleszar committed
365
  YV12_BUFFER_CONFIG *post = cm->frame_to_show;
366
  int mb_row, mb_col;
Yaowu Xu's avatar
Yaowu Xu committed
367
368
369
370
371
372
  const int sb64_rows = cm->mb_rows / 4;
  const int sb64_cols = cm->mb_cols / 4;
  const int extra_sb32_row = (cm->mb_rows & 2) != 0;
  const int extra_sb32_col = (cm->mb_cols & 2) != 0;
  const int extra_mb_col = cm->mb_cols & 1;
  const int extra_mb_row = cm->mb_rows & 1;
373
374
375
376
377
378
  // Set up the buffer pointers
  uint8_t *y_ptr = post->y_buffer;
  uint8_t *u_ptr = y_only ? 0 : post->u_buffer;
  uint8_t *v_ptr = y_only ? 0 : post->v_buffer;

  // Point at base of Mb MODE_INFO list
John Koleszar's avatar
John Koleszar committed
379
  const MODE_INFO *mode_info_context = cm->mi;
Yaowu Xu's avatar
Yaowu Xu committed
380
  const MODE_INFO *mi;
381
  const int mis = cm->mode_info_stride;
Yaowu Xu's avatar
Yaowu Xu committed
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
  const int y_stride = post->y_stride;
  const int uv_stride = post->uv_stride;
  // These two flags signal if MB left edge and above edge
  // should be filtered using MB edge filter. Currently, MB
  // edge filtering is not applied on MB edge internal to a
  // 32x32 superblock if:
  // 1) SB32 is using 32x32 prediction and 32x32 transform
  // 2) SB32 is using 32x32 prediction and 16x16 transform
  //    but all coefficients are zero.
  // MB edges are on 32x32 superblock boundary are always
  // filtered except on image frame boundary.
  int do_left_v, do_above_h;
  // These two flags signal if MB UV left edge and above edge
  // should be filtered using MB edge filter. Currently, MB
  // edge filtering is not applied for MB edges internal to
  // a 32x32 superblock if:
  // 1) SB32 is using 32x32 prediction and 32x32 transform
  // 2) SB32 is using 32x32 prediction and 16x16 transform
  //    but all coefficients are zero.
  // 3) SB32 UV edges internal to a SB64 and 32x32 transform
  //    is used, i.e. UV is doing 32x32 transform hence no
  //    transform boundary exists inside the SB64 for UV
  int do_left_v_mbuv, do_above_h_mbuv;
John Koleszar's avatar
John Koleszar committed
405

406
  // Initialize the loop filter for this frame.
407
  vp9_loop_filter_frame_init(cm, xd, frame_filter_level);
John Koleszar's avatar
John Koleszar committed
408

Yaowu Xu's avatar
Yaowu Xu committed
409
410
411
412
413
414
415
416
417
  // vp9_filter each 64x64 SB
  // For each SB64: the 4 SB32 are filtered in raster scan order
  // For each SB32: the 4 MBs are filtered in raster scan order
  // For each MB:   the left and above MB edges as well as the
  //                internal block edges are processed together
  for (mb_row = 0; mb_row < sb64_rows * 4; mb_row += 4) {
    for (mb_col = 0; mb_col < sb64_cols * 4; mb_col += 4) {
      lpf_sb64(cm, mode_info_context, mb_row, mb_col,
               y_ptr, u_ptr, v_ptr,
Paul Wilkins's avatar
Paul Wilkins committed
418
               y_stride, uv_stride, y_only);
Yaowu Xu's avatar
Yaowu Xu committed
419
420
421
      y_ptr += 64;
      u_ptr = y_only? 0 : u_ptr + 32;
      v_ptr = y_only? 0 : v_ptr + 32;
Jingning Han's avatar
Jingning Han committed
422
      mode_info_context += 8;       // step to next SB64
Yaowu Xu's avatar
Yaowu Xu committed
423
424
425
426
427
    }
    if (extra_sb32_col) {
      // process 2 SB32s in the extra SB32 col
      lpf_sb32(cm, mode_info_context, mb_row, mb_col,
               y_ptr, u_ptr, v_ptr,
Paul Wilkins's avatar
Paul Wilkins committed
428
               y_stride, uv_stride, y_only);
Jingning Han's avatar
Jingning Han committed
429
      lpf_sb32(cm, mode_info_context + mis * 4,
Yaowu Xu's avatar
Yaowu Xu committed
430
431
432
433
               mb_row + 2, mb_col,
               y_ptr + 32 * y_stride,
               u_ptr + 16 * uv_stride,
               v_ptr + 16 * uv_stride,
Paul Wilkins's avatar
Paul Wilkins committed
434
               y_stride, uv_stride, y_only);
Yaowu Xu's avatar
Yaowu Xu committed
435
436
437
      y_ptr += 32;
      u_ptr = y_only? 0 : u_ptr + 16;
      v_ptr = y_only? 0 : v_ptr + 16;
Jingning Han's avatar
Jingning Han committed
438
      mode_info_context += 4;       // step to next SB32
Yaowu Xu's avatar
Yaowu Xu committed
439
440
441
442
      mb_col += 2;
    }
    if (extra_mb_col) {
      // process 4 MB in the extra MB col
443
444
445
446
447
448
449
450
451
      int k;
      for (k = 0; k < 4; ++k) {
        mi = mode_info_context + (mis << 1) * k;
        do_left_v = (mb_col > 0);
        do_above_h = k == 0 ? mb_row > 0 : 1;
        do_left_v_mbuv =  1;
        do_above_h_mbuv = 1;
        lpf_mb(cm, mi, do_left_v, do_above_h,
               do_left_v_mbuv, do_above_h_mbuv,
452
               mb_row + k, mb_col,
453
454
455
               y_ptr + (k * 16) * y_stride,
               y_only ? 0 : (u_ptr + (k * 8) * uv_stride),
               y_only ? 0 : (v_ptr + (k * 8) * uv_stride),
Paul Wilkins's avatar
Paul Wilkins committed
456
               y_stride, uv_stride);
457
458
      }

John Koleszar's avatar
John Koleszar committed
459
      y_ptr += 16;
Yaowu Xu's avatar
Yaowu Xu committed
460
461
      u_ptr = y_only? 0 : u_ptr + 8;
      v_ptr = y_only? 0 : v_ptr + 8;
Jingning Han's avatar
Jingning Han committed
462
      mode_info_context += 2;       // step to next MB
John Koleszar's avatar
John Koleszar committed
463
    }
Yaowu Xu's avatar
Yaowu Xu committed
464
    // move pointers to the begining of next sb64 row
465
    y_ptr += y_stride  * 64 - cm->mb_cols * 16;
466
    if (!y_only) {
467
468
      u_ptr += uv_stride *  32 - cm->mb_cols * 8;
      v_ptr += uv_stride *  32 - cm->mb_cols * 8;
Yaowu Xu's avatar
Yaowu Xu committed
469
470
    }
    /* skip to next SB64 row */
471
    mode_info_context += mis * 8 - cm->mb_cols * 2;
Yaowu Xu's avatar
Yaowu Xu committed
472
473
474
475
476
477
  }
  if (extra_sb32_row) {
    const int sb32_cols = sb64_cols * 2 + extra_sb32_col;
    for (mb_col = 0; mb_col < sb32_cols * 2; mb_col += 2) {
      lpf_sb32(cm, mode_info_context, mb_row, mb_col,
               y_ptr, u_ptr, v_ptr,
Paul Wilkins's avatar
Paul Wilkins committed
478
               y_stride, uv_stride, y_only);
Yaowu Xu's avatar
Yaowu Xu committed
479
480
481
      y_ptr += 32;
      u_ptr = y_only? 0 : u_ptr + 16;
      v_ptr = y_only? 0 : v_ptr + 16;
Jingning Han's avatar
Jingning Han committed
482
      mode_info_context += 4;       // step to next SB32
Yaowu Xu's avatar
Yaowu Xu committed
483
484
485
486
487
488
489
490
491
492
    }
    if (extra_mb_col) {
      // process 1st MB
      mi = mode_info_context;
      do_left_v = (mb_col > 0);
      do_above_h = (mb_row > 0);
      do_left_v_mbuv =  1;
      do_above_h_mbuv = 1;
      lpf_mb(cm, mi, do_left_v, do_above_h,
             do_left_v_mbuv, do_above_h_mbuv,
493
             mb_row, mb_col,
Yaowu Xu's avatar
Yaowu Xu committed
494
495
496
             y_ptr,
             y_only? NULL : u_ptr,
             y_only? NULL : v_ptr,
Paul Wilkins's avatar
Paul Wilkins committed
497
             y_stride, uv_stride);
Yaowu Xu's avatar
Yaowu Xu committed
498
      // process 2nd MB
Jingning Han's avatar
Jingning Han committed
499
      mi = mode_info_context + (mis << 1);
Yaowu Xu's avatar
Yaowu Xu committed
500
501
502
503
504
505
      do_left_v = (mb_col > 0);
      do_above_h = 1;
      do_left_v_mbuv =  1;
      do_above_h_mbuv = 1;
      lpf_mb(cm, mi, do_left_v, do_above_h,
             do_left_v_mbuv, do_above_h_mbuv,
506
             mb_row + 1, mb_col,
Yaowu Xu's avatar
Yaowu Xu committed
507
508
509
             y_ptr + 16 * y_stride,
             y_only ? NULL : (u_ptr + 8 * uv_stride),
             y_only ? NULL : (v_ptr + 8 * uv_stride),
Paul Wilkins's avatar
Paul Wilkins committed
510
             y_stride, uv_stride);
Yaowu Xu's avatar
Yaowu Xu committed
511
512
513
      y_ptr += 16;
      u_ptr = y_only? 0 : u_ptr + 8;
      v_ptr = y_only? 0 : v_ptr + 8;
Jingning Han's avatar
Jingning Han committed
514
      mode_info_context += 2;       /* step to next MB */
Yaowu Xu's avatar
Yaowu Xu committed
515
516
    }
    // move pointers to the beginning of next sb64 row
517
518
519
    y_ptr += y_stride * 32 - cm->mb_cols * 16;
    u_ptr += y_only? 0 : uv_stride *  16 - cm->mb_cols * 8;
    v_ptr += y_only? 0 : uv_stride *  16 - cm->mb_cols * 8;
Yaowu Xu's avatar
Yaowu Xu committed
520
    // skip to next MB row if exist
521
    mode_info_context += mis * 4 - cm->mb_cols * 2;
Yaowu Xu's avatar
Yaowu Xu committed
522
523
524
525
526
527
528
529
530
531
532
    mb_row += 2;
  }
  if (extra_mb_row) {
    for (mb_col = 0; mb_col < cm->mb_cols; mb_col++) {
      const MODE_INFO *mi = mode_info_context;
      do_left_v =  (mb_col > 0);
      do_above_h = (mb_row > 0);
      do_left_v_mbuv = 1;
      do_above_h_mbuv = 1;
      lpf_mb(cm, mi, do_left_v, do_above_h,
             do_left_v_mbuv, do_above_h_mbuv,
533
             mb_row, mb_col,
Yaowu Xu's avatar
Yaowu Xu committed
534
535
536
             y_ptr,
             y_only? 0 : u_ptr,
             y_only? 0 : v_ptr,
Paul Wilkins's avatar
Paul Wilkins committed
537
             y_stride, uv_stride);
Yaowu Xu's avatar
Yaowu Xu committed
538
539
540
      y_ptr += 16;
      u_ptr = y_only? 0 : u_ptr + 8;
      v_ptr = y_only? 0 : v_ptr + 8;
Jingning Han's avatar
Jingning Han committed
541
      mode_info_context += 2;     // step to next MB
542
    }
John Koleszar's avatar
John Koleszar committed
543
  }
John Koleszar's avatar
John Koleszar committed
544
}
545
546
547
548
549
550
551
552

#else
static int build_lfi(const VP9_COMMON *cm, const MB_MODE_INFO *mbmi,
                      struct loop_filter_info *lfi) {
  const loop_filter_info_n *lfi_n = &cm->lf_info;
  int mode = mbmi->mode;
  int mode_index = lfi_n->mode_lf_lut[mode];
  int seg = mbmi->segment_id;
Yaowu Xu's avatar
Yaowu Xu committed
553
  int ref_frame = mbmi->ref_frame[0];
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
  int filter_level = lfi_n->lvl[seg][ref_frame][mode_index];

  if (filter_level) {
    const int hev_index = filter_level >> 4;
    lfi->mblim = lfi_n->mblim[filter_level];
    lfi->blim = lfi_n->blim[filter_level];
    lfi->lim = lfi_n->lim[filter_level];
    lfi->hev_thr = lfi_n->hev_thr[hev_index];
    return 1;
  }
  return 0;
}

static void filter_selectively_vert(uint8_t *s, int pitch,
                                    unsigned int mask_16x16,
                                    unsigned int mask_8x8,
                                    unsigned int mask_4x4,
                                    unsigned int mask_4x4_1,
                                    const struct loop_filter_info *lfi) {
  unsigned int mask;

  for (mask = mask_16x16 | mask_8x8 | mask_4x4; mask; mask >>= 1) {
    if (mask & 1) {
      if (mask_16x16 & 1) {
        vp9_mb_lpf_vertical_edge_w(s, pitch, lfi->mblim, lfi->lim,
                                   lfi->hev_thr, 1);
        assert(!(mask_8x8 & 1));
        assert(!(mask_4x4 & 1));
        assert(!(mask_4x4_1 & 1));
      } else if (mask_8x8 & 1) {
        vp9_mbloop_filter_vertical_edge(s, pitch, lfi->mblim, lfi->lim,
                                        lfi->hev_thr, 1);
        assert(!(mask_16x16 & 1));
        assert(!(mask_4x4 & 1));
      } else if (mask_4x4 & 1) {
        vp9_loop_filter_vertical_edge(s, pitch, lfi->mblim, lfi->lim,
                                      lfi->hev_thr, 1);
        assert(!(mask_16x16 & 1));
        assert(!(mask_8x8 & 1));
      } else {
        assert(0);
      }

      if (mask_4x4_1 & 1)
        vp9_loop_filter_vertical_edge(s + 4, pitch, lfi->mblim, lfi->lim,
                                      lfi->hev_thr, 1);
    }
    s += 8;
    lfi++;
    mask_16x16 >>= 1;
    mask_8x8 >>= 1;
    mask_4x4 >>= 1;
    mask_4x4_1 >>= 1;
  }
}

static void filter_selectively_horiz(uint8_t *s, int pitch,
                                     unsigned int mask_16x16,
                                     unsigned int mask_8x8,
                                     unsigned int mask_4x4,
                                     unsigned int mask_4x4_1,
                                     int only_4x4_1,
                                     const struct loop_filter_info *lfi) {
  unsigned int mask;

  for (mask = mask_16x16 | mask_8x8 | mask_4x4; mask; mask >>= 1) {
    if (mask & 1) {
      if (!only_4x4_1) {
        if (mask_16x16 & 1) {
          vp9_mb_lpf_horizontal_edge_w(s, pitch, lfi->mblim, lfi->lim,
                                       lfi->hev_thr, 1);
          assert(!(mask_8x8 & 1));
          assert(!(mask_4x4 & 1));
          assert(!(mask_4x4_1 & 1));
        } else if (mask_8x8 & 1) {
          vp9_mbloop_filter_horizontal_edge(s, pitch, lfi->mblim, lfi->lim,
                                            lfi->hev_thr, 1);
          assert(!(mask_16x16 & 1));
          assert(!(mask_4x4 & 1));
        } else if (mask_4x4 & 1) {
          vp9_loop_filter_horizontal_edge(s, pitch, lfi->mblim, lfi->lim,
                                          lfi->hev_thr, 1);
          assert(!(mask_16x16 & 1));
          assert(!(mask_8x8 & 1));
        } else {
          assert(0);
        }
      }

      if (mask_4x4_1 & 1)
        vp9_loop_filter_horizontal_edge(s + 4 * pitch, pitch, lfi->mblim,
                                        lfi->lim, lfi->hev_thr, 1);
    }
    s += 8;
    lfi++;
    mask_16x16 >>= 1;
    mask_8x8 >>= 1;
    mask_4x4 >>= 1;
    mask_4x4_1 >>= 1;
  }
}

static void filter_block_plane(VP9_COMMON *cm, MACROBLOCKD *xd,
                               int plane, int mi_row, int mi_col) {
  const int ss_x = xd->plane[plane].subsampling_x;
  const int row_step = 1 << xd->plane[plane].subsampling_y;
  const int col_step = 1 << xd->plane[plane].subsampling_x;
  struct buf_2d * const dst = &xd->plane[plane].dst;
  uint8_t* const dst0 = dst->buf;
  MODE_INFO* const mi0 = xd->mode_info_context;
  unsigned int mask_16x16[64 / MI_SIZE] = {0};
  unsigned int mask_8x8[64 / MI_SIZE] = {0};
  unsigned int mask_4x4[64 / MI_SIZE] = {0};
  unsigned int mask_4x4_1[64 / MI_SIZE] = {0};
  struct loop_filter_info lfi[64 / MI_SIZE][64 / MI_SIZE];
  int r, c;

  for (r = 0; r < 64 / MI_SIZE && mi_row + r < cm->mi_rows; r += row_step) {
    unsigned int mask_16x16_c = 0;
    unsigned int mask_8x8_c = 0;
    unsigned int mask_4x4_c = 0;
    unsigned int border_mask;

    // Determine the vertical edges that need filtering
    for (c = 0; c < 64 / MI_SIZE && mi_col + c < cm->mi_cols; c += col_step) {
      const MODE_INFO const *mi = xd->mode_info_context;
      const int skip_this = mi[c].mbmi.mb_skip_coeff;
681
682
683
      // left edge of current unit is block/partition edge -> no skip
      const int block_edge_left = b_width_log2(mi->mbmi.sb_type) ?
          !(c & ((1 << (b_width_log2(mi->mbmi.sb_type)-1)) - 1)) : 1;
684
      const int skip_this_c = skip_this && !block_edge_left;
685
686
687
      // top edge of current unit is block/partition edge -> no skip
      const int block_edge_above = b_height_log2(mi->mbmi.sb_type) ?
          !(r & ((1 << (b_height_log2(mi->mbmi.sb_type)-1)) - 1)) : 1;
688
      const int skip_this_r = skip_this && !block_edge_above;
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
      const TX_SIZE tx_size = plane ? get_uv_tx_size(xd) : mi[c].mbmi.txfm_size;

      // Filter level can vary per MI
      if (!build_lfi(cm, &mi[c].mbmi,
                     lfi[r] + (c >> xd->plane[plane].subsampling_x)))
        continue;

      // Build masks based on the transform size of each block
      if (tx_size == TX_32X32) {
        if (!skip_this_c && (c & 3) == 0)
          mask_16x16_c |= 1 << (c >> ss_x);
        if (!skip_this_r && (r & 3) == 0)
          mask_16x16[r] |= 1 << (c >> ss_x);
      } else if (tx_size == TX_16X16) {
        if (!skip_this_c && (c & 1) == 0)
          mask_16x16_c |= 1 << (c >> ss_x);
        if (!skip_this_r && (r & 1) == 0)
          mask_16x16[r] |= 1 << (c >> ss_x);
      } else {
        // force 8x8 filtering on 32x32 boundaries
        if (!skip_this_c) {
          if (tx_size == TX_8X8 || (c & 3) == 0)
            mask_8x8_c |= 1 << (c >> ss_x);
          else
            mask_4x4_c |= 1 << (c >> ss_x);
        }

        if (!skip_this_r) {
          if (tx_size == TX_8X8 || (r & 3) == 0)
            mask_8x8[r] |= 1 << (c >> ss_x);
          else
            mask_4x4[r] |= 1 << (c >> ss_x);
        }

        if (!skip_this && tx_size < TX_8X8)
          mask_4x4_1[r] |= 1 << (c >> ss_x);
      }
    }

    // Disable filtering on the leftmost column
    border_mask = ~(mi_col == 0);
    filter_selectively_vert(dst->buf, dst->stride,
                            mask_16x16_c & border_mask,
                            mask_8x8_c & border_mask,
                            mask_4x4_c & border_mask,
                            mask_4x4_1[r], lfi[r]);
    dst->buf += 8 * dst->stride;
    xd->mode_info_context += cm->mode_info_stride * row_step;
  }

  // Now do horizontal pass
  dst->buf = dst0;
  xd->mode_info_context = mi0;
  for (r = 0; r < 64 / MI_SIZE && mi_row + r < cm->mi_rows; r += row_step) {
    filter_selectively_horiz(dst->buf, dst->stride,
                             mask_16x16[r],
                             mask_8x8[r],
                             mask_4x4[r],
                             mask_4x4_1[r], mi_row + r == 0, lfi[r]);
    dst->buf += 8 * dst->stride;
    xd->mode_info_context += cm->mode_info_stride * row_step;
  }
}

void vp9_loop_filter_frame(VP9_COMMON *cm,
                           MACROBLOCKD *xd,
                           int frame_filter_level,
                           int y_only) {
  int mi_row, mi_col;

  // Initialize the loop filter for this frame.
  vp9_loop_filter_frame_init(cm, xd, frame_filter_level);

  for (mi_row = 0; mi_row < cm->mi_rows; mi_row += 64 / MI_SIZE) {
    MODE_INFO* const mi = cm->mi + mi_row * cm->mode_info_stride;

    for (mi_col = 0; mi_col < cm->mi_cols; mi_col += 64 / MI_SIZE) {
      int plane;

      setup_dst_planes(xd, cm->frame_to_show, mi_row, mi_col);
      for (plane = 0; plane < (y_only ? 1 : MAX_MB_PLANE); plane++) {
        xd->mode_info_context = mi + mi_col;
        filter_block_plane(cm, xd, plane, mi_row, mi_col);
      }
    }
  }
}
#endif