vp9_loopfilter.c 20.7 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10
 */

Attila Nagy's avatar
Attila Nagy committed
11
#include "vpx_config.h"
12 13
#include "vp9/common/vp9_loopfilter.h"
#include "vp9/common/vp9_onyxc_int.h"
Attila Nagy's avatar
Attila Nagy committed
14
#include "vpx_mem/vpx_mem.h"
John Koleszar's avatar
John Koleszar committed
15

16
#include "vp9/common/vp9_seg_common.h"
17

John Koleszar's avatar
John Koleszar committed
18 19 20 21 22 23 24 25 26 27 28
static void lf_init_lut(loop_filter_info_n *lfi) {
  lfi->mode_lf_lut[DC_PRED] = 1;
  lfi->mode_lf_lut[D45_PRED] = 1;
  lfi->mode_lf_lut[D135_PRED] = 1;
  lfi->mode_lf_lut[D117_PRED] = 1;
  lfi->mode_lf_lut[D153_PRED] = 1;
  lfi->mode_lf_lut[D27_PRED] = 1;
  lfi->mode_lf_lut[D63_PRED] = 1;
  lfi->mode_lf_lut[V_PRED] = 1;
  lfi->mode_lf_lut[H_PRED] = 1;
  lfi->mode_lf_lut[TM_PRED] = 1;
Yaowu Xu's avatar
Yaowu Xu committed
29
  lfi->mode_lf_lut[I4X4_PRED]  = 0;
John Koleszar's avatar
John Koleszar committed
30 31 32 33 34 35
  lfi->mode_lf_lut[I8X8_PRED] = 0;
  lfi->mode_lf_lut[ZEROMV]  = 1;
  lfi->mode_lf_lut[NEARESTMV] = 2;
  lfi->mode_lf_lut[NEARMV] = 2;
  lfi->mode_lf_lut[NEWMV] = 2;
  lfi->mode_lf_lut[SPLITMV] = 3;
Attila Nagy's avatar
Attila Nagy committed
36 37
}

38
void vp9_loop_filter_update_sharpness(loop_filter_info_n *lfi,
John Koleszar's avatar
John Koleszar committed
39 40 41 42 43 44 45
                                      int sharpness_lvl) {
  int i;

  /* For each possible value for the loop filter fill out limits */
  for (i = 0; i <= MAX_LOOP_FILTER; i++) {
    int filt_lvl = i;
    int block_inside_limit = 0;
John Koleszar's avatar
John Koleszar committed
46

John Koleszar's avatar
John Koleszar committed
47 48 49
    /* Set loop filter paramaeters that control sharpness. */
    block_inside_limit = filt_lvl >> (sharpness_lvl > 0);
    block_inside_limit = block_inside_limit >> (sharpness_lvl > 4);
John Koleszar's avatar
John Koleszar committed
50

John Koleszar's avatar
John Koleszar committed
51 52 53
    if (sharpness_lvl > 0) {
      if (block_inside_limit > (9 - sharpness_lvl))
        block_inside_limit = (9 - sharpness_lvl);
John Koleszar's avatar
John Koleszar committed
54
    }
John Koleszar's avatar
John Koleszar committed
55 56 57 58 59 60 61 62 63 64

    if (block_inside_limit < 1)
      block_inside_limit = 1;

    vpx_memset(lfi->lim[i], block_inside_limit, SIMD_WIDTH);
    vpx_memset(lfi->blim[i], (2 * filt_lvl + block_inside_limit),
               SIMD_WIDTH);
    vpx_memset(lfi->mblim[i], (2 * (filt_lvl + 2) + block_inside_limit),
               SIMD_WIDTH);
  }
Attila Nagy's avatar
Attila Nagy committed
65
}
John Koleszar's avatar
John Koleszar committed
66

67
void vp9_loop_filter_init(VP9_COMMON *cm) {
John Koleszar's avatar
John Koleszar committed
68 69
  loop_filter_info_n *lfi = &cm->lf_info;
  int i;
Attila Nagy's avatar
Attila Nagy committed
70

71
  // init limits for given sharpness
72
  vp9_loop_filter_update_sharpness(lfi, cm->sharpness_level);
John Koleszar's avatar
John Koleszar committed
73
  cm->last_sharpness_level = cm->sharpness_level;
Attila Nagy's avatar
Attila Nagy committed
74

75
  // init LUT for lvl  and hev thr picking
John Koleszar's avatar
John Koleszar committed
76
  lf_init_lut(lfi);
Attila Nagy's avatar
Attila Nagy committed
77

78 79
  // init hev threshold const vectors
  for (i = 0; i < 4; i++)
John Koleszar's avatar
John Koleszar committed
80
    vpx_memset(lfi->hev_thr[i], i, SIMD_WIDTH);
John Koleszar's avatar
John Koleszar committed
81 82
}

83
void vp9_loop_filter_frame_init(VP9_COMMON *cm,
84
                                MACROBLOCKD *xd,
John Koleszar's avatar
John Koleszar committed
85 86 87 88
                                int default_filt_lvl) {
  int seg,  /* segment number */
      ref,  /* index in ref_lf_deltas */
      mode; /* index in mode_lf_deltas */
John Koleszar's avatar
John Koleszar committed
89

John Koleszar's avatar
John Koleszar committed
90
  loop_filter_info_n *lfi = &cm->lf_info;
John Koleszar's avatar
John Koleszar committed
91

John Koleszar's avatar
John Koleszar committed
92
  /* update limits if sharpness has changed */
93 94 95
  // printf("vp9_loop_filter_frame_init %d\n", default_filt_lvl);
  // printf("sharpness level: %d [%d]\n",
  //        cm->sharpness_level, cm->last_sharpness_level);
John Koleszar's avatar
John Koleszar committed
96
  if (cm->last_sharpness_level != cm->sharpness_level) {
97
    vp9_loop_filter_update_sharpness(lfi, cm->sharpness_level);
John Koleszar's avatar
John Koleszar committed
98 99 100 101 102 103 104 105 106
    cm->last_sharpness_level = cm->sharpness_level;
  }

  for (seg = 0; seg < MAX_MB_SEGMENTS; seg++) {
    int lvl_seg = default_filt_lvl;
    int lvl_ref, lvl_mode;


    // Set the baseline filter values for each segment
107
    if (vp9_segfeature_active(xd, seg, SEG_LVL_ALT_LF)) {
John Koleszar's avatar
John Koleszar committed
108 109
      /* Abs value */
      if (xd->mb_segment_abs_delta == SEGMENT_ABSDATA) {
110
        lvl_seg = vp9_get_segdata(xd, seg, SEG_LVL_ALT_LF);
John Koleszar's avatar
John Koleszar committed
111
      } else { /* Delta Value */
112
        lvl_seg += vp9_get_segdata(xd, seg, SEG_LVL_ALT_LF);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
113
        lvl_seg = clamp(lvl_seg, 0, 63);
John Koleszar's avatar
John Koleszar committed
114 115
      }
    }
John Koleszar's avatar
John Koleszar committed
116

John Koleszar's avatar
John Koleszar committed
117 118 119 120 121 122 123
    if (!xd->mode_ref_lf_delta_enabled) {
      /* we could get rid of this if we assume that deltas are set to
       * zero when not in use; encoder always uses deltas
       */
      vpx_memset(lfi->lvl[seg][0], lvl_seg, 4 * 4);
      continue;
    }
John Koleszar's avatar
John Koleszar committed
124

John Koleszar's avatar
John Koleszar committed
125
    lvl_ref = lvl_seg;
John Koleszar's avatar
John Koleszar committed
126

John Koleszar's avatar
John Koleszar committed
127 128
    /* INTRA_FRAME */
    ref = INTRA_FRAME;
John Koleszar's avatar
John Koleszar committed
129

John Koleszar's avatar
John Koleszar committed
130 131
    /* Apply delta for reference frame */
    lvl_ref += xd->ref_lf_deltas[ref];
Attila Nagy's avatar
Attila Nagy committed
132

John Koleszar's avatar
John Koleszar committed
133
    /* Apply delta for Intra modes */
Yaowu Xu's avatar
Yaowu Xu committed
134 135
    mode = 0; /* I4X4_PRED */
    /* Only the split mode I4X4_PRED has a further special case */
Dmitry Kovalev's avatar
Dmitry Kovalev committed
136
    lvl_mode = clamp(lvl_ref +  xd->mode_lf_deltas[mode], 0, 63);
Attila Nagy's avatar
Attila Nagy committed
137

John Koleszar's avatar
John Koleszar committed
138
    lfi->lvl[seg][ref][mode] = lvl_mode;
Attila Nagy's avatar
Attila Nagy committed
139

John Koleszar's avatar
John Koleszar committed
140
    mode = 1; /* all the rest of Intra modes */
Dmitry Kovalev's avatar
Dmitry Kovalev committed
141
    lvl_mode = clamp(lvl_ref, 0, 63);
John Koleszar's avatar
John Koleszar committed
142
    lfi->lvl[seg][ref][mode] = lvl_mode;
John Koleszar's avatar
John Koleszar committed
143

John Koleszar's avatar
John Koleszar committed
144 145 146
    /* LAST, GOLDEN, ALT */
    for (ref = 1; ref < MAX_REF_FRAMES; ref++) {
      int lvl_ref = lvl_seg;
John Koleszar's avatar
John Koleszar committed
147

John Koleszar's avatar
John Koleszar committed
148 149
      /* Apply delta for reference frame */
      lvl_ref += xd->ref_lf_deltas[ref];
John Koleszar's avatar
John Koleszar committed
150

John Koleszar's avatar
John Koleszar committed
151 152
      /* Apply delta for Inter modes */
      for (mode = 1; mode < 4; mode++) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
153
        lvl_mode = clamp(lvl_ref + xd->mode_lf_deltas[mode], 0, 63);
John Koleszar's avatar
John Koleszar committed
154 155
        lfi->lvl[seg][ref][mode] = lvl_mode;
      }
John Koleszar's avatar
John Koleszar committed
156
    }
John Koleszar's avatar
John Koleszar committed
157
  }
John Koleszar's avatar
John Koleszar committed
158 159
}

Yaowu Xu's avatar
Yaowu Xu committed
160 161 162 163
// Determine if we should skip inner-MB loop filtering within a MB
// The current condition is that the loop filtering is skipped only
// the MB uses a prediction size of 16x16 and either 16x16 transform
// is used or there is no residue at all.
164 165 166 167
static int mb_lf_skip(const MB_MODE_INFO *const mbmi) {
  const MB_PREDICTION_MODE mode = mbmi->mode;
  const int skip_coef = mbmi->mb_skip_coeff;
  const int tx_size = mbmi->txfm_size;
Yaowu Xu's avatar
Yaowu Xu committed
168
  return mode != I4X4_PRED && mode != I8X8_PRED && mode != SPLITMV &&
Yaowu Xu's avatar
Yaowu Xu committed
169
         (tx_size >= TX_16X16 || skip_coef);
170
}
Yaowu Xu's avatar
Yaowu Xu committed
171 172 173 174 175

// Determine if we should skip MB loop filtering on a MB edge within
// a superblock, the current condition is that MB loop filtering is
// skipped only when both MBs do not use inner MB loop filtering, and
// same motion vector with same reference frame
176 177
static int sb_mb_lf_skip(const MODE_INFO *const mip0,
                         const MODE_INFO *const mip1) {
Yaowu Xu's avatar
Yaowu Xu committed
178
  const MB_MODE_INFO *mbmi0 = &mip0->mbmi;
Yaowu Xu's avatar
Yaowu Xu committed
179
  const MB_MODE_INFO *mbmi1 = &mip1->mbmi;
Yaowu Xu's avatar
Yaowu Xu committed
180
  return mb_lf_skip(mbmi0) && mb_lf_skip(mbmi1) &&
Yaowu Xu's avatar
Yaowu Xu committed
181 182
         mbmi0->ref_frame != INTRA_FRAME &&
         mbmi1->ref_frame != INTRA_FRAME;
183
}
184

Yaowu Xu's avatar
Yaowu Xu committed
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201
static void lpf_mb(VP9_COMMON *cm, const MODE_INFO *mi,
                   int do_left_mb_v, int do_above_mb_h,
                   int do_left_mbuv_v, int do_above_mbuv_h,
                   uint8_t *y_ptr, uint8_t *u_ptr, uint8_t *v_ptr,
                   int y_stride, int uv_stride, int dering) {
  loop_filter_info_n *lfi_n = &cm->lf_info;
  struct loop_filter_info lfi;
  int mode = mi->mbmi.mode;
  int mode_index = lfi_n->mode_lf_lut[mode];
  int seg = mi->mbmi.segment_id;
  int ref_frame = mi->mbmi.ref_frame;
  int filter_level = lfi_n->lvl[seg][ref_frame][mode_index];

  if (filter_level) {
    const int skip_lf = mb_lf_skip(&mi->mbmi);
    const int tx_size = mi->mbmi.txfm_size;
    if (cm->filter_type == NORMAL_LOOPFILTER) {
202
      const int hev_index = filter_level >> 4;
Yaowu Xu's avatar
Yaowu Xu committed
203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
      lfi.mblim = lfi_n->mblim[filter_level];
      lfi.blim = lfi_n->blim[filter_level];
      lfi.lim = lfi_n->lim[filter_level];
      lfi.hev_thr = lfi_n->hev_thr[hev_index];

      if (do_above_mb_h) {
        if (tx_size >= TX_16X16)
          vp9_lpf_mbh_w(y_ptr,
                        do_above_mbuv_h ? u_ptr : NULL,
                        do_above_mbuv_h ? v_ptr : NULL,
                        y_stride, uv_stride, &lfi);
        else
          vp9_loop_filter_mbh(y_ptr, u_ptr, v_ptr, y_stride, uv_stride, &lfi);
      }

      if (!skip_lf) {
        if (tx_size >= TX_8X8) {
          if (tx_size == TX_8X8 && (mode == I8X8_PRED || mode == SPLITMV))
            vp9_loop_filter_bh8x8(y_ptr, u_ptr, v_ptr,
                                  y_stride, uv_stride, &lfi);
          else
            vp9_loop_filter_bh8x8(y_ptr, NULL, NULL,
                                  y_stride, uv_stride, &lfi);
        } else {
          vp9_loop_filter_bh(y_ptr, u_ptr, v_ptr,
                             y_stride, uv_stride, &lfi);
        }
      }

      if (do_left_mb_v) {
        if (tx_size >= TX_16X16)
          vp9_lpf_mbv_w(y_ptr,
                        do_left_mbuv_v ? u_ptr : NULL,
                        do_left_mbuv_v ? v_ptr : NULL,
                        y_stride, uv_stride, &lfi);
        else
          vp9_loop_filter_mbv(y_ptr, u_ptr, v_ptr, y_stride, uv_stride, &lfi);
      }

      if (!skip_lf) {
        if (tx_size >= TX_8X8) {
          if (tx_size == TX_8X8 && (mode == I8X8_PRED || mode == SPLITMV))
            vp9_loop_filter_bv8x8(y_ptr, u_ptr, v_ptr,
                                  y_stride, uv_stride, &lfi);
          else
            vp9_loop_filter_bv8x8(y_ptr, NULL, NULL,
                                  y_stride, uv_stride, &lfi);
        } else {
          vp9_loop_filter_bv(y_ptr, u_ptr, v_ptr,
                             y_stride, uv_stride, &lfi);
        }
      }
      if (dering) {
#if CONFIG_LOOP_DERING
        vp9_post_proc_down_and_across(y_ptr, y_ptr,
          y_stride, y_stride,
          16, 16, dering);
        if (u_ptr && v_ptr) {
          vp9_post_proc_down_and_across(u_ptr, u_ptr,
            uv_stride, uv_stride,
            8, 8, dering);
          vp9_post_proc_down_and_across(v_ptr, v_ptr,
            uv_stride, uv_stride,
            8, 8, dering);
        }
#endif
      }
    } else {
      // TODO(yaowu): simple loop filter
    }
  }
}

static void lpf_sb32(VP9_COMMON *cm, const MODE_INFO *mode_info_context,
                     int mb_row, int mb_col,
                     uint8_t *y_ptr, uint8_t *u_ptr, uint8_t *v_ptr,
                     int y_stride, int uv_stride,
                     int y_only, int dering) {
    BLOCK_SIZE_TYPE sb_type = mode_info_context->mbmi.sb_type;
    TX_SIZE tx_size = mode_info_context->mbmi.txfm_size;
    int do_left_v, do_above_h;
    int do_left_v_mbuv, do_above_h_mbuv;
    int mis = cm->mode_info_stride;
    const MODE_INFO *mi;

    // process 1st MB top-left
    mi = mode_info_context;
    do_left_v = (mb_col > 0);
    do_above_h = (mb_row > 0);
    do_left_v_mbuv = !(sb_type >= BLOCK_SIZE_SB64X64 &&
      tx_size >= TX_32X32 && (mb_col & 2));
    do_above_h_mbuv = !(sb_type >= BLOCK_SIZE_SB64X64 &&
      tx_size >= TX_32X32 && (mb_row & 2));
    lpf_mb(cm, mi, do_left_v, do_above_h,
      do_left_v_mbuv, do_above_h_mbuv,
      y_ptr,
      y_only? 0 : u_ptr,
      y_only? 0 : v_ptr,
      y_stride, uv_stride, dering);
    // process 2nd MB top-right
    mi = mode_info_context + 1;
    do_left_v = !(sb_type && (tx_size >= TX_32X32 ||
      sb_mb_lf_skip(mode_info_context, mi)));
    do_above_h = (mb_row > 0);
    do_left_v_mbuv = do_left_v;
    do_above_h_mbuv = !(sb_type >= BLOCK_SIZE_SB64X64 &&
      tx_size >= TX_32X32 && (mb_row & 2));
    lpf_mb(cm, mi, do_left_v, do_above_h,
      do_left_v_mbuv, do_above_h_mbuv,
      y_ptr + 16,
      y_only ? 0 : (u_ptr + 8),
      y_only ? 0 : (v_ptr + 8),
      y_stride, uv_stride, dering);

    // process 3rd MB bottom-left
    mi = mode_info_context + mis;
    do_left_v = (mb_col > 0);
    do_above_h =!(sb_type && (tx_size >= TX_32X32 ||
      sb_mb_lf_skip(mode_info_context, mi)));
    do_left_v_mbuv =  !(sb_type >= BLOCK_SIZE_SB64X64 &&
      tx_size >= TX_32X32 && (mb_col & 2));
    do_above_h_mbuv = do_above_h;
    lpf_mb(cm, mi, do_left_v, do_above_h,
      do_left_v_mbuv, do_above_h_mbuv,
      y_ptr + 16 * y_stride,
      y_only ? 0 : (u_ptr + 8 * uv_stride),
      y_only ? 0 : (v_ptr + 8 * uv_stride),
      y_stride, uv_stride, dering);

    // process 4th MB bottom right
    mi = mode_info_context + mis + 1;
    do_left_v = !(sb_type && (tx_size >= TX_32X32 ||
      sb_mb_lf_skip(mi - 1, mi)));
    do_above_h =!(sb_type && (tx_size >= TX_32X32 ||
      sb_mb_lf_skip(mode_info_context + 1, mi)));
    do_left_v_mbuv = do_left_v;
    do_above_h_mbuv = do_above_h;
    lpf_mb(cm, mi, do_left_v, do_above_h,
      do_left_v_mbuv, do_above_h_mbuv,
      y_ptr + 16 * y_stride + 16,
      y_only ? 0 : (u_ptr + 8 * uv_stride + 8),
      y_only ? 0 : (v_ptr + 8 * uv_stride + 8),
      y_stride, uv_stride, dering);
}

static void lpf_sb64(VP9_COMMON *cm, const MODE_INFO *mode_info_context,
                     int mb_row, int mb_col,
                     uint8_t *y_ptr, uint8_t *u_ptr, uint8_t *v_ptr,
                     int y_stride, int uv_stride,
                     int y_only, int dering) {
    lpf_sb32(cm, mode_info_context, mb_row, mb_col,
      y_ptr, u_ptr, v_ptr,
      y_stride, uv_stride, y_only, dering);
    lpf_sb32(cm, mode_info_context + 2, mb_row, mb_col + 2,
      y_ptr + 32, u_ptr + 16, v_ptr + 16,
      y_stride, uv_stride, y_only, dering);
    lpf_sb32(cm, mode_info_context + cm->mode_info_stride * 2,
      mb_row + 2, mb_col,
      y_ptr + 32 * y_stride,
      u_ptr + 16 * uv_stride,
      v_ptr + 16 * uv_stride,
      y_stride, uv_stride, y_only, dering);
    lpf_sb32(cm, mode_info_context + cm->mode_info_stride * 2 + 2,
      mb_row + 2, mb_col + 2,
      y_ptr + 32 * y_stride + 32,
      u_ptr + 16 * uv_stride + 16,
      v_ptr + 16 * uv_stride + 16,
      y_stride, uv_stride, y_only, dering);
}
372 373 374
void vp9_loop_filter_frame(VP9_COMMON *cm,
                           MACROBLOCKD *xd,
                           int frame_filter_level,
375 376
                           int y_only,
                           int dering) {
John Koleszar's avatar
John Koleszar committed
377
  YV12_BUFFER_CONFIG *post = cm->frame_to_show;
378
  int mb_row, mb_col;
Yaowu Xu's avatar
Yaowu Xu committed
379 380 381 382 383 384
  const int sb64_rows = cm->mb_rows / 4;
  const int sb64_cols = cm->mb_cols / 4;
  const int extra_sb32_row = (cm->mb_rows & 2) != 0;
  const int extra_sb32_col = (cm->mb_cols & 2) != 0;
  const int extra_mb_col = cm->mb_cols & 1;
  const int extra_mb_row = cm->mb_rows & 1;
385 386 387 388 389 390
  // Set up the buffer pointers
  uint8_t *y_ptr = post->y_buffer;
  uint8_t *u_ptr = y_only ? 0 : post->u_buffer;
  uint8_t *v_ptr = y_only ? 0 : post->v_buffer;

  // Point at base of Mb MODE_INFO list
John Koleszar's avatar
John Koleszar committed
391
  const MODE_INFO *mode_info_context = cm->mi;
Yaowu Xu's avatar
Yaowu Xu committed
392
  const MODE_INFO *mi;
393
  const int mis = cm->mode_info_stride;
Yaowu Xu's avatar
Yaowu Xu committed
394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416
  const int y_stride = post->y_stride;
  const int uv_stride = post->uv_stride;
  // These two flags signal if MB left edge and above edge
  // should be filtered using MB edge filter. Currently, MB
  // edge filtering is not applied on MB edge internal to a
  // 32x32 superblock if:
  // 1) SB32 is using 32x32 prediction and 32x32 transform
  // 2) SB32 is using 32x32 prediction and 16x16 transform
  //    but all coefficients are zero.
  // MB edges are on 32x32 superblock boundary are always
  // filtered except on image frame boundary.
  int do_left_v, do_above_h;
  // These two flags signal if MB UV left edge and above edge
  // should be filtered using MB edge filter. Currently, MB
  // edge filtering is not applied for MB edges internal to
  // a 32x32 superblock if:
  // 1) SB32 is using 32x32 prediction and 32x32 transform
  // 2) SB32 is using 32x32 prediction and 16x16 transform
  //    but all coefficients are zero.
  // 3) SB32 UV edges internal to a SB64 and 32x32 transform
  //    is used, i.e. UV is doing 32x32 transform hence no
  //    transform boundary exists inside the SB64 for UV
  int do_left_v_mbuv, do_above_h_mbuv;
John Koleszar's avatar
John Koleszar committed
417

418
  // Initialize the loop filter for this frame.
419
  vp9_loop_filter_frame_init(cm, xd, frame_filter_level);
John Koleszar's avatar
John Koleszar committed
420

Yaowu Xu's avatar
Yaowu Xu committed
421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502
  // vp9_filter each 64x64 SB
  // For each SB64: the 4 SB32 are filtered in raster scan order
  // For each SB32: the 4 MBs are filtered in raster scan order
  // For each MB:   the left and above MB edges as well as the
  //                internal block edges are processed together
  for (mb_row = 0; mb_row < sb64_rows * 4; mb_row += 4) {
    for (mb_col = 0; mb_col < sb64_cols * 4; mb_col += 4) {
      lpf_sb64(cm, mode_info_context, mb_row, mb_col,
               y_ptr, u_ptr, v_ptr,
               y_stride, uv_stride, y_only, dering);
      y_ptr += 64;
      u_ptr = y_only? 0 : u_ptr + 32;
      v_ptr = y_only? 0 : v_ptr + 32;
      mode_info_context += 4;       // step to next SB64
    }
    if (extra_sb32_col) {
      // process 2 SB32s in the extra SB32 col
      lpf_sb32(cm, mode_info_context, mb_row, mb_col,
               y_ptr, u_ptr, v_ptr,
               y_stride, uv_stride, y_only, dering);
      lpf_sb32(cm, mode_info_context + mis * 2,
               mb_row + 2, mb_col,
               y_ptr + 32 * y_stride,
               u_ptr + 16 * uv_stride,
               v_ptr + 16 * uv_stride,
               y_stride, uv_stride, y_only, dering);
      y_ptr += 32;
      u_ptr = y_only? 0 : u_ptr + 16;
      v_ptr = y_only? 0 : v_ptr + 16;
      mode_info_context += 2;       // step to next SB32
      mb_col += 2;
    }
    if (extra_mb_col) {
      // process 4 MB in the extra MB col
      // process 1st MB
      mi = mode_info_context;
      do_left_v = (mb_col > 0);
      do_above_h = (mb_row > 0);
      do_left_v_mbuv =  1;
      do_above_h_mbuv = 1;
      lpf_mb(cm, mi, do_left_v, do_above_h,
             do_left_v_mbuv, do_above_h_mbuv,
             y_ptr,
             y_only? 0 : u_ptr,
             y_only? 0 : v_ptr,
             y_stride, uv_stride, dering);
      // process 2nd MB
      mi = mode_info_context + mis;
      do_left_v = (mb_col > 0);
      do_above_h = 1;
      do_left_v_mbuv =  1;
      do_above_h_mbuv = 1;
      lpf_mb(cm, mi, do_left_v, do_above_h,
             do_left_v_mbuv, do_above_h_mbuv,
             y_ptr + 16 * y_stride,
             y_only ? 0 : (u_ptr + 8 * uv_stride),
             y_only ? 0 : (v_ptr + 8 * uv_stride),
             y_stride, uv_stride, dering);
      // process 3nd MB
      mi = mode_info_context + mis * 2;
      do_left_v = (mb_col > 0);
      do_above_h = 1;
      do_left_v_mbuv =  1;
      do_above_h_mbuv = 1;
      lpf_mb(cm, mi, do_left_v, do_above_h,
             do_left_v_mbuv, do_above_h_mbuv,
             y_ptr + 32 * y_stride,
             y_only ? 0 : (u_ptr + 16 * uv_stride),
             y_only ? 0 : (v_ptr + 16 * uv_stride),
             y_stride, uv_stride, dering);
      // process 4th MB
      mi = mode_info_context + mis * 3;
      do_left_v = (mb_col > 0);
      do_above_h = 1;
      do_left_v_mbuv =  1;
      do_above_h_mbuv = 1;
      lpf_mb(cm, mi, do_left_v, do_above_h,
             do_left_v_mbuv, do_above_h_mbuv,
             y_ptr + 48 * y_stride,
             y_only ? 0 : (u_ptr + 24 * uv_stride),
             y_only ? 0 : (v_ptr + 24 * uv_stride),
             y_stride, uv_stride, dering);
John Koleszar's avatar
John Koleszar committed
503
      y_ptr += 16;
Yaowu Xu's avatar
Yaowu Xu committed
504 505 506
      u_ptr = y_only? 0 : u_ptr + 8;
      v_ptr = y_only? 0 : v_ptr + 8;
      mode_info_context++;       // step to next MB
John Koleszar's avatar
John Koleszar committed
507
    }
Yaowu Xu's avatar
Yaowu Xu committed
508 509
    // move pointers to the begining of next sb64 row
    y_ptr += y_stride  * 64 - post->y_width;
510
    if (!y_only) {
Yaowu Xu's avatar
Yaowu Xu committed
511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582
      u_ptr += uv_stride *  32 - post->uv_width;
      v_ptr += uv_stride *  32 - post->uv_width;
    }
    /* skip to next SB64 row */
    mode_info_context += mis * 4 - cm->mb_cols;
  }
  if (extra_sb32_row) {
    const int sb32_cols = sb64_cols * 2 + extra_sb32_col;
    for (mb_col = 0; mb_col < sb32_cols * 2; mb_col += 2) {
      lpf_sb32(cm, mode_info_context, mb_row, mb_col,
               y_ptr, u_ptr, v_ptr,
               y_stride, uv_stride, y_only, dering);
      y_ptr += 32;
      u_ptr = y_only? 0 : u_ptr + 16;
      v_ptr = y_only? 0 : v_ptr + 16;
      mode_info_context += 2;       // step to next SB32
    }
    if (extra_mb_col) {
      // process 1st MB
      mi = mode_info_context;
      do_left_v = (mb_col > 0);
      do_above_h = (mb_row > 0);
      do_left_v_mbuv =  1;
      do_above_h_mbuv = 1;
      lpf_mb(cm, mi, do_left_v, do_above_h,
             do_left_v_mbuv, do_above_h_mbuv,
             y_ptr,
             y_only? NULL : u_ptr,
             y_only? NULL : v_ptr,
             y_stride, uv_stride, dering);
      // process 2nd MB
      mi = mode_info_context + mis;
      do_left_v = (mb_col > 0);
      do_above_h = 1;
      do_left_v_mbuv =  1;
      do_above_h_mbuv = 1;
      lpf_mb(cm, mi, do_left_v, do_above_h,
             do_left_v_mbuv, do_above_h_mbuv,
             y_ptr + 16 * y_stride,
             y_only ? NULL : (u_ptr + 8 * uv_stride),
             y_only ? NULL : (v_ptr + 8 * uv_stride),
             y_stride, uv_stride, dering);
      y_ptr += 16;
      u_ptr = y_only? 0 : u_ptr + 8;
      v_ptr = y_only? 0 : v_ptr + 8;
      mode_info_context++;       /* step to next MB */
    }
    // move pointers to the beginning of next sb64 row
    y_ptr += y_stride * 32 - post->y_width;
    u_ptr += y_only? 0 : uv_stride *  16 - post->uv_width;
    v_ptr += y_only? 0 : uv_stride *  16 - post->uv_width;
    // skip to next MB row if exist
    mode_info_context += mis * 2 - cm->mb_cols;
    mb_row += 2;
  }
  if (extra_mb_row) {
    for (mb_col = 0; mb_col < cm->mb_cols; mb_col++) {
      const MODE_INFO *mi = mode_info_context;
      do_left_v =  (mb_col > 0);
      do_above_h = (mb_row > 0);
      do_left_v_mbuv = 1;
      do_above_h_mbuv = 1;
      lpf_mb(cm, mi, do_left_v, do_above_h,
             do_left_v_mbuv, do_above_h_mbuv,
             y_ptr,
             y_only? 0 : u_ptr,
             y_only? 0 : v_ptr,
             y_stride, uv_stride, dering);
      y_ptr += 16;
      u_ptr = y_only? 0 : u_ptr + 8;
      v_ptr = y_only? 0 : v_ptr + 8;
      mode_info_context++;     // step to next MB
583
    }
John Koleszar's avatar
John Koleszar committed
584
  }
John Koleszar's avatar
John Koleszar committed
585
}