loopfilter.c 63.9 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11
 */

Debargha Mukherjee's avatar
Debargha Mukherjee committed
12 13
#include <math.h>

Yaowu Xu's avatar
Yaowu Xu committed
14 15
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
16 17 18
#include "av1/common/loopfilter.h"
#include "av1/common/onyxc_int.h"
#include "av1/common/reconinter.h"
Yaowu Xu's avatar
Yaowu Xu committed
19 20
#include "aom_dsp/aom_dsp_common.h"
#include "aom_mem/aom_mem.h"
21
#include "aom_ports/mem.h"
Jingning Han's avatar
Jingning Han committed
22

23
#include "av1/common/seg_common.h"
Jingning Han's avatar
Jingning Han committed
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41

// 64 bit masks for left transform size. Each 1 represents a position where
// we should apply a loop filter across the left border of an 8x8 block
// boundary.
//
// In the case of TX_16X16->  ( in low order byte first we end up with
// a mask that looks like this
//
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//
// A loopfilter should be applied to every other 8x8 horizontally.
clang-format's avatar
clang-format committed
42
static const uint64_t left_64x64_txform_mask[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
  0xffffffffffffffffULL,  // TX_4X4
  0xffffffffffffffffULL,  // TX_8x8
  0x5555555555555555ULL,  // TX_16x16
  0x1111111111111111ULL,  // TX_32x32
};

// 64 bit masks for above transform size. Each 1 represents a position where
// we should apply a loop filter across the top border of an 8x8 block
// boundary.
//
// In the case of TX_32x32 ->  ( in low order byte first we end up with
// a mask that looks like this
//
//    11111111
//    00000000
//    00000000
//    00000000
//    11111111
//    00000000
//    00000000
//    00000000
//
// A loopfilter should be applied to every other 4 the row vertically.
clang-format's avatar
clang-format committed
66
static const uint64_t above_64x64_txform_mask[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139
  0xffffffffffffffffULL,  // TX_4X4
  0xffffffffffffffffULL,  // TX_8x8
  0x00ff00ff00ff00ffULL,  // TX_16x16
  0x000000ff000000ffULL,  // TX_32x32
};

// 64 bit masks for prediction sizes (left). Each 1 represents a position
// where left border of an 8x8 block. These are aligned to the right most
// appropriate bit, and then shifted into place.
//
// In the case of TX_16x32 ->  ( low order byte first ) we end up with
// a mask that looks like this :
//
//  10000000
//  10000000
//  10000000
//  10000000
//  00000000
//  00000000
//  00000000
//  00000000
static const uint64_t left_prediction_mask[BLOCK_SIZES] = {
  0x0000000000000001ULL,  // BLOCK_4X4,
  0x0000000000000001ULL,  // BLOCK_4X8,
  0x0000000000000001ULL,  // BLOCK_8X4,
  0x0000000000000001ULL,  // BLOCK_8X8,
  0x0000000000000101ULL,  // BLOCK_8X16,
  0x0000000000000001ULL,  // BLOCK_16X8,
  0x0000000000000101ULL,  // BLOCK_16X16,
  0x0000000001010101ULL,  // BLOCK_16X32,
  0x0000000000000101ULL,  // BLOCK_32X16,
  0x0000000001010101ULL,  // BLOCK_32X32,
  0x0101010101010101ULL,  // BLOCK_32X64,
  0x0000000001010101ULL,  // BLOCK_64X32,
  0x0101010101010101ULL,  // BLOCK_64X64
};

// 64 bit mask to shift and set for each prediction size.
static const uint64_t above_prediction_mask[BLOCK_SIZES] = {
  0x0000000000000001ULL,  // BLOCK_4X4
  0x0000000000000001ULL,  // BLOCK_4X8
  0x0000000000000001ULL,  // BLOCK_8X4
  0x0000000000000001ULL,  // BLOCK_8X8
  0x0000000000000001ULL,  // BLOCK_8X16,
  0x0000000000000003ULL,  // BLOCK_16X8
  0x0000000000000003ULL,  // BLOCK_16X16
  0x0000000000000003ULL,  // BLOCK_16X32,
  0x000000000000000fULL,  // BLOCK_32X16,
  0x000000000000000fULL,  // BLOCK_32X32,
  0x000000000000000fULL,  // BLOCK_32X64,
  0x00000000000000ffULL,  // BLOCK_64X32,
  0x00000000000000ffULL,  // BLOCK_64X64
};
// 64 bit mask to shift and set for each prediction size. A bit is set for
// each 8x8 block that would be in the left most block of the given block
// size in the 64x64 block.
static const uint64_t size_mask[BLOCK_SIZES] = {
  0x0000000000000001ULL,  // BLOCK_4X4
  0x0000000000000001ULL,  // BLOCK_4X8
  0x0000000000000001ULL,  // BLOCK_8X4
  0x0000000000000001ULL,  // BLOCK_8X8
  0x0000000000000101ULL,  // BLOCK_8X16,
  0x0000000000000003ULL,  // BLOCK_16X8
  0x0000000000000303ULL,  // BLOCK_16X16
  0x0000000003030303ULL,  // BLOCK_16X32,
  0x0000000000000f0fULL,  // BLOCK_32X16,
  0x000000000f0f0f0fULL,  // BLOCK_32X32,
  0x0f0f0f0f0f0f0f0fULL,  // BLOCK_32X64,
  0x00000000ffffffffULL,  // BLOCK_64X32,
  0xffffffffffffffffULL,  // BLOCK_64X64
};

// These are used for masking the left and above borders.
clang-format's avatar
clang-format committed
140
static const uint64_t left_border = 0x1111111111111111ULL;
Jingning Han's avatar
Jingning Han committed
141 142 143
static const uint64_t above_border = 0x000000ff000000ffULL;

// 16 bit masks for uv transform sizes.
clang-format's avatar
clang-format committed
144
static const uint16_t left_64x64_txform_mask_uv[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
145 146 147 148 149 150
  0xffff,  // TX_4X4
  0xffff,  // TX_8x8
  0x5555,  // TX_16x16
  0x1111,  // TX_32x32
};

clang-format's avatar
clang-format committed
151
static const uint16_t above_64x64_txform_mask_uv[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
  0xffff,  // TX_4X4
  0xffff,  // TX_8x8
  0x0f0f,  // TX_16x16
  0x000f,  // TX_32x32
};

// 16 bit left mask to shift and set for each uv prediction size.
static const uint16_t left_prediction_mask_uv[BLOCK_SIZES] = {
  0x0001,  // BLOCK_4X4,
  0x0001,  // BLOCK_4X8,
  0x0001,  // BLOCK_8X4,
  0x0001,  // BLOCK_8X8,
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8,
  0x0001,  // BLOCK_16X16,
  0x0011,  // BLOCK_16X32,
  0x0001,  // BLOCK_32X16,
  0x0011,  // BLOCK_32X32,
  0x1111,  // BLOCK_32X64
  0x0011,  // BLOCK_64X32,
  0x1111,  // BLOCK_64X64
};
// 16 bit above mask to shift and set for uv each prediction size.
static const uint16_t above_prediction_mask_uv[BLOCK_SIZES] = {
  0x0001,  // BLOCK_4X4
  0x0001,  // BLOCK_4X8
  0x0001,  // BLOCK_8X4
  0x0001,  // BLOCK_8X8
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8
  0x0001,  // BLOCK_16X16
  0x0001,  // BLOCK_16X32,
  0x0003,  // BLOCK_32X16,
  0x0003,  // BLOCK_32X32,
  0x0003,  // BLOCK_32X64,
  0x000f,  // BLOCK_64X32,
  0x000f,  // BLOCK_64X64
};

// 64 bit mask to shift and set for each uv prediction size
static const uint16_t size_mask_uv[BLOCK_SIZES] = {
  0x0001,  // BLOCK_4X4
  0x0001,  // BLOCK_4X8
  0x0001,  // BLOCK_8X4
  0x0001,  // BLOCK_8X8
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8
  0x0001,  // BLOCK_16X16
  0x0011,  // BLOCK_16X32,
  0x0003,  // BLOCK_32X16,
  0x0033,  // BLOCK_32X32,
  0x3333,  // BLOCK_32X64,
  0x00ff,  // BLOCK_64X32,
  0xffff,  // BLOCK_64X64
};
clang-format's avatar
clang-format committed
207
static const uint16_t left_border_uv = 0x1111;
Jingning Han's avatar
Jingning Han committed
208 209 210 211 212
static const uint16_t above_border_uv = 0x000f;

static const int mode_lf_lut[MB_MODE_COUNT] = {
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // INTRA_MODES
  1, 1, 0, 1                     // INTER_MODES (ZEROMV == 0)
Yue Chen's avatar
Yue Chen committed
213
#if CONFIG_EXT_INTER
clang-format's avatar
clang-format committed
214 215 216 217
  ,
  1,                            // NEWFROMNEARMV mode
  1, 1, 1, 1, 1, 1, 1, 1, 0, 1  // INTER_COMPOUND_MODES (ZERO_ZEROMV == 0)
#endif                          // CONFIG_EXT_INTER
Jingning Han's avatar
Jingning Han committed
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232
};

static void update_sharpness(loop_filter_info_n *lfi, int sharpness_lvl) {
  int lvl;

  // For each possible value for the loop filter fill out limits
  for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++) {
    // Set loop filter parameters that control sharpness.
    int block_inside_limit = lvl >> ((sharpness_lvl > 0) + (sharpness_lvl > 4));

    if (sharpness_lvl > 0) {
      if (block_inside_limit > (9 - sharpness_lvl))
        block_inside_limit = (9 - sharpness_lvl);
    }

clang-format's avatar
clang-format committed
233
    if (block_inside_limit < 1) block_inside_limit = 1;
Jingning Han's avatar
Jingning Han committed
234 235 236 237 238 239 240 241 242

    memset(lfi->lfthr[lvl].lim, block_inside_limit, SIMD_WIDTH);
    memset(lfi->lfthr[lvl].mblim, (2 * (lvl + 2) + block_inside_limit),
           SIMD_WIDTH);
  }
}

static uint8_t get_filter_level(const loop_filter_info_n *lfi_n,
                                const MB_MODE_INFO *mbmi) {
243
#if CONFIG_SUPERTX
Yaowu Xu's avatar
Yaowu Xu committed
244
  const int segment_id = AOMMIN(mbmi->segment_id, mbmi->segment_id_supertx);
clang-format's avatar
clang-format committed
245 246
  assert(
      IMPLIES(supertx_enabled(mbmi), mbmi->segment_id_supertx != MAX_SEGMENTS));
247 248 249 250 251 252
  assert(IMPLIES(supertx_enabled(mbmi),
                 mbmi->segment_id_supertx <= mbmi->segment_id));
#else
  const int segment_id = mbmi->segment_id;
#endif  // CONFIG_SUPERTX
  return lfi_n->lvl[segment_id][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]];
Jingning Han's avatar
Jingning Han committed
253 254
}

Yaowu Xu's avatar
Yaowu Xu committed
255
void av1_loop_filter_init(AV1_COMMON *cm) {
Jingning Han's avatar
Jingning Han committed
256 257 258 259 260 261 262 263 264 265 266 267 268
  loop_filter_info_n *lfi = &cm->lf_info;
  struct loopfilter *lf = &cm->lf;
  int lvl;

  // init limits for given sharpness
  update_sharpness(lfi, lf->sharpness_level);
  lf->last_sharpness_level = lf->sharpness_level;

  // init hev threshold const vectors
  for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++)
    memset(lfi->lfthr[lvl].hev_thr, (lvl >> 4), SIMD_WIDTH);
}

Yaowu Xu's avatar
Yaowu Xu committed
269
void av1_loop_filter_frame_init(AV1_COMMON *cm, int default_filt_lvl) {
Jingning Han's avatar
Jingning Han committed
270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288
  int seg_id;
  // n_shift is the multiplier for lf_deltas
  // the multiplier is 1 for when filter_lvl is between 0 and 31;
  // 2 when filter_lvl is between 32 and 63
  const int scale = 1 << (default_filt_lvl >> 5);
  loop_filter_info_n *const lfi = &cm->lf_info;
  struct loopfilter *const lf = &cm->lf;
  const struct segmentation *const seg = &cm->seg;

  // update limits if sharpness has changed
  if (lf->last_sharpness_level != lf->sharpness_level) {
    update_sharpness(lfi, lf->sharpness_level);
    lf->last_sharpness_level = lf->sharpness_level;
  }

  for (seg_id = 0; seg_id < MAX_SEGMENTS; seg_id++) {
    int lvl_seg = default_filt_lvl;
    if (segfeature_active(seg, seg_id, SEG_LVL_ALT_LF)) {
      const int data = get_segdata(seg, seg_id, SEG_LVL_ALT_LF);
clang-format's avatar
clang-format committed
289 290 291
      lvl_seg = clamp(
          seg->abs_delta == SEGMENT_ABSDATA ? data : default_filt_lvl + data, 0,
          MAX_LOOP_FILTER);
Jingning Han's avatar
Jingning Han committed
292 293 294 295 296 297 298 299 300 301 302
    }

    if (!lf->mode_ref_delta_enabled) {
      // we could get rid of this if we assume that deltas are set to
      // zero when not in use; encoder always uses deltas
      memset(lfi->lvl[seg_id], lvl_seg, sizeof(lfi->lvl[seg_id]));
    } else {
      int ref, mode;
      const int intra_lvl = lvl_seg + lf->ref_deltas[INTRA_FRAME] * scale;
      lfi->lvl[seg_id][INTRA_FRAME][0] = clamp(intra_lvl, 0, MAX_LOOP_FILTER);

303
      for (ref = LAST_FRAME; ref < TOTAL_REFS_PER_FRAME; ++ref) {
Jingning Han's avatar
Jingning Han committed
304
        for (mode = 0; mode < MAX_MODE_LF_DELTAS; ++mode) {
clang-format's avatar
clang-format committed
305 306
          const int inter_lvl = lvl_seg + lf->ref_deltas[ref] * scale +
                                lf->mode_deltas[mode] * scale;
Jingning Han's avatar
Jingning Han committed
307 308 309 310 311 312 313
          lfi->lvl[seg_id][ref][mode] = clamp(inter_lvl, 0, MAX_LOOP_FILTER);
        }
      }
    }
  }
}

clang-format's avatar
clang-format committed
314 315
static void filter_selectively_vert_row2(int subsampling_factor, uint8_t *s,
                                         int pitch, unsigned int mask_16x16_l,
Jingning Han's avatar
Jingning Han committed
316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343
                                         unsigned int mask_8x8_l,
                                         unsigned int mask_4x4_l,
                                         unsigned int mask_4x4_int_l,
                                         const loop_filter_info_n *lfi_n,
                                         const uint8_t *lfl) {
  const int mask_shift = subsampling_factor ? 4 : 8;
  const int mask_cutoff = subsampling_factor ? 0xf : 0xff;
  const int lfl_forward = subsampling_factor ? 4 : 8;

  unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff;
  unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff;
  unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff;
  unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff;
  unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff;
  unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff;
  unsigned int mask;

  for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 |
              mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1;
       mask; mask >>= 1) {
    const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl;
    const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward);

    if (mask & 1) {
      if ((mask_16x16_0 | mask_16x16_1) & 1) {
        if ((mask_16x16_0 & mask_16x16_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
344
          aom_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
345 346
                                   lfi0->hev_thr);
        } else if (mask_16x16_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
347
          aom_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
348
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
349
          aom_lpf_vertical_16(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
clang-format's avatar
clang-format committed
350
                              lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
351 352 353 354 355
        }
      }

      if ((mask_8x8_0 | mask_8x8_1) & 1) {
        if ((mask_8x8_0 & mask_8x8_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
356
          aom_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
357 358 359
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_8x8_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
360
          aom_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
361
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
362
          aom_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
363
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
364 365 366 367 368
        }
      }

      if ((mask_4x4_0 | mask_4x4_1) & 1) {
        if ((mask_4x4_0 & mask_4x4_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
369
          aom_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
370 371 372
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_4x4_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
373
          aom_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
374
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
375
          aom_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
376
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
377 378 379 380 381
        }
      }

      if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) {
        if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
382
          aom_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
383 384 385
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_4x4_int_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
386
          aom_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim,
387
                             lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
388
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
389
          aom_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim, lfi1->lim,
390
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407
        }
      }
    }

    s += 8;
    lfl += 1;
    mask_16x16_0 >>= 1;
    mask_8x8_0 >>= 1;
    mask_4x4_0 >>= 1;
    mask_4x4_int_0 >>= 1;
    mask_16x16_1 >>= 1;
    mask_8x8_1 >>= 1;
    mask_4x4_1 >>= 1;
    mask_4x4_int_1 >>= 1;
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
408
#if CONFIG_AOM_HIGHBITDEPTH
clang-format's avatar
clang-format committed
409 410 411 412 413
static void highbd_filter_selectively_vert_row2(
    int subsampling_factor, uint16_t *s, int pitch, unsigned int mask_16x16_l,
    unsigned int mask_8x8_l, unsigned int mask_4x4_l,
    unsigned int mask_4x4_int_l, const loop_filter_info_n *lfi_n,
    const uint8_t *lfl, int bd) {
Jingning Han's avatar
Jingning Han committed
414 415 416 417 418 419 420 421 422 423 424 425 426 427 428
  const int mask_shift = subsampling_factor ? 4 : 8;
  const int mask_cutoff = subsampling_factor ? 0xf : 0xff;
  const int lfl_forward = subsampling_factor ? 4 : 8;

  unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff;
  unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff;
  unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff;
  unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff;
  unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff;
  unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff;
  unsigned int mask;

  for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 |
clang-format's avatar
clang-format committed
429
              mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1;
Jingning Han's avatar
Jingning Han committed
430 431 432 433 434 435 436
       mask; mask >>= 1) {
    const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl;
    const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward);

    if (mask & 1) {
      if ((mask_16x16_0 | mask_16x16_1) & 1) {
        if ((mask_16x16_0 & mask_16x16_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
437
          aom_highbd_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
438 439
                                          lfi0->hev_thr, bd);
        } else if (mask_16x16_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
440
          aom_highbd_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
441 442
                                     lfi0->hev_thr, bd);
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
443
          aom_highbd_lpf_vertical_16(s + 8 * pitch, pitch, lfi1->mblim,
Jingning Han's avatar
Jingning Han committed
444 445 446 447 448 449
                                     lfi1->lim, lfi1->hev_thr, bd);
        }
      }

      if ((mask_8x8_0 | mask_8x8_1) & 1) {
        if ((mask_8x8_0 & mask_8x8_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
450
          aom_highbd_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
451 452 453
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_8x8_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
454
          aom_highbd_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim,
455
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
456
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
457
          aom_highbd_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim,
458
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
459 460 461 462 463
        }
      }

      if ((mask_4x4_0 | mask_4x4_1) & 1) {
        if ((mask_4x4_0 & mask_4x4_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
464
          aom_highbd_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
465 466 467
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_4x4_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
468
          aom_highbd_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim,
469
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
470
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
471
          aom_highbd_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim,
472
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
473 474 475 476 477
        }
      }

      if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) {
        if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
478
          aom_highbd_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
479 480 481
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_4x4_int_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
482
          aom_highbd_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim,
483
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
484
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
485
          aom_highbd_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim,
486
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502
        }
      }
    }

    s += 8;
    lfl += 1;
    mask_16x16_0 >>= 1;
    mask_8x8_0 >>= 1;
    mask_4x4_0 >>= 1;
    mask_4x4_int_0 >>= 1;
    mask_16x16_1 >>= 1;
    mask_8x8_1 >>= 1;
    mask_4x4_1 >>= 1;
    mask_4x4_int_1 >>= 1;
  }
}
Yaowu Xu's avatar
Yaowu Xu committed
503
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
504

clang-format's avatar
clang-format committed
505 506 507 508
static void filter_selectively_horiz(
    uint8_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8,
    unsigned int mask_4x4, unsigned int mask_4x4_int,
    const loop_filter_info_n *lfi_n, const uint8_t *lfl) {
Jingning Han's avatar
Jingning Han committed
509 510 511
  unsigned int mask;
  int count;

clang-format's avatar
clang-format committed
512 513
  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask;
       mask >>= count) {
Jingning Han's avatar
Jingning Han committed
514 515 516 517 518 519
    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;

    count = 1;
    if (mask & 1) {
      if (mask_16x16 & 1) {
        if ((mask_16x16 & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
520
          aom_lpf_horizontal_edge_16(s, pitch, lfi->mblim, lfi->lim,
521
                                     lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
522 523
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
524
          aom_lpf_horizontal_edge_8(s, pitch, lfi->mblim, lfi->lim,
525
                                    lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
526 527 528 529 530 531
        }
      } else if (mask_8x8 & 1) {
        if ((mask_8x8 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
532
          aom_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
533 534 535 536
                                    lfi->hev_thr, lfin->mblim, lfin->lim,
                                    lfin->hev_thr);

          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
537
            aom_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
Jingning Han's avatar
Jingning Han committed
538 539 540 541
                                      lfi->lim, lfi->hev_thr, lfin->mblim,
                                      lfin->lim, lfin->hev_thr);
          } else {
            if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
542
              aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
543
                                   lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
544
            else if (mask_4x4_int & 2)
Yaowu Xu's avatar
Yaowu Xu committed
545
              aom_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
546
                                   lfin->lim, lfin->hev_thr);
Jingning Han's avatar
Jingning Han committed
547 548 549
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
550
          aom_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
551 552

          if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
553
            aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
554
                                 lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
555 556 557 558 559 560
        }
      } else if (mask_4x4 & 1) {
        if ((mask_4x4 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
561
          aom_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
562 563 564
                                    lfi->hev_thr, lfin->mblim, lfin->lim,
                                    lfin->hev_thr);
          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
565
            aom_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
Jingning Han's avatar
Jingning Han committed
566 567 568 569
                                      lfi->lim, lfi->hev_thr, lfin->mblim,
                                      lfin->lim, lfin->hev_thr);
          } else {
            if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
570
              aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
571
                                   lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
572
            else if (mask_4x4_int & 2)
Yaowu Xu's avatar
Yaowu Xu committed
573
              aom_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
574
                                   lfin->lim, lfin->hev_thr);
Jingning Han's avatar
Jingning Han committed
575 576 577
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
578
          aom_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
579 580

          if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
581
            aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
582
                                 lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
583 584
        }
      } else if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
585
        aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
586
                             lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
587 588 589 590 591 592 593 594 595 596 597
      }
    }
    s += 8 * count;
    lfl += count;
    mask_16x16 >>= count;
    mask_8x8 >>= count;
    mask_4x4 >>= count;
    mask_4x4_int >>= count;
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
598
#if CONFIG_AOM_HIGHBITDEPTH
clang-format's avatar
clang-format committed
599 600 601 602
static void highbd_filter_selectively_horiz(
    uint16_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8,
    unsigned int mask_4x4, unsigned int mask_4x4_int,
    const loop_filter_info_n *lfi_n, const uint8_t *lfl, int bd) {
Jingning Han's avatar
Jingning Han committed
603 604 605
  unsigned int mask;
  int count;

clang-format's avatar
clang-format committed
606 607
  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask;
       mask >>= count) {
Jingning Han's avatar
Jingning Han committed
608 609 610 611 612 613
    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;

    count = 1;
    if (mask & 1) {
      if (mask_16x16 & 1) {
        if ((mask_16x16 & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
614
          aom_highbd_lpf_horizontal_edge_16(s, pitch, lfi->mblim, lfi->lim,
615
                                            lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
616 617
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
618
          aom_highbd_lpf_horizontal_edge_8(s, pitch, lfi->mblim, lfi->lim,
619
                                           lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
620 621 622 623 624 625
        }
      } else if (mask_8x8 & 1) {
        if ((mask_8x8 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
626
          aom_highbd_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
627 628 629 630
                                           lfi->hev_thr, lfin->mblim, lfin->lim,
                                           lfin->hev_thr, bd);

          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
631
            aom_highbd_lpf_horizontal_4_dual(
clang-format's avatar
clang-format committed
632 633
                s + 4 * pitch, pitch, lfi->mblim, lfi->lim, lfi->hev_thr,
                lfin->mblim, lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
634 635
          } else {
            if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
636
              aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
637
                                          lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
638
            } else if (mask_4x4_int & 2) {
Yaowu Xu's avatar
Yaowu Xu committed
639
              aom_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
640
                                          lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
641 642 643 644
            }
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
645
          aom_highbd_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim,
646
                                      lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
647 648

          if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
649
            aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
650
                                        lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
651 652 653 654 655 656 657
          }
        }
      } else if (mask_4x4 & 1) {
        if ((mask_4x4 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
658
          aom_highbd_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
659 660 661
                                           lfi->hev_thr, lfin->mblim, lfin->lim,
                                           lfin->hev_thr, bd);
          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
662
            aom_highbd_lpf_horizontal_4_dual(
clang-format's avatar
clang-format committed
663 664
                s + 4 * pitch, pitch, lfi->mblim, lfi->lim, lfi->hev_thr,
                lfin->mblim, lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
665 666
          } else {
            if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
667
              aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
668
                                          lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
669
            } else if (mask_4x4_int & 2) {
Yaowu Xu's avatar
Yaowu Xu committed
670
              aom_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
671
                                          lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
672 673 674 675
            }
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
676
          aom_highbd_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim,
677
                                      lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
678 679

          if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
680
            aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
681
                                        lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
682 683 684
          }
        }
      } else if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
685
        aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
686
                                    lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
687 688 689 690 691 692 693 694 695 696
      }
    }
    s += 8 * count;
    lfl += count;
    mask_16x16 >>= count;
    mask_8x8 >>= count;
    mask_4x4 >>= count;
    mask_4x4_int >>= count;
  }
}
Yaowu Xu's avatar
Yaowu Xu committed
697
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
698 699 700 701 702 703 704 705 706 707

// This function ors into the current lfm structure, where to do loop
// filters for the specific mi we are looking at. It uses information
// including the block_size_type (32x16, 32x32, etc.), the transform size,
// whether there were any coefficients encoded, and the loop filter strength
// block we are currently looking at. Shift is used to position the
// 1's we produce.
// TODO(JBB) Need another function for different resolution color..
static void build_masks(const loop_filter_info_n *const lfi_n,
                        const MODE_INFO *mi, const int shift_y,
clang-format's avatar
clang-format committed
708
                        const int shift_uv, LOOP_FILTER_MASK *lfm) {
Jingning Han's avatar
Jingning Han committed
709 710
  const MB_MODE_INFO *mbmi = &mi->mbmi;
  const BLOCK_SIZE block_size = mbmi->sb_type;
711 712
  // TODO(debargha): Check if masks can be setup correctly when
  // rectangular transfroms are used with the EXT_TX expt.
Debargha Mukherjee's avatar
Debargha Mukherjee committed
713
  const TX_SIZE tx_size_y = txsize_sqr_up_map[mbmi->tx_size];
714
  const TX_SIZE tx_size_uv =
715
      txsize_sqr_up_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
Jingning Han's avatar
Jingning Han committed
716 717 718 719 720 721
  const int filter_level = get_filter_level(lfi_n, mbmi);
  uint64_t *const left_y = &lfm->left_y[tx_size_y];
  uint64_t *const above_y = &lfm->above_y[tx_size_y];
  uint64_t *const int_4x4_y = &lfm->int_4x4_y;
  uint16_t *const left_uv = &lfm->left_uv[tx_size_uv];
  uint16_t *const above_uv = &lfm->above_uv[tx_size_uv];
722
  uint16_t *const int_4x4_uv = &lfm->left_int_4x4_uv;
Jingning Han's avatar
Jingning Han committed
723 724 725 726 727 728 729 730
  int i;

  // If filter level is 0 we don't loop filter.
  if (!filter_level) {
    return;
  } else {
    const int w = num_8x8_blocks_wide_lookup[block_size];
    const int h = num_8x8_blocks_high_lookup[block_size];
731 732 733
    const int row = (shift_y >> MAX_MIB_SIZE_LOG2);
    const int col = shift_y - (row << MAX_MIB_SIZE_LOG2);

clang-format's avatar
clang-format committed
734
    for (i = 0; i < h; i++) memset(&lfm->lfl_y[row + i][col], filter_level, w);
Jingning Han's avatar
Jingning Han committed
735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755
  }

  // These set 1 in the current block size for the block size edges.
  // For instance if the block size is 32x16, we'll set:
  //    above =   1111
  //              0000
  //    and
  //    left  =   1000
  //          =   1000
  // NOTE : In this example the low bit is left most ( 1000 ) is stored as
  //        1,  not 8...
  //
  // U and V set things on a 16 bit scale.
  //
  *above_y |= above_prediction_mask[block_size] << shift_y;
  *above_uv |= above_prediction_mask_uv[block_size] << shift_uv;
  *left_y |= left_prediction_mask[block_size] << shift_y;
  *left_uv |= left_prediction_mask_uv[block_size] << shift_uv;

  // If the block has no coefficients and is not intra we skip applying
  // the loop filter on block edges.
756
  if (mbmi->skip && is_inter_block(mbmi)) return;
Jingning Han's avatar
Jingning Han committed
757 758 759 760 761

  // Here we are adding a mask for the transform size. The transform
  // size mask is set to be correct for a 64x64 prediction block size. We
  // mask to match the size of the block we are working on and then shift it
  // into place..
clang-format's avatar
clang-format committed
762 763 764 765 766
  *above_y |= (size_mask[block_size] & above_64x64_txform_mask[tx_size_y])
              << shift_y;
  *above_uv |=
      (size_mask_uv[block_size] & above_64x64_txform_mask_uv[tx_size_uv])
      << shift_uv;
Jingning Han's avatar
Jingning Han committed
767

clang-format's avatar
clang-format committed
768 769 770 771
  *left_y |= (size_mask[block_size] & left_64x64_txform_mask[tx_size_y])
             << shift_y;
  *left_uv |= (size_mask_uv[block_size] & left_64x64_txform_mask_uv[tx_size_uv])
              << shift_uv;
Jingning Han's avatar
Jingning Han committed
772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788

  // Here we are trying to determine what to do with the internal 4x4 block
  // boundaries.  These differ from the 4x4 boundaries on the outside edge of
  // an 8x8 in that the internal ones can be skipped and don't depend on
  // the prediction block size.
  if (tx_size_y == TX_4X4)
    *int_4x4_y |= (size_mask[block_size] & 0xffffffffffffffffULL) << shift_y;

  if (tx_size_uv == TX_4X4)
    *int_4x4_uv |= (size_mask_uv[block_size] & 0xffff) << shift_uv;
}

// This function does the same thing as the one above with the exception that
// it only affects the y masks. It exists because for blocks < 16x16 in size,
// we only update u and v masks on the first block.
static void build_y_mask(const loop_filter_info_n *const lfi_n,
                         const MODE_INFO *mi, const int shift_y,
789 790 791
#if CONFIG_SUPERTX
                         int supertx_enabled,
#endif  // CONFIG_SUPERTX
Jingning Han's avatar
Jingning Han committed
792 793
                         LOOP_FILTER_MASK *lfm) {
  const MB_MODE_INFO *mbmi = &mi->mbmi;
Debargha Mukherjee's avatar
Debargha Mukherjee committed
794
  const TX_SIZE tx_size_y = txsize_sqr_up_map[mbmi->tx_size];
795 796 797 798 799 800
#if CONFIG_SUPERTX
  const BLOCK_SIZE block_size =
      supertx_enabled ? (BLOCK_SIZE)(3 * tx_size_y) : mbmi->sb_type;
#else
  const BLOCK_SIZE block_size = mbmi->sb_type;
#endif
Jingning Han's avatar
Jingning Han committed
801 802 803 804 805 806 807 808 809 810 811
  const int filter_level = get_filter_level(lfi_n, mbmi);
  uint64_t *const left_y = &lfm->left_y[tx_size_y];
  uint64_t *const above_y = &lfm->above_y[tx_size_y];
  uint64_t *const int_4x4_y = &lfm->int_4x4_y;
  int i;

  if (!filter_level) {
    return;
  } else {
    const int w = num_8x8_blocks_wide_lookup[block_size];
    const int h = num_8x8_blocks_high_lookup[block_size];
812 813 814
    const int row = (shift_y >> MAX_MIB_SIZE_LOG2);
    const int col = shift_y - (row << MAX_MIB_SIZE_LOG2);

clang-format's avatar
clang-format committed
815
    for (i = 0; i < h; i++) memset(&lfm->lfl_y[row + i][col], filter_level, w);
Jingning Han's avatar
Jingning Han committed
816 817 818 819 820
  }

  *above_y |= above_prediction_mask[block_size] << shift_y;
  *left_y |= left_prediction_mask[block_size] << shift_y;

821
  if (mbmi->skip && is_inter_block(mbmi)) return;
Jingning Han's avatar
Jingning Han committed
822

clang-format's avatar
clang-format committed
823 824
  *above_y |= (size_mask[block_size] & above_64x64_txform_mask[tx_size_y])
              << shift_y;
Jingning Han's avatar
Jingning Han committed
825

clang-format's avatar
clang-format committed
826 827
  *left_y |= (size_mask[block_size] & left_64x64_txform_mask[tx_size_y])
             << shift_y;
Jingning Han's avatar
Jingning Han committed
828 829 830 831 832 833 834 835

  if (tx_size_y == TX_4X4)
    *int_4x4_y |= (size_mask[block_size] & 0xffffffffffffffffULL) << shift_y;
}

// This function sets up the bit masks for the entire 64x64 region represented
// by mi_row, mi_col.
// TODO(JBB): This function only works for yv12.
Yaowu Xu's avatar
Yaowu Xu committed
836 837 838
void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
                    MODE_INFO **mi, const int mode_info_stride,
                    LOOP_FILTER_MASK *lfm) {
Jingning Han's avatar
Jingning Han committed
839 840 841 842 843 844 845 846 847
  int idx_32, idx_16, idx_8;
  const loop_filter_info_n *const lfi_n = &cm->lf_info;
  MODE_INFO **mip = mi;
  MODE_INFO **mip2 = mi;

  // These are offsets to the next mi in the 64x64 block. It is what gets
  // added to the mi ptr as we go through each loop. It helps us to avoid
  // setting up special row and column counters for each index. The last step
  // brings us out back to the starting position.
clang-format's avatar
clang-format committed
848 849 850 851 852
  const int offset_32[] = { 4, (mode_info_stride << 2) - 4, 4,
                            -(mode_info_stride << 2) - 4 };
  const int offset_16[] = { 2, (mode_info_stride << 1) - 2, 2,
                            -(mode_info_stride << 1) - 2 };
  const int offset[] = { 1, mode_info_stride - 1, 1, -mode_info_stride - 1 };
Jingning Han's avatar
Jingning Han committed
853 854 855 856 857

  // Following variables represent shifts to position the current block
  // mask over the appropriate block. A shift of 36 to the left will move
  // the bits for the final 32 by 32 block in the 64x64 up 4 rows and left
  // 4 rows to the appropriate spot.
clang-format's avatar
clang-format committed
858 859 860 861 862
  const int shift_32_y[] = { 0, 4, 32, 36 };
  const int shift_16_y[] = { 0, 2, 16, 18 };
  const int shift_8_y[] = { 0, 1, 8, 9 };
  const int shift_32_uv[] = { 0, 2, 8, 10 };
  const int shift_16_uv[] = { 0, 1, 4, 5 };
Jingning Han's avatar
Jingning Han committed
863
  int i;
Yaowu Xu's avatar
Yaowu Xu committed
864 865
  const int max_rows = AOMMIN(cm->mi_rows - mi_row, MAX_MIB_SIZE);
  const int max_cols = AOMMIN(cm->mi_cols - mi_col, MAX_MIB_SIZE);
866 867 868
#if CONFIG_EXT_PARTITION
  assert(0 && "Not yet updated");
#endif  // CONFIG_EXT_PARTITION
Jingning Han's avatar
Jingning Han committed
869

Yaowu Xu's avatar
Yaowu Xu committed
870
  av1_zero(*lfm);
Jingning Han's avatar
Jingning Han committed
871 872 873 874 875 876
  assert(mip[0] != NULL);

  // TODO(jimbankoski): Try moving most of the following code into decode
  // loop and storing lfm in the mbmi structure so that we don't have to go
  // through the recursive loop structure multiple times.
  switch (mip[0]->mbmi.sb_type) {
clang-format's avatar
clang-format committed
877
    case BLOCK_64X64: build_masks(lfi_n, mip[0], 0, 0, lfm); break;
Jingning Han's avatar
Jingning Han committed
878 879 880
    case BLOCK_64X32:
      build_masks(lfi_n, mip[0], 0, 0, lfm);
      mip2 = mip + mode_info_stride * 4;
clang-format's avatar
clang-format committed
881
      if (4 >= max_rows) break;
Jingning Han's avatar
Jingning Han committed
882 883 884 885 886
      build_masks(lfi_n, mip2[0], 32, 8, lfm);
      break;
    case BLOCK_32X64:
      build_masks(lfi_n, mip[0], 0, 0, lfm);
      mip2 = mip + 4;
clang-format's avatar
clang-format committed
887
      if (4 >= max_cols) break;
Jingning Han's avatar
Jingning Han committed
888 889 890 891
      build_masks(lfi_n, mip2[0], 4, 2, lfm);
      break;
    default:
      for (idx_32 = 0; idx_32 < 4; mip += offset_32[idx_32], ++idx_32) {
892 893
        const int shift_y_32 = shift_32_y[idx_32];
        const int shift_uv_32 = shift_32_uv[idx_32];
Jingning Han's avatar
Jingning Han committed
894 895 896 897 898 899
        const int mi_32_col_offset = ((idx_32 & 1) << 2);
        const int mi_32_row_offset = ((idx_32 >> 1) << 2);
        if (mi_32_col_offset >= max_cols || mi_32_row_offset >= max_rows)
          continue;
        switch (mip[0]->mbmi.sb_type) {
          case BLOCK_32X32:
900
            build_masks(lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
Jingning Han's avatar
Jingning Han committed
901
            break;
902 903
          case BLOCK_32X16:
            build_masks(lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
904
#if CONFIG_SUPERTX
clang-format's avatar
clang-format committed
905
            if (supertx_enabled(&mip[0]->mbmi)) break;
906
#endif
clang-format's avatar
clang-format committed
907
            if (mi_32_row_offset + 2 >= max_rows) continue;
Jingning Han's avatar
Jingning Han committed
908
            mip2 = mip + mode_info_stride * 2;
909
            build_masks(lfi_n, mip2[0], shift_y_32 + 16, shift_uv_32 + 4, lfm);
Jingning Han's avatar
Jingning Han committed
910
            break;
911 912
          case BLOCK_16X32:
            build_masks(lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
913
#if CONFIG_SUPERTX
clang-format's avatar
clang-format committed
914
            if (supertx_enabled(&mip[0]->mbmi)) break;
915
#endif
clang-format's avatar
clang-format committed
916
            if (mi_32_col_offset + 2 >= max_cols) continue;
Jingning Han's avatar
Jingning Han committed
917
            mip2 = mip + 2;
918
            build_masks(lfi_n, mip2[0], shift_y_32 + 2, shift_uv_32 + 1, lfm);
Jingning Han's avatar
Jingning Han committed
919 920
            break;
          default:
921 922
#if CONFIG_SUPERTX
            if (mip[0]->mbmi.tx_size == TX_32X32) {
923
              build_masks(lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
924 925 926
              break;
            }
#endif
Jingning Han's avatar
Jingning Han committed
927
            for (idx_16 = 0; idx_16 < 4; mip += offset_16[idx_16], ++idx_16) {
928 929
              const int shift_y_32_16 = shift_y_32 + shift_16_y[idx_16];
              const int shift_uv_32_16 = shift_uv_32 + shift_16_uv[idx_16];
clang-format's avatar
clang-format committed
930 931 932 933
              const int mi_16_col_offset =
                  mi_32_col_offset + ((idx_16 & 1) << 1);
              const int mi_16_row_offset =
                  mi_32_row_offset + ((idx_16 >> 1) << 1);
Jingning Han's avatar
Jingning Han committed
934 935 936 937 938 939

              if (mi_16_col_offset >= max_cols || mi_16_row_offset >= max_rows)
                continue;

              switch (mip[0]->mbmi.sb_type) {
                case BLOCK_16X16:
940 941
                  build_masks(lfi_n, mip[0], shift_y_32_16, shift_uv_32_16,
                              lfm);
Jingning Han's avatar
Jingning Han committed
942 943
                  break;
                case BLOCK_16X8:
944
#if CONFIG_SUPERTX
clang-format's avatar
clang-format committed
945
                  if (supertx_enabled(&mip[0]->mbmi)) break;
946
#endif
947 948
                  build_masks(lfi_n, mip[0], shift_y_32_16, shift_uv_32_16,
                              lfm);
clang-format's avatar
clang-format committed
949
                  if (mi_16_row_offset + 1 >= max_rows) continue;
Jingning Han's avatar
Jingning Han committed
950
                  mip2 = mip + mode_info_stride;
951
                  build_y_mask(lfi_n, mip2[0], shift_y_32_16 + 8,
952 953 954 955
#if CONFIG_SUPERTX
                               0,
#endif
                               lfm);
Jingning Han's avatar
Jingning Han committed
956 957
                  break;
                case BLOCK_8X16:
958
#if CONFIG_SUPERTX
clang-format's avatar
clang-format committed
959
                  if (supertx_enabled(&mip[0]->mbmi)) break;
960
#endif
961 962
                  build_masks(lfi_n, mip[0], shift_y_32_16, shift_uv_32_16,
                              lfm);
clang-format's avatar
clang-format committed
963
                  if (mi_16_col_offset + 1 >= max_cols) continue;
Jingning Han's avatar
Jingning Han committed
964
                  mip2 = mip + 1;
965
                  build_y_mask(lfi_n, mip2[0], shift_y_32_16 + 1,
966 967 968 969
#if CONFIG_SUPERTX
                               0,
#endif
                               lfm);
Jingning Han's avatar
Jingning Han committed
970 971
                  break;
                default: {
972
                  const int shift_y_32_16_8_zero = shift_y_32_16 + shift_8_y[0];
973 974
#if CONFIG_SUPERTX
                  if (mip[0]->mbmi.tx_size == TX_16X16) {
975 976
                    build_masks(lfi_n, mip[0], shift_y_32_16_8_zero,
                                shift_uv_32_16, lfm);
977 978 979
                    break;
                  }
#endif
980 981
                  build_masks(lfi_n, mip[0], shift_y_32_16_8_zero,
                              shift_uv_32_16, lfm);
Jingning Han's avatar
Jingning Han committed
982 983
                  mip += offset[0];
                  for (idx_8 = 1; idx_8 < 4; mip += offset[idx_8], ++idx_8) {
984 985
                    const int shift_y_32_16_8 =
                        shift_y_32_16 + shift_8_y[idx_8];
clang-format's avatar
clang-format committed
986 987 988 989
                    const int mi_8_col_offset =
                        mi_16_col_offset + ((idx_8 & 1));
                    const int mi_8_row_offset =
                        mi_16_row_offset + ((idx_8 >> 1));
Jingning Han's avatar
Jingning Han committed
990 991 992 993

                    if (mi_8_col_offset >= max_cols ||
                        mi_8_row_offset >= max_rows)
                      continue;
994
                    build_y_mask(lfi_n, mip[0], shift_y_32_16_8,
995 996 997 998
#if CONFIG_SUPERTX
                                 supertx_enabled(&mip[0]->mbmi),
#endif
                                 lfm);
Jingning Han's avatar
Jingning Han committed
999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028
                  }
                  break;
                }
              }
            }
            break;
        }
      }
      break;
  }
  // The largest loopfilter we have is 16x16 so we use the 16x16 mask
  // for 32x32 transforms also.
  lfm->left_y[TX_16X16] |= lfm->left_y[TX_32X32];
  lfm->above_y[TX_16X16] |= lfm->above_y[TX_32X32];
  lfm->left_uv[TX_16X16] |= lfm->left_uv[TX_32X32];
  lfm->above_uv[TX_16X16] |= lfm->above_uv[TX_32X32];

  // We do at least 8 tap filter on every 32x32 even if the transform size
  // is 4x4. So if the 4x4 is set on a border pixel add it to the 8x8 and
  // remove it from the 4x4.
  lfm->left_y[TX_8X8] |= lfm->left_y[TX_4X4] & left_border;
  lfm->left_y[TX_4X4] &= ~left_border;
  lfm->above_y[TX_8X8] |= lfm->above_y[TX_4X4] & above_border;
  lfm->above_y[TX_4X4] &= ~above_border;
  lfm->left_uv[TX_8X8] |= lfm->left_uv[TX_4X4] & left_border_uv;
  lfm->left_uv[TX_4X4] &= ~left_border_uv;
  lfm->above_uv[TX_8X8] |= lfm->above_uv[TX_4X4] & above_border_uv;
  lfm->above_uv[TX_4X4] &= ~above_border_uv;

  // We do some special edge handling.