av1_loopfilter.c 101 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11
 */

Debargha Mukherjee's avatar
Debargha Mukherjee committed
12 13
#include <math.h>

Yaowu Xu's avatar
Yaowu Xu committed
14 15 16 17
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_mem/aom_mem.h"
18
#include "aom_ports/mem.h"
19 20 21
#include "av1/common/av1_loopfilter.h"
#include "av1/common/onyxc_int.h"
#include "av1/common/reconinter.h"
22
#include "av1/common/seg_common.h"
Jingning Han's avatar
Jingning Han committed
23

24 25 26 27 28 29
#if CONFIG_LOOPFILTER_LEVEL
static const SEG_LVL_FEATURES seg_lvl_lf_lut[MAX_MB_PLANE][2] = {
  { SEG_LVL_ALT_LF_Y_V, SEG_LVL_ALT_LF_Y_H },
  { SEG_LVL_ALT_LF_U, SEG_LVL_ALT_LF_U },
  { SEG_LVL_ALT_LF_V, SEG_LVL_ALT_LF_V }
};
30 31 32 33 34 35

#if CONFIG_EXT_DELTA_Q
static const int delta_lf_id_lut[MAX_MB_PLANE][2] = {
  { 0, 1 }, { 2, 2 }, { 3, 3 }
};
#endif  // CONFIG_EXT_DELTA_Q
36 37
#endif  // CONFIG_LOOPFILTER_LEVEL

38
#define PARALLEL_DEBLOCKING_15TAPLUMAONLY 1
39
#define PARALLEL_DEBLOCKING_DISABLE_15TAP 0
Ola Hugosson's avatar
Ola Hugosson committed
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
#if CONFIG_DEBLOCK_13TAP
#define PARALLEL_DEBLOCKING_5_TAP_CHROMA 1
#else
#define PARALLEL_DEBLOCKING_5_TAP_CHROMA 0
#endif

#if PARALLEL_DEBLOCKING_5_TAP_CHROMA
extern void aom_lpf_vertical_6_c(uint8_t *s, int pitch, const uint8_t *blimit,
                                 const uint8_t *limit, const uint8_t *thresh);

extern void aom_lpf_horizontal_6_c(uint8_t *s, int p, const uint8_t *blimit,
                                   const uint8_t *limit, const uint8_t *thresh);

extern void aom_highbd_lpf_horizontal_6_c(uint16_t *s, int p,
                                          const uint8_t *blimit,
                                          const uint8_t *limit,
                                          const uint8_t *thresh, int bd);

extern void aom_highbd_lpf_vertical_6_c(uint16_t *s, int pitch,
                                        const uint8_t *blimit,
                                        const uint8_t *limit,
                                        const uint8_t *thresh, int bd);
#endif
63

Jingning Han's avatar
Jingning Han committed
64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80
// 64 bit masks for left transform size. Each 1 represents a position where
// we should apply a loop filter across the left border of an 8x8 block
// boundary.
//
// In the case of TX_16X16->  ( in low order byte first we end up with
// a mask that looks like this
//
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//
// A loopfilter should be applied to every other 8x8 horizontally.
clang-format's avatar
clang-format committed
81
static const uint64_t left_64x64_txform_mask[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
82 83 84 85
  0xffffffffffffffffULL,  // TX_4X4
  0xffffffffffffffffULL,  // TX_8x8
  0x5555555555555555ULL,  // TX_16x16
  0x1111111111111111ULL,  // TX_32x32
86 87 88
#if CONFIG_TX64X64
  0x0101010101010101ULL,  // TX_64x64
#endif                    // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107
};

// 64 bit masks for above transform size. Each 1 represents a position where
// we should apply a loop filter across the top border of an 8x8 block
// boundary.
//
// In the case of TX_32x32 ->  ( in low order byte first we end up with
// a mask that looks like this
//
//    11111111
//    00000000
//    00000000
//    00000000
//    11111111
//    00000000
//    00000000
//    00000000
//
// A loopfilter should be applied to every other 4 the row vertically.
clang-format's avatar
clang-format committed
108
static const uint64_t above_64x64_txform_mask[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
109 110 111 112
  0xffffffffffffffffULL,  // TX_4X4
  0xffffffffffffffffULL,  // TX_8x8
  0x00ff00ff00ff00ffULL,  // TX_16x16
  0x000000ff000000ffULL,  // TX_32x32
113 114 115
#if CONFIG_TX64X64
  0x00000000000000ffULL,  // TX_64x64
#endif                    // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
};

// 64 bit masks for prediction sizes (left). Each 1 represents a position
// where left border of an 8x8 block. These are aligned to the right most
// appropriate bit, and then shifted into place.
//
// In the case of TX_16x32 ->  ( low order byte first ) we end up with
// a mask that looks like this :
//
//  10000000
//  10000000
//  10000000
//  10000000
//  00000000
//  00000000
//  00000000
//  00000000
133
static const uint64_t left_prediction_mask[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
134 135 136
  0x0000000000000001ULL,  // BLOCK_2X2,
  0x0000000000000001ULL,  // BLOCK_2X4,
  0x0000000000000001ULL,  // BLOCK_4X2,
Jingning Han's avatar
Jingning Han committed
137 138 139 140 141 142 143 144 145 146 147 148
  0x0000000000000001ULL,  // BLOCK_4X4,
  0x0000000000000001ULL,  // BLOCK_4X8,
  0x0000000000000001ULL,  // BLOCK_8X4,
  0x0000000000000001ULL,  // BLOCK_8X8,
  0x0000000000000101ULL,  // BLOCK_8X16,
  0x0000000000000001ULL,  // BLOCK_16X8,
  0x0000000000000101ULL,  // BLOCK_16X16,
  0x0000000001010101ULL,  // BLOCK_16X32,
  0x0000000000000101ULL,  // BLOCK_32X16,
  0x0000000001010101ULL,  // BLOCK_32X32,
  0x0101010101010101ULL,  // BLOCK_32X64,
  0x0000000001010101ULL,  // BLOCK_64X32,
149 150 151 152
  0x0101010101010101ULL,  // BLOCK_64X64,
  0x0000000000000101ULL,  // BLOCK_4X16,
  0x0000000000000001ULL,  // BLOCK_16X4,
  0x0000000001010101ULL,  // BLOCK_8X32,
153 154 155
  0x0000000000000001ULL,  // BLOCK_32X8,
  0x0101010101010101ULL,  // BLOCK_16X64,
  0x0000000000000101ULL,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
156 157 158
};

// 64 bit mask to shift and set for each prediction size.
159
static const uint64_t above_prediction_mask[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
160 161 162
  0x0000000000000001ULL,  // BLOCK_2X2
  0x0000000000000001ULL,  // BLOCK_2X4
  0x0000000000000001ULL,  // BLOCK_4X2
Jingning Han's avatar
Jingning Han committed
163 164 165 166 167 168 169 170 171 172 173 174
  0x0000000000000001ULL,  // BLOCK_4X4
  0x0000000000000001ULL,  // BLOCK_4X8
  0x0000000000000001ULL,  // BLOCK_8X4
  0x0000000000000001ULL,  // BLOCK_8X8
  0x0000000000000001ULL,  // BLOCK_8X16,
  0x0000000000000003ULL,  // BLOCK_16X8
  0x0000000000000003ULL,  // BLOCK_16X16
  0x0000000000000003ULL,  // BLOCK_16X32,
  0x000000000000000fULL,  // BLOCK_32X16,
  0x000000000000000fULL,  // BLOCK_32X32,
  0x000000000000000fULL,  // BLOCK_32X64,
  0x00000000000000ffULL,  // BLOCK_64X32,
175 176 177 178
  0x00000000000000ffULL,  // BLOCK_64X64,
  0x0000000000000001ULL,  // BLOCK_4X16,
  0x0000000000000003ULL,  // BLOCK_16X4,
  0x0000000000000001ULL,  // BLOCK_8X32,
179 180 181
  0x000000000000000fULL,  // BLOCK_32X8,
  0x0000000000000003ULL,  // BLOCK_16X64,
  0x00000000000000ffULL,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
182 183
};
// 64 bit mask to shift and set for each prediction size. A bit is set for
184
// each 8x8 block that would be in the top left most block of the given block
Jingning Han's avatar
Jingning Han committed
185
// size in the 64x64 block.
186
static const uint64_t size_mask[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
187 188 189
  0x0000000000000001ULL,  // BLOCK_2X2
  0x0000000000000001ULL,  // BLOCK_2X4
  0x0000000000000001ULL,  // BLOCK_4X2
Jingning Han's avatar
Jingning Han committed
190 191 192 193 194 195 196 197 198 199 200 201
  0x0000000000000001ULL,  // BLOCK_4X4
  0x0000000000000001ULL,  // BLOCK_4X8
  0x0000000000000001ULL,  // BLOCK_8X4
  0x0000000000000001ULL,  // BLOCK_8X8
  0x0000000000000101ULL,  // BLOCK_8X16,
  0x0000000000000003ULL,  // BLOCK_16X8
  0x0000000000000303ULL,  // BLOCK_16X16
  0x0000000003030303ULL,  // BLOCK_16X32,
  0x0000000000000f0fULL,  // BLOCK_32X16,
  0x000000000f0f0f0fULL,  // BLOCK_32X32,
  0x0f0f0f0f0f0f0f0fULL,  // BLOCK_32X64,
  0x00000000ffffffffULL,  // BLOCK_64X32,
202 203 204 205
  0xffffffffffffffffULL,  // BLOCK_64X64,
  0x0000000000000101ULL,  // BLOCK_4X16,
  0x0000000000000003ULL,  // BLOCK_16X4,
  0x0000000001010101ULL,  // BLOCK_8X32,
206 207 208
  0x000000000000000fULL,  // BLOCK_32X8,
  0x0303030303030303ULL,  // BLOCK_16X64,
  0x000000000000ffffULL,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
209 210
};

211
// These are used for masking the left and above 32x32 borders.
clang-format's avatar
clang-format committed
212
static const uint64_t left_border = 0x1111111111111111ULL;
Jingning Han's avatar
Jingning Han committed
213 214 215
static const uint64_t above_border = 0x000000ff000000ffULL;

// 16 bit masks for uv transform sizes.
clang-format's avatar
clang-format committed
216
static const uint16_t left_64x64_txform_mask_uv[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
217 218 219 220
  0xffff,  // TX_4X4
  0xffff,  // TX_8x8
  0x5555,  // TX_16x16
  0x1111,  // TX_32x32
221 222 223
#if CONFIG_TX64X64
  0x0101,  // TX_64x64, never used
#endif     // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
224 225
};

clang-format's avatar
clang-format committed
226
static const uint16_t above_64x64_txform_mask_uv[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
227 228 229 230
  0xffff,  // TX_4X4
  0xffff,  // TX_8x8
  0x0f0f,  // TX_16x16
  0x000f,  // TX_32x32
231 232 233
#if CONFIG_TX64X64
  0x0003,  // TX_64x64, never used
#endif     // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
234 235 236
};

// 16 bit left mask to shift and set for each uv prediction size.
237
static const uint16_t left_prediction_mask_uv[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
238 239 240
  0x0001,  // BLOCK_2X2,
  0x0001,  // BLOCK_2X4,
  0x0001,  // BLOCK_4X2,
Jingning Han's avatar
Jingning Han committed
241 242 243 244 245 246 247 248 249 250 251 252
  0x0001,  // BLOCK_4X4,
  0x0001,  // BLOCK_4X8,
  0x0001,  // BLOCK_8X4,
  0x0001,  // BLOCK_8X8,
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8,
  0x0001,  // BLOCK_16X16,
  0x0011,  // BLOCK_16X32,
  0x0001,  // BLOCK_32X16,
  0x0011,  // BLOCK_32X32,
  0x1111,  // BLOCK_32X64
  0x0011,  // BLOCK_64X32,
253 254 255 256
  0x1111,  // BLOCK_64X64,
  0x0001,  // BLOCK_4X16,
  0x0001,  // BLOCK_16X4,
  0x0011,  // BLOCK_8X32,
257 258 259
  0x0001,  // BLOCK_32X8,
  0x1111,  // BLOCK_16X64,
  0x0001,  // BLOCK_64X16,
Jingning Han's avatar
Jingning Han committed
260
};
261

Jingning Han's avatar
Jingning Han committed
262
// 16 bit above mask to shift and set for uv each prediction size.
263
static const uint16_t above_prediction_mask_uv[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
264 265 266
  0x0001,  // BLOCK_2X2
  0x0001,  // BLOCK_2X4
  0x0001,  // BLOCK_4X2
Jingning Han's avatar
Jingning Han committed
267 268 269 270 271 272 273 274 275 276 277 278
  0x0001,  // BLOCK_4X4
  0x0001,  // BLOCK_4X8
  0x0001,  // BLOCK_8X4
  0x0001,  // BLOCK_8X8
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8
  0x0001,  // BLOCK_16X16
  0x0001,  // BLOCK_16X32,
  0x0003,  // BLOCK_32X16,
  0x0003,  // BLOCK_32X32,
  0x0003,  // BLOCK_32X64,
  0x000f,  // BLOCK_64X32,
279 280 281 282
  0x000f,  // BLOCK_64X64,
  0x0001,  // BLOCK_4X16,
  0x0001,  // BLOCK_16X4,
  0x0001,  // BLOCK_8X32,
283 284 285
  0x0003,  // BLOCK_32X8,
  0x0001,  // BLOCK_16X64,
  0x000f,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
286 287 288
};

// 64 bit mask to shift and set for each uv prediction size
289
static const uint16_t size_mask_uv[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
290 291 292
  0x0001,  // BLOCK_2X2
  0x0001,  // BLOCK_2X4
  0x0001,  // BLOCK_4X2
Jingning Han's avatar
Jingning Han committed
293 294 295 296 297 298 299 300 301 302 303 304
  0x0001,  // BLOCK_4X4
  0x0001,  // BLOCK_4X8
  0x0001,  // BLOCK_8X4
  0x0001,  // BLOCK_8X8
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8
  0x0001,  // BLOCK_16X16
  0x0011,  // BLOCK_16X32,
  0x0003,  // BLOCK_32X16,
  0x0033,  // BLOCK_32X32,
  0x3333,  // BLOCK_32X64,
  0x00ff,  // BLOCK_64X32,
305 306 307 308
  0xffff,  // BLOCK_64X64,
  0x0001,  // BLOCK_4X16,
  0x0001,  // BLOCK_16X4,
  0x0011,  // BLOCK_8X32,
309 310 311
  0x0003,  // BLOCK_32X8,
  0x1111,  // BLOCK_16X64,
  0x000f,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
312
};
clang-format's avatar
clang-format committed
313
static const uint16_t left_border_uv = 0x1111;
Jingning Han's avatar
Jingning Han committed
314 315
static const uint16_t above_border_uv = 0x000f;

316
static const int mode_lf_lut[] = {
317 318
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // INTRA_MODES
  1, 1, 0, 1,                             // INTER_MODES (GLOBALMV == 0)
Sarah Parker's avatar
Sarah Parker committed
319
  1, 1, 1, 1, 1, 1, 0, 1  // INTER_COMPOUND_MODES (GLOBAL_GLOBALMV == 0)
Jingning Han's avatar
Jingning Han committed
320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
};

static void update_sharpness(loop_filter_info_n *lfi, int sharpness_lvl) {
  int lvl;

  // For each possible value for the loop filter fill out limits
  for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++) {
    // Set loop filter parameters that control sharpness.
    int block_inside_limit = lvl >> ((sharpness_lvl > 0) + (sharpness_lvl > 4));

    if (sharpness_lvl > 0) {
      if (block_inside_limit > (9 - sharpness_lvl))
        block_inside_limit = (9 - sharpness_lvl);
    }

clang-format's avatar
clang-format committed
335
    if (block_inside_limit < 1) block_inside_limit = 1;
Jingning Han's avatar
Jingning Han committed
336 337 338 339 340 341

    memset(lfi->lfthr[lvl].lim, block_inside_limit, SIMD_WIDTH);
    memset(lfi->lfthr[lvl].mblim, (2 * (lvl + 2) + block_inside_limit),
           SIMD_WIDTH);
  }
}
Fangwen Fu's avatar
Fangwen Fu committed
342 343 344
#if CONFIG_EXT_DELTA_Q
static uint8_t get_filter_level(const AV1_COMMON *cm,
                                const loop_filter_info_n *lfi_n,
Cheng Chen's avatar
Cheng Chen committed
345
#if CONFIG_LOOPFILTER_LEVEL
346
                                const int dir_idx, int plane,
Cheng Chen's avatar
Cheng Chen committed
347 348 349
#endif
#if CONFIG_LPF_SB
                                int mi_row, int mi_col,
350
#endif
Fangwen Fu's avatar
Fangwen Fu committed
351
                                const MB_MODE_INFO *mbmi) {
352
#if CONFIG_LPF_SB
Cheng Chen's avatar
Cheng Chen committed
353
  return cm->mi[mi_row * cm->mi_stride + mi_col].mbmi.filt_lvl;
354 355
#endif

Fangwen Fu's avatar
Fangwen Fu committed
356 357
  const int segment_id = mbmi->segment_id;
  if (cm->delta_lf_present_flag) {
Cheng Chen's avatar
Cheng Chen committed
358
#if CONFIG_LOOPFILTER_LEVEL
359 360 361 362 363 364 365
    int delta_lf;
    if (cm->delta_lf_multi) {
      const int delta_lf_idx = delta_lf_id_lut[plane][dir_idx];
      delta_lf = mbmi->curr_delta_lf[delta_lf_idx];
    } else {
      delta_lf = mbmi->current_delta_lf_from_base;
    }
366
    int lvl_seg =
367
        clamp(delta_lf + cm->lf.filter_level[dir_idx], 0, MAX_LOOP_FILTER);
368
#else
Fangwen Fu's avatar
Fangwen Fu committed
369 370
    int lvl_seg = clamp(mbmi->current_delta_lf_from_base + cm->lf.filter_level,
                        0, MAX_LOOP_FILTER);
371
#endif
Fangwen Fu's avatar
Fangwen Fu committed
372
    const int scale = 1 << (lvl_seg >> 5);
373 374 375 376 377
#if CONFIG_LOOPFILTER_LEVEL
    assert(plane >= 0 && plane <= 2);
    const int seg_lf_feature_id = seg_lvl_lf_lut[plane][dir_idx];
    if (segfeature_active(&cm->seg, segment_id, seg_lf_feature_id)) {
      const int data = get_segdata(&cm->seg, segment_id, seg_lf_feature_id);
378
      lvl_seg = clamp(lvl_seg + data, 0, MAX_LOOP_FILTER);
379 380
    }
#else
Fangwen Fu's avatar
Fangwen Fu committed
381 382
    if (segfeature_active(&cm->seg, segment_id, SEG_LVL_ALT_LF)) {
      const int data = get_segdata(&cm->seg, segment_id, SEG_LVL_ALT_LF);
383
      lvl_seg = clamp(lvl_seg + data, 0, MAX_LOOP_FILTER);
Fangwen Fu's avatar
Fangwen Fu committed
384
    }
385
#endif  // CONFIG_LOOPFILTER_LEVEL
Jingning Han's avatar
Jingning Han committed
386

Fangwen Fu's avatar
Fangwen Fu committed
387 388 389 390 391 392 393 394
    if (cm->lf.mode_ref_delta_enabled) {
      lvl_seg += cm->lf.ref_deltas[mbmi->ref_frame[0]] * scale;
      if (mbmi->ref_frame[0] > INTRA_FRAME)
        lvl_seg += cm->lf.mode_deltas[mode_lf_lut[mbmi->mode]] * scale;
      lvl_seg = clamp(lvl_seg, 0, MAX_LOOP_FILTER);
    }
    return lvl_seg;
  } else {
Cheng Chen's avatar
Cheng Chen committed
395
#if CONFIG_LOOPFILTER_LEVEL
396 397 398
    return lfi_n
        ->lvl[segment_id][dir_idx][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]];
#else
Fangwen Fu's avatar
Fangwen Fu committed
399
    return lfi_n->lvl[segment_id][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]];
400
#endif
Fangwen Fu's avatar
Fangwen Fu committed
401 402 403
  }
}
#else
Jingning Han's avatar
Jingning Han committed
404 405
static uint8_t get_filter_level(const loop_filter_info_n *lfi_n,
                                const MB_MODE_INFO *mbmi) {
Cheng Chen's avatar
Cheng Chen committed
406 407 408 409
#if CONFIG_LPF_SB
  return mbmi->filt_lvl;
#endif

410 411
  const int segment_id = mbmi->segment_id;
  return lfi_n->lvl[segment_id][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]];
Jingning Han's avatar
Jingning Han committed
412
}
Fangwen Fu's avatar
Fangwen Fu committed
413
#endif
Jingning Han's avatar
Jingning Han committed
414

Yaowu Xu's avatar
Yaowu Xu committed
415
void av1_loop_filter_init(AV1_COMMON *cm) {
416
  assert(MB_MODE_COUNT == NELEMENTS(mode_lf_lut));
Jingning Han's avatar
Jingning Han committed
417 418 419 420 421 422 423 424 425 426 427 428 429
  loop_filter_info_n *lfi = &cm->lf_info;
  struct loopfilter *lf = &cm->lf;
  int lvl;

  // init limits for given sharpness
  update_sharpness(lfi, lf->sharpness_level);
  lf->last_sharpness_level = lf->sharpness_level;

  // init hev threshold const vectors
  for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++)
    memset(lfi->lfthr[lvl].hev_thr, (lvl >> 4), SIMD_WIDTH);
}

430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450
#if CONFIG_LPF_SB
void av1_loop_filter_sb_level_init(AV1_COMMON *cm, int mi_row, int mi_col,
                                   int lvl) {
  const int mi_row_start = AOMMAX(0, mi_row - FILT_BOUNDARY_MI_OFFSET);
  const int mi_col_start = AOMMAX(0, mi_col - FILT_BOUNDARY_MI_OFFSET);
  const int mi_row_range = mi_row - FILT_BOUNDARY_MI_OFFSET + MAX_MIB_SIZE;
  const int mi_col_range = mi_col - FILT_BOUNDARY_MI_OFFSET + MAX_MIB_SIZE;
  const int mi_row_end = AOMMIN(mi_row_range, cm->mi_rows);
  const int mi_col_end = AOMMIN(mi_col_range, cm->mi_cols);

  int row, col;
  for (row = mi_row_start; row < mi_row_end; ++row) {
    for (col = mi_col_start; col < mi_col_end; ++col) {
      // Note: can't use cm->mi_grid_visible. Because for each partition,
      // all visible pointers will point to the first of the partition.
      cm->mi[row * cm->mi_stride + col].mbmi.filt_lvl = lvl;
    }
  }
}
#endif  // CONFIG_LPF_SB

451
void av1_loop_filter_frame_init(AV1_COMMON *cm, int default_filt_lvl,
452 453 454 455 456 457
                                int default_filt_lvl_r
#if CONFIG_LOOPFILTER_LEVEL
                                ,
                                int plane
#endif
                                ) {
Jingning Han's avatar
Jingning Han committed
458 459 460 461
  int seg_id;
  // n_shift is the multiplier for lf_deltas
  // the multiplier is 1 for when filter_lvl is between 0 and 31;
  // 2 when filter_lvl is between 32 and 63
462
  int scale = 1 << (default_filt_lvl >> 5);
Jingning Han's avatar
Jingning Han committed
463 464 465 466 467 468 469 470 471 472 473
  loop_filter_info_n *const lfi = &cm->lf_info;
  struct loopfilter *const lf = &cm->lf;
  const struct segmentation *const seg = &cm->seg;

  // update limits if sharpness has changed
  if (lf->last_sharpness_level != lf->sharpness_level) {
    update_sharpness(lfi, lf->sharpness_level);
    lf->last_sharpness_level = lf->sharpness_level;
  }

  for (seg_id = 0; seg_id < MAX_SEGMENTS; seg_id++) {
474 475 476 477 478 479 480
    for (int dir = 0; dir < 2; ++dir) {
      int lvl_seg = (dir == 0) ? default_filt_lvl : default_filt_lvl_r;
#if CONFIG_LOOPFILTER_LEVEL
      assert(plane >= 0 && plane <= 2);
      const int seg_lf_feature_id = seg_lvl_lf_lut[plane][dir];
      if (segfeature_active(seg, seg_id, seg_lf_feature_id)) {
        const int data = get_segdata(&cm->seg, seg_id, seg_lf_feature_id);
481
        lvl_seg = clamp(lvl_seg + data, 0, MAX_LOOP_FILTER);
482 483 484 485
      }
#else
      if (segfeature_active(seg, seg_id, SEG_LVL_ALT_LF)) {
        const int data = get_segdata(seg, seg_id, SEG_LVL_ALT_LF);
486
        lvl_seg = clamp(lvl_seg + data, 0, MAX_LOOP_FILTER);
487 488
      }
#endif  // CONFIG_LOOPFILTER_LEVEL
Jingning Han's avatar
Jingning Han committed
489

490 491 492 493 494 495 496 497 498 499
      if (!lf->mode_ref_delta_enabled) {
// we could get rid of this if we assume that deltas are set to
// zero when not in use; encoder always uses deltas
#if CONFIG_LOOPFILTER_LEVEL
        memset(lfi->lvl[seg_id][dir], lvl_seg, sizeof(lfi->lvl[seg_id][dir]));
#else
        memset(lfi->lvl[seg_id], lvl_seg, sizeof(lfi->lvl[seg_id]));
#endif  // CONFIG_LOOPFILTER_LEVEL
      } else {
        int ref, mode;
Cheng Chen's avatar
Cheng Chen committed
500
#if CONFIG_LOOPFILTER_LEVEL
501 502 503 504 505 506 507 508 509 510 511 512 513 514 515
        scale = 1 << (lvl_seg >> 5);

        const int intra_lvl = lvl_seg + lf->ref_deltas[INTRA_FRAME] * scale;
        lfi->lvl[seg_id][dir][INTRA_FRAME][0] =
            clamp(intra_lvl, 0, MAX_LOOP_FILTER);

        for (ref = LAST_FRAME; ref < TOTAL_REFS_PER_FRAME; ++ref) {
          for (mode = 0; mode < MAX_MODE_LF_DELTAS; ++mode) {
            const int inter_lvl = lvl_seg + lf->ref_deltas[ref] * scale +
                                  lf->mode_deltas[mode] * scale;
            lfi->lvl[seg_id][dir][ref][mode] =
                clamp(inter_lvl, 0, MAX_LOOP_FILTER);
          }
        }
#else
516 517 518 519 520 521 522 523 524 525
        (void)default_filt_lvl_r;
        const int intra_lvl = lvl_seg + lf->ref_deltas[INTRA_FRAME] * scale;
        lfi->lvl[seg_id][INTRA_FRAME][0] = clamp(intra_lvl, 0, MAX_LOOP_FILTER);

        for (ref = LAST_FRAME; ref < TOTAL_REFS_PER_FRAME; ++ref) {
          for (mode = 0; mode < MAX_MODE_LF_DELTAS; ++mode) {
            const int inter_lvl = lvl_seg + lf->ref_deltas[ref] * scale +
                                  lf->mode_deltas[mode] * scale;
            lfi->lvl[seg_id][ref][mode] = clamp(inter_lvl, 0, MAX_LOOP_FILTER);
          }
Jingning Han's avatar
Jingning Han committed
526
        }
527
#endif
528
      }
Jingning Han's avatar
Jingning Han committed
529 530 531 532
    }
  }
}

clang-format's avatar
clang-format committed
533 534
static void filter_selectively_vert_row2(int subsampling_factor, uint8_t *s,
                                         int pitch, unsigned int mask_16x16_l,
Jingning Han's avatar
Jingning Han committed
535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562
                                         unsigned int mask_8x8_l,
                                         unsigned int mask_4x4_l,
                                         unsigned int mask_4x4_int_l,
                                         const loop_filter_info_n *lfi_n,
                                         const uint8_t *lfl) {
  const int mask_shift = subsampling_factor ? 4 : 8;
  const int mask_cutoff = subsampling_factor ? 0xf : 0xff;
  const int lfl_forward = subsampling_factor ? 4 : 8;

  unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff;
  unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff;
  unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff;
  unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff;
  unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff;
  unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff;
  unsigned int mask;

  for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 |
              mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1;
       mask; mask >>= 1) {
    const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl;
    const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward);

    if (mask & 1) {
      if ((mask_16x16_0 | mask_16x16_1) & 1) {
        if ((mask_16x16_0 & mask_16x16_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
563
          aom_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
564 565
                                   lfi0->hev_thr);
        } else if (mask_16x16_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
566
          aom_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
567
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
568
          aom_lpf_vertical_16(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
clang-format's avatar
clang-format committed
569
                              lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
570 571 572 573 574
        }
      }

      if ((mask_8x8_0 | mask_8x8_1) & 1) {
        if ((mask_8x8_0 & mask_8x8_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
575
          aom_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
576 577 578
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_8x8_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
579
          aom_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
580
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
581
          aom_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
582
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
583 584 585 586 587
        }
      }

      if ((mask_4x4_0 | mask_4x4_1) & 1) {
        if ((mask_4x4_0 & mask_4x4_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
588
          aom_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
589 590 591
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_4x4_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
592
          aom_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
593
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
594
          aom_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
595
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
596 597 598 599 600
        }
      }

      if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) {
        if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
601
          aom_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
602 603 604
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_4x4_int_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
605
          aom_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim,
606
                             lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
607
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
608
          aom_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim, lfi1->lim,
609
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626
        }
      }
    }

    s += 8;
    lfl += 1;
    mask_16x16_0 >>= 1;
    mask_8x8_0 >>= 1;
    mask_4x4_0 >>= 1;
    mask_4x4_int_0 >>= 1;
    mask_16x16_1 >>= 1;
    mask_8x8_1 >>= 1;
    mask_4x4_1 >>= 1;
    mask_4x4_int_1 >>= 1;
  }
}

627
#if CONFIG_HIGHBITDEPTH
clang-format's avatar
clang-format committed
628 629 630 631 632
static void highbd_filter_selectively_vert_row2(
    int subsampling_factor, uint16_t *s, int pitch, unsigned int mask_16x16_l,
    unsigned int mask_8x8_l, unsigned int mask_4x4_l,
    unsigned int mask_4x4_int_l, const loop_filter_info_n *lfi_n,
    const uint8_t *lfl, int bd) {
Jingning Han's avatar
Jingning Han committed
633 634 635 636 637 638 639 640 641 642 643 644 645 646 647
  const int mask_shift = subsampling_factor ? 4 : 8;
  const int mask_cutoff = subsampling_factor ? 0xf : 0xff;
  const int lfl_forward = subsampling_factor ? 4 : 8;

  unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff;
  unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff;
  unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff;
  unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff;
  unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff;
  unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff;
  unsigned int mask;

  for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 |
clang-format's avatar
clang-format committed
648
              mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1;
Jingning Han's avatar
Jingning Han committed
649 650 651 652 653 654 655
       mask; mask >>= 1) {
    const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl;
    const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward);

    if (mask & 1) {
      if ((mask_16x16_0 | mask_16x16_1) & 1) {
        if ((mask_16x16_0 & mask_16x16_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
656
          aom_highbd_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
657 658
                                          lfi0->hev_thr, bd);
        } else if (mask_16x16_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
659
          aom_highbd_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
660 661
                                     lfi0->hev_thr, bd);
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
662
          aom_highbd_lpf_vertical_16(s + 8 * pitch, pitch, lfi1->mblim,
Jingning Han's avatar
Jingning Han committed
663 664 665 666 667 668
                                     lfi1->lim, lfi1->hev_thr, bd);
        }
      }

      if ((mask_8x8_0 | mask_8x8_1) & 1) {
        if ((mask_8x8_0 & mask_8x8_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
669
          aom_highbd_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
670 671 672
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_8x8_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
673
          aom_highbd_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim,
674
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
675
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
676
          aom_highbd_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim,
677
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
678 679 680 681 682
        }
      }

      if ((mask_4x4_0 | mask_4x4_1) & 1) {
        if ((mask_4x4_0 & mask_4x4_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
683
          aom_highbd_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
684 685 686
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_4x4_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
687
          aom_highbd_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim,
688
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
689
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
690
          aom_highbd_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim,
691
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
692 693 694 695 696
        }
      }

      if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) {
        if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
697
          aom_highbd_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
698 699 700
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_4x4_int_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
701
          aom_highbd_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim,
702
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
703
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
704
          aom_highbd_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim,
705
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721
        }
      }
    }

    s += 8;
    lfl += 1;
    mask_16x16_0 >>= 1;
    mask_8x8_0 >>= 1;
    mask_4x4_0 >>= 1;
    mask_4x4_int_0 >>= 1;
    mask_16x16_1 >>= 1;
    mask_8x8_1 >>= 1;
    mask_4x4_1 >>= 1;
    mask_4x4_int_1 >>= 1;
  }
}
722
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
723

clang-format's avatar
clang-format committed
724 725 726
static void filter_selectively_horiz(
    uint8_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8,
    unsigned int mask_4x4, unsigned int mask_4x4_int,
727
    const loop_filter_info_n *lfi_n, const uint8_t *lfl) {
Jingning Han's avatar
Jingning Han committed
728 729 730
  unsigned int mask;
  int count;

clang-format's avatar
clang-format committed
731 732
  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask;
       mask >>= count) {
Jingning Han's avatar
Jingning Han committed
733 734 735 736 737 738
    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;

    count = 1;
    if (mask & 1) {
      if (mask_16x16 & 1) {
        if ((mask_16x16 & 3) == 3) {
James Zern's avatar
James Zern committed
739
          aom_lpf_horizontal_16_dual(s, pitch, lfi->mblim, lfi->lim,
740
                                     lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
741 742
          count = 2;
        } else {
James Zern's avatar
James Zern committed
743
          aom_lpf_horizontal_16(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
744 745 746 747 748 749
        }
      } else if (mask_8x8 & 1) {
        if ((mask_8x8 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
750
          aom_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
751 752 753 754
                                    lfi->hev_thr, lfin->mblim, lfin->lim,
                                    lfin->hev_thr);

          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
755
            aom_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
Jingning Han's avatar
Jingning Han committed
756 757 758 759
                                      lfi->lim, lfi->hev_thr, lfin->mblim,
                                      lfin->lim, lfin->hev_thr);
          } else {
            if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
760
              aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
761
                                   lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
762
            else if (mask_4x4_int & 2)
Yaowu Xu's avatar
Yaowu Xu committed
763
              aom_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
764
                                   lfin->lim, lfin->hev_thr);
Jingning Han's avatar
Jingning Han committed
765 766 767
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
768
          aom_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
769 770

          if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
771
            aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
772
                                 lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
773 774 775 776 777 778
        }
      } else if (mask_4x4 & 1) {
        if ((mask_4x4 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
779
          aom_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
780 781
                                    lfi->hev_thr, lfin->mblim, lfin->lim,
                                    lfin->hev_thr);
Cheng Chen's avatar
Cheng Chen committed
782

Jingning Han's avatar
Jingning Han committed
783
          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
784
            aom_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
Jingning Han's avatar
Jingning Han committed
785 786 787 788
                                      lfi->lim, lfi->hev_thr, lfin->mblim,
                                      lfin->lim, lfin->hev_thr);
          } else {
            if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
789
              aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
790
                                   lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
791
            else if (mask_4x4_int & 2)
Yaowu Xu's avatar
Yaowu Xu committed
792
              aom_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
793
                                   lfin->lim, lfin->hev_thr);
Jingning Han's avatar
Jingning Han committed
794 795 796
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
797
          aom_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
798 799

          if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
800
            aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
801
                                 lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
802 803
        }
      } else if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
804
        aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
805
                             lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
806 807 808 809 810 811 812 813 814 815 816
      }
    }
    s += 8 * count;
    lfl += count;
    mask_16x16 >>= count;
    mask_8x8 >>= count;
    mask_4x4 >>= count;
    mask_4x4_int >>= count;
  }
}

817
#if CONFIG_HIGHBITDEPTH
clang-format's avatar
clang-format committed
818 819 820 821
static void highbd_filter_selectively_horiz(
    uint16_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8,
    unsigned int mask_4x4, unsigned int mask_4x4_int,
    const loop_filter_info_n *lfi_n, const uint8_t *lfl, int bd) {
Jingning Han's avatar
Jingning Han committed
822 823 824
  unsigned int mask;
  int count;

clang-format's avatar
clang-format committed
825 826
  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask;
       mask >>= count) {
Jingning Han's avatar
Jingning Han committed
827 828 829 830 831 832
    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;

    count = 1;
    if (mask & 1) {
      if (mask_16x16 & 1) {
        if ((mask_16x16 & 3) == 3) {
833
          aom_highbd_lpf_horizontal_16_dual(s, pitch, lfi->mblim, lfi->lim,
834
                                            lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
835 836
          count = 2;
        } else {
837 838
          aom_highbd_lpf_horizontal_16(s, pitch, lfi->mblim, lfi->lim,
                                       lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
839 840 841 842 843 844
        }
      } else if (mask_8x8 & 1) {
        if ((mask_8x8 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
845
          aom_highbd_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
846 847 848 849
                                           lfi->hev_thr, lfin->mblim, lfin->lim,
                                           lfin->hev_thr, bd);

          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
850
            aom_highbd_lpf_horizontal_4_dual(
clang-format's avatar
clang-format committed
851 852
                s + 4 * pitch, pitch, lfi->mblim, lfi->lim, lfi->hev_thr,
                lfin->mblim, lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
853 854
          } else {
            if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
855
              aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
856
                                          lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
857
            } else if (mask_4x4_int & 2) {
Yaowu Xu's avatar
Yaowu Xu committed
858
              aom_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
859
                                          lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
860 861 862 863
            }
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
864
          aom_highbd_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim,
865
                                      lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
866 867

          if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
868
            aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
869
                                        lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
870 871 872 873 874 875 876
          }
        }
      } else if (mask_4x4 & 1) {
        if ((mask_4x4 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
877
          aom_highbd_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
878 879 880
                                           lfi->hev_thr, lfin->mblim, lfin->lim,
                                           lfin->hev_thr, bd);
          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
881
            aom_highbd_lpf_horizontal_4_dual(
clang-format's avatar
clang-format committed
882 883
                s + 4 * pitch, pitch, lfi->mblim, lfi->lim, lfi->hev_thr,
                lfin->mblim, lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
884 885
          } else {
            if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
886
              aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
887
                                          lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
888
            } else if (mask_4x4_int & 2) {
Yaowu Xu's avatar
Yaowu Xu committed
889
              aom_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
890
                                          lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
891 892 893 894
            }
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
895
          aom_highbd_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim,
896
                                      lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
897 898

          if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
899
            aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
900
                                        lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
901 902 903
          }
        }
      } else if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
904
        aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
905
                                    lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
906 907 908 909 910 911 912 913 914 915
      }
    }
    s += 8 * count;
    lfl += count;
    mask_16x16 >>= count;
    mask_8x8 >>= count;
    mask_4x4 >>= count;
    mask_4x4_int >>= count;
  }
}
916
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
917 918 919 920 921 922 923 924

// This function ors into the current lfm structure, where to do loop
// filters for the specific mi we are looking at. It uses information
// including the block_size_type (32x16, 32x32, etc.), the transform size,
// whether there were any coefficients encoded, and the loop filter strength
// block we are currently looking at. Shift is used to position the
// 1's we produce.
// TODO(JBB) Need another function for different resolution color..
925 926 927 928
static void build_masks(AV1_COMMON *const cm,
                        const loop_filter_info_n *const lfi_n,
                        const MODE_INFO *mi, const int shift_y,
                        const int shift_uv, LOOP_FILTER_MASK *lfm) {
Jingning Han's avatar
Jingning Han committed
929 930
  const MB_MODE_INFO *mbmi = &mi->mbmi;
  const BLOCK_SIZE block_size = mbmi->sb_type;
931 932
  // TODO(debargha): Check if masks can be setup correctly when
  // rectangular transfroms are used with the EXT_TX expt.
933 934 935
  const TX_SIZE tx_size_y = txsize_sqr_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_left = txsize_horz_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_above = txsize_vert_map[mbmi->tx_size];
936
  const TX_SIZE tx_size_uv =
937 938 939 940 941
      txsize_sqr_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
  const TX_SIZE tx_size_uv_left =
      txsize_horz_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
  const TX_SIZE tx_size_uv_above =
      txsize_vert_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
Fangwen Fu's avatar
Fangwen Fu committed
942
#if CONFIG_EXT_DELTA_Q
Cheng Chen's avatar
Cheng Chen committed
943
#if CONFIG_LOOPFILTER_LEVEL
944
  const int filter_level = get_filter_level(cm, lfi_n, 0, 0, mbmi);
Cheng Chen's avatar
Cheng Chen committed
945 946 947
#else
#if CONFIG_LPF_SB
  const int filter_level = get_filter_level(cm, lfi_n, 0, 0, mbmi);
948
#else
Fangwen Fu's avatar
Fangwen Fu committed
949
  const int filter_level = get_filter_level(cm, lfi_n, mbmi);
Cheng Chen's avatar
Cheng Chen committed
950
#endif  // CONFIG_LPF_SB
951
#endif
Fangwen Fu's avatar
Fangwen Fu committed
952
#else
Jingning Han's avatar
Jingning Han committed
953
  const int filter_level = get_filter_level(lfi_n, mbmi);
954
  (void)cm;
Fangwen Fu's avatar
Fangwen Fu committed
955
#endif
956 957
  uint64_t *const left_y = &lfm->left_y[tx_size_y_left];
  uint64_t *const above_y = &lfm->above_y[tx_size_y_above];
Jingning Han's avatar
Jingning Han committed
958
  uint64_t *const int_4x4_y = &lfm->int_4x4_y;
959 960
  uint16_t *const left_uv = &lfm->left_uv[tx_size_uv_left];
  uint16_t *const above_uv = &lfm->above_uv[tx_size_uv_above];
961
  uint16_t *const int_4x4_uv = &lfm->left_int_4x4_uv;
Jingning Han's avatar
Jingning Han committed
962 963 964 965 966 967 968 969
  int i;

  // If filter level is 0 we don't loop filter.
  if (!filter_level) {
    return;
  } else {
    const int w = num_8x8_blocks_wide_lookup[block_size];
    const int h = num_8x8_blocks_high_lookup[block_size];
970 971 972
    const int row = (shift_y >> MAX_MIB_SIZE_LOG2);
    const int col = shift_y - (row << MAX_MIB_SIZE_LOG2);

clang-format's avatar
clang-format committed
973
    for (i = 0; i < h; i++) memset(&lfm->lfl_y[row + i][col], filter_level, w);
Jingning Han's avatar
Jingning Han committed
974 975 976 977 978 979 980 981 982 983 984 985