av1_loopfilter.c 101 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
 */

Debargha Mukherjee's avatar
Debargha Mukherjee committed
12
13
#include <math.h>

Yaowu Xu's avatar
Yaowu Xu committed
14
15
16
17
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_mem/aom_mem.h"
18
#include "aom_ports/mem.h"
19
20
21
#include "av1/common/av1_loopfilter.h"
#include "av1/common/onyxc_int.h"
#include "av1/common/reconinter.h"
22
#include "av1/common/seg_common.h"
Jingning Han's avatar
Jingning Han committed
23

24
25
26
27
28
29
#if CONFIG_LOOPFILTER_LEVEL
static const SEG_LVL_FEATURES seg_lvl_lf_lut[MAX_MB_PLANE][2] = {
  { SEG_LVL_ALT_LF_Y_V, SEG_LVL_ALT_LF_Y_H },
  { SEG_LVL_ALT_LF_U, SEG_LVL_ALT_LF_U },
  { SEG_LVL_ALT_LF_V, SEG_LVL_ALT_LF_V }
};
30
31
32
33
34
35

#if CONFIG_EXT_DELTA_Q
static const int delta_lf_id_lut[MAX_MB_PLANE][2] = {
  { 0, 1 }, { 2, 2 }, { 3, 3 }
};
#endif  // CONFIG_EXT_DELTA_Q
36
37
#endif  // CONFIG_LOOPFILTER_LEVEL

38
#define PARALLEL_DEBLOCKING_15TAPLUMAONLY 1
39
#define PARALLEL_DEBLOCKING_DISABLE_15TAP 0
Ola Hugosson's avatar
Ola Hugosson committed
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
#if CONFIG_DEBLOCK_13TAP
#define PARALLEL_DEBLOCKING_5_TAP_CHROMA 1
#else
#define PARALLEL_DEBLOCKING_5_TAP_CHROMA 0
#endif

#if PARALLEL_DEBLOCKING_5_TAP_CHROMA
extern void aom_lpf_vertical_6_c(uint8_t *s, int pitch, const uint8_t *blimit,
                                 const uint8_t *limit, const uint8_t *thresh);

extern void aom_lpf_horizontal_6_c(uint8_t *s, int p, const uint8_t *blimit,
                                   const uint8_t *limit, const uint8_t *thresh);

extern void aom_highbd_lpf_horizontal_6_c(uint16_t *s, int p,
                                          const uint8_t *blimit,
                                          const uint8_t *limit,
                                          const uint8_t *thresh, int bd);

extern void aom_highbd_lpf_vertical_6_c(uint16_t *s, int pitch,
                                        const uint8_t *blimit,
                                        const uint8_t *limit,
                                        const uint8_t *thresh, int bd);
#endif
63

Jingning Han's avatar
Jingning Han committed
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
// 64 bit masks for left transform size. Each 1 represents a position where
// we should apply a loop filter across the left border of an 8x8 block
// boundary.
//
// In the case of TX_16X16->  ( in low order byte first we end up with
// a mask that looks like this
//
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//
// A loopfilter should be applied to every other 8x8 horizontally.
clang-format's avatar
clang-format committed
81
static const uint64_t left_64x64_txform_mask[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
82
83
84
85
  0xffffffffffffffffULL,  // TX_4X4
  0xffffffffffffffffULL,  // TX_8x8
  0x5555555555555555ULL,  // TX_16x16
  0x1111111111111111ULL,  // TX_32x32
86
87
88
#if CONFIG_TX64X64
  0x0101010101010101ULL,  // TX_64x64
#endif                    // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
};

// 64 bit masks for above transform size. Each 1 represents a position where
// we should apply a loop filter across the top border of an 8x8 block
// boundary.
//
// In the case of TX_32x32 ->  ( in low order byte first we end up with
// a mask that looks like this
//
//    11111111
//    00000000
//    00000000
//    00000000
//    11111111
//    00000000
//    00000000
//    00000000
//
// A loopfilter should be applied to every other 4 the row vertically.
clang-format's avatar
clang-format committed
108
static const uint64_t above_64x64_txform_mask[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
109
110
111
112
  0xffffffffffffffffULL,  // TX_4X4
  0xffffffffffffffffULL,  // TX_8x8
  0x00ff00ff00ff00ffULL,  // TX_16x16
  0x000000ff000000ffULL,  // TX_32x32
113
114
115
#if CONFIG_TX64X64
  0x00000000000000ffULL,  // TX_64x64
#endif                    // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
};

// 64 bit masks for prediction sizes (left). Each 1 represents a position
// where left border of an 8x8 block. These are aligned to the right most
// appropriate bit, and then shifted into place.
//
// In the case of TX_16x32 ->  ( low order byte first ) we end up with
// a mask that looks like this :
//
//  10000000
//  10000000
//  10000000
//  10000000
//  00000000
//  00000000
//  00000000
//  00000000
133
static const uint64_t left_prediction_mask[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
134
135
136
  0x0000000000000001ULL,  // BLOCK_2X2,
  0x0000000000000001ULL,  // BLOCK_2X4,
  0x0000000000000001ULL,  // BLOCK_4X2,
Jingning Han's avatar
Jingning Han committed
137
138
139
140
141
142
143
144
145
146
147
148
  0x0000000000000001ULL,  // BLOCK_4X4,
  0x0000000000000001ULL,  // BLOCK_4X8,
  0x0000000000000001ULL,  // BLOCK_8X4,
  0x0000000000000001ULL,  // BLOCK_8X8,
  0x0000000000000101ULL,  // BLOCK_8X16,
  0x0000000000000001ULL,  // BLOCK_16X8,
  0x0000000000000101ULL,  // BLOCK_16X16,
  0x0000000001010101ULL,  // BLOCK_16X32,
  0x0000000000000101ULL,  // BLOCK_32X16,
  0x0000000001010101ULL,  // BLOCK_32X32,
  0x0101010101010101ULL,  // BLOCK_32X64,
  0x0000000001010101ULL,  // BLOCK_64X32,
149
150
151
152
  0x0101010101010101ULL,  // BLOCK_64X64,
  0x0000000000000101ULL,  // BLOCK_4X16,
  0x0000000000000001ULL,  // BLOCK_16X4,
  0x0000000001010101ULL,  // BLOCK_8X32,
153
154
155
  0x0000000000000001ULL,  // BLOCK_32X8,
  0x0101010101010101ULL,  // BLOCK_16X64,
  0x0000000000000101ULL,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
156
157
158
};

// 64 bit mask to shift and set for each prediction size.
159
static const uint64_t above_prediction_mask[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
160
161
162
  0x0000000000000001ULL,  // BLOCK_2X2
  0x0000000000000001ULL,  // BLOCK_2X4
  0x0000000000000001ULL,  // BLOCK_4X2
Jingning Han's avatar
Jingning Han committed
163
164
165
166
167
168
169
170
171
172
173
174
  0x0000000000000001ULL,  // BLOCK_4X4
  0x0000000000000001ULL,  // BLOCK_4X8
  0x0000000000000001ULL,  // BLOCK_8X4
  0x0000000000000001ULL,  // BLOCK_8X8
  0x0000000000000001ULL,  // BLOCK_8X16,
  0x0000000000000003ULL,  // BLOCK_16X8
  0x0000000000000003ULL,  // BLOCK_16X16
  0x0000000000000003ULL,  // BLOCK_16X32,
  0x000000000000000fULL,  // BLOCK_32X16,
  0x000000000000000fULL,  // BLOCK_32X32,
  0x000000000000000fULL,  // BLOCK_32X64,
  0x00000000000000ffULL,  // BLOCK_64X32,
175
176
177
178
  0x00000000000000ffULL,  // BLOCK_64X64,
  0x0000000000000001ULL,  // BLOCK_4X16,
  0x0000000000000003ULL,  // BLOCK_16X4,
  0x0000000000000001ULL,  // BLOCK_8X32,
179
180
181
  0x000000000000000fULL,  // BLOCK_32X8,
  0x0000000000000003ULL,  // BLOCK_16X64,
  0x00000000000000ffULL,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
182
183
};
// 64 bit mask to shift and set for each prediction size. A bit is set for
184
// each 8x8 block that would be in the top left most block of the given block
Jingning Han's avatar
Jingning Han committed
185
// size in the 64x64 block.
186
static const uint64_t size_mask[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
187
188
189
  0x0000000000000001ULL,  // BLOCK_2X2
  0x0000000000000001ULL,  // BLOCK_2X4
  0x0000000000000001ULL,  // BLOCK_4X2
Jingning Han's avatar
Jingning Han committed
190
191
192
193
194
195
196
197
198
199
200
201
  0x0000000000000001ULL,  // BLOCK_4X4
  0x0000000000000001ULL,  // BLOCK_4X8
  0x0000000000000001ULL,  // BLOCK_8X4
  0x0000000000000001ULL,  // BLOCK_8X8
  0x0000000000000101ULL,  // BLOCK_8X16,
  0x0000000000000003ULL,  // BLOCK_16X8
  0x0000000000000303ULL,  // BLOCK_16X16
  0x0000000003030303ULL,  // BLOCK_16X32,
  0x0000000000000f0fULL,  // BLOCK_32X16,
  0x000000000f0f0f0fULL,  // BLOCK_32X32,
  0x0f0f0f0f0f0f0f0fULL,  // BLOCK_32X64,
  0x00000000ffffffffULL,  // BLOCK_64X32,
202
203
204
205
  0xffffffffffffffffULL,  // BLOCK_64X64,
  0x0000000000000101ULL,  // BLOCK_4X16,
  0x0000000000000003ULL,  // BLOCK_16X4,
  0x0000000001010101ULL,  // BLOCK_8X32,
206
207
208
  0x000000000000000fULL,  // BLOCK_32X8,
  0x0303030303030303ULL,  // BLOCK_16X64,
  0x000000000000ffffULL,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
209
210
};

211
// These are used for masking the left and above 32x32 borders.
clang-format's avatar
clang-format committed
212
static const uint64_t left_border = 0x1111111111111111ULL;
Jingning Han's avatar
Jingning Han committed
213
214
215
static const uint64_t above_border = 0x000000ff000000ffULL;

// 16 bit masks for uv transform sizes.
clang-format's avatar
clang-format committed
216
static const uint16_t left_64x64_txform_mask_uv[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
217
218
219
220
  0xffff,  // TX_4X4
  0xffff,  // TX_8x8
  0x5555,  // TX_16x16
  0x1111,  // TX_32x32
221
222
223
#if CONFIG_TX64X64
  0x0101,  // TX_64x64, never used
#endif     // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
224
225
};

clang-format's avatar
clang-format committed
226
static const uint16_t above_64x64_txform_mask_uv[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
227
228
229
230
  0xffff,  // TX_4X4
  0xffff,  // TX_8x8
  0x0f0f,  // TX_16x16
  0x000f,  // TX_32x32
231
232
233
#if CONFIG_TX64X64
  0x0003,  // TX_64x64, never used
#endif     // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
234
235
236
};

// 16 bit left mask to shift and set for each uv prediction size.
237
static const uint16_t left_prediction_mask_uv[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
238
239
240
  0x0001,  // BLOCK_2X2,
  0x0001,  // BLOCK_2X4,
  0x0001,  // BLOCK_4X2,
Jingning Han's avatar
Jingning Han committed
241
242
243
244
245
246
247
248
249
250
251
252
  0x0001,  // BLOCK_4X4,
  0x0001,  // BLOCK_4X8,
  0x0001,  // BLOCK_8X4,
  0x0001,  // BLOCK_8X8,
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8,
  0x0001,  // BLOCK_16X16,
  0x0011,  // BLOCK_16X32,
  0x0001,  // BLOCK_32X16,
  0x0011,  // BLOCK_32X32,
  0x1111,  // BLOCK_32X64
  0x0011,  // BLOCK_64X32,
253
254
255
256
  0x1111,  // BLOCK_64X64,
  0x0001,  // BLOCK_4X16,
  0x0001,  // BLOCK_16X4,
  0x0011,  // BLOCK_8X32,
257
258
259
  0x0001,  // BLOCK_32X8,
  0x1111,  // BLOCK_16X64,
  0x0001,  // BLOCK_64X16,
Jingning Han's avatar
Jingning Han committed
260
};
261

Jingning Han's avatar
Jingning Han committed
262
// 16 bit above mask to shift and set for uv each prediction size.
263
static const uint16_t above_prediction_mask_uv[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
264
265
266
  0x0001,  // BLOCK_2X2
  0x0001,  // BLOCK_2X4
  0x0001,  // BLOCK_4X2
Jingning Han's avatar
Jingning Han committed
267
268
269
270
271
272
273
274
275
276
277
278
  0x0001,  // BLOCK_4X4
  0x0001,  // BLOCK_4X8
  0x0001,  // BLOCK_8X4
  0x0001,  // BLOCK_8X8
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8
  0x0001,  // BLOCK_16X16
  0x0001,  // BLOCK_16X32,
  0x0003,  // BLOCK_32X16,
  0x0003,  // BLOCK_32X32,
  0x0003,  // BLOCK_32X64,
  0x000f,  // BLOCK_64X32,
279
280
281
282
  0x000f,  // BLOCK_64X64,
  0x0001,  // BLOCK_4X16,
  0x0001,  // BLOCK_16X4,
  0x0001,  // BLOCK_8X32,
283
284
285
  0x0003,  // BLOCK_32X8,
  0x0001,  // BLOCK_16X64,
  0x000f,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
286
287
288
};

// 64 bit mask to shift and set for each uv prediction size
289
static const uint16_t size_mask_uv[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
290
291
292
  0x0001,  // BLOCK_2X2
  0x0001,  // BLOCK_2X4
  0x0001,  // BLOCK_4X2
Jingning Han's avatar
Jingning Han committed
293
294
295
296
297
298
299
300
301
302
303
304
  0x0001,  // BLOCK_4X4
  0x0001,  // BLOCK_4X8
  0x0001,  // BLOCK_8X4
  0x0001,  // BLOCK_8X8
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8
  0x0001,  // BLOCK_16X16
  0x0011,  // BLOCK_16X32,
  0x0003,  // BLOCK_32X16,
  0x0033,  // BLOCK_32X32,
  0x3333,  // BLOCK_32X64,
  0x00ff,  // BLOCK_64X32,
305
306
307
308
  0xffff,  // BLOCK_64X64,
  0x0001,  // BLOCK_4X16,
  0x0001,  // BLOCK_16X4,
  0x0011,  // BLOCK_8X32,
309
310
311
  0x0003,  // BLOCK_32X8,
  0x1111,  // BLOCK_16X64,
  0x000f,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
312
};
clang-format's avatar
clang-format committed
313
static const uint16_t left_border_uv = 0x1111;
Jingning Han's avatar
Jingning Han committed
314
315
static const uint16_t above_border_uv = 0x000f;

316
static const int mode_lf_lut[] = {
317
318
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // INTRA_MODES
  1, 1, 0, 1,                             // INTER_MODES (GLOBALMV == 0)
Sarah Parker's avatar
Sarah Parker committed
319
  1, 1, 1, 1, 1, 1, 0, 1  // INTER_COMPOUND_MODES (GLOBAL_GLOBALMV == 0)
Jingning Han's avatar
Jingning Han committed
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
};

static void update_sharpness(loop_filter_info_n *lfi, int sharpness_lvl) {
  int lvl;

  // For each possible value for the loop filter fill out limits
  for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++) {
    // Set loop filter parameters that control sharpness.
    int block_inside_limit = lvl >> ((sharpness_lvl > 0) + (sharpness_lvl > 4));

    if (sharpness_lvl > 0) {
      if (block_inside_limit > (9 - sharpness_lvl))
        block_inside_limit = (9 - sharpness_lvl);
    }

clang-format's avatar
clang-format committed
335
    if (block_inside_limit < 1) block_inside_limit = 1;
Jingning Han's avatar
Jingning Han committed
336
337
338
339
340
341

    memset(lfi->lfthr[lvl].lim, block_inside_limit, SIMD_WIDTH);
    memset(lfi->lfthr[lvl].mblim, (2 * (lvl + 2) + block_inside_limit),
           SIMD_WIDTH);
  }
}
Fangwen Fu's avatar
Fangwen Fu committed
342
343
344
#if CONFIG_EXT_DELTA_Q
static uint8_t get_filter_level(const AV1_COMMON *cm,
                                const loop_filter_info_n *lfi_n,
Cheng Chen's avatar
Cheng Chen committed
345
#if CONFIG_LOOPFILTER_LEVEL
346
                                const int dir_idx, int plane,
Cheng Chen's avatar
Cheng Chen committed
347
348
349
#endif
#if CONFIG_LPF_SB
                                int mi_row, int mi_col,
350
#endif
Fangwen Fu's avatar
Fangwen Fu committed
351
                                const MB_MODE_INFO *mbmi) {
352
#if CONFIG_LPF_SB
Cheng Chen's avatar
Cheng Chen committed
353
  return cm->mi[mi_row * cm->mi_stride + mi_col].mbmi.filt_lvl;
354
355
#endif

Fangwen Fu's avatar
Fangwen Fu committed
356
357
  const int segment_id = mbmi->segment_id;
  if (cm->delta_lf_present_flag) {
Cheng Chen's avatar
Cheng Chen committed
358
#if CONFIG_LOOPFILTER_LEVEL
359
360
361
362
363
364
365
    int delta_lf;
    if (cm->delta_lf_multi) {
      const int delta_lf_idx = delta_lf_id_lut[plane][dir_idx];
      delta_lf = mbmi->curr_delta_lf[delta_lf_idx];
    } else {
      delta_lf = mbmi->current_delta_lf_from_base;
    }
366
    int lvl_seg =
367
        clamp(delta_lf + cm->lf.filter_level[dir_idx], 0, MAX_LOOP_FILTER);
368
#else
Fangwen Fu's avatar
Fangwen Fu committed
369
370
    int lvl_seg = clamp(mbmi->current_delta_lf_from_base + cm->lf.filter_level,
                        0, MAX_LOOP_FILTER);
371
#endif
Fangwen Fu's avatar
Fangwen Fu committed
372
    const int scale = 1 << (lvl_seg >> 5);
373
374
375
376
377
#if CONFIG_LOOPFILTER_LEVEL
    assert(plane >= 0 && plane <= 2);
    const int seg_lf_feature_id = seg_lvl_lf_lut[plane][dir_idx];
    if (segfeature_active(&cm->seg, segment_id, seg_lf_feature_id)) {
      const int data = get_segdata(&cm->seg, segment_id, seg_lf_feature_id);
378
      lvl_seg = clamp(lvl_seg + data, 0, MAX_LOOP_FILTER);
379
380
    }
#else
Fangwen Fu's avatar
Fangwen Fu committed
381
382
    if (segfeature_active(&cm->seg, segment_id, SEG_LVL_ALT_LF)) {
      const int data = get_segdata(&cm->seg, segment_id, SEG_LVL_ALT_LF);
383
      lvl_seg = clamp(lvl_seg + data, 0, MAX_LOOP_FILTER);
Fangwen Fu's avatar
Fangwen Fu committed
384
    }
385
#endif  // CONFIG_LOOPFILTER_LEVEL
Jingning Han's avatar
Jingning Han committed
386

Fangwen Fu's avatar
Fangwen Fu committed
387
388
389
390
391
392
393
394
    if (cm->lf.mode_ref_delta_enabled) {
      lvl_seg += cm->lf.ref_deltas[mbmi->ref_frame[0]] * scale;
      if (mbmi->ref_frame[0] > INTRA_FRAME)
        lvl_seg += cm->lf.mode_deltas[mode_lf_lut[mbmi->mode]] * scale;
      lvl_seg = clamp(lvl_seg, 0, MAX_LOOP_FILTER);
    }
    return lvl_seg;
  } else {
Cheng Chen's avatar
Cheng Chen committed
395
#if CONFIG_LOOPFILTER_LEVEL
396
397
398
    return lfi_n
        ->lvl[segment_id][dir_idx][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]];
#else
Fangwen Fu's avatar
Fangwen Fu committed
399
    return lfi_n->lvl[segment_id][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]];
400
#endif
Fangwen Fu's avatar
Fangwen Fu committed
401
402
403
  }
}
#else
Jingning Han's avatar
Jingning Han committed
404
405
static uint8_t get_filter_level(const loop_filter_info_n *lfi_n,
                                const MB_MODE_INFO *mbmi) {
Cheng Chen's avatar
Cheng Chen committed
406
407
408
409
#if CONFIG_LPF_SB
  return mbmi->filt_lvl;
#endif

410
411
  const int segment_id = mbmi->segment_id;
  return lfi_n->lvl[segment_id][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]];
Jingning Han's avatar
Jingning Han committed
412
}
Fangwen Fu's avatar
Fangwen Fu committed
413
#endif
Jingning Han's avatar
Jingning Han committed
414

Yaowu Xu's avatar
Yaowu Xu committed
415
void av1_loop_filter_init(AV1_COMMON *cm) {
416
  assert(MB_MODE_COUNT == NELEMENTS(mode_lf_lut));
Jingning Han's avatar
Jingning Han committed
417
418
419
420
421
422
423
424
425
426
427
428
429
  loop_filter_info_n *lfi = &cm->lf_info;
  struct loopfilter *lf = &cm->lf;
  int lvl;

  // init limits for given sharpness
  update_sharpness(lfi, lf->sharpness_level);
  lf->last_sharpness_level = lf->sharpness_level;

  // init hev threshold const vectors
  for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++)
    memset(lfi->lfthr[lvl].hev_thr, (lvl >> 4), SIMD_WIDTH);
}

430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
#if CONFIG_LPF_SB
void av1_loop_filter_sb_level_init(AV1_COMMON *cm, int mi_row, int mi_col,
                                   int lvl) {
  const int mi_row_start = AOMMAX(0, mi_row - FILT_BOUNDARY_MI_OFFSET);
  const int mi_col_start = AOMMAX(0, mi_col - FILT_BOUNDARY_MI_OFFSET);
  const int mi_row_range = mi_row - FILT_BOUNDARY_MI_OFFSET + MAX_MIB_SIZE;
  const int mi_col_range = mi_col - FILT_BOUNDARY_MI_OFFSET + MAX_MIB_SIZE;
  const int mi_row_end = AOMMIN(mi_row_range, cm->mi_rows);
  const int mi_col_end = AOMMIN(mi_col_range, cm->mi_cols);

  int row, col;
  for (row = mi_row_start; row < mi_row_end; ++row) {
    for (col = mi_col_start; col < mi_col_end; ++col) {
      // Note: can't use cm->mi_grid_visible. Because for each partition,
      // all visible pointers will point to the first of the partition.
      cm->mi[row * cm->mi_stride + col].mbmi.filt_lvl = lvl;
    }
  }
}
#endif  // CONFIG_LPF_SB

451
void av1_loop_filter_frame_init(AV1_COMMON *cm, int default_filt_lvl,
452
453
454
455
456
457
                                int default_filt_lvl_r
#if CONFIG_LOOPFILTER_LEVEL
                                ,
                                int plane
#endif
                                ) {
Jingning Han's avatar
Jingning Han committed
458
459
460
461
  int seg_id;
  // n_shift is the multiplier for lf_deltas
  // the multiplier is 1 for when filter_lvl is between 0 and 31;
  // 2 when filter_lvl is between 32 and 63
462
  int scale = 1 << (default_filt_lvl >> 5);
Jingning Han's avatar
Jingning Han committed
463
464
465
466
467
468
469
470
471
472
473
  loop_filter_info_n *const lfi = &cm->lf_info;
  struct loopfilter *const lf = &cm->lf;
  const struct segmentation *const seg = &cm->seg;

  // update limits if sharpness has changed
  if (lf->last_sharpness_level != lf->sharpness_level) {
    update_sharpness(lfi, lf->sharpness_level);
    lf->last_sharpness_level = lf->sharpness_level;
  }

  for (seg_id = 0; seg_id < MAX_SEGMENTS; seg_id++) {
474
475
476
477
478
479
480
    for (int dir = 0; dir < 2; ++dir) {
      int lvl_seg = (dir == 0) ? default_filt_lvl : default_filt_lvl_r;
#if CONFIG_LOOPFILTER_LEVEL
      assert(plane >= 0 && plane <= 2);
      const int seg_lf_feature_id = seg_lvl_lf_lut[plane][dir];
      if (segfeature_active(seg, seg_id, seg_lf_feature_id)) {
        const int data = get_segdata(&cm->seg, seg_id, seg_lf_feature_id);
481
        lvl_seg = clamp(lvl_seg + data, 0, MAX_LOOP_FILTER);
482
483
484
485
      }
#else
      if (segfeature_active(seg, seg_id, SEG_LVL_ALT_LF)) {
        const int data = get_segdata(seg, seg_id, SEG_LVL_ALT_LF);
486
        lvl_seg = clamp(lvl_seg + data, 0, MAX_LOOP_FILTER);
487
488
      }
#endif  // CONFIG_LOOPFILTER_LEVEL
Jingning Han's avatar
Jingning Han committed
489

490
491
492
493
494
495
496
497
498
499
      if (!lf->mode_ref_delta_enabled) {
// we could get rid of this if we assume that deltas are set to
// zero when not in use; encoder always uses deltas
#if CONFIG_LOOPFILTER_LEVEL
        memset(lfi->lvl[seg_id][dir], lvl_seg, sizeof(lfi->lvl[seg_id][dir]));
#else
        memset(lfi->lvl[seg_id], lvl_seg, sizeof(lfi->lvl[seg_id]));
#endif  // CONFIG_LOOPFILTER_LEVEL
      } else {
        int ref, mode;
Cheng Chen's avatar
Cheng Chen committed
500
#if CONFIG_LOOPFILTER_LEVEL
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
        scale = 1 << (lvl_seg >> 5);

        const int intra_lvl = lvl_seg + lf->ref_deltas[INTRA_FRAME] * scale;
        lfi->lvl[seg_id][dir][INTRA_FRAME][0] =
            clamp(intra_lvl, 0, MAX_LOOP_FILTER);

        for (ref = LAST_FRAME; ref < TOTAL_REFS_PER_FRAME; ++ref) {
          for (mode = 0; mode < MAX_MODE_LF_DELTAS; ++mode) {
            const int inter_lvl = lvl_seg + lf->ref_deltas[ref] * scale +
                                  lf->mode_deltas[mode] * scale;
            lfi->lvl[seg_id][dir][ref][mode] =
                clamp(inter_lvl, 0, MAX_LOOP_FILTER);
          }
        }
#else
516
517
518
519
520
521
522
523
524
525
        (void)default_filt_lvl_r;
        const int intra_lvl = lvl_seg + lf->ref_deltas[INTRA_FRAME] * scale;
        lfi->lvl[seg_id][INTRA_FRAME][0] = clamp(intra_lvl, 0, MAX_LOOP_FILTER);

        for (ref = LAST_FRAME; ref < TOTAL_REFS_PER_FRAME; ++ref) {
          for (mode = 0; mode < MAX_MODE_LF_DELTAS; ++mode) {
            const int inter_lvl = lvl_seg + lf->ref_deltas[ref] * scale +
                                  lf->mode_deltas[mode] * scale;
            lfi->lvl[seg_id][ref][mode] = clamp(inter_lvl, 0, MAX_LOOP_FILTER);
          }
Jingning Han's avatar
Jingning Han committed
526
        }
527
#endif
528
      }
Jingning Han's avatar
Jingning Han committed
529
530
531
532
    }
  }
}

clang-format's avatar
clang-format committed
533
534
static void filter_selectively_vert_row2(int subsampling_factor, uint8_t *s,
                                         int pitch, unsigned int mask_16x16_l,
Jingning Han's avatar
Jingning Han committed
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
                                         unsigned int mask_8x8_l,
                                         unsigned int mask_4x4_l,
                                         unsigned int mask_4x4_int_l,
                                         const loop_filter_info_n *lfi_n,
                                         const uint8_t *lfl) {
  const int mask_shift = subsampling_factor ? 4 : 8;
  const int mask_cutoff = subsampling_factor ? 0xf : 0xff;
  const int lfl_forward = subsampling_factor ? 4 : 8;

  unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff;
  unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff;
  unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff;
  unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff;
  unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff;
  unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff;
  unsigned int mask;

  for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 |
              mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1;
       mask; mask >>= 1) {
    const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl;
    const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward);

    if (mask & 1) {
      if ((mask_16x16_0 | mask_16x16_1) & 1) {
        if ((mask_16x16_0 & mask_16x16_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
563
          aom_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
564
565
                                   lfi0->hev_thr);
        } else if (mask_16x16_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
566
          aom_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
567
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
568
          aom_lpf_vertical_16(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
clang-format's avatar
clang-format committed
569
                              lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
570
571
572
573
574
        }
      }

      if ((mask_8x8_0 | mask_8x8_1) & 1) {
        if ((mask_8x8_0 & mask_8x8_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
575
          aom_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
576
577
578
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_8x8_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
579
          aom_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
580
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
581
          aom_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
582
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
583
584
585
586
587
        }
      }

      if ((mask_4x4_0 | mask_4x4_1) & 1) {
        if ((mask_4x4_0 & mask_4x4_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
588
          aom_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
589
590
591
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_4x4_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
592
          aom_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
593
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
594
          aom_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
595
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
596
597
598
599
600
        }
      }

      if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) {
        if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
601
          aom_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
602
603
604
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_4x4_int_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
605
          aom_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim,
606
                             lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
607
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
608
          aom_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim, lfi1->lim,
609
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
        }
      }
    }

    s += 8;
    lfl += 1;
    mask_16x16_0 >>= 1;
    mask_8x8_0 >>= 1;
    mask_4x4_0 >>= 1;
    mask_4x4_int_0 >>= 1;
    mask_16x16_1 >>= 1;
    mask_8x8_1 >>= 1;
    mask_4x4_1 >>= 1;
    mask_4x4_int_1 >>= 1;
  }
}

627
#if CONFIG_HIGHBITDEPTH
clang-format's avatar
clang-format committed
628
629
630
631
632
static void highbd_filter_selectively_vert_row2(
    int subsampling_factor, uint16_t *s, int pitch, unsigned int mask_16x16_l,
    unsigned int mask_8x8_l, unsigned int mask_4x4_l,
    unsigned int mask_4x4_int_l, const loop_filter_info_n *lfi_n,
    const uint8_t *lfl, int bd) {
Jingning Han's avatar
Jingning Han committed
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
  const int mask_shift = subsampling_factor ? 4 : 8;
  const int mask_cutoff = subsampling_factor ? 0xf : 0xff;
  const int lfl_forward = subsampling_factor ? 4 : 8;

  unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff;
  unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff;
  unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff;
  unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff;
  unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff;
  unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff;
  unsigned int mask;

  for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 |
clang-format's avatar
clang-format committed
648
              mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1;
Jingning Han's avatar
Jingning Han committed
649
650
651
652
653
654
655
       mask; mask >>= 1) {
    const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl;
    const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward);

    if (mask & 1) {
      if ((mask_16x16_0 | mask_16x16_1) & 1) {
        if ((mask_16x16_0 & mask_16x16_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
656
          aom_highbd_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
657
658
                                          lfi0->hev_thr, bd);
        } else if (mask_16x16_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
659
          aom_highbd_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
660
661
                                     lfi0->hev_thr, bd);
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
662
          aom_highbd_lpf_vertical_16(s + 8 * pitch, pitch, lfi1->mblim,
Jingning Han's avatar
Jingning Han committed
663
664
665
666
667
668
                                     lfi1->lim, lfi1->hev_thr, bd);
        }
      }

      if ((mask_8x8_0 | mask_8x8_1) & 1) {
        if ((mask_8x8_0 & mask_8x8_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
669
          aom_highbd_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
670
671
672
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_8x8_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
673
          aom_highbd_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim,
674
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
675
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
676
          aom_highbd_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim,
677
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
678
679
680
681
682
        }
      }

      if ((mask_4x4_0 | mask_4x4_1) & 1) {
        if ((mask_4x4_0 & mask_4x4_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
683
          aom_highbd_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
684
685
686
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_4x4_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
687
          aom_highbd_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim,
688
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
689
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
690
          aom_highbd_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim,
691
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
692
693
694
695
696
        }
      }

      if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) {
        if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
697
          aom_highbd_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
698
699
700
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_4x4_int_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
701
          aom_highbd_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim,
702
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
703
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
704
          aom_highbd_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim,
705
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
        }
      }
    }

    s += 8;
    lfl += 1;
    mask_16x16_0 >>= 1;
    mask_8x8_0 >>= 1;
    mask_4x4_0 >>= 1;
    mask_4x4_int_0 >>= 1;
    mask_16x16_1 >>= 1;
    mask_8x8_1 >>= 1;
    mask_4x4_1 >>= 1;
    mask_4x4_int_1 >>= 1;
  }
}
722
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
723

clang-format's avatar
clang-format committed
724
725
726
static void filter_selectively_horiz(
    uint8_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8,
    unsigned int mask_4x4, unsigned int mask_4x4_int,
727
    const loop_filter_info_n *lfi_n, const uint8_t *lfl) {
Jingning Han's avatar
Jingning Han committed
728
729
730
  unsigned int mask;
  int count;

clang-format's avatar
clang-format committed
731
732
  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask;
       mask >>= count) {
Jingning Han's avatar
Jingning Han committed
733
734
735
736
737
738
    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;

    count = 1;
    if (mask & 1) {
      if (mask_16x16 & 1) {
        if ((mask_16x16 & 3) == 3) {
James Zern's avatar
James Zern committed
739
          aom_lpf_horizontal_16_dual(s, pitch, lfi->mblim, lfi->lim,
740
                                     lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
741
742
          count = 2;
        } else {
James Zern's avatar
James Zern committed
743
          aom_lpf_horizontal_16(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
744
745
746
747
748
749
        }
      } else if (mask_8x8 & 1) {
        if ((mask_8x8 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
750
          aom_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
751
752
753
754
                                    lfi->hev_thr, lfin->mblim, lfin->lim,
                                    lfin->hev_thr);

          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
755
            aom_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
Jingning Han's avatar
Jingning Han committed
756
757
758
759
                                      lfi->lim, lfi->hev_thr, lfin->mblim,
                                      lfin->lim, lfin->hev_thr);
          } else {
            if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
760
              aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
761
                                   lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
762
            else if (mask_4x4_int & 2)
Yaowu Xu's avatar
Yaowu Xu committed
763
              aom_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
764
                                   lfin->lim, lfin->hev_thr);
Jingning Han's avatar
Jingning Han committed
765
766
767
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
768
          aom_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
769
770

          if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
771
            aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
772
                                 lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
773
774
775
776
777
778
        }
      } else if (mask_4x4 & 1) {
        if ((mask_4x4 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
779
          aom_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
780
781
                                    lfi->hev_thr, lfin->mblim, lfin->lim,
                                    lfin->hev_thr);
Cheng Chen's avatar
Cheng Chen committed
782

Jingning Han's avatar
Jingning Han committed
783
          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
784
            aom_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
Jingning Han's avatar
Jingning Han committed
785
786
787
788
                                      lfi->lim, lfi->hev_thr, lfin->mblim,
                                      lfin->lim, lfin->hev_thr);
          } else {
            if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
789
              aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
790
                                   lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
791
            else if (mask_4x4_int & 2)
Yaowu Xu's avatar
Yaowu Xu committed
792
              aom_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
793
                                   lfin->lim, lfin->hev_thr);
Jingning Han's avatar
Jingning Han committed
794
795
796
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
797
          aom_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
798
799

          if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
800
            aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
801
                                 lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
802
803
        }
      } else if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
804
        aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
805
                             lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
806
807
808
809
810
811
812
813
814
815
816
      }
    }
    s += 8 * count;
    lfl += count;
    mask_16x16 >>= count;
    mask_8x8 >>= count;
    mask_4x4 >>= count;
    mask_4x4_int >>= count;
  }
}

817
#if CONFIG_HIGHBITDEPTH
clang-format's avatar
clang-format committed
818
819
820
821
static void highbd_filter_selectively_horiz(
    uint16_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8,
    unsigned int mask_4x4, unsigned int mask_4x4_int,
    const loop_filter_info_n *lfi_n, const uint8_t *lfl, int bd) {
Jingning Han's avatar
Jingning Han committed
822
823
824
  unsigned int mask;
  int count;

clang-format's avatar
clang-format committed
825
826
  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask;
       mask >>= count) {
Jingning Han's avatar
Jingning Han committed
827
828
829
830
831
832
    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;

    count = 1;
    if (mask & 1) {
      if (mask_16x16 & 1) {
        if ((mask_16x16 & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
833
          aom_highbd_lpf_horizontal_edge_16(s, pitch, lfi->mblim, lfi->lim,
834
                                            lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
835
836
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
837
          aom_highbd_lpf_horizontal_edge_8(s, pitch, lfi->mblim, lfi->lim,
838
                                           lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
839
840
841
842
843
844
        }
      } else if (mask_8x8 & 1) {
        if ((mask_8x8 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
845
          aom_highbd_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
846
847
848
849
                                           lfi->hev_thr, lfin->mblim, lfin->lim,
                                           lfin->hev_thr, bd);

          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
850
            aom_highbd_lpf_horizontal_4_dual(
clang-format's avatar
clang-format committed
851
852
                s + 4 * pitch, pitch, lfi->mblim, lfi->lim, lfi->hev_thr,
                lfin->mblim, lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
853
854
          } else {
            if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
855
              aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
856
                                          lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
857
            } else if (mask_4x4_int & 2) {
Yaowu Xu's avatar
Yaowu Xu committed
858
              aom_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
859
                                          lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
860
861
862
863
            }
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
864
          aom_highbd_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim,
865
                                      lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
866
867

          if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
868
            aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
869
                                        lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
870
871
872
873
874
875
876
          }
        }
      } else if (mask_4x4 & 1) {
        if ((mask_4x4 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
877
          aom_highbd_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
878
879
880
                                           lfi->hev_thr, lfin->mblim, lfin->lim,
                                           lfin->hev_thr, bd);
          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
881
            aom_highbd_lpf_horizontal_4_dual(
clang-format's avatar
clang-format committed
882
883
                s + 4 * pitch, pitch, lfi->mblim, lfi->lim, lfi->hev_thr,
                lfin->mblim, lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
884
885
          } else {
            if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
886
              aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
887
                                          lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
888
            } else if (mask_4x4_int & 2) {
Yaowu Xu's avatar
Yaowu Xu committed
889
              aom_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
890
                                          lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
891
892
893
894
            }
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
895
          aom_highbd_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim,
896
                                      lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
897
898

          if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
899
            aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
900
                                        lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
901
902
903
          }
        }
      } else if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
904
        aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
905
                                    lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
906
907
908
909
910
911
912
913
914
915
      }
    }
    s += 8 * count;
    lfl += count;
    mask_16x16 >>= count;
    mask_8x8 >>= count;
    mask_4x4 >>= count;
    mask_4x4_int >>= count;
  }
}
916
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
917
918
919
920
921
922
923
924

// This function ors into the current lfm structure, where to do loop
// filters for the specific mi we are looking at. It uses information
// including the block_size_type (32x16, 32x32, etc.), the transform size,
// whether there were any coefficients encoded, and the loop filter strength
// block we are currently looking at. Shift is used to position the
// 1's we produce.
// TODO(JBB) Need another function for different resolution color..
925
926
927
928
static void build_masks(AV1_COMMON *const cm,
                        const loop_filter_info_n *const lfi_n,
                        const MODE_INFO *mi, const int shift_y,
                        const int shift_uv, LOOP_FILTER_MASK *lfm) {
Jingning Han's avatar
Jingning Han committed
929
930
  const MB_MODE_INFO *mbmi = &mi->mbmi;
  const BLOCK_SIZE block_size = mbmi->sb_type;
931
932
  // TODO(debargha): Check if masks can be setup correctly when
  // rectangular transfroms are used with the EXT_TX expt.
933
934
935
  const TX_SIZE tx_size_y = txsize_sqr_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_left = txsize_horz_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_above = txsize_vert_map[mbmi->tx_size];
936
  const TX_SIZE tx_size_uv =
937
938
939
940
941
      txsize_sqr_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
  const TX_SIZE tx_size_uv_left =
      txsize_horz_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
  const TX_SIZE tx_size_uv_above =
      txsize_vert_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
Fangwen Fu's avatar
Fangwen Fu committed
942
#if CONFIG_EXT_DELTA_Q
Cheng Chen's avatar
Cheng Chen committed
943
#if CONFIG_LOOPFILTER_LEVEL
944
  const int filter_level = get_filter_level(cm, lfi_n, 0, 0, mbmi);
Cheng Chen's avatar
Cheng Chen committed
945
946
947
#else
#if CONFIG_LPF_SB
  const int filter_level = get_filter_level(cm, lfi_n, 0, 0, mbmi);
948
#else
Fangwen Fu's avatar
Fangwen Fu committed
949
  const int filter_level = get_filter_level(cm, lfi_n, mbmi);
Cheng Chen's avatar
Cheng Chen committed
950
#endif  // CONFIG_LPF_SB
951
#endif
Fangwen Fu's avatar
Fangwen Fu committed
952
#else
Jingning Han's avatar
Jingning Han committed
953
  const int filter_level = get_filter_level(lfi_n, mbmi);
954
  (void)cm;
Fangwen Fu's avatar
Fangwen Fu committed
955
#endif
956
957
  uint64_t *const left_y = &lfm->left_y[tx_size_y_left];
  uint64_t *const above_y = &lfm->above_y[tx_size_y_above];
Jingning Han's avatar
Jingning Han committed
958
  uint64_t *const int_4x4_y = &lfm->int_4x4_y;
959
960
  uint16_t *const left_uv = &lfm->left_uv[tx_size_uv_left];
  uint16_t *const above_uv = &lfm->above_uv[tx_size_uv_above];
961
  uint16_t *const int_4x4_uv = &lfm->left_int_4x4_uv;
Jingning Han's avatar
Jingning Han committed
962
963
964
965
966
967
968
969
  int i;

  // If filter level is 0 we don't loop filter.
  if (!filter_level) {
    return;
  } else {
    const int w = num_8x8_blocks_wide_lookup[block_size];
    const int h = num_8x8_blocks_high_lookup[block_size];
970
971
972
    const int row = (shift_y >> MAX_MIB_SIZE_LOG2);
    const int col = shift_y - (row << MAX_MIB_SIZE_LOG2);

clang-format's avatar
clang-format committed
973
    for (i = 0; i < h; i++) memset(&lfm->lfl_y[row + i][col], filter_level, w);
Jingning Han's avatar
Jingning Han committed
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
  }

  // These set 1 in the current block size for the block size edges.
  // For instance if the block size is 32x16, we'll set:
  //    above =   1111
  //              0000
  //    and
  //    left  =   1000
  //          =   1000
  // NOTE : In this example the low bit is left most ( 1000 ) is stored as
  //        1,  not 8...
  //
  // U and V set things on a 16 bit scale.
  //
  *above_y |= above_prediction_mask[block_size] << shift_y;
  *above_uv |= above_prediction_mask_uv[block_size] << shift_uv;
  *left_y |= left_prediction_mask[block_size] << shift_y;
  *left_uv |= left_prediction_mask_uv[block_size] << shift_uv;

  // If the block has no coefficients and is not intra we skip applying
  // the loop filter on block edges.
995
  if (mbmi->skip && is_inter_block(mbmi)) return;
Jingning Han's avatar
Jingning Han committed
996
997
998
999
1000

  // Here we are adding a mask for the transform size. The transform
  // size mask is set to be correct for a 64x64 prediction block size. We
  // mask to match the size of the block we are working on and then shift it
  // into place..
1001
  *above_y |= (size_mask[block_size] & above_64x64_txform_mask[tx_size_y_above])
clang-format's avatar
clang-format committed
1002
1003
              << shift_y;
  *above_uv |=
1004
      (size_mask_uv[block_size] & above_64x64_txform_mask_uv[tx_size_uv_above])
clang-format's avatar
clang-format committed
1005
      << shift_uv;
Jingning Han's avatar
Jingning Han committed
1006

1007
  *left_y |= (size_mask[block_size] & left_64x64_txform_mask[tx_size_y_left])
clang-format's avatar
clang-format committed
1008
             << shift_y;
1009
1010
1011
  *left_uv |=
      (size_mask_uv[block_size] & left_64x64_txform_mask_uv[tx_size_uv_left])
      << shift_uv;
Jingning Han's avatar
Jingning Han committed
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026

  // Here we are trying to determine what to do with the internal 4x4 block
  // boundaries.  These differ from the 4x4 boundaries on the outside edge of
  // an 8x8 in that the internal ones can be skipped and don't depend on
  // the prediction block size.
  if (tx_size_y == TX_4X4)
    *int_4x4_y |= (size_mask[block_size] & 0xffffffffffffffffULL) << shift_y;

  if (tx_size_uv == TX_4X4)
    *int_4x4_uv |= (size_mask_uv[block_size] & 0xffff) << shift_uv;
}

// This function does the same thing as the one above with the exception that
// it only affects the y masks. It exists because for blocks < 16x16 in size,
// we only update u and v masks on the first block.
1027
1028
1029
1030
static void build_y_mask(AV1_COMMON *const cm,
                         const loop_filter_info_n *const lfi_n,
                         const MODE_INFO *mi, const int shift_y,
                         LOOP_FILTER_MASK *lfm) {
Jingning Han's avatar
Jingning Han committed
1031
  const MB_MODE_INFO *mbmi = &mi->mbmi;
1032
1033
1034
  const TX_SIZE tx_size_y = txsize_sqr_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_left = txsize_horz_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_above = txsize_vert_map[mbmi->tx_size];
1035
  const BLOCK_SIZE block_size = mbmi->sb_type;
Fangwen Fu's avatar
Fangwen Fu committed
1036
#if CONFIG_EXT_DELTA_Q
Cheng Chen's avatar
Cheng Chen committed
1037
#if CONFIG_LOOPFILTER_LEVEL
1038
  const int filter_level = get_filter_level(cm, lfi_n, 0, 0, mbmi);
Cheng Chen's avatar
Cheng Chen committed
1039
1040
1041
#else
#if CONFIG_LPF_SB
  const int filter_level = get_filter_level(cm, lfi_n, 0, 0, mbmi);
1042
#else
Fangwen Fu's avatar
Fangwen Fu committed
1043
  const int filter_level = get_filter_level(cm, lfi_n, mbmi);
Cheng Chen's avatar
Cheng Chen committed
1044
#endif  // CONFIG_LPF_SB
1045
#endif
Fangwen Fu's avatar
Fangwen Fu committed
1046
#else
Jingning Han's avatar
Jingning Han committed
1047
  const int filter_level = get_filter_level(lfi_n, mbmi);
1048
  (void)cm;
Fangwen Fu's avatar
Fangwen Fu committed
1049
#endif
1050
1051
  uint64_t *const left_y = &lfm->left_y[tx_size_y_left];
  uint64_t *const above_y = &lfm->above_y[tx_size_y_above];
Jingning Han's avatar
Jingning Han committed
1052
1053
1054
1055
1056
1057
1058
1059
  uint64_t *const int_4x4_y = &lfm->int_4x4_y;
  int i;

  if (!filter_level) {
    return;
  } else {
    const int w = num_8x8_blocks_wide_lookup[block_size];
    const int h = num_8x8_blocks_high_lookup[block_size];
1060
1061
1062
    const int row = (shift_y >> MAX_MIB_SIZE_LOG2);
    const int col = shift_y - (row << MAX_MIB_SIZE_LOG2);

clang-format's avatar
clang-format committed
1063
    for (i = 0; i < h; i++) memset(&lfm->lfl_y[row + i][col], filter_level, w);
Jingning Han's avatar
Jingning Han committed
1064
1065
1066
1067
1068
  }

  *above_y |= above_prediction_mask[block_size] << shift_y;
  *left_y |= left_prediction_mask[block_size] << shift_y;

1069
  if (mbmi->skip && is_inter_block(mbmi)) return;
Jingning Han's avatar
Jingning Han committed
1070

1071
  *above_y |= (size_mask[block_size] & above_64x64_txform_mask[tx_size_y_above])
clang-format's avatar
clang-format committed
1072
              << shift_y;
Jingning Han's avatar
Jingning Han committed
1073

1074
  *left_y |= (size_mask[block_size] & left_64x64_txform_mask[tx_size_y_left])
clang-format's avatar
clang-format committed
1075
             << shift_y;
Jingning Han's avatar
Jingning Han committed
1076
1077
1078
1079
1080

  if (tx_size_y == TX_4X4)
    *int_4x4_y |= (size_mask[block_size] & 0xffffffffffffffffULL) << shift_y;
}

1081
#if CONFIG_LOOPFILTERING_ACROSS_TILES
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
// This function update the bit masks for the entire 64x64 region represented
// by mi_row, mi_col. In case one of the edge is a tile boundary, loop filtering
// for that edge is disabled. This function only check the tile boundary info
// for the top left corner mi to determine the boundary information for the
// top and left edge of the whole super block
static void update_tile_boundary_filter_mask(AV1_COMMON *const cm,
                                             const int mi_row, const int mi_col,
                                             LOOP_FILTER_MASK *lfm) {
  int i;
  MODE_INFO *const mi = cm->mi + mi_row * cm->mi_stride + mi_col;

1093
  if (mi->mbmi.boundary_info & TILE_LEFT_BOUNDARY) {
1094
1095
1096
1097
1098
1099
    for (i = 0; i <= TX_32X32; i++) {
      lfm->left_y[i] &= 0xfefefefefefefefeULL;
      lfm->left_uv[i] &= 0xeeee;
    }
  }

1100
  if (mi->mbmi.boundary_info & TILE_ABOVE_BOUNDARY) {
1101
1102
1103
1104
1105
1106
    for (i = 0; i <= TX_32X32; i++) {
      lfm->above_y[i] &= 0xffffffffffffff00ULL;
      lfm->above_uv[i] &= 0xfff0;
    }
  }
}
1107
#endif  // CONFIG_LOOPFILTERING_ACROSS_TILES
1108

Jingning Han's avatar
Jingning Han committed
1109
1110
1111
// This function sets up the bit masks for the entire 64x64 region represented
// by mi_row, mi_col.
// TODO(JBB): This function only works for yv12.
Yaowu Xu's avatar
Yaowu Xu committed
1112
1113
1114
void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
                    MODE_INFO **mi, const int mode_info_stride,
                    LOOP_FILTER_MASK *lfm) {
1115
1116
1117
#if CONFIG_EXT_PARTITION
  assert(0 && "Not yet updated");
#endif  // CONFIG_EXT_PARTITION
Jingning Han's avatar
Jingning Han committed
1118
1119
1120
1121
1122
1123
1124
1125
1126
  int idx_32, idx_16, idx_8;
  const loop_filter_info_n *const lfi_n = &cm->lf_info;
  MODE_INFO **mip = mi;
  MODE_INFO **mip2 = mi;

  // These are offsets to the next mi in the 64x64 block. It is what gets
  // added to the mi ptr as we go through each loop. It helps us to avoid
  // setting up special row and column counters for each index. The last step
  // brings us out back to the starting position.
clang-format's avatar
clang-format committed
1127
1128
1129
1130
1131
  const int offset_32[] = { 4, (mode_info_stride << 2) - 4, 4,
                            -(mode_info_stride << 2) - 4 };
  const int offset_16[] = { 2, (mode_info_stride << 1) - 2, 2,
                            -(mode_info_stride << 1) - 2 };
  const int offset[] = { 1, mode_info_stride - 1, 1, -mode_info_stride - 1 };
Jingning Han's avatar
Jingning Han committed
1132
1133
1134
1135
1136

  // Following variables represent shifts to position the current block
  // mask over the appropriate block. A shift of 36 to the left will move
  // the bits for the final 32 by 32 block in the 64x64 up 4 rows and left
  // 4 rows to the appropriate spot.
clang-format's avatar
clang-format committed
1137
1138
1139
1140
1141
  const int shift_32_y[] = { 0, 4, 32, 36 };
  const int shift_16_y[] = { 0, 2, 16, 18 };
  const int shift_8_y[] = { 0, 1, 8, 9 };
  const int shift_32_uv[] = { 0, 2, 8, 10 };
  const int shift_16_uv[] = { 0, 1, 4, 5 };
Jingning Han's avatar
Jingning Han committed
1142
  int i;
Yaowu Xu's avatar
Yaowu Xu committed
1143
1144
  const int max_rows = AOMMIN(cm->mi_rows - mi_row, MAX_MIB_SIZE);