av1_loopfilter.c 98.6 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
 */

Debargha Mukherjee's avatar
Debargha Mukherjee committed
12
13
#include <math.h>

Yaowu Xu's avatar
Yaowu Xu committed
14
15
16
17
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_mem/aom_mem.h"
18
#include "aom_ports/mem.h"
19
20
21
#include "av1/common/av1_loopfilter.h"
#include "av1/common/onyxc_int.h"
#include "av1/common/reconinter.h"
22
#include "av1/common/seg_common.h"
Jingning Han's avatar
Jingning Han committed
23

24
25
26
27
28
29
#if CONFIG_LOOPFILTER_LEVEL
static const SEG_LVL_FEATURES seg_lvl_lf_lut[MAX_MB_PLANE][2] = {
  { SEG_LVL_ALT_LF_Y_V, SEG_LVL_ALT_LF_Y_H },
  { SEG_LVL_ALT_LF_U, SEG_LVL_ALT_LF_U },
  { SEG_LVL_ALT_LF_V, SEG_LVL_ALT_LF_V }
};
30
31
32
33
34
35

#if CONFIG_EXT_DELTA_Q
static const int delta_lf_id_lut[MAX_MB_PLANE][2] = {
  { 0, 1 }, { 2, 2 }, { 3, 3 }
};
#endif  // CONFIG_EXT_DELTA_Q
36
37
#endif  // CONFIG_LOOPFILTER_LEVEL

Ola Hugosson's avatar
Ola Hugosson committed
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
#if CONFIG_DEBLOCK_13TAP
#define PARALLEL_DEBLOCKING_5_TAP_CHROMA 1
#else
#define PARALLEL_DEBLOCKING_5_TAP_CHROMA 0
#endif

#if PARALLEL_DEBLOCKING_5_TAP_CHROMA
extern void aom_lpf_vertical_6_c(uint8_t *s, int pitch, const uint8_t *blimit,
                                 const uint8_t *limit, const uint8_t *thresh);

extern void aom_lpf_horizontal_6_c(uint8_t *s, int p, const uint8_t *blimit,
                                   const uint8_t *limit, const uint8_t *thresh);

extern void aom_highbd_lpf_horizontal_6_c(uint16_t *s, int p,
                                          const uint8_t *blimit,
                                          const uint8_t *limit,
                                          const uint8_t *thresh, int bd);

extern void aom_highbd_lpf_vertical_6_c(uint16_t *s, int pitch,
                                        const uint8_t *blimit,
                                        const uint8_t *limit,
                                        const uint8_t *thresh, int bd);
#endif
61

Jingning Han's avatar
Jingning Han committed
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
// 64 bit masks for left transform size. Each 1 represents a position where
// we should apply a loop filter across the left border of an 8x8 block
// boundary.
//
// In the case of TX_16X16->  ( in low order byte first we end up with
// a mask that looks like this
//
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//
// A loopfilter should be applied to every other 8x8 horizontally.
clang-format's avatar
clang-format committed
79
static const uint64_t left_64x64_txform_mask[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
80
81
82
83
  0xffffffffffffffffULL,  // TX_4X4
  0xffffffffffffffffULL,  // TX_8x8
  0x5555555555555555ULL,  // TX_16x16
  0x1111111111111111ULL,  // TX_32x32
84
85
86
#if CONFIG_TX64X64
  0x0101010101010101ULL,  // TX_64x64
#endif                    // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
};

// 64 bit masks for above transform size. Each 1 represents a position where
// we should apply a loop filter across the top border of an 8x8 block
// boundary.
//
// In the case of TX_32x32 ->  ( in low order byte first we end up with
// a mask that looks like this
//
//    11111111
//    00000000
//    00000000
//    00000000
//    11111111
//    00000000
//    00000000
//    00000000
//
// A loopfilter should be applied to every other 4 the row vertically.
clang-format's avatar
clang-format committed
106
static const uint64_t above_64x64_txform_mask[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
107
108
109
110
  0xffffffffffffffffULL,  // TX_4X4
  0xffffffffffffffffULL,  // TX_8x8
  0x00ff00ff00ff00ffULL,  // TX_16x16
  0x000000ff000000ffULL,  // TX_32x32
111
112
113
#if CONFIG_TX64X64
  0x00000000000000ffULL,  // TX_64x64
#endif                    // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
};

// 64 bit masks for prediction sizes (left). Each 1 represents a position
// where left border of an 8x8 block. These are aligned to the right most
// appropriate bit, and then shifted into place.
//
// In the case of TX_16x32 ->  ( low order byte first ) we end up with
// a mask that looks like this :
//
//  10000000
//  10000000
//  10000000
//  10000000
//  00000000
//  00000000
//  00000000
//  00000000
131
static const uint64_t left_prediction_mask[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
132
133
134
135
136
137
138
139
140
141
142
143
  0x0000000000000001ULL,  // BLOCK_4X4,
  0x0000000000000001ULL,  // BLOCK_4X8,
  0x0000000000000001ULL,  // BLOCK_8X4,
  0x0000000000000001ULL,  // BLOCK_8X8,
  0x0000000000000101ULL,  // BLOCK_8X16,
  0x0000000000000001ULL,  // BLOCK_16X8,
  0x0000000000000101ULL,  // BLOCK_16X16,
  0x0000000001010101ULL,  // BLOCK_16X32,
  0x0000000000000101ULL,  // BLOCK_32X16,
  0x0000000001010101ULL,  // BLOCK_32X32,
  0x0101010101010101ULL,  // BLOCK_32X64,
  0x0000000001010101ULL,  // BLOCK_64X32,
144
145
146
147
  0x0101010101010101ULL,  // BLOCK_64X64,
  0x0000000000000101ULL,  // BLOCK_4X16,
  0x0000000000000001ULL,  // BLOCK_16X4,
  0x0000000001010101ULL,  // BLOCK_8X32,
148
149
150
  0x0000000000000001ULL,  // BLOCK_32X8,
  0x0101010101010101ULL,  // BLOCK_16X64,
  0x0000000000000101ULL,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
151
152
153
};

// 64 bit mask to shift and set for each prediction size.
154
static const uint64_t above_prediction_mask[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
155
156
157
158
159
160
161
162
163
164
165
166
  0x0000000000000001ULL,  // BLOCK_4X4
  0x0000000000000001ULL,  // BLOCK_4X8
  0x0000000000000001ULL,  // BLOCK_8X4
  0x0000000000000001ULL,  // BLOCK_8X8
  0x0000000000000001ULL,  // BLOCK_8X16,
  0x0000000000000003ULL,  // BLOCK_16X8
  0x0000000000000003ULL,  // BLOCK_16X16
  0x0000000000000003ULL,  // BLOCK_16X32,
  0x000000000000000fULL,  // BLOCK_32X16,
  0x000000000000000fULL,  // BLOCK_32X32,
  0x000000000000000fULL,  // BLOCK_32X64,
  0x00000000000000ffULL,  // BLOCK_64X32,
167
168
169
170
  0x00000000000000ffULL,  // BLOCK_64X64,
  0x0000000000000001ULL,  // BLOCK_4X16,
  0x0000000000000003ULL,  // BLOCK_16X4,
  0x0000000000000001ULL,  // BLOCK_8X32,
171
172
173
  0x000000000000000fULL,  // BLOCK_32X8,
  0x0000000000000003ULL,  // BLOCK_16X64,
  0x00000000000000ffULL,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
174
175
};
// 64 bit mask to shift and set for each prediction size. A bit is set for
176
// each 8x8 block that would be in the top left most block of the given block
Jingning Han's avatar
Jingning Han committed
177
// size in the 64x64 block.
178
static const uint64_t size_mask[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
179
180
181
182
183
184
185
186
187
188
189
190
  0x0000000000000001ULL,  // BLOCK_4X4
  0x0000000000000001ULL,  // BLOCK_4X8
  0x0000000000000001ULL,  // BLOCK_8X4
  0x0000000000000001ULL,  // BLOCK_8X8
  0x0000000000000101ULL,  // BLOCK_8X16,
  0x0000000000000003ULL,  // BLOCK_16X8
  0x0000000000000303ULL,  // BLOCK_16X16
  0x0000000003030303ULL,  // BLOCK_16X32,
  0x0000000000000f0fULL,  // BLOCK_32X16,
  0x000000000f0f0f0fULL,  // BLOCK_32X32,
  0x0f0f0f0f0f0f0f0fULL,  // BLOCK_32X64,
  0x00000000ffffffffULL,  // BLOCK_64X32,
191
192
193
194
  0xffffffffffffffffULL,  // BLOCK_64X64,
  0x0000000000000101ULL,  // BLOCK_4X16,
  0x0000000000000003ULL,  // BLOCK_16X4,
  0x0000000001010101ULL,  // BLOCK_8X32,
195
196
197
  0x000000000000000fULL,  // BLOCK_32X8,
  0x0303030303030303ULL,  // BLOCK_16X64,
  0x000000000000ffffULL,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
198
199
};

200
// These are used for masking the left and above 32x32 borders.
clang-format's avatar
clang-format committed
201
static const uint64_t left_border = 0x1111111111111111ULL;
Jingning Han's avatar
Jingning Han committed
202
203
204
static const uint64_t above_border = 0x000000ff000000ffULL;

// 16 bit masks for uv transform sizes.
clang-format's avatar
clang-format committed
205
static const uint16_t left_64x64_txform_mask_uv[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
206
207
208
209
  0xffff,  // TX_4X4
  0xffff,  // TX_8x8
  0x5555,  // TX_16x16
  0x1111,  // TX_32x32
210
211
212
#if CONFIG_TX64X64
  0x0101,  // TX_64x64, never used
#endif     // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
213
214
};

clang-format's avatar
clang-format committed
215
static const uint16_t above_64x64_txform_mask_uv[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
216
217
218
219
  0xffff,  // TX_4X4
  0xffff,  // TX_8x8
  0x0f0f,  // TX_16x16
  0x000f,  // TX_32x32
220
221
222
#if CONFIG_TX64X64
  0x0003,  // TX_64x64, never used
#endif     // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
223
224
225
};

// 16 bit left mask to shift and set for each uv prediction size.
226
static const uint16_t left_prediction_mask_uv[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
227
228
229
230
231
232
233
234
235
236
237
238
  0x0001,  // BLOCK_4X4,
  0x0001,  // BLOCK_4X8,
  0x0001,  // BLOCK_8X4,
  0x0001,  // BLOCK_8X8,
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8,
  0x0001,  // BLOCK_16X16,
  0x0011,  // BLOCK_16X32,
  0x0001,  // BLOCK_32X16,
  0x0011,  // BLOCK_32X32,
  0x1111,  // BLOCK_32X64
  0x0011,  // BLOCK_64X32,
239
240
241
242
  0x1111,  // BLOCK_64X64,
  0x0001,  // BLOCK_4X16,
  0x0001,  // BLOCK_16X4,
  0x0011,  // BLOCK_8X32,
243
244
245
  0x0001,  // BLOCK_32X8,
  0x1111,  // BLOCK_16X64,
  0x0001,  // BLOCK_64X16,
Jingning Han's avatar
Jingning Han committed
246
};
247

Jingning Han's avatar
Jingning Han committed
248
// 16 bit above mask to shift and set for uv each prediction size.
249
static const uint16_t above_prediction_mask_uv[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
250
251
252
253
254
255
256
257
258
259
260
261
  0x0001,  // BLOCK_4X4
  0x0001,  // BLOCK_4X8
  0x0001,  // BLOCK_8X4
  0x0001,  // BLOCK_8X8
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8
  0x0001,  // BLOCK_16X16
  0x0001,  // BLOCK_16X32,
  0x0003,  // BLOCK_32X16,
  0x0003,  // BLOCK_32X32,
  0x0003,  // BLOCK_32X64,
  0x000f,  // BLOCK_64X32,
262
263
264
265
  0x000f,  // BLOCK_64X64,
  0x0001,  // BLOCK_4X16,
  0x0001,  // BLOCK_16X4,
  0x0001,  // BLOCK_8X32,
266
267
268
  0x0003,  // BLOCK_32X8,
  0x0001,  // BLOCK_16X64,
  0x000f,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
269
270
271
};

// 64 bit mask to shift and set for each uv prediction size
272
static const uint16_t size_mask_uv[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
273
274
275
276
277
278
279
280
281
282
283
284
  0x0001,  // BLOCK_4X4
  0x0001,  // BLOCK_4X8
  0x0001,  // BLOCK_8X4
  0x0001,  // BLOCK_8X8
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8
  0x0001,  // BLOCK_16X16
  0x0011,  // BLOCK_16X32,
  0x0003,  // BLOCK_32X16,
  0x0033,  // BLOCK_32X32,
  0x3333,  // BLOCK_32X64,
  0x00ff,  // BLOCK_64X32,
285
286
287
288
  0xffff,  // BLOCK_64X64,
  0x0001,  // BLOCK_4X16,
  0x0001,  // BLOCK_16X4,
  0x0011,  // BLOCK_8X32,
289
290
291
  0x0003,  // BLOCK_32X8,
  0x1111,  // BLOCK_16X64,
  0x000f,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
292
};
clang-format's avatar
clang-format committed
293
static const uint16_t left_border_uv = 0x1111;
Jingning Han's avatar
Jingning Han committed
294
295
static const uint16_t above_border_uv = 0x000f;

296
static const int mode_lf_lut[] = {
297
298
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // INTRA_MODES
  1, 1, 0, 1,                             // INTER_MODES (GLOBALMV == 0)
Sarah Parker's avatar
Sarah Parker committed
299
  1, 1, 1, 1, 1, 1, 0, 1  // INTER_COMPOUND_MODES (GLOBAL_GLOBALMV == 0)
Jingning Han's avatar
Jingning Han committed
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
};

static void update_sharpness(loop_filter_info_n *lfi, int sharpness_lvl) {
  int lvl;

  // For each possible value for the loop filter fill out limits
  for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++) {
    // Set loop filter parameters that control sharpness.
    int block_inside_limit = lvl >> ((sharpness_lvl > 0) + (sharpness_lvl > 4));

    if (sharpness_lvl > 0) {
      if (block_inside_limit > (9 - sharpness_lvl))
        block_inside_limit = (9 - sharpness_lvl);
    }

clang-format's avatar
clang-format committed
315
    if (block_inside_limit < 1) block_inside_limit = 1;
Jingning Han's avatar
Jingning Han committed
316
317
318
319
320
321

    memset(lfi->lfthr[lvl].lim, block_inside_limit, SIMD_WIDTH);
    memset(lfi->lfthr[lvl].mblim, (2 * (lvl + 2) + block_inside_limit),
           SIMD_WIDTH);
  }
}
Fangwen Fu's avatar
Fangwen Fu committed
322
323
324
#if CONFIG_EXT_DELTA_Q
static uint8_t get_filter_level(const AV1_COMMON *cm,
                                const loop_filter_info_n *lfi_n,
Cheng Chen's avatar
Cheng Chen committed
325
#if CONFIG_LOOPFILTER_LEVEL
326
                                const int dir_idx, int plane,
327
#endif
Fangwen Fu's avatar
Fangwen Fu committed
328
329
330
                                const MB_MODE_INFO *mbmi) {
  const int segment_id = mbmi->segment_id;
  if (cm->delta_lf_present_flag) {
Cheng Chen's avatar
Cheng Chen committed
331
#if CONFIG_LOOPFILTER_LEVEL
332
333
334
335
336
337
338
    int delta_lf;
    if (cm->delta_lf_multi) {
      const int delta_lf_idx = delta_lf_id_lut[plane][dir_idx];
      delta_lf = mbmi->curr_delta_lf[delta_lf_idx];
    } else {
      delta_lf = mbmi->current_delta_lf_from_base;
    }
339
    int lvl_seg =
340
        clamp(delta_lf + cm->lf.filter_level[dir_idx], 0, MAX_LOOP_FILTER);
341
#else
Fangwen Fu's avatar
Fangwen Fu committed
342
343
    int lvl_seg = clamp(mbmi->current_delta_lf_from_base + cm->lf.filter_level,
                        0, MAX_LOOP_FILTER);
344
#endif
345
346
347
348
349
#if CONFIG_LOOPFILTER_LEVEL
    assert(plane >= 0 && plane <= 2);
    const int seg_lf_feature_id = seg_lvl_lf_lut[plane][dir_idx];
    if (segfeature_active(&cm->seg, segment_id, seg_lf_feature_id)) {
      const int data = get_segdata(&cm->seg, segment_id, seg_lf_feature_id);
350
      lvl_seg = clamp(lvl_seg + data, 0, MAX_LOOP_FILTER);
351
352
    }
#else
Fangwen Fu's avatar
Fangwen Fu committed
353
354
    if (segfeature_active(&cm->seg, segment_id, SEG_LVL_ALT_LF)) {
      const int data = get_segdata(&cm->seg, segment_id, SEG_LVL_ALT_LF);
355
      lvl_seg = clamp(lvl_seg + data, 0, MAX_LOOP_FILTER);
Fangwen Fu's avatar
Fangwen Fu committed
356
    }
357
#endif  // CONFIG_LOOPFILTER_LEVEL
Jingning Han's avatar
Jingning Han committed
358

Fangwen Fu's avatar
Fangwen Fu committed
359
    if (cm->lf.mode_ref_delta_enabled) {
Luc Trudeau's avatar
Luc Trudeau committed
360
      const int scale = 1 << (lvl_seg >> 5);
Fangwen Fu's avatar
Fangwen Fu committed
361
362
363
364
365
366
367
      lvl_seg += cm->lf.ref_deltas[mbmi->ref_frame[0]] * scale;
      if (mbmi->ref_frame[0] > INTRA_FRAME)
        lvl_seg += cm->lf.mode_deltas[mode_lf_lut[mbmi->mode]] * scale;
      lvl_seg = clamp(lvl_seg, 0, MAX_LOOP_FILTER);
    }
    return lvl_seg;
  } else {
Cheng Chen's avatar
Cheng Chen committed
368
#if CONFIG_LOOPFILTER_LEVEL
369
370
371
    return lfi_n
        ->lvl[segment_id][dir_idx][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]];
#else
Fangwen Fu's avatar
Fangwen Fu committed
372
    return lfi_n->lvl[segment_id][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]];
373
#endif
Fangwen Fu's avatar
Fangwen Fu committed
374
375
376
  }
}
#else
Jingning Han's avatar
Jingning Han committed
377
378
static uint8_t get_filter_level(const loop_filter_info_n *lfi_n,
                                const MB_MODE_INFO *mbmi) {
379
380
  const int segment_id = mbmi->segment_id;
  return lfi_n->lvl[segment_id][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]];
Jingning Han's avatar
Jingning Han committed
381
}
Fangwen Fu's avatar
Fangwen Fu committed
382
#endif
Jingning Han's avatar
Jingning Han committed
383

Yaowu Xu's avatar
Yaowu Xu committed
384
void av1_loop_filter_init(AV1_COMMON *cm) {
385
  assert(MB_MODE_COUNT == NELEMENTS(mode_lf_lut));
Jingning Han's avatar
Jingning Han committed
386
387
388
389
390
391
392
393
394
395
396
397
398
  loop_filter_info_n *lfi = &cm->lf_info;
  struct loopfilter *lf = &cm->lf;
  int lvl;

  // init limits for given sharpness
  update_sharpness(lfi, lf->sharpness_level);
  lf->last_sharpness_level = lf->sharpness_level;

  // init hev threshold const vectors
  for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++)
    memset(lfi->lfthr[lvl].hev_thr, (lvl >> 4), SIMD_WIDTH);
}

399
void av1_loop_filter_frame_init(AV1_COMMON *cm, int default_filt_lvl,
400
401
402
403
404
405
                                int default_filt_lvl_r
#if CONFIG_LOOPFILTER_LEVEL
                                ,
                                int plane
#endif
                                ) {
Jingning Han's avatar
Jingning Han committed
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
  int seg_id;
  // n_shift is the multiplier for lf_deltas
  // the multiplier is 1 for when filter_lvl is between 0 and 31;
  // 2 when filter_lvl is between 32 and 63
  loop_filter_info_n *const lfi = &cm->lf_info;
  struct loopfilter *const lf = &cm->lf;
  const struct segmentation *const seg = &cm->seg;

  // update limits if sharpness has changed
  if (lf->last_sharpness_level != lf->sharpness_level) {
    update_sharpness(lfi, lf->sharpness_level);
    lf->last_sharpness_level = lf->sharpness_level;
  }

  for (seg_id = 0; seg_id < MAX_SEGMENTS; seg_id++) {
421
422
423
424
425
426
427
    for (int dir = 0; dir < 2; ++dir) {
      int lvl_seg = (dir == 0) ? default_filt_lvl : default_filt_lvl_r;
#if CONFIG_LOOPFILTER_LEVEL
      assert(plane >= 0 && plane <= 2);
      const int seg_lf_feature_id = seg_lvl_lf_lut[plane][dir];
      if (segfeature_active(seg, seg_id, seg_lf_feature_id)) {
        const int data = get_segdata(&cm->seg, seg_id, seg_lf_feature_id);
428
        lvl_seg = clamp(lvl_seg + data, 0, MAX_LOOP_FILTER);
429
430
431
432
      }
#else
      if (segfeature_active(seg, seg_id, SEG_LVL_ALT_LF)) {
        const int data = get_segdata(seg, seg_id, SEG_LVL_ALT_LF);
433
        lvl_seg = clamp(lvl_seg + data, 0, MAX_LOOP_FILTER);
434
435
      }
#endif  // CONFIG_LOOPFILTER_LEVEL
Jingning Han's avatar
Jingning Han committed
436

437
438
439
440
441
442
443
444
445
446
      if (!lf->mode_ref_delta_enabled) {
// we could get rid of this if we assume that deltas are set to
// zero when not in use; encoder always uses deltas
#if CONFIG_LOOPFILTER_LEVEL
        memset(lfi->lvl[seg_id][dir], lvl_seg, sizeof(lfi->lvl[seg_id][dir]));
#else
        memset(lfi->lvl[seg_id], lvl_seg, sizeof(lfi->lvl[seg_id]));
#endif  // CONFIG_LOOPFILTER_LEVEL
      } else {
        int ref, mode;
Cheng Chen's avatar
Cheng Chen committed
447
#if CONFIG_LOOPFILTER_LEVEL
Luc Trudeau's avatar
Luc Trudeau committed
448
        const int scale = 1 << (lvl_seg >> 5);
449
450
451
452
453
454
455
456
457
458
459
460
461
        const int intra_lvl = lvl_seg + lf->ref_deltas[INTRA_FRAME] * scale;
        lfi->lvl[seg_id][dir][INTRA_FRAME][0] =
            clamp(intra_lvl, 0, MAX_LOOP_FILTER);

        for (ref = LAST_FRAME; ref < TOTAL_REFS_PER_FRAME; ++ref) {
          for (mode = 0; mode < MAX_MODE_LF_DELTAS; ++mode) {
            const int inter_lvl = lvl_seg + lf->ref_deltas[ref] * scale +
                                  lf->mode_deltas[mode] * scale;
            lfi->lvl[seg_id][dir][ref][mode] =
                clamp(inter_lvl, 0, MAX_LOOP_FILTER);
          }
        }
#else
Luc Trudeau's avatar
Luc Trudeau committed
462
        const int scale = 1 << (default_filt_lvl >> 5);
463
464
465
466
467
468
469
470
471
        const int intra_lvl = lvl_seg + lf->ref_deltas[INTRA_FRAME] * scale;
        lfi->lvl[seg_id][INTRA_FRAME][0] = clamp(intra_lvl, 0, MAX_LOOP_FILTER);

        for (ref = LAST_FRAME; ref < TOTAL_REFS_PER_FRAME; ++ref) {
          for (mode = 0; mode < MAX_MODE_LF_DELTAS; ++mode) {
            const int inter_lvl = lvl_seg + lf->ref_deltas[ref] * scale +
                                  lf->mode_deltas[mode] * scale;
            lfi->lvl[seg_id][ref][mode] = clamp(inter_lvl, 0, MAX_LOOP_FILTER);
          }
Jingning Han's avatar
Jingning Han committed
472
        }
473
#endif
474
      }
Jingning Han's avatar
Jingning Han committed
475
476
477
478
    }
  }
}

clang-format's avatar
clang-format committed
479
480
static void filter_selectively_vert_row2(int subsampling_factor, uint8_t *s,
                                         int pitch, unsigned int mask_16x16_l,
Jingning Han's avatar
Jingning Han committed
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
                                         unsigned int mask_8x8_l,
                                         unsigned int mask_4x4_l,
                                         unsigned int mask_4x4_int_l,
                                         const loop_filter_info_n *lfi_n,
                                         const uint8_t *lfl) {
  const int mask_shift = subsampling_factor ? 4 : 8;
  const int mask_cutoff = subsampling_factor ? 0xf : 0xff;
  const int lfl_forward = subsampling_factor ? 4 : 8;

  unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff;
  unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff;
  unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff;
  unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff;
  unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff;
  unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff;
  unsigned int mask;

  for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 |
              mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1;
       mask; mask >>= 1) {
    const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl;
    const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward);

    if (mask & 1) {
      if ((mask_16x16_0 | mask_16x16_1) & 1) {
        if ((mask_16x16_0 & mask_16x16_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
509
          aom_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
510
511
                                   lfi0->hev_thr);
        } else if (mask_16x16_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
512
          aom_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
513
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
514
          aom_lpf_vertical_16(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
clang-format's avatar
clang-format committed
515
                              lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
516
517
518
519
520
        }
      }

      if ((mask_8x8_0 | mask_8x8_1) & 1) {
        if ((mask_8x8_0 & mask_8x8_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
521
          aom_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
522
523
524
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_8x8_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
525
          aom_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
526
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
527
          aom_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
528
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
529
530
531
532
533
        }
      }

      if ((mask_4x4_0 | mask_4x4_1) & 1) {
        if ((mask_4x4_0 & mask_4x4_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
534
          aom_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
535
536
537
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_4x4_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
538
          aom_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
539
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
540
          aom_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
541
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
542
543
544
545
546
        }
      }

      if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) {
        if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
547
          aom_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
548
549
550
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_4x4_int_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
551
          aom_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim,
552
                             lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
553
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
554
          aom_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim, lfi1->lim,
555
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
        }
      }
    }

    s += 8;
    lfl += 1;
    mask_16x16_0 >>= 1;
    mask_8x8_0 >>= 1;
    mask_4x4_0 >>= 1;
    mask_4x4_int_0 >>= 1;
    mask_16x16_1 >>= 1;
    mask_8x8_1 >>= 1;
    mask_4x4_1 >>= 1;
    mask_4x4_int_1 >>= 1;
  }
}

573
#if CONFIG_HIGHBITDEPTH
clang-format's avatar
clang-format committed
574
575
576
577
578
static void highbd_filter_selectively_vert_row2(
    int subsampling_factor, uint16_t *s, int pitch, unsigned int mask_16x16_l,
    unsigned int mask_8x8_l, unsigned int mask_4x4_l,
    unsigned int mask_4x4_int_l, const loop_filter_info_n *lfi_n,
    const uint8_t *lfl, int bd) {
Jingning Han's avatar
Jingning Han committed
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
  const int mask_shift = subsampling_factor ? 4 : 8;
  const int mask_cutoff = subsampling_factor ? 0xf : 0xff;
  const int lfl_forward = subsampling_factor ? 4 : 8;

  unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff;
  unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff;
  unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff;
  unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff;
  unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff;
  unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff;
  unsigned int mask;

  for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 |
clang-format's avatar
clang-format committed
594
              mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1;
Jingning Han's avatar
Jingning Han committed
595
596
597
598
599
600
601
       mask; mask >>= 1) {
    const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl;
    const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward);

    if (mask & 1) {
      if ((mask_16x16_0 | mask_16x16_1) & 1) {
        if ((mask_16x16_0 & mask_16x16_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
602
          aom_highbd_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
603
604
                                          lfi0->hev_thr, bd);
        } else if (mask_16x16_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
605
          aom_highbd_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
606
607
                                     lfi0->hev_thr, bd);
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
608
          aom_highbd_lpf_vertical_16(s + 8 * pitch, pitch, lfi1->mblim,
Jingning Han's avatar
Jingning Han committed
609
610
611
612
613
614
                                     lfi1->lim, lfi1->hev_thr, bd);
        }
      }

      if ((mask_8x8_0 | mask_8x8_1) & 1) {
        if ((mask_8x8_0 & mask_8x8_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
615
          aom_highbd_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
616
617
618
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_8x8_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
619
          aom_highbd_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim,
620
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
621
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
622
          aom_highbd_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim,
623
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
624
625
626
627
628
        }
      }

      if ((mask_4x4_0 | mask_4x4_1) & 1) {
        if ((mask_4x4_0 & mask_4x4_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
629
          aom_highbd_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
630
631
632
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_4x4_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
633
          aom_highbd_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim,
634
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
635
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
636
          aom_highbd_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim,
637
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
638
639
640
641
642
        }
      }

      if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) {
        if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
643
          aom_highbd_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
644
645
646
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_4x4_int_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
647
          aom_highbd_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim,
648
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
649
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
650
          aom_highbd_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim,
651
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
        }
      }
    }

    s += 8;
    lfl += 1;
    mask_16x16_0 >>= 1;
    mask_8x8_0 >>= 1;
    mask_4x4_0 >>= 1;
    mask_4x4_int_0 >>= 1;
    mask_16x16_1 >>= 1;
    mask_8x8_1 >>= 1;
    mask_4x4_1 >>= 1;
    mask_4x4_int_1 >>= 1;
  }
}
668
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
669

clang-format's avatar
clang-format committed
670
671
672
static void filter_selectively_horiz(
    uint8_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8,
    unsigned int mask_4x4, unsigned int mask_4x4_int,
673
    const loop_filter_info_n *lfi_n, const uint8_t *lfl) {
Jingning Han's avatar
Jingning Han committed
674
675
676
  unsigned int mask;
  int count;

clang-format's avatar
clang-format committed
677
678
  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask;
       mask >>= count) {
Jingning Han's avatar
Jingning Han committed
679
680
681
682
683
684
    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;

    count = 1;
    if (mask & 1) {
      if (mask_16x16 & 1) {
        if ((mask_16x16 & 3) == 3) {
James Zern's avatar
James Zern committed
685
          aom_lpf_horizontal_16_dual(s, pitch, lfi->mblim, lfi->lim,
686
                                     lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
687
688
          count = 2;
        } else {
James Zern's avatar
James Zern committed
689
          aom_lpf_horizontal_16(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
690
691
692
693
694
695
        }
      } else if (mask_8x8 & 1) {
        if ((mask_8x8 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
696
          aom_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
697
698
699
700
                                    lfi->hev_thr, lfin->mblim, lfin->lim,
                                    lfin->hev_thr);

          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
701
            aom_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
Jingning Han's avatar
Jingning Han committed
702
703
704
705
                                      lfi->lim, lfi->hev_thr, lfin->mblim,
                                      lfin->lim, lfin->hev_thr);
          } else {
            if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
706
              aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
707
                                   lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
708
            else if (mask_4x4_int & 2)
Yaowu Xu's avatar
Yaowu Xu committed
709
              aom_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
710
                                   lfin->lim, lfin->hev_thr);
Jingning Han's avatar
Jingning Han committed
711
712
713
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
714
          aom_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
715
716

          if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
717
            aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
718
                                 lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
719
720
721
722
723
724
        }
      } else if (mask_4x4 & 1) {
        if ((mask_4x4 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
725
          aom_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
726
727
                                    lfi->hev_thr, lfin->mblim, lfin->lim,
                                    lfin->hev_thr);
Cheng Chen's avatar
Cheng Chen committed
728

Jingning Han's avatar
Jingning Han committed
729
          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
730
            aom_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
Jingning Han's avatar
Jingning Han committed
731
732
733
734
                                      lfi->lim, lfi->hev_thr, lfin->mblim,
                                      lfin->lim, lfin->hev_thr);
          } else {
            if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
735
              aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
736
                                   lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
737
            else if (mask_4x4_int & 2)
Yaowu Xu's avatar
Yaowu Xu committed
738
              aom_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
739
                                   lfin->lim, lfin->hev_thr);
Jingning Han's avatar
Jingning Han committed
740
741
742
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
743
          aom_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
744
745

          if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
746
            aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
747
                                 lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
748
749
        }
      } else if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
750
        aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
751
                             lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
752
753
754
755
756
757
758
759
760
761
762
      }
    }
    s += 8 * count;
    lfl += count;
    mask_16x16 >>= count;
    mask_8x8 >>= count;
    mask_4x4 >>= count;
    mask_4x4_int >>= count;
  }
}

763
#if CONFIG_HIGHBITDEPTH
clang-format's avatar
clang-format committed
764
765
766
767
static void highbd_filter_selectively_horiz(
    uint16_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8,
    unsigned int mask_4x4, unsigned int mask_4x4_int,
    const loop_filter_info_n *lfi_n, const uint8_t *lfl, int bd) {
Jingning Han's avatar
Jingning Han committed
768
769
770
  unsigned int mask;
  int count;

clang-format's avatar
clang-format committed
771
772
  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask;
       mask >>= count) {
Jingning Han's avatar
Jingning Han committed
773
774
775
776
777
778
    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;

    count = 1;
    if (mask & 1) {
      if (mask_16x16 & 1) {
        if ((mask_16x16 & 3) == 3) {
779
          aom_highbd_lpf_horizontal_16_dual(s, pitch, lfi->mblim, lfi->lim,
780
                                            lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
781
782
          count = 2;
        } else {
783
784
          aom_highbd_lpf_horizontal_16(s, pitch, lfi->mblim, lfi->lim,
                                       lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
785
786
787
788
789
790
        }
      } else if (mask_8x8 & 1) {
        if ((mask_8x8 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
791
          aom_highbd_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
792
793
794
795
                                           lfi->hev_thr, lfin->mblim, lfin->lim,
                                           lfin->hev_thr, bd);

          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
796
            aom_highbd_lpf_horizontal_4_dual(
clang-format's avatar
clang-format committed
797
798
                s + 4 * pitch, pitch, lfi->mblim, lfi->lim, lfi->hev_thr,
                lfin->mblim, lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
799
800
          } else {
            if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
801
              aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
802
                                          lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
803
            } else if (mask_4x4_int & 2) {
Yaowu Xu's avatar
Yaowu Xu committed
804
              aom_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
805
                                          lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
806
807
808
809
            }
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
810
          aom_highbd_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim,
811
                                      lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
812
813

          if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
814
            aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
815
                                        lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
816
817
818
819
820
821
822
          }
        }
      } else if (mask_4x4 & 1) {
        if ((mask_4x4 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
823
          aom_highbd_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
824
825
826
                                           lfi->hev_thr, lfin->mblim, lfin->lim,
                                           lfin->hev_thr, bd);
          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
827
            aom_highbd_lpf_horizontal_4_dual(
clang-format's avatar
clang-format committed
828
829
                s + 4 * pitch, pitch, lfi->mblim, lfi->lim, lfi->hev_thr,
                lfin->mblim, lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
830
831
          } else {
            if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
832
              aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
833
                                          lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
834
            } else if (mask_4x4_int & 2) {
Yaowu Xu's avatar
Yaowu Xu committed
835
              aom_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
836
                                          lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
837
838
839
840
            }
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
841
          aom_highbd_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim,
842
                                      lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
843
844

          if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
845
            aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
846
                                        lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
847
848
849
          }
        }
      } else if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
850
        aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
851
                                    lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
852
853
854
855
856
857
858
859
860
861
      }
    }
    s += 8 * count;
    lfl += count;
    mask_16x16 >>= count;
    mask_8x8 >>= count;
    mask_4x4 >>= count;
    mask_4x4_int >>= count;
  }
}
862
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
863
864
865
866
867
868
869
870

// This function ors into the current lfm structure, where to do loop
// filters for the specific mi we are looking at. It uses information
// including the block_size_type (32x16, 32x32, etc.), the transform size,
// whether there were any coefficients encoded, and the loop filter strength
// block we are currently looking at. Shift is used to position the
// 1's we produce.
// TODO(JBB) Need another function for different resolution color..
871
872
873
874
static void build_masks(AV1_COMMON *const cm,
                        const loop_filter_info_n *const lfi_n,
                        const MODE_INFO *mi, const int shift_y,
                        const int shift_uv, LOOP_FILTER_MASK *lfm) {
Jingning Han's avatar
Jingning Han committed
875
876
  const MB_MODE_INFO *mbmi = &mi->mbmi;
  const BLOCK_SIZE block_size = mbmi->sb_type;
877
878
  // TODO(debargha): Check if masks can be setup correctly when
  // rectangular transfroms are used with the EXT_TX expt.
879
880
881
  const TX_SIZE tx_size_y = txsize_sqr_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_left = txsize_horz_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_above = txsize_vert_map[mbmi->tx_size];
882
  const TX_SIZE tx_size_uv =
883
884
885
886
887
      txsize_sqr_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
  const TX_SIZE tx_size_uv_left =
      txsize_horz_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
  const TX_SIZE tx_size_uv_above =
      txsize_vert_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
Fangwen Fu's avatar
Fangwen Fu committed
888
#if CONFIG_EXT_DELTA_Q
Cheng Chen's avatar
Cheng Chen committed
889
#if CONFIG_LOOPFILTER_LEVEL
Cheng Chen's avatar
Cheng Chen committed
890
  const int filter_level = get_filter_level(cm, lfi_n, 0, 0, mbmi);
891
#else
Fangwen Fu's avatar
Fangwen Fu committed
892
  const int filter_level = get_filter_level(cm, lfi_n, mbmi);
893
#endif
Fangwen Fu's avatar
Fangwen Fu committed
894
#else
Jingning Han's avatar
Jingning Han committed
895
  const int filter_level = get_filter_level(lfi_n, mbmi);
896
  (void)cm;
Fangwen Fu's avatar
Fangwen Fu committed
897
#endif
898
899
  uint64_t *const left_y = &lfm->left_y[tx_size_y_left];
  uint64_t *const above_y = &lfm->above_y[tx_size_y_above];
Jingning Han's avatar
Jingning Han committed
900
  uint64_t *const int_4x4_y = &lfm->int_4x4_y;
901
902
  uint16_t *const left_uv = &lfm->left_uv[tx_size_uv_left];
  uint16_t *const above_uv = &lfm->above_uv[tx_size_uv_above];
903
  uint16_t *const int_4x4_uv = &lfm->left_int_4x4_uv;
Jingning Han's avatar
Jingning Han committed
904
905
906
907
908
909
910
911
  int i;

  // If filter level is 0 we don't loop filter.
  if (!filter_level) {
    return;
  } else {
    const int w = num_8x8_blocks_wide_lookup[block_size];
    const int h = num_8x8_blocks_high_lookup[block_size];
912
913
914
    const int row = (shift_y >> MAX_MIB_SIZE_LOG2);
    const int col = shift_y - (row << MAX_MIB_SIZE_LOG2);

clang-format's avatar
clang-format committed
915
    for (i = 0; i < h; i++) memset(&lfm->lfl_y[row + i][col], filter_level, w);
Jingning Han's avatar
Jingning Han committed
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
  }

  // These set 1 in the current block size for the block size edges.
  // For instance if the block size is 32x16, we'll set:
  //    above =   1111
  //              0000
  //    and
  //    left  =   1000
  //          =   1000
  // NOTE : In this example the low bit is left most ( 1000 ) is stored as
  //        1,  not 8...
  //
  // U and V set things on a 16 bit scale.
  //
  *above_y |= above_prediction_mask[block_size] << shift_y;
  *above_uv |= above_prediction_mask_uv[block_size] << shift_uv;
  *left_y |= left_prediction_mask[block_size] << shift_y;
  *left_uv |= left_prediction_mask_uv[block_size] << shift_uv;

  // If the block has no coefficients and is not intra we skip applying
  // the loop filter on block edges.
937
  if (mbmi->skip && is_inter_block(mbmi)) return;
Jingning Han's avatar
Jingning Han committed
938
939
940
941
942

  // Here we are adding a mask for the transform size. The transform
  // size mask is set to be correct for a 64x64 prediction block size. We
  // mask to match the size of the block we are working on and then shift it
  // into place..
943
  *above_y |= (size_mask[block_size] & above_64x64_txform_mask[tx_size_y_above])
clang-format's avatar
clang-format committed
944
945
              << shift_y;
  *above_uv |=
946
      (size_mask_uv[block_size] & above_64x64_txform_mask_uv[tx_size_uv_above])
clang-format's avatar
clang-format committed
947
      << shift_uv;
Jingning Han's avatar
Jingning Han committed
948

949
  *left_y |= (size_mask[block_size] & left_64x64_txform_mask[tx_size_y_left])
clang-format's avatar
clang-format committed
950
             << shift_y;
951
952
953
  *left_uv |=
      (size_mask_uv[block_size] & left_64x64_txform_mask_uv[tx_size_uv_left])
      << shift_uv;
Jingning Han's avatar
Jingning Han committed
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968

  // Here we are trying to determine what to do with the internal 4x4 block
  // boundaries.  These differ from the 4x4 boundaries on the outside edge of
  // an 8x8 in that the internal ones can be skipped and don't depend on
  // the prediction block size.
  if (tx_size_y == TX_4X4)
    *int_4x4_y |= (size_mask[block_size] & 0xffffffffffffffffULL) << shift_y;

  if (tx_size_uv == TX_4X4)
    *int_4x4_uv |= (size_mask_uv[block_size] & 0xffff) << shift_uv;
}

// This function does the same thing as the one above with the exception that
// it only affects the y masks. It exists because for blocks < 16x16 in size,
// we only update u and v masks on the first block.
969
970
971
972
static void build_y_mask(AV1_COMMON *const cm,
                         const loop_filter_info_n *const lfi_n,
                         const MODE_INFO *mi, const int shift_y,
                         LOOP_FILTER_MASK *lfm) {
Jingning Han's avatar
Jingning Han committed
973
  const MB_MODE_INFO *mbmi = &mi->mbmi;
974
975
976
  const TX_SIZE tx_size_y = txsize_sqr_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_left = txsize_horz_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_above = txsize_vert_map[mbmi->tx_size];
977
  const BLOCK_SIZE block_size = mbmi->sb_type;
Fangwen Fu's avatar
Fangwen Fu committed
978
#if CONFIG_EXT_DELTA_Q
Cheng Chen's avatar
Cheng Chen committed
979
#if CONFIG_LOOPFILTER_LEVEL
Cheng Chen's avatar
Cheng Chen committed
980
  const int filter_level = get_filter_level(cm, lfi_n, 0, 0, mbmi);
981
#else
Fangwen Fu's avatar
Fangwen Fu committed
982
  const int filter_level = get_filter_level(cm, lfi_n, mbmi);
983
#endif
Fangwen Fu's avatar
Fangwen Fu committed
984
#else
Jingning Han's avatar
Jingning Han committed
985
  const int filter_level = get_filter_level(lfi_n, mbmi);
986
  (void)cm;
Fangwen Fu's avatar
Fangwen Fu committed
987
#endif
988
989
  uint64_t *const left_y = &lfm->left_y[tx_size_y_left];
  uint64_t *const above_y = &lfm->above_y[tx_size_y_above];
Jingning Han's avatar
Jingning Han committed
990
991
992
993
994
995
996
997
  uint64_t *const int_4x4_y = &lfm->int_4x4_y;
  int i;

  if (!filter_level) {
    return;
  } else {
    const int w = num_8x8_blocks_wide_lookup[block_size];
    const int h = num_8x8_blocks_high_lookup[block_size];
998
999
1000
    const int row = (shift_y >> MAX_MIB_SIZE_LOG2);
    const int col = shift_y - (row << MAX_MIB_SIZE_LOG2);

clang-format's avatar
clang-format committed
1001
    for (i = 0; i < h; i++) memset(&lfm->lfl_y[row + i][col], filter_level, w);
Jingning Han's avatar
Jingning Han committed
1002
1003
1004
1005
1006
  }

  *above_y |= above_prediction_mask[block_size] << shift_y;
  *left_y |= left_prediction_mask[block_size] << shift_y;

1007
  if (mbmi->skip && is_inter_block(mbmi)) return;
Jingning Han's avatar
Jingning Han committed
1008

1009
  *above_y |= (size_mask[block_size] & above_64x64_txform_mask[tx_size_y_above])
clang-format's avatar
clang-format committed
1010
              << shift_y;
Jingning Han's avatar
Jingning Han committed
1011

1012
  *left_y |= (size_mask[block_size] & left_64x64_txform_mask[tx_size_y_left])
clang-format's avatar
clang-format committed
1013
             << shift_y;
Jingning Han's avatar
Jingning Han committed
1014
1015
1016
1017
1018

  if (tx_size_y == TX_4X4)
    *int_4x4_y |= (size_mask[block_size] & 0xffffffffffffffffULL) << shift_y;
}

1019
#if CONFIG_LOOPFILTERING_ACROSS_TILES || CONFIG_LOOPFILTERING_ACROSS_TILES_EXT
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
// This function update the bit masks for the entire 64x64 region represented
// by mi_row, mi_col. In case one of the edge is a tile boundary, loop filtering
// for that edge is disabled. This function only check the tile boundary info
// for the top left corner mi to determine the boundary information for the
// top and left edge of the whole super block
static void update_tile_boundary_filter_mask(AV1_COMMON *const cm,
                                             const int mi_row, const int mi_col,
                                             LOOP_FILTER_MASK *lfm) {
  int i;
  MODE_INFO *const mi = cm->mi + mi_row * cm->mi_stride + mi_col;

1031
  if (mi->mbmi.boundary_info & TILE_LEFT_BOUNDARY) {
1032
1033
1034
1035
1036
1037
    for (i = 0; i <= TX_32X32; i++) {
      lfm->left_y[i] &= 0xfefefefefefefefeULL;
      lfm->left_uv[i] &= 0xeeee;
    }
  }

1038
  if (mi->mbmi.boundary_info & TILE_ABOVE_BOUNDARY) {
1039
1040
1041
1042
1043
1044
    for (i = 0; i <= TX_32X32; i++) {
      lfm->above_y[i] &= 0xffffffffffffff00ULL;
      lfm->above_uv[i] &= 0xfff0;
    }
  }
}
1045
#endif  // CONFIG_LOOPFILTERING_ACROSS_TILES
1046

Jingning Han's avatar
Jingning Han committed
1047
1048
1049
// This function sets up the bit masks for the entire 64x64 region represented
// by mi_row, mi_col.
// TODO(JBB): This function only works for yv12.
Yaowu Xu's avatar
Yaowu Xu committed
1050
1051
1052
void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
                    MODE_INFO **mi, const int mode_info_stride,
                    LOOP_FILTER_MASK *lfm) {
1053
1054
1055
#if CONFIG_EXT_PARTITION
  assert(0 && "Not yet updated");
#endif  // CONFIG_EXT_PARTITION
Jingning Han's avatar
Jingning Han committed
1056
1057
1058
1059
1060
1061
1062
1063
1064
  int idx_32, idx_16, idx_8;
  const loop_filter_info_n *const lfi_n = &cm->lf_info;
  MODE_INFO **mip = mi;
  MODE_INFO **mip2 = mi;

  // These are offsets to the next mi in the 64x64 block. It is what gets
  // added to the mi ptr as we go through each loop. It helps us to avoid
  // setting up special row and column counters for each index. The last step
  // brings us out back to the starting position.
clang-format's avatar
clang-format committed
1065
1066
1067
1068
1069
  const int offset_32[] = { 4, (mode_info_stride << 2) - 4, 4,
                            -(mode_info_stride << 2) - 4 };
  const int offset_16[] = { 2, (mode_info_stride << 1) - 2, 2,
                            -(mode_info_stride << 1) - 2 };
  const int offset[] = { 1, mode_info_stride - 1, 1, -mode_info_stride - 1 };
Jingning Han's avatar
Jingning Han committed
1070
1071
1072
1073
1074

  // Following variables represent shifts to position the current block
  // mask over the appropriate block. A shift of 36 to the left will move
  // the bits for the final 32 by 32 block in the 64x64 up 4 rows and left
  // 4 rows to the appropriate spot.
clang-format's avatar
clang-format committed
1075
1076
1077
1078
1079
  const int shift_32_y[] = { 0, 4, 32, 36 };
  const int shift_16_y[] = { 0, 2, 16, 18 };
  const int shift_8_y[] = { 0, 1, 8, 9 };
  const int shift_32_uv[] = { 0, 2, 8, 10 };
  const int shift_16_uv[] = { 0, 1, 4, 5 };
Jingning Han's avatar
Jingning Han committed
1080
  int i;
Yaowu Xu's avatar
Yaowu Xu committed
1081
1082
  const int max_rows = AOMMIN(cm->mi_rows - mi_row, MAX_MIB_SIZE);
  const int max_cols = AOMMIN(cm->mi_cols - mi_col, MAX_MIB_SIZE);
Jingning Han's avatar
Jingning Han committed
1083

Yaowu Xu's avatar
Yaowu Xu committed
1084
  av1_zero(*lfm);
Jingning Han's avatar
Jingning Han committed
1085
1086
1087
1088
1089
1090
  assert(mip[0] != NULL);

  // TODO(jimbankoski): Try moving most of the following code into decode
  // loop and storing lfm in the mbmi structure so that we don't have to go
  // through the recursive loop structure multiple times.
  switch (mip[0]->mbmi.sb_type) {
Fangwen Fu's avatar
Fangwen Fu committed
1091
    case BLOCK_64X64: build_masks(cm, lfi_n, mip[0], 0, 0, lfm); break;
1092
1093
    case BLOCK_64X32:
      build_masks(cm, lfi_n, mip[0], 0, 0, lfm);
Jingning Han's avatar
Jingning Han committed
1094
      mip2 = mip + mode_info_stride * 4;
clang-format's avatar
clang-format committed
1095
      if (4 >= max_rows) break;
Fangwen Fu's avatar
Fangwen Fu committed
1096
      build_masks(cm, lfi_n, mip2[0], 32, 8, lfm);
Jingning Han's avatar
Jingning Han committed
1097
      break;
1098
1099
    case BLOCK_32X64:
      build_masks(cm, lfi_n, mip[0], 0, 0, lfm);
Jingning Han's avatar
Jingning Han committed
1100
      mip2 = mip + 4;
clang-format's avatar
clang-format committed
1101
      if (4 >= max_cols) break;
Fangwen Fu's avatar
Fangwen Fu committed
1102
      build_masks(cm, lfi_n, mip2[0], 4, 2, lfm);
Jingning Han's avatar
Jingning Han committed
1103
1104
      break;
    default:
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
      for (idx_32 = 0; idx_32 < 4; mip += offset_32[idx_32], ++idx_32) {
        const int shift_y_32 = shift_32_y[idx_32];
        const int shift_uv_32 = shift_32_uv[idx_32];
        const int mi_32_col_offset = ((idx_32 & 1) << 2);
        const int mi_32_row_offset = ((idx_32 >> 1) << 2);
        if (mi_32_col_offset >= max_cols || mi_32_row_offset >= max_rows)
          continue;
        switch (mip[0]->mbmi.sb_type) {
          case BLOCK_32X32:
            build_masks(cm, lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
            break;
          case BLOCK_32X16:
            build_masks(cm, lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
            if (mi_32_row_offset + 2 >= max_rows) continue;
            mip2 = mip + mode_info_stride * 2;
            build_masks(cm, lfi_n, mip2[0], shift_y_32 + 16, shift_uv_32 + 4,
                        lfm);
            break;
          case BLOCK_16X32:
            build_masks(cm, lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
            if (mi_32_col_offset + 2 >= max_cols) continue;
            mip2 = mip + 2;
            build_masks(cm, lfi_n, mip2[0], shift_y_32 + 2, shift_uv_32 + 1,
                        lfm);
            break;
          default:
            for (idx_16 = 0; idx_16 < 4