av1_loopfilter.c 87.5 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
 */

Debargha Mukherjee's avatar
Debargha Mukherjee committed
12
13
#include <math.h>

Yaowu Xu's avatar
Yaowu Xu committed
14
15
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
16
#include "av1/common/av1_loopfilter.h"
17
18
#include "av1/common/onyxc_int.h"
#include "av1/common/reconinter.h"
Yaowu Xu's avatar
Yaowu Xu committed
19
20
#include "aom_dsp/aom_dsp_common.h"
#include "aom_mem/aom_mem.h"
21
#include "aom_ports/mem.h"
Jingning Han's avatar
Jingning Han committed
22

23
#include "av1/common/seg_common.h"
Jingning Han's avatar
Jingning Han committed
24

25
26
#define CONFIG_PARALLEL_DEBLOCKING_15TAPLUMAONLY 0

Jingning Han's avatar
Jingning Han committed
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
// 64 bit masks for left transform size. Each 1 represents a position where
// we should apply a loop filter across the left border of an 8x8 block
// boundary.
//
// In the case of TX_16X16->  ( in low order byte first we end up with
// a mask that looks like this
//
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//
// A loopfilter should be applied to every other 8x8 horizontally.
clang-format's avatar
clang-format committed
44
static const uint64_t left_64x64_txform_mask[TX_SIZES] = {
45
46
47
#if CONFIG_CB4X4
  0xffffffffffffffffULL,  // TX_2X2
#endif
Jingning Han's avatar
Jingning Han committed
48
49
50
51
  0xffffffffffffffffULL,  // TX_4X4
  0xffffffffffffffffULL,  // TX_8x8
  0x5555555555555555ULL,  // TX_16x16
  0x1111111111111111ULL,  // TX_32x32
52
53
54
#if CONFIG_TX64X64
  0x0101010101010101ULL,  // TX_64x64
#endif                    // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
};

// 64 bit masks for above transform size. Each 1 represents a position where
// we should apply a loop filter across the top border of an 8x8 block
// boundary.
//
// In the case of TX_32x32 ->  ( in low order byte first we end up with
// a mask that looks like this
//
//    11111111
//    00000000
//    00000000
//    00000000
//    11111111
//    00000000
//    00000000
//    00000000
//
// A loopfilter should be applied to every other 4 the row vertically.
clang-format's avatar
clang-format committed
74
static const uint64_t above_64x64_txform_mask[TX_SIZES] = {
75
76
77
#if CONFIG_CB4X4
  0xffffffffffffffffULL,  // TX_4X4
#endif
Jingning Han's avatar
Jingning Han committed
78
79
80
81
  0xffffffffffffffffULL,  // TX_4X4
  0xffffffffffffffffULL,  // TX_8x8
  0x00ff00ff00ff00ffULL,  // TX_16x16
  0x000000ff000000ffULL,  // TX_32x32
82
83
84
#if CONFIG_TX64X64
  0x00000000000000ffULL,  // TX_64x64
#endif                    // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
};

// 64 bit masks for prediction sizes (left). Each 1 represents a position
// where left border of an 8x8 block. These are aligned to the right most
// appropriate bit, and then shifted into place.
//
// In the case of TX_16x32 ->  ( low order byte first ) we end up with
// a mask that looks like this :
//
//  10000000
//  10000000
//  10000000
//  10000000
//  00000000
//  00000000
//  00000000
//  00000000
static const uint64_t left_prediction_mask[BLOCK_SIZES] = {
Jingning Han's avatar
Jingning Han committed
103
104
105
106
107
#if CONFIG_CB4X4
  0x0000000000000001ULL,  // BLOCK_2X2,
  0x0000000000000001ULL,  // BLOCK_2X4,
  0x0000000000000001ULL,  // BLOCK_4X2,
#endif
Jingning Han's avatar
Jingning Han committed
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
  0x0000000000000001ULL,  // BLOCK_4X4,
  0x0000000000000001ULL,  // BLOCK_4X8,
  0x0000000000000001ULL,  // BLOCK_8X4,
  0x0000000000000001ULL,  // BLOCK_8X8,
  0x0000000000000101ULL,  // BLOCK_8X16,
  0x0000000000000001ULL,  // BLOCK_16X8,
  0x0000000000000101ULL,  // BLOCK_16X16,
  0x0000000001010101ULL,  // BLOCK_16X32,
  0x0000000000000101ULL,  // BLOCK_32X16,
  0x0000000001010101ULL,  // BLOCK_32X32,
  0x0101010101010101ULL,  // BLOCK_32X64,
  0x0000000001010101ULL,  // BLOCK_64X32,
  0x0101010101010101ULL,  // BLOCK_64X64
};

// 64 bit mask to shift and set for each prediction size.
static const uint64_t above_prediction_mask[BLOCK_SIZES] = {
Jingning Han's avatar
Jingning Han committed
125
126
127
128
129
#if CONFIG_CB4X4
  0x0000000000000001ULL,  // BLOCK_2X2
  0x0000000000000001ULL,  // BLOCK_2X4
  0x0000000000000001ULL,  // BLOCK_4X2
#endif
Jingning Han's avatar
Jingning Han committed
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
  0x0000000000000001ULL,  // BLOCK_4X4
  0x0000000000000001ULL,  // BLOCK_4X8
  0x0000000000000001ULL,  // BLOCK_8X4
  0x0000000000000001ULL,  // BLOCK_8X8
  0x0000000000000001ULL,  // BLOCK_8X16,
  0x0000000000000003ULL,  // BLOCK_16X8
  0x0000000000000003ULL,  // BLOCK_16X16
  0x0000000000000003ULL,  // BLOCK_16X32,
  0x000000000000000fULL,  // BLOCK_32X16,
  0x000000000000000fULL,  // BLOCK_32X32,
  0x000000000000000fULL,  // BLOCK_32X64,
  0x00000000000000ffULL,  // BLOCK_64X32,
  0x00000000000000ffULL,  // BLOCK_64X64
};
// 64 bit mask to shift and set for each prediction size. A bit is set for
// each 8x8 block that would be in the left most block of the given block
// size in the 64x64 block.
static const uint64_t size_mask[BLOCK_SIZES] = {
Jingning Han's avatar
Jingning Han committed
148
149
150
151
152
#if CONFIG_CB4X4
  0x0000000000000001ULL,  // BLOCK_2X2
  0x0000000000000001ULL,  // BLOCK_2X4
  0x0000000000000001ULL,  // BLOCK_4X2
#endif
Jingning Han's avatar
Jingning Han committed
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
  0x0000000000000001ULL,  // BLOCK_4X4
  0x0000000000000001ULL,  // BLOCK_4X8
  0x0000000000000001ULL,  // BLOCK_8X4
  0x0000000000000001ULL,  // BLOCK_8X8
  0x0000000000000101ULL,  // BLOCK_8X16,
  0x0000000000000003ULL,  // BLOCK_16X8
  0x0000000000000303ULL,  // BLOCK_16X16
  0x0000000003030303ULL,  // BLOCK_16X32,
  0x0000000000000f0fULL,  // BLOCK_32X16,
  0x000000000f0f0f0fULL,  // BLOCK_32X32,
  0x0f0f0f0f0f0f0f0fULL,  // BLOCK_32X64,
  0x00000000ffffffffULL,  // BLOCK_64X32,
  0xffffffffffffffffULL,  // BLOCK_64X64
};

168
// These are used for masking the left and above 32x32 borders.
clang-format's avatar
clang-format committed
169
static const uint64_t left_border = 0x1111111111111111ULL;
Jingning Han's avatar
Jingning Han committed
170
171
172
static const uint64_t above_border = 0x000000ff000000ffULL;

// 16 bit masks for uv transform sizes.
clang-format's avatar
clang-format committed
173
static const uint16_t left_64x64_txform_mask_uv[TX_SIZES] = {
174
175
176
#if CONFIG_CB4X4
  0xffff,  // TX_2X2
#endif
Jingning Han's avatar
Jingning Han committed
177
178
179
180
  0xffff,  // TX_4X4
  0xffff,  // TX_8x8
  0x5555,  // TX_16x16
  0x1111,  // TX_32x32
181
182
183
#if CONFIG_TX64X64
  0x0101,  // TX_64x64, never used
#endif     // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
184
185
};

clang-format's avatar
clang-format committed
186
static const uint16_t above_64x64_txform_mask_uv[TX_SIZES] = {
187
188
189
#if CONFIG_CB4X4
  0xffff,  // TX_2X2
#endif
Jingning Han's avatar
Jingning Han committed
190
191
192
193
  0xffff,  // TX_4X4
  0xffff,  // TX_8x8
  0x0f0f,  // TX_16x16
  0x000f,  // TX_32x32
194
195
196
#if CONFIG_TX64X64
  0x0003,  // TX_64x64, never used
#endif     // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
197
198
199
200
};

// 16 bit left mask to shift and set for each uv prediction size.
static const uint16_t left_prediction_mask_uv[BLOCK_SIZES] = {
Jingning Han's avatar
Jingning Han committed
201
202
203
204
205
#if CONFIG_CB4X4
  0x0001,  // BLOCK_2X2,
  0x0001,  // BLOCK_2X4,
  0x0001,  // BLOCK_4X2,
#endif
Jingning Han's avatar
Jingning Han committed
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
  0x0001,  // BLOCK_4X4,
  0x0001,  // BLOCK_4X8,
  0x0001,  // BLOCK_8X4,
  0x0001,  // BLOCK_8X8,
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8,
  0x0001,  // BLOCK_16X16,
  0x0011,  // BLOCK_16X32,
  0x0001,  // BLOCK_32X16,
  0x0011,  // BLOCK_32X32,
  0x1111,  // BLOCK_32X64
  0x0011,  // BLOCK_64X32,
  0x1111,  // BLOCK_64X64
};
// 16 bit above mask to shift and set for uv each prediction size.
static const uint16_t above_prediction_mask_uv[BLOCK_SIZES] = {
Jingning Han's avatar
Jingning Han committed
222
223
224
225
226
#if CONFIG_CB4X4
  0x0001,  // BLOCK_2X2
  0x0001,  // BLOCK_2X4
  0x0001,  // BLOCK_4X2
#endif
Jingning Han's avatar
Jingning Han committed
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
  0x0001,  // BLOCK_4X4
  0x0001,  // BLOCK_4X8
  0x0001,  // BLOCK_8X4
  0x0001,  // BLOCK_8X8
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8
  0x0001,  // BLOCK_16X16
  0x0001,  // BLOCK_16X32,
  0x0003,  // BLOCK_32X16,
  0x0003,  // BLOCK_32X32,
  0x0003,  // BLOCK_32X64,
  0x000f,  // BLOCK_64X32,
  0x000f,  // BLOCK_64X64
};

// 64 bit mask to shift and set for each uv prediction size
static const uint16_t size_mask_uv[BLOCK_SIZES] = {
Jingning Han's avatar
Jingning Han committed
244
245
246
247
248
#if CONFIG_CB4X4
  0x0001,  // BLOCK_2X2
  0x0001,  // BLOCK_2X4
  0x0001,  // BLOCK_4X2
#endif
Jingning Han's avatar
Jingning Han committed
249
250
251
252
253
254
255
256
257
258
259
260
261
262
  0x0001,  // BLOCK_4X4
  0x0001,  // BLOCK_4X8
  0x0001,  // BLOCK_8X4
  0x0001,  // BLOCK_8X8
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8
  0x0001,  // BLOCK_16X16
  0x0011,  // BLOCK_16X32,
  0x0003,  // BLOCK_32X16,
  0x0033,  // BLOCK_32X32,
  0x3333,  // BLOCK_32X64,
  0x00ff,  // BLOCK_64X32,
  0xffff,  // BLOCK_64X64
};
clang-format's avatar
clang-format committed
263
static const uint16_t left_border_uv = 0x1111;
Jingning Han's avatar
Jingning Han committed
264
265
static const uint16_t above_border_uv = 0x000f;

266
static const int mode_lf_lut[] = {
Jingning Han's avatar
Jingning Han committed
267
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // INTRA_MODES
268
269
#if CONFIG_ALT_INTRA
  0,
Urvang Joshi's avatar
Urvang Joshi committed
270
271
272
273
#if CONFIG_SMOOTH_HV
  0, 0,
#endif         // CONFIG_SMOOTH_HV
#endif         // CONFIG_ALT_INTRA
274
  1, 1, 0, 1,  // INTER_MODES (ZEROMV == 0)
Yue Chen's avatar
Yue Chen committed
275
#if CONFIG_EXT_INTER
276
277
  1, 1, 1, 1, 1, 1, 0, 1  // INTER_COMPOUND_MODES (ZERO_ZEROMV == 0)
#endif                    // CONFIG_EXT_INTER
Jingning Han's avatar
Jingning Han committed
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
};

static void update_sharpness(loop_filter_info_n *lfi, int sharpness_lvl) {
  int lvl;

  // For each possible value for the loop filter fill out limits
  for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++) {
    // Set loop filter parameters that control sharpness.
    int block_inside_limit = lvl >> ((sharpness_lvl > 0) + (sharpness_lvl > 4));

    if (sharpness_lvl > 0) {
      if (block_inside_limit > (9 - sharpness_lvl))
        block_inside_limit = (9 - sharpness_lvl);
    }

clang-format's avatar
clang-format committed
293
    if (block_inside_limit < 1) block_inside_limit = 1;
Jingning Han's avatar
Jingning Han committed
294
295
296
297
298
299

    memset(lfi->lfthr[lvl].lim, block_inside_limit, SIMD_WIDTH);
    memset(lfi->lfthr[lvl].mblim, (2 * (lvl + 2) + block_inside_limit),
           SIMD_WIDTH);
  }
}
Fangwen Fu's avatar
Fangwen Fu committed
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
#if CONFIG_EXT_DELTA_Q
static uint8_t get_filter_level(const AV1_COMMON *cm,
                                const loop_filter_info_n *lfi_n,
                                const MB_MODE_INFO *mbmi) {
#if CONFIG_SUPERTX
  const int segment_id = AOMMIN(mbmi->segment_id, mbmi->segment_id_supertx);
  assert(
      IMPLIES(supertx_enabled(mbmi), mbmi->segment_id_supertx != MAX_SEGMENTS));
  assert(IMPLIES(supertx_enabled(mbmi),
                 mbmi->segment_id_supertx <= mbmi->segment_id));
#else
  const int segment_id = mbmi->segment_id;
#endif  // CONFIG_SUPERTX
  if (cm->delta_lf_present_flag) {
    int lvl_seg = clamp(mbmi->current_delta_lf_from_base + cm->lf.filter_level,
                        0, MAX_LOOP_FILTER);
    const int scale = 1 << (lvl_seg >> 5);
    if (segfeature_active(&cm->seg, segment_id, SEG_LVL_ALT_LF)) {
      const int data = get_segdata(&cm->seg, segment_id, SEG_LVL_ALT_LF);
      lvl_seg =
          clamp(cm->seg.abs_delta == SEGMENT_ABSDATA ? data : lvl_seg + data, 0,
                MAX_LOOP_FILTER);
    }
Jingning Han's avatar
Jingning Han committed
323

Fangwen Fu's avatar
Fangwen Fu committed
324
325
326
327
328
329
330
331
332
333
334
335
    if (cm->lf.mode_ref_delta_enabled) {
      lvl_seg += cm->lf.ref_deltas[mbmi->ref_frame[0]] * scale;
      if (mbmi->ref_frame[0] > INTRA_FRAME)
        lvl_seg += cm->lf.mode_deltas[mode_lf_lut[mbmi->mode]] * scale;
      lvl_seg = clamp(lvl_seg, 0, MAX_LOOP_FILTER);
    }
    return lvl_seg;
  } else {
    return lfi_n->lvl[segment_id][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]];
  }
}
#else
Jingning Han's avatar
Jingning Han committed
336
337
static uint8_t get_filter_level(const loop_filter_info_n *lfi_n,
                                const MB_MODE_INFO *mbmi) {
338
#if CONFIG_SUPERTX
Yaowu Xu's avatar
Yaowu Xu committed
339
  const int segment_id = AOMMIN(mbmi->segment_id, mbmi->segment_id_supertx);
clang-format's avatar
clang-format committed
340
341
  assert(
      IMPLIES(supertx_enabled(mbmi), mbmi->segment_id_supertx != MAX_SEGMENTS));
342
343
344
345
346
347
  assert(IMPLIES(supertx_enabled(mbmi),
                 mbmi->segment_id_supertx <= mbmi->segment_id));
#else
  const int segment_id = mbmi->segment_id;
#endif  // CONFIG_SUPERTX
  return lfi_n->lvl[segment_id][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]];
Jingning Han's avatar
Jingning Han committed
348
}
Fangwen Fu's avatar
Fangwen Fu committed
349
#endif
Jingning Han's avatar
Jingning Han committed
350

Yaowu Xu's avatar
Yaowu Xu committed
351
void av1_loop_filter_init(AV1_COMMON *cm) {
352
  assert(MB_MODE_COUNT == NELEMENTS(mode_lf_lut));
Jingning Han's avatar
Jingning Han committed
353
354
355
356
357
358
359
360
361
362
363
364
365
  loop_filter_info_n *lfi = &cm->lf_info;
  struct loopfilter *lf = &cm->lf;
  int lvl;

  // init limits for given sharpness
  update_sharpness(lfi, lf->sharpness_level);
  lf->last_sharpness_level = lf->sharpness_level;

  // init hev threshold const vectors
  for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++)
    memset(lfi->lfthr[lvl].hev_thr, (lvl >> 4), SIMD_WIDTH);
}

Yaowu Xu's avatar
Yaowu Xu committed
366
void av1_loop_filter_frame_init(AV1_COMMON *cm, int default_filt_lvl) {
Jingning Han's avatar
Jingning Han committed
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
  int seg_id;
  // n_shift is the multiplier for lf_deltas
  // the multiplier is 1 for when filter_lvl is between 0 and 31;
  // 2 when filter_lvl is between 32 and 63
  const int scale = 1 << (default_filt_lvl >> 5);
  loop_filter_info_n *const lfi = &cm->lf_info;
  struct loopfilter *const lf = &cm->lf;
  const struct segmentation *const seg = &cm->seg;

  // update limits if sharpness has changed
  if (lf->last_sharpness_level != lf->sharpness_level) {
    update_sharpness(lfi, lf->sharpness_level);
    lf->last_sharpness_level = lf->sharpness_level;
  }

  for (seg_id = 0; seg_id < MAX_SEGMENTS; seg_id++) {
    int lvl_seg = default_filt_lvl;
    if (segfeature_active(seg, seg_id, SEG_LVL_ALT_LF)) {
      const int data = get_segdata(seg, seg_id, SEG_LVL_ALT_LF);
clang-format's avatar
clang-format committed
386
387
388
      lvl_seg = clamp(
          seg->abs_delta == SEGMENT_ABSDATA ? data : default_filt_lvl + data, 0,
          MAX_LOOP_FILTER);
Jingning Han's avatar
Jingning Han committed
389
390
391
392
393
394
395
396
397
398
399
    }

    if (!lf->mode_ref_delta_enabled) {
      // we could get rid of this if we assume that deltas are set to
      // zero when not in use; encoder always uses deltas
      memset(lfi->lvl[seg_id], lvl_seg, sizeof(lfi->lvl[seg_id]));
    } else {
      int ref, mode;
      const int intra_lvl = lvl_seg + lf->ref_deltas[INTRA_FRAME] * scale;
      lfi->lvl[seg_id][INTRA_FRAME][0] = clamp(intra_lvl, 0, MAX_LOOP_FILTER);

400
      for (ref = LAST_FRAME; ref < TOTAL_REFS_PER_FRAME; ++ref) {
Jingning Han's avatar
Jingning Han committed
401
        for (mode = 0; mode < MAX_MODE_LF_DELTAS; ++mode) {
clang-format's avatar
clang-format committed
402
403
          const int inter_lvl = lvl_seg + lf->ref_deltas[ref] * scale +
                                lf->mode_deltas[mode] * scale;
Jingning Han's avatar
Jingning Han committed
404
405
406
407
408
409
410
          lfi->lvl[seg_id][ref][mode] = clamp(inter_lvl, 0, MAX_LOOP_FILTER);
        }
      }
    }
  }
}

clang-format's avatar
clang-format committed
411
412
static void filter_selectively_vert_row2(int subsampling_factor, uint8_t *s,
                                         int pitch, unsigned int mask_16x16_l,
Jingning Han's avatar
Jingning Han committed
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
                                         unsigned int mask_8x8_l,
                                         unsigned int mask_4x4_l,
                                         unsigned int mask_4x4_int_l,
                                         const loop_filter_info_n *lfi_n,
                                         const uint8_t *lfl) {
  const int mask_shift = subsampling_factor ? 4 : 8;
  const int mask_cutoff = subsampling_factor ? 0xf : 0xff;
  const int lfl_forward = subsampling_factor ? 4 : 8;

  unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff;
  unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff;
  unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff;
  unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff;
  unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff;
  unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff;
  unsigned int mask;

  for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 |
              mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1;
       mask; mask >>= 1) {
    const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl;
    const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward);

    if (mask & 1) {
      if ((mask_16x16_0 | mask_16x16_1) & 1) {
        if ((mask_16x16_0 & mask_16x16_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
441
          aom_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
442
443
                                   lfi0->hev_thr);
        } else if (mask_16x16_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
444
          aom_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
445
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
446
          aom_lpf_vertical_16(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
clang-format's avatar
clang-format committed
447
                              lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
448
449
450
451
452
        }
      }

      if ((mask_8x8_0 | mask_8x8_1) & 1) {
        if ((mask_8x8_0 & mask_8x8_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
453
          aom_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
454
455
456
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_8x8_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
457
          aom_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
458
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
459
          aom_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
460
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
461
462
463
464
465
        }
      }

      if ((mask_4x4_0 | mask_4x4_1) & 1) {
        if ((mask_4x4_0 & mask_4x4_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
466
          aom_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
467
468
469
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_4x4_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
470
          aom_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
471
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
472
          aom_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
473
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
474
475
476
477
478
        }
      }

      if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) {
        if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
479
          aom_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
480
481
482
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_4x4_int_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
483
          aom_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim,
484
                             lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
485
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
486
          aom_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim, lfi1->lim,
487
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
        }
      }
    }

    s += 8;
    lfl += 1;
    mask_16x16_0 >>= 1;
    mask_8x8_0 >>= 1;
    mask_4x4_0 >>= 1;
    mask_4x4_int_0 >>= 1;
    mask_16x16_1 >>= 1;
    mask_8x8_1 >>= 1;
    mask_4x4_1 >>= 1;
    mask_4x4_int_1 >>= 1;
  }
}

505
#if CONFIG_HIGHBITDEPTH
clang-format's avatar
clang-format committed
506
507
508
509
510
static void highbd_filter_selectively_vert_row2(
    int subsampling_factor, uint16_t *s, int pitch, unsigned int mask_16x16_l,
    unsigned int mask_8x8_l, unsigned int mask_4x4_l,
    unsigned int mask_4x4_int_l, const loop_filter_info_n *lfi_n,
    const uint8_t *lfl, int bd) {
Jingning Han's avatar
Jingning Han committed
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
  const int mask_shift = subsampling_factor ? 4 : 8;
  const int mask_cutoff = subsampling_factor ? 0xf : 0xff;
  const int lfl_forward = subsampling_factor ? 4 : 8;

  unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff;
  unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff;
  unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff;
  unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff;
  unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff;
  unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff;
  unsigned int mask;

  for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 |
clang-format's avatar
clang-format committed
526
              mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1;
Jingning Han's avatar
Jingning Han committed
527
528
529
530
531
532
533
       mask; mask >>= 1) {
    const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl;
    const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward);

    if (mask & 1) {
      if ((mask_16x16_0 | mask_16x16_1) & 1) {
        if ((mask_16x16_0 & mask_16x16_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
534
          aom_highbd_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
535
536
                                          lfi0->hev_thr, bd);
        } else if (mask_16x16_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
537
          aom_highbd_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
538
539
                                     lfi0->hev_thr, bd);
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
540
          aom_highbd_lpf_vertical_16(s + 8 * pitch, pitch, lfi1->mblim,
Jingning Han's avatar
Jingning Han committed
541
542
543
544
545
546
                                     lfi1->lim, lfi1->hev_thr, bd);
        }
      }

      if ((mask_8x8_0 | mask_8x8_1) & 1) {
        if ((mask_8x8_0 & mask_8x8_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
547
          aom_highbd_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
548
549
550
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_8x8_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
551
          aom_highbd_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim,
552
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
553
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
554
          aom_highbd_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim,
555
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
556
557
558
559
560
        }
      }

      if ((mask_4x4_0 | mask_4x4_1) & 1) {
        if ((mask_4x4_0 & mask_4x4_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
561
          aom_highbd_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
562
563
564
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_4x4_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
565
          aom_highbd_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim,
566
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
567
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
568
          aom_highbd_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim,
569
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
570
571
572
573
574
        }
      }

      if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) {
        if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
575
          aom_highbd_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
576
577
578
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_4x4_int_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
579
          aom_highbd_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim,
580
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
581
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
582
          aom_highbd_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim,
583
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
        }
      }
    }

    s += 8;
    lfl += 1;
    mask_16x16_0 >>= 1;
    mask_8x8_0 >>= 1;
    mask_4x4_0 >>= 1;
    mask_4x4_int_0 >>= 1;
    mask_16x16_1 >>= 1;
    mask_8x8_1 >>= 1;
    mask_4x4_1 >>= 1;
    mask_4x4_int_1 >>= 1;
  }
}
600
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
601

clang-format's avatar
clang-format committed
602
603
604
605
static void filter_selectively_horiz(
    uint8_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8,
    unsigned int mask_4x4, unsigned int mask_4x4_int,
    const loop_filter_info_n *lfi_n, const uint8_t *lfl) {
Jingning Han's avatar
Jingning Han committed
606
607
608
  unsigned int mask;
  int count;

clang-format's avatar
clang-format committed
609
610
  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask;
       mask >>= count) {
Jingning Han's avatar
Jingning Han committed
611
612
613
614
615
616
    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;

    count = 1;
    if (mask & 1) {
      if (mask_16x16 & 1) {
        if ((mask_16x16 & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
617
          aom_lpf_horizontal_edge_16(s, pitch, lfi->mblim, lfi->lim,
618
                                     lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
619
620
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
621
          aom_lpf_horizontal_edge_8(s, pitch, lfi->mblim, lfi->lim,
622
                                    lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
623
624
625
626
627
628
        }
      } else if (mask_8x8 & 1) {
        if ((mask_8x8 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
629
          aom_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
630
631
632
633
                                    lfi->hev_thr, lfin->mblim, lfin->lim,
                                    lfin->hev_thr);

          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
634
            aom_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
Jingning Han's avatar
Jingning Han committed
635
636
637
638
                                      lfi->lim, lfi->hev_thr, lfin->mblim,
                                      lfin->lim, lfin->hev_thr);
          } else {
            if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
639
              aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
640
                                   lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
641
            else if (mask_4x4_int & 2)
Yaowu Xu's avatar
Yaowu Xu committed
642
              aom_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
643
                                   lfin->lim, lfin->hev_thr);
Jingning Han's avatar
Jingning Han committed
644
645
646
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
647
          aom_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
648
649

          if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
650
            aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
651
                                 lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
652
653
654
655
656
657
        }
      } else if (mask_4x4 & 1) {
        if ((mask_4x4 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
658
          aom_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
659
660
661
                                    lfi->hev_thr, lfin->mblim, lfin->lim,
                                    lfin->hev_thr);
          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
662
            aom_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
Jingning Han's avatar
Jingning Han committed
663
664
665
666
                                      lfi->lim, lfi->hev_thr, lfin->mblim,
                                      lfin->lim, lfin->hev_thr);
          } else {
            if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
667
              aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
668
                                   lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
669
            else if (mask_4x4_int & 2)
Yaowu Xu's avatar
Yaowu Xu committed
670
              aom_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
671
                                   lfin->lim, lfin->hev_thr);
Jingning Han's avatar
Jingning Han committed
672
673
674
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
675
          aom_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
676
677

          if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
678
            aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
679
                                 lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
680
681
        }
      } else if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
682
        aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
683
                             lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
684
685
686
687
688
689
690
691
692
693
694
      }
    }
    s += 8 * count;
    lfl += count;
    mask_16x16 >>= count;
    mask_8x8 >>= count;
    mask_4x4 >>= count;
    mask_4x4_int >>= count;
  }
}

695
#if CONFIG_HIGHBITDEPTH
clang-format's avatar
clang-format committed
696
697
698
699
static void highbd_filter_selectively_horiz(
    uint16_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8,
    unsigned int mask_4x4, unsigned int mask_4x4_int,
    const loop_filter_info_n *lfi_n, const uint8_t *lfl, int bd) {
Jingning Han's avatar
Jingning Han committed
700
701
702
  unsigned int mask;
  int count;

clang-format's avatar
clang-format committed
703
704
  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask;
       mask >>= count) {
Jingning Han's avatar
Jingning Han committed
705
706
707
708
709
710
    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;

    count = 1;
    if (mask & 1) {
      if (mask_16x16 & 1) {
        if ((mask_16x16 & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
711
          aom_highbd_lpf_horizontal_edge_16(s, pitch, lfi->mblim, lfi->lim,
712
                                            lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
713
714
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
715
          aom_highbd_lpf_horizontal_edge_8(s, pitch, lfi->mblim, lfi->lim,
716
                                           lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
717
718
719
720
721
722
        }
      } else if (mask_8x8 & 1) {
        if ((mask_8x8 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
723
          aom_highbd_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
724
725
726
727
                                           lfi->hev_thr, lfin->mblim, lfin->lim,
                                           lfin->hev_thr, bd);

          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
728
            aom_highbd_lpf_horizontal_4_dual(
clang-format's avatar
clang-format committed
729
730
                s + 4 * pitch, pitch, lfi->mblim, lfi->lim, lfi->hev_thr,
                lfin->mblim, lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
731
732
          } else {
            if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
733
              aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
734
                                          lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
735
            } else if (mask_4x4_int & 2) {
Yaowu Xu's avatar
Yaowu Xu committed
736
              aom_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
737
                                          lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
738
739
740
741
            }
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
742
          aom_highbd_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim,
743
                                      lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
744
745

          if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
746
            aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
747
                                        lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
748
749
750
751
752
753
754
          }
        }
      } else if (mask_4x4 & 1) {
        if ((mask_4x4 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
755
          aom_highbd_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
756
757
758
                                           lfi->hev_thr, lfin->mblim, lfin->lim,
                                           lfin->hev_thr, bd);
          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
759
            aom_highbd_lpf_horizontal_4_dual(
clang-format's avatar
clang-format committed
760
761
                s + 4 * pitch, pitch, lfi->mblim, lfi->lim, lfi->hev_thr,
                lfin->mblim, lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
762
763
          } else {
            if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
764
              aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
765
                                          lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
766
            } else if (mask_4x4_int & 2) {
Yaowu Xu's avatar
Yaowu Xu committed
767
              aom_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
768
                                          lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
769
770
771
772
            }
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
773
          aom_highbd_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim,
774
                                      lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
775
776

          if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
777
            aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
778
                                        lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
779
780
781
          }
        }
      } else if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
782
        aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
783
                                    lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
784
785
786
787
788
789
790
791
792
793
      }
    }
    s += 8 * count;
    lfl += count;
    mask_16x16 >>= count;
    mask_8x8 >>= count;
    mask_4x4 >>= count;
    mask_4x4_int >>= count;
  }
}
794
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
795
796
797
798
799
800
801
802

// This function ors into the current lfm structure, where to do loop
// filters for the specific mi we are looking at. It uses information
// including the block_size_type (32x16, 32x32, etc.), the transform size,
// whether there were any coefficients encoded, and the loop filter strength
// block we are currently looking at. Shift is used to position the
// 1's we produce.
// TODO(JBB) Need another function for different resolution color..
803
804
805
806
static void build_masks(AV1_COMMON *const cm,
                        const loop_filter_info_n *const lfi_n,
                        const MODE_INFO *mi, const int shift_y,
                        const int shift_uv, LOOP_FILTER_MASK *lfm) {
Jingning Han's avatar
Jingning Han committed
807
808
  const MB_MODE_INFO *mbmi = &mi->mbmi;
  const BLOCK_SIZE block_size = mbmi->sb_type;
809
810
  // TODO(debargha): Check if masks can be setup correctly when
  // rectangular transfroms are used with the EXT_TX expt.
811
812
813
  const TX_SIZE tx_size_y = txsize_sqr_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_left = txsize_horz_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_above = txsize_vert_map[mbmi->tx_size];
814
  const TX_SIZE tx_size_uv =
815
816
817
818
819
      txsize_sqr_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
  const TX_SIZE tx_size_uv_left =
      txsize_horz_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
  const TX_SIZE tx_size_uv_above =
      txsize_vert_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
Fangwen Fu's avatar
Fangwen Fu committed
820
821
822
#if CONFIG_EXT_DELTA_Q
  const int filter_level = get_filter_level(cm, lfi_n, mbmi);
#else
Jingning Han's avatar
Jingning Han committed
823
  const int filter_level = get_filter_level(lfi_n, mbmi);
824
  (void)cm;
Fangwen Fu's avatar
Fangwen Fu committed
825
#endif
826
827
  uint64_t *const left_y = &lfm->left_y[tx_size_y_left];
  uint64_t *const above_y = &lfm->above_y[tx_size_y_above];
Jingning Han's avatar
Jingning Han committed
828
  uint64_t *const int_4x4_y = &lfm->int_4x4_y;
829
830
  uint16_t *const left_uv = &lfm->left_uv[tx_size_uv_left];
  uint16_t *const above_uv = &lfm->above_uv[tx_size_uv_above];
831
  uint16_t *const int_4x4_uv = &lfm->left_int_4x4_uv;
Jingning Han's avatar
Jingning Han committed
832
833
834
835
836
837
838
839
  int i;

  // If filter level is 0 we don't loop filter.
  if (!filter_level) {
    return;
  } else {
    const int w = num_8x8_blocks_wide_lookup[block_size];
    const int h = num_8x8_blocks_high_lookup[block_size];
840
841
842
    const int row = (shift_y >> MAX_MIB_SIZE_LOG2);
    const int col = shift_y - (row << MAX_MIB_SIZE_LOG2);

clang-format's avatar
clang-format committed
843
    for (i = 0; i < h; i++) memset(&lfm->lfl_y[row + i][col], filter_level, w);
Jingning Han's avatar
Jingning Han committed
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
  }

  // These set 1 in the current block size for the block size edges.
  // For instance if the block size is 32x16, we'll set:
  //    above =   1111
  //              0000
  //    and
  //    left  =   1000
  //          =   1000
  // NOTE : In this example the low bit is left most ( 1000 ) is stored as
  //        1,  not 8...
  //
  // U and V set things on a 16 bit scale.
  //
  *above_y |= above_prediction_mask[block_size] << shift_y;
  *above_uv |= above_prediction_mask_uv[block_size] << shift_uv;
  *left_y |= left_prediction_mask[block_size] << shift_y;
  *left_uv |= left_prediction_mask_uv[block_size] << shift_uv;

  // If the block has no coefficients and is not intra we skip applying
  // the loop filter on block edges.
865
  if (mbmi->skip && is_inter_block(mbmi)) return;
Jingning Han's avatar
Jingning Han committed
866
867
868
869
870

  // Here we are adding a mask for the transform size. The transform
  // size mask is set to be correct for a 64x64 prediction block size. We
  // mask to match the size of the block we are working on and then shift it
  // into place..
871
  *above_y |= (size_mask[block_size] & above_64x64_txform_mask[tx_size_y_above])
clang-format's avatar
clang-format committed
872
873
              << shift_y;
  *above_uv |=
874
      (size_mask_uv[block_size] & above_64x64_txform_mask_uv[tx_size_uv_above])
clang-format's avatar
clang-format committed
875
      << shift_uv;
Jingning Han's avatar
Jingning Han committed
876

877
  *left_y |= (size_mask[block_size] & left_64x64_txform_mask[tx_size_y_left])
clang-format's avatar
clang-format committed
878
             << shift_y;
879
880
881
  *left_uv |=
      (size_mask_uv[block_size] & left_64x64_txform_mask_uv[tx_size_uv_left])
      << shift_uv;
Jingning Han's avatar
Jingning Han committed
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896

  // Here we are trying to determine what to do with the internal 4x4 block
  // boundaries.  These differ from the 4x4 boundaries on the outside edge of
  // an 8x8 in that the internal ones can be skipped and don't depend on
  // the prediction block size.
  if (tx_size_y == TX_4X4)
    *int_4x4_y |= (size_mask[block_size] & 0xffffffffffffffffULL) << shift_y;

  if (tx_size_uv == TX_4X4)
    *int_4x4_uv |= (size_mask_uv[block_size] & 0xffff) << shift_uv;
}

// This function does the same thing as the one above with the exception that
// it only affects the y masks. It exists because for blocks < 16x16 in size,
// we only update u and v masks on the first block.
897
898
899
static void build_y_mask(AV1_COMMON *const cm,
                         const loop_filter_info_n *const lfi_n,
                         const MODE_INFO *mi, const int shift_y,
900
#if CONFIG_SUPERTX
901
                         int supertx_enabled,
902
#endif  // CONFIG_SUPERTX
903
                         LOOP_FILTER_MASK *lfm) {
Jingning Han's avatar
Jingning Han committed
904
  const MB_MODE_INFO *mbmi = &mi->mbmi;
905
906
907
  const TX_SIZE tx_size_y = txsize_sqr_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_left = txsize_horz_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_above = txsize_vert_map[mbmi->tx_size];
908
909
910
911
912
913
#if CONFIG_SUPERTX
  const BLOCK_SIZE block_size =
      supertx_enabled ? (BLOCK_SIZE)(3 * tx_size_y) : mbmi->sb_type;
#else
  const BLOCK_SIZE block_size = mbmi->sb_type;
#endif
Fangwen Fu's avatar
Fangwen Fu committed
914
915
916
#if CONFIG_EXT_DELTA_Q
  const int filter_level = get_filter_level(cm, lfi_n, mbmi);
#else
Jingning Han's avatar
Jingning Han committed
917
  const int filter_level = get_filter_level(lfi_n, mbmi);
918
  (void)cm;
Fangwen Fu's avatar
Fangwen Fu committed
919
#endif
920
921
  uint64_t *const left_y = &lfm->left_y[tx_size_y_left];
  uint64_t *const above_y = &lfm->above_y[tx_size_y_above];
Jingning Han's avatar
Jingning Han committed
922
923
924
925
926
927
928
929
  uint64_t *const int_4x4_y = &lfm->int_4x4_y;
  int i;

  if (!filter_level) {
    return;
  } else {
    const int w = num_8x8_blocks_wide_lookup[block_size];
    const int h = num_8x8_blocks_high_lookup[block_size];
930
931
932
    const int row = (shift_y >> MAX_MIB_SIZE_LOG2);
    const int col = shift_y - (row << MAX_MIB_SIZE_LOG2);

clang-format's avatar
clang-format committed
933
    for (i = 0; i < h; i++) memset(&lfm->lfl_y[row + i][col], filter_level, w);
Jingning Han's avatar
Jingning Han committed
934
935
936
937
938
  }

  *above_y |= above_prediction_mask[block_size] << shift_y;
  *left_y |= left_prediction_mask[block_size] << shift_y;

939
  if (mbmi->skip && is_inter_block(mbmi)) return;
Jingning Han's avatar
Jingning Han committed
940

941
  *above_y |= (size_mask[block_size] & above_64x64_txform_mask[tx_size_y_above])
clang-format's avatar
clang-format committed
942
              << shift_y;
Jingning Han's avatar
Jingning Han committed
943

944
  *left_y |= (size_mask[block_size] & left_64x64_txform_mask[tx_size_y_left])
clang-format's avatar
clang-format committed
945
             << shift_y;
Jingning Han's avatar
Jingning Han committed
946
947
948
949
950

  if (tx_size_y == TX_4X4)
    *int_4x4_y |= (size_mask[block_size] & 0xffffffffffffffffULL) << shift_y;
}

951
#if CONFIG_LOOPFILTERING_ACROSS_TILES
952
953
954
955
956
957
958
959
960
961
962
// This function update the bit masks for the entire 64x64 region represented
// by mi_row, mi_col. In case one of the edge is a tile boundary, loop filtering
// for that edge is disabled. This function only check the tile boundary info
// for the top left corner mi to determine the boundary information for the
// top and left edge of the whole super block
static void update_tile_boundary_filter_mask(AV1_COMMON *const cm,
                                             const int mi_row, const int mi_col,
                                             LOOP_FILTER_MASK *lfm) {
  int i;
  MODE_INFO *const mi = cm->mi + mi_row * cm->mi_stride + mi_col;

963
  if (mi->mbmi.boundary_info & TILE_LEFT_BOUNDARY) {
964
965
966
967
968
969
    for (i = 0; i <= TX_32X32; i++) {
      lfm->left_y[i] &= 0xfefefefefefefefeULL;
      lfm->left_uv[i] &= 0xeeee;
    }
  }

970
  if (mi->mbmi.boundary_info & TILE_ABOVE_BOUNDARY) {
971
972
973
974
975
976
    for (i = 0; i <= TX_32X32; i++) {
      lfm->above_y[i] &= 0xffffffffffffff00ULL;
      lfm->above_uv[i] &= 0xfff0;
    }
  }
}
977
#endif  // CONFIG_LOOPFILTERING_ACROSS_TILES
978

Jingning Han's avatar
Jingning Han committed
979
980
981
// This function sets up the bit masks for the entire 64x64 region represented
// by mi_row, mi_col.
// TODO(JBB): This function only works for yv12.
Yaowu Xu's avatar
Yaowu Xu committed
982
983
984
void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
                    MODE_INFO **mi, const int mode_info_stride,
                    LOOP_FILTER_MASK *lfm) {
Jingning Han's avatar
Jingning Han committed
985
986
987
988
989
990
991
992
993
  int idx_32, idx_16, idx_8;
  const loop_filter_info_n *const lfi_n = &cm->lf_info;
  MODE_INFO **mip = mi;
  MODE_INFO **mip2 = mi;

  // These are offsets to the next mi in the 64x64 block. It is what gets
  // added to the mi ptr as we go through each loop. It helps us to avoid
  // setting up special row and column counters for each index. The last step
  // brings us out back to the starting position.
clang-format's avatar
clang-format committed
994
995
996
997
998
  const int offset_32[] = { 4, (mode_info_stride << 2) - 4, 4,
                            -(mode_info_stride << 2) - 4 };
  const int offset_16[] = { 2, (mode_info_stride << 1) - 2, 2,
                            -(mode_info_stride << 1) - 2 };
  const int offset[] = { 1, mode_info_stride - 1, 1, -mode_info_stride - 1 };
Jingning Han's avatar
Jingning Han committed
999
1000
1001
1002
1003

  // Following variables represent shifts to position the current block
  // mask over the appropriate block. A shift of 36 to the left will move
  // the bits for the final 32 by 32 block in the 64x64 up 4 rows and left
  // 4 rows to the appropriate spot.
clang-format's avatar
clang-format committed
1004
1005
1006
1007
1008
  const int shift_32_y[] = { 0, 4, 32, 36 };
  const int shift_16_y[] = { 0, 2, 16, 18 };
  const int shift_8_y[] = { 0, 1, 8, 9 };
  const int shift_32_uv[] = { 0, 2, 8, 10 };
  const int shift_16_uv[] = { 0, 1, 4, 5 };
Jingning Han's avatar
Jingning Han committed
1009
  int i;
Yaowu Xu's avatar
Yaowu Xu committed
1010
1011
  const int max_rows = AOMMIN(cm->mi_rows - mi_row, MAX_MIB_SIZE);
  const int max_cols = AOMMIN(cm->mi_cols - mi_col, MAX_MIB_SIZE);
1012
1013
1014
#if CONFIG_EXT_PARTITION
  assert(0 && "Not yet updated");
#endif  // CONFIG_EXT_PARTITION
Jingning Han's avatar
Jingning Han committed
1015

Yaowu Xu's avatar
Yaowu Xu committed
1016
  av1_zero(*lfm);
Jingning Han's avatar
Jingning Han committed
1017
1018
1019
1020
1021
1022
  assert(mip[0] != NULL);

  // TODO(jimbankoski): Try moving most of the following code into decode
  // loop and storing lfm in the mbmi structure so that we don't have to go
  // through the recursive loop structure multiple times.
  switch (mip[0]->mbmi.sb_type) {
Fangwen Fu's avatar
Fangwen Fu committed
1023
1024
    case BLOCK_64X64: build_masks(cm, lfi_n, mip[0], 0, 0, lfm); break;
    case BLOCK_64X32: build_masks(cm, lfi_n, mip[0], 0, 0, lfm);
1025
#if CONFIG_SUPERTX && CONFIG_TX64X64
1026
      if (supertx_enabled(&mip[0]->mbmi)) break;
1027
#endif  // CONFIG_SUPERTX && CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
1028
      mip2 = mip + mode_info_stride * 4;
clang-format's avatar
clang-format committed
1029
      if (4 >= max_rows) break;
Fangwen Fu's avatar
Fangwen Fu committed
1030
      build_masks(cm, lfi_n, mip2[0], 32, 8, lfm);
Jingning Han's avatar
Jingning Han committed
1031
      break;
Fangwen Fu's avatar
Fangwen Fu committed
1032
    case BLOCK_32X64: build_masks(cm, lfi_n, mip[0], 0, 0, lfm);
1033
#if CONFIG_SUPERTX && CONFIG_TX64X64
1034
      if (supertx_enabled(&mip[0]->mbmi)) break;
1035
#endif  // CONFIG_SUPERTX && CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
1036
      mip2 = mip + 4;
clang-format's avatar
clang-format committed
1037
      if (4 >= max_cols) break;
Fangwen Fu's avatar
Fangwen Fu committed
1038
      build_masks(cm, lfi_n, mip2[0], 4, 2, lfm);
Jingning Han's avatar
Jingning Han committed
1039
1040
      break;
    default:
1041
#if CONFIG_SUPERTX && CONFIG_TX64X64
1042
      if (mip[0]->mbmi.tx_size == TX_64X64) {
Fangwen Fu's avatar
Fangwen Fu committed
1043
        build_masks(cm, lfi_n, mip[0], 0, 0, lfm);
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
      } else {
#endif  // CONFIG_SUPERTX && CONFIG_TX64X64
        for (idx_32 = 0; idx_32 < 4; mip += offset_32[idx_32], ++idx_32) {
          const int shift_y_32 = shift_32_y[idx_32];
          const int shift_uv_32 = shift_32_uv[idx_32];
          const int mi_32_col_offset = ((idx_32 & 1) << 2);
          const int mi_32_row_offset = ((idx_32 >> 1) << 2);
          if (mi_32_col_offset >= max_cols || mi_32_row_offset >= max_rows)
            continue;
          switch (mip[0]->mbmi.sb_type) {
            case BLOCK_32X32:
Fangwen Fu's avatar
Fangwen Fu committed
1055
              build_masks(cm, lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
1056
1057
              break;
            case BLOCK_32X16:
Fangwen Fu's avatar
Fangwen Fu committed
1058
              build_masks(cm, lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
1059
#if CONFIG_SUPERTX
1060
              if (supertx_enabled(&mip[0]->mbmi)) break;
1061
#endif
1062
1063
              if (mi_32_row_offset + 2 >= max_rows) continue;
              mip2 = mip + mode_info_stride * 2;
Fangwen Fu's avatar
Fangwen Fu committed
1064
              build_masks(cm, lfi_n, mip2[0], shift_y_32 + 16, shift_uv_32 + 4,
1065
1066
1067
                          lfm);
              break;
            case BLOCK_16X32:
Fangwen Fu's avatar
Fangwen Fu committed
1068
              build_masks(cm, lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
1069
#if CONFIG_SUPERTX
1070
              if (supertx_enabled(&mip[0]->mbmi)) break;
1071
#endif
1072
1073
              if (mi_32_col_offset + 2 >= max_cols) continue;
              mip2 = mip + 2;
Fangwen Fu's avatar
Fangwen Fu committed
1074
1075
              build_masks(cm, lfi_n, mip2[0], shift_y_32 + 2, shift_uv_32 + 1,
                          lfm);
1076
              break;
1077
1078
1079
            default:
#if CONFIG_SUPERTX
              if (mip[0]->mbmi.tx_size == TX_32X32) {
Fangwen Fu's avatar
Fangwen Fu committed
1080
                build_masks(cm, lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
1081
1082
                break;
              }
1083
#endif
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
              for (idx_16 = 0; idx_16 < 4; mip += offset_16[idx_16], ++idx_16) {
                const int shift_y_32_16 = shift_y_32 + shift_16_y[idx_16];
                const int shift_uv_32_16 = shift_uv_32 + shift_16_uv[idx_16];
                const int mi_16_col_offset =
                    mi_32_col_offset + ((idx_16 & 1) << 1);
                const int mi_16_row_offset =
                    mi_32_row_offset + ((idx_16 >> 1) << 1);

                if (mi_16_col_offset >= max_cols ||
                    mi_16_row_offset >= max_rows)
                  continue;

                switch (mip[0]->mbmi.sb_type) {
                  case BLOCK_16X16:
Fangwen Fu's avatar
Fangwen Fu committed
1098
1099
                    build_masks(cm, lfi_n, mip[0], shift_y_32_16,
                                shift_uv_32_16, lfm);
1100
1101
                    break;
                  case BLOCK_16X8:
1102
#if CONFIG_SUPERTX
1103
                    if (supertx_enabled(&mip[0]->mbmi)) break;
1104
#endif
Fangwen Fu's avatar
Fangwen Fu committed
1105
1106
                    build_masks(cm, lfi_n, mip[0], shift_y_32_16,
                                shift_uv_32_16, lfm);
1107
1108
                    if (mi_16_row_offset + 1 >= max_rows) continue;
                    mip2 = mip + mode_info_stride;
1109
                    build_y_mask(cm, lfi_n, mip2[0], shift_y_32_16 + 8,
1110
#if CONFIG_SUPERTX
1111
                                 0,
1112
#endif
1113
                                 lfm);
1114
1115
                    break;
                  case BLOCK_8X16:
1116
#if CONFIG_SUPERTX
1117
                    if (supertx_enabled(&mip[0]->mbmi)) break;
1118
#endif
Fangwen Fu's avatar
Fangwen Fu committed
1119
1120
                    build_masks(cm, lfi_n, mip[0], shift_y_32_16,
                                shift_uv_32_16, lfm);
1121
1122
                    if (mi_16_col_offset + 1 >= max_cols) continue;
                    mip2 = mip + 1;
1123
                    build_y_mask(cm, lfi_n, mip2[0], shift_y_32_16 + 1,
1124
#if CONFIG_SUPERTX
1125
                                 0,
1126
#endif
1127
                                 lfm);
1128
1129
1130
1131
                    break;
                  default: {
                    const int shift_y_32_16_8_zero =
                        shift_y_32_16 + shift_8_y[0];
1132
#if CONFIG_SUPERTX
1133
                    if (mip[0]->mbmi.tx_size == TX_16X16) {
Fangwen Fu's avatar
Fangwen Fu committed
1134
1135
                      build_masks(cm, lfi_n, mip[0], shift_y_32_16_8_zero,
                                  shift_uv_32_16, lfm);
1136
1137
1138
                      break;
                    }
#endif
Fangwen Fu's avatar
Fangwen Fu committed
1139
                    build_masks(cm, lfi_n, mip[0], shift_y_32_16_8_zero,
1140
                                shift_uv_32_16, lfm);
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151