av1_loopfilter.c 98.9 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
 */

Debargha Mukherjee's avatar
Debargha Mukherjee committed
12
13
#include <math.h>

Yaowu Xu's avatar
Yaowu Xu committed
14
15
16
17
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_mem/aom_mem.h"
18
#include "aom_ports/mem.h"
19
20
21
#include "av1/common/av1_loopfilter.h"
#include "av1/common/onyxc_int.h"
#include "av1/common/reconinter.h"
22
#include "av1/common/seg_common.h"
Jingning Han's avatar
Jingning Han committed
23

24
25
26
27
28
29
#if CONFIG_LOOPFILTER_LEVEL
static const SEG_LVL_FEATURES seg_lvl_lf_lut[MAX_MB_PLANE][2] = {
  { SEG_LVL_ALT_LF_Y_V, SEG_LVL_ALT_LF_Y_H },
  { SEG_LVL_ALT_LF_U, SEG_LVL_ALT_LF_U },
  { SEG_LVL_ALT_LF_V, SEG_LVL_ALT_LF_V }
};
30
31
32
33
34
35

#if CONFIG_EXT_DELTA_Q
static const int delta_lf_id_lut[MAX_MB_PLANE][2] = {
  { 0, 1 }, { 2, 2 }, { 3, 3 }
};
#endif  // CONFIG_EXT_DELTA_Q
36
37
#endif  // CONFIG_LOOPFILTER_LEVEL

38
#define PARALLEL_DEBLOCKING_15TAPLUMAONLY 1
39
#define PARALLEL_DEBLOCKING_DISABLE_15TAP 0
Ola Hugosson's avatar
Ola Hugosson committed
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
#if CONFIG_DEBLOCK_13TAP
#define PARALLEL_DEBLOCKING_5_TAP_CHROMA 1
#else
#define PARALLEL_DEBLOCKING_5_TAP_CHROMA 0
#endif

#if PARALLEL_DEBLOCKING_5_TAP_CHROMA
extern void aom_lpf_vertical_6_c(uint8_t *s, int pitch, const uint8_t *blimit,
                                 const uint8_t *limit, const uint8_t *thresh);

extern void aom_lpf_horizontal_6_c(uint8_t *s, int p, const uint8_t *blimit,
                                   const uint8_t *limit, const uint8_t *thresh);

extern void aom_highbd_lpf_horizontal_6_c(uint16_t *s, int p,
                                          const uint8_t *blimit,
                                          const uint8_t *limit,
                                          const uint8_t *thresh, int bd);

extern void aom_highbd_lpf_vertical_6_c(uint16_t *s, int pitch,
                                        const uint8_t *blimit,
                                        const uint8_t *limit,
                                        const uint8_t *thresh, int bd);
#endif
63

Jingning Han's avatar
Jingning Han committed
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
// 64 bit masks for left transform size. Each 1 represents a position where
// we should apply a loop filter across the left border of an 8x8 block
// boundary.
//
// In the case of TX_16X16->  ( in low order byte first we end up with
// a mask that looks like this
//
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//
// A loopfilter should be applied to every other 8x8 horizontally.
clang-format's avatar
clang-format committed
81
static const uint64_t left_64x64_txform_mask[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
82
83
84
85
  0xffffffffffffffffULL,  // TX_4X4
  0xffffffffffffffffULL,  // TX_8x8
  0x5555555555555555ULL,  // TX_16x16
  0x1111111111111111ULL,  // TX_32x32
86
87
88
#if CONFIG_TX64X64
  0x0101010101010101ULL,  // TX_64x64
#endif                    // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
};

// 64 bit masks for above transform size. Each 1 represents a position where
// we should apply a loop filter across the top border of an 8x8 block
// boundary.
//
// In the case of TX_32x32 ->  ( in low order byte first we end up with
// a mask that looks like this
//
//    11111111
//    00000000
//    00000000
//    00000000
//    11111111
//    00000000
//    00000000
//    00000000
//
// A loopfilter should be applied to every other 4 the row vertically.
clang-format's avatar
clang-format committed
108
static const uint64_t above_64x64_txform_mask[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
109
110
111
112
  0xffffffffffffffffULL,  // TX_4X4
  0xffffffffffffffffULL,  // TX_8x8
  0x00ff00ff00ff00ffULL,  // TX_16x16
  0x000000ff000000ffULL,  // TX_32x32
113
114
115
#if CONFIG_TX64X64
  0x00000000000000ffULL,  // TX_64x64
#endif                    // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
};

// 64 bit masks for prediction sizes (left). Each 1 represents a position
// where left border of an 8x8 block. These are aligned to the right most
// appropriate bit, and then shifted into place.
//
// In the case of TX_16x32 ->  ( low order byte first ) we end up with
// a mask that looks like this :
//
//  10000000
//  10000000
//  10000000
//  10000000
//  00000000
//  00000000
//  00000000
//  00000000
133
static const uint64_t left_prediction_mask[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
134
135
136
137
138
139
140
141
142
143
144
145
  0x0000000000000001ULL,  // BLOCK_4X4,
  0x0000000000000001ULL,  // BLOCK_4X8,
  0x0000000000000001ULL,  // BLOCK_8X4,
  0x0000000000000001ULL,  // BLOCK_8X8,
  0x0000000000000101ULL,  // BLOCK_8X16,
  0x0000000000000001ULL,  // BLOCK_16X8,
  0x0000000000000101ULL,  // BLOCK_16X16,
  0x0000000001010101ULL,  // BLOCK_16X32,
  0x0000000000000101ULL,  // BLOCK_32X16,
  0x0000000001010101ULL,  // BLOCK_32X32,
  0x0101010101010101ULL,  // BLOCK_32X64,
  0x0000000001010101ULL,  // BLOCK_64X32,
146
147
148
149
  0x0101010101010101ULL,  // BLOCK_64X64,
  0x0000000000000101ULL,  // BLOCK_4X16,
  0x0000000000000001ULL,  // BLOCK_16X4,
  0x0000000001010101ULL,  // BLOCK_8X32,
150
151
152
  0x0000000000000001ULL,  // BLOCK_32X8,
  0x0101010101010101ULL,  // BLOCK_16X64,
  0x0000000000000101ULL,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
153
154
155
};

// 64 bit mask to shift and set for each prediction size.
156
static const uint64_t above_prediction_mask[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
157
158
159
160
161
162
163
164
165
166
167
168
  0x0000000000000001ULL,  // BLOCK_4X4
  0x0000000000000001ULL,  // BLOCK_4X8
  0x0000000000000001ULL,  // BLOCK_8X4
  0x0000000000000001ULL,  // BLOCK_8X8
  0x0000000000000001ULL,  // BLOCK_8X16,
  0x0000000000000003ULL,  // BLOCK_16X8
  0x0000000000000003ULL,  // BLOCK_16X16
  0x0000000000000003ULL,  // BLOCK_16X32,
  0x000000000000000fULL,  // BLOCK_32X16,
  0x000000000000000fULL,  // BLOCK_32X32,
  0x000000000000000fULL,  // BLOCK_32X64,
  0x00000000000000ffULL,  // BLOCK_64X32,
169
170
171
172
  0x00000000000000ffULL,  // BLOCK_64X64,
  0x0000000000000001ULL,  // BLOCK_4X16,
  0x0000000000000003ULL,  // BLOCK_16X4,
  0x0000000000000001ULL,  // BLOCK_8X32,
173
174
175
  0x000000000000000fULL,  // BLOCK_32X8,
  0x0000000000000003ULL,  // BLOCK_16X64,
  0x00000000000000ffULL,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
176
177
};
// 64 bit mask to shift and set for each prediction size. A bit is set for
178
// each 8x8 block that would be in the top left most block of the given block
Jingning Han's avatar
Jingning Han committed
179
// size in the 64x64 block.
180
static const uint64_t size_mask[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
181
182
183
184
185
186
187
188
189
190
191
192
  0x0000000000000001ULL,  // BLOCK_4X4
  0x0000000000000001ULL,  // BLOCK_4X8
  0x0000000000000001ULL,  // BLOCK_8X4
  0x0000000000000001ULL,  // BLOCK_8X8
  0x0000000000000101ULL,  // BLOCK_8X16,
  0x0000000000000003ULL,  // BLOCK_16X8
  0x0000000000000303ULL,  // BLOCK_16X16
  0x0000000003030303ULL,  // BLOCK_16X32,
  0x0000000000000f0fULL,  // BLOCK_32X16,
  0x000000000f0f0f0fULL,  // BLOCK_32X32,
  0x0f0f0f0f0f0f0f0fULL,  // BLOCK_32X64,
  0x00000000ffffffffULL,  // BLOCK_64X32,
193
194
195
196
  0xffffffffffffffffULL,  // BLOCK_64X64,
  0x0000000000000101ULL,  // BLOCK_4X16,
  0x0000000000000003ULL,  // BLOCK_16X4,
  0x0000000001010101ULL,  // BLOCK_8X32,
197
198
199
  0x000000000000000fULL,  // BLOCK_32X8,
  0x0303030303030303ULL,  // BLOCK_16X64,
  0x000000000000ffffULL,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
200
201
};

202
// These are used for masking the left and above 32x32 borders.
clang-format's avatar
clang-format committed
203
static const uint64_t left_border = 0x1111111111111111ULL;
Jingning Han's avatar
Jingning Han committed
204
205
206
static const uint64_t above_border = 0x000000ff000000ffULL;

// 16 bit masks for uv transform sizes.
clang-format's avatar
clang-format committed
207
static const uint16_t left_64x64_txform_mask_uv[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
208
209
210
211
  0xffff,  // TX_4X4
  0xffff,  // TX_8x8
  0x5555,  // TX_16x16
  0x1111,  // TX_32x32
212
213
214
#if CONFIG_TX64X64
  0x0101,  // TX_64x64, never used
#endif     // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
215
216
};

clang-format's avatar
clang-format committed
217
static const uint16_t above_64x64_txform_mask_uv[TX_SIZES] = {
Jingning Han's avatar
Jingning Han committed
218
219
220
221
  0xffff,  // TX_4X4
  0xffff,  // TX_8x8
  0x0f0f,  // TX_16x16
  0x000f,  // TX_32x32
222
223
224
#if CONFIG_TX64X64
  0x0003,  // TX_64x64, never used
#endif     // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
225
226
227
};

// 16 bit left mask to shift and set for each uv prediction size.
228
static const uint16_t left_prediction_mask_uv[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
229
230
231
232
233
234
235
236
237
238
239
240
  0x0001,  // BLOCK_4X4,
  0x0001,  // BLOCK_4X8,
  0x0001,  // BLOCK_8X4,
  0x0001,  // BLOCK_8X8,
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8,
  0x0001,  // BLOCK_16X16,
  0x0011,  // BLOCK_16X32,
  0x0001,  // BLOCK_32X16,
  0x0011,  // BLOCK_32X32,
  0x1111,  // BLOCK_32X64
  0x0011,  // BLOCK_64X32,
241
242
243
244
  0x1111,  // BLOCK_64X64,
  0x0001,  // BLOCK_4X16,
  0x0001,  // BLOCK_16X4,
  0x0011,  // BLOCK_8X32,
245
246
247
  0x0001,  // BLOCK_32X8,
  0x1111,  // BLOCK_16X64,
  0x0001,  // BLOCK_64X16,
Jingning Han's avatar
Jingning Han committed
248
};
249

Jingning Han's avatar
Jingning Han committed
250
// 16 bit above mask to shift and set for uv each prediction size.
251
static const uint16_t above_prediction_mask_uv[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
252
253
254
255
256
257
258
259
260
261
262
263
  0x0001,  // BLOCK_4X4
  0x0001,  // BLOCK_4X8
  0x0001,  // BLOCK_8X4
  0x0001,  // BLOCK_8X8
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8
  0x0001,  // BLOCK_16X16
  0x0001,  // BLOCK_16X32,
  0x0003,  // BLOCK_32X16,
  0x0003,  // BLOCK_32X32,
  0x0003,  // BLOCK_32X64,
  0x000f,  // BLOCK_64X32,
264
265
266
267
  0x000f,  // BLOCK_64X64,
  0x0001,  // BLOCK_4X16,
  0x0001,  // BLOCK_16X4,
  0x0001,  // BLOCK_8X32,
268
269
270
  0x0003,  // BLOCK_32X8,
  0x0001,  // BLOCK_16X64,
  0x000f,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
271
272
273
};

// 64 bit mask to shift and set for each uv prediction size
274
static const uint16_t size_mask_uv[BLOCK_SIZES_ALL] = {
Jingning Han's avatar
Jingning Han committed
275
276
277
278
279
280
281
282
283
284
285
286
  0x0001,  // BLOCK_4X4
  0x0001,  // BLOCK_4X8
  0x0001,  // BLOCK_8X4
  0x0001,  // BLOCK_8X8
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8
  0x0001,  // BLOCK_16X16
  0x0011,  // BLOCK_16X32,
  0x0003,  // BLOCK_32X16,
  0x0033,  // BLOCK_32X32,
  0x3333,  // BLOCK_32X64,
  0x00ff,  // BLOCK_64X32,
287
288
289
290
  0xffff,  // BLOCK_64X64,
  0x0001,  // BLOCK_4X16,
  0x0001,  // BLOCK_16X4,
  0x0011,  // BLOCK_8X32,
291
292
293
  0x0003,  // BLOCK_32X8,
  0x1111,  // BLOCK_16X64,
  0x000f,  // BLOCK_64X16
Jingning Han's avatar
Jingning Han committed
294
};
clang-format's avatar
clang-format committed
295
static const uint16_t left_border_uv = 0x1111;
Jingning Han's avatar
Jingning Han committed
296
297
static const uint16_t above_border_uv = 0x000f;

298
static const int mode_lf_lut[] = {
299
300
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // INTRA_MODES
  1, 1, 0, 1,                             // INTER_MODES (GLOBALMV == 0)
Sarah Parker's avatar
Sarah Parker committed
301
  1, 1, 1, 1, 1, 1, 0, 1  // INTER_COMPOUND_MODES (GLOBAL_GLOBALMV == 0)
Jingning Han's avatar
Jingning Han committed
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
};

static void update_sharpness(loop_filter_info_n *lfi, int sharpness_lvl) {
  int lvl;

  // For each possible value for the loop filter fill out limits
  for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++) {
    // Set loop filter parameters that control sharpness.
    int block_inside_limit = lvl >> ((sharpness_lvl > 0) + (sharpness_lvl > 4));

    if (sharpness_lvl > 0) {
      if (block_inside_limit > (9 - sharpness_lvl))
        block_inside_limit = (9 - sharpness_lvl);
    }

clang-format's avatar
clang-format committed
317
    if (block_inside_limit < 1) block_inside_limit = 1;
Jingning Han's avatar
Jingning Han committed
318
319
320
321
322
323

    memset(lfi->lfthr[lvl].lim, block_inside_limit, SIMD_WIDTH);
    memset(lfi->lfthr[lvl].mblim, (2 * (lvl + 2) + block_inside_limit),
           SIMD_WIDTH);
  }
}
Fangwen Fu's avatar
Fangwen Fu committed
324
325
326
#if CONFIG_EXT_DELTA_Q
static uint8_t get_filter_level(const AV1_COMMON *cm,
                                const loop_filter_info_n *lfi_n,
Cheng Chen's avatar
Cheng Chen committed
327
#if CONFIG_LOOPFILTER_LEVEL
328
                                const int dir_idx, int plane,
329
#endif
Fangwen Fu's avatar
Fangwen Fu committed
330
331
332
                                const MB_MODE_INFO *mbmi) {
  const int segment_id = mbmi->segment_id;
  if (cm->delta_lf_present_flag) {
Cheng Chen's avatar
Cheng Chen committed
333
#if CONFIG_LOOPFILTER_LEVEL
334
335
336
337
338
339
340
    int delta_lf;
    if (cm->delta_lf_multi) {
      const int delta_lf_idx = delta_lf_id_lut[plane][dir_idx];
      delta_lf = mbmi->curr_delta_lf[delta_lf_idx];
    } else {
      delta_lf = mbmi->current_delta_lf_from_base;
    }
341
    int lvl_seg =
342
        clamp(delta_lf + cm->lf.filter_level[dir_idx], 0, MAX_LOOP_FILTER);
343
#else
Fangwen Fu's avatar
Fangwen Fu committed
344
345
    int lvl_seg = clamp(mbmi->current_delta_lf_from_base + cm->lf.filter_level,
                        0, MAX_LOOP_FILTER);
346
#endif
347
348
349
350
351
#if CONFIG_LOOPFILTER_LEVEL
    assert(plane >= 0 && plane <= 2);
    const int seg_lf_feature_id = seg_lvl_lf_lut[plane][dir_idx];
    if (segfeature_active(&cm->seg, segment_id, seg_lf_feature_id)) {
      const int data = get_segdata(&cm->seg, segment_id, seg_lf_feature_id);
352
      lvl_seg = clamp(lvl_seg + data, 0, MAX_LOOP_FILTER);
353
354
    }
#else
Fangwen Fu's avatar
Fangwen Fu committed
355
356
    if (segfeature_active(&cm->seg, segment_id, SEG_LVL_ALT_LF)) {
      const int data = get_segdata(&cm->seg, segment_id, SEG_LVL_ALT_LF);
357
      lvl_seg = clamp(lvl_seg + data, 0, MAX_LOOP_FILTER);
Fangwen Fu's avatar
Fangwen Fu committed
358
    }
359
#endif  // CONFIG_LOOPFILTER_LEVEL
Jingning Han's avatar
Jingning Han committed
360

Fangwen Fu's avatar
Fangwen Fu committed
361
    if (cm->lf.mode_ref_delta_enabled) {
Luc Trudeau's avatar
Luc Trudeau committed
362
      const int scale = 1 << (lvl_seg >> 5);
Fangwen Fu's avatar
Fangwen Fu committed
363
364
365
366
367
368
369
      lvl_seg += cm->lf.ref_deltas[mbmi->ref_frame[0]] * scale;
      if (mbmi->ref_frame[0] > INTRA_FRAME)
        lvl_seg += cm->lf.mode_deltas[mode_lf_lut[mbmi->mode]] * scale;
      lvl_seg = clamp(lvl_seg, 0, MAX_LOOP_FILTER);
    }
    return lvl_seg;
  } else {
Cheng Chen's avatar
Cheng Chen committed
370
#if CONFIG_LOOPFILTER_LEVEL
371
372
373
    return lfi_n
        ->lvl[segment_id][dir_idx][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]];
#else
Fangwen Fu's avatar
Fangwen Fu committed
374
    return lfi_n->lvl[segment_id][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]];
375
#endif
Fangwen Fu's avatar
Fangwen Fu committed
376
377
378
  }
}
#else
Jingning Han's avatar
Jingning Han committed
379
380
static uint8_t get_filter_level(const loop_filter_info_n *lfi_n,
                                const MB_MODE_INFO *mbmi) {
381
382
  const int segment_id = mbmi->segment_id;
  return lfi_n->lvl[segment_id][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]];
Jingning Han's avatar
Jingning Han committed
383
}
Fangwen Fu's avatar
Fangwen Fu committed
384
#endif
Jingning Han's avatar
Jingning Han committed
385

Yaowu Xu's avatar
Yaowu Xu committed
386
void av1_loop_filter_init(AV1_COMMON *cm) {
387
  assert(MB_MODE_COUNT == NELEMENTS(mode_lf_lut));
Jingning Han's avatar
Jingning Han committed
388
389
390
391
392
393
394
395
396
397
398
399
400
  loop_filter_info_n *lfi = &cm->lf_info;
  struct loopfilter *lf = &cm->lf;
  int lvl;

  // init limits for given sharpness
  update_sharpness(lfi, lf->sharpness_level);
  lf->last_sharpness_level = lf->sharpness_level;

  // init hev threshold const vectors
  for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++)
    memset(lfi->lfthr[lvl].hev_thr, (lvl >> 4), SIMD_WIDTH);
}

401
void av1_loop_filter_frame_init(AV1_COMMON *cm, int default_filt_lvl,
402
403
404
405
406
407
                                int default_filt_lvl_r
#if CONFIG_LOOPFILTER_LEVEL
                                ,
                                int plane
#endif
                                ) {
Jingning Han's avatar
Jingning Han committed
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
  int seg_id;
  // n_shift is the multiplier for lf_deltas
  // the multiplier is 1 for when filter_lvl is between 0 and 31;
  // 2 when filter_lvl is between 32 and 63
  loop_filter_info_n *const lfi = &cm->lf_info;
  struct loopfilter *const lf = &cm->lf;
  const struct segmentation *const seg = &cm->seg;

  // update limits if sharpness has changed
  if (lf->last_sharpness_level != lf->sharpness_level) {
    update_sharpness(lfi, lf->sharpness_level);
    lf->last_sharpness_level = lf->sharpness_level;
  }

  for (seg_id = 0; seg_id < MAX_SEGMENTS; seg_id++) {
423
424
425
426
427
428
429
    for (int dir = 0; dir < 2; ++dir) {
      int lvl_seg = (dir == 0) ? default_filt_lvl : default_filt_lvl_r;
#if CONFIG_LOOPFILTER_LEVEL
      assert(plane >= 0 && plane <= 2);
      const int seg_lf_feature_id = seg_lvl_lf_lut[plane][dir];
      if (segfeature_active(seg, seg_id, seg_lf_feature_id)) {
        const int data = get_segdata(&cm->seg, seg_id, seg_lf_feature_id);
430
        lvl_seg = clamp(lvl_seg + data, 0, MAX_LOOP_FILTER);
431
432
433
434
      }
#else
      if (segfeature_active(seg, seg_id, SEG_LVL_ALT_LF)) {
        const int data = get_segdata(seg, seg_id, SEG_LVL_ALT_LF);
435
        lvl_seg = clamp(lvl_seg + data, 0, MAX_LOOP_FILTER);
436
437
      }
#endif  // CONFIG_LOOPFILTER_LEVEL
Jingning Han's avatar
Jingning Han committed
438

439
440
441
442
443
444
445
446
447
448
      if (!lf->mode_ref_delta_enabled) {
// we could get rid of this if we assume that deltas are set to
// zero when not in use; encoder always uses deltas
#if CONFIG_LOOPFILTER_LEVEL
        memset(lfi->lvl[seg_id][dir], lvl_seg, sizeof(lfi->lvl[seg_id][dir]));
#else
        memset(lfi->lvl[seg_id], lvl_seg, sizeof(lfi->lvl[seg_id]));
#endif  // CONFIG_LOOPFILTER_LEVEL
      } else {
        int ref, mode;
Cheng Chen's avatar
Cheng Chen committed
449
#if CONFIG_LOOPFILTER_LEVEL
Luc Trudeau's avatar
Luc Trudeau committed
450
        const int scale = 1 << (lvl_seg >> 5);
451
452
453
454
455
456
457
458
459
460
461
462
463
        const int intra_lvl = lvl_seg + lf->ref_deltas[INTRA_FRAME] * scale;
        lfi->lvl[seg_id][dir][INTRA_FRAME][0] =
            clamp(intra_lvl, 0, MAX_LOOP_FILTER);

        for (ref = LAST_FRAME; ref < TOTAL_REFS_PER_FRAME; ++ref) {
          for (mode = 0; mode < MAX_MODE_LF_DELTAS; ++mode) {
            const int inter_lvl = lvl_seg + lf->ref_deltas[ref] * scale +
                                  lf->mode_deltas[mode] * scale;
            lfi->lvl[seg_id][dir][ref][mode] =
                clamp(inter_lvl, 0, MAX_LOOP_FILTER);
          }
        }
#else
Luc Trudeau's avatar
Luc Trudeau committed
464
        const int scale = 1 << (default_filt_lvl >> 5);
465
466
467
468
469
470
471
472
473
        const int intra_lvl = lvl_seg + lf->ref_deltas[INTRA_FRAME] * scale;
        lfi->lvl[seg_id][INTRA_FRAME][0] = clamp(intra_lvl, 0, MAX_LOOP_FILTER);

        for (ref = LAST_FRAME; ref < TOTAL_REFS_PER_FRAME; ++ref) {
          for (mode = 0; mode < MAX_MODE_LF_DELTAS; ++mode) {
            const int inter_lvl = lvl_seg + lf->ref_deltas[ref] * scale +
                                  lf->mode_deltas[mode] * scale;
            lfi->lvl[seg_id][ref][mode] = clamp(inter_lvl, 0, MAX_LOOP_FILTER);
          }
Jingning Han's avatar
Jingning Han committed
474
        }
475
#endif
476
      }
Jingning Han's avatar
Jingning Han committed
477
478
479
480
    }
  }
}

clang-format's avatar
clang-format committed
481
482
static void filter_selectively_vert_row2(int subsampling_factor, uint8_t *s,
                                         int pitch, unsigned int mask_16x16_l,
Jingning Han's avatar
Jingning Han committed
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
                                         unsigned int mask_8x8_l,
                                         unsigned int mask_4x4_l,
                                         unsigned int mask_4x4_int_l,
                                         const loop_filter_info_n *lfi_n,
                                         const uint8_t *lfl) {
  const int mask_shift = subsampling_factor ? 4 : 8;
  const int mask_cutoff = subsampling_factor ? 0xf : 0xff;
  const int lfl_forward = subsampling_factor ? 4 : 8;

  unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff;
  unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff;
  unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff;
  unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff;
  unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff;
  unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff;
  unsigned int mask;

  for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 |
              mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1;
       mask; mask >>= 1) {
    const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl;
    const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward);

    if (mask & 1) {
      if ((mask_16x16_0 | mask_16x16_1) & 1) {
        if ((mask_16x16_0 & mask_16x16_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
511
          aom_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
512
513
                                   lfi0->hev_thr);
        } else if (mask_16x16_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
514
          aom_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
515
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
516
          aom_lpf_vertical_16(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
clang-format's avatar
clang-format committed
517
                              lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
518
519
520
521
522
        }
      }

      if ((mask_8x8_0 | mask_8x8_1) & 1) {
        if ((mask_8x8_0 & mask_8x8_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
523
          aom_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
524
525
526
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_8x8_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
527
          aom_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
528
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
529
          aom_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
530
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
531
532
533
534
535
        }
      }

      if ((mask_4x4_0 | mask_4x4_1) & 1) {
        if ((mask_4x4_0 & mask_4x4_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
536
          aom_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
537
538
539
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_4x4_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
540
          aom_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
541
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
542
          aom_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
543
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
544
545
546
547
548
        }
      }

      if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) {
        if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
549
          aom_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
550
551
552
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_4x4_int_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
553
          aom_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim,
554
                             lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
555
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
556
          aom_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim, lfi1->lim,
557
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
        }
      }
    }

    s += 8;
    lfl += 1;
    mask_16x16_0 >>= 1;
    mask_8x8_0 >>= 1;
    mask_4x4_0 >>= 1;
    mask_4x4_int_0 >>= 1;
    mask_16x16_1 >>= 1;
    mask_8x8_1 >>= 1;
    mask_4x4_1 >>= 1;
    mask_4x4_int_1 >>= 1;
  }
}

575
#if CONFIG_HIGHBITDEPTH
clang-format's avatar
clang-format committed
576
577
578
579
580
static void highbd_filter_selectively_vert_row2(
    int subsampling_factor, uint16_t *s, int pitch, unsigned int mask_16x16_l,
    unsigned int mask_8x8_l, unsigned int mask_4x4_l,
    unsigned int mask_4x4_int_l, const loop_filter_info_n *lfi_n,
    const uint8_t *lfl, int bd) {
Jingning Han's avatar
Jingning Han committed
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
  const int mask_shift = subsampling_factor ? 4 : 8;
  const int mask_cutoff = subsampling_factor ? 0xf : 0xff;
  const int lfl_forward = subsampling_factor ? 4 : 8;

  unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff;
  unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff;
  unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff;
  unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff;
  unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff;
  unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff;
  unsigned int mask;

  for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 |
clang-format's avatar
clang-format committed
596
              mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1;
Jingning Han's avatar
Jingning Han committed
597
598
599
600
601
602
603
       mask; mask >>= 1) {
    const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl;
    const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward);

    if (mask & 1) {
      if ((mask_16x16_0 | mask_16x16_1) & 1) {
        if ((mask_16x16_0 & mask_16x16_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
604
          aom_highbd_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
605
606
                                          lfi0->hev_thr, bd);
        } else if (mask_16x16_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
607
          aom_highbd_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
608
609
                                     lfi0->hev_thr, bd);
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
610
          aom_highbd_lpf_vertical_16(s + 8 * pitch, pitch, lfi1->mblim,
Jingning Han's avatar
Jingning Han committed
611
612
613
614
615
616
                                     lfi1->lim, lfi1->hev_thr, bd);
        }
      }

      if ((mask_8x8_0 | mask_8x8_1) & 1) {
        if ((mask_8x8_0 & mask_8x8_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
617
          aom_highbd_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
618
619
620
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_8x8_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
621
          aom_highbd_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim,
622
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
623
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
624
          aom_highbd_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim,
625
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
626
627
628
629
630
        }
      }

      if ((mask_4x4_0 | mask_4x4_1) & 1) {
        if ((mask_4x4_0 & mask_4x4_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
631
          aom_highbd_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
632
633
634
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_4x4_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
635
          aom_highbd_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim,
636
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
637
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
638
          aom_highbd_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim,
639
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
640
641
642
643
644
        }
      }

      if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) {
        if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
645
          aom_highbd_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
646
647
648
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_4x4_int_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
649
          aom_highbd_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim,
650
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
651
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
652
          aom_highbd_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim,
653
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
        }
      }
    }

    s += 8;
    lfl += 1;
    mask_16x16_0 >>= 1;
    mask_8x8_0 >>= 1;
    mask_4x4_0 >>= 1;
    mask_4x4_int_0 >>= 1;
    mask_16x16_1 >>= 1;
    mask_8x8_1 >>= 1;
    mask_4x4_1 >>= 1;
    mask_4x4_int_1 >>= 1;
  }
}
670
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
671

clang-format's avatar
clang-format committed
672
673
674
static void filter_selectively_horiz(
    uint8_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8,
    unsigned int mask_4x4, unsigned int mask_4x4_int,
675
    const loop_filter_info_n *lfi_n, const uint8_t *lfl) {
Jingning Han's avatar
Jingning Han committed
676
677
678
  unsigned int mask;
  int count;

clang-format's avatar
clang-format committed
679
680
  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask;
       mask >>= count) {
Jingning Han's avatar
Jingning Han committed
681
682
683
684
685
686
    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;

    count = 1;
    if (mask & 1) {
      if (mask_16x16 & 1) {
        if ((mask_16x16 & 3) == 3) {
James Zern's avatar
James Zern committed
687
          aom_lpf_horizontal_16_dual(s, pitch, lfi->mblim, lfi->lim,
688
                                     lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
689
690
          count = 2;
        } else {
James Zern's avatar
James Zern committed
691
          aom_lpf_horizontal_16(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
692
693
694
695
696
697
        }
      } else if (mask_8x8 & 1) {
        if ((mask_8x8 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
698
          aom_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
699
700
701
702
                                    lfi->hev_thr, lfin->mblim, lfin->lim,
                                    lfin->hev_thr);

          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
703
            aom_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
Jingning Han's avatar
Jingning Han committed
704
705
706
707
                                      lfi->lim, lfi->hev_thr, lfin->mblim,
                                      lfin->lim, lfin->hev_thr);
          } else {
            if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
708
              aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
709
                                   lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
710
            else if (mask_4x4_int & 2)
Yaowu Xu's avatar
Yaowu Xu committed
711
              aom_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
712
                                   lfin->lim, lfin->hev_thr);
Jingning Han's avatar
Jingning Han committed
713
714
715
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
716
          aom_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
717
718

          if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
719
            aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
720
                                 lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
721
722
723
724
725
726
        }
      } else if (mask_4x4 & 1) {
        if ((mask_4x4 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
727
          aom_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
728
729
                                    lfi->hev_thr, lfin->mblim, lfin->lim,
                                    lfin->hev_thr);
Cheng Chen's avatar
Cheng Chen committed
730

Jingning Han's avatar
Jingning Han committed
731
          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
732
            aom_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
Jingning Han's avatar
Jingning Han committed
733
734
735
736
                                      lfi->lim, lfi->hev_thr, lfin->mblim,
                                      lfin->lim, lfin->hev_thr);
          } else {
            if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
737
              aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
738
                                   lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
739
            else if (mask_4x4_int & 2)
Yaowu Xu's avatar
Yaowu Xu committed
740
              aom_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
741
                                   lfin->lim, lfin->hev_thr);
Jingning Han's avatar
Jingning Han committed
742
743
744
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
745
          aom_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
746
747

          if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
748
            aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
749
                                 lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
750
751
        }
      } else if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
752
        aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
753
                             lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
754
755
756
757
758
759
760
761
762
763
764
      }
    }
    s += 8 * count;
    lfl += count;
    mask_16x16 >>= count;
    mask_8x8 >>= count;
    mask_4x4 >>= count;
    mask_4x4_int >>= count;
  }
}

765
#if CONFIG_HIGHBITDEPTH
clang-format's avatar
clang-format committed
766
767
768
769
static void highbd_filter_selectively_horiz(
    uint16_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8,
    unsigned int mask_4x4, unsigned int mask_4x4_int,
    const loop_filter_info_n *lfi_n, const uint8_t *lfl, int bd) {
Jingning Han's avatar
Jingning Han committed
770
771
772
  unsigned int mask;
  int count;

clang-format's avatar
clang-format committed
773
774
  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask;
       mask >>= count) {
Jingning Han's avatar
Jingning Han committed
775
776
777
778
779
780
    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;

    count = 1;
    if (mask & 1) {
      if (mask_16x16 & 1) {
        if ((mask_16x16 & 3) == 3) {
781
          aom_highbd_lpf_horizontal_16_dual(s, pitch, lfi->mblim, lfi->lim,
782
                                            lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
783
784
          count = 2;
        } else {
785
786
          aom_highbd_lpf_horizontal_16(s, pitch, lfi->mblim, lfi->lim,
                                       lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
787
788
789
790
791
792
        }
      } else if (mask_8x8 & 1) {
        if ((mask_8x8 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
793
          aom_highbd_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
794
795
796
797
                                           lfi->hev_thr, lfin->mblim, lfin->lim,
                                           lfin->hev_thr, bd);

          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
798
            aom_highbd_lpf_horizontal_4_dual(
clang-format's avatar
clang-format committed
799
800
                s + 4 * pitch, pitch, lfi->mblim, lfi->lim, lfi->hev_thr,
                lfin->mblim, lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
801
802
          } else {
            if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
803
              aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
804
                                          lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
805
            } else if (mask_4x4_int & 2) {
Yaowu Xu's avatar
Yaowu Xu committed
806
              aom_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
807
                                          lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
808
809
810
811
            }
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
812
          aom_highbd_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim,
813
                                      lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
814
815

          if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
816
            aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
817
                                        lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
818
819
820
821
822
823
824
          }
        }
      } else if (mask_4x4 & 1) {
        if ((mask_4x4 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
825
          aom_highbd_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
826
827
828
                                           lfi->hev_thr, lfin->mblim, lfin->lim,
                                           lfin->hev_thr, bd);
          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
829
            aom_highbd_lpf_horizontal_4_dual(
clang-format's avatar
clang-format committed
830
831
                s + 4 * pitch, pitch, lfi->mblim, lfi->lim, lfi->hev_thr,
                lfin->mblim, lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
832
833
          } else {
            if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
834
              aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
835
                                          lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
836
            } else if (mask_4x4_int & 2) {
Yaowu Xu's avatar
Yaowu Xu committed
837
              aom_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
838
                                          lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
839
840
841
842
            }
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
843
          aom_highbd_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim,
844
                                      lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
845
846

          if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
847
            aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
848
                                        lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
849
850
851
          }
        }
      } else if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
852
        aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
853
                                    lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
854
855
856
857
858
859
860
861
862
863
      }
    }
    s += 8 * count;
    lfl += count;
    mask_16x16 >>= count;
    mask_8x8 >>= count;
    mask_4x4 >>= count;
    mask_4x4_int >>= count;
  }
}
864
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
865
866
867
868
869
870
871
872

// This function ors into the current lfm structure, where to do loop
// filters for the specific mi we are looking at. It uses information
// including the block_size_type (32x16, 32x32, etc.), the transform size,
// whether there were any coefficients encoded, and the loop filter strength
// block we are currently looking at. Shift is used to position the
// 1's we produce.
// TODO(JBB) Need another function for different resolution color..
873
874
875
876
static void build_masks(AV1_COMMON *const cm,
                        const loop_filter_info_n *const lfi_n,
                        const MODE_INFO *mi, const int shift_y,
                        const int shift_uv, LOOP_FILTER_MASK *lfm) {
Jingning Han's avatar
Jingning Han committed
877
878
  const MB_MODE_INFO *mbmi = &mi->mbmi;
  const BLOCK_SIZE block_size = mbmi->sb_type;
879
880
  // TODO(debargha): Check if masks can be setup correctly when
  // rectangular transfroms are used with the EXT_TX expt.
881
882
883
  const TX_SIZE tx_size_y = txsize_sqr_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_left = txsize_horz_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_above = txsize_vert_map[mbmi->tx_size];
884
  const TX_SIZE tx_size_uv =
885
886
887
888
889
      txsize_sqr_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
  const TX_SIZE tx_size_uv_left =
      txsize_horz_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
  const TX_SIZE tx_size_uv_above =
      txsize_vert_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
Fangwen Fu's avatar
Fangwen Fu committed
890
#if CONFIG_EXT_DELTA_Q
Cheng Chen's avatar
Cheng Chen committed
891
#if CONFIG_LOOPFILTER_LEVEL
Cheng Chen's avatar
Cheng Chen committed
892
  const int filter_level = get_filter_level(cm, lfi_n, 0, 0, mbmi);
893
#else
Fangwen Fu's avatar
Fangwen Fu committed
894
  const int filter_level = get_filter_level(cm, lfi_n, mbmi);
895
#endif
Fangwen Fu's avatar
Fangwen Fu committed
896
#else
Jingning Han's avatar
Jingning Han committed
897
  const int filter_level = get_filter_level(lfi_n, mbmi);
898
  (void)cm;
Fangwen Fu's avatar
Fangwen Fu committed
899
#endif
900
901
  uint64_t *const left_y = &lfm->left_y[tx_size_y_left];
  uint64_t *const above_y = &lfm->above_y[tx_size_y_above];
Jingning Han's avatar
Jingning Han committed
902
  uint64_t *const int_4x4_y = &lfm->int_4x4_y;
903
904
  uint16_t *const left_uv = &lfm->left_uv[tx_size_uv_left];
  uint16_t *const above_uv = &lfm->above_uv[tx_size_uv_above];
905
  uint16_t *const int_4x4_uv = &lfm->left_int_4x4_uv;
Jingning Han's avatar
Jingning Han committed
906
907
908
909
910
911
912
913
  int i;

  // If filter level is 0 we don't loop filter.
  if (!filter_level) {
    return;
  } else {
    const int w = num_8x8_blocks_wide_lookup[block_size];
    const int h = num_8x8_blocks_high_lookup[block_size];
914
915
916
    const int row = (shift_y >> MAX_MIB_SIZE_LOG2);
    const int col = shift_y - (row << MAX_MIB_SIZE_LOG2);

clang-format's avatar
clang-format committed
917
    for (i = 0; i < h; i++) memset(&lfm->lfl_y[row + i][col], filter_level, w);
Jingning Han's avatar
Jingning Han committed
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
  }

  // These set 1 in the current block size for the block size edges.
  // For instance if the block size is 32x16, we'll set:
  //    above =   1111
  //              0000
  //    and
  //    left  =   1000
  //          =   1000
  // NOTE : In this example the low bit is left most ( 1000 ) is stored as
  //        1,  not 8...
  //
  // U and V set things on a 16 bit scale.
  //
  *above_y |= above_prediction_mask[block_size] << shift_y;
  *above_uv |= above_prediction_mask_uv[block_size] << shift_uv;
  *left_y |= left_prediction_mask[block_size] << shift_y;
  *left_uv |= left_prediction_mask_uv[block_size] << shift_uv;

  // If the block has no coefficients and is not intra we skip applying
  // the loop filter on block edges.
939
  if (mbmi->skip && is_inter_block(mbmi)) return;
Jingning Han's avatar
Jingning Han committed
940
941
942
943
944

  // Here we are adding a mask for the transform size. The transform
  // size mask is set to be correct for a 64x64 prediction block size. We
  // mask to match the size of the block we are working on and then shift it
  // into place..
945
  *above_y |= (size_mask[block_size] & above_64x64_txform_mask[tx_size_y_above])
clang-format's avatar
clang-format committed
946
947
              << shift_y;
  *above_uv |=
948
      (size_mask_uv[block_size] & above_64x64_txform_mask_uv[tx_size_uv_above])
clang-format's avatar
clang-format committed
949
      << shift_uv;
Jingning Han's avatar
Jingning Han committed
950

951
  *left_y |= (size_mask[block_size] & left_64x64_txform_mask[tx_size_y_left])
clang-format's avatar
clang-format committed
952
             << shift_y;
953
954
955
  *left_uv |=
      (size_mask_uv[block_size] & left_64x64_txform_mask_uv[tx_size_uv_left])
      << shift_uv;
Jingning Han's avatar
Jingning Han committed
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970

  // Here we are trying to determine what to do with the internal 4x4 block
  // boundaries.  These differ from the 4x4 boundaries on the outside edge of
  // an 8x8 in that the internal ones can be skipped and don't depend on
  // the prediction block size.
  if (tx_size_y == TX_4X4)
    *int_4x4_y |= (size_mask[block_size] & 0xffffffffffffffffULL) << shift_y;

  if (tx_size_uv == TX_4X4)
    *int_4x4_uv |= (size_mask_uv[block_size] & 0xffff) << shift_uv;
}

// This function does the same thing as the one above with the exception that
// it only affects the y masks. It exists because for blocks < 16x16 in size,
// we only update u and v masks on the first block.
971
972
973
974
static void build_y_mask(AV1_COMMON *const cm,
                         const loop_filter_info_n *const lfi_n,
                         const MODE_INFO *mi, const int shift_y,
                         LOOP_FILTER_MASK *lfm) {
Jingning Han's avatar
Jingning Han committed
975
  const MB_MODE_INFO *mbmi = &mi->mbmi;
976
977
978
  const TX_SIZE tx_size_y = txsize_sqr_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_left = txsize_horz_map[mbmi->tx_size];
  const TX_SIZE tx_size_y_above = txsize_vert_map[mbmi->tx_size];
979
  const BLOCK_SIZE block_size = mbmi->sb_type;
Fangwen Fu's avatar
Fangwen Fu committed
980
#if CONFIG_EXT_DELTA_Q
Cheng Chen's avatar
Cheng Chen committed
981
#if CONFIG_LOOPFILTER_LEVEL
Cheng Chen's avatar
Cheng Chen committed
982
  const int filter_level = get_filter_level(cm, lfi_n, 0, 0, mbmi);
983
#else
Fangwen Fu's avatar
Fangwen Fu committed
984
  const int filter_level = get_filter_level(cm, lfi_n, mbmi);
985
#endif
Fangwen Fu's avatar
Fangwen Fu committed
986
#else
Jingning Han's avatar
Jingning Han committed
987
  const int filter_level = get_filter_level(lfi_n, mbmi);
988
  (void)cm;
Fangwen Fu's avatar
Fangwen Fu committed
989
#endif
990
991
  uint64_t *const left_y = &lfm->left_y[tx_size_y_left];
  uint64_t *const above_y = &lfm->above_y[tx_size_y_above];
Jingning Han's avatar
Jingning Han committed
992
993
994
995
996
997
998
999
  uint64_t *const int_4x4_y = &lfm->int_4x4_y;
  int i;

  if (!filter_level) {
    return;
  } else {
    const int w = num_8x8_blocks_wide_lookup[block_size];
    const int h = num_8x8_blocks_high_lookup[block_size];
1000
1001
1002
    const int row = (shift_y >> MAX_MIB_SIZE_LOG2);
    const int col = shift_y - (row << MAX_MIB_SIZE_LOG2);

clang-format's avatar
clang-format committed
1003
    for (i = 0; i < h; i++) memset(&lfm->lfl_y[row + i][col], filter_level, w);
Jingning Han's avatar
Jingning Han committed
1004
1005
1006
1007
1008
  }

  *above_y |= above_prediction_mask[block_size] << shift_y;
  *left_y |= left_prediction_mask[block_size] << shift_y;

1009
  if (mbmi->skip && is_inter_block(mbmi)) return;
Jingning Han's avatar
Jingning Han committed
1010

1011
  *above_y |= (size_mask[block_size] & above_64x64_txform_mask[tx_size_y_above])
clang-format's avatar
clang-format committed
1012
              << shift_y;
Jingning Han's avatar
Jingning Han committed
1013

1014
  *left_y |= (size_mask[block_size] & left_64x64_txform_mask[tx_size_y_left])
clang-format's avatar
clang-format committed
1015
             << shift_y;
Jingning Han's avatar
Jingning Han committed
1016
1017
1018
1019
1020

  if (tx_size_y == TX_4X4)
    *int_4x4_y |= (size_mask[block_size] & 0xffffffffffffffffULL) << shift_y;
}

1021
#if CONFIG_LOOPFILTERING_ACROSS_TILES || CONFIG_LOOPFILTERING_ACROSS_TILES_EXT
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
// This function update the bit masks for the entire 64x64 region represented
// by mi_row, mi_col. In case one of the edge is a tile boundary, loop filtering
// for that edge is disabled. This function only check the tile boundary info
// for the top left corner mi to determine the boundary information for the
// top and left edge of the whole super block
static void update_tile_boundary_filter_mask(AV1_COMMON *const cm,
                                             const int mi_row, const int mi_col,
                                             LOOP_FILTER_MASK *lfm) {
  int i;
  MODE_INFO *const mi = cm->mi + mi_row * cm->mi_stride + mi_col;

1033
  if (mi->mbmi.boundary_info & TILE_LEFT_BOUNDARY) {
1034
1035
1036
1037
1038
1039
    for (i = 0; i <= TX_32X32; i++) {
      lfm->left_y[i] &= 0xfefefefefefefefeULL;
      lfm->left_uv[i] &= 0xeeee;
    }
  }

1040
  if (mi->mbmi.boundary_info & TILE_ABOVE_BOUNDARY) {
1041
1042
1043
1044
1045
1046
    for (i = 0; i <= TX_32X32; i++) {
      lfm->above_y[i] &= 0xffffffffffffff00ULL;
      lfm->above_uv[i] &= 0xfff0;
    }
  }
}
1047
#endif  // CONFIG_LOOPFILTERING_ACROSS_TILES
1048

Jingning Han's avatar
Jingning Han committed
1049
1050
1051
// This function sets up the bit masks for the entire 64x64 region represented
// by mi_row, mi_col.
// TODO(JBB): This function only works for yv12.
Yaowu Xu's avatar
Yaowu Xu committed
1052
1053
1054
void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
                    MODE_INFO **mi, const int mode_info_stride,
                    LOOP_FILTER_MASK *lfm) {
1055
1056
1057
#if CONFIG_EXT_PARTITION
  assert(0 && "Not yet updated");
#endif  // CONFIG_EXT_PARTITION
Jingning Han's avatar
Jingning Han committed
1058
1059
1060
1061
1062
1063
1064
1065
1066
  int idx_32, idx_16, idx_8;
  const loop_filter_info_n *const lfi_n = &cm->lf_info;
  MODE_INFO **mip = mi;
  MODE_INFO **mip2 = mi;

  // These are offsets to the next mi in the 64x64 block. It is what gets
  // added to the mi ptr as we go through each loop. It helps us to avoid
  // setting up special row and column counters for each index. The last step
  // brings us out back to the starting position.
clang-format's avatar
clang-format committed
1067
1068
1069
1070
1071
  const int offset_32[] = { 4, (mode_info_stride << 2) - 4, 4,
                            -(mode_info_stride << 2) - 4 };
  const int offset_16[] = { 2, (mode_info_stride << 1) - 2, 2,
                            -(mode_info_stride << 1) - 2 };
  const int offset[] = { 1, mode_info_stride - 1, 1, -mode_info_stride - 1 };
Jingning Han's avatar
Jingning Han committed
1072
1073
1074
1075
1076

  // Following variables represent shifts to position the current block
  // mask over the appropriate block. A shift of 36 to the left will move
  // the bits for the final 32 by 32 block in the 64x64 up 4 rows and left
  // 4 rows to the appropriate spot.
clang-format's avatar
clang-format committed
1077
1078
1079
1080
1081
  const int shift_32_y[] = { 0, 4, 32, 36 };
  const int shift_16_y[] = { 0, 2, 16, 18 };
  const int shift_8_y[] = { 0, 1, 8, 9 };
  const int shift_32_uv[] = { 0, 2, 8, 10 };
  const int shift_16_uv[] = { 0, 1, 4, 5 };
Jingning Han's avatar
Jingning Han committed
1082
  int i;
Yaowu Xu's avatar
Yaowu Xu committed
1083
1084
  const int max_rows = AOMMIN(cm->mi_rows - mi_row, MAX_MIB_SIZE);
  const int max_cols = AOMMIN(cm->mi_cols - mi_col, MAX_MIB_SIZE);
Jingning Han's avatar
Jingning Han committed
1085

Yaowu Xu's avatar
Yaowu Xu committed
1086
  av1_zero(*lfm);
Jingning Han's avatar
Jingning Han committed
1087
1088
1089
1090
1091
1092
  assert(mip[0] != NULL);

  // TODO(jimbankoski): Try moving most of the following code into decode
  // loop and storing lfm in the mbmi structure so that we don't have to go
  // through the recursive loop structure multiple times.
  switch (mip[0]->mbmi.sb_type) {
Fangwen Fu's avatar
Fangwen Fu committed
1093
    case BLOCK_64X64: build_masks(cm, lfi_n, mip[0], 0, 0, lfm); break;
1094
1095
    case BLOCK_64X32:
      build_masks(cm, lfi_n, mip[0], 0, 0, lfm);
Jingning Han's avatar
Jingning Han committed
1096
      mip2 = mip + mode_info_stride * 4;
clang-format's avatar
clang-format committed
1097
      if (4 >= max_rows) break;
Fangwen Fu's avatar
Fangwen Fu committed
1098
      build_masks(cm, lfi_n, mip2[0], 32, 8, lfm);
Jingning Han's avatar
Jingning Han committed
1099
      break;
1100
1101
    case BLOCK_32X64:
      build_masks(cm, lfi_n, mip[0], 0, 0, lfm);
Jingning Han's avatar
Jingning Han committed
1102
      mip2 = mip + 4;
clang-format's avatar
clang-format committed
1103
      if (4 >= max_cols) break;
Fangwen Fu's avatar
Fangwen Fu committed
1104
      build_masks(cm, lfi_n, mip2[0], 4, 2, lfm);
Jingning Han's avatar
Jingning Han committed
1105
1106
      break;
    default:
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
      for (idx_32 = 0; idx_32 < 4; mip += offset_32[idx_32], ++idx_32) {
        const int shift_y_32 = shift_32_y[idx_32];
        const int shift_uv_32 = shift_32_uv[idx_32];
        const int mi_32_col_offset = ((idx_32 & 1) << 2);
        const int mi_32_row_offset = ((idx_32 >> 1) << 2);
        if (mi_32_col_offset >= max_cols || mi_32_row_offset >= max_rows)