loopfilter.c 74.7 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
 */

Debargha Mukherjee's avatar
Debargha Mukherjee committed
12
13
#include <math.h>

Yaowu Xu's avatar
Yaowu Xu committed
14
15
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
16
17
18
#include "av1/common/loopfilter.h"
#include "av1/common/onyxc_int.h"
#include "av1/common/reconinter.h"
Yaowu Xu's avatar
Yaowu Xu committed
19
20
#include "aom_dsp/aom_dsp_common.h"
#include "aom_mem/aom_mem.h"
21
#include "aom_ports/mem.h"
Jingning Han's avatar
Jingning Han committed
22

23
#include "av1/common/seg_common.h"
Jingning Han's avatar
Jingning Han committed
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

// 64 bit masks for left transform size. Each 1 represents a position where
// we should apply a loop filter across the left border of an 8x8 block
// boundary.
//
// In the case of TX_16X16->  ( in low order byte first we end up with
// a mask that looks like this
//
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//    10101010
//
// A loopfilter should be applied to every other 8x8 horizontally.
clang-format's avatar
clang-format committed
42
static const uint64_t left_64x64_txform_mask[TX_SIZES] = {
43
44
45
#if CONFIG_CB4X4
  0xffffffffffffffffULL,  // TX_2X2
#endif
Jingning Han's avatar
Jingning Han committed
46
47
48
49
  0xffffffffffffffffULL,  // TX_4X4
  0xffffffffffffffffULL,  // TX_8x8
  0x5555555555555555ULL,  // TX_16x16
  0x1111111111111111ULL,  // TX_32x32
50
51
52
#if CONFIG_TX64X64
  0x0101010101010101ULL,  // TX_64x64
#endif                    // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
};

// 64 bit masks for above transform size. Each 1 represents a position where
// we should apply a loop filter across the top border of an 8x8 block
// boundary.
//
// In the case of TX_32x32 ->  ( in low order byte first we end up with
// a mask that looks like this
//
//    11111111
//    00000000
//    00000000
//    00000000
//    11111111
//    00000000
//    00000000
//    00000000
//
// A loopfilter should be applied to every other 4 the row vertically.
clang-format's avatar
clang-format committed
72
static const uint64_t above_64x64_txform_mask[TX_SIZES] = {
73
74
75
#if CONFIG_CB4X4
  0xffffffffffffffffULL,  // TX_4X4
#endif
Jingning Han's avatar
Jingning Han committed
76
77
78
79
  0xffffffffffffffffULL,  // TX_4X4
  0xffffffffffffffffULL,  // TX_8x8
  0x00ff00ff00ff00ffULL,  // TX_16x16
  0x000000ff000000ffULL,  // TX_32x32
80
81
82
#if CONFIG_TX64X64
  0x00000000000000ffULL,  // TX_64x64
#endif                    // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
};

// 64 bit masks for prediction sizes (left). Each 1 represents a position
// where left border of an 8x8 block. These are aligned to the right most
// appropriate bit, and then shifted into place.
//
// In the case of TX_16x32 ->  ( low order byte first ) we end up with
// a mask that looks like this :
//
//  10000000
//  10000000
//  10000000
//  10000000
//  00000000
//  00000000
//  00000000
//  00000000
static const uint64_t left_prediction_mask[BLOCK_SIZES] = {
Jingning Han's avatar
Jingning Han committed
101
102
103
104
105
#if CONFIG_CB4X4
  0x0000000000000001ULL,  // BLOCK_2X2,
  0x0000000000000001ULL,  // BLOCK_2X4,
  0x0000000000000001ULL,  // BLOCK_4X2,
#endif
Jingning Han's avatar
Jingning Han committed
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
  0x0000000000000001ULL,  // BLOCK_4X4,
  0x0000000000000001ULL,  // BLOCK_4X8,
  0x0000000000000001ULL,  // BLOCK_8X4,
  0x0000000000000001ULL,  // BLOCK_8X8,
  0x0000000000000101ULL,  // BLOCK_8X16,
  0x0000000000000001ULL,  // BLOCK_16X8,
  0x0000000000000101ULL,  // BLOCK_16X16,
  0x0000000001010101ULL,  // BLOCK_16X32,
  0x0000000000000101ULL,  // BLOCK_32X16,
  0x0000000001010101ULL,  // BLOCK_32X32,
  0x0101010101010101ULL,  // BLOCK_32X64,
  0x0000000001010101ULL,  // BLOCK_64X32,
  0x0101010101010101ULL,  // BLOCK_64X64
};

// 64 bit mask to shift and set for each prediction size.
static const uint64_t above_prediction_mask[BLOCK_SIZES] = {
Jingning Han's avatar
Jingning Han committed
123
124
125
126
127
#if CONFIG_CB4X4
  0x0000000000000001ULL,  // BLOCK_2X2
  0x0000000000000001ULL,  // BLOCK_2X4
  0x0000000000000001ULL,  // BLOCK_4X2
#endif
Jingning Han's avatar
Jingning Han committed
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
  0x0000000000000001ULL,  // BLOCK_4X4
  0x0000000000000001ULL,  // BLOCK_4X8
  0x0000000000000001ULL,  // BLOCK_8X4
  0x0000000000000001ULL,  // BLOCK_8X8
  0x0000000000000001ULL,  // BLOCK_8X16,
  0x0000000000000003ULL,  // BLOCK_16X8
  0x0000000000000003ULL,  // BLOCK_16X16
  0x0000000000000003ULL,  // BLOCK_16X32,
  0x000000000000000fULL,  // BLOCK_32X16,
  0x000000000000000fULL,  // BLOCK_32X32,
  0x000000000000000fULL,  // BLOCK_32X64,
  0x00000000000000ffULL,  // BLOCK_64X32,
  0x00000000000000ffULL,  // BLOCK_64X64
};
// 64 bit mask to shift and set for each prediction size. A bit is set for
// each 8x8 block that would be in the left most block of the given block
// size in the 64x64 block.
static const uint64_t size_mask[BLOCK_SIZES] = {
Jingning Han's avatar
Jingning Han committed
146
147
148
149
150
#if CONFIG_CB4X4
  0x0000000000000001ULL,  // BLOCK_2X2
  0x0000000000000001ULL,  // BLOCK_2X4
  0x0000000000000001ULL,  // BLOCK_4X2
#endif
Jingning Han's avatar
Jingning Han committed
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
  0x0000000000000001ULL,  // BLOCK_4X4
  0x0000000000000001ULL,  // BLOCK_4X8
  0x0000000000000001ULL,  // BLOCK_8X4
  0x0000000000000001ULL,  // BLOCK_8X8
  0x0000000000000101ULL,  // BLOCK_8X16,
  0x0000000000000003ULL,  // BLOCK_16X8
  0x0000000000000303ULL,  // BLOCK_16X16
  0x0000000003030303ULL,  // BLOCK_16X32,
  0x0000000000000f0fULL,  // BLOCK_32X16,
  0x000000000f0f0f0fULL,  // BLOCK_32X32,
  0x0f0f0f0f0f0f0f0fULL,  // BLOCK_32X64,
  0x00000000ffffffffULL,  // BLOCK_64X32,
  0xffffffffffffffffULL,  // BLOCK_64X64
};

// These are used for masking the left and above borders.
clang-format's avatar
clang-format committed
167
static const uint64_t left_border = 0x1111111111111111ULL;
Jingning Han's avatar
Jingning Han committed
168
169
170
static const uint64_t above_border = 0x000000ff000000ffULL;

// 16 bit masks for uv transform sizes.
clang-format's avatar
clang-format committed
171
static const uint16_t left_64x64_txform_mask_uv[TX_SIZES] = {
172
173
174
#if CONFIG_CB4X4
  0xffff,  // TX_2X2
#endif
Jingning Han's avatar
Jingning Han committed
175
176
177
178
  0xffff,  // TX_4X4
  0xffff,  // TX_8x8
  0x5555,  // TX_16x16
  0x1111,  // TX_32x32
179
180
181
#if CONFIG_TX64X64
  0x0101,  // TX_64x64, never used
#endif     // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
182
183
};

clang-format's avatar
clang-format committed
184
static const uint16_t above_64x64_txform_mask_uv[TX_SIZES] = {
185
186
187
#if CONFIG_CB4X4
  0xffff,  // TX_2X2
#endif
Jingning Han's avatar
Jingning Han committed
188
189
190
191
  0xffff,  // TX_4X4
  0xffff,  // TX_8x8
  0x0f0f,  // TX_16x16
  0x000f,  // TX_32x32
192
193
194
#if CONFIG_TX64X64
  0x0003,  // TX_64x64, never used
#endif     // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
195
196
197
198
};

// 16 bit left mask to shift and set for each uv prediction size.
static const uint16_t left_prediction_mask_uv[BLOCK_SIZES] = {
Jingning Han's avatar
Jingning Han committed
199
200
201
202
203
#if CONFIG_CB4X4
  0x0001,  // BLOCK_2X2,
  0x0001,  // BLOCK_2X4,
  0x0001,  // BLOCK_4X2,
#endif
Jingning Han's avatar
Jingning Han committed
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
  0x0001,  // BLOCK_4X4,
  0x0001,  // BLOCK_4X8,
  0x0001,  // BLOCK_8X4,
  0x0001,  // BLOCK_8X8,
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8,
  0x0001,  // BLOCK_16X16,
  0x0011,  // BLOCK_16X32,
  0x0001,  // BLOCK_32X16,
  0x0011,  // BLOCK_32X32,
  0x1111,  // BLOCK_32X64
  0x0011,  // BLOCK_64X32,
  0x1111,  // BLOCK_64X64
};
// 16 bit above mask to shift and set for uv each prediction size.
static const uint16_t above_prediction_mask_uv[BLOCK_SIZES] = {
Jingning Han's avatar
Jingning Han committed
220
221
222
223
224
#if CONFIG_CB4X4
  0x0001,  // BLOCK_2X2
  0x0001,  // BLOCK_2X4
  0x0001,  // BLOCK_4X2
#endif
Jingning Han's avatar
Jingning Han committed
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
  0x0001,  // BLOCK_4X4
  0x0001,  // BLOCK_4X8
  0x0001,  // BLOCK_8X4
  0x0001,  // BLOCK_8X8
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8
  0x0001,  // BLOCK_16X16
  0x0001,  // BLOCK_16X32,
  0x0003,  // BLOCK_32X16,
  0x0003,  // BLOCK_32X32,
  0x0003,  // BLOCK_32X64,
  0x000f,  // BLOCK_64X32,
  0x000f,  // BLOCK_64X64
};

// 64 bit mask to shift and set for each uv prediction size
static const uint16_t size_mask_uv[BLOCK_SIZES] = {
Jingning Han's avatar
Jingning Han committed
242
243
244
245
246
#if CONFIG_CB4X4
  0x0001,  // BLOCK_2X2
  0x0001,  // BLOCK_2X4
  0x0001,  // BLOCK_4X2
#endif
Jingning Han's avatar
Jingning Han committed
247
248
249
250
251
252
253
254
255
256
257
258
259
260
  0x0001,  // BLOCK_4X4
  0x0001,  // BLOCK_4X8
  0x0001,  // BLOCK_8X4
  0x0001,  // BLOCK_8X8
  0x0001,  // BLOCK_8X16,
  0x0001,  // BLOCK_16X8
  0x0001,  // BLOCK_16X16
  0x0011,  // BLOCK_16X32,
  0x0003,  // BLOCK_32X16,
  0x0033,  // BLOCK_32X32,
  0x3333,  // BLOCK_32X64,
  0x00ff,  // BLOCK_64X32,
  0xffff,  // BLOCK_64X64
};
clang-format's avatar
clang-format committed
261
static const uint16_t left_border_uv = 0x1111;
Jingning Han's avatar
Jingning Han committed
262
263
264
265
266
static const uint16_t above_border_uv = 0x000f;

static const int mode_lf_lut[MB_MODE_COUNT] = {
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // INTRA_MODES
  1, 1, 0, 1                     // INTER_MODES (ZEROMV == 0)
Yue Chen's avatar
Yue Chen committed
267
#if CONFIG_EXT_INTER
clang-format's avatar
clang-format committed
268
269
270
271
  ,
  1,                            // NEWFROMNEARMV mode
  1, 1, 1, 1, 1, 1, 1, 1, 0, 1  // INTER_COMPOUND_MODES (ZERO_ZEROMV == 0)
#endif                          // CONFIG_EXT_INTER
Jingning Han's avatar
Jingning Han committed
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
};

static void update_sharpness(loop_filter_info_n *lfi, int sharpness_lvl) {
  int lvl;

  // For each possible value for the loop filter fill out limits
  for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++) {
    // Set loop filter parameters that control sharpness.
    int block_inside_limit = lvl >> ((sharpness_lvl > 0) + (sharpness_lvl > 4));

    if (sharpness_lvl > 0) {
      if (block_inside_limit > (9 - sharpness_lvl))
        block_inside_limit = (9 - sharpness_lvl);
    }

clang-format's avatar
clang-format committed
287
    if (block_inside_limit < 1) block_inside_limit = 1;
Jingning Han's avatar
Jingning Han committed
288
289
290
291
292
293
294
295
296

    memset(lfi->lfthr[lvl].lim, block_inside_limit, SIMD_WIDTH);
    memset(lfi->lfthr[lvl].mblim, (2 * (lvl + 2) + block_inside_limit),
           SIMD_WIDTH);
  }
}

static uint8_t get_filter_level(const loop_filter_info_n *lfi_n,
                                const MB_MODE_INFO *mbmi) {
297
#if CONFIG_SUPERTX
Yaowu Xu's avatar
Yaowu Xu committed
298
  const int segment_id = AOMMIN(mbmi->segment_id, mbmi->segment_id_supertx);
clang-format's avatar
clang-format committed
299
300
  assert(
      IMPLIES(supertx_enabled(mbmi), mbmi->segment_id_supertx != MAX_SEGMENTS));
301
302
303
304
305
306
  assert(IMPLIES(supertx_enabled(mbmi),
                 mbmi->segment_id_supertx <= mbmi->segment_id));
#else
  const int segment_id = mbmi->segment_id;
#endif  // CONFIG_SUPERTX
  return lfi_n->lvl[segment_id][mbmi->ref_frame[0]][mode_lf_lut[mbmi->mode]];
Jingning Han's avatar
Jingning Han committed
307
308
}

Yaowu Xu's avatar
Yaowu Xu committed
309
void av1_loop_filter_init(AV1_COMMON *cm) {
Jingning Han's avatar
Jingning Han committed
310
311
312
313
314
315
316
317
318
319
320
321
322
  loop_filter_info_n *lfi = &cm->lf_info;
  struct loopfilter *lf = &cm->lf;
  int lvl;

  // init limits for given sharpness
  update_sharpness(lfi, lf->sharpness_level);
  lf->last_sharpness_level = lf->sharpness_level;

  // init hev threshold const vectors
  for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++)
    memset(lfi->lfthr[lvl].hev_thr, (lvl >> 4), SIMD_WIDTH);
}

Yaowu Xu's avatar
Yaowu Xu committed
323
void av1_loop_filter_frame_init(AV1_COMMON *cm, int default_filt_lvl) {
Jingning Han's avatar
Jingning Han committed
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
  int seg_id;
  // n_shift is the multiplier for lf_deltas
  // the multiplier is 1 for when filter_lvl is between 0 and 31;
  // 2 when filter_lvl is between 32 and 63
  const int scale = 1 << (default_filt_lvl >> 5);
  loop_filter_info_n *const lfi = &cm->lf_info;
  struct loopfilter *const lf = &cm->lf;
  const struct segmentation *const seg = &cm->seg;

  // update limits if sharpness has changed
  if (lf->last_sharpness_level != lf->sharpness_level) {
    update_sharpness(lfi, lf->sharpness_level);
    lf->last_sharpness_level = lf->sharpness_level;
  }

  for (seg_id = 0; seg_id < MAX_SEGMENTS; seg_id++) {
    int lvl_seg = default_filt_lvl;
    if (segfeature_active(seg, seg_id, SEG_LVL_ALT_LF)) {
      const int data = get_segdata(seg, seg_id, SEG_LVL_ALT_LF);
clang-format's avatar
clang-format committed
343
344
345
      lvl_seg = clamp(
          seg->abs_delta == SEGMENT_ABSDATA ? data : default_filt_lvl + data, 0,
          MAX_LOOP_FILTER);
Jingning Han's avatar
Jingning Han committed
346
347
348
349
350
351
352
353
354
355
356
    }

    if (!lf->mode_ref_delta_enabled) {
      // we could get rid of this if we assume that deltas are set to
      // zero when not in use; encoder always uses deltas
      memset(lfi->lvl[seg_id], lvl_seg, sizeof(lfi->lvl[seg_id]));
    } else {
      int ref, mode;
      const int intra_lvl = lvl_seg + lf->ref_deltas[INTRA_FRAME] * scale;
      lfi->lvl[seg_id][INTRA_FRAME][0] = clamp(intra_lvl, 0, MAX_LOOP_FILTER);

357
      for (ref = LAST_FRAME; ref < TOTAL_REFS_PER_FRAME; ++ref) {
Jingning Han's avatar
Jingning Han committed
358
        for (mode = 0; mode < MAX_MODE_LF_DELTAS; ++mode) {
clang-format's avatar
clang-format committed
359
360
          const int inter_lvl = lvl_seg + lf->ref_deltas[ref] * scale +
                                lf->mode_deltas[mode] * scale;
Jingning Han's avatar
Jingning Han committed
361
362
363
364
365
366
367
          lfi->lvl[seg_id][ref][mode] = clamp(inter_lvl, 0, MAX_LOOP_FILTER);
        }
      }
    }
  }
}

clang-format's avatar
clang-format committed
368
369
static void filter_selectively_vert_row2(int subsampling_factor, uint8_t *s,
                                         int pitch, unsigned int mask_16x16_l,
Jingning Han's avatar
Jingning Han committed
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
                                         unsigned int mask_8x8_l,
                                         unsigned int mask_4x4_l,
                                         unsigned int mask_4x4_int_l,
                                         const loop_filter_info_n *lfi_n,
                                         const uint8_t *lfl) {
  const int mask_shift = subsampling_factor ? 4 : 8;
  const int mask_cutoff = subsampling_factor ? 0xf : 0xff;
  const int lfl_forward = subsampling_factor ? 4 : 8;

  unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff;
  unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff;
  unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff;
  unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff;
  unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff;
  unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff;
  unsigned int mask;

  for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 |
              mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1;
       mask; mask >>= 1) {
    const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl;
    const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward);

    if (mask & 1) {
      if ((mask_16x16_0 | mask_16x16_1) & 1) {
        if ((mask_16x16_0 & mask_16x16_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
398
          aom_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
399
400
                                   lfi0->hev_thr);
        } else if (mask_16x16_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
401
          aom_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
402
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
403
          aom_lpf_vertical_16(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
clang-format's avatar
clang-format committed
404
                              lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
405
406
407
408
409
        }
      }

      if ((mask_8x8_0 | mask_8x8_1) & 1) {
        if ((mask_8x8_0 & mask_8x8_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
410
          aom_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
411
412
413
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_8x8_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
414
          aom_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
415
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
416
          aom_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
417
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
418
419
420
421
422
        }
      }

      if ((mask_4x4_0 | mask_4x4_1) & 1) {
        if ((mask_4x4_0 & mask_4x4_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
423
          aom_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
424
425
426
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_4x4_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
427
          aom_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
428
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
429
          aom_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
430
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
431
432
433
434
435
        }
      }

      if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) {
        if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
436
          aom_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
437
438
439
                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                  lfi1->hev_thr);
        } else if (mask_4x4_int_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
440
          aom_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim,
441
                             lfi0->hev_thr);
Jingning Han's avatar
Jingning Han committed
442
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
443
          aom_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim, lfi1->lim,
444
                             lfi1->hev_thr);
Jingning Han's avatar
Jingning Han committed
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
        }
      }
    }

    s += 8;
    lfl += 1;
    mask_16x16_0 >>= 1;
    mask_8x8_0 >>= 1;
    mask_4x4_0 >>= 1;
    mask_4x4_int_0 >>= 1;
    mask_16x16_1 >>= 1;
    mask_8x8_1 >>= 1;
    mask_4x4_1 >>= 1;
    mask_4x4_int_1 >>= 1;
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
462
#if CONFIG_AOM_HIGHBITDEPTH
clang-format's avatar
clang-format committed
463
464
465
466
467
static void highbd_filter_selectively_vert_row2(
    int subsampling_factor, uint16_t *s, int pitch, unsigned int mask_16x16_l,
    unsigned int mask_8x8_l, unsigned int mask_4x4_l,
    unsigned int mask_4x4_int_l, const loop_filter_info_n *lfi_n,
    const uint8_t *lfl, int bd) {
Jingning Han's avatar
Jingning Han committed
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
  const int mask_shift = subsampling_factor ? 4 : 8;
  const int mask_cutoff = subsampling_factor ? 0xf : 0xff;
  const int lfl_forward = subsampling_factor ? 4 : 8;

  unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff;
  unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff;
  unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff;
  unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff;
  unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff;
  unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff;
  unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff;
  unsigned int mask;

  for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 |
clang-format's avatar
clang-format committed
483
              mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1;
Jingning Han's avatar
Jingning Han committed
484
485
486
487
488
489
490
       mask; mask >>= 1) {
    const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl;
    const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward);

    if (mask & 1) {
      if ((mask_16x16_0 | mask_16x16_1) & 1) {
        if ((mask_16x16_0 & mask_16x16_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
491
          aom_highbd_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
492
493
                                          lfi0->hev_thr, bd);
        } else if (mask_16x16_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
494
          aom_highbd_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
495
496
                                     lfi0->hev_thr, bd);
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
497
          aom_highbd_lpf_vertical_16(s + 8 * pitch, pitch, lfi1->mblim,
Jingning Han's avatar
Jingning Han committed
498
499
500
501
502
503
                                     lfi1->lim, lfi1->hev_thr, bd);
        }
      }

      if ((mask_8x8_0 | mask_8x8_1) & 1) {
        if ((mask_8x8_0 & mask_8x8_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
504
          aom_highbd_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
505
506
507
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_8x8_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
508
          aom_highbd_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim,
509
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
510
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
511
          aom_highbd_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim,
512
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
513
514
515
516
517
        }
      }

      if ((mask_4x4_0 | mask_4x4_1) & 1) {
        if ((mask_4x4_0 & mask_4x4_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
518
          aom_highbd_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
519
520
521
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_4x4_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
522
          aom_highbd_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim,
523
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
524
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
525
          aom_highbd_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim,
526
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
527
528
529
530
531
        }
      }

      if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) {
        if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
532
          aom_highbd_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim,
Jingning Han's avatar
Jingning Han committed
533
534
535
                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
                                         lfi1->hev_thr, bd);
        } else if (mask_4x4_int_0 & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
536
          aom_highbd_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim,
537
                                    lfi0->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
538
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
539
          aom_highbd_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim,
540
                                    lfi1->lim, lfi1->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
        }
      }
    }

    s += 8;
    lfl += 1;
    mask_16x16_0 >>= 1;
    mask_8x8_0 >>= 1;
    mask_4x4_0 >>= 1;
    mask_4x4_int_0 >>= 1;
    mask_16x16_1 >>= 1;
    mask_8x8_1 >>= 1;
    mask_4x4_1 >>= 1;
    mask_4x4_int_1 >>= 1;
  }
}
Yaowu Xu's avatar
Yaowu Xu committed
557
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
558

clang-format's avatar
clang-format committed
559
560
561
562
static void filter_selectively_horiz(
    uint8_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8,
    unsigned int mask_4x4, unsigned int mask_4x4_int,
    const loop_filter_info_n *lfi_n, const uint8_t *lfl) {
Jingning Han's avatar
Jingning Han committed
563
564
565
  unsigned int mask;
  int count;

clang-format's avatar
clang-format committed
566
567
  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask;
       mask >>= count) {
Jingning Han's avatar
Jingning Han committed
568
569
570
571
572
573
    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;

    count = 1;
    if (mask & 1) {
      if (mask_16x16 & 1) {
        if ((mask_16x16 & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
574
          aom_lpf_horizontal_edge_16(s, pitch, lfi->mblim, lfi->lim,
575
                                     lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
576
577
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
578
          aom_lpf_horizontal_edge_8(s, pitch, lfi->mblim, lfi->lim,
579
                                    lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
580
581
582
583
584
585
        }
      } else if (mask_8x8 & 1) {
        if ((mask_8x8 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
586
          aom_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
587
588
589
590
                                    lfi->hev_thr, lfin->mblim, lfin->lim,
                                    lfin->hev_thr);

          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
591
            aom_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
Jingning Han's avatar
Jingning Han committed
592
593
594
595
                                      lfi->lim, lfi->hev_thr, lfin->mblim,
                                      lfin->lim, lfin->hev_thr);
          } else {
            if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
596
              aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
597
                                   lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
598
            else if (mask_4x4_int & 2)
Yaowu Xu's avatar
Yaowu Xu committed
599
              aom_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
600
                                   lfin->lim, lfin->hev_thr);
Jingning Han's avatar
Jingning Han committed
601
602
603
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
604
          aom_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
605
606

          if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
607
            aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
608
                                 lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
609
610
611
612
613
614
        }
      } else if (mask_4x4 & 1) {
        if ((mask_4x4 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
615
          aom_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
616
617
618
                                    lfi->hev_thr, lfin->mblim, lfin->lim,
                                    lfin->hev_thr);
          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
619
            aom_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
Jingning Han's avatar
Jingning Han committed
620
621
622
623
                                      lfi->lim, lfi->hev_thr, lfin->mblim,
                                      lfin->lim, lfin->hev_thr);
          } else {
            if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
624
              aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
625
                                   lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
626
            else if (mask_4x4_int & 2)
Yaowu Xu's avatar
Yaowu Xu committed
627
              aom_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
628
                                   lfin->lim, lfin->hev_thr);
Jingning Han's avatar
Jingning Han committed
629
630
631
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
632
          aom_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
633
634

          if (mask_4x4_int & 1)
Yaowu Xu's avatar
Yaowu Xu committed
635
            aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
636
                                 lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
637
638
        }
      } else if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
639
        aom_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
640
                             lfi->hev_thr);
Jingning Han's avatar
Jingning Han committed
641
642
643
644
645
646
647
648
649
650
651
      }
    }
    s += 8 * count;
    lfl += count;
    mask_16x16 >>= count;
    mask_8x8 >>= count;
    mask_4x4 >>= count;
    mask_4x4_int >>= count;
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
652
#if CONFIG_AOM_HIGHBITDEPTH
clang-format's avatar
clang-format committed
653
654
655
656
static void highbd_filter_selectively_horiz(
    uint16_t *s, int pitch, unsigned int mask_16x16, unsigned int mask_8x8,
    unsigned int mask_4x4, unsigned int mask_4x4_int,
    const loop_filter_info_n *lfi_n, const uint8_t *lfl, int bd) {
Jingning Han's avatar
Jingning Han committed
657
658
659
  unsigned int mask;
  int count;

clang-format's avatar
clang-format committed
660
661
  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int; mask;
       mask >>= count) {
Jingning Han's avatar
Jingning Han committed
662
663
664
665
666
667
    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;

    count = 1;
    if (mask & 1) {
      if (mask_16x16 & 1) {
        if ((mask_16x16 & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
668
          aom_highbd_lpf_horizontal_edge_16(s, pitch, lfi->mblim, lfi->lim,
669
                                            lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
670
671
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
672
          aom_highbd_lpf_horizontal_edge_8(s, pitch, lfi->mblim, lfi->lim,
673
                                           lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
674
675
676
677
678
679
        }
      } else if (mask_8x8 & 1) {
        if ((mask_8x8 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
680
          aom_highbd_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
681
682
683
684
                                           lfi->hev_thr, lfin->mblim, lfin->lim,
                                           lfin->hev_thr, bd);

          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
685
            aom_highbd_lpf_horizontal_4_dual(
clang-format's avatar
clang-format committed
686
687
                s + 4 * pitch, pitch, lfi->mblim, lfi->lim, lfi->hev_thr,
                lfin->mblim, lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
688
689
          } else {
            if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
690
              aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
691
                                          lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
692
            } else if (mask_4x4_int & 2) {
Yaowu Xu's avatar
Yaowu Xu committed
693
              aom_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
694
                                          lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
695
696
697
698
            }
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
699
          aom_highbd_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim,
700
                                      lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
701
702

          if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
703
            aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
704
                                        lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
705
706
707
708
709
710
711
          }
        }
      } else if (mask_4x4 & 1) {
        if ((mask_4x4 & 3) == 3) {
          // Next block's thresholds.
          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);

Yaowu Xu's avatar
Yaowu Xu committed
712
          aom_highbd_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim,
Jingning Han's avatar
Jingning Han committed
713
714
715
                                           lfi->hev_thr, lfin->mblim, lfin->lim,
                                           lfin->hev_thr, bd);
          if ((mask_4x4_int & 3) == 3) {
Yaowu Xu's avatar
Yaowu Xu committed
716
            aom_highbd_lpf_horizontal_4_dual(
clang-format's avatar
clang-format committed
717
718
                s + 4 * pitch, pitch, lfi->mblim, lfi->lim, lfi->hev_thr,
                lfin->mblim, lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
719
720
          } else {
            if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
721
              aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
722
                                          lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
723
            } else if (mask_4x4_int & 2) {
Yaowu Xu's avatar
Yaowu Xu committed
724
              aom_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
725
                                          lfin->lim, lfin->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
726
727
728
729
            }
          }
          count = 2;
        } else {
Yaowu Xu's avatar
Yaowu Xu committed
730
          aom_highbd_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim,
731
                                      lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
732
733

          if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
734
            aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
735
                                        lfi->lim, lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
736
737
738
          }
        }
      } else if (mask_4x4_int & 1) {
Yaowu Xu's avatar
Yaowu Xu committed
739
        aom_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
740
                                    lfi->hev_thr, bd);
Jingning Han's avatar
Jingning Han committed
741
742
743
744
745
746
747
748
749
750
      }
    }
    s += 8 * count;
    lfl += count;
    mask_16x16 >>= count;
    mask_8x8 >>= count;
    mask_4x4 >>= count;
    mask_4x4_int >>= count;
  }
}
Yaowu Xu's avatar
Yaowu Xu committed
751
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
752
753
754
755
756
757
758
759
760
761

// This function ors into the current lfm structure, where to do loop
// filters for the specific mi we are looking at. It uses information
// including the block_size_type (32x16, 32x32, etc.), the transform size,
// whether there were any coefficients encoded, and the loop filter strength
// block we are currently looking at. Shift is used to position the
// 1's we produce.
// TODO(JBB) Need another function for different resolution color..
static void build_masks(const loop_filter_info_n *const lfi_n,
                        const MODE_INFO *mi, const int shift_y,
clang-format's avatar
clang-format committed
762
                        const int shift_uv, LOOP_FILTER_MASK *lfm) {
Jingning Han's avatar
Jingning Han committed
763
764
  const MB_MODE_INFO *mbmi = &mi->mbmi;
  const BLOCK_SIZE block_size = mbmi->sb_type;
765
766
  // TODO(debargha): Check if masks can be setup correctly when
  // rectangular transfroms are used with the EXT_TX expt.
Debargha Mukherjee's avatar
Debargha Mukherjee committed
767
  const TX_SIZE tx_size_y = txsize_sqr_up_map[mbmi->tx_size];
768
  const TX_SIZE tx_size_uv =
769
      txsize_sqr_up_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
Jingning Han's avatar
Jingning Han committed
770
771
772
773
774
775
  const int filter_level = get_filter_level(lfi_n, mbmi);
  uint64_t *const left_y = &lfm->left_y[tx_size_y];
  uint64_t *const above_y = &lfm->above_y[tx_size_y];
  uint64_t *const int_4x4_y = &lfm->int_4x4_y;
  uint16_t *const left_uv = &lfm->left_uv[tx_size_uv];
  uint16_t *const above_uv = &lfm->above_uv[tx_size_uv];
776
  uint16_t *const int_4x4_uv = &lfm->left_int_4x4_uv;
Jingning Han's avatar
Jingning Han committed
777
778
779
780
781
782
783
784
  int i;

  // If filter level is 0 we don't loop filter.
  if (!filter_level) {
    return;
  } else {
    const int w = num_8x8_blocks_wide_lookup[block_size];
    const int h = num_8x8_blocks_high_lookup[block_size];
785
786
787
    const int row = (shift_y >> MAX_MIB_SIZE_LOG2);
    const int col = shift_y - (row << MAX_MIB_SIZE_LOG2);

clang-format's avatar
clang-format committed
788
    for (i = 0; i < h; i++) memset(&lfm->lfl_y[row + i][col], filter_level, w);
Jingning Han's avatar
Jingning Han committed
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
  }

  // These set 1 in the current block size for the block size edges.
  // For instance if the block size is 32x16, we'll set:
  //    above =   1111
  //              0000
  //    and
  //    left  =   1000
  //          =   1000
  // NOTE : In this example the low bit is left most ( 1000 ) is stored as
  //        1,  not 8...
  //
  // U and V set things on a 16 bit scale.
  //
  *above_y |= above_prediction_mask[block_size] << shift_y;
  *above_uv |= above_prediction_mask_uv[block_size] << shift_uv;
  *left_y |= left_prediction_mask[block_size] << shift_y;
  *left_uv |= left_prediction_mask_uv[block_size] << shift_uv;

  // If the block has no coefficients and is not intra we skip applying
  // the loop filter on block edges.
810
  if (mbmi->skip && is_inter_block(mbmi)) return;
Jingning Han's avatar
Jingning Han committed
811
812
813
814
815

  // Here we are adding a mask for the transform size. The transform
  // size mask is set to be correct for a 64x64 prediction block size. We
  // mask to match the size of the block we are working on and then shift it
  // into place..
clang-format's avatar
clang-format committed
816
817
818
819
820
  *above_y |= (size_mask[block_size] & above_64x64_txform_mask[tx_size_y])
              << shift_y;
  *above_uv |=
      (size_mask_uv[block_size] & above_64x64_txform_mask_uv[tx_size_uv])
      << shift_uv;
Jingning Han's avatar
Jingning Han committed
821

clang-format's avatar
clang-format committed
822
823
824
825
  *left_y |= (size_mask[block_size] & left_64x64_txform_mask[tx_size_y])
             << shift_y;
  *left_uv |= (size_mask_uv[block_size] & left_64x64_txform_mask_uv[tx_size_uv])
              << shift_uv;
Jingning Han's avatar
Jingning Han committed
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842

  // Here we are trying to determine what to do with the internal 4x4 block
  // boundaries.  These differ from the 4x4 boundaries on the outside edge of
  // an 8x8 in that the internal ones can be skipped and don't depend on
  // the prediction block size.
  if (tx_size_y == TX_4X4)
    *int_4x4_y |= (size_mask[block_size] & 0xffffffffffffffffULL) << shift_y;

  if (tx_size_uv == TX_4X4)
    *int_4x4_uv |= (size_mask_uv[block_size] & 0xffff) << shift_uv;
}

// This function does the same thing as the one above with the exception that
// it only affects the y masks. It exists because for blocks < 16x16 in size,
// we only update u and v masks on the first block.
static void build_y_mask(const loop_filter_info_n *const lfi_n,
                         const MODE_INFO *mi, const int shift_y,
843
844
845
#if CONFIG_SUPERTX
                         int supertx_enabled,
#endif  // CONFIG_SUPERTX
Jingning Han's avatar
Jingning Han committed
846
847
                         LOOP_FILTER_MASK *lfm) {
  const MB_MODE_INFO *mbmi = &mi->mbmi;
Debargha Mukherjee's avatar
Debargha Mukherjee committed
848
  const TX_SIZE tx_size_y = txsize_sqr_up_map[mbmi->tx_size];
849
850
851
852
853
854
#if CONFIG_SUPERTX
  const BLOCK_SIZE block_size =
      supertx_enabled ? (BLOCK_SIZE)(3 * tx_size_y) : mbmi->sb_type;
#else
  const BLOCK_SIZE block_size = mbmi->sb_type;
#endif
Jingning Han's avatar
Jingning Han committed
855
856
857
858
859
860
861
862
863
864
865
  const int filter_level = get_filter_level(lfi_n, mbmi);
  uint64_t *const left_y = &lfm->left_y[tx_size_y];
  uint64_t *const above_y = &lfm->above_y[tx_size_y];
  uint64_t *const int_4x4_y = &lfm->int_4x4_y;
  int i;

  if (!filter_level) {
    return;
  } else {
    const int w = num_8x8_blocks_wide_lookup[block_size];
    const int h = num_8x8_blocks_high_lookup[block_size];
866
867
868
    const int row = (shift_y >> MAX_MIB_SIZE_LOG2);
    const int col = shift_y - (row << MAX_MIB_SIZE_LOG2);

clang-format's avatar
clang-format committed
869
    for (i = 0; i < h; i++) memset(&lfm->lfl_y[row + i][col], filter_level, w);
Jingning Han's avatar
Jingning Han committed
870
871
872
873
874
  }

  *above_y |= above_prediction_mask[block_size] << shift_y;
  *left_y |= left_prediction_mask[block_size] << shift_y;

875
  if (mbmi->skip && is_inter_block(mbmi)) return;
Jingning Han's avatar
Jingning Han committed
876

clang-format's avatar
clang-format committed
877
878
  *above_y |= (size_mask[block_size] & above_64x64_txform_mask[tx_size_y])
              << shift_y;
Jingning Han's avatar
Jingning Han committed
879

clang-format's avatar
clang-format committed
880
881
  *left_y |= (size_mask[block_size] & left_64x64_txform_mask[tx_size_y])
             << shift_y;
Jingning Han's avatar
Jingning Han committed
882
883
884
885
886
887
888
889

  if (tx_size_y == TX_4X4)
    *int_4x4_y |= (size_mask[block_size] & 0xffffffffffffffffULL) << shift_y;
}

// This function sets up the bit masks for the entire 64x64 region represented
// by mi_row, mi_col.
// TODO(JBB): This function only works for yv12.
Yaowu Xu's avatar
Yaowu Xu committed
890
891
892
void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
                    MODE_INFO **mi, const int mode_info_stride,
                    LOOP_FILTER_MASK *lfm) {
Jingning Han's avatar
Jingning Han committed
893
894
895
896
897
898
899
900
901
  int idx_32, idx_16, idx_8;
  const loop_filter_info_n *const lfi_n = &cm->lf_info;
  MODE_INFO **mip = mi;
  MODE_INFO **mip2 = mi;

  // These are offsets to the next mi in the 64x64 block. It is what gets
  // added to the mi ptr as we go through each loop. It helps us to avoid
  // setting up special row and column counters for each index. The last step
  // brings us out back to the starting position.
clang-format's avatar
clang-format committed
902
903
904
905
906
  const int offset_32[] = { 4, (mode_info_stride << 2) - 4, 4,
                            -(mode_info_stride << 2) - 4 };
  const int offset_16[] = { 2, (mode_info_stride << 1) - 2, 2,
                            -(mode_info_stride << 1) - 2 };
  const int offset[] = { 1, mode_info_stride - 1, 1, -mode_info_stride - 1 };
Jingning Han's avatar
Jingning Han committed
907
908
909
910
911

  // Following variables represent shifts to position the current block
  // mask over the appropriate block. A shift of 36 to the left will move
  // the bits for the final 32 by 32 block in the 64x64 up 4 rows and left
  // 4 rows to the appropriate spot.
clang-format's avatar
clang-format committed
912
913
914
915
916
  const int shift_32_y[] = { 0, 4, 32, 36 };
  const int shift_16_y[] = { 0, 2, 16, 18 };
  const int shift_8_y[] = { 0, 1, 8, 9 };
  const int shift_32_uv[] = { 0, 2, 8, 10 };
  const int shift_16_uv[] = { 0, 1, 4, 5 };
Jingning Han's avatar
Jingning Han committed
917
  int i;
Yaowu Xu's avatar
Yaowu Xu committed
918
919
  const int max_rows = AOMMIN(cm->mi_rows - mi_row, MAX_MIB_SIZE);
  const int max_cols = AOMMIN(cm->mi_cols - mi_col, MAX_MIB_SIZE);
920
921
922
#if CONFIG_EXT_PARTITION
  assert(0 && "Not yet updated");
#endif  // CONFIG_EXT_PARTITION
Jingning Han's avatar
Jingning Han committed
923

Yaowu Xu's avatar
Yaowu Xu committed
924
  av1_zero(*lfm);
Jingning Han's avatar
Jingning Han committed
925
926
927
928
929
930
  assert(mip[0] != NULL);

  // TODO(jimbankoski): Try moving most of the following code into decode
  // loop and storing lfm in the mbmi structure so that we don't have to go
  // through the recursive loop structure multiple times.
  switch (mip[0]->mbmi.sb_type) {
clang-format's avatar
clang-format committed
931
    case BLOCK_64X64: build_masks(lfi_n, mip[0], 0, 0, lfm); break;
932
933
    case BLOCK_64X32: build_masks(lfi_n, mip[0], 0, 0, lfm);
#if CONFIG_SUPERTX && CONFIG_TX64X64
934
      if (supertx_enabled(&mip[0]->mbmi)) break;
935
#endif  // CONFIG_SUPERTX && CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
936
      mip2 = mip + mode_info_stride * 4;
clang-format's avatar
clang-format committed
937
      if (4 >= max_rows) break;
Jingning Han's avatar
Jingning Han committed
938
939
      build_masks(lfi_n, mip2[0], 32, 8, lfm);
      break;
940
941
    case BLOCK_32X64: build_masks(lfi_n, mip[0], 0, 0, lfm);
#if CONFIG_SUPERTX && CONFIG_TX64X64
942
      if (supertx_enabled(&mip[0]->mbmi)) break;
943
#endif  // CONFIG_SUPERTX && CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
944
      mip2 = mip + 4;
clang-format's avatar
clang-format committed
945
      if (4 >= max_cols) break;
Jingning Han's avatar
Jingning Han committed
946
947
948
      build_masks(lfi_n, mip2[0], 4, 2, lfm);
      break;
    default:
949
#if CONFIG_SUPERTX && CONFIG_TX64X64
950
951
      if (mip[0]->mbmi.tx_size == TX_64X64) {
        build_masks(lfi_n, mip[0], 0, 0, lfm);
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
      } else {
#endif  // CONFIG_SUPERTX && CONFIG_TX64X64
        for (idx_32 = 0; idx_32 < 4; mip += offset_32[idx_32], ++idx_32) {
          const int shift_y_32 = shift_32_y[idx_32];
          const int shift_uv_32 = shift_32_uv[idx_32];
          const int mi_32_col_offset = ((idx_32 & 1) << 2);
          const int mi_32_row_offset = ((idx_32 >> 1) << 2);
          if (mi_32_col_offset >= max_cols || mi_32_row_offset >= max_rows)
            continue;
          switch (mip[0]->mbmi.sb_type) {
            case BLOCK_32X32:
              build_masks(lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
              break;
            case BLOCK_32X16:
              build_masks(lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
967
#if CONFIG_SUPERTX
968
              if (supertx_enabled(&mip[0]->mbmi)) break;
969
#endif
970
971
972
973
974
975
976
              if (mi_32_row_offset + 2 >= max_rows) continue;
              mip2 = mip + mode_info_stride * 2;
              build_masks(lfi_n, mip2[0], shift_y_32 + 16, shift_uv_32 + 4,
                          lfm);
              break;
            case BLOCK_16X32:
              build_masks(lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
977
#if CONFIG_SUPERTX
978
              if (supertx_enabled(&mip[0]->mbmi)) break;
979
#endif
980
981
982
              if (mi_32_col_offset + 2 >= max_cols) continue;
              mip2 = mip + 2;
              build_masks(lfi_n, mip2[0], shift_y_32 + 2, shift_uv_32 + 1, lfm);
983
              break;
984
985
986
987
988
989
            default:
#if CONFIG_SUPERTX
              if (mip[0]->mbmi.tx_size == TX_32X32) {
                build_masks(lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
                break;
              }
990
#endif
991
992
993
994
995
996
997
998
999
1000
              for (idx_16 = 0; idx_16 < 4; mip += offset_16[idx_16], ++idx_16) {
                const int shift_y_32_16 = shift_y_32 + shift_16_y[idx_16];
                const int shift_uv_32_16 = shift_uv_32 + shift_16_uv[idx_16];
                const int mi_16_col_offset =
                    mi_32_col_offset + ((idx_16 & 1) << 1);
                const int mi_16_row_offset =
                    mi_32_row_offset + ((idx_16 >> 1) << 1);

                if (mi_16_col_offset >= max_cols ||
                    mi_16_row_offset >= max_rows)
For faster browsing, not all history is shown. View entire blame