cdef.c 15.7 KB
Newer Older
Yaowu Xu's avatar
Yaowu Xu committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Yaowu Xu's avatar
Yaowu Xu committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Yaowu Xu's avatar
Yaowu Xu committed
10
11
 */

12
#include <assert.h>
Yaowu Xu's avatar
Yaowu Xu committed
13
#include <math.h>
14
#include <string.h>
Yaowu Xu's avatar
Yaowu Xu committed
15

Yaowu Xu's avatar
Yaowu Xu committed
16
17
#include "./aom_scale_rtcd.h"
#include "aom/aom_integer.h"
18
#include "av1/common/cdef.h"
Steinar Midtskogen's avatar
Steinar Midtskogen committed
19
#include "av1/common/cdef_block.h"
20
21
#include "av1/common/onyxc_int.h"
#include "av1/common/reconinter.h"
Yaowu Xu's avatar
Yaowu Xu committed
22

Yaowu Xu's avatar
Yaowu Xu committed
23
int sb_all_skip(const AV1_COMMON *const cm, int mi_row, int mi_col) {
Yaowu Xu's avatar
Yaowu Xu committed
24
25
26
27
28
  int r, c;
  int maxc, maxr;
  int skip = 1;
  maxc = cm->mi_cols - mi_col;
  maxr = cm->mi_rows - mi_row;
29

Cheng Chen's avatar
Cheng Chen committed
30
31
  maxr = AOMMIN(maxr, MI_SIZE_64X64);
  maxc = AOMMIN(maxc, MI_SIZE_64X64);
32

Yaowu Xu's avatar
Yaowu Xu committed
33
34
35
  for (r = 0; r < maxr; r++) {
    for (c = 0; c < maxc; c++) {
      skip = skip &&
clang-format's avatar
clang-format committed
36
37
             cm->mi_grid_visible[(mi_row + r) * cm->mi_stride + mi_col + c]
                 ->mbmi.skip;
Yaowu Xu's avatar
Yaowu Xu committed
38
39
40
41
42
    }
  }
  return skip;
}

Jingning Han's avatar
Jingning Han committed
43
44
45
46
47
48
49
50
51
52
static int is_8x8_block_skip(MODE_INFO **grid, int mi_row, int mi_col,
                             int mi_stride) {
  int is_skip = 1;
  for (int r = 0; r < mi_size_high[BLOCK_8X8]; ++r)
    for (int c = 0; c < mi_size_wide[BLOCK_8X8]; ++c)
      is_skip &= grid[(mi_row + r) * mi_stride + (mi_col + c)]->mbmi.skip;

  return is_skip;
}

Steinar Midtskogen's avatar
Steinar Midtskogen committed
53
54
int sb_compute_cdef_list(const AV1_COMMON *const cm, int mi_row, int mi_col,
                         cdef_list *dlist, int filter_skip) {
Jean-Marc Valin's avatar
Jean-Marc Valin committed
55
56
57
  int r, c;
  int maxc, maxr;
  MODE_INFO **grid;
58
  int count = 0;
Jean-Marc Valin's avatar
Jean-Marc Valin committed
59
60
61
  grid = cm->mi_grid_visible;
  maxc = cm->mi_cols - mi_col;
  maxr = cm->mi_rows - mi_row;
62

Cheng Chen's avatar
Cheng Chen committed
63
64
  maxr = AOMMIN(maxr, MI_SIZE_64X64);
  maxc = AOMMIN(maxc, MI_SIZE_64X64);
65
66
67

  const int r_step = mi_size_high[BLOCK_8X8];
  const int c_step = mi_size_wide[BLOCK_8X8];
Jingning Han's avatar
Jingning Han committed
68
69
70
71
72
  const int r_shift = (r_step == 2);
  const int c_shift = (c_step == 2);

  assert(r_step == 1 || r_step == 2);
  assert(c_step == 1 || c_step == 2);
73

74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
  if (filter_skip) {
    for (r = 0; r < maxr; r += r_step) {
      for (c = 0; c < maxc; c += c_step) {
        dlist[count].by = r >> r_shift;
        dlist[count].bx = c >> c_shift;
        dlist[count].skip =
            is_8x8_block_skip(grid, mi_row + r, mi_col + c, cm->mi_stride);
        count++;
      }
    }
  } else {
    for (r = 0; r < maxr; r += r_step) {
      for (c = 0; c < maxc; c += c_step) {
        if (!is_8x8_block_skip(grid, mi_row + r, mi_col + c, cm->mi_stride)) {
          dlist[count].by = r >> r_shift;
          dlist[count].bx = c >> c_shift;
          dlist[count].skip = 0;
          count++;
        }
      }
Jean-Marc Valin's avatar
Jean-Marc Valin committed
94
95
    }
  }
96
  return count;
Jean-Marc Valin's avatar
Jean-Marc Valin committed
97
98
}

Michael Bebenita's avatar
Michael Bebenita committed
99
100
void copy_rect8_8bit_to_16bit_c(uint16_t *dst, int dstride, const uint8_t *src,
                                int sstride, int v, int h) {
101
  int i, j;
Michael Bebenita's avatar
Michael Bebenita committed
102
103
  for (i = 0; i < v; i++) {
    for (j = 0; j < h; j++) {
104
105
106
107
108
      dst[i * dstride + j] = src[i * sstride + j];
    }
  }
}

Michael Bebenita's avatar
Michael Bebenita committed
109
110
111
void copy_rect8_16bit_to_16bit_c(uint16_t *dst, int dstride,
                                 const uint16_t *src, int sstride, int v,
                                 int h) {
112
  int i, j;
Michael Bebenita's avatar
Michael Bebenita committed
113
114
  for (i = 0; i < v; i++) {
    for (j = 0; j < h; j++) {
115
116
117
118
119
      dst[i * dstride + j] = src[i * sstride + j];
    }
  }
}

Cheng Chen's avatar
Cheng Chen committed
120
121
122
static void copy_sb8_16(UNUSED AV1_COMMON *cm, uint16_t *dst, int dstride,
                        const uint8_t *src, int src_voffset, int src_hoffset,
                        int sstride, int vsize, int hsize) {
123
#if CONFIG_HIGHBITDEPTH
124
  if (cm->use_highbitdepth) {
125
126
    const uint16_t *base =
        &CONVERT_TO_SHORTPTR(src)[src_voffset * sstride + src_hoffset];
Michael Bebenita's avatar
Michael Bebenita committed
127
    copy_rect8_16bit_to_16bit(dst, dstride, base, sstride, vsize, hsize);
128
  } else {
129
130
#endif
    const uint8_t *base = &src[src_voffset * sstride + src_hoffset];
Michael Bebenita's avatar
Michael Bebenita committed
131
    copy_rect8_8bit_to_16bit(dst, dstride, base, sstride, vsize, hsize);
132
#if CONFIG_HIGHBITDEPTH
133
  }
134
#endif
135
136
}

Michael Bebenita's avatar
Michael Bebenita committed
137
138
139
140
static INLINE void fill_rect(uint16_t *dst, int dstride, int v, int h,
                             uint16_t x) {
  int i, j;
  for (i = 0; i < v; i++) {
141
142
    for (j = 0; j < h; j++) {
      dst[i * dstride + j] = x;
Michael Bebenita's avatar
Michael Bebenita committed
143
144
145
146
    }
  }
}

Michael Bebenita's avatar
Michael Bebenita committed
147
148
149
150
static INLINE void copy_rect(uint16_t *dst, int dstride, const uint16_t *src,
                             int sstride, int v, int h) {
  int i, j;
  for (i = 0; i < v; i++) {
151
152
    for (j = 0; j < h; j++) {
      dst[i * dstride + j] = src[i * sstride + j];
Michael Bebenita's avatar
Michael Bebenita committed
153
154
155
156
    }
  }
}

157
158
void av1_cdef_frame(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm,
                    MACROBLOCKD *xd) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
159
160
161
  int fbr, fbc;
  int nhfb, nvfb;
  uint16_t src[CDEF_INBUF_SIZE];
162
  uint16_t *linebuf[3];
163
  uint16_t *colbuf[3];
Steinar Midtskogen's avatar
Steinar Midtskogen committed
164
165
166
167
168
  cdef_list dlist[MI_SIZE_64X64 * MI_SIZE_64X64];
  unsigned char *row_cdef, *prev_row_cdef, *curr_row_cdef;
  int cdef_count;
  int dir[CDEF_NBLOCKS][CDEF_NBLOCKS] = { { 0 } };
  int var[CDEF_NBLOCKS][CDEF_NBLOCKS] = { { 0 } };
Yaowu Xu's avatar
Yaowu Xu committed
169
  int stride;
Jingning Han's avatar
Jingning Han committed
170
171
  int mi_wide_l2[3];
  int mi_high_l2[3];
172
173
  int xdec[3];
  int ydec[3];
Yaowu Xu's avatar
Yaowu Xu committed
174
  int pli;
Steinar Midtskogen's avatar
Steinar Midtskogen committed
175
  int cdef_left;
Yaowu Xu's avatar
Yaowu Xu committed
176
  int coeff_shift = AOMMAX(cm->bit_depth - 8, 0);
Steinar Midtskogen's avatar
Steinar Midtskogen committed
177
178
179
180
181
  int nplanes = MAX_MB_PLANE;
  int chroma_cdef = xd->plane[1].subsampling_x == xd->plane[1].subsampling_y &&
                    xd->plane[2].subsampling_x == xd->plane[2].subsampling_y;
  nvfb = (cm->mi_rows + MI_SIZE_64X64 - 1) / MI_SIZE_64X64;
  nhfb = (cm->mi_cols + MI_SIZE_64X64 - 1) / MI_SIZE_64X64;
182
  av1_setup_dst_planes(xd->plane, cm->sb_size, frame, 0, 0);
Steinar Midtskogen's avatar
Steinar Midtskogen committed
183
184
185
186
  row_cdef = aom_malloc(sizeof(*row_cdef) * (nhfb + 2) * 2);
  memset(row_cdef, 1, sizeof(*row_cdef) * (nhfb + 2) * 2);
  prev_row_cdef = row_cdef + 1;
  curr_row_cdef = prev_row_cdef + nhfb + 2;
187
  for (pli = 0; pli < nplanes; pli++) {
188
189
    xdec[pli] = xd->plane[pli].subsampling_x;
    ydec[pli] = xd->plane[pli].subsampling_y;
Jingning Han's avatar
Jingning Han committed
190
    mi_wide_l2[pli] = MI_SIZE_LOG2 - xd->plane[pli].subsampling_x;
191
    mi_high_l2[pli] = MI_SIZE_LOG2 - xd->plane[pli].subsampling_y;
Yaowu Xu's avatar
Yaowu Xu committed
192
  }
Steinar Midtskogen's avatar
Steinar Midtskogen committed
193
  stride = (cm->mi_cols << MI_SIZE_LOG2) + 2 * CDEF_HBORDER;
194
  for (pli = 0; pli < nplanes; pli++) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
195
    linebuf[pli] = aom_malloc(sizeof(*linebuf) * CDEF_VBORDER * stride);
196
197
    colbuf[pli] =
        aom_malloc(sizeof(*colbuf) *
Steinar Midtskogen's avatar
Steinar Midtskogen committed
198
199
                   ((CDEF_BLOCKSIZE << mi_high_l2[pli]) + 2 * CDEF_VBORDER) *
                   CDEF_HBORDER);
200
  }
Steinar Midtskogen's avatar
Steinar Midtskogen committed
201
  for (fbr = 0; fbr < nvfb; fbr++) {
202
    for (pli = 0; pli < nplanes; pli++) {
Jingning Han's avatar
Jingning Han committed
203
      const int block_height =
Steinar Midtskogen's avatar
Steinar Midtskogen committed
204
205
206
          (MI_SIZE_64X64 << mi_high_l2[pli]) + 2 * CDEF_VBORDER;
      fill_rect(colbuf[pli], CDEF_HBORDER, block_height, CDEF_HBORDER,
                CDEF_VERY_LARGE);
Yaowu Xu's avatar
Yaowu Xu committed
207
    }
Steinar Midtskogen's avatar
Steinar Midtskogen committed
208
209
210
211
    cdef_left = 1;
    for (fbc = 0; fbc < nhfb; fbc++) {
      int level, sec_strength;
      int uv_level, uv_sec_strength;
Yaowu Xu's avatar
Yaowu Xu committed
212
      int nhb, nvb;
213
      int cstart = 0;
Steinar Midtskogen's avatar
Steinar Midtskogen committed
214
215
216
217
218
      curr_row_cdef[fbc] = 0;
      if (cm->mi_grid_visible[MI_SIZE_64X64 * fbr * cm->mi_stride +
                              MI_SIZE_64X64 * fbc] == NULL ||
          cm->mi_grid_visible[MI_SIZE_64X64 * fbr * cm->mi_stride +
                              MI_SIZE_64X64 * fbc]
219
                  ->mbmi.cdef_strength == -1) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
220
        cdef_left = 0;
221
222
        continue;
      }
Steinar Midtskogen's avatar
Steinar Midtskogen committed
223
224
225
      if (!cdef_left) cstart = -CDEF_HBORDER;
      nhb = AOMMIN(MI_SIZE_64X64, cm->mi_cols - MI_SIZE_64X64 * fbc);
      nvb = AOMMIN(MI_SIZE_64X64, cm->mi_rows - MI_SIZE_64X64 * fbr);
Jean-Marc Valin's avatar
Jean-Marc Valin committed
226
      int tile_top, tile_left, tile_bottom, tile_right;
Steinar Midtskogen's avatar
Steinar Midtskogen committed
227
      int mi_idx = MI_SIZE_64X64 * fbr * cm->mi_stride + MI_SIZE_64X64 * fbc;
228
229
      MODE_INFO *const mi_tl = cm->mi + mi_idx;
      BOUNDARY_TYPE boundary_tl = mi_tl->mbmi.boundary_info;
Jean-Marc Valin's avatar
Jean-Marc Valin committed
230
231
      tile_top = boundary_tl & TILE_ABOVE_BOUNDARY;
      tile_left = boundary_tl & TILE_LEFT_BOUNDARY;
232

Steinar Midtskogen's avatar
Steinar Midtskogen committed
233
      if (fbr != nvfb - 1 &&
Cheng Chen's avatar
Cheng Chen committed
234
235
          (&cm->mi[mi_idx + (MI_SIZE_64X64 - 1) * cm->mi_stride]))
        tile_bottom = cm->mi[mi_idx + (MI_SIZE_64X64 - 1) * cm->mi_stride]
236
237
                          .mbmi.boundary_info &
                      TILE_BOTTOM_BOUNDARY;
238
239
      else
        tile_bottom = 1;
240

Steinar Midtskogen's avatar
Steinar Midtskogen committed
241
      if (fbc != nhfb - 1 && (&cm->mi[mi_idx + MI_SIZE_64X64 - 1]))
Cheng Chen's avatar
Cheng Chen committed
242
        tile_right = cm->mi[mi_idx + MI_SIZE_64X64 - 1].mbmi.boundary_info &
243
                     TILE_RIGHT_BOUNDARY;
244
245
      else
        tile_right = 1;
246

247
      const int mbmi_cdef_strength =
Steinar Midtskogen's avatar
Steinar Midtskogen committed
248
249
          cm->mi_grid_visible[MI_SIZE_64X64 * fbr * cm->mi_stride +
                              MI_SIZE_64X64 * fbc]
250
              ->mbmi.cdef_strength;
Steinar Midtskogen's avatar
Steinar Midtskogen committed
251
252
253
254
255
256
257
258
259
260
261
262
      level = cm->cdef_strengths[mbmi_cdef_strength] / CDEF_SEC_STRENGTHS;
      sec_strength =
          cm->cdef_strengths[mbmi_cdef_strength] % CDEF_SEC_STRENGTHS;
      sec_strength += sec_strength == 3;
      uv_level = cm->cdef_uv_strengths[mbmi_cdef_strength] / CDEF_SEC_STRENGTHS;
      uv_sec_strength =
          cm->cdef_uv_strengths[mbmi_cdef_strength] % CDEF_SEC_STRENGTHS;
      uv_sec_strength += uv_sec_strength == 3;
      if ((level == 0 && sec_strength == 0 && uv_level == 0 &&
           uv_sec_strength == 0) ||
          (cdef_count = sb_compute_cdef_list(
               cm, fbr * MI_SIZE_64X64, fbc * MI_SIZE_64X64, dlist,
263
               get_filter_skip(level) || get_filter_skip(uv_level))) == 0) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
264
        cdef_left = 0;
265
        continue;
266
      }
267

Steinar Midtskogen's avatar
Steinar Midtskogen committed
268
      curr_row_cdef[fbc] = 1;
269
      for (pli = 0; pli < nplanes; pli++) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
270
        uint16_t dst[CDEF_BLOCKSIZE * CDEF_BLOCKSIZE];
271
272
        int coffset;
        int rend, cend;
Steinar Midtskogen's avatar
Steinar Midtskogen committed
273
274
        int pri_damping = cm->cdef_pri_damping;
        int sec_damping = cm->cdef_sec_damping;
275
276
        int hsize = nhb << mi_wide_l2[pli];
        int vsize = nvb << mi_high_l2[pli];
277
278

        if (pli) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
279
          if (chroma_cdef)
280
281
282
            level = uv_level;
          else
            level = 0;
Steinar Midtskogen's avatar
Steinar Midtskogen committed
283
          sec_strength = uv_sec_strength;
284
        }
Jingning Han's avatar
Jingning Han committed
285

Steinar Midtskogen's avatar
Steinar Midtskogen committed
286
        if (fbc == nhfb - 1)
287
          cend = hsize;
288
        else
Steinar Midtskogen's avatar
Steinar Midtskogen committed
289
          cend = hsize + CDEF_HBORDER;
Jingning Han's avatar
Jingning Han committed
290

Steinar Midtskogen's avatar
Steinar Midtskogen committed
291
        if (fbr == nvfb - 1)
292
          rend = vsize;
293
        else
Steinar Midtskogen's avatar
Steinar Midtskogen committed
294
          rend = vsize + CDEF_VBORDER;
Jingning Han's avatar
Jingning Han committed
295

Steinar Midtskogen's avatar
Steinar Midtskogen committed
296
297
        coffset = fbc * MI_SIZE_64X64 << mi_wide_l2[pli];
        if (fbc == nhfb - 1) {
298
          /* On the last superblock column, fill in the right border with
Steinar Midtskogen's avatar
Steinar Midtskogen committed
299
300
301
302
             CDEF_VERY_LARGE to avoid filtering with the outside. */
          fill_rect(&src[cend + CDEF_HBORDER], CDEF_BSTRIDE,
                    rend + CDEF_VBORDER, hsize + CDEF_HBORDER - cend,
                    CDEF_VERY_LARGE);
303
        }
Steinar Midtskogen's avatar
Steinar Midtskogen committed
304
        if (fbr == nvfb - 1) {
305
          /* On the last superblock row, fill in the bottom border with
Steinar Midtskogen's avatar
Steinar Midtskogen committed
306
307
308
             CDEF_VERY_LARGE to avoid filtering with the outside. */
          fill_rect(&src[(rend + CDEF_VBORDER) * CDEF_BSTRIDE], CDEF_BSTRIDE,
                    CDEF_VBORDER, hsize + 2 * CDEF_HBORDER, CDEF_VERY_LARGE);
309
        }
310
311
        /* Copy in the pixels we need from the current superblock for
           deringing.*/
Steinar Midtskogen's avatar
Steinar Midtskogen committed
312
313
314
315
316
317
318
319
320
321
322
323
324
        copy_sb8_16(cm,
                    &src[CDEF_VBORDER * CDEF_BSTRIDE + CDEF_HBORDER + cstart],
                    CDEF_BSTRIDE, xd->plane[pli].dst.buf,
                    (MI_SIZE_64X64 << mi_high_l2[pli]) * fbr, coffset + cstart,
                    xd->plane[pli].dst.stride, rend, cend - cstart);
        if (!prev_row_cdef[fbc]) {
          copy_sb8_16(cm, &src[CDEF_HBORDER], CDEF_BSTRIDE,
                      xd->plane[pli].dst.buf,
                      (MI_SIZE_64X64 << mi_high_l2[pli]) * fbr - CDEF_VBORDER,
                      coffset, xd->plane[pli].dst.stride, CDEF_VBORDER, hsize);
        } else if (fbr > 0) {
          copy_rect(&src[CDEF_HBORDER], CDEF_BSTRIDE, &linebuf[pli][coffset],
                    stride, CDEF_VBORDER, hsize);
325
        } else {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
326
327
          fill_rect(&src[CDEF_HBORDER], CDEF_BSTRIDE, CDEF_VBORDER, hsize,
                    CDEF_VERY_LARGE);
328
        }
Steinar Midtskogen's avatar
Steinar Midtskogen committed
329
330
331
332
333
334
335
336
        if (!prev_row_cdef[fbc - 1]) {
          copy_sb8_16(cm, src, CDEF_BSTRIDE, xd->plane[pli].dst.buf,
                      (MI_SIZE_64X64 << mi_high_l2[pli]) * fbr - CDEF_VBORDER,
                      coffset - CDEF_HBORDER, xd->plane[pli].dst.stride,
                      CDEF_VBORDER, CDEF_HBORDER);
        } else if (fbr > 0 && fbc > 0) {
          copy_rect(src, CDEF_BSTRIDE, &linebuf[pli][coffset - CDEF_HBORDER],
                    stride, CDEF_VBORDER, CDEF_HBORDER);
337
        } else {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
338
339
          fill_rect(src, CDEF_BSTRIDE, CDEF_VBORDER, CDEF_HBORDER,
                    CDEF_VERY_LARGE);
340
        }
Steinar Midtskogen's avatar
Steinar Midtskogen committed
341
342
343
344
345
346
347
348
349
350
        if (!prev_row_cdef[fbc + 1]) {
          copy_sb8_16(cm, &src[CDEF_HBORDER + (nhb << mi_wide_l2[pli])],
                      CDEF_BSTRIDE, xd->plane[pli].dst.buf,
                      (MI_SIZE_64X64 << mi_high_l2[pli]) * fbr - CDEF_VBORDER,
                      coffset + hsize, xd->plane[pli].dst.stride, CDEF_VBORDER,
                      CDEF_HBORDER);
        } else if (fbr > 0 && fbc < nhfb - 1) {
          copy_rect(&src[hsize + CDEF_HBORDER], CDEF_BSTRIDE,
                    &linebuf[pli][coffset + hsize], stride, CDEF_VBORDER,
                    CDEF_HBORDER);
351
        } else {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
352
353
          fill_rect(&src[hsize + CDEF_HBORDER], CDEF_BSTRIDE, CDEF_VBORDER,
                    CDEF_HBORDER, CDEF_VERY_LARGE);
354
        }
Steinar Midtskogen's avatar
Steinar Midtskogen committed
355
        if (cdef_left) {
356
357
          /* If we deringed the superblock on the left then we need to copy in
             saved pixels. */
Steinar Midtskogen's avatar
Steinar Midtskogen committed
358
359
          copy_rect(src, CDEF_BSTRIDE, colbuf[pli], CDEF_HBORDER,
                    rend + CDEF_VBORDER, CDEF_HBORDER);
360
        }
Michael Bebenita's avatar
Michael Bebenita committed
361
362
        /* Saving pixels in case we need to dering the superblock on the
            right. */
Steinar Midtskogen's avatar
Steinar Midtskogen committed
363
364
        copy_rect(colbuf[pli], CDEF_HBORDER, src + hsize, CDEF_BSTRIDE,
                  rend + CDEF_VBORDER, CDEF_HBORDER);
Jingning Han's avatar
Jingning Han committed
365
366
        copy_sb8_16(
            cm, &linebuf[pli][coffset], stride, xd->plane[pli].dst.buf,
Steinar Midtskogen's avatar
Steinar Midtskogen committed
367
368
            (MI_SIZE_64X64 << mi_high_l2[pli]) * (fbr + 1) - CDEF_VBORDER,
            coffset, xd->plane[pli].dst.stride, CDEF_VBORDER, hsize);
369

Jean-Marc Valin's avatar
Jean-Marc Valin committed
370
        if (tile_top) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
371
372
          fill_rect(src, CDEF_BSTRIDE, CDEF_VBORDER, hsize + 2 * CDEF_HBORDER,
                    CDEF_VERY_LARGE);
Jean-Marc Valin's avatar
Jean-Marc Valin committed
373
374
        }
        if (tile_left) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
375
376
          fill_rect(src, CDEF_BSTRIDE, vsize + 2 * CDEF_VBORDER, CDEF_HBORDER,
                    CDEF_VERY_LARGE);
Jean-Marc Valin's avatar
Jean-Marc Valin committed
377
378
        }
        if (tile_bottom) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
379
380
          fill_rect(&src[(vsize + CDEF_VBORDER) * CDEF_BSTRIDE], CDEF_BSTRIDE,
                    CDEF_VBORDER, hsize + 2 * CDEF_HBORDER, CDEF_VERY_LARGE);
Jean-Marc Valin's avatar
Jean-Marc Valin committed
381
382
        }
        if (tile_right) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
383
384
          fill_rect(&src[hsize + CDEF_HBORDER], CDEF_BSTRIDE,
                    vsize + 2 * CDEF_VBORDER, CDEF_HBORDER, CDEF_VERY_LARGE);
Jean-Marc Valin's avatar
Jean-Marc Valin committed
385
        }
386
#if CONFIG_HIGHBITDEPTH
387
        if (cm->use_highbitdepth) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
388
          cdef_filter_fb(
Jingning Han's avatar
Jingning Han committed
389
390
391
              (uint8_t *)&CONVERT_TO_SHORTPTR(
                  xd->plane[pli]
                      .dst.buf)[xd->plane[pli].dst.stride *
Steinar Midtskogen's avatar
Steinar Midtskogen committed
392
393
                                    (MI_SIZE_64X64 * fbr << mi_high_l2[pli]) +
                                (fbc * MI_SIZE_64X64 << mi_wide_l2[pli])],
394
              xd->plane[pli].dst.stride, dst,
Steinar Midtskogen's avatar
Steinar Midtskogen committed
395
396
397
              &src[CDEF_VBORDER * CDEF_BSTRIDE + CDEF_HBORDER], xdec[pli],
              ydec[pli], dir, NULL, var, pli, dlist, cdef_count, level,
              sec_strength, sec_damping, pri_damping, coeff_shift, 0, 1);
Jingning Han's avatar
Jingning Han committed
398
399
        } else {
#endif
Steinar Midtskogen's avatar
Steinar Midtskogen committed
400
401
402
403
404
405
406
407
408
          cdef_filter_fb(
              &xd->plane[pli]
                   .dst.buf[xd->plane[pli].dst.stride *
                                (MI_SIZE_64X64 * fbr << mi_high_l2[pli]) +
                            (fbc * MI_SIZE_64X64 << mi_wide_l2[pli])],
              xd->plane[pli].dst.stride, dst,
              &src[CDEF_VBORDER * CDEF_BSTRIDE + CDEF_HBORDER], xdec[pli],
              ydec[pli], dir, NULL, var, pli, dlist, cdef_count, level,
              sec_strength, sec_damping, pri_damping, coeff_shift, 0, 0);
409

410
#if CONFIG_HIGHBITDEPTH
Yaowu Xu's avatar
Yaowu Xu committed
411
        }
412
#endif
Yaowu Xu's avatar
Yaowu Xu committed
413
      }
Steinar Midtskogen's avatar
Steinar Midtskogen committed
414
      cdef_left = 1;
Yaowu Xu's avatar
Yaowu Xu committed
415
    }
416
417
    {
      unsigned char *tmp;
Steinar Midtskogen's avatar
Steinar Midtskogen committed
418
419
420
      tmp = prev_row_cdef;
      prev_row_cdef = curr_row_cdef;
      curr_row_cdef = tmp;
421
    }
Yaowu Xu's avatar
Yaowu Xu committed
422
  }
Steinar Midtskogen's avatar
Steinar Midtskogen committed
423
  aom_free(row_cdef);
424
  for (pli = 0; pli < nplanes; pli++) {
425
    aom_free(linebuf[pli]);
426
    aom_free(colbuf[pli]);
Yaowu Xu's avatar
Yaowu Xu committed
427
428
  }
}