cdef_block.c 14.8 KB
Newer Older
Yaowu Xu's avatar
Yaowu Xu committed
1 2 3 4 5 6 7 8 9 10
/*
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */
11 12 13 14

#include <math.h>
#include <stdlib.h>

Yaowu Xu's avatar
Yaowu Xu committed
15
#ifdef HAVE_CONFIG_H
16
#include "./config.h"
Yaowu Xu's avatar
Yaowu Xu committed
17 18
#endif

19
#include "./aom_dsp_rtcd.h"
20
#include "./av1_rtcd.h"
21
#include "./cdef.h"
Yaowu Xu's avatar
Yaowu Xu committed
22 23

/* Generated from gen_filter_tables.c. */
Steinar Midtskogen's avatar
Steinar Midtskogen committed
24 25 26 27 28 29 30 31 32
const int cdef_directions[8][3] = {
  { -1 * CDEF_BSTRIDE + 1, -2 * CDEF_BSTRIDE + 2, -3 * CDEF_BSTRIDE + 3 },
  { 0 * CDEF_BSTRIDE + 1, -1 * CDEF_BSTRIDE + 2, -1 * CDEF_BSTRIDE + 3 },
  { 0 * CDEF_BSTRIDE + 1, 0 * CDEF_BSTRIDE + 2, 0 * CDEF_BSTRIDE + 3 },
  { 0 * CDEF_BSTRIDE + 1, 1 * CDEF_BSTRIDE + 2, 1 * CDEF_BSTRIDE + 3 },
  { 1 * CDEF_BSTRIDE + 1, 2 * CDEF_BSTRIDE + 2, 3 * CDEF_BSTRIDE + 3 },
  { 1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE + 1, 3 * CDEF_BSTRIDE + 1 },
  { 1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE + 0, 3 * CDEF_BSTRIDE + 0 },
  { 1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE - 1, 3 * CDEF_BSTRIDE - 1 }
Yaowu Xu's avatar
Yaowu Xu committed
33 34 35 36 37 38 39 40 41
};

/* Detect direction. 0 means 45-degree up-right, 2 is horizontal, and so on.
   The search minimizes the weighted variance along all the lines in a
   particular direction, i.e. the squared error between the input and a
   "predicted" block where each pixel is replaced by the average along a line
   in a particular direction. Since each direction have the same sum(x^2) term,
   that term is never computed. See Section 2, step 2, of:
   http://jmvalin.ca/notes/intra_paint.pdf */
Steinar Midtskogen's avatar
Steinar Midtskogen committed
42 43
int cdef_find_dir_c(const uint16_t *img, int stride, int32_t *var,
                    int coeff_shift) {
Yaowu Xu's avatar
Yaowu Xu committed
44
  int i;
clang-format's avatar
clang-format committed
45 46
  int32_t cost[8] = { 0 };
  int partial[8][15] = { { 0 } };
Yaowu Xu's avatar
Yaowu Xu committed
47 48 49 50 51
  int32_t best_cost = 0;
  int best_dir = 0;
  /* Instead of dividing by n between 2 and 8, we multiply by 3*5*7*8/n.
     The output is then 840 times larger, but we don't care for finding
     the max. */
clang-format's avatar
clang-format committed
52
  static const int div_table[] = { 0, 840, 420, 280, 210, 168, 140, 120, 105 };
Yaowu Xu's avatar
Yaowu Xu committed
53 54 55 56 57 58
  for (i = 0; i < 8; i++) {
    int j;
    for (j = 0; j < 8; j++) {
      int x;
      /* We subtract 128 here to reduce the maximum range of the squared
         partial sums. */
clang-format's avatar
clang-format committed
59
      x = (img[i * stride + j] >> coeff_shift) - 128;
Yaowu Xu's avatar
Yaowu Xu committed
60
      partial[0][i + j] += x;
clang-format's avatar
clang-format committed
61
      partial[1][i + j / 2] += x;
Yaowu Xu's avatar
Yaowu Xu committed
62
      partial[2][i] += x;
clang-format's avatar
clang-format committed
63
      partial[3][3 + i - j / 2] += x;
Yaowu Xu's avatar
Yaowu Xu committed
64
      partial[4][7 + i - j] += x;
clang-format's avatar
clang-format committed
65
      partial[5][3 - i / 2 + j] += x;
Yaowu Xu's avatar
Yaowu Xu committed
66
      partial[6][j] += x;
clang-format's avatar
clang-format committed
67
      partial[7][i / 2 + j] += x;
Yaowu Xu's avatar
Yaowu Xu committed
68 69 70
    }
  }
  for (i = 0; i < 8; i++) {
clang-format's avatar
clang-format committed
71 72
    cost[2] += partial[2][i] * partial[2][i];
    cost[6] += partial[6][i] * partial[6][i];
Yaowu Xu's avatar
Yaowu Xu committed
73 74 75 76
  }
  cost[2] *= div_table[8];
  cost[6] *= div_table[8];
  for (i = 0; i < 7; i++) {
clang-format's avatar
clang-format committed
77 78 79 80 81 82
    cost[0] += (partial[0][i] * partial[0][i] +
                partial[0][14 - i] * partial[0][14 - i]) *
               div_table[i + 1];
    cost[4] += (partial[4][i] * partial[4][i] +
                partial[4][14 - i] * partial[4][14 - i]) *
               div_table[i + 1];
Yaowu Xu's avatar
Yaowu Xu committed
83
  }
clang-format's avatar
clang-format committed
84 85
  cost[0] += partial[0][7] * partial[0][7] * div_table[8];
  cost[4] += partial[4][7] * partial[4][7] * div_table[8];
Yaowu Xu's avatar
Yaowu Xu committed
86 87 88
  for (i = 1; i < 8; i += 2) {
    int j;
    for (j = 0; j < 4 + 1; j++) {
clang-format's avatar
clang-format committed
89
      cost[i] += partial[i][3 + j] * partial[i][3 + j];
Yaowu Xu's avatar
Yaowu Xu committed
90 91 92
    }
    cost[i] *= div_table[8];
    for (j = 0; j < 4 - 1; j++) {
clang-format's avatar
clang-format committed
93 94 95
      cost[i] += (partial[i][j] * partial[i][j] +
                  partial[i][10 - j] * partial[i][10 - j]) *
                 div_table[2 * j + 2];
Yaowu Xu's avatar
Yaowu Xu committed
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
    }
  }
  for (i = 0; i < 8; i++) {
    if (cost[i] > best_cost) {
      best_cost = cost[i];
      best_dir = i;
    }
  }
  /* Difference between the optimal variance and the variance along the
     orthogonal direction. Again, the sum(x^2) terms cancel out. */
  *var = best_cost - cost[(best_dir + 4) & 7];
  /* We'd normally divide by 840, but dividing by 1024 is close enough
     for what we're going to do with this. */
  *var >>= 10;
  return best_dir;
}

/* Smooth in the direction detected. */
Steinar Midtskogen's avatar
Steinar Midtskogen committed
114 115
void cdef_direction_8x8_c(uint16_t *y, int ystride, const uint16_t *in,
                          int threshold, int dir, int damping) {
Yaowu Xu's avatar
Yaowu Xu committed
116 117 118
  int i;
  int j;
  int k;
119
  static const int taps[3] = { 3, 2, 1 };
120 121
  for (i = 0; i < 8; i++) {
    for (j = 0; j < 8; j++) {
Yaowu Xu's avatar
Yaowu Xu committed
122 123 124
      int16_t sum;
      int16_t xx;
      int16_t yy;
Steinar Midtskogen's avatar
Steinar Midtskogen committed
125
      xx = in[i * CDEF_BSTRIDE + j];
clang-format's avatar
clang-format committed
126
      sum = 0;
Yaowu Xu's avatar
Yaowu Xu committed
127 128 129
      for (k = 0; k < 3; k++) {
        int16_t p0;
        int16_t p1;
Steinar Midtskogen's avatar
Steinar Midtskogen committed
130 131
        p0 = in[i * CDEF_BSTRIDE + j + cdef_directions[dir][k]] - xx;
        p1 = in[i * CDEF_BSTRIDE + j - cdef_directions[dir][k]] - xx;
132 133
        sum += taps[k] * constrain(p0, threshold, damping);
        sum += taps[k] * constrain(p1, threshold, damping);
Yaowu Xu's avatar
Yaowu Xu committed
134
      }
135 136
      sum = (sum + 8) >> 4;
      yy = xx + sum;
clang-format's avatar
clang-format committed
137
      y[i * ystride + j] = yy;
Yaowu Xu's avatar
Yaowu Xu committed
138 139 140 141
    }
  }
}

142
/* Smooth in the direction detected. */
Steinar Midtskogen's avatar
Steinar Midtskogen committed
143 144
void cdef_direction_4x4_c(uint16_t *y, int ystride, const uint16_t *in,
                          int threshold, int dir, int damping) {
145 146 147 148 149 150 151 152 153
  int i;
  int j;
  int k;
  static const int taps[2] = { 4, 1 };
  for (i = 0; i < 4; i++) {
    for (j = 0; j < 4; j++) {
      int16_t sum;
      int16_t xx;
      int16_t yy;
Steinar Midtskogen's avatar
Steinar Midtskogen committed
154
      xx = in[i * CDEF_BSTRIDE + j];
155 156 157 158
      sum = 0;
      for (k = 0; k < 2; k++) {
        int16_t p0;
        int16_t p1;
Steinar Midtskogen's avatar
Steinar Midtskogen committed
159 160
        p0 = in[i * CDEF_BSTRIDE + j + cdef_directions[dir][k]] - xx;
        p1 = in[i * CDEF_BSTRIDE + j - cdef_directions[dir][k]] - xx;
161 162
        sum += taps[k] * constrain(p0, threshold, damping);
        sum += taps[k] * constrain(p1, threshold, damping);
163
      }
164 165
      sum = (sum + 8) >> 4;
      yy = xx + sum;
166 167 168
      y[i * ystride + j] = yy;
    }
  }
Yaowu Xu's avatar
Yaowu Xu committed
169 170
}

Steinar Midtskogen's avatar
Steinar Midtskogen committed
171 172 173 174 175 176 177
/* Compute the primary filter strength for an 8x8 block based on the
   directional variance difference. A high variance difference means
   that we have a highly directional pattern (e.g. a high contrast
   edge), so we can apply more deringing. A low variance means that we
   either have a low contrast edge, or a non-directional texture, so
   we want to be careful not to blur. */
static INLINE int adjust_strength(int strength, int32_t var) {
178
  const int i = var >> 6 ? AOMMIN(get_msb(var >> 6), 12) : 0;
Steinar Midtskogen's avatar
Steinar Midtskogen committed
179 180
  /* We use the variance of 8x8 blocks to adjust the strength. */
  return var ? (strength * (4 + i) + 8) >> 4 : 0;
Yaowu Xu's avatar
Yaowu Xu committed
181 182
}

183 184
void copy_8x8_16bit_to_16bit_c(uint16_t *dst, int dstride, const uint16_t *src,
                               int sstride) {
185 186
  int i, j;
  for (i = 0; i < 8; i++)
187
    for (j = 0; j < 8; j++) dst[i * dstride + j] = src[i * sstride + j];
188 189
}

190 191
void copy_4x4_16bit_to_16bit_c(uint16_t *dst, int dstride, const uint16_t *src,
                               int sstride) {
192 193
  int i, j;
  for (i = 0; i < 4; i++)
194
    for (j = 0; j < 4; j++) dst[i * dstride + j] = src[i * sstride + j];
195 196
}

Steinar Midtskogen's avatar
Steinar Midtskogen committed
197 198 199
static void copy_block_16bit_to_16bit(uint16_t *dst, int dstride, uint16_t *src,
                                      cdef_list *dlist, int cdef_count,
                                      int bsize) {
200
  int bi, bx, by;
201

202
  if (bsize == BLOCK_8X8) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
203
    for (bi = 0; bi < cdef_count; bi++) {
204 205
      by = dlist[bi].by;
      bx = dlist[bi].bx;
206
      copy_8x8_16bit_to_16bit(&dst[(by << 3) * dstride + (bx << 3)], dstride,
207 208 209
                              &src[bi << (3 + 3)], 8);
    }
  } else if (bsize == BLOCK_4X8) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
210
    for (bi = 0; bi < cdef_count; bi++) {
211 212 213 214 215 216 217 218
      by = dlist[bi].by;
      bx = dlist[bi].bx;
      copy_4x4_16bit_to_16bit(&dst[(by << 3) * dstride + (bx << 2)], dstride,
                              &src[bi << (3 + 2)], 4);
      copy_4x4_16bit_to_16bit(&dst[((by << 3) + 4) * dstride + (bx << 2)],
                              dstride, &src[(bi << (3 + 2)) + 4 * 4], 4);
    }
  } else if (bsize == BLOCK_8X4) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
219
    for (bi = 0; bi < cdef_count; bi++) {
220 221 222 223 224 225
      by = dlist[bi].by;
      bx = dlist[bi].bx;
      copy_4x4_16bit_to_16bit(&dst[(by << 2) * dstride + (bx << 3)], dstride,
                              &src[bi << (2 + 3)], 8);
      copy_4x4_16bit_to_16bit(&dst[(by << 2) * dstride + (bx << 3) + 4],
                              dstride, &src[(bi << (2 + 3)) + 4], 8);
226 227
    }
  } else {
228
    assert(bsize == BLOCK_4X4);
Steinar Midtskogen's avatar
Steinar Midtskogen committed
229
    for (bi = 0; bi < cdef_count; bi++) {
230 231
      by = dlist[bi].by;
      bx = dlist[bi].bx;
232
      copy_4x4_16bit_to_16bit(&dst[(by << 2) * dstride + (bx << 2)], dstride,
233
                              &src[bi << (2 + 2)], 4);
234 235 236 237
    }
  }
}

238 239
void copy_8x8_16bit_to_8bit_c(uint8_t *dst, int dstride, const uint16_t *src,
                              int sstride) {
240 241 242 243 244 245
  int i, j;
  for (i = 0; i < 8; i++)
    for (j = 0; j < 8; j++)
      dst[i * dstride + j] = (uint8_t)src[i * sstride + j];
}

246 247
void copy_4x4_16bit_to_8bit_c(uint8_t *dst, int dstride, const uint16_t *src,
                              int sstride) {
248 249 250 251 252 253
  int i, j;
  for (i = 0; i < 4; i++)
    for (j = 0; j < 4; j++)
      dst[i * dstride + j] = (uint8_t)src[i * sstride + j];
}

Steinar Midtskogen's avatar
Steinar Midtskogen committed
254 255 256
static void copy_block_16bit_to_8bit(uint8_t *dst, int dstride,
                                     const uint16_t *src, cdef_list *dlist,
                                     int cdef_count, int bsize) {
257
  int bi, bx, by;
258
  if (bsize == BLOCK_8X8) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
259
    for (bi = 0; bi < cdef_count; bi++) {
260 261 262
      by = dlist[bi].by;
      bx = dlist[bi].bx;
      copy_8x8_16bit_to_8bit(&dst[(by << 3) * dstride + (bx << 3)], dstride,
263 264 265
                             &src[bi << (3 + 3)], 8);
    }
  } else if (bsize == BLOCK_4X8) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
266
    for (bi = 0; bi < cdef_count; bi++) {
267 268 269 270 271 272 273 274
      by = dlist[bi].by;
      bx = dlist[bi].bx;
      copy_4x4_16bit_to_8bit(&dst[(by << 3) * dstride + (bx << 2)], dstride,
                             &src[bi << (3 + 2)], 4);
      copy_4x4_16bit_to_8bit(&dst[((by << 3) + 4) * dstride + (bx << 2)],
                             dstride, &src[(bi << (3 + 2)) + 4 * 4], 4);
    }
  } else if (bsize == BLOCK_8X4) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
275
    for (bi = 0; bi < cdef_count; bi++) {
276 277 278 279 280 281
      by = dlist[bi].by;
      bx = dlist[bi].bx;
      copy_4x4_16bit_to_8bit(&dst[(by << 2) * dstride + (bx << 3)], dstride,
                             &src[bi << (2 + 3)], 8);
      copy_4x4_16bit_to_8bit(&dst[(by << 2) * dstride + (bx << 3) + 4], dstride,
                             &src[(bi << (2 + 3)) + 4], 8);
282 283
    }
  } else {
284
    assert(bsize == BLOCK_4X4);
Steinar Midtskogen's avatar
Steinar Midtskogen committed
285
    for (bi = 0; bi < cdef_count; bi++) {
286 287 288
      by = dlist[bi].by;
      bx = dlist[bi].bx;
      copy_4x4_16bit_to_8bit(&dst[(by << 2) * dstride + (bx << 2)], dstride,
289
                             &src[bi << (2 * 2)], 4);
290 291 292 293
    }
  }
}

294 295 296 297 298 299
int get_filter_skip(int level) {
  int filter_skip = level & 1;
  if (level == 1) filter_skip = 0;
  return filter_skip;
}

Steinar Midtskogen's avatar
Steinar Midtskogen committed
300 301 302 303 304 305
void cdef_filter_fb(uint8_t *dst, int dstride, uint16_t *y, uint16_t *in,
                    int xdec, int ydec, int dir[CDEF_NBLOCKS][CDEF_NBLOCKS],
                    int *dirinit, int var[CDEF_NBLOCKS][CDEF_NBLOCKS], int pli,
                    cdef_list *dlist, int cdef_count, int level,
                    int sec_strength, int sec_damping, int pri_damping,
                    int coeff_shift, int skip_dering, int hbd) {
306
  int bi;
Yaowu Xu's avatar
Yaowu Xu committed
307 308
  int bx;
  int by;
309
  int bsize, bsizex, bsizey;
310

311
  int threshold = (level >> 1) << coeff_shift;
312 313
  int filter_skip = get_filter_skip(level);
  if (level == 1) threshold = 31 << coeff_shift;
314

Steinar Midtskogen's avatar
Steinar Midtskogen committed
315 316 317 318
  cdef_direction_func cdef_direction[] = { cdef_direction_4x4,
                                           cdef_direction_8x8 };
  sec_damping += coeff_shift - (pli != AOM_PLANE_Y);
  pri_damping += coeff_shift - (pli != AOM_PLANE_Y);
319 320 321 322 323
  bsize =
      ydec ? (xdec ? BLOCK_4X4 : BLOCK_8X4) : (xdec ? BLOCK_4X8 : BLOCK_8X8);
  bsizex = 3 - xdec;
  bsizey = 3 - ydec;

324 325 326
  if (!skip_dering) {
    if (pli == 0) {
      if (!dirinit || !*dirinit) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
327
        for (bi = 0; bi < cdef_count; bi++) {
328 329
          by = dlist[bi].by;
          bx = dlist[bi].bx;
Steinar Midtskogen's avatar
Steinar Midtskogen committed
330 331
          dir[by][bx] = cdef_find_dir(&in[8 * by * CDEF_BSTRIDE + 8 * bx],
                                      CDEF_BSTRIDE, &var[by][bx], coeff_shift);
332 333 334
        }
        if (dirinit) *dirinit = 1;
      }
335 336 337 338 339 340
    }
    // Only run dering for non-zero threshold (which is always the case for
    // 4:2:2 or 4:4:0). If we don't dering, we still need to eventually write
    // something out in y[] later.
    if (threshold != 0) {
      assert(bsize == BLOCK_8X8 || bsize == BLOCK_4X4);
Steinar Midtskogen's avatar
Steinar Midtskogen committed
341
      for (bi = 0; bi < cdef_count; bi++) {
342
        int t = !filter_skip && dlist[bi].skip ? 0 : threshold;
343 344
        by = dlist[bi].by;
        bx = dlist[bi].bx;
Steinar Midtskogen's avatar
Steinar Midtskogen committed
345
        (cdef_direction[bsize == BLOCK_8X8])(
346
            &y[bi << (bsizex + bsizey)], 1 << bsizex,
Steinar Midtskogen's avatar
Steinar Midtskogen committed
347 348 349
            &in[(by * CDEF_BSTRIDE << bsizey) + (bx << bsizex)],
            pli ? t : adjust_strength(t, var[by][bx]), dir[by][bx],
            pri_damping);
350
      }
Yaowu Xu's avatar
Yaowu Xu committed
351 352
    }
  }
353

Steinar Midtskogen's avatar
Steinar Midtskogen committed
354
  if (sec_strength) {
355
    if (threshold && !skip_dering)
Steinar Midtskogen's avatar
Steinar Midtskogen committed
356 357
      copy_block_16bit_to_16bit(in, CDEF_BSTRIDE, y, dlist, cdef_count, bsize);
    for (bi = 0; bi < cdef_count; bi++) {
358 359
      by = dlist[bi].by;
      bx = dlist[bi].bx;
360 361
      int py = by << bsizey;
      int px = bx << bsizex;
362

363
      if (!filter_skip && dlist[bi].skip) continue;
364 365 366 367
      if (!dst || hbd) {
        // 16 bit destination if high bitdepth or 8 bit destination not given
        (!threshold || (dir[by][bx] < 4 && dir[by][bx]) ? aom_clpf_block_hbd
                                                        : aom_clpf_hblock_hbd)(
368 369
            dst ? (uint16_t *)dst + py * dstride + px
                : &y[bi << (bsizex + bsizey)],
Steinar Midtskogen's avatar
Steinar Midtskogen committed
370 371 372
            in + py * CDEF_BSTRIDE + px, dst && hbd ? dstride : 1 << bsizex,
            CDEF_BSTRIDE, 1 << bsizex, 1 << bsizey, sec_strength << coeff_shift,
            sec_damping);
373 374 375 376
      } else {
        // Do clpf and write the result to an 8 bit destination
        (!threshold || (dir[by][bx] < 4 && dir[by][bx]) ? aom_clpf_block
                                                        : aom_clpf_hblock)(
Steinar Midtskogen's avatar
Steinar Midtskogen committed
377 378 379
            dst + py * dstride + px, in + py * CDEF_BSTRIDE + px, dstride,
            CDEF_BSTRIDE, 1 << bsizex, 1 << bsizey, sec_strength << coeff_shift,
            sec_damping);
380
      }
381
    }
382
  } else if (threshold != 0) {
383
    // No clpf, so copy instead
384
    if (hbd) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
385 386
      copy_block_16bit_to_16bit((uint16_t *)dst, dstride, y, dlist, cdef_count,
                                bsize);
387
    } else {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
388
      copy_block_16bit_to_8bit(dst, dstride, y, dlist, cdef_count, bsize);
389
    }
390 391 392 393
  } else if (dirinit) {
    // If we're here, both dering and clpf are off, and we still haven't written
    // anything to y[] yet, so we just copy the input to y[]. This is necessary
    // only for av1_cdef_search() and only av1_cdef_search() sets dirinit.
Steinar Midtskogen's avatar
Steinar Midtskogen committed
394
    for (bi = 0; bi < cdef_count; bi++) {
395 396 397 398 399 400 401
      by = dlist[bi].by;
      bx = dlist[bi].bx;
      int iy, ix;
      // TODO(stemidts/jmvalin): SIMD optimisations
      for (iy = 0; iy < 1 << bsizey; iy++)
        for (ix = 0; ix < 1 << bsizex; ix++)
          y[(bi << (bsizex + bsizey)) + (iy << bsizex) + ix] =
Steinar Midtskogen's avatar
Steinar Midtskogen committed
402
              in[((by << bsizey) + iy) * CDEF_BSTRIDE + (bx << bsizex) + ix];
403
    }
404
  }
Yaowu Xu's avatar
Yaowu Xu committed
405
}