clpf_simd.h 15.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
/*
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#include "./aom_dsp_rtcd.h"
Steinar Midtskogen's avatar
Steinar Midtskogen committed
13
#include "aom_ports/mem.h"
14

Steinar Midtskogen's avatar
Steinar Midtskogen committed
15
16
17
18
19
20
21
22
23
// delta = 4/16 * clamp(a - o, -s, s) + 1/16 * clamp(b - o, -s, s) +
//         3/16 * clamp(c - o, -s, s) + 3/16 * clamp(d - o, -s, s) +
//         1/16 * clamp(e - o, -s, s) + 4/16 * clamp(f - o, -s, s)
SIMD_INLINE v128 calc_delta(v128 o, v128 a, v128 b, v128 c, v128 d, v128 e,
                            v128 f, v128 sp, v128 sm) {
  // The difference will be 9 bit, offset by 128 so we can use saturated
  // sub to avoid going to 16 bit temporarily before "strength" clipping.
  const v128 c128 = v128_dup_8(128);
  const v128 x = v128_add_8(c128, o);
24
  const v128 c8 = v128_dup_8(8);
Steinar Midtskogen's avatar
Steinar Midtskogen committed
25
26
27
  const v128 tmp = v128_add_8(
      v128_max_s8(v128_min_s8(v128_ssub_s8(v128_add_8(c128, c), x), sp), sm),
      v128_max_s8(v128_min_s8(v128_ssub_s8(v128_add_8(c128, d), x), sp), sm));
28
29
30
  const v128 delta = v128_add_8(
      v128_add_8(
          v128_shl_8(
Steinar Midtskogen's avatar
Steinar Midtskogen committed
31
32
33
34
35
36
37
              v128_add_8(
                  v128_max_s8(
                      v128_min_s8(v128_ssub_s8(v128_add_8(c128, a), x), sp),
                      sm),
                  v128_max_s8(
                      v128_min_s8(v128_ssub_s8(v128_add_8(c128, f), x), sp),
                      sm)),
38
              2),
Steinar Midtskogen's avatar
Steinar Midtskogen committed
39
40
41
42
43
          v128_add_8(
              v128_max_s8(v128_min_s8(v128_ssub_s8(v128_add_8(c128, b), x), sp),
                          sm),
              v128_max_s8(v128_min_s8(v128_ssub_s8(v128_add_8(c128, e), x), sp),
                          sm))),
44
      v128_add_8(v128_add_8(tmp, tmp), tmp));
Steinar Midtskogen's avatar
Steinar Midtskogen committed
45
  return v128_add_8(
46
47
48
49
50
51
      o,
      v128_shr_s8(
          v128_add_8(c8, v128_add_8(delta, v128_cmplt_s8(delta, v128_zero()))),
          4));
}

Steinar Midtskogen's avatar
Steinar Midtskogen committed
52
53
// Process blocks of width 8, two lines at a time, 8 bit.
static void clpf_block8(const uint8_t *src, uint8_t *dst, int sstride,
54
55
56
57
58
59
                        int dstride, int x0, int y0, int sizey,
                        unsigned int strength, BOUNDARY_TYPE bt) {
  const int bottom = bt & TILE_BOTTOM_BOUNDARY ? sizey - 2 : -1;
  const int right = !(bt & TILE_RIGHT_BOUNDARY);
  const int left = !(bt & TILE_LEFT_BOUNDARY);
  const int top = bt & TILE_ABOVE_BOUNDARY ? y0 : -1;
60
61
  const v128 sp = v128_dup_8(strength);
  const v128 sm = v128_dup_8(-(int)strength);
Steinar Midtskogen's avatar
Steinar Midtskogen committed
62
63
64
65
66
67
68
69
70
71
  DECLARE_ALIGNED(16, static const uint64_t,
                  b_shuff[]) = { 0x0504030201000000LL, 0x0d0c0b0a09080808LL };
  DECLARE_ALIGNED(16, static const uint64_t,
                  c_shuff[]) = { 0x0605040302010000LL, 0x0e0d0c0b0a090808LL };
  DECLARE_ALIGNED(16, static const uint64_t,
                  d_shuff[]) = { 0x0707060504030201LL, 0x0f0f0e0d0c0b0a09LL };
  DECLARE_ALIGNED(16, static const uint64_t,
                  e_shuff[]) = { 0x0707070605040302LL, 0x0f0f0f0e0d0c0b0aLL };
  int y;

72
73
  dst += x0 + y0 * dstride;
  src += x0 + y0 * sstride;
74

Steinar Midtskogen's avatar
Steinar Midtskogen committed
75
76
77
78
79
  for (y = 0; y < sizey; y += 2) {
    const v64 l1 = v64_load_aligned(src);
    const v64 l2 = v64_load_aligned(src + sstride);
    v128 o = v128_from_v64(l1, l2);
    const v128 a =
80
        v128_from_v64(v64_load_aligned(src - (y != top) * sstride), l1);
Steinar Midtskogen's avatar
Steinar Midtskogen committed
81
82
    const v128 f = v128_from_v64(
        l2, v64_load_aligned(src + ((y != bottom) + 1) * sstride));
83
    v128 b, c, d, e;
84

85
    if (left) {
86
87
88
89
90
91
92
      b = v128_from_v64(v64_load_unaligned(src - 2),
                        v64_load_unaligned(src - 2 + sstride));
      c = v128_from_v64(v64_load_unaligned(src - 1),
                        v64_load_unaligned(src - 1 + sstride));
    } else {  // Left clipping
      b = v128_shuffle_8(o, v128_load_aligned(b_shuff));
      c = v128_shuffle_8(o, v128_load_aligned(c_shuff));
Steinar Midtskogen's avatar
Steinar Midtskogen committed
93
    }
94
95
96
97
98
99
100
101
    if (right) {
      d = v128_from_v64(v64_load_unaligned(src + 1),
                        v64_load_unaligned(src + 1 + sstride));
      e = v128_from_v64(v64_load_unaligned(src + 2),
                        v64_load_unaligned(src + 2 + sstride));
    } else {  // Right clipping
      d = v128_shuffle_8(o, v128_load_aligned(d_shuff));
      e = v128_shuffle_8(o, v128_load_aligned(e_shuff));
102
    }
103

Steinar Midtskogen's avatar
Steinar Midtskogen committed
104
105
106
107
108
109
110
111
112
113
    o = calc_delta(o, a, b, c, d, e, f, sp, sm);
    v64_store_aligned(dst, v128_high_v64(o));
    v64_store_aligned(dst + dstride, v128_low_v64(o));
    src += sstride * 2;
    dst += dstride * 2;
  }
}

// Process blocks of width 4, four lines at a time, 8 bit.
static void clpf_block4(const uint8_t *src, uint8_t *dst, int sstride,
114
115
                        int dstride, int x0, int y0, int sizey,
                        unsigned int strength, BOUNDARY_TYPE bt) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
116
117
  const v128 sp = v128_dup_8(strength);
  const v128 sm = v128_dup_8(-(int)strength);
118
119
120
121
122
  const int right = !(bt & TILE_RIGHT_BOUNDARY);
  const int bottom = bt & TILE_BOTTOM_BOUNDARY ? sizey - 4 : -1;
  const int left = !(bt & TILE_LEFT_BOUNDARY);
  const int top = bt & TILE_ABOVE_BOUNDARY ? y0 : -1;

Steinar Midtskogen's avatar
Steinar Midtskogen committed
123
124
125
126
127
128
129
130
131
132
133
134
135
136
  DECLARE_ALIGNED(16, static const uint64_t,
                  b_shuff[]) = { 0x0504040401000000LL, 0x0d0c0c0c09080808LL };
  DECLARE_ALIGNED(16, static const uint64_t,
                  c_shuff[]) = { 0x0605040402010000LL, 0x0e0d0c0c0a090808LL };
  DECLARE_ALIGNED(16, static const uint64_t,
                  d_shuff[]) = { 0x0707060503030201LL, 0x0f0f0e0d0b0b0a09LL };
  DECLARE_ALIGNED(16, static const uint64_t,
                  e_shuff[]) = { 0x0707070603030302LL, 0x0f0f0f0e0b0b0b0aLL };
  int y;

  dst += x0 + y0 * dstride;
  src += x0 + y0 * sstride;

  for (y = 0; y < sizey; y += 4) {
137
    const uint32_t l0 = u32_load_aligned(src - (y != top) * sstride);
Steinar Midtskogen's avatar
Steinar Midtskogen committed
138
139
140
141
142
143
144
145
    const uint32_t l1 = u32_load_aligned(src);
    const uint32_t l2 = u32_load_aligned(src + sstride);
    const uint32_t l3 = u32_load_aligned(src + 2 * sstride);
    const uint32_t l4 = u32_load_aligned(src + 3 * sstride);
    const uint32_t l5 = u32_load_aligned(src + ((y != bottom) + 3) * sstride);
    v128 o = v128_from_32(l1, l2, l3, l4);
    const v128 a = v128_from_32(l0, l1, l2, l3);
    const v128 f = v128_from_32(l2, l3, l4, l5);
146
    v128 b, c, d, e;
Steinar Midtskogen's avatar
Steinar Midtskogen committed
147

148
    if (left) {
149
150
151
152
153
154
155
156
157
158
159
      b = v128_from_32(u32_load_unaligned(src - 2),
                       u32_load_unaligned(src + sstride - 2),
                       u32_load_unaligned(src + 2 * sstride - 2),
                       u32_load_unaligned(src + 3 * sstride - 2));
      c = v128_from_32(u32_load_unaligned(src - 1),
                       u32_load_unaligned(src + sstride - 1),
                       u32_load_unaligned(src + 2 * sstride - 1),
                       u32_load_unaligned(src + 3 * sstride - 1));
    } else {  // Left clipping
      b = v128_shuffle_8(o, v128_load_aligned(b_shuff));
      c = v128_shuffle_8(o, v128_load_aligned(c_shuff));
160
    }
161
162
163
164
165
166
167
168
169
170
171
172
    if (right) {
      d = v128_from_32(u32_load_unaligned(src + 1),
                       u32_load_unaligned(src + sstride + 1),
                       u32_load_unaligned(src + 2 * sstride + 1),
                       u32_load_unaligned(src + 3 * sstride + 1));
      e = v128_from_32(u32_load_unaligned(src + 2 * !!right),
                       u32_load_unaligned(src + sstride + 2),
                       u32_load_unaligned(src + 2 * sstride + 2),
                       u32_load_unaligned(src + 3 * sstride + 2));
    } else {  // Right clipping
      d = v128_shuffle_8(o, v128_load_aligned(d_shuff));
      e = v128_shuffle_8(o, v128_load_aligned(e_shuff));
173
    }
Steinar Midtskogen's avatar
Steinar Midtskogen committed
174
175
176
177
178
179
180
181
182

    o = calc_delta(o, a, b, c, d, e, f, sp, sm);
    u32_store_aligned(dst, v128_low_u32(v128_shr_n_byte(o, 12)));
    u32_store_aligned(dst + dstride, v128_low_u32(v128_shr_n_byte(o, 8)));
    u32_store_aligned(dst + 2 * dstride, v128_low_u32(v128_shr_n_byte(o, 4)));
    u32_store_aligned(dst + 3 * dstride, v128_low_u32(o));

    dst += 4 * dstride;
    src += 4 * sstride;
183
184
185
  }
}

186
187
void SIMD_FUNC(aom_clpf_block)(const uint8_t *src, uint8_t *dst, int sstride,
                               int dstride, int x0, int y0, int sizex,
188
189
                               int sizey, unsigned int strength,
                               BOUNDARY_TYPE bt) {
190
191
192
193
  if ((sizex != 4 && sizex != 8) || ((sizey & 3) && sizex == 4)) {
    // Fallback to C for odd sizes:
    // * block widths not 4 or 8
    // * block heights not a multiple of 4 if the block width is 4
194
195
    aom_clpf_block_c(src, dst, sstride, dstride, x0, y0, sizex, sizey, strength,
                     bt);
196
  } else {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
197
    (sizex == 4 ? clpf_block4 : clpf_block8)(src, dst, sstride, dstride, x0, y0,
198
                                             sizey, strength, bt);
199
200
  }
}
201
202

#if CONFIG_AOM_HIGHBITDEPTH
Steinar Midtskogen's avatar
Steinar Midtskogen committed
203
204
205
206
207
// delta = 4/16 * clamp(a - o, -s, s) + 1/16 * clamp(b - o, -s, s) +
//         3/16 * clamp(c - o, -s, s) + 3/16 * clamp(d - o, -s, s) +
//         1/16 * clamp(e - o, -s, s) + 4/16 * clamp(f - o, -s, s)
SIMD_INLINE v128 calc_delta_hbd(v128 o, v128 a, v128 b, v128 c, v128 d, v128 e,
                                v128 f, v128 sp, v128 sm) {
208
209
210
211
212
213
214
215
216
217
218
219
220
221
  const v128 c8 = v128_dup_16(8);
  const v128 tmp =
      v128_add_16(v128_max_s16(v128_min_s16(v128_sub_16(c, o), sp), sm),
                  v128_max_s16(v128_min_s16(v128_sub_16(d, o), sp), sm));
  const v128 delta = v128_add_16(
      v128_add_16(
          v128_shl_16(
              v128_add_16(
                  v128_max_s16(v128_min_s16(v128_sub_16(a, o), sp), sm),
                  v128_max_s16(v128_min_s16(v128_sub_16(f, o), sp), sm)),
              2),
          v128_add_16(v128_max_s16(v128_min_s16(v128_sub_16(b, o), sp), sm),
                      v128_max_s16(v128_min_s16(v128_sub_16(e, o), sp), sm))),
      v128_add_16(v128_add_16(tmp, tmp), tmp));
Steinar Midtskogen's avatar
Steinar Midtskogen committed
222
223
224
225
226
  return v128_add_16(
      o, v128_shr_s16(
             v128_add_16(
                 c8, v128_add_16(delta, v128_cmplt_s16(delta, v128_zero()))),
             4));
227
228
}

Steinar Midtskogen's avatar
Steinar Midtskogen committed
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
static void calc_delta_hbd4(v128 o, v128 a, v128 b, v128 c, v128 d, v128 e,
                            v128 f, uint16_t *dst, v128 sp, v128 sm,
                            int dstride) {
  o = calc_delta_hbd(o, a, b, c, d, e, f, sp, sm);
  v64_store_aligned(dst, v128_high_v64(o));
  v64_store_aligned(dst + dstride, v128_low_v64(o));
}

static void calc_delta_hbd8(v128 o, v128 a, v128 b, v128 c, v128 d, v128 e,
                            v128 f, uint16_t *dst, v128 sp, v128 sm) {
  v128_store_aligned(dst, calc_delta_hbd(o, a, b, c, d, e, f, sp, sm));
}

// Process blocks of width 4, two lines at time.
SIMD_INLINE void clpf_block_hbd4(const uint16_t *src, uint16_t *dst,
                                 int sstride, int dstride, int x0, int y0,
245
246
                                 int sizey, unsigned int strength,
                                 BOUNDARY_TYPE bt) {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
247
248
  const v128 sp = v128_dup_16(strength);
  const v128 sm = v128_dup_16(-(int)strength);
249
250
251
252
253
  const int right = !(bt & TILE_RIGHT_BOUNDARY);
  const int bottom = bt & TILE_BOTTOM_BOUNDARY ? sizey - 2 : -1;
  const int left = !(bt & TILE_LEFT_BOUNDARY);
  const int top = bt & TILE_ABOVE_BOUNDARY ? y0 : -1;

Steinar Midtskogen's avatar
Steinar Midtskogen committed
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
  DECLARE_ALIGNED(16, static const uint64_t,
                  b_shuff[]) = { 0x0302010001000100LL, 0x0b0a090809080908LL };
  DECLARE_ALIGNED(16, static const uint64_t,
                  c_shuff[]) = { 0x0504030201000100LL, 0x0d0c0b0a09080908LL };
  DECLARE_ALIGNED(16, static const uint64_t,
                  d_shuff[]) = { 0x0706070605040302LL, 0x0f0e0f0e0d0c0b0aLL };
  DECLARE_ALIGNED(16, static const uint64_t,
                  e_shuff[]) = { 0x0706070607060504LL, 0x0f0e0f0e0f0e0d0cLL };
  int y;

  dst += x0 + y0 * dstride;
  src += x0 + y0 * sstride;

  for (y = 0; y < sizey; y += 2) {
    const v64 l1 = v64_load_aligned(src);
    const v64 l2 = v64_load_aligned(src + sstride);
    v128 o = v128_from_v64(l1, l2);
    const v128 a =
272
        v128_from_v64(v64_load_aligned(src - (y != top) * sstride), l1);
Steinar Midtskogen's avatar
Steinar Midtskogen committed
273
274
    const v128 f = v128_from_v64(
        l2, v64_load_aligned(src + ((y != bottom) + 1) * sstride));
275
    v128 b, c, d, e;
Steinar Midtskogen's avatar
Steinar Midtskogen committed
276

277
    if (left) {
278
279
280
281
282
283
284
      b = v128_from_v64(v64_load_unaligned(src - 2),
                        v64_load_unaligned(src - 2 + sstride));
      c = v128_from_v64(v64_load_unaligned(src - 1),
                        v64_load_unaligned(src - 1 + sstride));
    } else {  // Left clipping
      b = v128_shuffle_8(o, v128_load_aligned(b_shuff));
      c = v128_shuffle_8(o, v128_load_aligned(c_shuff));
Steinar Midtskogen's avatar
Steinar Midtskogen committed
285
    }
286
287
288
289
290
291
292
293
    if (right) {
      d = v128_from_v64(v64_load_unaligned(src + 1),
                        v64_load_unaligned(src + 1 + sstride));
      e = v128_from_v64(v64_load_unaligned(src + 2),
                        v64_load_unaligned(src + 2 + sstride));
    } else {  // Right clipping
      d = v128_shuffle_8(o, v128_load_aligned(d_shuff));
      e = v128_shuffle_8(o, v128_load_aligned(e_shuff));
Steinar Midtskogen's avatar
Steinar Midtskogen committed
294
295
296
297
298
299
300
301
    }
    calc_delta_hbd4(o, a, b, c, d, e, f, dst, sp, sm, dstride);
    src += sstride * 2;
    dst += dstride * 2;
  }
}

// The most simple case.  Start here if you need to understand the functions.
302
303
SIMD_INLINE void clpf_block_hbd(const uint16_t *src, uint16_t *dst, int sstride,
                                int dstride, int x0, int y0, int sizey,
304
                                unsigned int strength, BOUNDARY_TYPE bt) {
305
306
  const v128 sp = v128_dup_16(strength);
  const v128 sm = v128_dup_16(-(int)strength);
307
308
309
310
311
  const int right = !(bt & TILE_RIGHT_BOUNDARY);
  const int bottom = bt & TILE_BOTTOM_BOUNDARY ? sizey - 2 : -2;
  const int left = !(bt & TILE_LEFT_BOUNDARY);
  const int top = bt & TILE_ABOVE_BOUNDARY ? y0 : -1;

Steinar Midtskogen's avatar
Steinar Midtskogen committed
312
313
314
315
316
317
318
319
320
  DECLARE_ALIGNED(16, static const uint64_t,
                  b_shuff[]) = { 0x0302010001000100LL, 0x0b0a090807060504LL };
  DECLARE_ALIGNED(16, static const uint64_t,
                  c_shuff[]) = { 0x0504030201000100LL, 0x0d0c0b0a09080706LL };
  DECLARE_ALIGNED(16, static const uint64_t,
                  d_shuff[]) = { 0x0908070605040302LL, 0x0f0e0f0e0d0c0b0aLL };
  DECLARE_ALIGNED(16, static const uint64_t,
                  e_shuff[]) = { 0x0b0a090807060504LL, 0x0f0e0f0e0f0e0d0cLL };
  int y;
321
322
323
324

  dst += x0 + y0 * dstride;
  src += x0 + y0 * sstride;

Steinar Midtskogen's avatar
Steinar Midtskogen committed
325
326
327
328
329
330
  // Read 8 set of pixels at a time.  Clipping along upper and lower
  // edges is handled by reading the upper or lower line twice.
  // Clipping along the left and right edges is handled by shuffle
  // instructions doing shift and pad.
  for (y = 0; y < sizey; y++) {
    const v128 o = v128_load_aligned(src);
331
    const v128 a = v128_load_aligned(src - (y != top) * sstride);
Steinar Midtskogen's avatar
Steinar Midtskogen committed
332
    const v128 f = v128_load_aligned(src + (y - 1 != bottom) * sstride);
333
    v128 b, c, d, e;
Steinar Midtskogen's avatar
Steinar Midtskogen committed
334

335
    if (left) {
336
337
338
339
340
      b = v128_load_unaligned(src - 2);
      c = v128_load_unaligned(src - 1);
    } else {  // Left clipping
      b = v128_shuffle_8(o, v128_load_aligned(b_shuff));
      c = v128_shuffle_8(o, v128_load_aligned(c_shuff));
341
    }
342
343
344
345
346
347
    if (right) {
      d = v128_load_unaligned(src + 1);
      e = v128_load_unaligned(src + 2);
    } else {  // Right clipping
      d = v128_shuffle_8(o, v128_load_aligned(d_shuff));
      e = v128_shuffle_8(o, v128_load_aligned(e_shuff));
348
    }
Steinar Midtskogen's avatar
Steinar Midtskogen committed
349
350
351
    calc_delta_hbd8(o, a, b, c, d, e, f, dst, sp, sm);
    src += sstride;
    dst += dstride;
352
353
354
355
356
  }
}

void SIMD_FUNC(aom_clpf_block_hbd)(const uint16_t *src, uint16_t *dst,
                                   int sstride, int dstride, int x0, int y0,
357
358
                                   int sizex, int sizey, unsigned int strength,
                                   BOUNDARY_TYPE bt) {
359
360
361
362
  if ((sizex != 4 && sizex != 8) || ((sizey & 1) && sizex == 4)) {
    // Fallback to C for odd sizes:
    // * block width not 4 or 8
    // * block heights not a multiple of 2 if the block width is 4
363
    aom_clpf_block_hbd_c(src, dst, sstride, dstride, x0, y0, sizex, sizey,
364
                         strength, bt);
365
  } else {
Steinar Midtskogen's avatar
Steinar Midtskogen committed
366
    (sizex == 4 ? clpf_block_hbd4 : clpf_block_hbd)(
367
        src, dst, sstride, dstride, x0, y0, sizey, strength, bt);
368
369
370
  }
}
#endif