aom_convolve.c 34.7 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
John Koleszar's avatar
John Koleszar committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
John Koleszar's avatar
John Koleszar committed
10
 */
Christian Duvivier's avatar
Christian Duvivier committed
11

John Koleszar's avatar
John Koleszar committed
12
#include <assert.h>
Zoe Liu's avatar
Zoe Liu committed
13
#include <string.h>
John Koleszar's avatar
John Koleszar committed
14

Yaowu Xu's avatar
Yaowu Xu committed
15 16 17 18 19 20
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
#include "aom/aom_integer.h"
#include "aom_dsp/aom_convolve.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/aom_filter.h"
21
#include "aom_ports/mem.h"
John Koleszar's avatar
John Koleszar committed
22

Dmitry Kovalev's avatar
Dmitry Kovalev committed
23 24
static void convolve_horiz(const uint8_t *src, ptrdiff_t src_stride,
                           uint8_t *dst, ptrdiff_t dst_stride,
clang-format's avatar
clang-format committed
25 26
                           const InterpKernel *x_filters, int x0_q4,
                           int x_step_q4, int w, int h) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
27 28
  int x, y;
  src -= SUBPEL_TAPS / 2 - 1;
John Koleszar's avatar
John Koleszar committed
29
  for (y = 0; y < h; ++y) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
30
    int x_q4 = x0_q4;
John Koleszar's avatar
John Koleszar committed
31
    for (x = 0; x < w; ++x) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
32 33 34
      const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS];
      const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK];
      int k, sum = 0;
clang-format's avatar
clang-format committed
35
      for (k = 0; k < SUBPEL_TAPS; ++k) sum += src_x[k] * x_filter[k];
36
      dst[x] = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS));
John Koleszar's avatar
John Koleszar committed
37 38 39 40 41 42 43
      x_q4 += x_step_q4;
    }
    src += src_stride;
    dst += dst_stride;
  }
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
44 45
static void convolve_avg_horiz(const uint8_t *src, ptrdiff_t src_stride,
                               uint8_t *dst, ptrdiff_t dst_stride,
clang-format's avatar
clang-format committed
46 47
                               const InterpKernel *x_filters, int x0_q4,
                               int x_step_q4, int w, int h) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
48 49
  int x, y;
  src -= SUBPEL_TAPS / 2 - 1;
John Koleszar's avatar
John Koleszar committed
50
  for (y = 0; y < h; ++y) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
51
    int x_q4 = x0_q4;
John Koleszar's avatar
John Koleszar committed
52
    for (x = 0; x < w; ++x) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
53 54 55
      const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS];
      const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK];
      int k, sum = 0;
clang-format's avatar
clang-format committed
56 57 58
      for (k = 0; k < SUBPEL_TAPS; ++k) sum += src_x[k] * x_filter[k];
      dst[x] = ROUND_POWER_OF_TWO(
          dst[x] + clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS)), 1);
John Koleszar's avatar
John Koleszar committed
59 60 61 62 63 64 65
      x_q4 += x_step_q4;
    }
    src += src_stride;
    dst += dst_stride;
  }
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
66 67
static void convolve_vert(const uint8_t *src, ptrdiff_t src_stride,
                          uint8_t *dst, ptrdiff_t dst_stride,
clang-format's avatar
clang-format committed
68 69
                          const InterpKernel *y_filters, int y0_q4,
                          int y_step_q4, int w, int h) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
70 71
  int x, y;
  src -= src_stride * (SUBPEL_TAPS / 2 - 1);
John Koleszar's avatar
John Koleszar committed
72

73
  for (x = 0; x < w; ++x) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
74
    int y_q4 = y0_q4;
John Koleszar's avatar
John Koleszar committed
75
    for (y = 0; y < h; ++y) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
76 77 78 79 80 81
      const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
      const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK];
      int k, sum = 0;
      for (k = 0; k < SUBPEL_TAPS; ++k)
        sum += src_y[k * src_stride] * y_filter[k];
      dst[y * dst_stride] = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS));
John Koleszar's avatar
John Koleszar committed
82 83 84 85 86 87 88
      y_q4 += y_step_q4;
    }
    ++src;
    ++dst;
  }
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
89 90
static void convolve_avg_vert(const uint8_t *src, ptrdiff_t src_stride,
                              uint8_t *dst, ptrdiff_t dst_stride,
clang-format's avatar
clang-format committed
91 92
                              const InterpKernel *y_filters, int y0_q4,
                              int y_step_q4, int w, int h) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
93 94
  int x, y;
  src -= src_stride * (SUBPEL_TAPS / 2 - 1);
John Koleszar's avatar
John Koleszar committed
95

96
  for (x = 0; x < w; ++x) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
97
    int y_q4 = y0_q4;
John Koleszar's avatar
John Koleszar committed
98
    for (y = 0; y < h; ++y) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
99 100 101 102 103
      const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
      const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK];
      int k, sum = 0;
      for (k = 0; k < SUBPEL_TAPS; ++k)
        sum += src_y[k * src_stride] * y_filter[k];
clang-format's avatar
clang-format committed
104 105 106 107
      dst[y * dst_stride] = ROUND_POWER_OF_TWO(
          dst[y * dst_stride] +
              clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS)),
          1);
John Koleszar's avatar
John Koleszar committed
108 109 110 111 112 113 114
      y_q4 += y_step_q4;
    }
    ++src;
    ++dst;
  }
}

clang-format's avatar
clang-format committed
115 116
static void convolve(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
                     ptrdiff_t dst_stride, const InterpKernel *const x_filters,
Dmitry Kovalev's avatar
Dmitry Kovalev committed
117
                     int x0_q4, int x_step_q4,
clang-format's avatar
clang-format committed
118 119
                     const InterpKernel *const y_filters, int y0_q4,
                     int y_step_q4, int w, int h) {
120 121 122 123 124 125 126 127 128 129 130 131
  // Note: Fixed size intermediate buffer, temp, places limits on parameters.
  // 2d filtering proceeds in 2 steps:
  //   (1) Interpolate horizontally into an intermediate buffer, temp.
  //   (2) Interpolate temp vertically to derive the sub-pixel result.
  // Deriving the maximum number of rows in the temp buffer (135):
  // --Smallest scaling factor is x1/2 ==> y_step_q4 = 32 (Normative).
  // --Largest block size is 64x64 pixels.
  // --64 rows in the downscaled frame span a distance of (64 - 1) * 32 in the
  //   original frame (in 1/16th pixel units).
  // --Must round-up because block may be located at sub-pixel position.
  // --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails.
  // --((64 - 1) * 32 + 15) >> 4 + 8 = 135.
132
  uint8_t temp[MAX_EXT_SIZE * MAX_SB_SIZE];
133
  int intermediate_height =
clang-format's avatar
clang-format committed
134
      (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS;
135

136 137
  assert(w <= MAX_SB_SIZE);
  assert(h <= MAX_SB_SIZE);
138

139 140
  assert(y_step_q4 <= 32);
  assert(x_step_q4 <= 32);
141

clang-format's avatar
clang-format committed
142 143 144 145 146
  convolve_horiz(src - src_stride * (SUBPEL_TAPS / 2 - 1), src_stride, temp,
                 MAX_SB_SIZE, x_filters, x0_q4, x_step_q4, w,
                 intermediate_height);
  convolve_vert(temp + MAX_SB_SIZE * (SUBPEL_TAPS / 2 - 1), MAX_SB_SIZE, dst,
                dst_stride, y_filters, y0_q4, y_step_q4, w, h);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
147 148
}

149
static const InterpKernel *get_filter_base(const int16_t *filter) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
150 151
  // NOTE: This assumes that the filter table is 256-byte aligned.
  // TODO(agrange) Modify to make independent of table alignment.
152
  return (const InterpKernel *)(((intptr_t)filter) & ~((intptr_t)0xFF));
Dmitry Kovalev's avatar
Dmitry Kovalev committed
153 154
}

155
static int get_filter_offset(const int16_t *f, const InterpKernel *base) {
156
  return (int)((const InterpKernel *)(intptr_t)f - base);
John Koleszar's avatar
John Koleszar committed
157 158
}

Yaowu Xu's avatar
Yaowu Xu committed
159
void aom_convolve8_horiz_c(const uint8_t *src, ptrdiff_t src_stride,
160
                           uint8_t *dst, ptrdiff_t dst_stride,
John Koleszar's avatar
John Koleszar committed
161
                           const int16_t *filter_x, int x_step_q4,
clang-format's avatar
clang-format committed
162 163
                           const int16_t *filter_y, int y_step_q4, int w,
                           int h) {
164
  const InterpKernel *const filters_x = get_filter_base(filter_x);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
165 166
  const int x0_q4 = get_filter_offset(filter_x, filters_x);

167 168 169
  (void)filter_y;
  (void)y_step_q4;

clang-format's avatar
clang-format committed
170 171
  convolve_horiz(src, src_stride, dst, dst_stride, filters_x, x0_q4, x_step_q4,
                 w, h);
John Koleszar's avatar
John Koleszar committed
172 173
}

Yaowu Xu's avatar
Yaowu Xu committed
174
void aom_convolve8_avg_horiz_c(const uint8_t *src, ptrdiff_t src_stride,
175
                               uint8_t *dst, ptrdiff_t dst_stride,
John Koleszar's avatar
John Koleszar committed
176
                               const int16_t *filter_x, int x_step_q4,
clang-format's avatar
clang-format committed
177 178
                               const int16_t *filter_y, int y_step_q4, int w,
                               int h) {
179
  const InterpKernel *const filters_x = get_filter_base(filter_x);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
180 181
  const int x0_q4 = get_filter_offset(filter_x, filters_x);

182 183 184
  (void)filter_y;
  (void)y_step_q4;

clang-format's avatar
clang-format committed
185 186
  convolve_avg_horiz(src, src_stride, dst, dst_stride, filters_x, x0_q4,
                     x_step_q4, w, h);
John Koleszar's avatar
John Koleszar committed
187 188
}

Yaowu Xu's avatar
Yaowu Xu committed
189
void aom_convolve8_vert_c(const uint8_t *src, ptrdiff_t src_stride,
190
                          uint8_t *dst, ptrdiff_t dst_stride,
John Koleszar's avatar
John Koleszar committed
191
                          const int16_t *filter_x, int x_step_q4,
clang-format's avatar
clang-format committed
192 193
                          const int16_t *filter_y, int y_step_q4, int w,
                          int h) {
194
  const InterpKernel *const filters_y = get_filter_base(filter_y);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
195
  const int y0_q4 = get_filter_offset(filter_y, filters_y);
196 197 198 199

  (void)filter_x;
  (void)x_step_q4;

clang-format's avatar
clang-format committed
200 201
  convolve_vert(src, src_stride, dst, dst_stride, filters_y, y0_q4, y_step_q4,
                w, h);
John Koleszar's avatar
John Koleszar committed
202 203
}

Yaowu Xu's avatar
Yaowu Xu committed
204
void aom_convolve8_avg_vert_c(const uint8_t *src, ptrdiff_t src_stride,
205
                              uint8_t *dst, ptrdiff_t dst_stride,
John Koleszar's avatar
John Koleszar committed
206
                              const int16_t *filter_x, int x_step_q4,
clang-format's avatar
clang-format committed
207 208
                              const int16_t *filter_y, int y_step_q4, int w,
                              int h) {
209
  const InterpKernel *const filters_y = get_filter_base(filter_y);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
210
  const int y0_q4 = get_filter_offset(filter_y, filters_y);
211 212 213 214

  (void)filter_x;
  (void)x_step_q4;

clang-format's avatar
clang-format committed
215 216
  convolve_avg_vert(src, src_stride, dst, dst_stride, filters_y, y0_q4,
                    y_step_q4, w, h);
John Koleszar's avatar
John Koleszar committed
217 218
}

Yaowu Xu's avatar
Yaowu Xu committed
219
void aom_convolve8_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
clang-format's avatar
clang-format committed
220 221
                     ptrdiff_t dst_stride, const int16_t *filter_x,
                     int x_step_q4, const int16_t *filter_y, int y_step_q4,
John Koleszar's avatar
John Koleszar committed
222
                     int w, int h) {
223
  const InterpKernel *const filters_x = get_filter_base(filter_x);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
224 225
  const int x0_q4 = get_filter_offset(filter_x, filters_x);

226
  const InterpKernel *const filters_y = get_filter_base(filter_y);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
227 228
  const int y0_q4 = get_filter_offset(filter_y, filters_y);

clang-format's avatar
clang-format committed
229
  convolve(src, src_stride, dst, dst_stride, filters_x, x0_q4, x_step_q4,
Dmitry Kovalev's avatar
Dmitry Kovalev committed
230
           filters_y, y0_q4, y_step_q4, w, h);
John Koleszar's avatar
John Koleszar committed
231 232
}

Yaowu Xu's avatar
Yaowu Xu committed
233
void aom_convolve8_avg_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
clang-format's avatar
clang-format committed
234 235
                         ptrdiff_t dst_stride, const int16_t *filter_x,
                         int x_step_q4, const int16_t *filter_y, int y_step_q4,
John Koleszar's avatar
John Koleszar committed
236
                         int w, int h) {
Christian Duvivier's avatar
Christian Duvivier committed
237
  /* Fixed size intermediate buffer places limits on parameters. */
238 239 240
  DECLARE_ALIGNED(16, uint8_t, temp[MAX_SB_SIZE * MAX_SB_SIZE]);
  assert(w <= MAX_SB_SIZE);
  assert(h <= MAX_SB_SIZE);
Christian Duvivier's avatar
Christian Duvivier committed
241

Yaowu Xu's avatar
Yaowu Xu committed
242
  aom_convolve8_c(src, src_stride, temp, MAX_SB_SIZE, filter_x, x_step_q4,
clang-format's avatar
clang-format committed
243
                  filter_y, y_step_q4, w, h);
Yaowu Xu's avatar
Yaowu Xu committed
244
  aom_convolve_avg_c(temp, MAX_SB_SIZE, dst, dst_stride, NULL, 0, NULL, 0, w,
clang-format's avatar
clang-format committed
245
                     h);
John Koleszar's avatar
John Koleszar committed
246
}
247

Yaowu Xu's avatar
Yaowu Xu committed
248
void aom_convolve_copy_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
clang-format's avatar
clang-format committed
249 250 251
                         ptrdiff_t dst_stride, const int16_t *filter_x,
                         int filter_x_stride, const int16_t *filter_y,
                         int filter_y_stride, int w, int h) {
252 253
  int r;

clang-format's avatar
clang-format committed
254 255 256 257
  (void)filter_x;
  (void)filter_x_stride;
  (void)filter_y;
  (void)filter_y_stride;
258

259
  for (r = h; r > 0; --r) {
James Zern's avatar
James Zern committed
260
    memcpy(dst, src, w);
261 262
    src += src_stride;
    dst += dst_stride;
263 264 265
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
266
void aom_convolve_avg_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
clang-format's avatar
clang-format committed
267 268 269
                        ptrdiff_t dst_stride, const int16_t *filter_x,
                        int filter_x_stride, const int16_t *filter_y,
                        int filter_y_stride, int w, int h) {
270 271
  int x, y;

clang-format's avatar
clang-format committed
272 273 274 275
  (void)filter_x;
  (void)filter_x_stride;
  (void)filter_y;
  (void)filter_y_stride;
276

277
  for (y = 0; y < h; ++y) {
clang-format's avatar
clang-format committed
278
    for (x = 0; x < w; ++x) dst[x] = ROUND_POWER_OF_TWO(dst[x] + src[x], 1);
279

280 281 282 283
    src += src_stride;
    dst += dst_stride;
  }
}
284

Yaowu Xu's avatar
Yaowu Xu committed
285
void aom_scaled_horiz_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
clang-format's avatar
clang-format committed
286 287
                        ptrdiff_t dst_stride, const int16_t *filter_x,
                        int x_step_q4, const int16_t *filter_y, int y_step_q4,
288
                        int w, int h) {
Yaowu Xu's avatar
Yaowu Xu committed
289
  aom_convolve8_horiz_c(src, src_stride, dst, dst_stride, filter_x, x_step_q4,
290 291 292
                        filter_y, y_step_q4, w, h);
}

Yaowu Xu's avatar
Yaowu Xu committed
293
void aom_scaled_vert_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
clang-format's avatar
clang-format committed
294 295
                       ptrdiff_t dst_stride, const int16_t *filter_x,
                       int x_step_q4, const int16_t *filter_y, int y_step_q4,
296
                       int w, int h) {
Yaowu Xu's avatar
Yaowu Xu committed
297
  aom_convolve8_vert_c(src, src_stride, dst, dst_stride, filter_x, x_step_q4,
298 299 300
                       filter_y, y_step_q4, w, h);
}

Yaowu Xu's avatar
Yaowu Xu committed
301
void aom_scaled_2d_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
clang-format's avatar
clang-format committed
302 303
                     ptrdiff_t dst_stride, const int16_t *filter_x,
                     int x_step_q4, const int16_t *filter_y, int y_step_q4,
304
                     int w, int h) {
Yaowu Xu's avatar
Yaowu Xu committed
305
  aom_convolve8_c(src, src_stride, dst, dst_stride, filter_x, x_step_q4,
306 307 308
                  filter_y, y_step_q4, w, h);
}

Yaowu Xu's avatar
Yaowu Xu committed
309
void aom_scaled_avg_horiz_c(const uint8_t *src, ptrdiff_t src_stride,
310 311
                            uint8_t *dst, ptrdiff_t dst_stride,
                            const int16_t *filter_x, int x_step_q4,
clang-format's avatar
clang-format committed
312 313
                            const int16_t *filter_y, int y_step_q4, int w,
                            int h) {
Yaowu Xu's avatar
Yaowu Xu committed
314
  aom_convolve8_avg_horiz_c(src, src_stride, dst, dst_stride, filter_x,
315 316 317
                            x_step_q4, filter_y, y_step_q4, w, h);
}

Yaowu Xu's avatar
Yaowu Xu committed
318
void aom_scaled_avg_vert_c(const uint8_t *src, ptrdiff_t src_stride,
319 320
                           uint8_t *dst, ptrdiff_t dst_stride,
                           const int16_t *filter_x, int x_step_q4,
clang-format's avatar
clang-format committed
321 322
                           const int16_t *filter_y, int y_step_q4, int w,
                           int h) {
Yaowu Xu's avatar
Yaowu Xu committed
323
  aom_convolve8_avg_vert_c(src, src_stride, dst, dst_stride, filter_x,
324 325 326
                           x_step_q4, filter_y, y_step_q4, w, h);
}

Yaowu Xu's avatar
Yaowu Xu committed
327
void aom_scaled_avg_2d_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
clang-format's avatar
clang-format committed
328 329 330
                         ptrdiff_t dst_stride, const int16_t *filter_x,
                         int x_step_q4, const int16_t *filter_y, int y_step_q4,
                         int w, int h) {
Yaowu Xu's avatar
Yaowu Xu committed
331
  aom_convolve8_avg_c(src, src_stride, dst, dst_stride, filter_x, x_step_q4,
332 333 334
                      filter_y, y_step_q4, w, h);
}

335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450
#if CONFIG_LOOP_RESTORATION
static void convolve_add_src_horiz(const uint8_t *src, ptrdiff_t src_stride,
                                   uint8_t *dst, ptrdiff_t dst_stride,
                                   const InterpKernel *x_filters, int x0_q4,
                                   int x_step_q4, int w, int h) {
  int x, y;
  src -= SUBPEL_TAPS / 2 - 1;
  for (y = 0; y < h; ++y) {
    int x_q4 = x0_q4;
    for (x = 0; x < w; ++x) {
      const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS];
      const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK];
      int k, sum = 0;
      for (k = 0; k < SUBPEL_TAPS; ++k) sum += src_x[k] * x_filter[k];
      dst[x] = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS) +
                          src_x[SUBPEL_TAPS / 2 - 1]);
      x_q4 += x_step_q4;
    }
    src += src_stride;
    dst += dst_stride;
  }
}

static void convolve_add_src_vert(const uint8_t *src, ptrdiff_t src_stride,
                                  uint8_t *dst, ptrdiff_t dst_stride,
                                  const InterpKernel *y_filters, int y0_q4,
                                  int y_step_q4, int w, int h) {
  int x, y;
  src -= src_stride * (SUBPEL_TAPS / 2 - 1);

  for (x = 0; x < w; ++x) {
    int y_q4 = y0_q4;
    for (y = 0; y < h; ++y) {
      const unsigned char *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
      const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK];
      int k, sum = 0;
      for (k = 0; k < SUBPEL_TAPS; ++k)
        sum += src_y[k * src_stride] * y_filter[k];
      dst[y * dst_stride] =
          clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS) +
                     src_y[(SUBPEL_TAPS / 2 - 1) * src_stride]);
      y_q4 += y_step_q4;
    }
    ++src;
    ++dst;
  }
}

static void convolve_add_src(const uint8_t *src, ptrdiff_t src_stride,
                             uint8_t *dst, ptrdiff_t dst_stride,
                             const InterpKernel *const x_filters, int x0_q4,
                             int x_step_q4, const InterpKernel *const y_filters,
                             int y0_q4, int y_step_q4, int w, int h) {
  uint8_t temp[MAX_EXT_SIZE * MAX_SB_SIZE];
  int intermediate_height =
      (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS;

  assert(w <= MAX_SB_SIZE);
  assert(h <= MAX_SB_SIZE);

  assert(y_step_q4 <= 32);
  assert(x_step_q4 <= 32);

  convolve_add_src_horiz(src - src_stride * (SUBPEL_TAPS / 2 - 1), src_stride,
                         temp, MAX_SB_SIZE, x_filters, x0_q4, x_step_q4, w,
                         intermediate_height);
  convolve_add_src_vert(temp + MAX_SB_SIZE * (SUBPEL_TAPS / 2 - 1), MAX_SB_SIZE,
                        dst, dst_stride, y_filters, y0_q4, y_step_q4, w, h);
}

void aom_convolve8_add_src_horiz_c(const uint8_t *src, ptrdiff_t src_stride,
                                   uint8_t *dst, ptrdiff_t dst_stride,
                                   const int16_t *filter_x, int x_step_q4,
                                   const int16_t *filter_y, int y_step_q4,
                                   int w, int h) {
  const InterpKernel *const filters_x = get_filter_base(filter_x);
  const int x0_q4 = get_filter_offset(filter_x, filters_x);

  (void)filter_y;
  (void)y_step_q4;

  convolve_add_src_horiz(src, src_stride, dst, dst_stride, filters_x, x0_q4,
                         x_step_q4, w, h);
}

void aom_convolve8_add_src_vert_c(const uint8_t *src, ptrdiff_t src_stride,
                                  uint8_t *dst, ptrdiff_t dst_stride,
                                  const int16_t *filter_x, int x_step_q4,
                                  const int16_t *filter_y, int y_step_q4, int w,
                                  int h) {
  const InterpKernel *const filters_y = get_filter_base(filter_y);
  const int y0_q4 = get_filter_offset(filter_y, filters_y);

  (void)filter_x;
  (void)x_step_q4;

  convolve_add_src_vert(src, src_stride, dst, dst_stride, filters_y, y0_q4,
                        y_step_q4, w, h);
}

void aom_convolve8_add_src_c(const uint8_t *src, ptrdiff_t src_stride,
                             uint8_t *dst, ptrdiff_t dst_stride,
                             const int16_t *filter_x, int x_step_q4,
                             const int16_t *filter_y, int y_step_q4, int w,
                             int h) {
  const InterpKernel *const filters_x = get_filter_base(filter_x);
  const int x0_q4 = get_filter_offset(filter_x, filters_x);

  const InterpKernel *const filters_y = get_filter_base(filter_y);
  const int y0_q4 = get_filter_offset(filter_y, filters_y);

  convolve_add_src(src, src_stride, dst, dst_stride, filters_x, x0_q4,
                   x_step_q4, filters_y, y0_q4, y_step_q4, w, h);
}
#endif  // CONFIG_LOOP_RESTORATION

451
#if CONFIG_HIGHBITDEPTH
452 453
static void highbd_convolve_horiz(const uint8_t *src8, ptrdiff_t src_stride,
                                  uint8_t *dst8, ptrdiff_t dst_stride,
clang-format's avatar
clang-format committed
454 455
                                  const InterpKernel *x_filters, int x0_q4,
                                  int x_step_q4, int w, int h, int bd) {
456 457 458 459 460 461 462 463 464 465
  int x, y;
  uint16_t *src = CONVERT_TO_SHORTPTR(src8);
  uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
  src -= SUBPEL_TAPS / 2 - 1;
  for (y = 0; y < h; ++y) {
    int x_q4 = x0_q4;
    for (x = 0; x < w; ++x) {
      const uint16_t *const src_x = &src[x_q4 >> SUBPEL_BITS];
      const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK];
      int k, sum = 0;
clang-format's avatar
clang-format committed
466
      for (k = 0; k < SUBPEL_TAPS; ++k) sum += src_x[k] * x_filter[k];
467
      dst[x] = clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd);
468 469 470 471 472 473 474
      x_q4 += x_step_q4;
    }
    src += src_stride;
    dst += dst_stride;
  }
}

475 476
static void highbd_convolve_avg_horiz(const uint8_t *src8, ptrdiff_t src_stride,
                                      uint8_t *dst8, ptrdiff_t dst_stride,
clang-format's avatar
clang-format committed
477 478
                                      const InterpKernel *x_filters, int x0_q4,
                                      int x_step_q4, int w, int h, int bd) {
479 480 481 482 483 484 485 486 487 488
  int x, y;
  uint16_t *src = CONVERT_TO_SHORTPTR(src8);
  uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
  src -= SUBPEL_TAPS / 2 - 1;
  for (y = 0; y < h; ++y) {
    int x_q4 = x0_q4;
    for (x = 0; x < w; ++x) {
      const uint16_t *const src_x = &src[x_q4 >> SUBPEL_BITS];
      const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK];
      int k, sum = 0;
clang-format's avatar
clang-format committed
489 490 491 492
      for (k = 0; k < SUBPEL_TAPS; ++k) sum += src_x[k] * x_filter[k];
      dst[x] = ROUND_POWER_OF_TWO(
          dst[x] + clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd),
          1);
493 494 495 496 497 498 499
      x_q4 += x_step_q4;
    }
    src += src_stride;
    dst += dst_stride;
  }
}

500 501
static void highbd_convolve_vert(const uint8_t *src8, ptrdiff_t src_stride,
                                 uint8_t *dst8, ptrdiff_t dst_stride,
clang-format's avatar
clang-format committed
502 503
                                 const InterpKernel *y_filters, int y0_q4,
                                 int y_step_q4, int w, int h, int bd) {
504 505 506 507 508 509 510 511 512 513 514 515
  int x, y;
  uint16_t *src = CONVERT_TO_SHORTPTR(src8);
  uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
  src -= src_stride * (SUBPEL_TAPS / 2 - 1);
  for (x = 0; x < w; ++x) {
    int y_q4 = y0_q4;
    for (y = 0; y < h; ++y) {
      const uint16_t *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
      const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK];
      int k, sum = 0;
      for (k = 0; k < SUBPEL_TAPS; ++k)
        sum += src_y[k * src_stride] * y_filter[k];
clang-format's avatar
clang-format committed
516 517
      dst[y * dst_stride] =
          clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd);
518 519 520 521 522 523 524
      y_q4 += y_step_q4;
    }
    ++src;
    ++dst;
  }
}

525 526
static void highbd_convolve_avg_vert(const uint8_t *src8, ptrdiff_t src_stride,
                                     uint8_t *dst8, ptrdiff_t dst_stride,
clang-format's avatar
clang-format committed
527 528
                                     const InterpKernel *y_filters, int y0_q4,
                                     int y_step_q4, int w, int h, int bd) {
529 530 531 532 533 534 535 536 537 538 539 540
  int x, y;
  uint16_t *src = CONVERT_TO_SHORTPTR(src8);
  uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
  src -= src_stride * (SUBPEL_TAPS / 2 - 1);
  for (x = 0; x < w; ++x) {
    int y_q4 = y0_q4;
    for (y = 0; y < h; ++y) {
      const uint16_t *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
      const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK];
      int k, sum = 0;
      for (k = 0; k < SUBPEL_TAPS; ++k)
        sum += src_y[k * src_stride] * y_filter[k];
clang-format's avatar
clang-format committed
541 542 543 544
      dst[y * dst_stride] = ROUND_POWER_OF_TWO(
          dst[y * dst_stride] +
              clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd),
          1);
545 546 547 548 549 550 551
      y_q4 += y_step_q4;
    }
    ++src;
    ++dst;
  }
}

552 553
static void highbd_convolve(const uint8_t *src, ptrdiff_t src_stride,
                            uint8_t *dst, ptrdiff_t dst_stride,
clang-format's avatar
clang-format committed
554 555 556
                            const InterpKernel *const x_filters, int x0_q4,
                            int x_step_q4, const InterpKernel *const y_filters,
                            int y0_q4, int y_step_q4, int w, int h, int bd) {
557 558 559 560 561 562 563 564 565 566 567 568
  // Note: Fixed size intermediate buffer, temp, places limits on parameters.
  // 2d filtering proceeds in 2 steps:
  //   (1) Interpolate horizontally into an intermediate buffer, temp.
  //   (2) Interpolate temp vertically to derive the sub-pixel result.
  // Deriving the maximum number of rows in the temp buffer (135):
  // --Smallest scaling factor is x1/2 ==> y_step_q4 = 32 (Normative).
  // --Largest block size is 64x64 pixels.
  // --64 rows in the downscaled frame span a distance of (64 - 1) * 32 in the
  //   original frame (in 1/16th pixel units).
  // --Must round-up because block may be located at sub-pixel position.
  // --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails.
  // --((64 - 1) * 32 + 15) >> 4 + 8 = 135.
569
  uint16_t temp[MAX_EXT_SIZE * MAX_SB_SIZE];
570
  int intermediate_height =
clang-format's avatar
clang-format committed
571
      (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS;
572

573 574
  assert(w <= MAX_SB_SIZE);
  assert(h <= MAX_SB_SIZE);
575 576 577
  assert(y_step_q4 <= 32);
  assert(x_step_q4 <= 32);

578
  highbd_convolve_horiz(src - src_stride * (SUBPEL_TAPS / 2 - 1), src_stride,
clang-format's avatar
clang-format committed
579 580
                        CONVERT_TO_BYTEPTR(temp), MAX_SB_SIZE, x_filters, x0_q4,
                        x_step_q4, w, intermediate_height, bd);
581
  highbd_convolve_vert(
clang-format's avatar
clang-format committed
582 583
      CONVERT_TO_BYTEPTR(temp) + MAX_SB_SIZE * (SUBPEL_TAPS / 2 - 1),
      MAX_SB_SIZE, dst, dst_stride, y_filters, y0_q4, y_step_q4, w, h, bd);
584 585
}

Yaowu Xu's avatar
Yaowu Xu committed
586
void aom_highbd_convolve8_horiz_c(const uint8_t *src, ptrdiff_t src_stride,
587 588
                                  uint8_t *dst, ptrdiff_t dst_stride,
                                  const int16_t *filter_x, int x_step_q4,
clang-format's avatar
clang-format committed
589 590
                                  const int16_t *filter_y, int y_step_q4, int w,
                                  int h, int bd) {
591 592 593 594 595
  const InterpKernel *const filters_x = get_filter_base(filter_x);
  const int x0_q4 = get_filter_offset(filter_x, filters_x);
  (void)filter_y;
  (void)y_step_q4;

clang-format's avatar
clang-format committed
596 597
  highbd_convolve_horiz(src, src_stride, dst, dst_stride, filters_x, x0_q4,
                        x_step_q4, w, h, bd);
598 599
}

Yaowu Xu's avatar
Yaowu Xu committed
600
void aom_highbd_convolve8_avg_horiz_c(const uint8_t *src, ptrdiff_t src_stride,
601 602 603 604
                                      uint8_t *dst, ptrdiff_t dst_stride,
                                      const int16_t *filter_x, int x_step_q4,
                                      const int16_t *filter_y, int y_step_q4,
                                      int w, int h, int bd) {
605 606 607 608 609
  const InterpKernel *const filters_x = get_filter_base(filter_x);
  const int x0_q4 = get_filter_offset(filter_x, filters_x);
  (void)filter_y;
  (void)y_step_q4;

clang-format's avatar
clang-format committed
610 611
  highbd_convolve_avg_horiz(src, src_stride, dst, dst_stride, filters_x, x0_q4,
                            x_step_q4, w, h, bd);
612 613
}

Yaowu Xu's avatar
Yaowu Xu committed
614
void aom_highbd_convolve8_vert_c(const uint8_t *src, ptrdiff_t src_stride,
615 616
                                 uint8_t *dst, ptrdiff_t dst_stride,
                                 const int16_t *filter_x, int x_step_q4,
clang-format's avatar
clang-format committed
617 618
                                 const int16_t *filter_y, int y_step_q4, int w,
                                 int h, int bd) {
619 620 621 622 623
  const InterpKernel *const filters_y = get_filter_base(filter_y);
  const int y0_q4 = get_filter_offset(filter_y, filters_y);
  (void)filter_x;
  (void)x_step_q4;

clang-format's avatar
clang-format committed
624 625
  highbd_convolve_vert(src, src_stride, dst, dst_stride, filters_y, y0_q4,
                       y_step_q4, w, h, bd);
626 627
}

Yaowu Xu's avatar
Yaowu Xu committed
628
void aom_highbd_convolve8_avg_vert_c(const uint8_t *src, ptrdiff_t src_stride,
629 630 631 632
                                     uint8_t *dst, ptrdiff_t dst_stride,
                                     const int16_t *filter_x, int x_step_q4,
                                     const int16_t *filter_y, int y_step_q4,
                                     int w, int h, int bd) {
633 634 635 636 637
  const InterpKernel *const filters_y = get_filter_base(filter_y);
  const int y0_q4 = get_filter_offset(filter_y, filters_y);
  (void)filter_x;
  (void)x_step_q4;

clang-format's avatar
clang-format committed
638 639
  highbd_convolve_avg_vert(src, src_stride, dst, dst_stride, filters_y, y0_q4,
                           y_step_q4, w, h, bd);
640 641
}

Yaowu Xu's avatar
Yaowu Xu committed
642
void aom_highbd_convolve8_c(const uint8_t *src, ptrdiff_t src_stride,
643 644
                            uint8_t *dst, ptrdiff_t dst_stride,
                            const int16_t *filter_x, int x_step_q4,
clang-format's avatar
clang-format committed
645 646
                            const int16_t *filter_y, int y_step_q4, int w,
                            int h, int bd) {
647 648 649 650 651 652
  const InterpKernel *const filters_x = get_filter_base(filter_x);
  const int x0_q4 = get_filter_offset(filter_x, filters_x);

  const InterpKernel *const filters_y = get_filter_base(filter_y);
  const int y0_q4 = get_filter_offset(filter_y, filters_y);

clang-format's avatar
clang-format committed
653
  highbd_convolve(src, src_stride, dst, dst_stride, filters_x, x0_q4, x_step_q4,
654
                  filters_y, y0_q4, y_step_q4, w, h, bd);
655 656
}

Yaowu Xu's avatar
Yaowu Xu committed
657
void aom_highbd_convolve8_avg_c(const uint8_t *src, ptrdiff_t src_stride,
658 659
                                uint8_t *dst, ptrdiff_t dst_stride,
                                const int16_t *filter_x, int x_step_q4,
clang-format's avatar
clang-format committed
660 661
                                const int16_t *filter_y, int y_step_q4, int w,
                                int h, int bd) {
662
  // Fixed size intermediate buffer places limits on parameters.
663 664 665
  DECLARE_ALIGNED(16, uint16_t, temp[MAX_SB_SIZE * MAX_SB_SIZE]);
  assert(w <= MAX_SB_SIZE);
  assert(h <= MAX_SB_SIZE);
666

Yaowu Xu's avatar
Yaowu Xu committed
667
  aom_highbd_convolve8_c(src, src_stride, CONVERT_TO_BYTEPTR(temp), MAX_SB_SIZE,
668
                         filter_x, x_step_q4, filter_y, y_step_q4, w, h, bd);
Yaowu Xu's avatar
Yaowu Xu committed
669
  aom_highbd_convolve_avg_c(CONVERT_TO_BYTEPTR(temp), MAX_SB_SIZE, dst,
clang-format's avatar
clang-format committed
670
                            dst_stride, NULL, 0, NULL, 0, w, h, bd);
671 672
}

Yaowu Xu's avatar
Yaowu Xu committed
673
void aom_highbd_convolve_copy_c(const uint8_t *src8, ptrdiff_t src_stride,
674 675 676 677
                                uint8_t *dst8, ptrdiff_t dst_stride,
                                const int16_t *filter_x, int filter_x_stride,
                                const int16_t *filter_y, int filter_y_stride,
                                int w, int h, int bd) {
678 679 680 681 682 683 684 685 686 687
  int r;
  uint16_t *src = CONVERT_TO_SHORTPTR(src8);
  uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
  (void)filter_x;
  (void)filter_y;
  (void)filter_x_stride;
  (void)filter_y_stride;
  (void)bd;

  for (r = h; r > 0; --r) {
James Zern's avatar
James Zern committed
688
    memcpy(dst, src, w * sizeof(uint16_t));
689 690 691 692 693
    src += src_stride;
    dst += dst_stride;
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
694
void aom_highbd_convolve_avg_c(const uint8_t *src8, ptrdiff_t src_stride,
695 696 697 698
                               uint8_t *dst8, ptrdiff_t dst_stride,
                               const int16_t *filter_x, int filter_x_stride,
                               const int16_t *filter_y, int filter_y_stride,
                               int w, int h, int bd) {
699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715
  int x, y;
  uint16_t *src = CONVERT_TO_SHORTPTR(src8);
  uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
  (void)filter_x;
  (void)filter_y;
  (void)filter_x_stride;
  (void)filter_y_stride;
  (void)bd;

  for (y = 0; y < h; ++y) {
    for (x = 0; x < w; ++x) {
      dst[x] = ROUND_POWER_OF_TWO(dst[x] + src[x], 1);
    }
    src += src_stride;
    dst += dst_stride;
  }
}
716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853

#if CONFIG_LOOP_RESTORATION
static void highbd_convolve_add_src_horiz(const uint8_t *src8,
                                          ptrdiff_t src_stride, uint8_t *dst8,
                                          ptrdiff_t dst_stride,
                                          const InterpKernel *x_filters,
                                          int x0_q4, int x_step_q4, int w,
                                          int h, int bd) {
  int x, y;
  uint16_t *src = CONVERT_TO_SHORTPTR(src8);
  uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
  src -= SUBPEL_TAPS / 2 - 1;
  for (y = 0; y < h; ++y) {
    int x_q4 = x0_q4;
    for (x = 0; x < w; ++x) {
      const uint16_t *const src_x = &src[x_q4 >> SUBPEL_BITS];
      const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK];
      int k, sum = 0;
      for (k = 0; k < SUBPEL_TAPS; ++k) sum += src_x[k] * x_filter[k];
      dst[x] = clip_pixel_highbd(
          ROUND_POWER_OF_TWO(sum, FILTER_BITS) + src_x[SUBPEL_TAPS / 2 - 1],
          bd);
      x_q4 += x_step_q4;
    }
    src += src_stride;
    dst += dst_stride;
  }
}

static void highbd_convolve_add_src_vert(const uint8_t *src8,
                                         ptrdiff_t src_stride, uint8_t *dst8,
                                         ptrdiff_t dst_stride,
                                         const InterpKernel *y_filters,
                                         int y0_q4, int y_step_q4, int w, int h,
                                         int bd) {
  int x, y;
  uint16_t *src = CONVERT_TO_SHORTPTR(src8);
  uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
  src -= src_stride * (SUBPEL_TAPS / 2 - 1);
  for (x = 0; x < w; ++x) {
    int y_q4 = y0_q4;
    for (y = 0; y < h; ++y) {
      const uint16_t *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
      const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK];
      int k, sum = 0;
      for (k = 0; k < SUBPEL_TAPS; ++k)
        sum += src_y[k * src_stride] * y_filter[k];
      dst[y * dst_stride] =
          clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS) +
                                src_y[(SUBPEL_TAPS / 2 - 1) * src_stride],
                            bd);
      y_q4 += y_step_q4;
    }
    ++src;
    ++dst;
  }
}

static void highbd_convolve_add_src(const uint8_t *src, ptrdiff_t src_stride,
                                    uint8_t *dst, ptrdiff_t dst_stride,
                                    const InterpKernel *const x_filters,
                                    int x0_q4, int x_step_q4,
                                    const InterpKernel *const y_filters,
                                    int y0_q4, int y_step_q4, int w, int h,
                                    int bd) {
  // Note: Fixed size intermediate buffer, temp, places limits on parameters.
  // 2d filtering proceeds in 2 steps:
  //   (1) Interpolate horizontally into an intermediate buffer, temp.
  //   (2) Interpolate temp vertically to derive the sub-pixel result.
  // Deriving the maximum number of rows in the temp buffer (135):
  // --Smallest scaling factor is x1/2 ==> y_step_q4 = 32 (Normative).
  // --Largest block size is 64x64 pixels.
  // --64 rows in the downscaled frame span a distance of (64 - 1) * 32 in the
  //   original frame (in 1/16th pixel units).
  // --Must round-up because block may be located at sub-pixel position.
  // --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails.
  // --((64 - 1) * 32 + 15) >> 4 + 8 = 135.
  uint16_t temp[MAX_EXT_SIZE * MAX_SB_SIZE];
  int intermediate_height =
      (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS;

  assert(w <= MAX_SB_SIZE);
  assert(h <= MAX_SB_SIZE);
  assert(y_step_q4 <= 32);
  assert(x_step_q4 <= 32);

  highbd_convolve_add_src_horiz(src - src_stride * (SUBPEL_TAPS / 2 - 1),
                                src_stride, CONVERT_TO_BYTEPTR(temp),
                                MAX_SB_SIZE, x_filters, x0_q4, x_step_q4, w,
                                intermediate_height, bd);
  highbd_convolve_add_src_vert(
      CONVERT_TO_BYTEPTR(temp) + MAX_SB_SIZE * (SUBPEL_TAPS / 2 - 1),
      MAX_SB_SIZE, dst, dst_stride, y_filters, y0_q4, y_step_q4, w, h, bd);
}

void aom_highbd_convolve8_add_src_horiz_c(
    const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
    ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4,
    const int16_t *filter_y, int y_step_q4, int w, int h, int bd) {
  const InterpKernel *const filters_x = get_filter_base(filter_x);
  const int x0_q4 = get_filter_offset(filter_x, filters_x);
  (void)filter_y;
  (void)y_step_q4;

  highbd_convolve_add_src_horiz(src, src_stride, dst, dst_stride, filters_x,
                                x0_q4, x_step_q4, w, h, bd);
}

void aom_highbd_convolve8_add_src_vert_c(const uint8_t *src,
                                         ptrdiff_t src_stride, uint8_t *dst,
                                         ptrdiff_t dst_stride,
                                         const int16_t *filter_x, int x_step_q4,
                                         const int16_t *filter_y, int y_step_q4,
                                         int w, int h, int bd) {
  const InterpKernel *const filters_y = get_filter_base(filter_y);
  const int y0_q4 = get_filter_offset(filter_y, filters_y);
  (void)filter_x;
  (void)x_step_q4;

  highbd_convolve_add_src_vert(src, src_stride, dst, dst_stride, filters_y,
                               y0_q4, y_step_q4, w, h, bd);
}

void aom_highbd_convolve8_add_src_c(const uint8_t *src, ptrdiff_t src_stride,
                                    uint8_t *dst, ptrdiff_t dst_stride,
                                    const int16_t *filter_x, int x_step_q4,
                                    const int16_t *filter_y, int y_step_q4,
                                    int w, int h, int bd) {
  const InterpKernel *const filters_x = get_filter_base(filter_x);
  const int x0_q4 = get_filter_offset(filter_x, filters_x);

  const InterpKernel *const filters_y = get_filter_base(filter_y);
  const int y0_q4 = get_filter_offset(filter_y, filters_y);

  highbd_convolve_add_src(src, src_stride, dst, dst_stride, filters_x, x0_q4,
                          x_step_q4, filters_y, y0_q4, y_step_q4, w, h, bd);
}
#endif  // CONFIG_LOOP_RESTORATION
854
#endif  // CONFIG_HIGHBITDEPTH