reconintra.c 55.4 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
2
3
4
5
6
7
8
9
10
/*
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

hui su's avatar
hui su committed
11
12
#include <math.h>

Yaowu Xu's avatar
Yaowu Xu committed
13
14
15
#include "./av1_rtcd.h"
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
16
#include "aom_ports/system_state.h"
Jingning Han's avatar
Jingning Han committed
17

Yaowu Xu's avatar
Yaowu Xu committed
18
19
20
21
#if CONFIG_AOM_HIGHBITDEPTH
#include "aom_dsp/aom_dsp_common.h"
#endif  // CONFIG_AOM_HIGHBITDEPTH
#include "aom_mem/aom_mem.h"
22
#include "aom_ports/mem.h"
Yaowu Xu's avatar
Yaowu Xu committed
23
#include "aom_ports/aom_once.h"
24
#if CONFIG_EXT_INTRA
25
#include "av1/common/intra_filters.h"
26
#endif
27
28
#include "av1/common/reconintra.h"
#include "av1/common/onyxc_int.h"
Jingning Han's avatar
Jingning Han committed
29

30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
enum {
  NEED_LEFT = 1 << 1,
  NEED_ABOVE = 1 << 2,
  NEED_ABOVERIGHT = 1 << 3,
  NEED_ABOVELEFT = 1 << 4,
  NEED_BOTTOMLEFT = 1 << 5,
};

static const uint8_t extend_modes[INTRA_MODES] = {
  NEED_ABOVE | NEED_LEFT,                   // DC
  NEED_ABOVE,                               // V
  NEED_LEFT,                                // H
  NEED_ABOVE | NEED_ABOVERIGHT,             // D45
  NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT,  // D135
  NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT,  // D117
  NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT,  // D153
  NEED_LEFT | NEED_BOTTOMLEFT,              // D207
  NEED_ABOVE | NEED_ABOVERIGHT,             // D63
  NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT,  // TM
};
Jingning Han's avatar
Jingning Han committed
50

51
52
53
54
static const uint8_t orders_128x128[1] = { 0 };
static const uint8_t orders_128x64[2] = { 0, 1 };
static const uint8_t orders_64x128[2] = { 0, 1 };
static const uint8_t orders_64x64[4] = {
clang-format's avatar
clang-format committed
55
  0, 1, 2, 3,
56
};
57
static const uint8_t orders_64x32[8] = {
clang-format's avatar
clang-format committed
58
  0, 2, 1, 3, 4, 6, 5, 7,
59
};
60
static const uint8_t orders_32x64[8] = {
clang-format's avatar
clang-format committed
61
  0, 1, 2, 3, 4, 5, 6, 7,
62
};
63
static const uint8_t orders_32x32[16] = {
clang-format's avatar
clang-format committed
64
  0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15,
65
};
66
static const uint8_t orders_32x16[32] = {
clang-format's avatar
clang-format committed
67
68
  0,  2,  8,  10, 1,  3,  9,  11, 4,  6,  12, 14, 5,  7,  13, 15,
  16, 18, 24, 26, 17, 19, 25, 27, 20, 22, 28, 30, 21, 23, 29, 31,
69
};
70
static const uint8_t orders_16x32[32] = {
clang-format's avatar
clang-format committed
71
72
  0,  1,  2,  3,  8,  9,  10, 11, 4,  5,  6,  7,  12, 13, 14, 15,
  16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31,
73
};
74
static const uint8_t orders_16x16[64] = {
clang-format's avatar
clang-format committed
75
76
77
78
  0,  1,  4,  5,  16, 17, 20, 21, 2,  3,  6,  7,  18, 19, 22, 23,
  8,  9,  12, 13, 24, 25, 28, 29, 10, 11, 14, 15, 26, 27, 30, 31,
  32, 33, 36, 37, 48, 49, 52, 53, 34, 35, 38, 39, 50, 51, 54, 55,
  40, 41, 44, 45, 56, 57, 60, 61, 42, 43, 46, 47, 58, 59, 62, 63,
79
};
80
81
82

#if CONFIG_EXT_PARTITION
static const uint8_t orders_16x8[128] = {
clang-format's avatar
clang-format committed
83
84
85
86
87
88
89
90
  0,  2,  8,  10, 32,  34,  40,  42,  1,  3,  9,  11, 33,  35,  41,  43,
  4,  6,  12, 14, 36,  38,  44,  46,  5,  7,  13, 15, 37,  39,  45,  47,
  16, 18, 24, 26, 48,  50,  56,  58,  17, 19, 25, 27, 49,  51,  57,  59,
  20, 22, 28, 30, 52,  54,  60,  62,  21, 23, 29, 31, 53,  55,  61,  63,
  64, 66, 72, 74, 96,  98,  104, 106, 65, 67, 73, 75, 97,  99,  105, 107,
  68, 70, 76, 78, 100, 102, 108, 110, 69, 71, 77, 79, 101, 103, 109, 111,
  80, 82, 88, 90, 112, 114, 120, 122, 81, 83, 89, 91, 113, 115, 121, 123,
  84, 86, 92, 94, 116, 118, 124, 126, 85, 87, 93, 95, 117, 119, 125, 127,
91
92
};
static const uint8_t orders_8x16[128] = {
clang-format's avatar
clang-format committed
93
94
95
96
97
  0,  1,  2,  3,  8,  9,  10, 11, 32,  33,  34,  35,  40,  41,  42,  43,
  4,  5,  6,  7,  12, 13, 14, 15, 36,  37,  38,  39,  44,  45,  46,  47,
  16, 17, 18, 19, 24, 25, 26, 27, 48,  49,  50,  51,  56,  57,  58,  59,
  20, 21, 22, 23, 28, 29, 30, 31, 52,  53,  54,  55,  60,  61,  62,  63,
  64, 65, 66, 67, 72, 73, 74, 75, 96,  97,  98,  99,  104, 105, 106, 107,
98
99
100
101
102
  68, 69, 70, 71, 76, 77, 78, 79, 100, 101, 102, 103, 108, 109, 110, 111,
  80, 81, 82, 83, 88, 89, 90, 91, 112, 113, 114, 115, 120, 121, 122, 123,
  84, 85, 86, 87, 92, 93, 94, 95, 116, 117, 118, 119, 124, 125, 126, 127,
};
static const uint8_t orders_8x8[256] = {
clang-format's avatar
clang-format committed
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
  0,   1,   4,   5,   16,  17,  20,  21,  64,  65,  68,  69,  80,  81,  84,
  85,  2,   3,   6,   7,   18,  19,  22,  23,  66,  67,  70,  71,  82,  83,
  86,  87,  8,   9,   12,  13,  24,  25,  28,  29,  72,  73,  76,  77,  88,
  89,  92,  93,  10,  11,  14,  15,  26,  27,  30,  31,  74,  75,  78,  79,
  90,  91,  94,  95,  32,  33,  36,  37,  48,  49,  52,  53,  96,  97,  100,
  101, 112, 113, 116, 117, 34,  35,  38,  39,  50,  51,  54,  55,  98,  99,
  102, 103, 114, 115, 118, 119, 40,  41,  44,  45,  56,  57,  60,  61,  104,
  105, 108, 109, 120, 121, 124, 125, 42,  43,  46,  47,  58,  59,  62,  63,
  106, 107, 110, 111, 122, 123, 126, 127, 128, 129, 132, 133, 144, 145, 148,
  149, 192, 193, 196, 197, 208, 209, 212, 213, 130, 131, 134, 135, 146, 147,
  150, 151, 194, 195, 198, 199, 210, 211, 214, 215, 136, 137, 140, 141, 152,
  153, 156, 157, 200, 201, 204, 205, 216, 217, 220, 221, 138, 139, 142, 143,
  154, 155, 158, 159, 202, 203, 206, 207, 218, 219, 222, 223, 160, 161, 164,
  165, 176, 177, 180, 181, 224, 225, 228, 229, 240, 241, 244, 245, 162, 163,
  166, 167, 178, 179, 182, 183, 226, 227, 230, 231, 242, 243, 246, 247, 168,
  169, 172, 173, 184, 185, 188, 189, 232, 233, 236, 237, 248, 249, 252, 253,
  170, 171, 174, 175, 186, 187, 190, 191, 234, 235, 238, 239, 250, 251, 254,
  255,
121
122
};

clang-format's avatar
clang-format committed
123
/* clang-format off */
124
static const uint8_t *const orders[BLOCK_SIZES] = {
125
126
127
128
129
130
131
132
133
134
135
136
  //                              4X4
                                  orders_8x8,
  // 4X8,         8X4,            8X8
  orders_8x8,     orders_8x8,     orders_8x8,
  // 8X16,        16X8,           16X16
  orders_8x16,    orders_16x8,    orders_16x16,
  // 16X32,       32X16,          32X32
  orders_16x32,   orders_32x16,   orders_32x32,
  // 32X64,       64X32,          64X64
  orders_32x64,   orders_64x32,   orders_64x64,
  // 64x128,      128x64,         128x128
  orders_64x128,  orders_128x64,  orders_128x128
137
};
clang-format's avatar
clang-format committed
138
/* clang-format on */
139
#else
clang-format's avatar
clang-format committed
140
/* clang-format off */
141
142
143
144
145
146
147
148
149
150
151
152
static const uint8_t *const orders[BLOCK_SIZES] = {
  //                              4X4
                                  orders_16x16,
  // 4X8,         8X4,            8X8
  orders_16x16,   orders_16x16,   orders_16x16,
  // 8X16,        16X8,           16X16
  orders_16x32,   orders_32x16,   orders_32x32,
  // 16X32,       32X16,          32X32
  orders_32x64,   orders_64x32,   orders_64x64,
  // 32X64,       64X32,          64X64
  orders_64x128,  orders_128x64,  orders_128x128
};
clang-format's avatar
clang-format committed
153
/* clang-format on */
154
155
#endif  // CONFIG_EXT_PARTITION

156
#if CONFIG_EXT_PARTITION_TYPES
157
static const uint8_t orders_verta_64x64[4] = {
clang-format's avatar
clang-format committed
158
  0, 2, 1, 2,
159
};
160
static const uint8_t orders_verta_32x32[16] = {
clang-format's avatar
clang-format committed
161
  0, 2, 4, 6, 1, 2, 5, 6, 8, 10, 12, 14, 9, 10, 13, 14,
162
};
163
static const uint8_t orders_verta_16x16[64] = {
clang-format's avatar
clang-format committed
164
165
166
167
  0,  2,  4,  6,  16, 18, 20, 22, 1,  2,  5,  6,  17, 18, 21, 22,
  8,  10, 12, 14, 24, 26, 28, 30, 9,  10, 13, 14, 25, 26, 29, 30,
  32, 34, 36, 38, 48, 50, 52, 54, 33, 34, 37, 38, 49, 50, 53, 54,
  40, 42, 44, 46, 56, 58, 60, 62, 41, 42, 45, 46, 57, 58, 61, 62,
168
};
169
170
#if CONFIG_EXT_PARTITION
static const uint8_t orders_verta_8x8[256] = {
clang-format's avatar
clang-format committed
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
  0,   2,   4,   6,   16,  18,  20,  22,  64,  66,  68,  70,  80,  82,  84,
  86,  1,   2,   5,   6,   17,  18,  21,  22,  65,  66,  69,  70,  81,  82,
  85,  86,  8,   10,  12,  14,  24,  26,  28,  30,  72,  74,  76,  78,  88,
  90,  92,  94,  9,   10,  13,  14,  25,  26,  29,  30,  73,  74,  77,  78,
  89,  90,  93,  94,  32,  34,  36,  38,  48,  50,  52,  54,  96,  98,  100,
  102, 112, 114, 116, 118, 33,  34,  37,  38,  49,  50,  53,  54,  97,  98,
  101, 102, 113, 114, 117, 118, 40,  42,  44,  46,  56,  58,  60,  62,  104,
  106, 108, 110, 120, 122, 124, 126, 41,  42,  45,  46,  57,  58,  61,  62,
  105, 106, 109, 110, 121, 122, 125, 126, 128, 130, 132, 134, 144, 146, 148,
  150, 192, 194, 196, 198, 208, 210, 212, 214, 129, 130, 133, 134, 145, 146,
  149, 150, 193, 194, 197, 198, 209, 210, 213, 214, 136, 138, 140, 142, 152,
  154, 156, 158, 200, 202, 204, 206, 216, 218, 220, 222, 137, 138, 141, 142,
  153, 154, 157, 158, 201, 202, 205, 206, 217, 218, 221, 222, 160, 162, 164,
  166, 176, 178, 180, 182, 224, 226, 228, 230, 240, 242, 244, 246, 161, 162,
  165, 166, 177, 178, 181, 182, 225, 226, 229, 230, 241, 242, 245, 246, 168,
  170, 172, 174, 184, 186, 188, 190, 232, 234, 236, 238, 248, 250, 252, 254,
  169, 170, 173, 174, 185, 186, 189, 190, 233, 234, 237, 238, 249, 250, 253,
  254,
189
};
clang-format's avatar
clang-format committed
190
191

/* clang-format off */
192
193
194
195
196
197
198
199
200
201
202
203
204
205
static const uint8_t *const orders_verta[BLOCK_SIZES] = {
  //                                  4X4
                                      orders_verta_8x8,
  // 4X8,           8X4,              8X8
  orders_verta_8x8, orders_verta_8x8, orders_verta_8x8,
  // 8X16,          16X8,             16X16
  orders_8x16,      orders_16x8,      orders_verta_16x16,
  // 16X32,         32X16,            32X32
  orders_16x32,     orders_32x16,     orders_verta_32x32,
  // 32X64,         64X32,            64X64
  orders_32x64,     orders_64x32,     orders_verta_64x64,
  // 64x128,        128x64,           128x128
  orders_64x128,    orders_128x64,    orders_128x128
};
clang-format's avatar
clang-format committed
206
/* clang-format on */
207
#else
clang-format's avatar
clang-format committed
208
/* clang-format off */
209
static const uint8_t *const orders_verta[BLOCK_SIZES] = {
210
211
212
213
214
215
216
217
218
219
  //                                      4X4
                                          orders_verta_16x16,
  // 4X8,             8X4,                8X8
  orders_verta_16x16, orders_verta_16x16, orders_verta_16x16,
  // 8X16,            16X8,               16X16
  orders_16x32,       orders_32x16,       orders_verta_32x32,
  // 16X32,           32X16,              32X32
  orders_32x64,       orders_64x32,       orders_verta_64x64,
  // 32X64,           64X32,              64X64
  orders_64x128,      orders_128x64,      orders_128x128
220
};
clang-format's avatar
clang-format committed
221
/* clang-format on */
222
#endif  // CONFIG_EXT_PARTITION
223
#endif  // CONFIG_EXT_PARTITION_TYPES
224

Yaowu Xu's avatar
Yaowu Xu committed
225
226
static int av1_has_right(BLOCK_SIZE bsize, int mi_row, int mi_col,
                         int right_available,
227
#if CONFIG_EXT_PARTITION_TYPES
Yaowu Xu's avatar
Yaowu Xu committed
228
                         PARTITION_TYPE partition,
229
#endif
Yaowu Xu's avatar
Yaowu Xu committed
230
                         TX_SIZE txsz, int y, int x, int ss_x) {
hui su's avatar
hui su committed
231
  const int wl = mi_width_log2_lookup[bsize];
Yaowu Xu's avatar
Yaowu Xu committed
232
  const int w = AOMMAX(num_4x4_blocks_wide_lookup[bsize] >> ss_x, 1);
hui su's avatar
hui su committed
233
234
  const int step = 1 << txsz;

235
236
237
238
  if (!right_available) {
    return 0;
  } else {
    // Handle block size 4x8 and 4x4
clang-format's avatar
clang-format committed
239
    if (ss_x == 0 && num_4x4_blocks_wide_lookup[bsize] < 2 && x == 0) return 1;
240

241
242
243
244
    if (y == 0) {
      const int hl = mi_height_log2_lookup[bsize];
      const uint8_t *order;
      int my_order, tr_order;
245
#if CONFIG_EXT_PARTITION_TYPES
246
247
248
      if (partition == PARTITION_VERT_A)
        order = orders_verta[bsize];
      else
249
#endif  // CONFIG_EXT_PARTITION_TYPES
clang-format's avatar
clang-format committed
250
        order = orders[bsize];
251

clang-format's avatar
clang-format committed
252
      if (x + step < w) return 1;
253

254
255
      mi_row = (mi_row & MAX_MIB_MASK) >> hl;
      mi_col = (mi_col & MAX_MIB_MASK) >> wl;
256

257
      // If top row of coding unit
clang-format's avatar
clang-format committed
258
      if (mi_row == 0) return 1;
259

260
      // If rightmost column of coding unit
clang-format's avatar
clang-format committed
261
      if (((mi_col + 1) << wl) >= MAX_MIB_SIZE) return 0;
262

263
264
      my_order = order[((mi_row + 0) << (MAX_MIB_SIZE_LOG2 - wl)) + mi_col + 0];
      tr_order = order[((mi_row - 1) << (MAX_MIB_SIZE_LOG2 - wl)) + mi_col + 1];
265

266
267
268
269
      return my_order > tr_order;
    } else {
      return x + step < w;
    }
270
271
272
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
273
274
275
static int av1_has_bottom(BLOCK_SIZE bsize, int mi_row, int mi_col,
                          int bottom_available, TX_SIZE txsz, int y, int x,
                          int ss_y) {
276
277
278
  if (!bottom_available || x != 0) {
    return 0;
  } else {
hui su's avatar
hui su committed
279
280
281
282
    const int wl = mi_width_log2_lookup[bsize];
    const int hl = mi_height_log2_lookup[bsize];
    const int h = 1 << (hl + 1 - ss_y);
    const int step = 1 << txsz;
283
284
285
    const uint8_t *order = orders[bsize];
    int my_order, bl_order;

286
    // Handle block size 8x4 and 4x4
clang-format's avatar
clang-format committed
287
    if (ss_y == 0 && num_4x4_blocks_high_lookup[bsize] < 2 && y == 0) return 1;
288

clang-format's avatar
clang-format committed
289
    if (y + step < h) return 1;
290

291
292
    mi_row = (mi_row & MAX_MIB_MASK) >> hl;
    mi_col = (mi_col & MAX_MIB_MASK) >> wl;
293
294

    if (mi_col == 0)
295
      return (mi_row << (hl + !ss_y)) + y + step < (MAX_MIB_SIZE << !ss_y);
296

clang-format's avatar
clang-format committed
297
    if (((mi_row + 1) << hl) >= MAX_MIB_SIZE) return 0;
298

299
300
    my_order = order[((mi_row + 0) << (MAX_MIB_SIZE_LOG2 - wl)) + mi_col + 0];
    bl_order = order[((mi_row + 1) << (MAX_MIB_SIZE_LOG2 - wl)) + mi_col - 1];
301

302
    return bl_order < my_order;
303
304
  }
}
Jingning Han's avatar
Jingning Han committed
305
306
307
308
309
310
311

typedef void (*intra_pred_fn)(uint8_t *dst, ptrdiff_t stride,
                              const uint8_t *above, const uint8_t *left);

static intra_pred_fn pred[INTRA_MODES][TX_SIZES];
static intra_pred_fn dc_pred[2][2][TX_SIZES];

Yaowu Xu's avatar
Yaowu Xu committed
312
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
313
314
315
316
317
typedef void (*intra_high_pred_fn)(uint16_t *dst, ptrdiff_t stride,
                                   const uint16_t *above, const uint16_t *left,
                                   int bd);
static intra_high_pred_fn pred_high[INTRA_MODES][4];
static intra_high_pred_fn dc_pred_high[2][2][4];
Yaowu Xu's avatar
Yaowu Xu committed
318
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
319

Yaowu Xu's avatar
Yaowu Xu committed
320
static void av1_init_intra_predictors_internal(void) {
clang-format's avatar
clang-format committed
321
#define INIT_NO_4X4(p, type)                  \
Yaowu Xu's avatar
Yaowu Xu committed
322
323
324
  p[TX_8X8] = aom_##type##_predictor_8x8;     \
  p[TX_16X16] = aom_##type##_predictor_16x16; \
  p[TX_32X32] = aom_##type##_predictor_32x32
Jingning Han's avatar
Jingning Han committed
325

clang-format's avatar
clang-format committed
326
#define INIT_ALL_SIZES(p, type)           \
Yaowu Xu's avatar
Yaowu Xu committed
327
  p[TX_4X4] = aom_##type##_predictor_4x4; \
328
329
  INIT_NO_4X4(p, type)

Jingning Han's avatar
Jingning Han committed
330
331
  INIT_ALL_SIZES(pred[V_PRED], v);
  INIT_ALL_SIZES(pred[H_PRED], h);
332
333
334
  INIT_ALL_SIZES(pred[D207_PRED], d207e);
  INIT_ALL_SIZES(pred[D45_PRED], d45e);
  INIT_ALL_SIZES(pred[D63_PRED], d63e);
Jingning Han's avatar
Jingning Han committed
335
336
337
338
339
340
341
342
343
344
  INIT_ALL_SIZES(pred[D117_PRED], d117);
  INIT_ALL_SIZES(pred[D135_PRED], d135);
  INIT_ALL_SIZES(pred[D153_PRED], d153);
  INIT_ALL_SIZES(pred[TM_PRED], tm);

  INIT_ALL_SIZES(dc_pred[0][0], dc_128);
  INIT_ALL_SIZES(dc_pred[0][1], dc_top);
  INIT_ALL_SIZES(dc_pred[1][0], dc_left);
  INIT_ALL_SIZES(dc_pred[1][1], dc);

Yaowu Xu's avatar
Yaowu Xu committed
345
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
346
347
  INIT_ALL_SIZES(pred_high[V_PRED], highbd_v);
  INIT_ALL_SIZES(pred_high[H_PRED], highbd_h);
348
349
350
  INIT_ALL_SIZES(pred_high[D207_PRED], highbd_d207e);
  INIT_ALL_SIZES(pred_high[D45_PRED], highbd_d45e);
  INIT_ALL_SIZES(pred_high[D63_PRED], highbd_d63e);
Jingning Han's avatar
Jingning Han committed
351
352
353
354
355
356
357
358
359
  INIT_ALL_SIZES(pred_high[D117_PRED], highbd_d117);
  INIT_ALL_SIZES(pred_high[D135_PRED], highbd_d135);
  INIT_ALL_SIZES(pred_high[D153_PRED], highbd_d153);
  INIT_ALL_SIZES(pred_high[TM_PRED], highbd_tm);

  INIT_ALL_SIZES(dc_pred_high[0][0], highbd_dc_128);
  INIT_ALL_SIZES(dc_pred_high[0][1], highbd_dc_top);
  INIT_ALL_SIZES(dc_pred_high[1][0], highbd_dc_left);
  INIT_ALL_SIZES(dc_pred_high[1][1], highbd_dc);
Yaowu Xu's avatar
Yaowu Xu committed
360
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
361
362
363
364

#undef intra_pred_allsizes
}

hui su's avatar
hui su committed
365
366
367
#if CONFIG_EXT_INTRA

static const uint8_t ext_intra_extend_modes[FILTER_INTRA_MODES] = {
clang-format's avatar
clang-format committed
368
369
370
371
372
373
374
375
376
377
  NEED_LEFT | NEED_ABOVE,  // FILTER_DC
  NEED_LEFT | NEED_ABOVE,  // FILTER_V
  NEED_LEFT | NEED_ABOVE,  // FILTER_H
  NEED_LEFT | NEED_ABOVE,  // FILTER_D45
  NEED_LEFT | NEED_ABOVE,  // FILTER_D135
  NEED_LEFT | NEED_ABOVE,  // FILTER_D117
  NEED_LEFT | NEED_ABOVE,  // FILTER_D153
  NEED_LEFT | NEED_ABOVE,  // FILTER_D207
  NEED_LEFT | NEED_ABOVE,  // FILTER_D63
  NEED_LEFT | NEED_ABOVE,  // FILTER_TM
hui su's avatar
hui su committed
378
379
};

380
381
382
383
384
385
386
387
388
389
390
static int intra_subpel_interp(int base, int shift, const uint8_t *ref,
                               int ref_start_idx, int ref_end_idx,
                               INTRA_FILTER filter_type) {
  int val, k, idx, filter_idx = 0;
  const int16_t *filter = NULL;

  if (filter_type == INTRA_FILTER_LINEAR) {
    val = ref[base] * (256 - shift) + ref[base + 1] * shift;
    val = ROUND_POWER_OF_TWO(val, 8);
  } else {
    filter_idx = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS);
Yaowu Xu's avatar
Yaowu Xu committed
391
    filter = av1_intra_filter_kernels[filter_type][filter_idx];
392
393
394
395
396

    if (filter_idx < (1 << SUBPEL_BITS)) {
      val = 0;
      for (k = 0; k < SUBPEL_TAPS; ++k) {
        idx = base + 1 - (SUBPEL_TAPS / 2) + k;
Yaowu Xu's avatar
Yaowu Xu committed
397
        idx = AOMMAX(AOMMIN(idx, ref_end_idx), ref_start_idx);
398
399
400
401
402
403
404
405
406
407
408
        val += ref[idx] * filter[k];
      }
      val = ROUND_POWER_OF_TWO(val, FILTER_BITS);
    } else {
      val = ref[base + 1];
    }
  }

  return val;
}

hui su's avatar
hui su committed
409
410
// Directional prediction, zone 1: 0 < angle < 90
static void dr_prediction_z1(uint8_t *dst, ptrdiff_t stride, int bs,
clang-format's avatar
clang-format committed
411
412
                             const uint8_t *above, const uint8_t *left, int dx,
                             int dy, INTRA_FILTER filter_type) {
hui su's avatar
hui su committed
413
  int r, c, x, base, shift, val;
hui su's avatar
hui su committed
414
415
416
417
418
419

  (void)left;
  (void)dy;
  assert(dy == 1);
  assert(dx < 0);

hui su's avatar
hui su committed
420
421
422
  if (filter_type != INTRA_FILTER_LINEAR) {
    const int pad_size = SUBPEL_TAPS >> 1;
    int len;
423
424
    DECLARE_ALIGNED(16, uint8_t, buf[SUBPEL_SHIFTS][MAX_SB_SIZE]);
    DECLARE_ALIGNED(16, uint8_t, src[MAX_SB_SIZE + SUBPEL_TAPS]);
hui su's avatar
hui su committed
425
426
427
428
429
430
431
432
433
434
    uint8_t flags[SUBPEL_SHIFTS];

    memset(flags, 0, SUBPEL_SHIFTS * sizeof(flags[0]));
    memset(src, above[0], pad_size * sizeof(above[0]));
    memcpy(src + pad_size, above, 2 * bs * sizeof(above[0]));
    memset(src + pad_size + 2 * bs, above[2 * bs - 1],
           pad_size * sizeof(above[0]));
    flags[0] = 1;
    x = -dx;
    for (r = 0; r < bs; ++r, dst += stride, x -= dx) {
hui su's avatar
hui su committed
435
      base = x >> 8;
hui su's avatar
hui su committed
436
437
438
439
440
441
      shift = x & 0xFF;
      shift = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS);
      if (shift == SUBPEL_SHIFTS) {
        base += 1;
        shift = 0;
      }
Yaowu Xu's avatar
Yaowu Xu committed
442
      len = AOMMIN(bs, 2 * bs - 1 - base);
hui su's avatar
hui su committed
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
      if (len <= 0) {
        int i;
        for (i = r; i < bs; ++i) {
          memset(dst, above[2 * bs - 1], bs * sizeof(dst[0]));
          dst += stride;
        }
        return;
      }

      if (len <= (bs >> 1) && !flags[shift]) {
        base = x >> 8;
        shift = x & 0xFF;
        for (c = 0; c < len; ++c) {
          val = intra_subpel_interp(base, shift, above, 0, 2 * bs - 1,
                                    filter_type);
          dst[c] = clip_pixel(val);
          ++base;
        }
      } else {
        if (!flags[shift]) {
Yaowu Xu's avatar
Yaowu Xu committed
463
464
          const int16_t *filter = av1_intra_filter_kernels[filter_type][shift];
          aom_convolve8_horiz(src + pad_size, 2 * bs, buf[shift], 2 * bs,
clang-format's avatar
clang-format committed
465
466
                              filter, 16, NULL, 16, 2 * bs,
                              2 * bs < 16 ? 2 : 1);
hui su's avatar
hui su committed
467
468
469
          flags[shift] = 1;
        }
        memcpy(dst, shift == 0 ? src + pad_size + base : &buf[shift][base],
clang-format's avatar
clang-format committed
470
               len * sizeof(dst[0]));
hui su's avatar
hui su committed
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
      }

      if (len < bs)
        memset(dst + len, above[2 * bs - 1], (bs - len) * sizeof(dst[0]));
    }
    return;
  }

  // For linear filter, C code is faster.
  x = -dx;
  for (r = 0; r < bs; ++r, dst += stride, x -= dx) {
    base = x >> 8;
    shift = x & 0xFF;

    if (base >= 2 * bs - 1) {
      int i;
      for (i = r; i < bs; ++i) {
        memset(dst, above[2 * bs - 1], bs * sizeof(dst[0]));
        dst += stride;
      }
      return;
    }

    for (c = 0; c < bs; ++c, ++base) {
hui su's avatar
hui su committed
495
      if (base < 2 * bs - 1) {
hui su's avatar
hui su committed
496
497
        val = above[base] * (256 - shift) + above[base + 1] * shift;
        val = ROUND_POWER_OF_TWO(val, 8);
hui su's avatar
hui su committed
498
499
500
501
502
503
504
505
506
507
        dst[c] = clip_pixel(val);
      } else {
        dst[c] = above[2 * bs - 1];
      }
    }
  }
}

// Directional prediction, zone 2: 90 < angle < 180
static void dr_prediction_z2(uint8_t *dst, ptrdiff_t stride, int bs,
clang-format's avatar
clang-format committed
508
509
                             const uint8_t *above, const uint8_t *left, int dx,
                             int dy, INTRA_FILTER filter_type) {
Debargha Mukherjee's avatar
Debargha Mukherjee committed
510
  int r, c, x, y, shift1, shift2, val, base1, base2;
hui su's avatar
hui su committed
511
512
513
514

  assert(dx > 0);
  assert(dy > 0);

hui su's avatar
hui su committed
515
516
517
518
519
520
521
  x = -dx;
  for (r = 0; r < bs; ++r, x -= dx, dst += stride) {
    base1 = x >> 8;
    y = (r << 8) - dy;
    for (c = 0; c < bs; ++c, ++base1, y -= dy) {
      if (base1 >= -1) {
        shift1 = x & 0xFF;
clang-format's avatar
clang-format committed
522
523
        val =
            intra_subpel_interp(base1, shift1, above, -1, bs - 1, filter_type);
hui su's avatar
hui su committed
524
      } else {
hui su's avatar
hui su committed
525
526
527
        base2 = y >> 8;
        if (base2 >= 0) {
          shift2 = y & 0xFF;
clang-format's avatar
clang-format committed
528
529
          val =
              intra_subpel_interp(base2, shift2, left, 0, bs - 1, filter_type);
hui su's avatar
hui su committed
530
        } else {
531
          val = left[0];
hui su's avatar
hui su committed
532
533
534
535
536
537
538
539
540
        }
      }
      dst[c] = clip_pixel(val);
    }
  }
}

// Directional prediction, zone 3: 180 < angle < 270
static void dr_prediction_z3(uint8_t *dst, ptrdiff_t stride, int bs,
clang-format's avatar
clang-format committed
541
542
                             const uint8_t *above, const uint8_t *left, int dx,
                             int dy, INTRA_FILTER filter_type) {
hui su's avatar
hui su committed
543
  int r, c, y, base, shift, val;
hui su's avatar
hui su committed
544
545
546

  (void)above;
  (void)dx;
547

hui su's avatar
hui su committed
548
549
550
  assert(dx == 1);
  assert(dy < 0);

hui su's avatar
hui su committed
551
552
553
  if (filter_type != INTRA_FILTER_LINEAR) {
    const int pad_size = SUBPEL_TAPS >> 1;
    int len, i;
554
555
    DECLARE_ALIGNED(16, uint8_t, buf[MAX_SB_SIZE][4 * SUBPEL_SHIFTS]);
    DECLARE_ALIGNED(16, uint8_t, src[(MAX_SB_SIZE + SUBPEL_TAPS) * 4]);
hui su's avatar
hui su committed
556
557
558
    uint8_t flags[SUBPEL_SHIFTS];

    memset(flags, 0, SUBPEL_SHIFTS * sizeof(flags[0]));
clang-format's avatar
clang-format committed
559
560
    for (i = 0; i < pad_size; ++i) src[4 * i] = left[0];
    for (i = 0; i < 2 * bs; ++i) src[4 * (i + pad_size)] = left[i];
hui su's avatar
hui su committed
561
562
563
564
565
    for (i = 0; i < pad_size; ++i)
      src[4 * (i + 2 * bs + pad_size)] = left[2 * bs - 1];
    flags[0] = 1;
    y = -dy;
    for (c = 0; c < bs; ++c, y -= dy) {
hui su's avatar
hui su committed
566
      base = y >> 8;
hui su's avatar
hui su committed
567
568
569
570
571
572
      shift = y & 0xFF;
      shift = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS);
      if (shift == SUBPEL_SHIFTS) {
        base += 1;
        shift = 0;
      }
Yaowu Xu's avatar
Yaowu Xu committed
573
      len = AOMMIN(bs, 2 * bs - 1 - base);
hui su's avatar
hui su committed
574
575

      if (len <= 0) {
Debargha Mukherjee's avatar
Debargha Mukherjee committed
576
        for (r = 0; r < bs; ++r) {
clang-format's avatar
clang-format committed
577
          dst[r * stride + c] = left[2 * bs - 1];
hui su's avatar
hui su committed
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
        }
        continue;
      }

      if (len <= (bs >> 1) && !flags[shift]) {
        base = y >> 8;
        shift = y & 0xFF;
        for (r = 0; r < len; ++r) {
          val = intra_subpel_interp(base, shift, left, 0, 2 * bs - 1,
                                    filter_type);
          dst[r * stride + c] = clip_pixel(val);
          ++base;
        }
      } else {
        if (!flags[shift]) {
Yaowu Xu's avatar
Yaowu Xu committed
593
594
          const int16_t *filter = av1_intra_filter_kernels[filter_type][shift];
          aom_convolve8_vert(src + 4 * pad_size, 4, buf[0] + 4 * shift,
clang-format's avatar
clang-format committed
595
                             4 * SUBPEL_SHIFTS, NULL, 16, filter, 16,
hui su's avatar
hui su committed
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
                             2 * bs < 16 ? 4 : 4, 2 * bs);
          flags[shift] = 1;
        }

        if (shift == 0) {
          for (r = 0; r < len; ++r) {
            dst[r * stride + c] = left[r + base];
          }
        } else {
          for (r = 0; r < len; ++r) {
            dst[r * stride + c] = buf[r + base][4 * shift];
          }
        }
      }

      if (len < bs) {
        for (r = len; r < bs; ++r) {
clang-format's avatar
clang-format committed
613
          dst[r * stride + c] = left[2 * bs - 1];
hui su's avatar
hui su committed
614
615
616
617
618
619
620
621
622
623
624
625
626
        }
      }
    }
    return;
  }

  // For linear filter, C code is faster.
  y = -dy;
  for (c = 0; c < bs; ++c, y -= dy) {
    base = y >> 8;
    shift = y & 0xFF;

    for (r = 0; r < bs; ++r, ++base) {
627
      if (base < 2 * bs - 1) {
hui su's avatar
hui su committed
628
629
630
        val = left[base] * (256 - shift) + left[base + 1] * shift;
        val = ROUND_POWER_OF_TWO(val, 8);
        dst[r * stride + c] = clip_pixel(val);
hui su's avatar
hui su committed
631
      } else {
clang-format's avatar
clang-format committed
632
        for (; r < bs; ++r) dst[r * stride + c] = left[2 * bs - 1];
hui su's avatar
hui su committed
633
        break;
hui su's avatar
hui su committed
634
635
636
637
638
      }
    }
  }
}

639
640
641
642
// Get the shift (up-scaled by 256) in X w.r.t a unit change in Y.
// If angle > 0 && angle < 90, dx = -((int)(256 / t));
// If angle > 90 && angle < 180, dx = (int)(256 / t);
// If angle > 180 && angle < 270, dx = 1;
Yaowu Xu's avatar
Yaowu Xu committed
643
static INLINE int get_dx(int angle) {
644
645
646
647
648
649
650
651
652
653
654
655
656
657
  if (angle > 0 && angle < 90) {
    return -dr_intra_derivative[angle];
  } else if (angle > 90 && angle < 180) {
    return dr_intra_derivative[180 - angle];
  } else {
    // In this case, we are not really going to use dx. We may return any value.
    return 1;
  }
}

// Get the shift (up-scaled by 256) in Y w.r.t a unit change in X.
// If angle > 0 && angle < 90, dy = 1;
// If angle > 90 && angle < 180, dy = (int)(256 * t);
// If angle > 180 && angle < 270, dy = -((int)(256 * t));
Yaowu Xu's avatar
Yaowu Xu committed
658
static INLINE int get_dy(int angle) {
659
  if (angle > 90 && angle < 180) {
clang-format's avatar
clang-format committed
660
661
662
663
    return dr_intra_derivative[angle - 90];
  } else if (angle > 180 && angle < 270) {
    return -dr_intra_derivative[270 - angle];
  } else {
664
665
666
667
668
    // In this case, we are not really going to use dy. We may return any value.
    return 1;
  }
}

hui su's avatar
hui su committed
669
static void dr_predictor(uint8_t *dst, ptrdiff_t stride, TX_SIZE tx_size,
670
671
                         const uint8_t *above, const uint8_t *left, int angle,
                         INTRA_FILTER filter_type) {
672
673
  const int dx = get_dx(angle);
  const int dy = get_dy(angle);
674
  const int bs = 4 * num_4x4_blocks_wide_txsize_lookup[tx_size];
675
676
  assert(angle > 0 && angle < 270);

hui su's avatar
hui su committed
677
  if (angle > 0 && angle < 90) {
678
    dr_prediction_z1(dst, stride, bs, above, left, dx, dy, filter_type);
hui su's avatar
hui su committed
679
  } else if (angle > 90 && angle < 180) {
680
    dr_prediction_z2(dst, stride, bs, above, left, dx, dy, filter_type);
hui su's avatar
hui su committed
681
  } else if (angle > 180 && angle < 270) {
682
    dr_prediction_z3(dst, stride, bs, above, left, dx, dy, filter_type);
hui su's avatar
hui su committed
683
  } else if (angle == 90) {
hui su's avatar
hui su committed
684
    pred[V_PRED][tx_size](dst, stride, above, left);
hui su's avatar
hui su committed
685
  } else if (angle == 180) {
hui su's avatar
hui su committed
686
    pred[H_PRED][tx_size](dst, stride, above, left);
hui su's avatar
hui su committed
687
688
689
690
691
  }
}

static void filter_intra_predictors_4tap(uint8_t *dst, ptrdiff_t stride, int bs,
                                         const uint8_t *above,
clang-format's avatar
clang-format committed
692
                                         const uint8_t *left, int mode) {
hui su's avatar
hui su committed
693
694
695
  int k, r, c;
  int pred[33][65];
  int mean, ipred;
clang-format's avatar
clang-format committed
696
697
698
  const TX_SIZE tx_size =
      (bs == 32) ? TX_32X32
                 : ((bs == 16) ? TX_16X16 : ((bs == 8) ? TX_8X8 : (TX_4X4)));
hui su's avatar
hui su committed
699
700
701
702
703
704
705
706
707
708
709
710
711
712
  const int c0 = filter_intra_taps_4[tx_size][mode][0];
  const int c1 = filter_intra_taps_4[tx_size][mode][1];
  const int c2 = filter_intra_taps_4[tx_size][mode][2];
  const int c3 = filter_intra_taps_4[tx_size][mode][3];

  k = 0;
  mean = 0;
  while (k < bs) {
    mean = mean + (int)left[k];
    mean = mean + (int)above[k];
    k++;
  }
  mean = (mean + bs) / (2 * bs);

clang-format's avatar
clang-format committed
713
  for (r = 0; r < bs; ++r) pred[r + 1][0] = (int)left[r] - mean;
hui su's avatar
hui su committed
714

clang-format's avatar
clang-format committed
715
  for (c = 0; c < 2 * bs + 1; ++c) pred[0][c] = (int)above[c - 1] - mean;
hui su's avatar
hui su committed
716
717
718
719

  for (r = 1; r < bs + 1; ++r)
    for (c = 1; c < 2 * bs + 1 - r; ++c) {
      ipred = c0 * pred[r - 1][c] + c1 * pred[r][c - 1] +
clang-format's avatar
clang-format committed
720
              c2 * pred[r - 1][c - 1] + c3 * pred[r - 1][c + 1];
721
      pred[r][c] = ROUND_POWER_OF_TWO_SIGNED(ipred, FILTER_INTRA_PREC_BITS);
hui su's avatar
hui su committed
722
723
724
725
726
727
728
729
730
731
732
    }

  for (r = 0; r < bs; ++r) {
    for (c = 0; c < bs; ++c) {
      ipred = pred[r + 1][c + 1] + mean;
      dst[c] = clip_pixel(ipred);
    }
    dst += stride;
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
733
734
void av1_dc_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                               const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
735
736
737
  filter_intra_predictors_4tap(dst, stride, bs, above, left, DC_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
738
739
void av1_v_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                              const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
740
741
742
  filter_intra_predictors_4tap(dst, stride, bs, above, left, V_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
743
744
void av1_h_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                              const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
745
746
747
  filter_intra_predictors_4tap(dst, stride, bs, above, left, H_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
748
749
void av1_d45_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                                const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
750
751
752
  filter_intra_predictors_4tap(dst, stride, bs, above, left, D45_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
753
754
void av1_d135_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                                 const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
755
756
757
  filter_intra_predictors_4tap(dst, stride, bs, above, left, D135_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
758
759
void av1_d117_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                                 const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
760
761
762
  filter_intra_predictors_4tap(dst, stride, bs, above, left, D117_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
763
764
void av1_d153_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                                 const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
765
766
767
  filter_intra_predictors_4tap(dst, stride, bs, above, left, D153_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
768
769
void av1_d207_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                                 const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
770
771
772
  filter_intra_predictors_4tap(dst, stride, bs, above, left, D207_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
773
774
void av1_d63_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                                const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
775
776
777
  filter_intra_predictors_4tap(dst, stride, bs, above, left, D63_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
778
779
void av1_tm_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                               const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
780
781
782
  filter_intra_predictors_4tap(dst, stride, bs, above, left, TM_PRED);
}

clang-format's avatar
clang-format committed
783
784
785
static void filter_intra_predictors(int mode, uint8_t *dst, ptrdiff_t stride,
                                    int bs, const uint8_t *above,
                                    const uint8_t *left) {
786
  switch (mode) {
Yaowu Xu's avatar
Yaowu Xu committed
787
788
789
    case DC_PRED: av1_dc_filter_predictor(dst, stride, bs, above, left); break;
    case V_PRED: av1_v_filter_predictor(dst, stride, bs, above, left); break;
    case H_PRED: av1_h_filter_predictor(dst, stride, bs, above, left); break;
790
    case D45_PRED:
Yaowu Xu's avatar
Yaowu Xu committed
791
      av1_d45_filter_predictor(dst, stride, bs, above, left);
792
793
      break;
    case D135_PRED:
Yaowu Xu's avatar
Yaowu Xu committed
794
      av1_d135_filter_predictor(dst, stride, bs, above, left);
795
796
      break;
    case D117_PRED:
Yaowu Xu's avatar
Yaowu Xu committed
797
      av1_d117_filter_predictor(dst, stride, bs, above, left);
798
799
      break;
    case D153_PRED:
Yaowu Xu's avatar
Yaowu Xu committed
800
      av1_d153_filter_predictor(dst, stride, bs, above, left);
801
802
      break;
    case D207_PRED:
Yaowu Xu's avatar
Yaowu Xu committed
803
      av1_d207_filter_predictor(dst, stride, bs, above, left);
804
805
      break;
    case D63_PRED:
Yaowu Xu's avatar
Yaowu Xu committed
806
      av1_d63_filter_predictor(dst, stride, bs, above, left);
807
      break;
Yaowu Xu's avatar
Yaowu Xu committed
808
    case TM_PRED: av1_tm_filter_predictor(dst, stride, bs, above, left); break;
clang-format's avatar
clang-format committed
809
    default: assert(0);
810
811
  }
}
hui su's avatar
hui su committed
812

Yaowu Xu's avatar
Yaowu Xu committed
813
#if CONFIG_AOM_HIGHBITDEPTH
814
815
816
817
818
819
820
821
822
823
824
static int highbd_intra_subpel_interp(int base, int shift, const uint16_t *ref,
                                      int ref_start_idx, int ref_end_idx,
                                      INTRA_FILTER filter_type) {
  int val, k, idx, filter_idx = 0;
  const int16_t *filter = NULL;

  if (filter_type == INTRA_FILTER_LINEAR) {
    val = ref[base] * (256 - shift) + ref[base + 1] * shift;
    val = ROUND_POWER_OF_TWO(val, 8);
  } else {
    filter_idx = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS);
Yaowu Xu's avatar
Yaowu Xu committed
825
    filter = av1_intra_filter_kernels[filter_type][filter_idx];
826
827
828
829
830

    if (filter_idx < (1 << SUBPEL_BITS)) {
      val = 0;
      for (k = 0; k < SUBPEL_TAPS; ++k) {
        idx = base + 1 - (SUBPEL_TAPS / 2) + k;
Yaowu Xu's avatar
Yaowu Xu committed
831
        idx = AOMMAX(AOMMIN(idx, ref_end_idx), ref_start_idx);
832
833
834
835
836
837
838
839
840
841
842
        val += ref[idx] * filter[k];
      }
      val = ROUND_POWER_OF_TWO(val, FILTER_BITS);
    } else {
      val = ref[base + 1];
    }
  }

  return val;
}

hui su's avatar
hui su committed
843
844
845
// Directional prediction, zone 1: 0 < angle < 90
static void highbd_dr_prediction_z1(uint16_t *dst, ptrdiff_t stride, int bs,
                                    const uint16_t *above, const uint16_t *left,
846
847
                                    int dx, int dy, int bd,
                                    INTRA_FILTER filter_type) {
hui su's avatar
hui su committed
848
849
850
851
852
853
854
855
856
857
  int r, c, x, y, base, shift, val;

  (void)left;
  (void)dy;
  assert(dy == 1);
  assert(dx < 0);

  for (r = 0; r < bs; ++r) {
    y = r + 1;
    for (c = 0; c < bs; ++c) {
858
      x = (c << 8) - y * dx;
hui su's avatar
hui su committed
859
      base = x >> 8;
860
      shift = x - (base << 8);
hui su's avatar
hui su committed
861
      if (base < 2 * bs - 1) {
862
863
        val = highbd_intra_subpel_interp(base, shift, above, 0, 2 * bs - 1,
                                         filter_type);
hui su's avatar
hui su committed
864
865
866
867
868
869
870
871
872
873
874
875
        dst[c] = clip_pixel_highbd(val, bd);
      } else {
        dst[c] = above[2 * bs - 1];
      }
    }
    dst += stride;
  }
}

// Directional prediction, zone 2: 90 < angle < 180
static void highbd_dr_prediction_z2(uint16_t *dst, ptrdiff_t stride, int bs,
                                    const uint16_t *above, const uint16_t *left,
876
877
878
                                    int dx, int dy, int bd,
                                    INTRA_FILTER filter_type) {
  int r, c, x, y, shift, val, base;
hui su's avatar
hui su committed
879
880
881
882
883
884
885

  assert(dx > 0);
  assert(dy > 0);

  for (r = 0; r < bs; ++r) {
    for (c = 0; c < bs; ++c) {
      y = r + 1;
886
887
888
889
890
891
      x = (c << 8) - y * dx;
      base = x >> 8;
      if (base >= -1) {
        shift = x - (base << 8);
        val = highbd_intra_subpel_interp(base, shift, above, -1, bs - 1,
                                         filter_type);
hui su's avatar
hui su committed
892
893
      } else {
        x = c + 1;
894
        y = (r << 8) - x * dy;
hui su's avatar
hui su committed
895
896
        base = y >> 8;
        if (base >= 0) {
clang-format's avatar
clang-format committed
897
          shift = y - (base << 8);
898
899
          val = highbd_intra_subpel_interp(base, shift, left, 0, bs - 1,
                                           filter_type);
hui su's avatar
hui su committed
900
        } else {
901
          val = left[0];
hui su's avatar
hui su committed
902
903
904
905
906
907
908
909
910
911
912
        }
      }
      dst[c] = clip_pixel_highbd(val, bd);
    }
    dst += stride;
  }
}

// Directional prediction, zone 3: 180 < angle < 270
static void highbd_dr_prediction_z3(uint16_t *dst, ptrdiff_t stride, int bs,
                                    const uint16_t *above, const uint16_t *left,
913
914
                                    int dx, int dy, int bd,
                                    INTRA_FILTER filter_type) {
hui su's avatar
hui su committed
915
916
917
918
919
920
921
922
923
924
  int r, c, x, y, base, shift, val;

  (void)above;
  (void)dx;
  assert(dx == 1);
  assert(dy < 0);

  for (r = 0; r < bs; ++r) {
    for (c = 0; c < bs; ++c) {
      x = c + 1;
925
      y = (r << 8) - x * dy;
hui su's avatar
hui su committed
926
      base = y >> 8;
927
      shift = y - (base << 8);
928
      if (base < 2 * bs - 1) {
929
930
        val = highbd_intra_subpel_interp(base, shift, left, 0, 2 * bs - 1,
                                         filter_type);
hui su's avatar
hui su committed
931
932
        dst[c] = clip_pixel_highbd(val, bd);
      } else {
933
        dst[c] = left[2 * bs - 1];
hui su's avatar
hui su committed
934
935
936
937
938
939
      }
    }
    dst += stride;
  }
}

clang-format's avatar
clang-format committed
940
941
static INLINE void highbd_v_predictor(uint16_t *dst, ptrdiff_t stride, int bs,
                                      const uint16_t *above,
hui su's avatar
hui su committed
942
943
                                      const uint16_t *left, int bd) {
  int r;
clang-format's avatar
clang-format committed
944
945
  (void)left;
  (void)bd;
hui su's avatar
hui su committed
946
947
948
949
950
951
  for (r = 0; r < bs; r++) {
    memcpy(dst, above, bs * sizeof(uint16_t));
    dst += stride;
  }
}

clang-format's avatar
clang-format committed
952
953
static INLINE void highbd_h_predictor(uint16_t *dst, ptrdiff_t stride, int bs,
                                      const uint16_t *above,
hui su's avatar
hui su committed
954
955
                                      const uint16_t *left, int bd) {
  int r;
clang-format's avatar
clang-format committed
956
957
  (void)above;
  (void)bd;
hui su's avatar
hui su committed
958
  for (r = 0; r < bs; r++) {
Yaowu Xu's avatar
Yaowu Xu committed
959
    aom_memset16(dst, left[r], bs);
hui su's avatar
hui su committed
960
961
962
963
964
965
    dst += stride;
  }
}

static void highbd_dr_predictor(uint16_t *dst, ptrdiff_t stride, int bs,
                                const uint16_t *above, const uint16_t *left,
966
                                int angle, int bd, INTRA_FILTER filter) {
967
968
  const int dx = get_dx(angle);
  const int dy = get_dy(angle);
969
  assert(angle > 0 && angle < 270);
hui su's avatar
hui su committed
970
971

  if (angle > 0 && angle < 90) {
972
    highbd_dr_prediction_z1(dst, stride, bs, above, left, dx, dy, bd, filter);
hui su's avatar
hui su committed
973
  } else if (angle > 90 && angle < 180) {
974
    highbd_dr_prediction_z2(dst, stride, bs, above, left, dx, dy, bd, filter);
hui su's avatar
hui su committed
975
  } else if (angle > 180 && angle < 270) {
976
    highbd_dr_prediction_z3(dst, stride, bs, above, left, dx, dy, bd, filter);
hui su's avatar
hui su committed
977
978
979
980
981
982
983
984
985
986
987
988
989
990
  } else if (angle == 90) {
    highbd_v_predictor(dst, stride, bs, above, left, bd);
  } else if (angle == 180) {
    highbd_h_predictor(dst, stride, bs, above, left, bd);
  }
}

static void highbd_filter_intra_predictors_4tap(uint16_t *dst, ptrdiff_t stride,
                                                int bs, const uint16_t *above,
                                                const uint16_t *left, int mode,
                                                int bd) {
  int k, r, c;
  int pred[33][65];
  int mean, ipred;
clang-format's avatar
clang-format committed
991
992
993
  const TX_SIZE tx_size =
      (bs == 32) ? TX_32X32
                 : ((bs == 16) ? TX_16X16 : ((bs == 8) ? TX_8X8 : (TX_4X4)));
hui su's avatar
hui su committed
994
995
996
997
998
999
1000
  const int c0 = filter_intra_taps_4[tx_size][mode][0];
  const int c1 = filter_intra_taps_4[tx_size][mode][1];
  const int c2 = filter_intra_taps_4[tx_size][mode][2];
  const int c3 = filter_intra_taps_4[tx_size][mode][3];

  k = 0;
  mean = 0;