reconintra.c 55.8 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11
 */

hui su's avatar
hui su committed
12 13
#include <math.h>

Yaowu Xu's avatar
Yaowu Xu committed
14 15 16
#include "./av1_rtcd.h"
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
17
#include "aom_ports/system_state.h"
Jingning Han's avatar
Jingning Han committed
18

Yaowu Xu's avatar
Yaowu Xu committed
19 20 21 22
#if CONFIG_AOM_HIGHBITDEPTH
#include "aom_dsp/aom_dsp_common.h"
#endif  // CONFIG_AOM_HIGHBITDEPTH
#include "aom_mem/aom_mem.h"
23
#include "aom_ports/mem.h"
Yaowu Xu's avatar
Yaowu Xu committed
24
#include "aom_ports/aom_once.h"
25
#if CONFIG_EXT_INTRA
26
#include "av1/common/intra_filters.h"
27
#endif
28 29
#include "av1/common/reconintra.h"
#include "av1/common/onyxc_int.h"
Jingning Han's avatar
Jingning Han committed
30

31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50
enum {
  NEED_LEFT = 1 << 1,
  NEED_ABOVE = 1 << 2,
  NEED_ABOVERIGHT = 1 << 3,
  NEED_ABOVELEFT = 1 << 4,
  NEED_BOTTOMLEFT = 1 << 5,
};

static const uint8_t extend_modes[INTRA_MODES] = {
  NEED_ABOVE | NEED_LEFT,                   // DC
  NEED_ABOVE,                               // V
  NEED_LEFT,                                // H
  NEED_ABOVE | NEED_ABOVERIGHT,             // D45
  NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT,  // D135
  NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT,  // D117
  NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT,  // D153
  NEED_LEFT | NEED_BOTTOMLEFT,              // D207
  NEED_ABOVE | NEED_ABOVERIGHT,             // D63
  NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT,  // TM
};
Jingning Han's avatar
Jingning Han committed
51

52 53 54 55
static const uint8_t orders_128x128[1] = { 0 };
static const uint8_t orders_128x64[2] = { 0, 1 };
static const uint8_t orders_64x128[2] = { 0, 1 };
static const uint8_t orders_64x64[4] = {
clang-format's avatar
clang-format committed
56
  0, 1, 2, 3,
57
};
58
static const uint8_t orders_64x32[8] = {
clang-format's avatar
clang-format committed
59
  0, 2, 1, 3, 4, 6, 5, 7,
60
};
61
static const uint8_t orders_32x64[8] = {
clang-format's avatar
clang-format committed
62
  0, 1, 2, 3, 4, 5, 6, 7,
63
};
64
static const uint8_t orders_32x32[16] = {
clang-format's avatar
clang-format committed
65
  0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15,
66
};
67
static const uint8_t orders_32x16[32] = {
clang-format's avatar
clang-format committed
68 69
  0,  2,  8,  10, 1,  3,  9,  11, 4,  6,  12, 14, 5,  7,  13, 15,
  16, 18, 24, 26, 17, 19, 25, 27, 20, 22, 28, 30, 21, 23, 29, 31,
70
};
71
static const uint8_t orders_16x32[32] = {
clang-format's avatar
clang-format committed
72 73
  0,  1,  2,  3,  8,  9,  10, 11, 4,  5,  6,  7,  12, 13, 14, 15,
  16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31,
74
};
75
static const uint8_t orders_16x16[64] = {
clang-format's avatar
clang-format committed
76 77 78 79
  0,  1,  4,  5,  16, 17, 20, 21, 2,  3,  6,  7,  18, 19, 22, 23,
  8,  9,  12, 13, 24, 25, 28, 29, 10, 11, 14, 15, 26, 27, 30, 31,
  32, 33, 36, 37, 48, 49, 52, 53, 34, 35, 38, 39, 50, 51, 54, 55,
  40, 41, 44, 45, 56, 57, 60, 61, 42, 43, 46, 47, 58, 59, 62, 63,
80
};
81 82 83

#if CONFIG_EXT_PARTITION
static const uint8_t orders_16x8[128] = {
clang-format's avatar
clang-format committed
84 85 86 87 88 89 90 91
  0,  2,  8,  10, 32,  34,  40,  42,  1,  3,  9,  11, 33,  35,  41,  43,
  4,  6,  12, 14, 36,  38,  44,  46,  5,  7,  13, 15, 37,  39,  45,  47,
  16, 18, 24, 26, 48,  50,  56,  58,  17, 19, 25, 27, 49,  51,  57,  59,
  20, 22, 28, 30, 52,  54,  60,  62,  21, 23, 29, 31, 53,  55,  61,  63,
  64, 66, 72, 74, 96,  98,  104, 106, 65, 67, 73, 75, 97,  99,  105, 107,
  68, 70, 76, 78, 100, 102, 108, 110, 69, 71, 77, 79, 101, 103, 109, 111,
  80, 82, 88, 90, 112, 114, 120, 122, 81, 83, 89, 91, 113, 115, 121, 123,
  84, 86, 92, 94, 116, 118, 124, 126, 85, 87, 93, 95, 117, 119, 125, 127,
92 93
};
static const uint8_t orders_8x16[128] = {
clang-format's avatar
clang-format committed
94 95 96 97 98
  0,  1,  2,  3,  8,  9,  10, 11, 32,  33,  34,  35,  40,  41,  42,  43,
  4,  5,  6,  7,  12, 13, 14, 15, 36,  37,  38,  39,  44,  45,  46,  47,
  16, 17, 18, 19, 24, 25, 26, 27, 48,  49,  50,  51,  56,  57,  58,  59,
  20, 21, 22, 23, 28, 29, 30, 31, 52,  53,  54,  55,  60,  61,  62,  63,
  64, 65, 66, 67, 72, 73, 74, 75, 96,  97,  98,  99,  104, 105, 106, 107,
99 100 101 102 103
  68, 69, 70, 71, 76, 77, 78, 79, 100, 101, 102, 103, 108, 109, 110, 111,
  80, 81, 82, 83, 88, 89, 90, 91, 112, 113, 114, 115, 120, 121, 122, 123,
  84, 85, 86, 87, 92, 93, 94, 95, 116, 117, 118, 119, 124, 125, 126, 127,
};
static const uint8_t orders_8x8[256] = {
clang-format's avatar
clang-format committed
104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121
  0,   1,   4,   5,   16,  17,  20,  21,  64,  65,  68,  69,  80,  81,  84,
  85,  2,   3,   6,   7,   18,  19,  22,  23,  66,  67,  70,  71,  82,  83,
  86,  87,  8,   9,   12,  13,  24,  25,  28,  29,  72,  73,  76,  77,  88,
  89,  92,  93,  10,  11,  14,  15,  26,  27,  30,  31,  74,  75,  78,  79,
  90,  91,  94,  95,  32,  33,  36,  37,  48,  49,  52,  53,  96,  97,  100,
  101, 112, 113, 116, 117, 34,  35,  38,  39,  50,  51,  54,  55,  98,  99,
  102, 103, 114, 115, 118, 119, 40,  41,  44,  45,  56,  57,  60,  61,  104,
  105, 108, 109, 120, 121, 124, 125, 42,  43,  46,  47,  58,  59,  62,  63,
  106, 107, 110, 111, 122, 123, 126, 127, 128, 129, 132, 133, 144, 145, 148,
  149, 192, 193, 196, 197, 208, 209, 212, 213, 130, 131, 134, 135, 146, 147,
  150, 151, 194, 195, 198, 199, 210, 211, 214, 215, 136, 137, 140, 141, 152,
  153, 156, 157, 200, 201, 204, 205, 216, 217, 220, 221, 138, 139, 142, 143,
  154, 155, 158, 159, 202, 203, 206, 207, 218, 219, 222, 223, 160, 161, 164,
  165, 176, 177, 180, 181, 224, 225, 228, 229, 240, 241, 244, 245, 162, 163,
  166, 167, 178, 179, 182, 183, 226, 227, 230, 231, 242, 243, 246, 247, 168,
  169, 172, 173, 184, 185, 188, 189, 232, 233, 236, 237, 248, 249, 252, 253,
  170, 171, 174, 175, 186, 187, 190, 191, 234, 235, 238, 239, 250, 251, 254,
  255,
122 123
};

clang-format's avatar
clang-format committed
124
/* clang-format off */
125
static const uint8_t *const orders[BLOCK_SIZES] = {
126 127 128 129 130 131 132 133 134 135 136 137
  //                              4X4
                                  orders_8x8,
  // 4X8,         8X4,            8X8
  orders_8x8,     orders_8x8,     orders_8x8,
  // 8X16,        16X8,           16X16
  orders_8x16,    orders_16x8,    orders_16x16,
  // 16X32,       32X16,          32X32
  orders_16x32,   orders_32x16,   orders_32x32,
  // 32X64,       64X32,          64X64
  orders_32x64,   orders_64x32,   orders_64x64,
  // 64x128,      128x64,         128x128
  orders_64x128,  orders_128x64,  orders_128x128
138
};
clang-format's avatar
clang-format committed
139
/* clang-format on */
140
#else
clang-format's avatar
clang-format committed
141
/* clang-format off */
142 143 144 145 146 147 148 149 150 151 152 153
static const uint8_t *const orders[BLOCK_SIZES] = {
  //                              4X4
                                  orders_16x16,
  // 4X8,         8X4,            8X8
  orders_16x16,   orders_16x16,   orders_16x16,
  // 8X16,        16X8,           16X16
  orders_16x32,   orders_32x16,   orders_32x32,
  // 16X32,       32X16,          32X32
  orders_32x64,   orders_64x32,   orders_64x64,
  // 32X64,       64X32,          64X64
  orders_64x128,  orders_128x64,  orders_128x128
};
clang-format's avatar
clang-format committed
154
/* clang-format on */
155 156
#endif  // CONFIG_EXT_PARTITION

157
#if CONFIG_EXT_PARTITION_TYPES
158
static const uint8_t orders_verta_64x64[4] = {
clang-format's avatar
clang-format committed
159
  0, 2, 1, 2,
160
};
161
static const uint8_t orders_verta_32x32[16] = {
clang-format's avatar
clang-format committed
162
  0, 2, 4, 6, 1, 2, 5, 6, 8, 10, 12, 14, 9, 10, 13, 14,
163
};
164
static const uint8_t orders_verta_16x16[64] = {
clang-format's avatar
clang-format committed
165 166 167 168
  0,  2,  4,  6,  16, 18, 20, 22, 1,  2,  5,  6,  17, 18, 21, 22,
  8,  10, 12, 14, 24, 26, 28, 30, 9,  10, 13, 14, 25, 26, 29, 30,
  32, 34, 36, 38, 48, 50, 52, 54, 33, 34, 37, 38, 49, 50, 53, 54,
  40, 42, 44, 46, 56, 58, 60, 62, 41, 42, 45, 46, 57, 58, 61, 62,
169
};
170 171
#if CONFIG_EXT_PARTITION
static const uint8_t orders_verta_8x8[256] = {
clang-format's avatar
clang-format committed
172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
  0,   2,   4,   6,   16,  18,  20,  22,  64,  66,  68,  70,  80,  82,  84,
  86,  1,   2,   5,   6,   17,  18,  21,  22,  65,  66,  69,  70,  81,  82,
  85,  86,  8,   10,  12,  14,  24,  26,  28,  30,  72,  74,  76,  78,  88,
  90,  92,  94,  9,   10,  13,  14,  25,  26,  29,  30,  73,  74,  77,  78,
  89,  90,  93,  94,  32,  34,  36,  38,  48,  50,  52,  54,  96,  98,  100,
  102, 112, 114, 116, 118, 33,  34,  37,  38,  49,  50,  53,  54,  97,  98,
  101, 102, 113, 114, 117, 118, 40,  42,  44,  46,  56,  58,  60,  62,  104,
  106, 108, 110, 120, 122, 124, 126, 41,  42,  45,  46,  57,  58,  61,  62,
  105, 106, 109, 110, 121, 122, 125, 126, 128, 130, 132, 134, 144, 146, 148,
  150, 192, 194, 196, 198, 208, 210, 212, 214, 129, 130, 133, 134, 145, 146,
  149, 150, 193, 194, 197, 198, 209, 210, 213, 214, 136, 138, 140, 142, 152,
  154, 156, 158, 200, 202, 204, 206, 216, 218, 220, 222, 137, 138, 141, 142,
  153, 154, 157, 158, 201, 202, 205, 206, 217, 218, 221, 222, 160, 162, 164,
  166, 176, 178, 180, 182, 224, 226, 228, 230, 240, 242, 244, 246, 161, 162,
  165, 166, 177, 178, 181, 182, 225, 226, 229, 230, 241, 242, 245, 246, 168,
  170, 172, 174, 184, 186, 188, 190, 232, 234, 236, 238, 248, 250, 252, 254,
  169, 170, 173, 174, 185, 186, 189, 190, 233, 234, 237, 238, 249, 250, 253,
  254,
190
};
clang-format's avatar
clang-format committed
191 192

/* clang-format off */
193 194 195 196 197 198 199 200 201 202 203 204 205 206
static const uint8_t *const orders_verta[BLOCK_SIZES] = {
  //                                  4X4
                                      orders_verta_8x8,
  // 4X8,           8X4,              8X8
  orders_verta_8x8, orders_verta_8x8, orders_verta_8x8,
  // 8X16,          16X8,             16X16
  orders_8x16,      orders_16x8,      orders_verta_16x16,
  // 16X32,         32X16,            32X32
  orders_16x32,     orders_32x16,     orders_verta_32x32,
  // 32X64,         64X32,            64X64
  orders_32x64,     orders_64x32,     orders_verta_64x64,
  // 64x128,        128x64,           128x128
  orders_64x128,    orders_128x64,    orders_128x128
};
clang-format's avatar
clang-format committed
207
/* clang-format on */
208
#else
clang-format's avatar
clang-format committed
209
/* clang-format off */
210
static const uint8_t *const orders_verta[BLOCK_SIZES] = {
211 212 213 214 215 216 217 218 219 220
  //                                      4X4
                                          orders_verta_16x16,
  // 4X8,             8X4,                8X8
  orders_verta_16x16, orders_verta_16x16, orders_verta_16x16,
  // 8X16,            16X8,               16X16
  orders_16x32,       orders_32x16,       orders_verta_32x32,
  // 16X32,           32X16,              32X32
  orders_32x64,       orders_64x32,       orders_verta_64x64,
  // 32X64,           64X32,              64X64
  orders_64x128,      orders_128x64,      orders_128x128
221
};
clang-format's avatar
clang-format committed
222
/* clang-format on */
223
#endif  // CONFIG_EXT_PARTITION
224
#endif  // CONFIG_EXT_PARTITION_TYPES
225

Yaowu Xu's avatar
Yaowu Xu committed
226 227
static int av1_has_right(BLOCK_SIZE bsize, int mi_row, int mi_col,
                         int right_available,
228
#if CONFIG_EXT_PARTITION_TYPES
Yaowu Xu's avatar
Yaowu Xu committed
229
                         PARTITION_TYPE partition,
230
#endif
Yaowu Xu's avatar
Yaowu Xu committed
231
                         TX_SIZE txsz, int y, int x, int ss_x) {
hui su's avatar
hui su committed
232
  const int wl = mi_width_log2_lookup[bsize];
Yaowu Xu's avatar
Yaowu Xu committed
233
  const int w = AOMMAX(num_4x4_blocks_wide_lookup[bsize] >> ss_x, 1);
hui su's avatar
hui su committed
234 235
  const int step = 1 << txsz;

236 237 238 239
  if (!right_available) {
    return 0;
  } else {
    // Handle block size 4x8 and 4x4
clang-format's avatar
clang-format committed
240
    if (ss_x == 0 && num_4x4_blocks_wide_lookup[bsize] < 2 && x == 0) return 1;
241

242 243 244 245
    if (y == 0) {
      const int hl = mi_height_log2_lookup[bsize];
      const uint8_t *order;
      int my_order, tr_order;
246
#if CONFIG_EXT_PARTITION_TYPES
247 248 249
      if (partition == PARTITION_VERT_A)
        order = orders_verta[bsize];
      else
250
#endif  // CONFIG_EXT_PARTITION_TYPES
clang-format's avatar
clang-format committed
251
        order = orders[bsize];
252

clang-format's avatar
clang-format committed
253
      if (x + step < w) return 1;
254

255 256
      mi_row = (mi_row & MAX_MIB_MASK) >> hl;
      mi_col = (mi_col & MAX_MIB_MASK) >> wl;
257

258
      // If top row of coding unit
clang-format's avatar
clang-format committed
259
      if (mi_row == 0) return 1;
260

261
      // If rightmost column of coding unit
clang-format's avatar
clang-format committed
262
      if (((mi_col + 1) << wl) >= MAX_MIB_SIZE) return 0;
263

264 265
      my_order = order[((mi_row + 0) << (MAX_MIB_SIZE_LOG2 - wl)) + mi_col + 0];
      tr_order = order[((mi_row - 1) << (MAX_MIB_SIZE_LOG2 - wl)) + mi_col + 1];
266

267 268 269 270
      return my_order > tr_order;
    } else {
      return x + step < w;
    }
271 272 273
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
274 275 276
static int av1_has_bottom(BLOCK_SIZE bsize, int mi_row, int mi_col,
                          int bottom_available, TX_SIZE txsz, int y, int x,
                          int ss_y) {
277 278 279
  if (!bottom_available || x != 0) {
    return 0;
  } else {
hui su's avatar
hui su committed
280 281 282 283
    const int wl = mi_width_log2_lookup[bsize];
    const int hl = mi_height_log2_lookup[bsize];
    const int h = 1 << (hl + 1 - ss_y);
    const int step = 1 << txsz;
284 285 286
    const uint8_t *order = orders[bsize];
    int my_order, bl_order;

287
    // Handle block size 8x4 and 4x4
clang-format's avatar
clang-format committed
288
    if (ss_y == 0 && num_4x4_blocks_high_lookup[bsize] < 2 && y == 0) return 1;
289

clang-format's avatar
clang-format committed
290
    if (y + step < h) return 1;
291

292 293
    mi_row = (mi_row & MAX_MIB_MASK) >> hl;
    mi_col = (mi_col & MAX_MIB_MASK) >> wl;
294 295

    if (mi_col == 0)
296
      return (mi_row << (hl + !ss_y)) + y + step < (MAX_MIB_SIZE << !ss_y);
297

clang-format's avatar
clang-format committed
298
    if (((mi_row + 1) << hl) >= MAX_MIB_SIZE) return 0;
299

300 301
    my_order = order[((mi_row + 0) << (MAX_MIB_SIZE_LOG2 - wl)) + mi_col + 0];
    bl_order = order[((mi_row + 1) << (MAX_MIB_SIZE_LOG2 - wl)) + mi_col - 1];
302

303
    return bl_order < my_order;
304 305
  }
}
Jingning Han's avatar
Jingning Han committed
306 307 308 309 310 311 312

typedef void (*intra_pred_fn)(uint8_t *dst, ptrdiff_t stride,
                              const uint8_t *above, const uint8_t *left);

static intra_pred_fn pred[INTRA_MODES][TX_SIZES];
static intra_pred_fn dc_pred[2][2][TX_SIZES];

Yaowu Xu's avatar
Yaowu Xu committed
313
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
314 315 316 317 318
typedef void (*intra_high_pred_fn)(uint16_t *dst, ptrdiff_t stride,
                                   const uint16_t *above, const uint16_t *left,
                                   int bd);
static intra_high_pred_fn pred_high[INTRA_MODES][4];
static intra_high_pred_fn dc_pred_high[2][2][4];
Yaowu Xu's avatar
Yaowu Xu committed
319
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
320

Yaowu Xu's avatar
Yaowu Xu committed
321
static void av1_init_intra_predictors_internal(void) {
clang-format's avatar
clang-format committed
322
#define INIT_NO_4X4(p, type)                  \
Yaowu Xu's avatar
Yaowu Xu committed
323 324 325
  p[TX_8X8] = aom_##type##_predictor_8x8;     \
  p[TX_16X16] = aom_##type##_predictor_16x16; \
  p[TX_32X32] = aom_##type##_predictor_32x32
Jingning Han's avatar
Jingning Han committed
326

clang-format's avatar
clang-format committed
327
#define INIT_ALL_SIZES(p, type)           \
Yaowu Xu's avatar
Yaowu Xu committed
328
  p[TX_4X4] = aom_##type##_predictor_4x4; \
329 330
  INIT_NO_4X4(p, type)

Jingning Han's avatar
Jingning Han committed
331 332
  INIT_ALL_SIZES(pred[V_PRED], v);
  INIT_ALL_SIZES(pred[H_PRED], h);
333 334 335
  INIT_ALL_SIZES(pred[D207_PRED], d207e);
  INIT_ALL_SIZES(pred[D45_PRED], d45e);
  INIT_ALL_SIZES(pred[D63_PRED], d63e);
Jingning Han's avatar
Jingning Han committed
336 337 338
  INIT_ALL_SIZES(pred[D117_PRED], d117);
  INIT_ALL_SIZES(pred[D135_PRED], d135);
  INIT_ALL_SIZES(pred[D153_PRED], d153);
Urvang Joshi's avatar
Urvang Joshi committed
339 340 341 342

#if CONFIG_ALT_INTRA
  INIT_ALL_SIZES(pred[TM_PRED], paeth);
#else
Jingning Han's avatar
Jingning Han committed
343
  INIT_ALL_SIZES(pred[TM_PRED], tm);
Urvang Joshi's avatar
Urvang Joshi committed
344
#endif  // CONFIG_ALT_INTRA
Jingning Han's avatar
Jingning Han committed
345 346 347 348 349 350

  INIT_ALL_SIZES(dc_pred[0][0], dc_128);
  INIT_ALL_SIZES(dc_pred[0][1], dc_top);
  INIT_ALL_SIZES(dc_pred[1][0], dc_left);
  INIT_ALL_SIZES(dc_pred[1][1], dc);

Yaowu Xu's avatar
Yaowu Xu committed
351
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
352 353
  INIT_ALL_SIZES(pred_high[V_PRED], highbd_v);
  INIT_ALL_SIZES(pred_high[H_PRED], highbd_h);
354 355 356
  INIT_ALL_SIZES(pred_high[D207_PRED], highbd_d207e);
  INIT_ALL_SIZES(pred_high[D45_PRED], highbd_d45e);
  INIT_ALL_SIZES(pred_high[D63_PRED], highbd_d63e);
Jingning Han's avatar
Jingning Han committed
357 358 359
  INIT_ALL_SIZES(pred_high[D117_PRED], highbd_d117);
  INIT_ALL_SIZES(pred_high[D135_PRED], highbd_d135);
  INIT_ALL_SIZES(pred_high[D153_PRED], highbd_d153);
Urvang Joshi's avatar
Urvang Joshi committed
360 361 362 363

#if CONFIG_ALT_INTRA
  INIT_ALL_SIZES(pred_high[TM_PRED], highbd_paeth);
#else
Jingning Han's avatar
Jingning Han committed
364
  INIT_ALL_SIZES(pred_high[TM_PRED], highbd_tm);
Urvang Joshi's avatar
Urvang Joshi committed
365
#endif  // CONFIG_ALT_INTRA
Jingning Han's avatar
Jingning Han committed
366 367 368 369 370

  INIT_ALL_SIZES(dc_pred_high[0][0], highbd_dc_128);
  INIT_ALL_SIZES(dc_pred_high[0][1], highbd_dc_top);
  INIT_ALL_SIZES(dc_pred_high[1][0], highbd_dc_left);
  INIT_ALL_SIZES(dc_pred_high[1][1], highbd_dc);
Yaowu Xu's avatar
Yaowu Xu committed
371
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
372 373 374 375

#undef intra_pred_allsizes
}

hui su's avatar
hui su committed
376 377 378
#if CONFIG_EXT_INTRA

static const uint8_t ext_intra_extend_modes[FILTER_INTRA_MODES] = {
clang-format's avatar
clang-format committed
379 380 381 382 383 384 385 386 387 388
  NEED_LEFT | NEED_ABOVE,  // FILTER_DC
  NEED_LEFT | NEED_ABOVE,  // FILTER_V
  NEED_LEFT | NEED_ABOVE,  // FILTER_H
  NEED_LEFT | NEED_ABOVE,  // FILTER_D45
  NEED_LEFT | NEED_ABOVE,  // FILTER_D135
  NEED_LEFT | NEED_ABOVE,  // FILTER_D117
  NEED_LEFT | NEED_ABOVE,  // FILTER_D153
  NEED_LEFT | NEED_ABOVE,  // FILTER_D207
  NEED_LEFT | NEED_ABOVE,  // FILTER_D63
  NEED_LEFT | NEED_ABOVE,  // FILTER_TM
hui su's avatar
hui su committed
389 390
};

391 392 393 394 395 396 397 398 399 400 401
static int intra_subpel_interp(int base, int shift, const uint8_t *ref,
                               int ref_start_idx, int ref_end_idx,
                               INTRA_FILTER filter_type) {
  int val, k, idx, filter_idx = 0;
  const int16_t *filter = NULL;

  if (filter_type == INTRA_FILTER_LINEAR) {
    val = ref[base] * (256 - shift) + ref[base + 1] * shift;
    val = ROUND_POWER_OF_TWO(val, 8);
  } else {
    filter_idx = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS);
Yaowu Xu's avatar
Yaowu Xu committed
402
    filter = av1_intra_filter_kernels[filter_type][filter_idx];
403 404 405 406 407

    if (filter_idx < (1 << SUBPEL_BITS)) {
      val = 0;
      for (k = 0; k < SUBPEL_TAPS; ++k) {
        idx = base + 1 - (SUBPEL_TAPS / 2) + k;
Yaowu Xu's avatar
Yaowu Xu committed
408
        idx = AOMMAX(AOMMIN(idx, ref_end_idx), ref_start_idx);
409 410 411 412 413 414 415 416 417 418 419
        val += ref[idx] * filter[k];
      }
      val = ROUND_POWER_OF_TWO(val, FILTER_BITS);
    } else {
      val = ref[base + 1];
    }
  }

  return val;
}

hui su's avatar
hui su committed
420 421
// Directional prediction, zone 1: 0 < angle < 90
static void dr_prediction_z1(uint8_t *dst, ptrdiff_t stride, int bs,
clang-format's avatar
clang-format committed
422 423
                             const uint8_t *above, const uint8_t *left, int dx,
                             int dy, INTRA_FILTER filter_type) {
hui su's avatar
hui su committed
424
  int r, c, x, base, shift, val;
hui su's avatar
hui su committed
425 426 427 428 429 430

  (void)left;
  (void)dy;
  assert(dy == 1);
  assert(dx < 0);

hui su's avatar
hui su committed
431 432 433
  if (filter_type != INTRA_FILTER_LINEAR) {
    const int pad_size = SUBPEL_TAPS >> 1;
    int len;
434 435
    DECLARE_ALIGNED(16, uint8_t, buf[SUBPEL_SHIFTS][MAX_SB_SIZE]);
    DECLARE_ALIGNED(16, uint8_t, src[MAX_SB_SIZE + SUBPEL_TAPS]);
hui su's avatar
hui su committed
436 437 438 439 440 441 442 443 444 445
    uint8_t flags[SUBPEL_SHIFTS];

    memset(flags, 0, SUBPEL_SHIFTS * sizeof(flags[0]));
    memset(src, above[0], pad_size * sizeof(above[0]));
    memcpy(src + pad_size, above, 2 * bs * sizeof(above[0]));
    memset(src + pad_size + 2 * bs, above[2 * bs - 1],
           pad_size * sizeof(above[0]));
    flags[0] = 1;
    x = -dx;
    for (r = 0; r < bs; ++r, dst += stride, x -= dx) {
hui su's avatar
hui su committed
446
      base = x >> 8;
hui su's avatar
hui su committed
447 448 449 450 451 452
      shift = x & 0xFF;
      shift = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS);
      if (shift == SUBPEL_SHIFTS) {
        base += 1;
        shift = 0;
      }
Yaowu Xu's avatar
Yaowu Xu committed
453
      len = AOMMIN(bs, 2 * bs - 1 - base);
hui su's avatar
hui su committed
454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473
      if (len <= 0) {
        int i;
        for (i = r; i < bs; ++i) {
          memset(dst, above[2 * bs - 1], bs * sizeof(dst[0]));
          dst += stride;
        }
        return;
      }

      if (len <= (bs >> 1) && !flags[shift]) {
        base = x >> 8;
        shift = x & 0xFF;
        for (c = 0; c < len; ++c) {
          val = intra_subpel_interp(base, shift, above, 0, 2 * bs - 1,
                                    filter_type);
          dst[c] = clip_pixel(val);
          ++base;
        }
      } else {
        if (!flags[shift]) {
Yaowu Xu's avatar
Yaowu Xu committed
474 475
          const int16_t *filter = av1_intra_filter_kernels[filter_type][shift];
          aom_convolve8_horiz(src + pad_size, 2 * bs, buf[shift], 2 * bs,
clang-format's avatar
clang-format committed
476 477
                              filter, 16, NULL, 16, 2 * bs,
                              2 * bs < 16 ? 2 : 1);
hui su's avatar
hui su committed
478 479 480
          flags[shift] = 1;
        }
        memcpy(dst, shift == 0 ? src + pad_size + base : &buf[shift][base],
clang-format's avatar
clang-format committed
481
               len * sizeof(dst[0]));
hui su's avatar
hui su committed
482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505
      }

      if (len < bs)
        memset(dst + len, above[2 * bs - 1], (bs - len) * sizeof(dst[0]));
    }
    return;
  }

  // For linear filter, C code is faster.
  x = -dx;
  for (r = 0; r < bs; ++r, dst += stride, x -= dx) {
    base = x >> 8;
    shift = x & 0xFF;

    if (base >= 2 * bs - 1) {
      int i;
      for (i = r; i < bs; ++i) {
        memset(dst, above[2 * bs - 1], bs * sizeof(dst[0]));
        dst += stride;
      }
      return;
    }

    for (c = 0; c < bs; ++c, ++base) {
hui su's avatar
hui su committed
506
      if (base < 2 * bs - 1) {
hui su's avatar
hui su committed
507 508
        val = above[base] * (256 - shift) + above[base + 1] * shift;
        val = ROUND_POWER_OF_TWO(val, 8);
hui su's avatar
hui su committed
509 510 511 512 513 514 515 516 517 518
        dst[c] = clip_pixel(val);
      } else {
        dst[c] = above[2 * bs - 1];
      }
    }
  }
}

// Directional prediction, zone 2: 90 < angle < 180
static void dr_prediction_z2(uint8_t *dst, ptrdiff_t stride, int bs,
clang-format's avatar
clang-format committed
519 520
                             const uint8_t *above, const uint8_t *left, int dx,
                             int dy, INTRA_FILTER filter_type) {
Debargha Mukherjee's avatar
Debargha Mukherjee committed
521
  int r, c, x, y, shift1, shift2, val, base1, base2;
hui su's avatar
hui su committed
522 523 524 525

  assert(dx > 0);
  assert(dy > 0);

hui su's avatar
hui su committed
526 527 528 529 530 531 532
  x = -dx;
  for (r = 0; r < bs; ++r, x -= dx, dst += stride) {
    base1 = x >> 8;
    y = (r << 8) - dy;
    for (c = 0; c < bs; ++c, ++base1, y -= dy) {
      if (base1 >= -1) {
        shift1 = x & 0xFF;
clang-format's avatar
clang-format committed
533 534
        val =
            intra_subpel_interp(base1, shift1, above, -1, bs - 1, filter_type);
hui su's avatar
hui su committed
535
      } else {
hui su's avatar
hui su committed
536 537 538
        base2 = y >> 8;
        if (base2 >= 0) {
          shift2 = y & 0xFF;
clang-format's avatar
clang-format committed
539 540
          val =
              intra_subpel_interp(base2, shift2, left, 0, bs - 1, filter_type);
hui su's avatar
hui su committed
541
        } else {
542
          val = left[0];
hui su's avatar
hui su committed
543 544 545 546 547 548 549 550 551
        }
      }
      dst[c] = clip_pixel(val);
    }
  }
}

// Directional prediction, zone 3: 180 < angle < 270
static void dr_prediction_z3(uint8_t *dst, ptrdiff_t stride, int bs,
clang-format's avatar
clang-format committed
552 553
                             const uint8_t *above, const uint8_t *left, int dx,
                             int dy, INTRA_FILTER filter_type) {
hui su's avatar
hui su committed
554
  int r, c, y, base, shift, val;
hui su's avatar
hui su committed
555 556 557

  (void)above;
  (void)dx;
558

hui su's avatar
hui su committed
559 560 561
  assert(dx == 1);
  assert(dy < 0);

hui su's avatar
hui su committed
562 563 564
  if (filter_type != INTRA_FILTER_LINEAR) {
    const int pad_size = SUBPEL_TAPS >> 1;
    int len, i;
565 566
    DECLARE_ALIGNED(16, uint8_t, buf[MAX_SB_SIZE][4 * SUBPEL_SHIFTS]);
    DECLARE_ALIGNED(16, uint8_t, src[(MAX_SB_SIZE + SUBPEL_TAPS) * 4]);
hui su's avatar
hui su committed
567 568 569
    uint8_t flags[SUBPEL_SHIFTS];

    memset(flags, 0, SUBPEL_SHIFTS * sizeof(flags[0]));
clang-format's avatar
clang-format committed
570 571
    for (i = 0; i < pad_size; ++i) src[4 * i] = left[0];
    for (i = 0; i < 2 * bs; ++i) src[4 * (i + pad_size)] = left[i];
hui su's avatar
hui su committed
572 573 574 575 576
    for (i = 0; i < pad_size; ++i)
      src[4 * (i + 2 * bs + pad_size)] = left[2 * bs - 1];
    flags[0] = 1;
    y = -dy;
    for (c = 0; c < bs; ++c, y -= dy) {
hui su's avatar
hui su committed
577
      base = y >> 8;
hui su's avatar
hui su committed
578 579 580 581 582 583
      shift = y & 0xFF;
      shift = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS);
      if (shift == SUBPEL_SHIFTS) {
        base += 1;
        shift = 0;
      }
Yaowu Xu's avatar
Yaowu Xu committed
584
      len = AOMMIN(bs, 2 * bs - 1 - base);
hui su's avatar
hui su committed
585 586

      if (len <= 0) {
Debargha Mukherjee's avatar
Debargha Mukherjee committed
587
        for (r = 0; r < bs; ++r) {
clang-format's avatar
clang-format committed
588
          dst[r * stride + c] = left[2 * bs - 1];
hui su's avatar
hui su committed
589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
        }
        continue;
      }

      if (len <= (bs >> 1) && !flags[shift]) {
        base = y >> 8;
        shift = y & 0xFF;
        for (r = 0; r < len; ++r) {
          val = intra_subpel_interp(base, shift, left, 0, 2 * bs - 1,
                                    filter_type);
          dst[r * stride + c] = clip_pixel(val);
          ++base;
        }
      } else {
        if (!flags[shift]) {
Yaowu Xu's avatar
Yaowu Xu committed
604 605
          const int16_t *filter = av1_intra_filter_kernels[filter_type][shift];
          aom_convolve8_vert(src + 4 * pad_size, 4, buf[0] + 4 * shift,
clang-format's avatar
clang-format committed
606
                             4 * SUBPEL_SHIFTS, NULL, 16, filter, 16,
hui su's avatar
hui su committed
607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623
                             2 * bs < 16 ? 4 : 4, 2 * bs);
          flags[shift] = 1;
        }

        if (shift == 0) {
          for (r = 0; r < len; ++r) {
            dst[r * stride + c] = left[r + base];
          }
        } else {
          for (r = 0; r < len; ++r) {
            dst[r * stride + c] = buf[r + base][4 * shift];
          }
        }
      }

      if (len < bs) {
        for (r = len; r < bs; ++r) {
clang-format's avatar
clang-format committed
624
          dst[r * stride + c] = left[2 * bs - 1];
hui su's avatar
hui su committed
625 626 627 628 629 630 631 632 633 634 635 636 637
        }
      }
    }
    return;
  }

  // For linear filter, C code is faster.
  y = -dy;
  for (c = 0; c < bs; ++c, y -= dy) {
    base = y >> 8;
    shift = y & 0xFF;

    for (r = 0; r < bs; ++r, ++base) {
638
      if (base < 2 * bs - 1) {
hui su's avatar
hui su committed
639 640 641
        val = left[base] * (256 - shift) + left[base + 1] * shift;
        val = ROUND_POWER_OF_TWO(val, 8);
        dst[r * stride + c] = clip_pixel(val);
hui su's avatar
hui su committed
642
      } else {
clang-format's avatar
clang-format committed
643
        for (; r < bs; ++r) dst[r * stride + c] = left[2 * bs - 1];
hui su's avatar
hui su committed
644
        break;
hui su's avatar
hui su committed
645 646 647 648 649
      }
    }
  }
}

650 651 652 653
// Get the shift (up-scaled by 256) in X w.r.t a unit change in Y.
// If angle > 0 && angle < 90, dx = -((int)(256 / t));
// If angle > 90 && angle < 180, dx = (int)(256 / t);
// If angle > 180 && angle < 270, dx = 1;
Yaowu Xu's avatar
Yaowu Xu committed
654
static INLINE int get_dx(int angle) {
655 656 657 658 659 660 661 662 663 664 665 666 667 668
  if (angle > 0 && angle < 90) {
    return -dr_intra_derivative[angle];
  } else if (angle > 90 && angle < 180) {
    return dr_intra_derivative[180 - angle];
  } else {
    // In this case, we are not really going to use dx. We may return any value.
    return 1;
  }
}

// Get the shift (up-scaled by 256) in Y w.r.t a unit change in X.
// If angle > 0 && angle < 90, dy = 1;
// If angle > 90 && angle < 180, dy = (int)(256 * t);
// If angle > 180 && angle < 270, dy = -((int)(256 * t));
Yaowu Xu's avatar
Yaowu Xu committed
669
static INLINE int get_dy(int angle) {
670
  if (angle > 90 && angle < 180) {
clang-format's avatar
clang-format committed
671 672 673 674
    return dr_intra_derivative[angle - 90];
  } else if (angle > 180 && angle < 270) {
    return -dr_intra_derivative[270 - angle];
  } else {
675 676 677 678 679
    // In this case, we are not really going to use dy. We may return any value.
    return 1;
  }
}

hui su's avatar
hui su committed
680
static void dr_predictor(uint8_t *dst, ptrdiff_t stride, TX_SIZE tx_size,
681 682
                         const uint8_t *above, const uint8_t *left, int angle,
                         INTRA_FILTER filter_type) {
683 684
  const int dx = get_dx(angle);
  const int dy = get_dy(angle);
685
  const int bs = 4 * num_4x4_blocks_wide_txsize_lookup[tx_size];
686 687
  assert(angle > 0 && angle < 270);

hui su's avatar
hui su committed
688
  if (angle > 0 && angle < 90) {
689
    dr_prediction_z1(dst, stride, bs, above, left, dx, dy, filter_type);
hui su's avatar
hui su committed
690
  } else if (angle > 90 && angle < 180) {
691
    dr_prediction_z2(dst, stride, bs, above, left, dx, dy, filter_type);
hui su's avatar
hui su committed
692
  } else if (angle > 180 && angle < 270) {
693
    dr_prediction_z3(dst, stride, bs, above, left, dx, dy, filter_type);
hui su's avatar
hui su committed
694
  } else if (angle == 90) {
hui su's avatar
hui su committed
695
    pred[V_PRED][tx_size](dst, stride, above, left);
hui su's avatar
hui su committed
696
  } else if (angle == 180) {
hui su's avatar
hui su committed
697
    pred[H_PRED][tx_size](dst, stride, above, left);
hui su's avatar
hui su committed
698 699 700 701 702
  }
}

static void filter_intra_predictors_4tap(uint8_t *dst, ptrdiff_t stride, int bs,
                                         const uint8_t *above,
clang-format's avatar
clang-format committed
703
                                         const uint8_t *left, int mode) {
hui su's avatar
hui su committed
704 705 706
  int k, r, c;
  int pred[33][65];
  int mean, ipred;
clang-format's avatar
clang-format committed
707 708 709
  const TX_SIZE tx_size =
      (bs == 32) ? TX_32X32
                 : ((bs == 16) ? TX_16X16 : ((bs == 8) ? TX_8X8 : (TX_4X4)));
hui su's avatar
hui su committed
710 711 712 713 714 715 716 717 718 719 720 721 722 723
  const int c0 = filter_intra_taps_4[tx_size][mode][0];
  const int c1 = filter_intra_taps_4[tx_size][mode][1];
  const int c2 = filter_intra_taps_4[tx_size][mode][2];
  const int c3 = filter_intra_taps_4[tx_size][mode][3];

  k = 0;
  mean = 0;
  while (k < bs) {
    mean = mean + (int)left[k];
    mean = mean + (int)above[k];
    k++;
  }
  mean = (mean + bs) / (2 * bs);

clang-format's avatar
clang-format committed
724
  for (r = 0; r < bs; ++r) pred[r + 1][0] = (int)left[r] - mean;
hui su's avatar
hui su committed
725

clang-format's avatar
clang-format committed
726
  for (c = 0; c < 2 * bs + 1; ++c) pred[0][c] = (int)above[c - 1] - mean;
hui su's avatar
hui su committed
727 728 729 730

  for (r = 1; r < bs + 1; ++r)
    for (c = 1; c < 2 * bs + 1 - r; ++c) {
      ipred = c0 * pred[r - 1][c] + c1 * pred[r][c - 1] +
clang-format's avatar
clang-format committed
731
              c2 * pred[r - 1][c - 1] + c3 * pred[r - 1][c + 1];
732
      pred[r][c] = ROUND_POWER_OF_TWO_SIGNED(ipred, FILTER_INTRA_PREC_BITS);
hui su's avatar
hui su committed
733 734 735 736 737 738 739 740 741 742 743
    }

  for (r = 0; r < bs; ++r) {
    for (c = 0; c < bs; ++c) {
      ipred = pred[r + 1][c + 1] + mean;
      dst[c] = clip_pixel(ipred);
    }
    dst += stride;
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
744 745
void av1_dc_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                               const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
746 747 748
  filter_intra_predictors_4tap(dst, stride, bs, above, left, DC_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
749 750
void av1_v_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                              const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
751 752 753
  filter_intra_predictors_4tap(dst, stride, bs, above, left, V_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
754 755
void av1_h_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                              const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
756 757 758
  filter_intra_predictors_4tap(dst, stride, bs, above, left, H_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
759 760
void av1_d45_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                                const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
761 762 763
  filter_intra_predictors_4tap(dst, stride, bs, above, left, D45_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
764 765
void av1_d135_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                                 const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
766 767 768
  filter_intra_predictors_4tap(dst, stride, bs, above, left, D135_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
769 770
void av1_d117_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                                 const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
771 772 773
  filter_intra_predictors_4tap(dst, stride, bs, above, left, D117_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
774 775
void av1_d153_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                                 const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
776 777 778
  filter_intra_predictors_4tap(dst, stride, bs, above, left, D153_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
779 780
void av1_d207_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                                 const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
781 782 783
  filter_intra_predictors_4tap(dst, stride, bs, above, left, D207_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
784 785
void av1_d63_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                                const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
786 787 788
  filter_intra_predictors_4tap(dst, stride, bs, above, left, D63_PRED);
}

Yaowu Xu's avatar
Yaowu Xu committed
789 790
void av1_tm_filter_predictor_c(uint8_t *dst, ptrdiff_t stride, int bs,
                               const uint8_t *above, const uint8_t *left) {
hui su's avatar
hui su committed
791 792 793
  filter_intra_predictors_4tap(dst, stride, bs, above, left, TM_PRED);
}

clang-format's avatar
clang-format committed
794 795 796
static void filter_intra_predictors(int mode, uint8_t *dst, ptrdiff_t stride,
                                    int bs, const uint8_t *above,
                                    const uint8_t *left) {
797
  switch (mode) {
Yaowu Xu's avatar
Yaowu Xu committed
798 799 800
    case DC_PRED: av1_dc_filter_predictor(dst, stride, bs, above, left); break;
    case V_PRED: av1_v_filter_predictor(dst, stride, bs, above, left); break;
    case H_PRED: av1_h_filter_predictor(dst, stride, bs, above, left); break;
801
    case D45_PRED:
Yaowu Xu's avatar
Yaowu Xu committed
802
      av1_d45_filter_predictor(dst, stride, bs, above, left);
803 804
      break;
    case D135_PRED:
Yaowu Xu's avatar
Yaowu Xu committed
805
      av1_d135_filter_predictor(dst, stride, bs, above, left);
806 807
      break;
    case D117_PRED:
Yaowu Xu's avatar
Yaowu Xu committed
808
      av1_d117_filter_predictor(dst, stride, bs, above, left);
809 810
      break;
    case D153_PRED:
Yaowu Xu's avatar
Yaowu Xu committed
811
      av1_d153_filter_predictor(dst, stride, bs, above, left);
812 813
      break;
    case D207_PRED:
Yaowu Xu's avatar
Yaowu Xu committed
814
      av1_d207_filter_predictor(dst, stride, bs, above, left);
815 816
      break;
    case D63_PRED:
Yaowu Xu's avatar
Yaowu Xu committed
817
      av1_d63_filter_predictor(dst, stride, bs, above, left);
818
      break;
Yaowu Xu's avatar
Yaowu Xu committed
819
    case TM_PRED: av1_tm_filter_predictor(dst, stride, bs, above, left); break;
clang-format's avatar
clang-format committed
820
    default: assert(0);
821 822
  }
}
hui su's avatar
hui su committed
823

Yaowu Xu's avatar
Yaowu Xu committed
824
#if CONFIG_AOM_HIGHBITDEPTH
825 826 827 828 829 830 831 832 833 834 835
static int highbd_intra_subpel_interp(int base, int shift, const uint16_t *ref,
                                      int ref_start_idx, int ref_end_idx,
                                      INTRA_FILTER filter_type) {
  int val, k, idx, filter_idx = 0;
  const int16_t *filter = NULL;

  if (filter_type == INTRA_FILTER_LINEAR) {
    val = ref[base] * (256 - shift) + ref[base + 1] * shift;
    val = ROUND_POWER_OF_TWO(val, 8);
  } else {
    filter_idx = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS);
Yaowu Xu's avatar
Yaowu Xu committed
836
    filter = av1_intra_filter_kernels[filter_type][filter_idx];
837 838 839 840 841

    if (filter_idx < (1 << SUBPEL_BITS)) {
      val = 0;
      for (k = 0; k < SUBPEL_TAPS; ++k) {
        idx = base + 1 - (SUBPEL_TAPS / 2) + k;
Yaowu Xu's avatar
Yaowu Xu committed
842
        idx = AOMMAX(AOMMIN(idx, ref_end_idx), ref_start_idx);
843 844 845 846 847 848 849 850 851 852 853
        val += ref[idx] * filter[k];
      }
      val = ROUND_POWER_OF_TWO(val, FILTER_BITS);
    } else {
      val = ref[base + 1];
    }
  }

  return val;
}

hui su's avatar
hui su committed
854 855 856
// Directional prediction, zone 1: 0 < angle < 90
static void highbd_dr_prediction_z1(uint16_t *dst, ptrdiff_t stride, int bs,
                                    const uint16_t *above, const uint16_t *left,
857 858
                                    int dx, int dy, int bd,
                                    INTRA_FILTER filter_type) {
hui su's avatar
hui su committed
859 860 861 862 863 864 865 866 867 868
  int r, c, x, y, base, shift, val;

  (void)left;
  (void)dy;
  assert(dy == 1);
  assert(dx < 0);

  for (r = 0; r < bs; ++r) {
    y = r + 1;
    for (c = 0; c < bs; ++c) {
869
      x = (c << 8) - y * dx;
hui su's avatar
hui su committed
870
      base = x >> 8;
871
      shift = x & 0xFF;
hui su's avatar
hui su committed
872
      if (base < 2 * bs - 1) {
873 874
        val = highbd_intra_subpel_interp(base, shift, above, 0, 2 * bs - 1,
                                         filter_type);
hui su's avatar
hui su committed
875 876 877 878 879 880 881 882 883 884 885 886
        dst[c] = clip_pixel_highbd(val, bd);
      } else {
        dst[c] = above[2 * bs - 1];
      }
    }
    dst += stride;
  }
}

// Directional prediction, zone 2: 90 < angle < 180
static void highbd_dr_prediction_z2(uint16_t *dst, ptrdiff_t stride, int bs,
                                    const uint16_t *above, const uint16_t *left,
887 888 889
                                    int dx, int dy, int bd,
                                    INTRA_FILTER filter_type) {
  int r, c, x, y, shift, val, base;
hui su's avatar
hui su committed
890 891 892 893 894 895 896

  assert(dx > 0);
  assert(dy > 0);

  for (r = 0; r < bs; ++r) {
    for (c = 0; c < bs; ++c) {
      y = r + 1;
897 898 899
      x = (c << 8) - y * dx;
      base = x >> 8;
      if (base >= -1) {
900
        shift = x & 0xFF;
901 902
        val = highbd_intra_subpel_interp(base, shift, above, -1, bs - 1,
                                         filter_type);
hui su's avatar
hui su committed
903 904
      } else {
        x = c + 1;
905
        y = (r << 8) - x * dy;
hui su's avatar
hui su committed
906 907
        base = y >> 8;
        if (base >= 0) {
908
          shift = y & 0xFF;
909 910
          val = highbd_intra_subpel_interp(base, shift, left, 0, bs - 1,
                                           filter_type);
hui su's avatar
hui su committed
911
        } else {
912
          val = left[0];
hui su's avatar
hui su committed
913 914 915 916 917 918 919 920 921 922 923
        }
      }
      dst[c] = clip_pixel_highbd(val, bd);
    }
    dst += stride;
  }
}

// Directional prediction, zone 3: 180 < angle < 270
static void highbd_dr_prediction_z3(uint16_t *dst, ptrdiff_t stride, int bs,
                                    const uint16_t *above, const uint16_t *left,
924 925
                                    int dx, int dy, int bd,
                                    INTRA_FILTER filter_type) {
hui su's avatar
hui su committed
926 927 928 929 930 931 932 933 934 935
  int r, c, x, y, base, shift, val;

  (void)above;
  (void)dx;
  assert(dx == 1);
  assert(dy < 0);

  for (r = 0; r < bs; ++r) {
    for (c = 0; c < bs; ++c) {
      x = c + 1;
936
      y = (r << 8) - x * dy;
hui su's avatar
hui su committed
937
      base = y >> 8;
938
      shift = y & 0xFF;
939
      if (base < 2 * bs - 1) {
940 941
        val = highbd_intra_subpel_interp(base, shift, left, 0, 2 * bs - 1,
                                         filter_type);
hui su's avatar
hui su committed
942 943
        dst[c] = clip_pixel_highbd(val, bd);
      } else {
944
        dst[c] = left[2 * bs - 1];
hui su's avatar
hui su committed
945 946 947 948 949 950
      }
    }
    dst += stride;
  }
}

clang-format's avatar
clang-format committed
951 952
static INLINE void highbd_v_predictor(uint16_t *dst, ptrdiff_t stride, int bs,
                                      const uint16_t *above,
hui su's avatar
hui su committed
953 954
                                      const uint16_t *left, int bd) {
  int r;
clang-format's avatar
clang-format committed
955 956
  (void)left;
  (void)bd;
hui su's avatar
hui su committed
957 958 959 960 961 962
  for (r = 0; r < bs; r++) {
    memcpy(dst, above, bs * sizeof(uint16_t));
    dst += stride;
  }
}

clang-format's avatar
clang-format committed
963 964
static INLINE void highbd_h_predictor(uint16_t *dst, ptrdiff_t stride, int bs,
                                      const uint16_t *above,
hui su's avatar
hui su committed
965 966
                                      const uint16_t *left, int bd) {
  int r;
clang-format's avatar
clang-format committed
967 968
  (void)above;
  (void)bd;
hui su's avatar
hui su committed
969
  for (r = 0; r < bs; r++) {
Yaowu Xu's avatar
Yaowu Xu committed
970
    aom_memset16(dst, left[r], bs);
hui su's avatar
hui su committed
971 972 973 974 975 976
    dst += stride;
  }
}

static void highbd_dr_predictor(uint16_t *dst, ptrdiff_t stride, int bs,
                                const uint16_t *above, const uint16_t *left,
977
                                int angle, int bd, INTRA_FILTER filter) {
978 979
  const int dx = get_dx(angle);
  const int dy = get_dy(angle);
980
  assert(angle > 0 && angle < 270);
hui su's avatar
hui su committed
981 982

  if (angle > 0 && angle < 90) {