reconintra.c 97.1 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
 */

hui su's avatar
hui su committed
12
13
#include <math.h>

Yaowu Xu's avatar
Yaowu Xu committed
14
15
16
#include "./av1_rtcd.h"
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
17
#include "aom_ports/system_state.h"
Jingning Han's avatar
Jingning Han committed
18

19
#if CONFIG_HIGHBITDEPTH
Yaowu Xu's avatar
Yaowu Xu committed
20
#include "aom_dsp/aom_dsp_common.h"
21
#endif  // CONFIG_HIGHBITDEPTH
Yaowu Xu's avatar
Yaowu Xu committed
22
#include "aom_mem/aom_mem.h"
23
#include "aom_ports/mem.h"
Yaowu Xu's avatar
Yaowu Xu committed
24
#include "aom_ports/aom_once.h"
25
26
#include "av1/common/reconintra.h"
#include "av1/common/onyxc_int.h"
Luc Trudeau's avatar
Luc Trudeau committed
27
28
29
#if CONFIG_CFL
#include "av1/common/cfl.h"
#endif
Jingning Han's avatar
Jingning Han committed
30

31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
enum {
  NEED_LEFT = 1 << 1,
  NEED_ABOVE = 1 << 2,
  NEED_ABOVERIGHT = 1 << 3,
  NEED_ABOVELEFT = 1 << 4,
  NEED_BOTTOMLEFT = 1 << 5,
};

static const uint8_t extend_modes[INTRA_MODES] = {
  NEED_ABOVE | NEED_LEFT,                   // DC
  NEED_ABOVE,                               // V
  NEED_LEFT,                                // H
  NEED_ABOVE | NEED_ABOVERIGHT,             // D45
  NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT,  // D135
  NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT,  // D117
  NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT,  // D153
  NEED_LEFT | NEED_BOTTOMLEFT,              // D207
  NEED_ABOVE | NEED_ABOVERIGHT,             // D63
49
#if CONFIG_ALT_INTRA
Urvang Joshi's avatar
Urvang Joshi committed
50
51
52
53
54
  NEED_LEFT | NEED_ABOVE,  // SMOOTH
#if CONFIG_SMOOTH_HV
  NEED_LEFT | NEED_ABOVE,                   // SMOOTH_V
  NEED_LEFT | NEED_ABOVE,                   // SMOOTH_H
#endif                                      // CONFIG_SMOOTH_HV
55
#endif                                      // CONFIG_ALT_INTRA
56
57
  NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT,  // TM
};
Jingning Han's avatar
Jingning Han committed
58

59
60
61
62
static const uint16_t orders_128x128[1] = { 0 };
static const uint16_t orders_128x64[2] = { 0, 1 };
static const uint16_t orders_64x128[2] = { 0, 1 };
static const uint16_t orders_64x64[4] = {
clang-format's avatar
clang-format committed
63
  0, 1, 2, 3,
64
};
65
static const uint16_t orders_64x32[8] = {
clang-format's avatar
clang-format committed
66
  0, 2, 1, 3, 4, 6, 5, 7,
67
};
68
static const uint16_t orders_32x64[8] = {
clang-format's avatar
clang-format committed
69
  0, 1, 2, 3, 4, 5, 6, 7,
70
};
71
static const uint16_t orders_32x32[16] = {
clang-format's avatar
clang-format committed
72
  0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15,
73
};
74
static const uint16_t orders_32x16[32] = {
clang-format's avatar
clang-format committed
75
76
  0,  2,  8,  10, 1,  3,  9,  11, 4,  6,  12, 14, 5,  7,  13, 15,
  16, 18, 24, 26, 17, 19, 25, 27, 20, 22, 28, 30, 21, 23, 29, 31,
77
};
78
static const uint16_t orders_16x32[32] = {
clang-format's avatar
clang-format committed
79
80
  0,  1,  2,  3,  8,  9,  10, 11, 4,  5,  6,  7,  12, 13, 14, 15,
  16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31,
81
};
82
static const uint16_t orders_16x16[64] = {
clang-format's avatar
clang-format committed
83
84
85
86
  0,  1,  4,  5,  16, 17, 20, 21, 2,  3,  6,  7,  18, 19, 22, 23,
  8,  9,  12, 13, 24, 25, 28, 29, 10, 11, 14, 15, 26, 27, 30, 31,
  32, 33, 36, 37, 48, 49, 52, 53, 34, 35, 38, 39, 50, 51, 54, 55,
  40, 41, 44, 45, 56, 57, 60, 61, 42, 43, 46, 47, 58, 59, 62, 63,
87
};
88

89
#if CONFIG_CB4X4 || CONFIG_EXT_PARTITION
90
static const uint16_t orders_16x8[128] = {
clang-format's avatar
clang-format committed
91
92
93
94
95
96
97
98
  0,  2,  8,  10, 32,  34,  40,  42,  1,  3,  9,  11, 33,  35,  41,  43,
  4,  6,  12, 14, 36,  38,  44,  46,  5,  7,  13, 15, 37,  39,  45,  47,
  16, 18, 24, 26, 48,  50,  56,  58,  17, 19, 25, 27, 49,  51,  57,  59,
  20, 22, 28, 30, 52,  54,  60,  62,  21, 23, 29, 31, 53,  55,  61,  63,
  64, 66, 72, 74, 96,  98,  104, 106, 65, 67, 73, 75, 97,  99,  105, 107,
  68, 70, 76, 78, 100, 102, 108, 110, 69, 71, 77, 79, 101, 103, 109, 111,
  80, 82, 88, 90, 112, 114, 120, 122, 81, 83, 89, 91, 113, 115, 121, 123,
  84, 86, 92, 94, 116, 118, 124, 126, 85, 87, 93, 95, 117, 119, 125, 127,
99
};
100
static const uint16_t orders_8x16[128] = {
clang-format's avatar
clang-format committed
101
102
103
104
105
  0,  1,  2,  3,  8,  9,  10, 11, 32,  33,  34,  35,  40,  41,  42,  43,
  4,  5,  6,  7,  12, 13, 14, 15, 36,  37,  38,  39,  44,  45,  46,  47,
  16, 17, 18, 19, 24, 25, 26, 27, 48,  49,  50,  51,  56,  57,  58,  59,
  20, 21, 22, 23, 28, 29, 30, 31, 52,  53,  54,  55,  60,  61,  62,  63,
  64, 65, 66, 67, 72, 73, 74, 75, 96,  97,  98,  99,  104, 105, 106, 107,
106
107
108
109
  68, 69, 70, 71, 76, 77, 78, 79, 100, 101, 102, 103, 108, 109, 110, 111,
  80, 81, 82, 83, 88, 89, 90, 91, 112, 113, 114, 115, 120, 121, 122, 123,
  84, 85, 86, 87, 92, 93, 94, 95, 116, 117, 118, 119, 124, 125, 126, 127,
};
110
static const uint16_t orders_8x8[256] = {
clang-format's avatar
clang-format committed
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
  0,   1,   4,   5,   16,  17,  20,  21,  64,  65,  68,  69,  80,  81,  84,
  85,  2,   3,   6,   7,   18,  19,  22,  23,  66,  67,  70,  71,  82,  83,
  86,  87,  8,   9,   12,  13,  24,  25,  28,  29,  72,  73,  76,  77,  88,
  89,  92,  93,  10,  11,  14,  15,  26,  27,  30,  31,  74,  75,  78,  79,
  90,  91,  94,  95,  32,  33,  36,  37,  48,  49,  52,  53,  96,  97,  100,
  101, 112, 113, 116, 117, 34,  35,  38,  39,  50,  51,  54,  55,  98,  99,
  102, 103, 114, 115, 118, 119, 40,  41,  44,  45,  56,  57,  60,  61,  104,
  105, 108, 109, 120, 121, 124, 125, 42,  43,  46,  47,  58,  59,  62,  63,
  106, 107, 110, 111, 122, 123, 126, 127, 128, 129, 132, 133, 144, 145, 148,
  149, 192, 193, 196, 197, 208, 209, 212, 213, 130, 131, 134, 135, 146, 147,
  150, 151, 194, 195, 198, 199, 210, 211, 214, 215, 136, 137, 140, 141, 152,
  153, 156, 157, 200, 201, 204, 205, 216, 217, 220, 221, 138, 139, 142, 143,
  154, 155, 158, 159, 202, 203, 206, 207, 218, 219, 222, 223, 160, 161, 164,
  165, 176, 177, 180, 181, 224, 225, 228, 229, 240, 241, 244, 245, 162, 163,
  166, 167, 178, 179, 182, 183, 226, 227, 230, 231, 242, 243, 246, 247, 168,
  169, 172, 173, 184, 185, 188, 189, 232, 233, 236, 237, 248, 249, 252, 253,
  170, 171, 174, 175, 186, 187, 190, 191, 234, 235, 238, 239, 250, 251, 254,
  255,
129
};
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289

#if CONFIG_CB4X4 && CONFIG_EXT_PARTITION
static const uint16_t orders_4x8[512] = {
  0,   1,   2,   3,   8,   9,   10,  11,  32,  33,  34,  35,  40,  41,  42,
  43,  128, 129, 130, 131, 136, 137, 138, 139, 160, 161, 162, 163, 168, 169,
  170, 171, 4,   5,   6,   7,   12,  13,  14,  15,  36,  37,  38,  39,  44,
  45,  46,  47,  132, 133, 134, 135, 140, 141, 142, 143, 164, 165, 166, 167,
  172, 173, 174, 175, 16,  17,  18,  19,  24,  25,  26,  27,  48,  49,  50,
  51,  56,  57,  58,  59,  144, 145, 146, 147, 152, 153, 154, 155, 176, 177,
  178, 179, 184, 185, 186, 187, 20,  21,  22,  23,  28,  29,  30,  31,  52,
  53,  54,  55,  60,  61,  62,  63,  148, 149, 150, 151, 156, 157, 158, 159,
  180, 181, 182, 183, 188, 189, 190, 191, 64,  65,  66,  67,  72,  73,  74,
  75,  96,  97,  98,  99,  104, 105, 106, 107, 192, 193, 194, 195, 200, 201,
  202, 203, 224, 225, 226, 227, 232, 233, 234, 235, 68,  69,  70,  71,  76,
  77,  78,  79,  100, 101, 102, 103, 108, 109, 110, 111, 196, 197, 198, 199,
  204, 205, 206, 207, 228, 229, 230, 231, 236, 237, 238, 239, 80,  81,  82,
  83,  88,  89,  90,  91,  112, 113, 114, 115, 120, 121, 122, 123, 208, 209,
  210, 211, 216, 217, 218, 219, 240, 241, 242, 243, 248, 249, 250, 251, 84,
  85,  86,  87,  92,  93,  94,  95,  116, 117, 118, 119, 124, 125, 126, 127,
  212, 213, 214, 215, 220, 221, 222, 223, 244, 245, 246, 247, 252, 253, 254,
  255, 256, 257, 258, 259, 264, 265, 266, 267, 288, 289, 290, 291, 296, 297,
  298, 299, 384, 385, 386, 387, 392, 393, 394, 395, 416, 417, 418, 419, 424,
  425, 426, 427, 260, 261, 262, 263, 268, 269, 270, 271, 292, 293, 294, 295,
  300, 301, 302, 303, 388, 389, 390, 391, 396, 397, 398, 399, 420, 421, 422,
  423, 428, 429, 430, 431, 272, 273, 274, 275, 280, 281, 282, 283, 304, 305,
  306, 307, 312, 313, 314, 315, 400, 401, 402, 403, 408, 409, 410, 411, 432,
  433, 434, 435, 440, 441, 442, 443, 276, 277, 278, 279, 284, 285, 286, 287,
  308, 309, 310, 311, 316, 317, 318, 319, 404, 405, 406, 407, 412, 413, 414,
  415, 436, 437, 438, 439, 444, 445, 446, 447, 320, 321, 322, 323, 328, 329,
  330, 331, 352, 353, 354, 355, 360, 361, 362, 363, 448, 449, 450, 451, 456,
  457, 458, 459, 480, 481, 482, 483, 488, 489, 490, 491, 324, 325, 326, 327,
  332, 333, 334, 335, 356, 357, 358, 359, 364, 365, 366, 367, 452, 453, 454,
  455, 460, 461, 462, 463, 484, 485, 486, 487, 492, 493, 494, 495, 336, 337,
  338, 339, 344, 345, 346, 347, 368, 369, 370, 371, 376, 377, 378, 379, 464,
  465, 466, 467, 472, 473, 474, 475, 496, 497, 498, 499, 504, 505, 506, 507,
  340, 341, 342, 343, 348, 349, 350, 351, 372, 373, 374, 375, 380, 381, 382,
  383, 468, 469, 470, 471, 476, 477, 478, 479, 500, 501, 502, 503, 508, 509,
  510, 511,
};

static const uint16_t orders_8x4[512] = {
  0,   2,   8,   10,  32,  34,  40,  42,  128, 130, 136, 138, 160, 162, 168,
  170, 1,   3,   9,   11,  33,  35,  41,  43,  129, 131, 137, 139, 161, 163,
  169, 171, 4,   6,   12,  14,  36,  38,  44,  46,  132, 134, 140, 142, 164,
  166, 172, 174, 5,   7,   13,  15,  37,  39,  45,  47,  133, 135, 141, 143,
  165, 167, 173, 175, 16,  18,  24,  26,  48,  50,  56,  58,  144, 146, 152,
  154, 176, 178, 184, 186, 17,  19,  25,  27,  49,  51,  57,  59,  145, 147,
  153, 155, 177, 179, 185, 187, 20,  22,  28,  30,  52,  54,  60,  62,  148,
  150, 156, 158, 180, 182, 188, 190, 21,  23,  29,  31,  53,  55,  61,  63,
  149, 151, 157, 159, 181, 183, 189, 191, 64,  66,  72,  74,  96,  98,  104,
  106, 192, 194, 200, 202, 224, 226, 232, 234, 65,  67,  73,  75,  97,  99,
  105, 107, 193, 195, 201, 203, 225, 227, 233, 235, 68,  70,  76,  78,  100,
  102, 108, 110, 196, 198, 204, 206, 228, 230, 236, 238, 69,  71,  77,  79,
  101, 103, 109, 111, 197, 199, 205, 207, 229, 231, 237, 239, 80,  82,  88,
  90,  112, 114, 120, 122, 208, 210, 216, 218, 240, 242, 248, 250, 81,  83,
  89,  91,  113, 115, 121, 123, 209, 211, 217, 219, 241, 243, 249, 251, 84,
  86,  92,  94,  116, 118, 124, 126, 212, 214, 220, 222, 244, 246, 252, 254,
  85,  87,  93,  95,  117, 119, 125, 127, 213, 215, 221, 223, 245, 247, 253,
  255, 256, 258, 264, 266, 288, 290, 296, 298, 384, 386, 392, 394, 416, 418,
  424, 426, 257, 259, 265, 267, 289, 291, 297, 299, 385, 387, 393, 395, 417,
  419, 425, 427, 260, 262, 268, 270, 292, 294, 300, 302, 388, 390, 396, 398,
  420, 422, 428, 430, 261, 263, 269, 271, 293, 295, 301, 303, 389, 391, 397,
  399, 421, 423, 429, 431, 272, 274, 280, 282, 304, 306, 312, 314, 400, 402,
  408, 410, 432, 434, 440, 442, 273, 275, 281, 283, 305, 307, 313, 315, 401,
  403, 409, 411, 433, 435, 441, 443, 276, 278, 284, 286, 308, 310, 316, 318,
  404, 406, 412, 414, 436, 438, 444, 446, 277, 279, 285, 287, 309, 311, 317,
  319, 405, 407, 413, 415, 437, 439, 445, 447, 320, 322, 328, 330, 352, 354,
  360, 362, 448, 450, 456, 458, 480, 482, 488, 490, 321, 323, 329, 331, 353,
  355, 361, 363, 449, 451, 457, 459, 481, 483, 489, 491, 324, 326, 332, 334,
  356, 358, 364, 366, 452, 454, 460, 462, 484, 486, 492, 494, 325, 327, 333,
  335, 357, 359, 365, 367, 453, 455, 461, 463, 485, 487, 493, 495, 336, 338,
  344, 346, 368, 370, 376, 378, 464, 466, 472, 474, 496, 498, 504, 506, 337,
  339, 345, 347, 369, 371, 377, 379, 465, 467, 473, 475, 497, 499, 505, 507,
  340, 342, 348, 350, 372, 374, 380, 382, 468, 470, 476, 478, 500, 502, 508,
  510, 341, 343, 349, 351, 373, 375, 381, 383, 469, 471, 477, 479, 501, 503,
  509, 511,
};

static const uint16_t orders_4x4[1024] = {
  0,    1,    4,    5,    16,   17,   20,   21,   64,   65,   68,   69,   80,
  81,   84,   85,   256,  257,  260,  261,  272,  273,  276,  277,  320,  321,
  324,  325,  336,  337,  340,  341,  2,    3,    6,    7,    18,   19,   22,
  23,   66,   67,   70,   71,   82,   83,   86,   87,   258,  259,  262,  263,
  274,  275,  278,  279,  322,  323,  326,  327,  338,  339,  342,  343,  8,
  9,    12,   13,   24,   25,   28,   29,   72,   73,   76,   77,   88,   89,
  92,   93,   264,  265,  268,  269,  280,  281,  284,  285,  328,  329,  332,
  333,  344,  345,  348,  349,  10,   11,   14,   15,   26,   27,   30,   31,
  74,   75,   78,   79,   90,   91,   94,   95,   266,  267,  270,  271,  282,
  283,  286,  287,  330,  331,  334,  335,  346,  347,  350,  351,  32,   33,
  36,   37,   48,   49,   52,   53,   96,   97,   100,  101,  112,  113,  116,
  117,  288,  289,  292,  293,  304,  305,  308,  309,  352,  353,  356,  357,
  368,  369,  372,  373,  34,   35,   38,   39,   50,   51,   54,   55,   98,
  99,   102,  103,  114,  115,  118,  119,  290,  291,  294,  295,  306,  307,
  310,  311,  354,  355,  358,  359,  370,  371,  374,  375,  40,   41,   44,
  45,   56,   57,   60,   61,   104,  105,  108,  109,  120,  121,  124,  125,
  296,  297,  300,  301,  312,  313,  316,  317,  360,  361,  364,  365,  376,
  377,  380,  381,  42,   43,   46,   47,   58,   59,   62,   63,   106,  107,
  110,  111,  122,  123,  126,  127,  298,  299,  302,  303,  314,  315,  318,
  319,  362,  363,  366,  367,  378,  379,  382,  383,  128,  129,  132,  133,
  144,  145,  148,  149,  192,  193,  196,  197,  208,  209,  212,  213,  384,
  385,  388,  389,  400,  401,  404,  405,  448,  449,  452,  453,  464,  465,
  468,  469,  130,  131,  134,  135,  146,  147,  150,  151,  194,  195,  198,
  199,  210,  211,  214,  215,  386,  387,  390,  391,  402,  403,  406,  407,
  450,  451,  454,  455,  466,  467,  470,  471,  136,  137,  140,  141,  152,
  153,  156,  157,  200,  201,  204,  205,  216,  217,  220,  221,  392,  393,
  396,  397,  408,  409,  412,  413,  456,  457,  460,  461,  472,  473,  476,
  477,  138,  139,  142,  143,  154,  155,  158,  159,  202,  203,  206,  207,
  218,  219,  222,  223,  394,  395,  398,  399,  410,  411,  414,  415,  458,
  459,  462,  463,  474,  475,  478,  479,  160,  161,  164,  165,  176,  177,
  180,  181,  224,  225,  228,  229,  240,  241,  244,  245,  416,  417,  420,
  421,  432,  433,  436,  437,  480,  481,  484,  485,  496,  497,  500,  501,
  162,  163,  166,  167,  178,  179,  182,  183,  226,  227,  230,  231,  242,
  243,  246,  247,  418,  419,  422,  423,  434,  435,  438,  439,  482,  483,
  486,  487,  498,  499,  502,  503,  168,  169,  172,  173,  184,  185,  188,
  189,  232,  233,  236,  237,  248,  249,  252,  253,  424,  425,  428,  429,
  440,  441,  444,  445,  488,  489,  492,  493,  504,  505,  508,  509,  170,
  171,  174,  175,  186,  187,  190,  191,  234,  235,  238,  239,  250,  251,
  254,  255,  426,  427,  430,  431,  442,  443,  446,  447,  490,  491,  494,
  495,  506,  507,  510,  511,  512,  513,  516,  517,  528,  529,  532,  533,
  576,  577,  580,  581,  592,  593,  596,  597,  768,  769,  772,  773,  784,
  785,  788,  789,  832,  833,  836,  837,  848,  849,  852,  853,  514,  515,
  518,  519,  530,  531,  534,  535,  578,  579,  582,  583,  594,  595,  598,
  599,  770,  771,  774,  775,  786,  787,  790,  791,  834,  835,  838,  839,
  850,  851,  854,  855,  520,  521,  524,  525,  536,  537,  540,  541,  584,
  585,  588,  589,  600,  601,  604,  605,  776,  777,  780,  781,  792,  793,
  796,  797,  840,  841,  844,  845,  856,  857,  860,  861,  522,  523,  526,
  527,  538,  539,  542,  543,  586,  587,  590,  591,  602,  603,  606,  607,
  778,  779,  782,  783,  794,  795,  798,  799,  842,  843,  846,  847,  858,
  859,  862,  863,  544,  545,  548,  549,  560,  561,  564,  565,  608,  609,
  612,  613,  624,  625,  628,  629,  800,  801,  804,  805,  816,  817,  820,
  821,  864,  865,  868,  869,  880,  881,  884,  885,  546,  547,  550,  551,
  562,  563,  566,  567,  610,  611,  614,  615,  626,  627,  630,  631,  802,
  803,  806,  807,  818,  819,  822,  823,  866,  867,  870,  871,  882,  883,
  886,  887,  552,  553,  556,  557,  568,  569,  572,  573,  616,  617,  620,
  621,  632,  633,  636,  637,  808,  809,  812,  813,  824,  825,  828,  829,
  872,  873,  876,  877,  888,  889,  892,  893,  554,  555,  558,  559,  570,
  571,  574,  575,  618,  619,  622,  623,  634,  635,  638,  639,  810,  811,
  814,  815,  826,  827,  830,  831,  874,  875,  878,  879,  890,  891,  894,
  895,  640,  641,  644,  645,  656,  657,  660,  661,  704,  705,  708,  709,
  720,  721,  724,  725,  896,  897,  900,  901,  912,  913,  916,  917,  960,
  961,  964,  965,  976,  977,  980,  981,  642,  643,  646,  647,  658,  659,
  662,  663,  706,  707,  710,  711,  722,  723,  726,  727,  898,  899,  902,
  903,  914,  915,  918,  919,  962,  963,  966,  967,  978,  979,  982,  983,
  648,  649,  652,  653,  664,  665,  668,  669,  712,  713,  716,  717,  728,
  729,  732,  733,  904,  905,  908,  909,  920,  921,  924,  925,  968,  969,
  972,  973,  984,  985,  988,  989,  650,  651,  654,  655,  666,  667,  670,
  671,  714,  715,  718,  719,  730,  731,  734,  735,  906,  907,  910,  911,
  922,  923,  926,  927,  970,  971,  974,  975,  986,  987,  990,  991,  672,
  673,  676,  677,  688,  689,  692,  693,  736,  737,  740,  741,  752,  753,
  756,  757,  928,  929,  932,  933,  944,  945,  948,  949,  992,  993,  996,
  997,  1008, 1009, 1012, 1013, 674,  675,  678,  679,  690,  691,  694,  695,
  738,  739,  742,  743,  754,  755,  758,  759,  930,  931,  934,  935,  946,
  947,  950,  951,  994,  995,  998,  999,  1010, 1011, 1014, 1015, 680,  681,
  684,  685,  696,  697,  700,  701,  744,  745,  748,  749,  760,  761,  764,
  765,  936,  937,  940,  941,  952,  953,  956,  957,  1000, 1001, 1004, 1005,
  1016, 1017, 1020, 1021, 682,  683,  686,  687,  698,  699,  702,  703,  746,
  747,  750,  751,  762,  763,  766,  767,  938,  939,  942,  943,  954,  955,
  958,  959,  1002, 1003, 1006, 1007, 1018, 1019, 1022, 1023,
};
#endif
290
#endif  // CONFIG_CB4X4 || CONFIG_EXT_PARTITION
291

292
#if CONFIG_EXT_PARTITION
clang-format's avatar
clang-format committed
293
/* clang-format off */
294
static const uint16_t *const orders[BLOCK_SIZES] = {
Jingning Han's avatar
Jingning Han committed
295
296
#if CONFIG_CB4X4
  // 2X2,         2X4,            4X2
297
298
299
300
301
302
  orders_4x4,     orders_4x4,     orders_4x4,
  //                              4X4
                                  orders_4x4,
  // 4X8,         8X4,            8X8
  orders_4x8,     orders_8x4,     orders_8x8,
#else
303
304
305
306
  //                              4X4
                                  orders_8x8,
  // 4X8,         8X4,            8X8
  orders_8x8,     orders_8x8,     orders_8x8,
307
#endif
308
309
310
311
312
313
314
315
  // 8X16,        16X8,           16X16
  orders_8x16,    orders_16x8,    orders_16x16,
  // 16X32,       32X16,          32X32
  orders_16x32,   orders_32x16,   orders_32x32,
  // 32X64,       64X32,          64X64
  orders_32x64,   orders_64x32,   orders_64x64,
  // 64x128,      128x64,         128x128
  orders_64x128,  orders_128x64,  orders_128x128
316
};
clang-format's avatar
clang-format committed
317
/* clang-format on */
318
#else
clang-format's avatar
clang-format committed
319
/* clang-format off */
320
static const uint16_t *const orders[BLOCK_SIZES] = {
Jingning Han's avatar
Jingning Han committed
321
322
#if CONFIG_CB4X4
  // 2X2,         2X4,            4X2
323
324
325
326
327
328
  orders_8x8,     orders_8x8,     orders_8x8,
  //                              4X4
                                  orders_8x8,
  // 4X8,         8X4,            8X8
  orders_8x16,    orders_16x8,    orders_16x16,
#else
329
330
331
332
  //                              4X4
                                  orders_16x16,
  // 4X8,         8X4,            8X8
  orders_16x16,   orders_16x16,   orders_16x16,
333
#endif
334
335
336
337
338
339
340
  // 8X16,        16X8,           16X16
  orders_16x32,   orders_32x16,   orders_32x32,
  // 16X32,       32X16,          32X32
  orders_32x64,   orders_64x32,   orders_64x64,
  // 32X64,       64X32,          64X64
  orders_64x128,  orders_128x64,  orders_128x128
};
clang-format's avatar
clang-format committed
341
/* clang-format on */
342
343
#endif  // CONFIG_EXT_PARTITION

344
#if CONFIG_EXT_PARTITION_TYPES
345
static const uint16_t orders_verta_64x64[4] = {
clang-format's avatar
clang-format committed
346
  0, 2, 1, 2,
347
};
348
static const uint16_t orders_verta_32x32[16] = {
clang-format's avatar
clang-format committed
349
  0, 2, 4, 6, 1, 2, 5, 6, 8, 10, 12, 14, 9, 10, 13, 14,
350
};
351
static const uint16_t orders_verta_16x16[64] = {
clang-format's avatar
clang-format committed
352
353
354
355
  0,  2,  4,  6,  16, 18, 20, 22, 1,  2,  5,  6,  17, 18, 21, 22,
  8,  10, 12, 14, 24, 26, 28, 30, 9,  10, 13, 14, 25, 26, 29, 30,
  32, 34, 36, 38, 48, 50, 52, 54, 33, 34, 37, 38, 49, 50, 53, 54,
  40, 42, 44, 46, 56, 58, 60, 62, 41, 42, 45, 46, 57, 58, 61, 62,
356
};
357
#if CONFIG_EXT_PARTITION || CONFIG_CB4X4
358
static const uint16_t orders_verta_8x8[256] = {
clang-format's avatar
clang-format committed
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
  0,   2,   4,   6,   16,  18,  20,  22,  64,  66,  68,  70,  80,  82,  84,
  86,  1,   2,   5,   6,   17,  18,  21,  22,  65,  66,  69,  70,  81,  82,
  85,  86,  8,   10,  12,  14,  24,  26,  28,  30,  72,  74,  76,  78,  88,
  90,  92,  94,  9,   10,  13,  14,  25,  26,  29,  30,  73,  74,  77,  78,
  89,  90,  93,  94,  32,  34,  36,  38,  48,  50,  52,  54,  96,  98,  100,
  102, 112, 114, 116, 118, 33,  34,  37,  38,  49,  50,  53,  54,  97,  98,
  101, 102, 113, 114, 117, 118, 40,  42,  44,  46,  56,  58,  60,  62,  104,
  106, 108, 110, 120, 122, 124, 126, 41,  42,  45,  46,  57,  58,  61,  62,
  105, 106, 109, 110, 121, 122, 125, 126, 128, 130, 132, 134, 144, 146, 148,
  150, 192, 194, 196, 198, 208, 210, 212, 214, 129, 130, 133, 134, 145, 146,
  149, 150, 193, 194, 197, 198, 209, 210, 213, 214, 136, 138, 140, 142, 152,
  154, 156, 158, 200, 202, 204, 206, 216, 218, 220, 222, 137, 138, 141, 142,
  153, 154, 157, 158, 201, 202, 205, 206, 217, 218, 221, 222, 160, 162, 164,
  166, 176, 178, 180, 182, 224, 226, 228, 230, 240, 242, 244, 246, 161, 162,
  165, 166, 177, 178, 181, 182, 225, 226, 229, 230, 241, 242, 245, 246, 168,
  170, 172, 174, 184, 186, 188, 190, 232, 234, 236, 238, 248, 250, 252, 254,
  169, 170, 173, 174, 185, 186, 189, 190, 233, 234, 237, 238, 249, 250, 253,
  254,
377
};
378
#endif  // CONFIG_EXT_PARTITION || CONFIG_CB4X4
clang-format's avatar
clang-format committed
379

380
#if CONFIG_EXT_PARTITION
clang-format's avatar
clang-format committed
381
/* clang-format off */
382
static const uint16_t *const orders_verta[BLOCK_SIZES] = {
383
384
385
386
#if CONFIG_CB4X4
  // 2X2,           2X4,              4X2
  orders_4x4,       orders_4x4,       orders_4x4,
#endif
387
388
389
390
391
392
393
394
395
396
397
398
399
  //                                  4X4
                                      orders_verta_8x8,
  // 4X8,           8X4,              8X8
  orders_verta_8x8, orders_verta_8x8, orders_verta_8x8,
  // 8X16,          16X8,             16X16
  orders_8x16,      orders_16x8,      orders_verta_16x16,
  // 16X32,         32X16,            32X32
  orders_16x32,     orders_32x16,     orders_verta_32x32,
  // 32X64,         64X32,            64X64
  orders_32x64,     orders_64x32,     orders_verta_64x64,
  // 64x128,        128x64,           128x128
  orders_64x128,    orders_128x64,    orders_128x128
};
clang-format's avatar
clang-format committed
400
/* clang-format on */
401
#else
clang-format's avatar
clang-format committed
402
/* clang-format off */
403
static const uint16_t *const orders_verta[BLOCK_SIZES] = {
404
405
406
407
408
409
410
411
#if CONFIG_CB4X4
  // 2X2,             2X4,                4X2
  orders_verta_8x8,   orders_verta_8x8,   orders_verta_8x8,
  //                                      4X4
                                          orders_verta_8x8,
  // 4X8,             8X4,                8X8
  orders_verta_8x8,   orders_verta_8x8,   orders_verta_16x16,
#else
412
413
414
415
  //                                      4X4
                                          orders_verta_16x16,
  // 4X8,             8X4,                8X8
  orders_verta_16x16, orders_verta_16x16, orders_verta_16x16,
416
#endif
417
418
419
420
421
422
  // 8X16,            16X8,               16X16
  orders_16x32,       orders_32x16,       orders_verta_32x32,
  // 16X32,           32X16,              32X32
  orders_32x64,       orders_64x32,       orders_verta_64x64,
  // 32X64,           64X32,              64X64
  orders_64x128,      orders_128x64,      orders_128x128
423
};
clang-format's avatar
clang-format committed
424
/* clang-format on */
425
#endif  // CONFIG_EXT_PARTITION
426
#endif  // CONFIG_EXT_PARTITION_TYPES
427

428
429
static int has_top_right(BLOCK_SIZE bsize, int mi_row, int mi_col,
                         int top_available, int right_available,
430
#if CONFIG_EXT_PARTITION_TYPES
Yaowu Xu's avatar
Yaowu Xu committed
431
                         PARTITION_TYPE partition,
432
#endif
433
434
435
                         TX_SIZE txsz, int row_off, int col_off, int ss_x) {
  if (!top_available || !right_available) return 0;

436
#if !CONFIG_CB4X4
437
438
439
440
441
442
  // TODO(bshacklett, huisu): Currently the RD loop traverses 4X8 blocks in
  // inverted N order while in the bitstream the subblocks are stored in Z
  // order. This discrepancy makes this function incorrect when considering 4X8
  // blocks in the RD loop, so we disable the extended right edge for these
  // blocks. The correct solution is to change the bitstream to store these
  // blocks in inverted N order, and then update this function appropriately.
443
  if (bsize == BLOCK_4X8 && row_off == 1) return 0;
444
#endif
445

446
447
448
  const int bw_unit = block_size_wide[bsize] >> tx_size_wide_log2[0];
  const int plane_bw_unit = AOMMAX(bw_unit >> ss_x, 1);
  const int top_right_count_unit = tx_size_wide_unit[txsz];
449

450
451
  // Special handling for block sizes 4x8 and 4x4.
  if (ss_x == 0 && bw_unit < 2 && col_off == 0) return 1;
452

453
454
455
456
457
  if (row_off > 0) {  // Just need to check if enough pixels on the right.
    return col_off + top_right_count_unit < plane_bw_unit;
  } else {
    // All top-right pixels are in the block above, which is already available.
    if (col_off + top_right_count_unit < plane_bw_unit) return 1;
458

459
460
461
462
    const int bw_in_mi_log2 = mi_width_log2_lookup[bsize];
    const int bh_in_mi_log2 = mi_height_log2_lookup[bsize];
    const int blk_row_in_sb = (mi_row & MAX_MIB_MASK) >> bh_in_mi_log2;
    const int blk_col_in_sb = (mi_col & MAX_MIB_MASK) >> bw_in_mi_log2;
463

464
465
466
    // Top row of superblock: so top-right pixels are in the top and/or
    // top-right superblocks, both of which are already available.
    if (blk_row_in_sb == 0) return 1;
467

468
469
470
    // Rightmost column of superblock (and not the top row): so top-right pixels
    // fall in the right superblock, which is not available yet.
    if (((blk_col_in_sb + 1) << bw_in_mi_log2) >= MAX_MIB_SIZE) return 0;
471

472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
    // General case (neither top row nor rightmost column): check if the
    // top-right block is coded before the current block.
    const uint16_t *const order =
#if CONFIG_EXT_PARTITION_TYPES
        (partition == PARTITION_VERT_A) ? orders_verta[bsize] :
#endif  // CONFIG_EXT_PARTITION_TYPES
                                        orders[bsize];
    const int this_blk_index =
        ((blk_row_in_sb + 0) << (MAX_MIB_SIZE_LOG2 - bw_in_mi_log2)) +
        blk_col_in_sb + 0;
    const uint16_t this_blk_order = order[this_blk_index];
    const int tr_blk_index =
        ((blk_row_in_sb - 1) << (MAX_MIB_SIZE_LOG2 - bw_in_mi_log2)) +
        blk_col_in_sb + 1;
    const uint16_t tr_blk_order = order[tr_blk_index];
    return tr_blk_order < this_blk_order;
488
489
490
  }
}

491
492
493
494
495
496
497
static int has_bottom_left(BLOCK_SIZE bsize, int mi_row, int mi_col,
                           int bottom_available, int left_available,
                           TX_SIZE txsz, int row_off, int col_off, int ss_y) {
  if (!bottom_available || !left_available) return 0;

  if (col_off > 0) {
    // Bottom-left pixels are in the bottom-left block, which is not available.
498
499
    return 0;
  } else {
500
501
502
    const int bh_unit = block_size_high[bsize] >> tx_size_high_log2[0];
    const int plane_bh_unit = AOMMAX(bh_unit >> ss_y, 1);
    const int bottom_left_count_unit = tx_size_high_unit[txsz];
503

504
#if !CONFIG_CB4X4
505
506
    // Special handling for block sizes 8x4 and 4x4.
    if (ss_y == 0 && bh_unit < 2 && row_off == 0) return 1;
507
#endif
508

509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
    // All bottom-left pixels are in the left block, which is already available.
    if (row_off + bottom_left_count_unit < plane_bh_unit) return 1;

    const int bw_in_mi_log2 = mi_width_log2_lookup[bsize];
    const int bh_in_mi_log2 = mi_height_log2_lookup[bsize];
    const int blk_row_in_sb = (mi_row & MAX_MIB_MASK) >> bh_in_mi_log2;
    const int blk_col_in_sb = (mi_col & MAX_MIB_MASK) >> bw_in_mi_log2;

    // Leftmost column of superblock: so bottom-left pixels maybe in the left
    // and/or bottom-left superblocks. But only the left superblock is
    // available, so check if all required pixels fall in that superblock.
    if (blk_col_in_sb == 0) {
      const int blk_start_row_off = blk_row_in_sb << (bh_in_mi_log2 + !ss_y);
      const int row_off_in_sb = blk_start_row_off + row_off;
      const int sb_height_unit = MAX_MIB_SIZE << !ss_y;
      return row_off_in_sb + bottom_left_count_unit < sb_height_unit;
    }
526

527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
    // Bottom row of superblock (and not the leftmost column): so bottom-left
    // pixels fall in the bottom superblock, which is not available yet.
    if (((blk_row_in_sb + 1) << bh_in_mi_log2) >= MAX_MIB_SIZE) return 0;

    // General case (neither leftmost column nor bottom row): check if the
    // bottom-left block is coded before the current block.
    const uint16_t *const order = orders[bsize];
    const int this_blk_index =
        ((blk_row_in_sb + 0) << (MAX_MIB_SIZE_LOG2 - bw_in_mi_log2)) +
        blk_col_in_sb + 0;
    const uint16_t this_blk_order = order[this_blk_index];
    const int bl_blk_index =
        ((blk_row_in_sb + 1) << (MAX_MIB_SIZE_LOG2 - bw_in_mi_log2)) +
        blk_col_in_sb - 1;
    const uint16_t bl_blk_order = order[bl_blk_index];
    return bl_blk_order < this_blk_order;
543
544
  }
}
Jingning Han's avatar
Jingning Han committed
545
546
547
548
549
550
551

typedef void (*intra_pred_fn)(uint8_t *dst, ptrdiff_t stride,
                              const uint8_t *above, const uint8_t *left);

static intra_pred_fn pred[INTRA_MODES][TX_SIZES];
static intra_pred_fn dc_pred[2][2][TX_SIZES];

552
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
553
554
555
typedef void (*intra_high_pred_fn)(uint16_t *dst, ptrdiff_t stride,
                                   const uint16_t *above, const uint16_t *left,
                                   int bd);
556
557
static intra_high_pred_fn pred_high[INTRA_MODES][TX_SIZES];
static intra_high_pred_fn dc_pred_high[2][2][TX_SIZES];
558
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
559

Yaowu Xu's avatar
Yaowu Xu committed
560
static void av1_init_intra_predictors_internal(void) {
561
562
563
564
565
566
567
#if CONFIG_TX64X64
#define INIT_NO_4X4(p, type)                  \
  p[TX_8X8] = aom_##type##_predictor_8x8;     \
  p[TX_16X16] = aom_##type##_predictor_16x16; \
  p[TX_32X32] = aom_##type##_predictor_32x32; \
  p[TX_64X64] = aom_##type##_predictor_64x64
#else
clang-format's avatar
clang-format committed
568
#define INIT_NO_4X4(p, type)                  \
Yaowu Xu's avatar
Yaowu Xu committed
569
570
571
  p[TX_8X8] = aom_##type##_predictor_8x8;     \
  p[TX_16X16] = aom_##type##_predictor_16x16; \
  p[TX_32X32] = aom_##type##_predictor_32x32
572
#endif  // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
573

Jingning Han's avatar
Jingning Han committed
574
575
576
577
578
579
#if CONFIG_CB4X4
#define INIT_ALL_SIZES(p, type)           \
  p[TX_2X2] = aom_##type##_predictor_2x2; \
  p[TX_4X4] = aom_##type##_predictor_4x4; \
  INIT_NO_4X4(p, type)
#else
clang-format's avatar
clang-format committed
580
#define INIT_ALL_SIZES(p, type)           \
Yaowu Xu's avatar
Yaowu Xu committed
581
  p[TX_4X4] = aom_##type##_predictor_4x4; \
582
  INIT_NO_4X4(p, type)
Jingning Han's avatar
Jingning Han committed
583
#endif
584

Jingning Han's avatar
Jingning Han committed
585
586
  INIT_ALL_SIZES(pred[V_PRED], v);
  INIT_ALL_SIZES(pred[H_PRED], h);
587
588
589
  INIT_ALL_SIZES(pred[D207_PRED], d207e);
  INIT_ALL_SIZES(pred[D45_PRED], d45e);
  INIT_ALL_SIZES(pred[D63_PRED], d63e);
Jingning Han's avatar
Jingning Han committed
590
591
592
  INIT_ALL_SIZES(pred[D117_PRED], d117);
  INIT_ALL_SIZES(pred[D135_PRED], d135);
  INIT_ALL_SIZES(pred[D153_PRED], d153);
Urvang Joshi's avatar
Urvang Joshi committed
593
594
595

#if CONFIG_ALT_INTRA
  INIT_ALL_SIZES(pred[TM_PRED], paeth);
596
  INIT_ALL_SIZES(pred[SMOOTH_PRED], smooth);
Urvang Joshi's avatar
Urvang Joshi committed
597
598
599
600
#if CONFIG_SMOOTH_HV
  INIT_ALL_SIZES(pred[SMOOTH_V_PRED], smooth_v);
  INIT_ALL_SIZES(pred[SMOOTH_H_PRED], smooth_h);
#endif  // CONFIG_SMOOTH_HV
Urvang Joshi's avatar
Urvang Joshi committed
601
#else
Jingning Han's avatar
Jingning Han committed
602
  INIT_ALL_SIZES(pred[TM_PRED], tm);
Urvang Joshi's avatar
Urvang Joshi committed
603
#endif  // CONFIG_ALT_INTRA
Jingning Han's avatar
Jingning Han committed
604
605
606
607
608
609

  INIT_ALL_SIZES(dc_pred[0][0], dc_128);
  INIT_ALL_SIZES(dc_pred[0][1], dc_top);
  INIT_ALL_SIZES(dc_pred[1][0], dc_left);
  INIT_ALL_SIZES(dc_pred[1][1], dc);

610
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
611
612
  INIT_ALL_SIZES(pred_high[V_PRED], highbd_v);
  INIT_ALL_SIZES(pred_high[H_PRED], highbd_h);
613
614
615
  INIT_ALL_SIZES(pred_high[D207_PRED], highbd_d207e);
  INIT_ALL_SIZES(pred_high[D45_PRED], highbd_d45e);
  INIT_ALL_SIZES(pred_high[D63_PRED], highbd_d63e);
Jingning Han's avatar
Jingning Han committed
616
617
618
  INIT_ALL_SIZES(pred_high[D117_PRED], highbd_d117);
  INIT_ALL_SIZES(pred_high[D135_PRED], highbd_d135);
  INIT_ALL_SIZES(pred_high[D153_PRED], highbd_d153);
Urvang Joshi's avatar
Urvang Joshi committed
619
620
621

#if CONFIG_ALT_INTRA
  INIT_ALL_SIZES(pred_high[TM_PRED], highbd_paeth);
622
  INIT_ALL_SIZES(pred_high[SMOOTH_PRED], highbd_smooth);
Urvang Joshi's avatar
Urvang Joshi committed
623
624
625
626
#if CONFIG_SMOOTH_HV
  INIT_ALL_SIZES(pred_high[SMOOTH_V_PRED], highbd_smooth_v);
  INIT_ALL_SIZES(pred_high[SMOOTH_H_PRED], highbd_smooth_h);
#endif  // CONFIG_SMOOTH_HV
Urvang Joshi's avatar
Urvang Joshi committed
627
#else
Jingning Han's avatar
Jingning Han committed
628
  INIT_ALL_SIZES(pred_high[TM_PRED], highbd_tm);
Urvang Joshi's avatar
Urvang Joshi committed
629
#endif  // CONFIG_ALT_INTRA
Jingning Han's avatar
Jingning Han committed
630
631
632
633
634

  INIT_ALL_SIZES(dc_pred_high[0][0], highbd_dc_128);
  INIT_ALL_SIZES(dc_pred_high[0][1], highbd_dc_top);
  INIT_ALL_SIZES(dc_pred_high[1][0], highbd_dc_left);
  INIT_ALL_SIZES(dc_pred_high[1][1], highbd_dc);
635
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
636
637
638
639

#undef intra_pred_allsizes
}

hui su's avatar
hui su committed
640
#if CONFIG_EXT_INTRA
hui su's avatar
hui su committed
641
#if CONFIG_INTRA_INTERP
642
643
644
645
646
647
648
649
650
651
652
static int intra_subpel_interp(int base, int shift, const uint8_t *ref,
                               int ref_start_idx, int ref_end_idx,
                               INTRA_FILTER filter_type) {
  int val, k, idx, filter_idx = 0;
  const int16_t *filter = NULL;

  if (filter_type == INTRA_FILTER_LINEAR) {
    val = ref[base] * (256 - shift) + ref[base + 1] * shift;
    val = ROUND_POWER_OF_TWO(val, 8);
  } else {
    filter_idx = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS);
Yaowu Xu's avatar
Yaowu Xu committed
653
    filter = av1_intra_filter_kernels[filter_type][filter_idx];
654
655
656
657
658

    if (filter_idx < (1 << SUBPEL_BITS)) {
      val = 0;
      for (k = 0; k < SUBPEL_TAPS; ++k) {
        idx = base + 1 - (SUBPEL_TAPS / 2) + k;
Yaowu Xu's avatar
Yaowu Xu committed
659
        idx = AOMMAX(AOMMIN(idx, ref_end_idx), ref_start_idx);
660
661
662
663
664
665
666
667
668
669
        val += ref[idx] * filter[k];
      }
      val = ROUND_POWER_OF_TWO(val, FILTER_BITS);
    } else {
      val = ref[base + 1];
    }
  }

  return val;
}
hui su's avatar
hui su committed
670
#endif  // CONFIG_INTRA_INTERP
671

hui su's avatar
hui su committed
672
673
// Directional prediction, zone 1: 0 < angle < 90
static void dr_prediction_z1(uint8_t *dst, ptrdiff_t stride, int bs,
hui su's avatar
hui su committed
674
675
676
677
678
                             const uint8_t *above, const uint8_t *left,
#if CONFIG_INTRA_INTERP
                             INTRA_FILTER filter_type,
#endif  // CONFIG_INTRA_INTERP
                             int dx, int dy) {
hui su's avatar
hui su committed
679
  int r, c, x, base, shift, val;
hui su's avatar
hui su committed
680
681
682
683

  (void)left;
  (void)dy;
  assert(dy == 1);
hui su's avatar
hui su committed
684
  assert(dx > 0);
hui su's avatar
hui su committed
685

hui su's avatar
hui su committed
686
#if CONFIG_INTRA_INTERP
hui su's avatar
hui su committed
687
688
689
  if (filter_type != INTRA_FILTER_LINEAR) {
    const int pad_size = SUBPEL_TAPS >> 1;
    int len;
690
691
    DECLARE_ALIGNED(16, uint8_t, buf[SUBPEL_SHIFTS][MAX_SB_SIZE]);
    DECLARE_ALIGNED(16, uint8_t, src[MAX_SB_SIZE + SUBPEL_TAPS]);
hui su's avatar
hui su committed
692
693
694
695
696
697
698
699
    uint8_t flags[SUBPEL_SHIFTS];

    memset(flags, 0, SUBPEL_SHIFTS * sizeof(flags[0]));
    memset(src, above[0], pad_size * sizeof(above[0]));
    memcpy(src + pad_size, above, 2 * bs * sizeof(above[0]));
    memset(src + pad_size + 2 * bs, above[2 * bs - 1],
           pad_size * sizeof(above[0]));
    flags[0] = 1;
hui su's avatar
hui su committed
700
701
    x = dx;
    for (r = 0; r < bs; ++r, dst += stride, x += dx) {
hui su's avatar
hui su committed
702
      base = x >> 8;
hui su's avatar
hui su committed
703
704
705
706
707
708
      shift = x & 0xFF;
      shift = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS);
      if (shift == SUBPEL_SHIFTS) {
        base += 1;
        shift = 0;
      }
Yaowu Xu's avatar
Yaowu Xu committed
709
      len = AOMMIN(bs, 2 * bs - 1 - base);
hui su's avatar
hui su committed
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
      if (len <= 0) {
        int i;
        for (i = r; i < bs; ++i) {
          memset(dst, above[2 * bs - 1], bs * sizeof(dst[0]));
          dst += stride;
        }
        return;
      }

      if (len <= (bs >> 1) && !flags[shift]) {
        base = x >> 8;
        shift = x & 0xFF;
        for (c = 0; c < len; ++c) {
          val = intra_subpel_interp(base, shift, above, 0, 2 * bs - 1,
                                    filter_type);
          dst[c] = clip_pixel(val);
          ++base;
        }
      } else {
        if (!flags[shift]) {
Yaowu Xu's avatar
Yaowu Xu committed
730
731
          const int16_t *filter = av1_intra_filter_kernels[filter_type][shift];
          aom_convolve8_horiz(src + pad_size, 2 * bs, buf[shift], 2 * bs,
clang-format's avatar
clang-format committed
732
733
                              filter, 16, NULL, 16, 2 * bs,
                              2 * bs < 16 ? 2 : 1);
hui su's avatar
hui su committed
734
735
736
          flags[shift] = 1;
        }
        memcpy(dst, shift == 0 ? src + pad_size + base : &buf[shift][base],
clang-format's avatar
clang-format committed
737
               len * sizeof(dst[0]));
hui su's avatar
hui su committed
738
739
740
741
742
743
744
      }

      if (len < bs)
        memset(dst + len, above[2 * bs - 1], (bs - len) * sizeof(dst[0]));
    }
    return;
  }
hui su's avatar
hui su committed
745
#endif  // CONFIG_INTRA_INTERP
hui su's avatar
hui su committed
746

hui su's avatar
hui su committed
747
748
  x = dx;
  for (r = 0; r < bs; ++r, dst += stride, x += dx) {
hui su's avatar
hui su committed
749
750
751
752
753
754
755
756
757
758
759
760
761
    base = x >> 8;
    shift = x & 0xFF;

    if (base >= 2 * bs - 1) {
      int i;
      for (i = r; i < bs; ++i) {
        memset(dst, above[2 * bs - 1], bs * sizeof(dst[0]));
        dst += stride;
      }
      return;
    }

    for (c = 0; c < bs; ++c, ++base) {
hui su's avatar
hui su committed
762
      if (base < 2 * bs - 1) {
hui su's avatar
hui su committed
763
764
        val = above[base] * (256 - shift) + above[base + 1] * shift;
        val = ROUND_POWER_OF_TWO(val, 8);
hui su's avatar
hui su committed
765
766
767
768
769
770
771
772
773
774
        dst[c] = clip_pixel(val);
      } else {
        dst[c] = above[2 * bs - 1];
      }
    }
  }
}

// Directional prediction, zone 2: 90 < angle < 180
static void dr_prediction_z2(uint8_t *dst, ptrdiff_t stride, int bs,
hui su's avatar
hui su committed
775
776
777
778
779
                             const uint8_t *above, const uint8_t *left,
#if CONFIG_INTRA_INTERP
                             INTRA_FILTER filter_type,
#endif  // CONFIG_INTRA_INTERP
                             int dx, int dy) {
Debargha Mukherjee's avatar
Debargha Mukherjee committed
780
  int r, c, x, y, shift1, shift2, val, base1, base2;
hui su's avatar
hui su committed
781
782
783
784

  assert(dx > 0);
  assert(dy > 0);

hui su's avatar
hui su committed
785
786
787
788
789
790
791
  x = -dx;
  for (r = 0; r < bs; ++r, x -= dx, dst += stride) {
    base1 = x >> 8;
    y = (r << 8) - dy;
    for (c = 0; c < bs; ++c, ++base1, y -= dy) {
      if (base1 >= -1) {
        shift1 = x & 0xFF;
hui su's avatar
hui su committed
792
#if CONFIG_INTRA_INTERP
clang-format's avatar
clang-format committed
793
794
        val =
            intra_subpel_interp(base1, shift1, above, -1, bs - 1, filter_type);
hui su's avatar
hui su committed
795
796
797
798
#else
        val = above[base1] * (256 - shift1) + above[base1 + 1] * shift1;
        val = ROUND_POWER_OF_TWO(val, 8);
#endif  // CONFIG_INTRA_INTERP
hui su's avatar
hui su committed
799
      } else {
hui su's avatar
hui su committed
800
        base2 = y >> 8;
801
        assert(base2 >= -1);
802
        shift2 = y & 0xFF;
hui su's avatar
hui su committed
803
#if CONFIG_INTRA_INTERP
804
        val = intra_subpel_interp(base2, shift2, left, -1, bs - 1, filter_type);
hui su's avatar
hui su committed
805
#else
806
807
        val = left[base2] * (256 - shift2) + left[base2 + 1] * shift2;
        val = ROUND_POWER_OF_TWO(val, 8);
hui su's avatar
hui su committed
808
#endif  // CONFIG_INTRA_INTERP
hui su's avatar
hui su committed
809
810
811
812
813
814
815
816
      }
      dst[c] = clip_pixel(val);
    }
  }
}

// Directional prediction, zone 3: 180 < angle < 270
static void dr_prediction_z3(uint8_t *dst, ptrdiff_t stride, int bs,
hui su's avatar
hui su committed
817
818
819
820
821
                             const uint8_t *above, const uint8_t *left,
#if CONFIG_INTRA_INTERP
                             INTRA_FILTER filter_type,
#endif  // CONFIG_INTRA_INTERP
                             int dx, int dy) {
hui su's avatar
hui su committed
822
  int r, c, y, base, shift, val;
hui su's avatar
hui su committed
823
824
825

  (void)above;
  (void)dx;
826

hui su's avatar
hui su committed
827
  assert(dx == 1);
hui su's avatar
hui su committed
828
  assert(dy > 0);
hui su's avatar
hui su committed
829

hui su's avatar
hui su committed
830
#if CONFIG_INTRA_INTERP
hui su's avatar
hui su committed
831
832
833
  if (filter_type != INTRA_FILTER_LINEAR) {
    const int pad_size = SUBPEL_TAPS >> 1;
    int len, i;
834
835
    DECLARE_ALIGNED(16, uint8_t, buf[MAX_SB_SIZE][4 * SUBPEL_SHIFTS]);
    DECLARE_ALIGNED(16, uint8_t, src[(MAX_SB_SIZE + SUBPEL_TAPS) * 4]);
hui su's avatar
hui su committed
836
837
838
    uint8_t flags[SUBPEL_SHIFTS];

    memset(flags, 0, SUBPEL_SHIFTS * sizeof(flags[0]));
clang-format's avatar
clang-format committed
839
840
    for (i = 0; i < pad_size; ++i) src[4 * i] = left[0];
    for (i = 0; i < 2 * bs; ++i) src[4 * (i + pad_size)] = left[i];
hui su's avatar
hui su committed
841
842
843
    for (i = 0; i < pad_size; ++i)
      src[4 * (i + 2 * bs + pad_size)] = left[2 * bs - 1];
    flags[0] = 1;
hui su's avatar
hui su committed
844
845
    y = dy;
    for (c = 0; c < bs; ++c, y += dy) {
hui su's avatar
hui su committed
846
      base = y >> 8;
hui su's avatar
hui su committed
847
848
849
850
851
852
      shift = y & 0xFF;
      shift = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS);
      if (shift == SUBPEL_SHIFTS) {
        base += 1;
        shift = 0;
      }
Yaowu Xu's avatar
Yaowu Xu committed
853
      len = AOMMIN(bs, 2 * bs - 1 - base);
hui su's avatar
hui su committed
854
855

      if (len <= 0) {
Debargha Mukherjee's avatar
Debargha Mukherjee committed
856
        for (r = 0; r < bs; ++r) {
clang-format's avatar
clang-format committed
857
          dst[r * stride + c] = left[2 * bs - 1];
hui su's avatar
hui su committed
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
        }
        continue;
      }

      if (len <= (bs >> 1) && !flags[shift]) {
        base = y >> 8;
        shift = y & 0xFF;
        for (r = 0; r < len; ++r) {
          val = intra_subpel_interp(base, shift, left, 0, 2 * bs - 1,
                                    filter_type);
          dst[r * stride + c] = clip_pixel(val);
          ++base;
        }
      } else {
        if (!flags[shift]) {
Yaowu Xu's avatar
Yaowu Xu committed
873
874
          const int16_t *filter = av1_intra_filter_kernels[filter_type][shift];
          aom_convolve8_vert(src + 4 * pad_size, 4, buf[0] + 4 * shift,
clang-format's avatar
clang-format committed
875
                             4 * SUBPEL_SHIFTS, NULL, 16, filter, 16,
hui su's avatar
hui su committed
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
                             2 * bs < 16 ? 4 : 4, 2 * bs);
          flags[shift] = 1;
        }

        if (shift == 0) {
          for (r = 0; r < len; ++r) {
            dst[r * stride + c] = left[r + base];
          }
        } else {
          for (r = 0; r < len; ++r) {
            dst[r * stride + c] = buf[r + base][4 * shift];
          }
        }
      }

      if (len < bs) {
        for (r = len; r < bs; ++r) {
clang-format's avatar
clang-format committed
893
          dst[r * stride + c] = left[2 * bs - 1];
hui su's avatar
hui su committed
894
895
896
897
898
        }
      }
    }
    return;
  }
hui su's avatar
hui su committed
899
#endif  // CONFIG_INTRA_INTERP
hui su's avatar
hui su committed
900

hui su's avatar
hui su committed
901
902
  y = dy;
  for (c = 0; c < bs; ++c, y += dy) {
hui su's avatar
hui su committed
903
904
905
906
    base = y >> 8;
    shift = y & 0xFF;

    for (r = 0; r < bs; ++r, ++base) {
907
      if (base < 2 * bs - 1) {
hui su's avatar
hui su committed
908
909
910
        val = left[base] * (256 - shift) + left[base + 1] * shift;
        val = ROUND_POWER_OF_TWO(val, 8);
        dst[r * stride + c] = clip_pixel(val);
hui su's avatar
hui su committed
911
      } else {
clang-format's avatar
clang-format committed
912
        for (; r < bs; ++r) dst[r * stride + c] = left[2 * bs - 1];
hui su's avatar
hui su committed
913
        break;
hui su's avatar
hui su committed
914
915
916
917
918
      }
    }
  }
}

919
920
921
922
// Get the shift (up-scaled by 256) in X w.r.t a unit change in Y.
// If angle > 0 && angle < 90, dx = -((int)(256 / t));
// If angle > 90 && angle < 180, dx = (int)(256 / t);
// If angle > 180 && angle < 270, dx = 1;
Yaowu Xu's avatar
Yaowu Xu committed
923
static INLINE int get_dx(int angle) {
924
  if (angle > 0 && angle < 90) {
hui su's avatar
hui su committed
925
    return dr_intra_derivative[angle];
926
927
928
929
930
931
932
933
934
935
936
937
  } else if (angle > 90 && angle < 180) {
    return dr_intra_derivative[180 - angle];
  } else {
    // In this case, we are not really going to use dx. We may return any value.
    return 1;
  }
}

// Get the shift (up-scaled by 256) in Y w.r.t a unit change in X.
// If angle > 0 && angle < 90, dy = 1;
// If angle > 90 && angle < 180, dy = (int)(256 * t);
// If angle > 180 && angle < 270, dy = -((int)(256 * t));
Yaowu Xu's avatar
Yaowu Xu committed
938
static INLINE int get_dy(int angle) {
939
  if (angle > 90 && angle < 180) {
clang-format's avatar
clang-format committed
940
941
    return dr_intra_derivative[angle - 90];
  } else if (angle > 180 && angle < 270) {
hui su's avatar
hui su committed
942
    return dr_intra_derivative[270 - angle];
clang-format's avatar
clang-format committed
943
  } else {
944
945
946
947
948
    // In this case, we are not really going to use dy. We may return any value.
    return 1;
  }
}

hui su's avatar
hui su committed
949
static void dr_predictor(uint8_t *dst, ptrdiff_t stride, TX_SIZE tx_size,
hui su's avatar
hui su committed
950
951
952
953
954
                         const uint8_t *above, const uint8_t *left,
#if CONFIG_INTRA_INTERP
                         INTRA_FILTER filter_type,
#endif  // CONFIG_INTRA_INTERP
                         int angle) {
955
956
  const int dx = get_dx(angle);
  const int dy = get_dy(angle);
957
  const int bs = tx_size_wide[tx_size];
958
959
  assert(angle > 0 && angle < 270);

hui su's avatar
hui su committed
960
  if (angle > 0 && angle < 90) {
hui su's avatar
hui su committed
961
962
963
964
965
    dr_prediction_z1(dst, stride, bs, above, left,
#if CONFIG_INTRA_INTERP
                     filter_type,
#endif  // CONFIG_INTRA_INTERP
                     dx, dy);
hui su's avatar
hui su committed
966
  } else if (angle > 90 && angle < 180) {
hui su's avatar
hui su committed
967
968
969
970
971
    dr_prediction_z2(dst, stride, bs, above, left,
#if CONFIG_INTRA_INTERP
                     filter_type,
#endif  // CONFIG_INTRA_INTERP
                     dx, dy);
hui su's avatar
hui su committed
972
  } else if (angle > 180 && angle < 270) {
hui su's avatar
hui su committed
973
974
975
976
977
    dr_prediction_z3(dst, stride, bs, above, left,
#if CONFIG_INTRA_INTERP
                     filter_type,
#endif  // CONFIG_INTRA_INTERP
                     dx, dy);
hui su's avatar
hui su committed
978
  } else if (angle == 90) {
hui su's avatar
hui su committed
979
    pred[V_PRED][tx_size](dst, stride, above, left);
hui su's avatar
hui su committed
980
  } else if (angle == 180) {
hui su's avatar
hui su committed
981
    pred[H_PRED][tx_size](dst, stride, above, left);
hui su's avatar
hui su committed
982
983
984
  }
}

985
#if CONFIG_HIGHBITDEPTH
hui su's avatar
hui su committed
986
#if CONFIG_INTRA_INTERP
987
988
989
990
991
992
993
994
995
996
997
static int highbd_intra_subpel_interp(int base, int shift, const uint16_t *ref,
                                      int ref_start_idx, int ref_end_idx,
                                      INTRA_FILTER filter_type) {
  int val, k, idx, filter_idx = 0;
  const int16_t *filter = NULL;

  if (filter_type == INTRA_FILTER_LINEAR) {
    val = ref[base] * (256 - shift) + ref[base + 1] * shift;
    val = ROUND_POWER_OF_TWO(val, 8);
  } else {
    filter_idx = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS);
Yaowu Xu's avatar
Yaowu Xu committed
998
    filter = av1_intra_filter_kernels[filter_type][filter_idx];
999
1000

    if (filter_idx < (1 << SUBPEL_BITS)) {