reconintra.c 80.2 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11
 */

hui su's avatar
hui su committed
12 13
#include <math.h>

Yaowu Xu's avatar
Yaowu Xu committed
14 15 16
#include "./av1_rtcd.h"
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
17
#include "aom_ports/system_state.h"
Jingning Han's avatar
Jingning Han committed
18

Yaowu Xu's avatar
Yaowu Xu committed
19 20 21 22
#if CONFIG_AOM_HIGHBITDEPTH
#include "aom_dsp/aom_dsp_common.h"
#endif  // CONFIG_AOM_HIGHBITDEPTH
#include "aom_mem/aom_mem.h"
23
#include "aom_ports/mem.h"
Yaowu Xu's avatar
Yaowu Xu committed
24
#include "aom_ports/aom_once.h"
25 26
#include "av1/common/reconintra.h"
#include "av1/common/onyxc_int.h"
Jingning Han's avatar
Jingning Han committed
27

28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45
enum {
  NEED_LEFT = 1 << 1,
  NEED_ABOVE = 1 << 2,
  NEED_ABOVERIGHT = 1 << 3,
  NEED_ABOVELEFT = 1 << 4,
  NEED_BOTTOMLEFT = 1 << 5,
};

static const uint8_t extend_modes[INTRA_MODES] = {
  NEED_ABOVE | NEED_LEFT,                   // DC
  NEED_ABOVE,                               // V
  NEED_LEFT,                                // H
  NEED_ABOVE | NEED_ABOVERIGHT,             // D45
  NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT,  // D135
  NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT,  // D117
  NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT,  // D153
  NEED_LEFT | NEED_BOTTOMLEFT,              // D207
  NEED_ABOVE | NEED_ABOVERIGHT,             // D63
46 47 48
#if CONFIG_ALT_INTRA
  NEED_LEFT | NEED_ABOVE,                   // SMOOTH
#endif                                      // CONFIG_ALT_INTRA
49 50
  NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT,  // TM
};
Jingning Han's avatar
Jingning Han committed
51

52 53 54 55
static const uint16_t orders_128x128[1] = { 0 };
static const uint16_t orders_128x64[2] = { 0, 1 };
static const uint16_t orders_64x128[2] = { 0, 1 };
static const uint16_t orders_64x64[4] = {
clang-format's avatar
clang-format committed
56
  0, 1, 2, 3,
57
};
58
static const uint16_t orders_64x32[8] = {
clang-format's avatar
clang-format committed
59
  0, 2, 1, 3, 4, 6, 5, 7,
60
};
61
static const uint16_t orders_32x64[8] = {
clang-format's avatar
clang-format committed
62
  0, 1, 2, 3, 4, 5, 6, 7,
63
};
64
static const uint16_t orders_32x32[16] = {
clang-format's avatar
clang-format committed
65
  0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15,
66
};
67
static const uint16_t orders_32x16[32] = {
clang-format's avatar
clang-format committed
68 69
  0,  2,  8,  10, 1,  3,  9,  11, 4,  6,  12, 14, 5,  7,  13, 15,
  16, 18, 24, 26, 17, 19, 25, 27, 20, 22, 28, 30, 21, 23, 29, 31,
70
};
71
static const uint16_t orders_16x32[32] = {
clang-format's avatar
clang-format committed
72 73
  0,  1,  2,  3,  8,  9,  10, 11, 4,  5,  6,  7,  12, 13, 14, 15,
  16, 17, 18, 19, 24, 25, 26, 27, 20, 21, 22, 23, 28, 29, 30, 31,
74
};
75
static const uint16_t orders_16x16[64] = {
clang-format's avatar
clang-format committed
76 77 78 79
  0,  1,  4,  5,  16, 17, 20, 21, 2,  3,  6,  7,  18, 19, 22, 23,
  8,  9,  12, 13, 24, 25, 28, 29, 10, 11, 14, 15, 26, 27, 30, 31,
  32, 33, 36, 37, 48, 49, 52, 53, 34, 35, 38, 39, 50, 51, 54, 55,
  40, 41, 44, 45, 56, 57, 60, 61, 42, 43, 46, 47, 58, 59, 62, 63,
80
};
81

82
#if CONFIG_CB4X4 || CONFIG_EXT_PARTITION
83
static const uint16_t orders_16x8[128] = {
clang-format's avatar
clang-format committed
84 85 86 87 88 89 90 91
  0,  2,  8,  10, 32,  34,  40,  42,  1,  3,  9,  11, 33,  35,  41,  43,
  4,  6,  12, 14, 36,  38,  44,  46,  5,  7,  13, 15, 37,  39,  45,  47,
  16, 18, 24, 26, 48,  50,  56,  58,  17, 19, 25, 27, 49,  51,  57,  59,
  20, 22, 28, 30, 52,  54,  60,  62,  21, 23, 29, 31, 53,  55,  61,  63,
  64, 66, 72, 74, 96,  98,  104, 106, 65, 67, 73, 75, 97,  99,  105, 107,
  68, 70, 76, 78, 100, 102, 108, 110, 69, 71, 77, 79, 101, 103, 109, 111,
  80, 82, 88, 90, 112, 114, 120, 122, 81, 83, 89, 91, 113, 115, 121, 123,
  84, 86, 92, 94, 116, 118, 124, 126, 85, 87, 93, 95, 117, 119, 125, 127,
92
};
93
static const uint16_t orders_8x16[128] = {
clang-format's avatar
clang-format committed
94 95 96 97 98
  0,  1,  2,  3,  8,  9,  10, 11, 32,  33,  34,  35,  40,  41,  42,  43,
  4,  5,  6,  7,  12, 13, 14, 15, 36,  37,  38,  39,  44,  45,  46,  47,
  16, 17, 18, 19, 24, 25, 26, 27, 48,  49,  50,  51,  56,  57,  58,  59,
  20, 21, 22, 23, 28, 29, 30, 31, 52,  53,  54,  55,  60,  61,  62,  63,
  64, 65, 66, 67, 72, 73, 74, 75, 96,  97,  98,  99,  104, 105, 106, 107,
99 100 101 102
  68, 69, 70, 71, 76, 77, 78, 79, 100, 101, 102, 103, 108, 109, 110, 111,
  80, 81, 82, 83, 88, 89, 90, 91, 112, 113, 114, 115, 120, 121, 122, 123,
  84, 85, 86, 87, 92, 93, 94, 95, 116, 117, 118, 119, 124, 125, 126, 127,
};
103
static const uint16_t orders_8x8[256] = {
clang-format's avatar
clang-format committed
104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121
  0,   1,   4,   5,   16,  17,  20,  21,  64,  65,  68,  69,  80,  81,  84,
  85,  2,   3,   6,   7,   18,  19,  22,  23,  66,  67,  70,  71,  82,  83,
  86,  87,  8,   9,   12,  13,  24,  25,  28,  29,  72,  73,  76,  77,  88,
  89,  92,  93,  10,  11,  14,  15,  26,  27,  30,  31,  74,  75,  78,  79,
  90,  91,  94,  95,  32,  33,  36,  37,  48,  49,  52,  53,  96,  97,  100,
  101, 112, 113, 116, 117, 34,  35,  38,  39,  50,  51,  54,  55,  98,  99,
  102, 103, 114, 115, 118, 119, 40,  41,  44,  45,  56,  57,  60,  61,  104,
  105, 108, 109, 120, 121, 124, 125, 42,  43,  46,  47,  58,  59,  62,  63,
  106, 107, 110, 111, 122, 123, 126, 127, 128, 129, 132, 133, 144, 145, 148,
  149, 192, 193, 196, 197, 208, 209, 212, 213, 130, 131, 134, 135, 146, 147,
  150, 151, 194, 195, 198, 199, 210, 211, 214, 215, 136, 137, 140, 141, 152,
  153, 156, 157, 200, 201, 204, 205, 216, 217, 220, 221, 138, 139, 142, 143,
  154, 155, 158, 159, 202, 203, 206, 207, 218, 219, 222, 223, 160, 161, 164,
  165, 176, 177, 180, 181, 224, 225, 228, 229, 240, 241, 244, 245, 162, 163,
  166, 167, 178, 179, 182, 183, 226, 227, 230, 231, 242, 243, 246, 247, 168,
  169, 172, 173, 184, 185, 188, 189, 232, 233, 236, 237, 248, 249, 252, 253,
  170, 171, 174, 175, 186, 187, 190, 191, 234, 235, 238, 239, 250, 251, 254,
  255,
122
};
123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282

#if CONFIG_CB4X4 && CONFIG_EXT_PARTITION
static const uint16_t orders_4x8[512] = {
  0,   1,   2,   3,   8,   9,   10,  11,  32,  33,  34,  35,  40,  41,  42,
  43,  128, 129, 130, 131, 136, 137, 138, 139, 160, 161, 162, 163, 168, 169,
  170, 171, 4,   5,   6,   7,   12,  13,  14,  15,  36,  37,  38,  39,  44,
  45,  46,  47,  132, 133, 134, 135, 140, 141, 142, 143, 164, 165, 166, 167,
  172, 173, 174, 175, 16,  17,  18,  19,  24,  25,  26,  27,  48,  49,  50,
  51,  56,  57,  58,  59,  144, 145, 146, 147, 152, 153, 154, 155, 176, 177,
  178, 179, 184, 185, 186, 187, 20,  21,  22,  23,  28,  29,  30,  31,  52,
  53,  54,  55,  60,  61,  62,  63,  148, 149, 150, 151, 156, 157, 158, 159,
  180, 181, 182, 183, 188, 189, 190, 191, 64,  65,  66,  67,  72,  73,  74,
  75,  96,  97,  98,  99,  104, 105, 106, 107, 192, 193, 194, 195, 200, 201,
  202, 203, 224, 225, 226, 227, 232, 233, 234, 235, 68,  69,  70,  71,  76,
  77,  78,  79,  100, 101, 102, 103, 108, 109, 110, 111, 196, 197, 198, 199,
  204, 205, 206, 207, 228, 229, 230, 231, 236, 237, 238, 239, 80,  81,  82,
  83,  88,  89,  90,  91,  112, 113, 114, 115, 120, 121, 122, 123, 208, 209,
  210, 211, 216, 217, 218, 219, 240, 241, 242, 243, 248, 249, 250, 251, 84,
  85,  86,  87,  92,  93,  94,  95,  116, 117, 118, 119, 124, 125, 126, 127,
  212, 213, 214, 215, 220, 221, 222, 223, 244, 245, 246, 247, 252, 253, 254,
  255, 256, 257, 258, 259, 264, 265, 266, 267, 288, 289, 290, 291, 296, 297,
  298, 299, 384, 385, 386, 387, 392, 393, 394, 395, 416, 417, 418, 419, 424,
  425, 426, 427, 260, 261, 262, 263, 268, 269, 270, 271, 292, 293, 294, 295,
  300, 301, 302, 303, 388, 389, 390, 391, 396, 397, 398, 399, 420, 421, 422,
  423, 428, 429, 430, 431, 272, 273, 274, 275, 280, 281, 282, 283, 304, 305,
  306, 307, 312, 313, 314, 315, 400, 401, 402, 403, 408, 409, 410, 411, 432,
  433, 434, 435, 440, 441, 442, 443, 276, 277, 278, 279, 284, 285, 286, 287,
  308, 309, 310, 311, 316, 317, 318, 319, 404, 405, 406, 407, 412, 413, 414,
  415, 436, 437, 438, 439, 444, 445, 446, 447, 320, 321, 322, 323, 328, 329,
  330, 331, 352, 353, 354, 355, 360, 361, 362, 363, 448, 449, 450, 451, 456,
  457, 458, 459, 480, 481, 482, 483, 488, 489, 490, 491, 324, 325, 326, 327,
  332, 333, 334, 335, 356, 357, 358, 359, 364, 365, 366, 367, 452, 453, 454,
  455, 460, 461, 462, 463, 484, 485, 486, 487, 492, 493, 494, 495, 336, 337,
  338, 339, 344, 345, 346, 347, 368, 369, 370, 371, 376, 377, 378, 379, 464,
  465, 466, 467, 472, 473, 474, 475, 496, 497, 498, 499, 504, 505, 506, 507,
  340, 341, 342, 343, 348, 349, 350, 351, 372, 373, 374, 375, 380, 381, 382,
  383, 468, 469, 470, 471, 476, 477, 478, 479, 500, 501, 502, 503, 508, 509,
  510, 511,
};

static const uint16_t orders_8x4[512] = {
  0,   2,   8,   10,  32,  34,  40,  42,  128, 130, 136, 138, 160, 162, 168,
  170, 1,   3,   9,   11,  33,  35,  41,  43,  129, 131, 137, 139, 161, 163,
  169, 171, 4,   6,   12,  14,  36,  38,  44,  46,  132, 134, 140, 142, 164,
  166, 172, 174, 5,   7,   13,  15,  37,  39,  45,  47,  133, 135, 141, 143,
  165, 167, 173, 175, 16,  18,  24,  26,  48,  50,  56,  58,  144, 146, 152,
  154, 176, 178, 184, 186, 17,  19,  25,  27,  49,  51,  57,  59,  145, 147,
  153, 155, 177, 179, 185, 187, 20,  22,  28,  30,  52,  54,  60,  62,  148,
  150, 156, 158, 180, 182, 188, 190, 21,  23,  29,  31,  53,  55,  61,  63,
  149, 151, 157, 159, 181, 183, 189, 191, 64,  66,  72,  74,  96,  98,  104,
  106, 192, 194, 200, 202, 224, 226, 232, 234, 65,  67,  73,  75,  97,  99,
  105, 107, 193, 195, 201, 203, 225, 227, 233, 235, 68,  70,  76,  78,  100,
  102, 108, 110, 196, 198, 204, 206, 228, 230, 236, 238, 69,  71,  77,  79,
  101, 103, 109, 111, 197, 199, 205, 207, 229, 231, 237, 239, 80,  82,  88,
  90,  112, 114, 120, 122, 208, 210, 216, 218, 240, 242, 248, 250, 81,  83,
  89,  91,  113, 115, 121, 123, 209, 211, 217, 219, 241, 243, 249, 251, 84,
  86,  92,  94,  116, 118, 124, 126, 212, 214, 220, 222, 244, 246, 252, 254,
  85,  87,  93,  95,  117, 119, 125, 127, 213, 215, 221, 223, 245, 247, 253,
  255, 256, 258, 264, 266, 288, 290, 296, 298, 384, 386, 392, 394, 416, 418,
  424, 426, 257, 259, 265, 267, 289, 291, 297, 299, 385, 387, 393, 395, 417,
  419, 425, 427, 260, 262, 268, 270, 292, 294, 300, 302, 388, 390, 396, 398,
  420, 422, 428, 430, 261, 263, 269, 271, 293, 295, 301, 303, 389, 391, 397,
  399, 421, 423, 429, 431, 272, 274, 280, 282, 304, 306, 312, 314, 400, 402,
  408, 410, 432, 434, 440, 442, 273, 275, 281, 283, 305, 307, 313, 315, 401,
  403, 409, 411, 433, 435, 441, 443, 276, 278, 284, 286, 308, 310, 316, 318,
  404, 406, 412, 414, 436, 438, 444, 446, 277, 279, 285, 287, 309, 311, 317,
  319, 405, 407, 413, 415, 437, 439, 445, 447, 320, 322, 328, 330, 352, 354,
  360, 362, 448, 450, 456, 458, 480, 482, 488, 490, 321, 323, 329, 331, 353,
  355, 361, 363, 449, 451, 457, 459, 481, 483, 489, 491, 324, 326, 332, 334,
  356, 358, 364, 366, 452, 454, 460, 462, 484, 486, 492, 494, 325, 327, 333,
  335, 357, 359, 365, 367, 453, 455, 461, 463, 485, 487, 493, 495, 336, 338,
  344, 346, 368, 370, 376, 378, 464, 466, 472, 474, 496, 498, 504, 506, 337,
  339, 345, 347, 369, 371, 377, 379, 465, 467, 473, 475, 497, 499, 505, 507,
  340, 342, 348, 350, 372, 374, 380, 382, 468, 470, 476, 478, 500, 502, 508,
  510, 341, 343, 349, 351, 373, 375, 381, 383, 469, 471, 477, 479, 501, 503,
  509, 511,
};

static const uint16_t orders_4x4[1024] = {
  0,    1,    4,    5,    16,   17,   20,   21,   64,   65,   68,   69,   80,
  81,   84,   85,   256,  257,  260,  261,  272,  273,  276,  277,  320,  321,
  324,  325,  336,  337,  340,  341,  2,    3,    6,    7,    18,   19,   22,
  23,   66,   67,   70,   71,   82,   83,   86,   87,   258,  259,  262,  263,
  274,  275,  278,  279,  322,  323,  326,  327,  338,  339,  342,  343,  8,
  9,    12,   13,   24,   25,   28,   29,   72,   73,   76,   77,   88,   89,
  92,   93,   264,  265,  268,  269,  280,  281,  284,  285,  328,  329,  332,
  333,  344,  345,  348,  349,  10,   11,   14,   15,   26,   27,   30,   31,
  74,   75,   78,   79,   90,   91,   94,   95,   266,  267,  270,  271,  282,
  283,  286,  287,  330,  331,  334,  335,  346,  347,  350,  351,  32,   33,
  36,   37,   48,   49,   52,   53,   96,   97,   100,  101,  112,  113,  116,
  117,  288,  289,  292,  293,  304,  305,  308,  309,  352,  353,  356,  357,
  368,  369,  372,  373,  34,   35,   38,   39,   50,   51,   54,   55,   98,
  99,   102,  103,  114,  115,  118,  119,  290,  291,  294,  295,  306,  307,
  310,  311,  354,  355,  358,  359,  370,  371,  374,  375,  40,   41,   44,
  45,   56,   57,   60,   61,   104,  105,  108,  109,  120,  121,  124,  125,
  296,  297,  300,  301,  312,  313,  316,  317,  360,  361,  364,  365,  376,
  377,  380,  381,  42,   43,   46,   47,   58,   59,   62,   63,   106,  107,
  110,  111,  122,  123,  126,  127,  298,  299,  302,  303,  314,  315,  318,
  319,  362,  363,  366,  367,  378,  379,  382,  383,  128,  129,  132,  133,
  144,  145,  148,  149,  192,  193,  196,  197,  208,  209,  212,  213,  384,
  385,  388,  389,  400,  401,  404,  405,  448,  449,  452,  453,  464,  465,
  468,  469,  130,  131,  134,  135,  146,  147,  150,  151,  194,  195,  198,
  199,  210,  211,  214,  215,  386,  387,  390,  391,  402,  403,  406,  407,
  450,  451,  454,  455,  466,  467,  470,  471,  136,  137,  140,  141,  152,
  153,  156,  157,  200,  201,  204,  205,  216,  217,  220,  221,  392,  393,
  396,  397,  408,  409,  412,  413,  456,  457,  460,  461,  472,  473,  476,
  477,  138,  139,  142,  143,  154,  155,  158,  159,  202,  203,  206,  207,
  218,  219,  222,  223,  394,  395,  398,  399,  410,  411,  414,  415,  458,
  459,  462,  463,  474,  475,  478,  479,  160,  161,  164,  165,  176,  177,
  180,  181,  224,  225,  228,  229,  240,  241,  244,  245,  416,  417,  420,
  421,  432,  433,  436,  437,  480,  481,  484,  485,  496,  497,  500,  501,
  162,  163,  166,  167,  178,  179,  182,  183,  226,  227,  230,  231,  242,
  243,  246,  247,  418,  419,  422,  423,  434,  435,  438,  439,  482,  483,
  486,  487,  498,  499,  502,  503,  168,  169,  172,  173,  184,  185,  188,
  189,  232,  233,  236,  237,  248,  249,  252,  253,  424,  425,  428,  429,
  440,  441,  444,  445,  488,  489,  492,  493,  504,  505,  508,  509,  170,
  171,  174,  175,  186,  187,  190,  191,  234,  235,  238,  239,  250,  251,
  254,  255,  426,  427,  430,  431,  442,  443,  446,  447,  490,  491,  494,
  495,  506,  507,  510,  511,  512,  513,  516,  517,  528,  529,  532,  533,
  576,  577,  580,  581,  592,  593,  596,  597,  768,  769,  772,  773,  784,
  785,  788,  789,  832,  833,  836,  837,  848,  849,  852,  853,  514,  515,
  518,  519,  530,  531,  534,  535,  578,  579,  582,  583,  594,  595,  598,
  599,  770,  771,  774,  775,  786,  787,  790,  791,  834,  835,  838,  839,
  850,  851,  854,  855,  520,  521,  524,  525,  536,  537,  540,  541,  584,
  585,  588,  589,  600,  601,  604,  605,  776,  777,  780,  781,  792,  793,
  796,  797,  840,  841,  844,  845,  856,  857,  860,  861,  522,  523,  526,
  527,  538,  539,  542,  543,  586,  587,  590,  591,  602,  603,  606,  607,
  778,  779,  782,  783,  794,  795,  798,  799,  842,  843,  846,  847,  858,
  859,  862,  863,  544,  545,  548,  549,  560,  561,  564,  565,  608,  609,
  612,  613,  624,  625,  628,  629,  800,  801,  804,  805,  816,  817,  820,
  821,  864,  865,  868,  869,  880,  881,  884,  885,  546,  547,  550,  551,
  562,  563,  566,  567,  610,  611,  614,  615,  626,  627,  630,  631,  802,
  803,  806,  807,  818,  819,  822,  823,  866,  867,  870,  871,  882,  883,
  886,  887,  552,  553,  556,  557,  568,  569,  572,  573,  616,  617,  620,
  621,  632,  633,  636,  637,  808,  809,  812,  813,  824,  825,  828,  829,
  872,  873,  876,  877,  888,  889,  892,  893,  554,  555,  558,  559,  570,
  571,  574,  575,  618,  619,  622,  623,  634,  635,  638,  639,  810,  811,
  814,  815,  826,  827,  830,  831,  874,  875,  878,  879,  890,  891,  894,
  895,  640,  641,  644,  645,  656,  657,  660,  661,  704,  705,  708,  709,
  720,  721,  724,  725,  896,  897,  900,  901,  912,  913,  916,  917,  960,
  961,  964,  965,  976,  977,  980,  981,  642,  643,  646,  647,  658,  659,
  662,  663,  706,  707,  710,  711,  722,  723,  726,  727,  898,  899,  902,
  903,  914,  915,  918,  919,  962,  963,  966,  967,  978,  979,  982,  983,
  648,  649,  652,  653,  664,  665,  668,  669,  712,  713,  716,  717,  728,
  729,  732,  733,  904,  905,  908,  909,  920,  921,  924,  925,  968,  969,
  972,  973,  984,  985,  988,  989,  650,  651,  654,  655,  666,  667,  670,
  671,  714,  715,  718,  719,  730,  731,  734,  735,  906,  907,  910,  911,
  922,  923,  926,  927,  970,  971,  974,  975,  986,  987,  990,  991,  672,
  673,  676,  677,  688,  689,  692,  693,  736,  737,  740,  741,  752,  753,
  756,  757,  928,  929,  932,  933,  944,  945,  948,  949,  992,  993,  996,
  997,  1008, 1009, 1012, 1013, 674,  675,  678,  679,  690,  691,  694,  695,
  738,  739,  742,  743,  754,  755,  758,  759,  930,  931,  934,  935,  946,
  947,  950,  951,  994,  995,  998,  999,  1010, 1011, 1014, 1015, 680,  681,
  684,  685,  696,  697,  700,  701,  744,  745,  748,  749,  760,  761,  764,
  765,  936,  937,  940,  941,  952,  953,  956,  957,  1000, 1001, 1004, 1005,
  1016, 1017, 1020, 1021, 682,  683,  686,  687,  698,  699,  702,  703,  746,
  747,  750,  751,  762,  763,  766,  767,  938,  939,  942,  943,  954,  955,
  958,  959,  1002, 1003, 1006, 1007, 1018, 1019, 1022, 1023,
};
#endif
283
#endif  // CONFIG_CB4X4 || CONFIG_EXT_PARTITION
284

285
#if CONFIG_EXT_PARTITION
clang-format's avatar
clang-format committed
286
/* clang-format off */
287
static const uint16_t *const orders[BLOCK_SIZES] = {
Jingning Han's avatar
Jingning Han committed
288 289
#if CONFIG_CB4X4
  // 2X2,         2X4,            4X2
290 291 292 293 294 295
  orders_4x4,     orders_4x4,     orders_4x4,
  //                              4X4
                                  orders_4x4,
  // 4X8,         8X4,            8X8
  orders_4x8,     orders_8x4,     orders_8x8,
#else
296 297 298 299
  //                              4X4
                                  orders_8x8,
  // 4X8,         8X4,            8X8
  orders_8x8,     orders_8x8,     orders_8x8,
300
#endif
301 302 303 304 305 306 307 308
  // 8X16,        16X8,           16X16
  orders_8x16,    orders_16x8,    orders_16x16,
  // 16X32,       32X16,          32X32
  orders_16x32,   orders_32x16,   orders_32x32,
  // 32X64,       64X32,          64X64
  orders_32x64,   orders_64x32,   orders_64x64,
  // 64x128,      128x64,         128x128
  orders_64x128,  orders_128x64,  orders_128x128
309
};
clang-format's avatar
clang-format committed
310
/* clang-format on */
311
#else
clang-format's avatar
clang-format committed
312
/* clang-format off */
313
static const uint16_t *const orders[BLOCK_SIZES] = {
Jingning Han's avatar
Jingning Han committed
314 315
#if CONFIG_CB4X4
  // 2X2,         2X4,            4X2
316 317 318 319 320 321
  orders_8x8,     orders_8x8,     orders_8x8,
  //                              4X4
                                  orders_8x8,
  // 4X8,         8X4,            8X8
  orders_8x16,    orders_16x8,    orders_16x16,
#else
322 323 324 325
  //                              4X4
                                  orders_16x16,
  // 4X8,         8X4,            8X8
  orders_16x16,   orders_16x16,   orders_16x16,
326
#endif
327 328 329 330 331 332 333
  // 8X16,        16X8,           16X16
  orders_16x32,   orders_32x16,   orders_32x32,
  // 16X32,       32X16,          32X32
  orders_32x64,   orders_64x32,   orders_64x64,
  // 32X64,       64X32,          64X64
  orders_64x128,  orders_128x64,  orders_128x128
};
clang-format's avatar
clang-format committed
334
/* clang-format on */
335 336
#endif  // CONFIG_EXT_PARTITION

337
#if CONFIG_EXT_PARTITION_TYPES
338
static const uint16_t orders_verta_64x64[4] = {
clang-format's avatar
clang-format committed
339
  0, 2, 1, 2,
340
};
341
static const uint16_t orders_verta_32x32[16] = {
clang-format's avatar
clang-format committed
342
  0, 2, 4, 6, 1, 2, 5, 6, 8, 10, 12, 14, 9, 10, 13, 14,
343
};
344
static const uint16_t orders_verta_16x16[64] = {
clang-format's avatar
clang-format committed
345 346 347 348
  0,  2,  4,  6,  16, 18, 20, 22, 1,  2,  5,  6,  17, 18, 21, 22,
  8,  10, 12, 14, 24, 26, 28, 30, 9,  10, 13, 14, 25, 26, 29, 30,
  32, 34, 36, 38, 48, 50, 52, 54, 33, 34, 37, 38, 49, 50, 53, 54,
  40, 42, 44, 46, 56, 58, 60, 62, 41, 42, 45, 46, 57, 58, 61, 62,
349
};
350
#if CONFIG_EXT_PARTITION || CONFIG_CB4X4
351
static const uint16_t orders_verta_8x8[256] = {
clang-format's avatar
clang-format committed
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
  0,   2,   4,   6,   16,  18,  20,  22,  64,  66,  68,  70,  80,  82,  84,
  86,  1,   2,   5,   6,   17,  18,  21,  22,  65,  66,  69,  70,  81,  82,
  85,  86,  8,   10,  12,  14,  24,  26,  28,  30,  72,  74,  76,  78,  88,
  90,  92,  94,  9,   10,  13,  14,  25,  26,  29,  30,  73,  74,  77,  78,
  89,  90,  93,  94,  32,  34,  36,  38,  48,  50,  52,  54,  96,  98,  100,
  102, 112, 114, 116, 118, 33,  34,  37,  38,  49,  50,  53,  54,  97,  98,
  101, 102, 113, 114, 117, 118, 40,  42,  44,  46,  56,  58,  60,  62,  104,
  106, 108, 110, 120, 122, 124, 126, 41,  42,  45,  46,  57,  58,  61,  62,
  105, 106, 109, 110, 121, 122, 125, 126, 128, 130, 132, 134, 144, 146, 148,
  150, 192, 194, 196, 198, 208, 210, 212, 214, 129, 130, 133, 134, 145, 146,
  149, 150, 193, 194, 197, 198, 209, 210, 213, 214, 136, 138, 140, 142, 152,
  154, 156, 158, 200, 202, 204, 206, 216, 218, 220, 222, 137, 138, 141, 142,
  153, 154, 157, 158, 201, 202, 205, 206, 217, 218, 221, 222, 160, 162, 164,
  166, 176, 178, 180, 182, 224, 226, 228, 230, 240, 242, 244, 246, 161, 162,
  165, 166, 177, 178, 181, 182, 225, 226, 229, 230, 241, 242, 245, 246, 168,
  170, 172, 174, 184, 186, 188, 190, 232, 234, 236, 238, 248, 250, 252, 254,
  169, 170, 173, 174, 185, 186, 189, 190, 233, 234, 237, 238, 249, 250, 253,
  254,
370
};
371
#endif  // CONFIG_EXT_PARTITION || CONFIG_CB4X4
clang-format's avatar
clang-format committed
372

373
#if CONFIG_EXT_PARTITION
clang-format's avatar
clang-format committed
374
/* clang-format off */
375
static const uint16_t *const orders_verta[BLOCK_SIZES] = {
376 377 378 379
#if CONFIG_CB4X4
  // 2X2,           2X4,              4X2
  orders_4x4,       orders_4x4,       orders_4x4,
#endif
380 381 382 383 384 385 386 387 388 389 390 391 392
  //                                  4X4
                                      orders_verta_8x8,
  // 4X8,           8X4,              8X8
  orders_verta_8x8, orders_verta_8x8, orders_verta_8x8,
  // 8X16,          16X8,             16X16
  orders_8x16,      orders_16x8,      orders_verta_16x16,
  // 16X32,         32X16,            32X32
  orders_16x32,     orders_32x16,     orders_verta_32x32,
  // 32X64,         64X32,            64X64
  orders_32x64,     orders_64x32,     orders_verta_64x64,
  // 64x128,        128x64,           128x128
  orders_64x128,    orders_128x64,    orders_128x128
};
clang-format's avatar
clang-format committed
393
/* clang-format on */
394
#else
clang-format's avatar
clang-format committed
395
/* clang-format off */
396
static const uint16_t *const orders_verta[BLOCK_SIZES] = {
397 398 399 400 401 402 403 404
#if CONFIG_CB4X4
  // 2X2,             2X4,                4X2
  orders_verta_8x8,   orders_verta_8x8,   orders_verta_8x8,
  //                                      4X4
                                          orders_verta_8x8,
  // 4X8,             8X4,                8X8
  orders_verta_8x8,   orders_verta_8x8,   orders_verta_16x16,
#else
405 406 407 408
  //                                      4X4
                                          orders_verta_16x16,
  // 4X8,             8X4,                8X8
  orders_verta_16x16, orders_verta_16x16, orders_verta_16x16,
409
#endif
410 411 412 413 414 415
  // 8X16,            16X8,               16X16
  orders_16x32,       orders_32x16,       orders_verta_32x32,
  // 16X32,           32X16,              32X32
  orders_32x64,       orders_64x32,       orders_verta_64x64,
  // 32X64,           64X32,              64X64
  orders_64x128,      orders_128x64,      orders_128x128
416
};
clang-format's avatar
clang-format committed
417
/* clang-format on */
418
#endif  // CONFIG_EXT_PARTITION
419
#endif  // CONFIG_EXT_PARTITION_TYPES
420

Yaowu Xu's avatar
Yaowu Xu committed
421 422
static int av1_has_right(BLOCK_SIZE bsize, int mi_row, int mi_col,
                         int right_available,
423
#if CONFIG_EXT_PARTITION_TYPES
Yaowu Xu's avatar
Yaowu Xu committed
424
                         PARTITION_TYPE partition,
425
#endif
Yaowu Xu's avatar
Yaowu Xu committed
426
                         TX_SIZE txsz, int y, int x, int ss_x) {
427 428
  const int width = block_size_wide[bsize] >> tx_size_wide_log2[0];
  const int w = AOMMAX(width >> ss_x, 1);
429
  const int step = tx_size_wide_unit[txsz];
430
#if !CONFIG_CB4X4
431 432 433 434 435 436 437
  // TODO(bshacklett, huisu): Currently the RD loop traverses 4X8 blocks in
  // inverted N order while in the bitstream the subblocks are stored in Z
  // order. This discrepancy makes this function incorrect when considering 4X8
  // blocks in the RD loop, so we disable the extended right edge for these
  // blocks. The correct solution is to change the bitstream to store these
  // blocks in inverted N order, and then update this function appropriately.
  if (bsize == BLOCK_4X8 && y == 1) return 0;
438
#endif
439

440 441 442
  if (!right_available) return 0;

  // Handle block size 4x8 and 4x4
443
  if (ss_x == 0 && width < 2 && x == 0) return 1;
444

445
  if (y == 0) {
446
    const int wl = mi_width_log2_lookup[bsize];
447
    const int hl = mi_height_log2_lookup[bsize];
448
    const uint16_t *order;
449
    int my_order, tr_order;
450

451
#if CONFIG_EXT_PARTITION_TYPES
452 453 454
    if (partition == PARTITION_VERT_A)
      order = orders_verta[bsize];
    else
455
#endif  // CONFIG_EXT_PARTITION_TYPES
456
      order = orders[bsize];
457

458
    if (x + step < w) return 1;
459

460 461
    mi_row = (mi_row & MAX_MIB_MASK) >> hl;
    mi_col = (mi_col & MAX_MIB_MASK) >> wl;
462

463 464
    // If top row of coding unit
    if (mi_row == 0) return 1;
465

466 467
    // If rightmost column of coding unit
    if (((mi_col + 1) << wl) >= MAX_MIB_SIZE) return 0;
468

469 470
    my_order = order[((mi_row + 0) << (MAX_MIB_SIZE_LOG2 - wl)) + mi_col + 0];
    tr_order = order[((mi_row - 1) << (MAX_MIB_SIZE_LOG2 - wl)) + mi_col + 1];
471

472 473 474
    return my_order > tr_order;
  } else {
    return x + step < w;
475 476 477
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
478 479 480
static int av1_has_bottom(BLOCK_SIZE bsize, int mi_row, int mi_col,
                          int bottom_available, TX_SIZE txsz, int y, int x,
                          int ss_y) {
481 482 483
  if (!bottom_available || x != 0) {
    return 0;
  } else {
hui su's avatar
hui su committed
484 485
    const int wl = mi_width_log2_lookup[bsize];
    const int hl = mi_height_log2_lookup[bsize];
486 487
    const int height = block_size_high[bsize] >> tx_size_high_log2[0];
    const int h = AOMMAX(height >> ss_y, 1);
488
    const int step = tx_size_wide_unit[txsz];
489
    const uint16_t *order = orders[bsize];
490 491
    int my_order, bl_order;

492
#if !CONFIG_CB4X4
493
    // Handle block size 8x4 and 4x4
494
    if (ss_y == 0 && height < 2 && y == 0) return 1;
495
#endif
496

clang-format's avatar
clang-format committed
497
    if (y + step < h) return 1;
498

499 500
    mi_row = (mi_row & MAX_MIB_MASK) >> hl;
    mi_col = (mi_col & MAX_MIB_MASK) >> wl;
501 502

    if (mi_col == 0)
503
      return (mi_row << (hl + !ss_y)) + y + step < (MAX_MIB_SIZE << !ss_y);
504

clang-format's avatar
clang-format committed
505
    if (((mi_row + 1) << hl) >= MAX_MIB_SIZE) return 0;
506

507 508
    my_order = order[((mi_row + 0) << (MAX_MIB_SIZE_LOG2 - wl)) + mi_col + 0];
    bl_order = order[((mi_row + 1) << (MAX_MIB_SIZE_LOG2 - wl)) + mi_col - 1];
509

510
    return bl_order < my_order;
511 512
  }
}
Jingning Han's avatar
Jingning Han committed
513 514 515 516 517 518 519

typedef void (*intra_pred_fn)(uint8_t *dst, ptrdiff_t stride,
                              const uint8_t *above, const uint8_t *left);

static intra_pred_fn pred[INTRA_MODES][TX_SIZES];
static intra_pred_fn dc_pred[2][2][TX_SIZES];

Yaowu Xu's avatar
Yaowu Xu committed
520
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
521 522 523
typedef void (*intra_high_pred_fn)(uint16_t *dst, ptrdiff_t stride,
                                   const uint16_t *above, const uint16_t *left,
                                   int bd);
524 525
static intra_high_pred_fn pred_high[INTRA_MODES][TX_SIZES];
static intra_high_pred_fn dc_pred_high[2][2][TX_SIZES];
Yaowu Xu's avatar
Yaowu Xu committed
526
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
527

Yaowu Xu's avatar
Yaowu Xu committed
528
static void av1_init_intra_predictors_internal(void) {
529 530 531 532 533 534 535
#if CONFIG_TX64X64
#define INIT_NO_4X4(p, type)                  \
  p[TX_8X8] = aom_##type##_predictor_8x8;     \
  p[TX_16X16] = aom_##type##_predictor_16x16; \
  p[TX_32X32] = aom_##type##_predictor_32x32; \
  p[TX_64X64] = aom_##type##_predictor_64x64
#else
clang-format's avatar
clang-format committed
536
#define INIT_NO_4X4(p, type)                  \
Yaowu Xu's avatar
Yaowu Xu committed
537 538 539
  p[TX_8X8] = aom_##type##_predictor_8x8;     \
  p[TX_16X16] = aom_##type##_predictor_16x16; \
  p[TX_32X32] = aom_##type##_predictor_32x32
540
#endif  // CONFIG_TX64X64
Jingning Han's avatar
Jingning Han committed
541

Jingning Han's avatar
Jingning Han committed
542 543 544 545 546 547
#if CONFIG_CB4X4
#define INIT_ALL_SIZES(p, type)           \
  p[TX_2X2] = aom_##type##_predictor_2x2; \
  p[TX_4X4] = aom_##type##_predictor_4x4; \
  INIT_NO_4X4(p, type)
#else
clang-format's avatar
clang-format committed
548
#define INIT_ALL_SIZES(p, type)           \
Yaowu Xu's avatar
Yaowu Xu committed
549
  p[TX_4X4] = aom_##type##_predictor_4x4; \
550
  INIT_NO_4X4(p, type)
Jingning Han's avatar
Jingning Han committed
551
#endif
552

Jingning Han's avatar
Jingning Han committed
553 554
  INIT_ALL_SIZES(pred[V_PRED], v);
  INIT_ALL_SIZES(pred[H_PRED], h);
555 556 557
  INIT_ALL_SIZES(pred[D207_PRED], d207e);
  INIT_ALL_SIZES(pred[D45_PRED], d45e);
  INIT_ALL_SIZES(pred[D63_PRED], d63e);
Jingning Han's avatar
Jingning Han committed
558 559 560
  INIT_ALL_SIZES(pred[D117_PRED], d117);
  INIT_ALL_SIZES(pred[D135_PRED], d135);
  INIT_ALL_SIZES(pred[D153_PRED], d153);
Urvang Joshi's avatar
Urvang Joshi committed
561 562 563

#if CONFIG_ALT_INTRA
  INIT_ALL_SIZES(pred[TM_PRED], paeth);
564
  INIT_ALL_SIZES(pred[SMOOTH_PRED], smooth);
Urvang Joshi's avatar
Urvang Joshi committed
565
#else
Jingning Han's avatar
Jingning Han committed
566
  INIT_ALL_SIZES(pred[TM_PRED], tm);
Urvang Joshi's avatar
Urvang Joshi committed
567
#endif  // CONFIG_ALT_INTRA
Jingning Han's avatar
Jingning Han committed
568 569 570 571 572 573

  INIT_ALL_SIZES(dc_pred[0][0], dc_128);
  INIT_ALL_SIZES(dc_pred[0][1], dc_top);
  INIT_ALL_SIZES(dc_pred[1][0], dc_left);
  INIT_ALL_SIZES(dc_pred[1][1], dc);

Yaowu Xu's avatar
Yaowu Xu committed
574
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
575 576
  INIT_ALL_SIZES(pred_high[V_PRED], highbd_v);
  INIT_ALL_SIZES(pred_high[H_PRED], highbd_h);
577 578 579
  INIT_ALL_SIZES(pred_high[D207_PRED], highbd_d207e);
  INIT_ALL_SIZES(pred_high[D45_PRED], highbd_d45e);
  INIT_ALL_SIZES(pred_high[D63_PRED], highbd_d63e);
Jingning Han's avatar
Jingning Han committed
580 581 582
  INIT_ALL_SIZES(pred_high[D117_PRED], highbd_d117);
  INIT_ALL_SIZES(pred_high[D135_PRED], highbd_d135);
  INIT_ALL_SIZES(pred_high[D153_PRED], highbd_d153);
Urvang Joshi's avatar
Urvang Joshi committed
583 584 585

#if CONFIG_ALT_INTRA
  INIT_ALL_SIZES(pred_high[TM_PRED], highbd_paeth);
586
  INIT_ALL_SIZES(pred_high[SMOOTH_PRED], highbd_smooth);
Urvang Joshi's avatar
Urvang Joshi committed
587
#else
Jingning Han's avatar
Jingning Han committed
588
  INIT_ALL_SIZES(pred_high[TM_PRED], highbd_tm);
Urvang Joshi's avatar
Urvang Joshi committed
589
#endif  // CONFIG_ALT_INTRA
Jingning Han's avatar
Jingning Han committed
590 591 592 593 594

  INIT_ALL_SIZES(dc_pred_high[0][0], highbd_dc_128);
  INIT_ALL_SIZES(dc_pred_high[0][1], highbd_dc_top);
  INIT_ALL_SIZES(dc_pred_high[1][0], highbd_dc_left);
  INIT_ALL_SIZES(dc_pred_high[1][1], highbd_dc);
Yaowu Xu's avatar
Yaowu Xu committed
595
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
596 597 598 599

#undef intra_pred_allsizes
}

hui su's avatar
hui su committed
600
#if CONFIG_EXT_INTRA
hui su's avatar
hui su committed
601
#if CONFIG_INTRA_INTERP
602 603 604 605 606 607 608 609 610 611 612
static int intra_subpel_interp(int base, int shift, const uint8_t *ref,
                               int ref_start_idx, int ref_end_idx,
                               INTRA_FILTER filter_type) {
  int val, k, idx, filter_idx = 0;
  const int16_t *filter = NULL;

  if (filter_type == INTRA_FILTER_LINEAR) {
    val = ref[base] * (256 - shift) + ref[base + 1] * shift;
    val = ROUND_POWER_OF_TWO(val, 8);
  } else {
    filter_idx = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS);
Yaowu Xu's avatar
Yaowu Xu committed
613
    filter = av1_intra_filter_kernels[filter_type][filter_idx];
614 615 616 617 618

    if (filter_idx < (1 << SUBPEL_BITS)) {
      val = 0;
      for (k = 0; k < SUBPEL_TAPS; ++k) {
        idx = base + 1 - (SUBPEL_TAPS / 2) + k;
Yaowu Xu's avatar
Yaowu Xu committed
619
        idx = AOMMAX(AOMMIN(idx, ref_end_idx), ref_start_idx);
620 621 622 623 624 625 626 627 628 629
        val += ref[idx] * filter[k];
      }
      val = ROUND_POWER_OF_TWO(val, FILTER_BITS);
    } else {
      val = ref[base + 1];
    }
  }

  return val;
}
hui su's avatar
hui su committed
630
#endif  // CONFIG_INTRA_INTERP
631

hui su's avatar
hui su committed
632 633
// Directional prediction, zone 1: 0 < angle < 90
static void dr_prediction_z1(uint8_t *dst, ptrdiff_t stride, int bs,
hui su's avatar
hui su committed
634 635 636 637 638
                             const uint8_t *above, const uint8_t *left,
#if CONFIG_INTRA_INTERP
                             INTRA_FILTER filter_type,
#endif  // CONFIG_INTRA_INTERP
                             int dx, int dy) {
hui su's avatar
hui su committed
639
  int r, c, x, base, shift, val;
hui su's avatar
hui su committed
640 641 642 643

  (void)left;
  (void)dy;
  assert(dy == 1);
hui su's avatar
hui su committed
644
  assert(dx > 0);
hui su's avatar
hui su committed
645

hui su's avatar
hui su committed
646
#if CONFIG_INTRA_INTERP
hui su's avatar
hui su committed
647 648 649
  if (filter_type != INTRA_FILTER_LINEAR) {
    const int pad_size = SUBPEL_TAPS >> 1;
    int len;
650 651
    DECLARE_ALIGNED(16, uint8_t, buf[SUBPEL_SHIFTS][MAX_SB_SIZE]);
    DECLARE_ALIGNED(16, uint8_t, src[MAX_SB_SIZE + SUBPEL_TAPS]);
hui su's avatar
hui su committed
652 653 654 655 656 657 658 659
    uint8_t flags[SUBPEL_SHIFTS];

    memset(flags, 0, SUBPEL_SHIFTS * sizeof(flags[0]));
    memset(src, above[0], pad_size * sizeof(above[0]));
    memcpy(src + pad_size, above, 2 * bs * sizeof(above[0]));
    memset(src + pad_size + 2 * bs, above[2 * bs - 1],
           pad_size * sizeof(above[0]));
    flags[0] = 1;
hui su's avatar
hui su committed
660 661
    x = dx;
    for (r = 0; r < bs; ++r, dst += stride, x += dx) {
hui su's avatar
hui su committed
662
      base = x >> 8;
hui su's avatar
hui su committed
663 664 665 666 667 668
      shift = x & 0xFF;
      shift = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS);
      if (shift == SUBPEL_SHIFTS) {
        base += 1;
        shift = 0;
      }
Yaowu Xu's avatar
Yaowu Xu committed
669
      len = AOMMIN(bs, 2 * bs - 1 - base);
hui su's avatar
hui su committed
670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689
      if (len <= 0) {
        int i;
        for (i = r; i < bs; ++i) {
          memset(dst, above[2 * bs - 1], bs * sizeof(dst[0]));
          dst += stride;
        }
        return;
      }

      if (len <= (bs >> 1) && !flags[shift]) {
        base = x >> 8;
        shift = x & 0xFF;
        for (c = 0; c < len; ++c) {
          val = intra_subpel_interp(base, shift, above, 0, 2 * bs - 1,
                                    filter_type);
          dst[c] = clip_pixel(val);
          ++base;
        }
      } else {
        if (!flags[shift]) {
Yaowu Xu's avatar
Yaowu Xu committed
690 691
          const int16_t *filter = av1_intra_filter_kernels[filter_type][shift];
          aom_convolve8_horiz(src + pad_size, 2 * bs, buf[shift], 2 * bs,
clang-format's avatar
clang-format committed
692 693
                              filter, 16, NULL, 16, 2 * bs,
                              2 * bs < 16 ? 2 : 1);
hui su's avatar
hui su committed
694 695 696
          flags[shift] = 1;
        }
        memcpy(dst, shift == 0 ? src + pad_size + base : &buf[shift][base],
clang-format's avatar
clang-format committed
697
               len * sizeof(dst[0]));
hui su's avatar
hui su committed
698 699 700 701 702 703 704
      }

      if (len < bs)
        memset(dst + len, above[2 * bs - 1], (bs - len) * sizeof(dst[0]));
    }
    return;
  }
hui su's avatar
hui su committed
705
#endif  // CONFIG_INTRA_INTERP
hui su's avatar
hui su committed
706

hui su's avatar
hui su committed
707 708
  x = dx;
  for (r = 0; r < bs; ++r, dst += stride, x += dx) {
hui su's avatar
hui su committed
709 710 711 712 713 714 715 716 717 718 719 720 721
    base = x >> 8;
    shift = x & 0xFF;

    if (base >= 2 * bs - 1) {
      int i;
      for (i = r; i < bs; ++i) {
        memset(dst, above[2 * bs - 1], bs * sizeof(dst[0]));
        dst += stride;
      }
      return;
    }

    for (c = 0; c < bs; ++c, ++base) {
hui su's avatar
hui su committed
722
      if (base < 2 * bs - 1) {
hui su's avatar
hui su committed
723 724
        val = above[base] * (256 - shift) + above[base + 1] * shift;
        val = ROUND_POWER_OF_TWO(val, 8);
hui su's avatar
hui su committed
725 726 727 728 729 730 731 732 733 734
        dst[c] = clip_pixel(val);
      } else {
        dst[c] = above[2 * bs - 1];
      }
    }
  }
}

// Directional prediction, zone 2: 90 < angle < 180
static void dr_prediction_z2(uint8_t *dst, ptrdiff_t stride, int bs,
hui su's avatar
hui su committed
735 736 737 738 739
                             const uint8_t *above, const uint8_t *left,
#if CONFIG_INTRA_INTERP
                             INTRA_FILTER filter_type,
#endif  // CONFIG_INTRA_INTERP
                             int dx, int dy) {
Debargha Mukherjee's avatar
Debargha Mukherjee committed
740
  int r, c, x, y, shift1, shift2, val, base1, base2;
hui su's avatar
hui su committed
741 742 743 744

  assert(dx > 0);
  assert(dy > 0);

hui su's avatar
hui su committed
745 746 747 748 749 750 751
  x = -dx;
  for (r = 0; r < bs; ++r, x -= dx, dst += stride) {
    base1 = x >> 8;
    y = (r << 8) - dy;
    for (c = 0; c < bs; ++c, ++base1, y -= dy) {
      if (base1 >= -1) {
        shift1 = x & 0xFF;
hui su's avatar
hui su committed
752
#if CONFIG_INTRA_INTERP
clang-format's avatar
clang-format committed
753 754
        val =
            intra_subpel_interp(base1, shift1, above, -1, bs - 1, filter_type);
hui su's avatar
hui su committed
755 756 757 758
#else
        val = above[base1] * (256 - shift1) + above[base1 + 1] * shift1;
        val = ROUND_POWER_OF_TWO(val, 8);
#endif  // CONFIG_INTRA_INTERP
hui su's avatar
hui su committed
759
      } else {
hui su's avatar
hui su committed
760 761 762
        base2 = y >> 8;
        if (base2 >= 0) {
          shift2 = y & 0xFF;
hui su's avatar
hui su committed
763
#if CONFIG_INTRA_INTERP
clang-format's avatar
clang-format committed
764 765
          val =
              intra_subpel_interp(base2, shift2, left, 0, bs - 1, filter_type);
hui su's avatar
hui su committed
766 767 768 769
#else
          val = left[base2] * (256 - shift2) + left[base2 + 1] * shift2;
          val = ROUND_POWER_OF_TWO(val, 8);
#endif  // CONFIG_INTRA_INTERP
hui su's avatar
hui su committed
770
        } else {
771
          val = left[0];
hui su's avatar
hui su committed
772 773 774 775 776 777 778 779 780
        }
      }
      dst[c] = clip_pixel(val);
    }
  }
}

// Directional prediction, zone 3: 180 < angle < 270
static void dr_prediction_z3(uint8_t *dst, ptrdiff_t stride, int bs,
hui su's avatar
hui su committed
781 782 783 784 785
                             const uint8_t *above, const uint8_t *left,
#if CONFIG_INTRA_INTERP
                             INTRA_FILTER filter_type,
#endif  // CONFIG_INTRA_INTERP
                             int dx, int dy) {
hui su's avatar
hui su committed
786
  int r, c, y, base, shift, val;
hui su's avatar
hui su committed
787 788 789

  (void)above;
  (void)dx;
790

hui su's avatar
hui su committed
791
  assert(dx == 1);
hui su's avatar
hui su committed
792
  assert(dy > 0);
hui su's avatar
hui su committed
793

hui su's avatar
hui su committed
794
#if CONFIG_INTRA_INTERP
hui su's avatar
hui su committed
795 796 797
  if (filter_type != INTRA_FILTER_LINEAR) {
    const int pad_size = SUBPEL_TAPS >> 1;
    int len, i;
798 799
    DECLARE_ALIGNED(16, uint8_t, buf[MAX_SB_SIZE][4 * SUBPEL_SHIFTS]);
    DECLARE_ALIGNED(16, uint8_t, src[(MAX_SB_SIZE + SUBPEL_TAPS) * 4]);
hui su's avatar
hui su committed
800 801 802
    uint8_t flags[SUBPEL_SHIFTS];

    memset(flags, 0, SUBPEL_SHIFTS * sizeof(flags[0]));
clang-format's avatar
clang-format committed
803 804
    for (i = 0; i < pad_size; ++i) src[4 * i] = left[0];
    for (i = 0; i < 2 * bs; ++i) src[4 * (i + pad_size)] = left[i];
hui su's avatar
hui su committed
805 806 807
    for (i = 0; i < pad_size; ++i)
      src[4 * (i + 2 * bs + pad_size)] = left[2 * bs - 1];
    flags[0] = 1;
hui su's avatar
hui su committed
808 809
    y = dy;
    for (c = 0; c < bs; ++c, y += dy) {
hui su's avatar
hui su committed
810
      base = y >> 8;
hui su's avatar
hui su committed
811 812 813 814 815 816
      shift = y & 0xFF;
      shift = ROUND_POWER_OF_TWO(shift, 8 - SUBPEL_BITS);
      if (shift == SUBPEL_SHIFTS) {
        base += 1;
        shift = 0;
      }
Yaowu Xu's avatar
Yaowu Xu committed
817
      len = AOMMIN(bs, 2 * bs - 1 - base);
hui su's avatar
hui su committed
818 819

      if (len <= 0) {
Debargha Mukherjee's avatar
Debargha Mukherjee committed
820
        for (r = 0; r < bs; ++r) {
clang-format's avatar
clang-format committed
821
          dst[r * stride + c] = left[2 * bs - 1];
hui su's avatar
hui su committed
822 823 824 825 826 827 828 829 830 831 832 833 834 835 836
        }
        continue;
      }

      if (len <= (bs >> 1) && !flags[shift]) {
        base = y >> 8;
        shift = y & 0xFF;
        for (r = 0; r < len; ++r) {
          val = intra_subpel_interp(base, shift, left, 0, 2 * bs - 1,
                                    filter_type);
          dst[r * stride + c] = clip_pixel(val);
          ++base;
        }
      } else {
        if (!flags[shift]) {
Yaowu Xu's avatar
Yaowu Xu committed
837 838
          const int16_t *filter = av1_intra_filter_kernels[filter_type][shift];
          aom_convolve8_vert(src + 4 * pad_size, 4, buf[0] + 4 * shift,
clang-format's avatar
clang-format committed
839
                             4 * SUBPEL_SHIFTS, NULL, 16, filter, 16,
hui su's avatar
hui su committed
840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856
                             2 * bs < 16 ? 4 : 4, 2 * bs);
          flags[shift] = 1;
        }

        if (shift == 0) {
          for (r = 0; r < len; ++r) {
            dst[r * stride + c] = left[r + base];
          }
        } else {
          for (r = 0; r < len; ++r) {
            dst[r * stride + c] = buf[r + base][4 * shift];
          }
        }
      }

      if (len < bs) {
        for (r = len; r < bs; ++r) {
clang-format's avatar
clang-format committed
857
          dst[r * stride + c] = left[2 * bs - 1];
hui su's avatar
hui su committed
858 859 860 861 862
        }
      }
    }
    return;
  }
hui su's avatar
hui su committed
863
#endif  // CONFIG_INTRA_INTERP
hui su's avatar
hui su committed
864

hui su's avatar
hui su committed
865 866
  y = dy;
  for (c = 0; c < bs; ++c, y += dy) {
hui su's avatar
hui su committed
867 868 869 870
    base = y >> 8;
    shift = y & 0xFF;

    for (r = 0; r < bs; ++r, ++base) {
871
      if (base < 2 * bs - 1) {
hui su's avatar
hui su committed
872 873 874
        val = left[base] * (256 - shift) + left[base + 1] * shift;
        val = ROUND_POWER_OF_TWO(val, 8);
        dst[r * stride + c] = clip_pixel(val);
hui su's avatar
hui su committed
875
      } else {
clang-format's avatar
clang-format committed
876
        for (; r < bs; ++r) dst[r * stride + c] = left[2 * bs - 1];
hui su's avatar
hui su committed
877
        break;
hui su's avatar
hui su committed
878 879 880 881 882
      }
    }
  }
}

883 884 885 886
// Get the shift (up-scaled by 256) in X w.r.t a unit change in Y.
// If angle > 0 && angle < 90, dx = -((int)(256 / t));
// If angle > 90 && angle < 180, dx = (int)(256 / t);
// If angle > 180 && angle < 270, dx = 1;
Yaowu Xu's avatar
Yaowu Xu committed
887
static INLINE int get_dx(int angle) {
888
  if (angle > 0 && angle < 90) {
hui su's avatar
hui su committed
889
    return dr_intra_derivative[angle];
890 891 892 893 894 895 896 897 898 899 900 901
  } else if (angle > 90 && angle < 180) {
    return dr_intra_derivative[180 - angle];
  } else {
    // In this case, we are not really going to use dx. We may return any value.
    return 1;
  }
}

// Get the shift (up-scaled by 256) in Y w.r.t a unit change in X.
// If angle > 0 && angle < 90, dy = 1;
// If angle > 90 && angle < 180, dy = (int)(256 * t);
// If angle > 180 && angle < 270, dy = -((int)(256 * t));
Yaowu Xu's avatar
Yaowu Xu committed
902
static INLINE int get_dy(int angle) {
903
  if (angle > 90 && angle < 180) {
clang-format's avatar
clang-format committed
904 905
    return dr_intra_derivative[angle - 90];
  } else if (angle > 180 && angle < 270) {
hui su's avatar
hui su committed
906
    return dr_intra_derivative[270 - angle];
clang-format's avatar
clang-format committed
907
  } else {
908 909 910 911 912
    // In this case, we are not really going to use dy. We may return any value.
    return 1;
  }
}

hui su's avatar
hui su committed
913
static void dr_predictor(uint8_t *dst, ptrdiff_t stride, TX_SIZE tx_size,
hui su's avatar
hui su committed
914 915 916 917 918
                         const uint8_t *above, const uint8_t *left,
#if CONFIG_INTRA_INTERP
                         INTRA_FILTER filter_type,
#endif  // CONFIG_INTRA_INTERP
                         int angle) {
919 920
  const int dx = get_dx(angle);
  const int dy = get_dy(angle);
921
  const int bs = tx_size_wide[tx_size];
922 923
  assert(angle > 0 && angle < 270);

hui su's avatar
hui su committed
924
  if (angle > 0 && angle < 90) {
hui su's avatar
hui su committed
925 926 927 928 929
    dr_prediction_z1(dst, stride, bs, above, left,
#if CONFIG_INTRA_INTERP
                     filter_type,
#endif  // CONFIG_INTRA_INTERP
                     dx, dy);
hui su's avatar
hui su committed
930
  } else if (angle > 90 && angle < 180) {
hui su's avatar
hui su committed
931 932 933 934 935
    dr_prediction_z2(dst, stride, bs, above, left,
#if CONFIG_INTRA_INTERP
                     filter_type,
#endif  // CONFIG_INTRA_INTERP
                     dx, dy);
hui su's avatar
hui su committed
936
  } else if (angle > 180 && angle < 270) {
hui su's avatar
hui su committed
937 938 939 940 941
    dr_prediction_z3(dst, stride, bs, above, left,
#if CONFIG_INTRA_INTERP
                     filter_type,
#endif  // CONFIG_INTRA_INTERP
                     dx, dy);
hui su's avatar
hui su committed
942
  } else if (angle == 90) {
hui su's avatar
hui su committed
943
    pred[V_PRED][tx_size](dst, stride, above, left);
hui su's avatar
hui su committed
944
  } else if (angle == 180) {
hui su's avatar
hui su committed
945
    pred[H_PRED][tx_size](dst, stride, above, left);