Commit b3ad8128 authored by Alex Converse's avatar Alex Converse

Port switch to 9-bit rate cost to vp10.

Brings the following commits to vp10:
269428e3 Tie the bit cost scale to a define.
d13385ce Switch to 9-bit rate cost constants built on a 256 probability denominator.
ad43a738 Fix a signed overflow in vp9 motion cost.
1c9b0918 Fix some interger overflow errors
fac947df Restore previous motion search bit-error scale.

Change-Id: I598ba7ee7efcde18439c31dfa96b86cbf297a580
parent 00380700
......@@ -81,8 +81,14 @@ struct macroblock {
int skip_optimize;
int q_index;
// The equivalent error at the current rdmult of one whole bit (not one
// bitcost unit).
int errorperbit;
// The equivalend SAD error of one (whole) bit at the current quantizer
// for large blocks.
int sadperbit16;
// The equivalend SAD error of one (whole) bit at the current quantizer
// for sub-8x8 blocks.
int sadperbit4;
int rddiv;
int rdmult;
......
......@@ -12,29 +12,32 @@
#include "vp10/encoder/cost.h"
#include "vp10/common/entropy.h"
const unsigned int vp10_prob_cost[256] = {
2047, 2047, 1791, 1641, 1535, 1452, 1385, 1328, 1279, 1235, 1196, 1161,
1129, 1099, 1072, 1046, 1023, 1000, 979, 959, 940, 922, 905, 889,
873, 858, 843, 829, 816, 803, 790, 778, 767, 755, 744, 733,
723, 713, 703, 693, 684, 675, 666, 657, 649, 641, 633, 625,
617, 609, 602, 594, 587, 580, 573, 567, 560, 553, 547, 541,
534, 528, 522, 516, 511, 505, 499, 494, 488, 483, 477, 472,
467, 462, 457, 452, 447, 442, 437, 433, 428, 424, 419, 415,
410, 406, 401, 397, 393, 389, 385, 381, 377, 373, 369, 365,
361, 357, 353, 349, 346, 342, 338, 335, 331, 328, 324, 321,
317, 314, 311, 307, 304, 301, 297, 294, 291, 288, 285, 281,
278, 275, 272, 269, 266, 263, 260, 257, 255, 252, 249, 246,
243, 240, 238, 235, 232, 229, 227, 224, 221, 219, 216, 214,
211, 208, 206, 203, 201, 198, 196, 194, 191, 189, 186, 184,
181, 179, 177, 174, 172, 170, 168, 165, 163, 161, 159, 156,
154, 152, 150, 148, 145, 143, 141, 139, 137, 135, 133, 131,
129, 127, 125, 123, 121, 119, 117, 115, 113, 111, 109, 107,
105, 103, 101, 99, 97, 95, 93, 92, 90, 88, 86, 84,
82, 81, 79, 77, 75, 73, 72, 70, 68, 66, 65, 63,
61, 60, 58, 56, 55, 53, 51, 50, 48, 46, 45, 43,
41, 40, 38, 37, 35, 33, 32, 30, 29, 27, 25, 24,
22, 21, 19, 18, 16, 15, 13, 12, 10, 9, 7, 6,
4, 3, 1, 1};
/* round(-log2(i/256.) * (1 << VP9_PROB_COST_SHIFT))
Begins and ends with a bogus entry to satisfy use of prob=0 in the firstpass.
https://code.google.com/p/webm/issues/detail?id=1089 */
const uint16_t vp10_prob_cost[257] = {
4096, 4096, 3584, 3284, 3072, 2907, 2772, 2659, 2560, 2473, 2395, 2325,
2260, 2201, 2147, 2096, 2048, 2003, 1961, 1921, 1883, 1847, 1813, 1780,
1748, 1718, 1689, 1661, 1635, 1609, 1584, 1559, 1536, 1513, 1491, 1470,
1449, 1429, 1409, 1390, 1371, 1353, 1335, 1318, 1301, 1284, 1268, 1252,
1236, 1221, 1206, 1192, 1177, 1163, 1149, 1136, 1123, 1110, 1097, 1084,
1072, 1059, 1047, 1036, 1024, 1013, 1001, 990, 979, 968, 958, 947,
937, 927, 917, 907, 897, 887, 878, 868, 859, 850, 841, 832,
823, 814, 806, 797, 789, 780, 772, 764, 756, 748, 740, 732,
724, 717, 709, 702, 694, 687, 680, 673, 665, 658, 651, 644,
637, 631, 624, 617, 611, 604, 598, 591, 585, 578, 572, 566,
560, 554, 547, 541, 535, 530, 524, 518, 512, 506, 501, 495,
489, 484, 478, 473, 467, 462, 456, 451, 446, 441, 435, 430,
425, 420, 415, 410, 405, 400, 395, 390, 385, 380, 375, 371,
366, 361, 356, 352, 347, 343, 338, 333, 329, 324, 320, 316,
311, 307, 302, 298, 294, 289, 285, 281, 277, 273, 268, 264,
260, 256, 252, 248, 244, 240, 236, 232, 228, 224, 220, 216,
212, 209, 205, 201, 197, 194, 190, 186, 182, 179, 175, 171,
168, 164, 161, 157, 153, 150, 146, 143, 139, 136, 132, 129,
125, 122, 119, 115, 112, 109, 105, 102, 99, 95, 92, 89,
86, 82, 79, 76, 73, 70, 66, 63, 60, 57, 54, 51,
48, 45, 42, 38, 35, 32, 29, 26, 23, 20, 18, 15,
12, 9, 6, 3, 3};
static void cost(int *costs, vpx_tree tree, const vpx_prob *probs,
int i, int c) {
......
......@@ -12,18 +12,22 @@
#define VP10_ENCODER_COST_H_
#include "vpx_dsp/prob.h"
#include "vpx/vpx_integer.h"
#ifdef __cplusplus
extern "C" {
#endif
extern const unsigned int vp10_prob_cost[256];
extern const uint16_t vp10_prob_cost[257];
// The factor to scale from cost in bits to cost in vp10_prob_cost units.
#define VP9_PROB_COST_SHIFT 9
#define vp10_cost_zero(prob) (vp10_prob_cost[prob])
#define vp10_cost_one(prob) vp10_cost_zero(vpx_complement(prob))
#define vp10_cost_one(prob) vp10_cost_zero(256 - (prob))
#define vp10_cost_bit(prob, bit) vp10_cost_zero((bit) ? vpx_complement(prob) \
#define vp10_cost_bit(prob, bit) vp10_cost_zero((bit) ? 256 - (prob) \
: (prob))
static INLINE unsigned int cost_branch256(const unsigned int ct[2],
......
......@@ -52,7 +52,9 @@ void vp10_subtract_plane(MACROBLOCK *x, BLOCK_SIZE bsize, int plane) {
pd->dst.buf, pd->dst.stride);
}
#define RDTRUNC(RM, DM, R, D) ((128 + (R) * (RM)) & 0xFF)
#define RDTRUNC(RM, DM, R, D) \
(((1 << (VP9_PROB_COST_SHIFT - 1)) + (R) * (RM)) & \
((1 << VP9_PROB_COST_SHIFT) - 1))
typedef struct vp10_token_state {
int rate;
......@@ -119,9 +121,9 @@ static int optimize_b(MACROBLOCK *mb, int plane, int block,
EXTRABIT e0;
int best, band, pt, i, final_eob;
#if CONFIG_VP9_HIGHBITDEPTH
const int16_t *cat6_high_cost = vp10_get_high_cost_table(xd->bd);
const int *cat6_high_cost = vp10_get_high_cost_table(xd->bd);
#else
const int16_t *cat6_high_cost = vp10_get_high_cost_table(8);
const int *cat6_high_cost = vp10_get_high_cost_table(8);
#endif
assert((!type && !plane) || (type && plane));
......
......@@ -80,24 +80,29 @@ int vp10_mv_bit_cost(const MV *mv, const MV *ref,
return ROUND_POWER_OF_TWO(mv_cost(&diff, mvjcost, mvcost) * weight, 7);
}
static int mv_err_cost(const MV *mv, const MV *ref,
const int *mvjcost, int *mvcost[2],
int error_per_bit) {
#define PIXEL_TRANSFORM_ERROR_SCALE 4
static int mv_err_cost(const MV *mv, const MV *ref, const int *mvjcost,
int *mvcost[2], int error_per_bit) {
if (mvcost) {
const MV diff = { mv->row - ref->row,
mv->col - ref->col };
return ROUND_POWER_OF_TWO(mv_cost(&diff, mvjcost, mvcost) *
error_per_bit, 13);
const MV diff = {mv->row - ref->row, mv->col - ref->col};
// This product sits at a 32-bit ceiling right now and any additional
// accuracy in either bit cost or error cost will cause it to overflow.
return ROUND_POWER_OF_TWO(
(unsigned)mv_cost(&diff, mvjcost, mvcost) * error_per_bit,
RDDIV_BITS + VP9_PROB_COST_SHIFT - RD_EPB_SHIFT +
PIXEL_TRANSFORM_ERROR_SCALE);
}
return 0;
}
static int mvsad_err_cost(const MACROBLOCK *x, const MV *mv, const MV *ref,
int error_per_bit) {
int sad_per_bit) {
const MV diff = { mv->row - ref->row,
mv->col - ref->col };
return ROUND_POWER_OF_TWO(mv_cost(&diff, x->nmvjointsadcost,
x->nmvsadcost) * error_per_bit, 8);
return ROUND_POWER_OF_TWO(
(unsigned)mv_cost(&diff, x->nmvjointsadcost, x->nmvsadcost) *
sad_per_bit,
VP9_PROB_COST_SHIFT);
}
void vp10_init_dsmotion_compensation(search_site_config *cfg, int stride) {
......@@ -155,12 +160,13 @@ void vp10_init3smotion_compensation(search_site_config *cfg, int stride) {
* could reduce the area.
*/
/* estimated cost of a motion vector (r,c) */
#define MVC(r, c) \
(mvcost ? \
((mvjcost[((r) != rr) * 2 + ((c) != rc)] + \
mvcost[0][((r) - rr)] + mvcost[1][((c) - rc)]) * \
error_per_bit + 4096) >> 13 : 0)
/* Estimated (square) error cost of a motion vector (r,c). The 14 scale comes
* from the same math as in mv_err_cost(). */
#define MVC(r, c) \
(mvcost ? \
((unsigned)(mvjcost[((r) != rr) * 2 + ((c) != rc)] + \
mvcost[0][((r) - rr)] + mvcost[1][((c) - rc)]) * \
error_per_bit + 8192) >> 14 : 0)
// convert motion vector component to offset for sv[a]f calc
......@@ -852,9 +858,9 @@ static INLINE void calc_int_cost_list(const MACROBLOCK *x,
cost_list[i + 1] = fn_ptr->vf(what->buf, what->stride,
get_buf_from_mv(in_what, &this_mv),
in_what->stride, &sse) +
// mvsad_err_cost(x, &this_mv, &fcenter_mv, sadpb);
mv_err_cost(&this_mv, &fcenter_mv, x->nmvjointcost, x->mvcost,
x->errorperbit);
mv_err_cost(&this_mv, &fcenter_mv,
x->nmvjointcost, x->mvcost,
x->errorperbit);
}
} else {
for (i = 0; i < 4; i++) {
......@@ -866,9 +872,9 @@ static INLINE void calc_int_cost_list(const MACROBLOCK *x,
cost_list[i + 1] = fn_ptr->vf(what->buf, what->stride,
get_buf_from_mv(in_what, &this_mv),
in_what->stride, &sse) +
// mvsad_err_cost(x, &this_mv, &fcenter_mv, sadpb);
mv_err_cost(&this_mv, &fcenter_mv, x->nmvjointcost, x->mvcost,
x->errorperbit);
mv_err_cost(&this_mv, &fcenter_mv,
x->nmvjointcost, x->mvcost,
x->errorperbit);
}
}
}
......
......@@ -519,8 +519,7 @@ void vp10_init_plane_quantizers(VP10_COMP *cpi, MACROBLOCK *x) {
x->skip_block = segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP);
x->q_index = qindex;
x->errorperbit = rdmult >> 6;
x->errorperbit += (x->errorperbit == 0);
set_error_per_bit(x, rdmult);
vp10_initialize_me_consts(cpi, x, x->q_index);
}
......
......@@ -41,7 +41,6 @@
#include "vp10/encoder/tokenize.h"
#define RD_THRESH_POW 1.25
#define RD_MULT_EPB_RATIO 64
// Factor to weigh the rate for switchable interp filters.
#define SWITCHABLE_INTERP_RATE_FACTOR 1
......@@ -343,8 +342,7 @@ void vp10_initialize_rd_consts(VP10_COMP *cpi) {
rd->RDDIV = RDDIV_BITS; // In bits (to multiply D by 128).
rd->RDMULT = vp10_compute_rd_mult(cpi, cm->base_qindex + cm->y_dc_delta_q);
x->errorperbit = rd->RDMULT / RD_MULT_EPB_RATIO;
x->errorperbit += (x->errorperbit == 0);
set_error_per_bit(x, rd->RDMULT);
x->select_tx_size = (cpi->sf.tx_size_search_method == USE_LARGESTALL &&
cm->frame_type != KEY_FRAME) ? 0 : 1;
......@@ -504,7 +502,7 @@ void vp10_model_rd_from_var_lapndz(unsigned int var, unsigned int n_log2,
(((uint64_t)qstep * qstep << (n_log2 + 10)) + (var >> 1)) / var;
const int xsq_q10 = (int)VPXMIN(xsq_q10_64, MAX_XSQ_Q10);
model_rd_norm(xsq_q10, &r_q10, &d_q10);
*rate = ((r_q10 << n_log2) + 2) >> 2;
*rate = ROUND_POWER_OF_TWO(r_q10 << n_log2, 10 - VP9_PROB_COST_SHIFT);
*dist = (var * (int64_t)d_q10 + 512) >> 10;
}
}
......
......@@ -17,18 +17,21 @@
#include "vp10/encoder/block.h"
#include "vp10/encoder/context_tree.h"
#include "vp10/encoder/cost.h"
#ifdef __cplusplus
extern "C" {
#endif
#define RDDIV_BITS 7
#define RD_EPB_SHIFT 6
#define RDCOST(RM, DM, R, D) \
(((128 + ((int64_t)R) * (RM)) >> 8) + (D << DM))
(ROUND_POWER_OF_TWO(((int64_t)R) * (RM), VP9_PROB_COST_SHIFT) + (D << DM))
#define RDCOST_DBL(RM, DM, R, D) \
(((((double)(R)) * (RM)) / 256.0) + ((double)(D) * (1 << (DM))))
#define RDCOST_DBL(RM, DM, R, D) \
(((((double)(R)) * (RM)) / (double)(1 << VP9_PROB_COST_SHIFT)) + \
((double)(D) * (1 << (DM))))
#define QIDX_SKIP_THRESH 115
......@@ -310,6 +313,11 @@ void vp10_mv_pred(struct VP10_COMP *cpi, MACROBLOCK *x,
uint8_t *ref_y_buffer, int ref_y_stride,
int ref_frame, BLOCK_SIZE block_size);
static INLINE void set_error_per_bit(MACROBLOCK *x, int rdmult) {
x->errorperbit = rdmult >> RD_EPB_SHIFT;
x->errorperbit += (x->errorperbit == 0);
}
void vp10_setup_pred_block(const MACROBLOCKD *xd,
struct buf_2d dst[MAX_MB_PLANE],
const YV12_BUFFER_CONFIG *src,
......
......@@ -411,7 +411,7 @@ static void model_rd_for_sb(VP10_COMP *cpi, BLOCK_SIZE bsize,
int quantizer = (pd->dequant[1] >> dequant_shift);
if (quantizer < 120)
rate = (square_error * (280 - quantizer)) >> 8;
rate = (square_error * (280 - quantizer)) >> (16 - VP9_PROB_COST_SHIFT);
else
rate = 0;
dist = (square_error * quantizer) >> 8;
......@@ -523,9 +523,9 @@ static int cost_coeffs(MACROBLOCK *x,
#endif
int c, cost;
#if CONFIG_VP9_HIGHBITDEPTH
const int16_t *cat6_high_cost = vp10_get_high_cost_table(xd->bd);
const int *cat6_high_cost = vp10_get_high_cost_table(xd->bd);
#else
const int16_t *cat6_high_cost = vp10_get_high_cost_table(8);
const int *cat6_high_cost = vp10_get_high_cost_table(8);
#endif
#if !CONFIG_VAR_TX && !CONFIG_SUPERTX
......
......@@ -80,7 +80,7 @@ static int remap_prob(int v, int m) {
static int prob_diff_update_cost(vpx_prob newp, vpx_prob oldp) {
int delp = remap_prob(newp, oldp);
return update_bits[delp] * 256;
return update_bits[delp] << VP9_PROB_COST_SHIFT;
}
static void encode_uniform(vpx_writer *w, int v) {
......
......@@ -75,292 +75,170 @@ static const vpx_tree_index cat6[28] = {2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12,
14, 14, 16, 16, 18, 18, 20, 20, 22, 22, 24, 24, 26, 26, 0, 0};
static const int16_t zero_cost[] = {0};
static const int16_t sign_cost[] = {255, 257};
static const int16_t cat1_cost[] = {429, 431, 616, 618};
static const int16_t cat2_cost[] = {624, 626, 727, 729, 848, 850, 951, 953};
static const int16_t cat3_cost[] = {
820, 822, 893, 895, 940, 942, 1013, 1015, 1096, 1098, 1169, 1171, 1216, 1218,
1289, 1291
};
static const int16_t cat4_cost[] = {
1032, 1034, 1075, 1077, 1105, 1107, 1148, 1150, 1194, 1196, 1237, 1239,
1267, 1269, 1310, 1312, 1328, 1330, 1371, 1373, 1401, 1403, 1444, 1446,
1490, 1492, 1533, 1535, 1563, 1565, 1606, 1608
};
static const int16_t cat5_cost[] = {
1269, 1271, 1283, 1285, 1306, 1308, 1320,
1322, 1347, 1349, 1361, 1363, 1384, 1386, 1398, 1400, 1443, 1445, 1457,
1459, 1480, 1482, 1494, 1496, 1521, 1523, 1535, 1537, 1558, 1560, 1572,
1574, 1592, 1594, 1606, 1608, 1629, 1631, 1643, 1645, 1670, 1672, 1684,
1686, 1707, 1709, 1721, 1723, 1766, 1768, 1780, 1782, 1803, 1805, 1817,
1819, 1844, 1846, 1858, 1860, 1881, 1883, 1895, 1897
};
static const int16_t sign_cost[1] = {512};
static const int16_t cat1_cost[1 << 1] = {864, 1229};
static const int16_t cat2_cost[1 << 2] = {1256, 1453, 1696, 1893};
static const int16_t cat3_cost[1 << 3] = {1652, 1791, 1884, 2023,
2195, 2334, 2427, 2566};
static const int16_t cat4_cost[1 << 4] = {2079, 2160, 2218, 2299, 2395, 2476,
2534, 2615, 2661, 2742, 2800, 2881,
2977, 3058, 3116, 3197};
static const int16_t cat5_cost[1 << 5] = {
2553, 2576, 2622, 2645, 2703, 2726, 2772, 2795, 2894, 2917, 2963,
2986, 3044, 3067, 3113, 3136, 3190, 3213, 3259, 3282, 3340, 3363,
3409, 3432, 3531, 3554, 3600, 3623, 3681, 3704, 3750, 3773};
const int16_t vp10_cat6_low_cost[256] = {
1638, 1640, 1646, 1648, 1652, 1654, 1660, 1662,
1670, 1672, 1678, 1680, 1684, 1686, 1692, 1694, 1711, 1713, 1719, 1721,
1725, 1727, 1733, 1735, 1743, 1745, 1751, 1753, 1757, 1759, 1765, 1767,
1787, 1789, 1795, 1797, 1801, 1803, 1809, 1811, 1819, 1821, 1827, 1829,
1833, 1835, 1841, 1843, 1860, 1862, 1868, 1870, 1874, 1876, 1882, 1884,
1892, 1894, 1900, 1902, 1906, 1908, 1914, 1916, 1940, 1942, 1948, 1950,
1954, 1956, 1962, 1964, 1972, 1974, 1980, 1982, 1986, 1988, 1994, 1996,
2013, 2015, 2021, 2023, 2027, 2029, 2035, 2037, 2045, 2047, 2053, 2055,
2059, 2061, 2067, 2069, 2089, 2091, 2097, 2099, 2103, 2105, 2111, 2113,
2121, 2123, 2129, 2131, 2135, 2137, 2143, 2145, 2162, 2164, 2170, 2172,
2176, 2178, 2184, 2186, 2194, 2196, 2202, 2204, 2208, 2210, 2216, 2218,
2082, 2084, 2090, 2092, 2096, 2098, 2104, 2106, 2114, 2116, 2122, 2124,
2128, 2130, 2136, 2138, 2155, 2157, 2163, 2165, 2169, 2171, 2177, 2179,
2187, 2189, 2195, 2197, 2201, 2203, 2209, 2211, 2231, 2233, 2239, 2241,
2245, 2247, 2253, 2255, 2263, 2265, 2271, 2273, 2277, 2279, 2285, 2287,
2304, 2306, 2312, 2314, 2318, 2320, 2326, 2328, 2336, 2338, 2344, 2346,
2350, 2352, 2358, 2360, 2384, 2386, 2392, 2394, 2398, 2400, 2406, 2408,
2416, 2418, 2424, 2426, 2430, 2432, 2438, 2440, 2457, 2459, 2465, 2467,
2471, 2473, 2479, 2481, 2489, 2491, 2497, 2499, 2503, 2505, 2511, 2513,
2533, 2535, 2541, 2543, 2547, 2549, 2555, 2557, 2565, 2567, 2573, 2575,
2579, 2581, 2587, 2589, 2606, 2608, 2614, 2616, 2620, 2622, 2628, 2630,
2638, 2640, 2646, 2648, 2652, 2654, 2660, 2662
};
const int16_t vp10_cat6_high_cost[128] = {
72, 892, 1183, 2003, 1448, 2268, 2559, 3379,
1709, 2529, 2820, 3640, 3085, 3905, 4196, 5016, 2118, 2938, 3229, 4049,
3494, 4314, 4605, 5425, 3755, 4575, 4866, 5686, 5131, 5951, 6242, 7062,
2118, 2938, 3229, 4049, 3494, 4314, 4605, 5425, 3755, 4575, 4866, 5686,
5131, 5951, 6242, 7062, 4164, 4984, 5275, 6095, 5540, 6360, 6651, 7471,
5801, 6621, 6912, 7732, 7177, 7997, 8288, 9108, 2118, 2938, 3229, 4049,
3494, 4314, 4605, 5425, 3755, 4575, 4866, 5686, 5131, 5951, 6242, 7062,
4164, 4984, 5275, 6095, 5540, 6360, 6651, 7471, 5801, 6621, 6912, 7732,
7177, 7997, 8288, 9108, 4164, 4984, 5275, 6095, 5540, 6360, 6651, 7471,
5801, 6621, 6912, 7732, 7177, 7997, 8288, 9108, 6210, 7030, 7321, 8141,
7586, 8406, 8697, 9517, 7847, 8667, 8958, 9778, 9223, 10043, 10334, 11154
};
3378, 3390, 3401, 3413, 3435, 3447, 3458, 3470, 3517, 3529, 3540, 3552,
3574, 3586, 3597, 3609, 3671, 3683, 3694, 3706, 3728, 3740, 3751, 3763,
3810, 3822, 3833, 3845, 3867, 3879, 3890, 3902, 3973, 3985, 3996, 4008,
4030, 4042, 4053, 4065, 4112, 4124, 4135, 4147, 4169, 4181, 4192, 4204,
4266, 4278, 4289, 4301, 4323, 4335, 4346, 4358, 4405, 4417, 4428, 4440,
4462, 4474, 4485, 4497, 4253, 4265, 4276, 4288, 4310, 4322, 4333, 4345,
4392, 4404, 4415, 4427, 4449, 4461, 4472, 4484, 4546, 4558, 4569, 4581,
4603, 4615, 4626, 4638, 4685, 4697, 4708, 4720, 4742, 4754, 4765, 4777,
4848, 4860, 4871, 4883, 4905, 4917, 4928, 4940, 4987, 4999, 5010, 5022,
5044, 5056, 5067, 5079, 5141, 5153, 5164, 5176, 5198, 5210, 5221, 5233,
5280, 5292, 5303, 5315, 5337, 5349, 5360, 5372, 4988, 5000, 5011, 5023,
5045, 5057, 5068, 5080, 5127, 5139, 5150, 5162, 5184, 5196, 5207, 5219,
5281, 5293, 5304, 5316, 5338, 5350, 5361, 5373, 5420, 5432, 5443, 5455,
5477, 5489, 5500, 5512, 5583, 5595, 5606, 5618, 5640, 5652, 5663, 5675,
5722, 5734, 5745, 5757, 5779, 5791, 5802, 5814, 5876, 5888, 5899, 5911,
5933, 5945, 5956, 5968, 6015, 6027, 6038, 6050, 6072, 6084, 6095, 6107,
5863, 5875, 5886, 5898, 5920, 5932, 5943, 5955, 6002, 6014, 6025, 6037,
6059, 6071, 6082, 6094, 6156, 6168, 6179, 6191, 6213, 6225, 6236, 6248,
6295, 6307, 6318, 6330, 6352, 6364, 6375, 6387, 6458, 6470, 6481, 6493,
6515, 6527, 6538, 6550, 6597, 6609, 6620, 6632, 6654, 6666, 6677, 6689,
6751, 6763, 6774, 6786, 6808, 6820, 6831, 6843, 6890, 6902, 6913, 6925,
6947, 6959, 6970, 6982};
const int vp10_cat6_high_cost[64] = {
88, 2251, 2727, 4890, 3148, 5311, 5787, 7950, 3666, 5829, 6305,
8468, 6726, 8889, 9365, 11528, 3666, 5829, 6305, 8468, 6726, 8889,
9365, 11528, 7244, 9407, 9883, 12046, 10304, 12467, 12943, 15106, 3666,
5829, 6305, 8468, 6726, 8889, 9365, 11528, 7244, 9407, 9883, 12046,
10304, 12467, 12943, 15106, 7244, 9407, 9883, 12046, 10304, 12467, 12943,
15106, 10822, 12985, 13461, 15624, 13882, 16045, 16521, 18684};
#if CONFIG_VP9_HIGHBITDEPTH
const int16_t vp10_cat6_high10_high_cost[512] = {
74, 894, 1185, 2005, 1450, 2270, 2561,
3381, 1711, 2531, 2822, 3642, 3087, 3907, 4198, 5018, 2120, 2940, 3231,
4051, 3496, 4316, 4607, 5427, 3757, 4577, 4868, 5688, 5133, 5953, 6244,
7064, 2120, 2940, 3231, 4051, 3496, 4316, 4607, 5427, 3757, 4577, 4868,
5688, 5133, 5953, 6244, 7064, 4166, 4986, 5277, 6097, 5542, 6362, 6653,
7473, 5803, 6623, 6914, 7734, 7179, 7999, 8290, 9110, 2120, 2940, 3231,
4051, 3496, 4316, 4607, 5427, 3757, 4577, 4868, 5688, 5133, 5953, 6244,
7064, 4166, 4986, 5277, 6097, 5542, 6362, 6653, 7473, 5803, 6623, 6914,
7734, 7179, 7999, 8290, 9110, 4166, 4986, 5277, 6097, 5542, 6362, 6653,
7473, 5803, 6623, 6914, 7734, 7179, 7999, 8290, 9110, 6212, 7032, 7323,
8143, 7588, 8408, 8699, 9519, 7849, 8669, 8960, 9780, 9225, 10045, 10336,
11156, 2120, 2940, 3231, 4051, 3496, 4316, 4607, 5427, 3757, 4577, 4868,
5688, 5133, 5953, 6244, 7064, 4166, 4986, 5277, 6097, 5542, 6362, 6653,
7473, 5803, 6623, 6914, 7734, 7179, 7999, 8290, 9110, 4166, 4986, 5277,
6097, 5542, 6362, 6653, 7473, 5803, 6623, 6914, 7734, 7179, 7999, 8290,
9110, 6212, 7032, 7323, 8143, 7588, 8408, 8699, 9519, 7849, 8669, 8960,
9780, 9225, 10045, 10336, 11156, 4166, 4986, 5277, 6097, 5542, 6362, 6653,
7473, 5803, 6623, 6914, 7734, 7179, 7999, 8290, 9110, 6212, 7032, 7323,
8143, 7588, 8408, 8699, 9519, 7849, 8669, 8960, 9780, 9225, 10045, 10336,
11156, 6212, 7032, 7323, 8143, 7588, 8408, 8699, 9519, 7849, 8669, 8960,
9780, 9225, 10045, 10336, 11156, 8258, 9078, 9369, 10189, 9634, 10454,
10745, 11565, 9895, 10715, 11006, 11826, 11271, 12091, 12382, 13202, 2120,
2940, 3231, 4051, 3496, 4316, 4607, 5427, 3757, 4577, 4868, 5688, 5133,
5953, 6244, 7064, 4166, 4986, 5277, 6097, 5542, 6362, 6653, 7473, 5803,
6623, 6914, 7734, 7179, 7999, 8290, 9110, 4166, 4986, 5277, 6097, 5542,
6362, 6653, 7473, 5803, 6623, 6914, 7734, 7179, 7999, 8290, 9110, 6212,
7032, 7323, 8143, 7588, 8408, 8699, 9519, 7849, 8669, 8960, 9780, 9225,
10045, 10336, 11156, 4166, 4986, 5277, 6097, 5542, 6362, 6653, 7473, 5803,
6623, 6914, 7734, 7179, 7999, 8290, 9110, 6212, 7032, 7323, 8143, 7588,
8408, 8699, 9519, 7849, 8669, 8960, 9780, 9225, 10045, 10336, 11156, 6212,
7032, 7323, 8143, 7588, 8408, 8699, 9519, 7849, 8669, 8960, 9780, 9225,
10045, 10336, 11156, 8258, 9078, 9369, 10189, 9634, 10454, 10745, 11565,
9895, 10715, 11006, 11826, 11271, 12091, 12382, 13202, 4166, 4986, 5277,
6097, 5542, 6362, 6653, 7473, 5803, 6623, 6914, 7734, 7179, 7999, 8290,
9110, 6212, 7032, 7323, 8143, 7588, 8408, 8699, 9519, 7849, 8669, 8960,
9780, 9225, 10045, 10336, 11156, 6212, 7032, 7323, 8143, 7588, 8408, 8699,
9519, 7849, 8669, 8960, 9780, 9225, 10045, 10336, 11156, 8258, 9078, 9369,
10189, 9634, 10454, 10745, 11565, 9895, 10715, 11006, 11826, 11271, 12091,
12382, 13202, 6212, 7032, 7323, 8143, 7588, 8408, 8699, 9519, 7849, 8669,
8960, 9780, 9225, 10045, 10336, 11156, 8258, 9078, 9369, 10189, 9634, 10454,
10745, 11565, 9895, 10715, 11006, 11826, 11271, 12091, 12382, 13202, 8258,
9078, 9369, 10189, 9634, 10454, 10745, 11565, 9895, 10715, 11006, 11826,
11271, 12091, 12382, 13202, 10304, 11124, 11415, 12235, 11680, 12500, 12791,
13611, 11941, 12761, 13052, 13872, 13317, 14137, 14428, 15248,
};
const int16_t vp10_cat6_high12_high_cost[2048] = {
76, 896, 1187, 2007, 1452, 2272, 2563,
3383, 1713, 2533, 2824, 3644, 3089, 3909, 4200, 5020, 2122, 2942, 3233,
4053, 3498, 4318, 4609, 5429, 3759, 4579, 4870, 5690, 5135, 5955, 6246,
7066, 2122, 2942, 3233, 4053, 3498, 4318, 4609, 5429, 3759, 4579, 4870,
5690, 5135, 5955, 6246, 7066, 4168, 4988, 5279, 6099, 5544, 6364, 6655,
7475, 5805, 6625, 6916, 7736, 7181, 8001, 8292, 9112, 2122, 2942, 3233,
4053, 3498, 4318, 4609, 5429, 3759, 4579, 4870, 5690, 5135, 5955, 6246,
7066, 4168, 4988, 5279, 6099, 5544, 6364, 6655, 7475, 5805, 6625, 6916,
7736, 7181, 8001, 8292, 9112, 4168, 4988, 5279, 6099, 5544, 6364, 6655,
7475, 5805, 6625, 6916, 7736, 7181, 8001, 8292, 9112, 6214, 7034, 7325,
8145, 7590, 8410, 8701, 9521, 7851, 8671, 8962, 9782, 9227, 10047, 10338,
11158, 2122, 2942, 3233, 4053, 3498, 4318, 4609, 5429, 3759, 4579, 4870,
5690, 5135, 5955, 6246, 7066, 4168, 4988, 5279, 6099, 5544, 6364, 6655,
7475, 5805, 6625, 6916, 7736, 7181, 8001, 8292, 9112, 4168, 4988, 5279,
6099, 5544, 6364, 6655, 7475, 5805, 6625, 6916, 7736, 7181, 8001, 8292,
9112, 6214, 7034, 7325, 8145, 7590, 8410, 8701, 9521, 7851, 8671, 8962,
9782, 9227, 10047, 10338, 11158, 4168, 4988, 5279, 6099, 5544, 6364, 6655,
7475, 5805, 6625, 6916, 7736, 7181, 8001, 8292, 9112, 6214, 7034, 7325,
8145, 7590, 8410, 8701, 9521, 7851, 8671, 8962, 9782, 9227, 10047, 10338,
11158, 6214, 7034, 7325, 8145, 7590, 8410, 8701, 9521, 7851, 8671, 8962,
9782, 9227, 10047, 10338, 11158, 8260, 9080, 9371, 10191, 9636, 10456,
10747, 11567, 9897, 10717, 11008, 11828, 11273, 12093, 12384, 13204, 2122,
2942, 3233, 4053, 3498, 4318, 4609, 5429, 3759, 4579, 4870, 5690, 5135,
5955, 6246, 7066, 4168, 4988, 5279, 6099, 5544, 6364, 6655, 7475, 5805,
6625, 6916, 7736, 7181, 8001, 8292, 9112, 4168, 4988, 5279, 6099, 5544,
6364, 6655, 7475, 5805, 6625, 6916, 7736, 7181, 8001, 8292, 9112, 6214,
7034, 7325, 8145, 7590, 8410, 8701, 9521, 7851, 8671, 8962, 9782, 9227,
10047, 10338, 11158, 4168, 4988, 5279, 6099, 5544, 6364, 6655, 7475, 5805,
6625, 6916, 7736, 7181, 8001, 8292, 9112, 6214, 7034, 7325, 8145, 7590,
8410, 8701, 9521, 7851, 8671, 8962, 9782, 9227, 10047, 10338, 11158, 6214,
7034, 7325, 8145, 7590, 8410, 8701, 9521, 7851, 8671, 8962, 9782, 9227,
10047, 10338, 11158, 8260, 9080, 9371, 10191, 9636, 10456, 10747, 11567,
9897, 10717, 11008, 11828, 11273, 12093, 12384, 13204, 4168, 4988, 5279,
6099, 5544, 6364, 6655, 7475, 5805, 6625, 6916, 7736, 7181, 8001, 8292,
9112, 6214, 7034, 7325, 8145, 7590, 8410, 8701, 9521, 7851, 8671, 8962,
9782, 9227, 10047, 10338, 11158, 6214, 7034, 7325, 8145, 7590, 8410, 8701,
9521, 7851, 8671, 8962, 9782, 9227, 10047, 10338, 11158, 8260, 9080, 9371,
10191, 9636, 10456, 10747, 11567, 9897, 10717, 11008, 11828, 11273, 12093,
12384, 13204, 6214, 7034, 7325, 8145, 7590, 8410, 8701, 9521, 7851, 8671,
8962, 9782, 9227, 10047, 10338, 11158, 8260, 9080, 9371, 10191, 9636, 10456,
10747, 11567, 9897, 10717, 11008, 11828, 11273, 12093, 12384, 13204, 8260,
9080, 9371, 10191, 9636, 10456, 10747, 11567, 9897, 10717, 11008, 11828,
11273, 12093, 12384, 13204, 10306, 11126, 11417, 12237, 11682, 12502, 12793,
13613, 11943, 12763, 13054, 13874, 13319, 14139, 14430, 15250, 2122, 2942,
3233, 4053, 3498, 4318, 4609, 5429, 3759, 4579, 4870, 5690, 5135, 5955,
6246, 7066, 4168, 4988, 5279, 6099, 5544, 6364, 6655, 7475, 5805, 6625,
6916, 7736, 7181, 8001, 8292, 9112, 4168, 4988, 5279, 6099, 5544, 6364,
6655, 7475, 5805, 6625, 6916, 7736, 7181, 8001, 8292, 9112, 6214, 7034,
7325, 8145, 7590, 8410, 8701, 9521, 7851, 8671, 8962, 9782, 9227, 10047,
10338, 11158, 4168, 4988, 5279, 6099, 5544, 6364, 6655, 7475, 5805, 6625,
6916, 7736, 7181, 8001, 8292, 9112, 6214, 7034, 7325, 8145, 7590, 8410,
8701, 9521, 7851, 8671, 8962, 9782, 9227, 10047, 10338, 11158, 6214, 7034,
7325, 8145, 7590, 8410, 8701, 9521, 7851, 8671, 8962, 9782, 9227, 10047,
10338, 11158, 8260, 9080, 9371, 10191, 9636, 10456, 10747, 11567, 9897,
10717, 11008, 11828, 11273, 12093, 12384, 13204, 4168, 4988, 5279, 6099,
5544, 6364, 6655, 7475, 5805, 6625, 6916, 7736, 7181, 8001, 8292, 9112,
6214, 7034, 7325, 8145, 7590, 8410, 8701, 9521, 7851, 8671, 8962, 9782,
9227, 10047, 10338, 11158, 6214, 7034, 7325, 8145, 7590, 8410, 8701, 9521,
7851, 8671, 8962, 9782, 9227, 10047, 10338, 11158, 8260, 9080, 9371, 10191,
9636, 10456, 10747, 11567, 9897, 10717, 11008, 11828, 11273, 12093, 12384,
13204, 6214, 7034, 7325, 8145, 7590, 8410, 8701, 9521, 7851, 8671, 8962,
9782, 9227, 10047, 10338, 11158, 8260, 9080, 9371, 10191, 9636, 10456,
10747, 11567, 9897, 10717, 11008, 11828, 11273, 12093, 12384, 13204, 8260,
9080, 9371, 10191, 9636, 10456, 10747, 11567, 9897, 10717, 11008, 11828,
11273, 12093, 12384, 13204, 10306, 11126, 11417, 12237, 11682, 12502, 12793,
13613, 11943, 12763, 13054, 13874, 13319, 14139, 14430, 15250, 4168, 4988,
5279, 6099, 5544, 6364, 6655, 7475, 5805, 6625, 6916, 7736, 7181, 8001,
8292, 9112, 6214, 7034, 7325, 8145, 7590, 8410, 8701, 9521, 7851, 8671,
8962, 9782, 9227, 10047, 10338, 11158, 6214, 7034, 7325, 8145, 7590, 8410,
8701, 9521, 7851, 8671, 8962, 9782, 9227, 10047, 10338, 11158, 8260, 9080,
9371, 10191, 9636, 10456, 10747, 11567, 9897, 10717, 11008, 11828, 11273,
12093, 12384, 13204, 6214, 7034, 7325, 8145, 7590, 8410, 8701, 9521, 7851,
8671, 8962, 9782, 9227, 10047, 10338, 11158, 8260, 9080, 9371, 10191, 9636,
10456, 10747, 11567, 9897, 10717, 11008, 11828, 11273, 12093, 12384, 13204,
8260, 9080, 9371, 10191, 9636, 10456, 10747, 11567, 9897, 10717, 11008,
11828, 11273, 12093, 12384, 13204, 10306, 11126, 11417, 12237, 11682, 12502,
12793, 13613, 11943, 12763, 13054, 13874, 13319, 14139, 14430, 15250, 6214,
7034, 7325, 8145, 7590, 8410, 8701, 9521, 7851, 8671, 8962, 9782, 9227,
10047, 10338, 11158, 8260, 9080, 9371, 10191, 9636, 10456, 10747, 11567,
9897, 10717, 11008, 11828, 11273, 12093, 12384, 13204, 8260, 9080, 9371,
10191, 9636, 10456, 10747, 11567, 9897, 10717, 11008, 11828, 11273, 12093,
12384, 13204, 10306, 11126, 11417, 12237, 11682, 12502, 12793, 13613, 11943,
12763, 13054, 13874, 13319, 14139, 14430, 15250, 8260, 9080, 9371, 10191,
9636, 10456, 10747, 11567, 9897, 10717, 11008, 11828, 11273, 12093, 12384,
13204, 10306, 11126, 11417, 12237, 11682, 12502, 12793, 13613, 11943, 12763,
13054, 13874, 13319, 14139, 14430, 15250, 10306, 11126, 11417, 12237, 11682,
12502, 12793, 13613, 11943, 12763, 13054, 13874, 13319, 14139, 14430, 15250,
12352, 13172, 13463, 14283, 13728, 14548, 14839, 15659, 13989, 14809, 15100,
15920, 15365, 16185, 16476, 17296, 2122, 2942, 3233, 4053, 3498, 4318, 4609,
5429, 3759, 4579, 4870, 5690, 5135, 5955, 6246, 7066, 4168, 4988, 5279,
6099, 5544, 6364, 6655, 7475, 5805, 6625, 6916, 7736, 7181, 8001, 8292,
9112, 4168, 4988, 5279, 6099, 5544, 6364, 6655, 7475, 5805, 6625, 6916,