From 0ccefe21af3842e0fd694d69f24d92286b1914e6 Mon Sep 17 00:00:00 2001 From: Thomas Davies Date: Mon, 21 Nov 2016 10:54:29 +0000 Subject: [PATCH] EC_MULTISYMBOL: merge ZERO_TOKEN into coding scheme. Zero, one, and two or more coded as one symbol (head). Remaining tokens coded as a tail symbol. The pareto CDF distribution is adjusted to cover tokens from two onwards. Change-Id: I98b33fab6b9f52690f6ad618ac55e725a97be056 --- av1/common/entropy.c | 548 +++++++++++++++--------------- av1/common/entropy.h | 3 + av1/common/entropymode.h | 3 +- av1/decoder/decodeframe.c | 2 +- av1/decoder/detokenize.c | 57 +++- av1/encoder/bitstream.c | 11 +- av1/encoder/tokenize.c | 22 +- av1/encoder/tokenize.h | 3 +- tools/gen_constrained_tokenset.py | 6 +- 9 files changed, 358 insertions(+), 297 deletions(-) diff --git a/av1/common/entropy.c b/av1/common/entropy.c index 20667a920..2fe7ace32 100644 --- a/av1/common/entropy.c +++ b/av1/common/entropy.c @@ -281,6 +281,7 @@ const uint8_t av1_pt_energy_class[ENTROPY_TOKENS] = { 0, 1, 2, 3, 3, 4, // by averaging : // av1_pareto8_full[l][node] = (av1_pareto8_full[l-1][node] + // av1_pareto8_full[l+1][node] ) >> 1; +// Values for tokens ONE_TOKEN through CATEGORY6_TOKEN included here. const aom_prob av1_pareto8_full[COEFF_PROB_MODELS][MODEL_NODES] = { { 3, 86, 128, 6, 86, 23, 88, 29 }, { 6, 86, 128, 11, 87, 42, 91, 52 }, @@ -551,266 +552,269 @@ const aom_prob av1_pareto8_full[COEFF_PROB_MODELS][MODEL_NODES] = { // The full source code of the generating program is available in: // tools/gen_constrained_tokenset.py // -// Values for tokens ONE_TOKEN through CATEGORY6_TOKEN included here. -// ZERO_TOKEN and EOB_TOKEN are coded as flags outside this coder. -const aom_cdf_prob - av1_pareto8_token_probs[COEFF_PROB_MODELS][ENTROPY_TOKENS - 2] = { - { 128, 127, 127, 126, 251, 495, 965, 1832, 3305, 25412 }, - { 256, 254, 252, 249, 492, 959, 1820, 3283, 5365, 19838 }, - { 384, 379, 374, 369, 724, 1392, 2574, 4417, 6568, 15587 }, - { 512, 503, 494, 486, 946, 1795, 3238, 5289, 7184, 12321 }, - { 640, 626, 612, 599, 1159, 2172, 3818, 5946, 7401, 9795 }, - { 768, 748, 728, 709, 1363, 2522, 4324, 6424, 7352, 7830 }, - { 896, 869, 842, 816, 1559, 2847, 4762, 6755, 7131, 6291 }, - { 1024, 988, 954, 921, 1747, 3148, 5139, 6966, 6803, 5078 }, - { 1152, 1107, 1063, 1022, 1926, 3427, 5460, 7080, 6412, 4119 }, - { 1280, 1224, 1171, 1120, 2098, 3685, 5730, 7113, 5991, 3356 }, - { 1408, 1340, 1276, 1216, 2261, 3923, 5955, 7083, 5560, 2746 }, - { 1536, 1455, 1380, 1308, 2418, 4142, 6140, 7001, 5133, 2255 }, - { 1664, 1569, 1481, 1398, 2567, 4342, 6287, 6879, 4721, 1860 }, - { 1792, 1683, 1580, 1485, 2709, 4525, 6401, 6725, 4329, 1539 }, - { 1920, 1794, 1678, 1570, 2845, 4692, 6486, 6546, 3959, 1278 }, - { 2048, 1905, 1773, 1651, 2974, 4844, 6543, 6350, 3615, 1065 }, - { 2176, 2015, 1867, 1731, 3096, 4980, 6576, 6140, 3296, 891 }, - { 2304, 2123, 1958, 1807, 3212, 5104, 6589, 5922, 3002, 747 }, - { 2432, 2231, 2048, 1882, 3322, 5214, 6581, 5698, 2732, 628 }, - { 2560, 2337, 2136, 1953, 3427, 5311, 6557, 5472, 2485, 530 }, - { 2688, 2442, 2222, 2023, 3525, 5397, 6518, 5246, 2259, 448 }, - { 2816, 2547, 2306, 2090, 3618, 5472, 6465, 5021, 2053, 380 }, - { 2944, 2650, 2388, 2154, 3706, 5537, 6401, 4799, 1866, 323 }, - { 3072, 2752, 2468, 2217, 3788, 5591, 6327, 4581, 1696, 276 }, - { 3200, 2853, 2547, 2277, 3866, 5637, 6243, 4369, 1541, 235 }, - { 3328, 2952, 2624, 2335, 3938, 5673, 6152, 4163, 1401, 202 }, - { 3456, 3051, 2699, 2391, 4006, 5702, 6054, 3962, 1274, 173 }, - { 3584, 3149, 2772, 2444, 4070, 5723, 5950, 3769, 1158, 149 }, - { 3712, 3246, 2843, 2496, 4128, 5736, 5842, 3583, 1054, 128 }, - { 3840, 3341, 2913, 2545, 4183, 5743, 5729, 3404, 959, 111 }, - { 3968, 3436, 2981, 2593, 4233, 5743, 5614, 3232, 872, 96 }, - { 4096, 3529, 3048, 2638, 4280, 5737, 5496, 3067, 794, 83 }, - { 4224, 3621, 3113, 2682, 4322, 5726, 5375, 2909, 724, 72 }, - { 4352, 3712, 3176, 2724, 4361, 5709, 5253, 2759, 659, 63 }, - { 4480, 3803, 3237, 2764, 4396, 5687, 5130, 2615, 601, 55 }, - { 4608, 3892, 3297, 2801, 4428, 5661, 5007, 2478, 548, 48 }, - { 4736, 3980, 3355, 2838, 4456, 5631, 4883, 2347, 500, 42 }, - { 4864, 4067, 3412, 2872, 4481, 5596, 4760, 2223, 456, 37 }, - { 4992, 4152, 3467, 2905, 4503, 5558, 4637, 2105, 417, 32 }, - { 5120, 4237, 3521, 2936, 4521, 5516, 4515, 1993, 381, 28 }, - { 5248, 4321, 3573, 2966, 4537, 5471, 4393, 1886, 348, 25 }, - { 5376, 4404, 3623, 2993, 4550, 5424, 4273, 1785, 318, 22 }, - { 5504, 4486, 3672, 3020, 4560, 5373, 4155, 1688, 291, 19 }, - { 5632, 4566, 3720, 3044, 4568, 5321, 4037, 1597, 266, 17 }, - { 5760, 4646, 3766, 3067, 4572, 5265, 3922, 1511, 244, 15 }, - { 5888, 4724, 3811, 3089, 4575, 5208, 3808, 1429, 223, 13 }, - { 6016, 4802, 3854, 3109, 4575, 5148, 3696, 1352, 204, 12 }, - { 6144, 4878, 3895, 3128, 4573, 5088, 3587, 1278, 187, 10 }, - { 6272, 4953, 3936, 3145, 4568, 5025, 3479, 1209, 172, 9 }, - { 6400, 5028, 3975, 3161, 4561, 4961, 3373, 1143, 158, 8 }, - { 6528, 5101, 4012, 3175, 4553, 4896, 3270, 1081, 145, 7 }, - { 6656, 5173, 4048, 3189, 4542, 4830, 3168, 1022, 133, 7 }, - { 6784, 5244, 4083, 3201, 4530, 4763, 3069, 966, 122, 6 }, - { 6912, 5314, 4117, 3212, 4516, 4694, 2973, 913, 112, 5 }, - { 7040, 5383, 4149, 3221, 4500, 4626, 2878, 863, 103, 5 }, - { 7168, 5452, 4180, 3229, 4482, 4556, 2786, 816, 95, 4 }, - { 7296, 5519, 4210, 3236, 4463, 4486, 2696, 771, 87, 4 }, - { 7424, 5585, 4238, 3242, 4442, 4416, 2609, 729, 80, 3 }, - { 7552, 5650, 4265, 3247, 4420, 4345, 2523, 689, 74, 3 }, - { 7680, 5714, 4291, 3251, 4396, 4274, 2440, 651, 68, 3 }, - { 7808, 5777, 4315, 3254, 4371, 4203, 2359, 616, 63, 2 }, - { 7936, 5838, 4339, 3255, 4345, 4132, 2281, 582, 58, 2 }, - { 8064, 5899, 4361, 3256, 4318, 4061, 2204, 550, 53, 2 }, - { 8192, 5959, 4382, 3255, 4289, 3990, 2130, 520, 49, 2 }, - { 8320, 6018, 4402, 3254, 4259, 3919, 2057, 492, 45, 2 }, - { 8448, 6075, 4421, 3252, 4229, 3848, 1987, 465, 42, 1 }, - { 8576, 6133, 4438, 3248, 4197, 3778, 1919, 439, 39, 1 }, - { 8704, 6188, 4455, 3244, 4164, 3708, 1853, 415, 36, 1 }, - { 8832, 6243, 4470, 3239, 4131, 3638, 1789, 392, 33, 1 }, - { 8960, 6297, 4484, 3233, 4096, 3569, 1727, 371, 30, 1 }, - { 9088, 6349, 4497, 3226, 4061, 3500, 1667, 351, 28, 1 }, - { 9216, 6401, 4509, 3219, 4025, 3432, 1608, 331, 26, 1 }, - { 9344, 6452, 4520, 3210, 3989, 3364, 1551, 313, 24, 1 }, - { 9472, 6501, 4530, 3201, 3952, 3297, 1496, 296, 22, 1 }, - { 9600, 6550, 4539, 3191, 3914, 3230, 1443, 280, 20, 1 }, - { 9728, 6597, 4547, 3180, 3875, 3164, 1392, 265, 19, 1 }, - { 9856, 6644, 4554, 3169, 3836, 3098, 1342, 250, 18, 1 }, - { 9984, 6690, 4560, 3157, 3796, 3034, 1293, 237, 16, 1 }, - { 10112, 6734, 4565, 3144, 3756, 2970, 1247, 224, 15, 1 }, - { 10240, 6778, 4568, 3131, 3716, 2907, 1202, 211, 14, 1 }, - { 10368, 6821, 4571, 3117, 3675, 2844, 1158, 200, 13, 1 }, - { 10496, 6862, 4573, 3102, 3634, 2783, 1116, 189, 12, 1 }, - { 10624, 6903, 4574, 3087, 3592, 2722, 1075, 179, 11, 1 }, - { 10752, 6942, 4575, 3071, 3551, 2662, 1035, 169, 10, 1 }, - { 10880, 6981, 4574, 3054, 3508, 2603, 997, 160, 10, 1 }, - { 11008, 7019, 4572, 3038, 3466, 2544, 960, 151, 9, 1 }, - { 11136, 7055, 4570, 3020, 3424, 2487, 924, 143, 8, 1 }, - { 11264, 7091, 4566, 3002, 3381, 2430, 890, 135, 8, 1 }, - { 11392, 7126, 4563, 2984, 3338, 2374, 856, 127, 7, 1 }, - { 11520, 7159, 4557, 2965, 3295, 2319, 824, 121, 7, 1 }, - { 11648, 7193, 4552, 2945, 3252, 2264, 793, 114, 6, 1 }, - { 11776, 7224, 4545, 2925, 3209, 2211, 763, 108, 6, 1 }, - { 11904, 7255, 4538, 2905, 3165, 2159, 734, 102, 5, 1 }, - { 12032, 7285, 4530, 2884, 3122, 2107, 706, 96, 5, 1 }, - { 12160, 7314, 4520, 2863, 3079, 2056, 679, 91, 5, 1 }, - { 12288, 7341, 4511, 2842, 3036, 2006, 653, 86, 4, 1 }, - { 12416, 7368, 4500, 2820, 2993, 1957, 628, 81, 4, 1 }, - { 12544, 7394, 4489, 2797, 2949, 1909, 604, 77, 4, 1 }, - { 12672, 7419, 4477, 2775, 2906, 1861, 581, 73, 3, 1 }, - { 12800, 7443, 4464, 2752, 2863, 1815, 558, 69, 3, 1 }, - { 12928, 7466, 4451, 2729, 2820, 1769, 536, 65, 3, 1 }, - { 13056, 7488, 4437, 2705, 2777, 1724, 516, 61, 3, 1 }, - { 13184, 7509, 4422, 2682, 2734, 1680, 495, 58, 3, 1 }, - { 13312, 7529, 4406, 2658, 2692, 1637, 476, 55, 2, 1 }, - { 13440, 7548, 4390, 2633, 2650, 1595, 457, 52, 2, 1 }, - { 13568, 7567, 4373, 2609, 2607, 1553, 439, 49, 2, 1 }, - { 13696, 7583, 4356, 2584, 2565, 1513, 422, 46, 2, 1 }, - { 13824, 7600, 4337, 2559, 2523, 1473, 405, 44, 2, 1 }, - { 13952, 7615, 4319, 2533, 2482, 1434, 389, 41, 2, 1 }, - { 14080, 7629, 4300, 2508, 2441, 1395, 373, 39, 2, 1 }, - { 14208, 7643, 4280, 2482, 2400, 1358, 358, 37, 1, 1 }, - { 14336, 7655, 4259, 2457, 2359, 1321, 344, 35, 1, 1 }, - { 14464, 7667, 4238, 2431, 2318, 1285, 330, 33, 1, 1 }, - { 14592, 7677, 4217, 2405, 2278, 1250, 316, 31, 1, 1 }, - { 14720, 7687, 4195, 2378, 2238, 1215, 304, 29, 1, 1 }, - { 14848, 7696, 4172, 2352, 2198, 1181, 291, 28, 1, 1 }, - { 14976, 7703, 4149, 2326, 2159, 1148, 279, 26, 1, 1 }, - { 15104, 7710, 4125, 2299, 2119, 1116, 268, 25, 1, 1 }, - { 15232, 7715, 4101, 2272, 2081, 1085, 257, 23, 1, 1 }, - { 15360, 7721, 4076, 2245, 2042, 1054, 246, 22, 1, 1 }, - { 15488, 7724, 4051, 2219, 2004, 1023, 236, 21, 1, 1 }, - { 15616, 7727, 4025, 2192, 1966, 994, 226, 20, 1, 1 }, - { 15744, 7729, 3999, 2164, 1929, 965, 217, 19, 1, 1 }, - { 15872, 7731, 3972, 2137, 1892, 937, 207, 18, 1, 1 }, - { 16000, 7731, 3945, 2110, 1855, 909, 199, 17, 1, 1 }, - { 16128, 7730, 3918, 2083, 1819, 882, 190, 16, 1, 1 }, - { 16256, 7728, 3890, 2056, 1783, 856, 182, 15, 1, 1 }, - { 16384, 7725, 3862, 2029, 1747, 831, 174, 14, 1, 1 }, - { 16512, 7721, 3833, 2002, 1712, 806, 167, 13, 1, 1 }, - { 16640, 7717, 3804, 1975, 1677, 781, 160, 12, 1, 1 }, - { 16768, 7712, 3775, 1947, 1642, 757, 153, 12, 1, 1 }, - { 16896, 7706, 3745, 1920, 1608, 734, 146, 11, 1, 1 }, - { 17024, 7699, 3714, 1893, 1575, 711, 140, 10, 1, 1 }, - { 17152, 7690, 3684, 1866, 1541, 689, 134, 10, 1, 1 }, - { 17280, 7681, 3653, 1839, 1508, 668, 128, 9, 1, 1 }, - { 17408, 7671, 3621, 1812, 1476, 647, 122, 9, 1, 1 }, - { 17536, 7660, 3590, 1785, 1444, 626, 117, 8, 1, 1 }, - { 17664, 7648, 3558, 1758, 1412, 606, 112, 8, 1, 1 }, - { 17792, 7635, 3526, 1731, 1381, 587, 107, 7, 1, 1 }, - { 17920, 7622, 3493, 1704, 1350, 568, 102, 7, 1, 1 }, - { 18048, 7607, 3461, 1678, 1319, 549, 98, 6, 1, 1 }, - { 18176, 7592, 3428, 1651, 1289, 531, 93, 6, 1, 1 }, - { 18304, 7575, 3394, 1625, 1259, 514, 89, 6, 1, 1 }, - { 18432, 7558, 3361, 1598, 1230, 497, 85, 5, 1, 1 }, - { 18560, 7540, 3327, 1572, 1201, 480, 81, 5, 1, 1 }, - { 18688, 7520, 3293, 1546, 1173, 464, 77, 5, 1, 1 }, - { 18816, 7500, 3258, 1520, 1145, 448, 74, 5, 1, 1 }, - { 18944, 7480, 3224, 1494, 1117, 433, 70, 4, 1, 1 }, - { 19072, 7458, 3189, 1468, 1090, 418, 67, 4, 1, 1 }, - { 19200, 7435, 3154, 1442, 1063, 404, 64, 4, 1, 1 }, - { 19328, 7410, 3119, 1417, 1037, 390, 61, 4, 1, 1 }, - { 19456, 7386, 3084, 1392, 1011, 376, 58, 3, 1, 1 }, - { 19584, 7361, 3048, 1366, 986, 363, 55, 3, 1, 1 }, - { 19712, 7335, 3012, 1341, 960, 350, 53, 3, 1, 1 }, - { 19840, 7307, 2977, 1316, 936, 337, 50, 3, 1, 1 }, - { 19968, 7279, 2941, 1291, 911, 325, 48, 3, 1, 1 }, - { 20096, 7251, 2905, 1267, 887, 313, 45, 2, 1, 1 }, - { 20224, 7220, 2868, 1243, 864, 302, 43, 2, 1, 1 }, - { 20352, 7189, 2832, 1218, 841, 291, 41, 2, 1, 1 }, - { 20480, 7158, 2795, 1194, 818, 280, 39, 2, 1, 1 }, - { 20608, 7124, 2759, 1170, 796, 270, 37, 2, 1, 1 }, - { 20736, 7091, 2722, 1147, 774, 259, 35, 2, 1, 1 }, - { 20864, 7056, 2685, 1123, 752, 250, 34, 2, 1, 1 }, - { 20992, 7021, 2648, 1100, 731, 240, 32, 2, 1, 1 }, - { 21120, 6985, 2612, 1077, 710, 231, 30, 1, 1, 1 }, - { 21248, 6948, 2574, 1054, 690, 222, 29, 1, 1, 1 }, - { 21376, 6911, 2537, 1031, 670, 213, 27, 1, 1, 1 }, - { 21504, 6872, 2500, 1008, 650, 205, 26, 1, 1, 1 }, - { 21632, 6831, 2463, 986, 631, 197, 25, 1, 1, 1 }, - { 21760, 6791, 2426, 964, 612, 189, 23, 1, 1, 1 }, - { 21888, 6749, 2389, 942, 594, 181, 22, 1, 1, 1 }, - { 22016, 6707, 2351, 921, 575, 174, 21, 1, 1, 1 }, - { 22144, 6663, 2314, 899, 558, 167, 20, 1, 1, 1 }, - { 22272, 6619, 2277, 878, 540, 160, 19, 1, 1, 1 }, - { 22400, 6574, 2240, 857, 523, 153, 18, 1, 1, 1 }, - { 22528, 6529, 2202, 836, 507, 146, 17, 1, 1, 1 }, - { 22656, 6482, 2165, 816, 490, 140, 16, 1, 1, 1 }, - { 22784, 6435, 2128, 795, 474, 134, 15, 1, 1, 1 }, - { 22912, 6386, 2091, 775, 459, 128, 14, 1, 1, 1 }, - { 23040, 6336, 2054, 756, 443, 123, 13, 1, 1, 1 }, - { 23168, 6286, 2017, 736, 428, 117, 13, 1, 1, 1 }, - { 23296, 6234, 1980, 717, 414, 112, 12, 1, 1, 1 }, - { 23424, 6183, 1943, 698, 399, 107, 11, 1, 1, 1 }, - { 23552, 6130, 1906, 679, 385, 102, 11, 1, 1, 1 }, - { 23680, 6077, 1869, 660, 372, 97, 10, 1, 1, 1 }, - { 23808, 6022, 1833, 642, 358, 93, 9, 1, 1, 1 }, - { 23936, 5966, 1796, 624, 345, 89, 9, 1, 1, 1 }, - { 24064, 5910, 1760, 606, 333, 84, 8, 1, 1, 1 }, - { 24192, 5853, 1724, 588, 320, 80, 8, 1, 1, 1 }, - { 24320, 5796, 1687, 571, 308, 76, 7, 1, 1, 1 }, - { 24448, 5735, 1651, 554, 297, 73, 7, 1, 1, 1 }, - { 24576, 5677, 1615, 537, 285, 69, 6, 1, 1, 1 }, - { 24704, 5615, 1579, 521, 274, 66, 6, 1, 1, 1 }, - { 24832, 5554, 1544, 504, 263, 62, 6, 1, 1, 1 }, - { 24960, 5492, 1508, 488, 253, 59, 5, 1, 1, 1 }, - { 25088, 5428, 1473, 473, 242, 56, 5, 1, 1, 1 }, - { 25216, 5364, 1438, 457, 232, 53, 5, 1, 1, 1 }, - { 25344, 5300, 1403, 442, 222, 50, 4, 1, 1, 1 }, - { 25472, 5233, 1368, 427, 213, 48, 4, 1, 1, 1 }, - { 25600, 5166, 1334, 412, 204, 45, 4, 1, 1, 1 }, - { 25728, 5098, 1299, 398, 195, 43, 4, 1, 1, 1 }, - { 25856, 5030, 1266, 384, 186, 40, 3, 1, 1, 1 }, - { 25984, 4960, 1232, 370, 178, 38, 3, 1, 1, 1 }, - { 26112, 4890, 1198, 356, 170, 36, 3, 1, 1, 1 }, - { 26240, 4819, 1164, 343, 162, 34, 3, 1, 1, 1 }, - { 26368, 4748, 1132, 329, 154, 32, 2, 1, 1, 1 }, - { 26496, 4675, 1098, 317, 147, 30, 2, 1, 1, 1 }, - { 26624, 4602, 1066, 304, 139, 28, 2, 1, 1, 1 }, - { 26752, 4527, 1034, 292, 132, 26, 2, 1, 1, 1 }, - { 26880, 4451, 1001, 280, 126, 25, 2, 1, 1, 1 }, - { 27008, 4375, 970, 268, 119, 23, 2, 1, 1, 1 }, - { 27136, 4299, 938, 256, 113, 21, 2, 1, 1, 1 }, - { 27264, 4221, 907, 245, 107, 20, 1, 1, 1, 1 }, - { 27392, 4142, 876, 234, 101, 19, 1, 1, 1, 1 }, - { 27520, 4063, 846, 223, 95, 17, 1, 1, 1, 1 }, - { 27648, 3982, 815, 213, 90, 16, 1, 1, 1, 1 }, - { 27776, 3900, 786, 202, 85, 15, 1, 1, 1, 1 }, - { 27904, 3818, 756, 192, 80, 14, 1, 1, 1, 1 }, - { 28032, 3734, 727, 183, 75, 13, 1, 1, 1, 1 }, - { 28160, 3651, 698, 173, 70, 12, 1, 1, 1, 1 }, - { 28288, 3566, 669, 164, 66, 11, 1, 1, 1, 1 }, - { 28416, 3481, 641, 155, 61, 10, 1, 1, 1, 1 }, - { 28544, 3393, 614, 147, 57, 9, 1, 1, 1, 1 }, - { 28672, 3306, 586, 138, 53, 9, 1, 1, 1, 1 }, - { 28800, 3217, 559, 130, 50, 8, 1, 1, 1, 1 }, - { 28928, 3128, 533, 122, 46, 7, 1, 1, 1, 1 }, - { 29056, 3037, 507, 114, 43, 7, 1, 1, 1, 1 }, - { 29184, 2947, 481, 107, 39, 6, 1, 1, 1, 1 }, - { 29312, 2855, 456, 100, 36, 5, 1, 1, 1, 1 }, - { 29440, 2762, 431, 93, 33, 5, 1, 1, 1, 1 }, - { 29568, 2668, 407, 86, 31, 4, 1, 1, 1, 1 }, - { 29696, 2573, 383, 80, 28, 4, 1, 1, 1, 1 }, - { 29824, 2478, 359, 74, 25, 4, 1, 1, 1, 1 }, - { 29952, 2381, 337, 68, 23, 3, 1, 1, 1, 1 }, - { 30080, 2284, 314, 62, 21, 3, 1, 1, 1, 1 }, - { 30208, 2185, 293, 57, 19, 2, 1, 1, 1, 1 }, - { 30336, 2086, 271, 52, 17, 2, 1, 1, 1, 1 }, - { 30464, 1986, 250, 47, 15, 2, 1, 1, 1, 1 }, - { 30592, 1885, 230, 42, 13, 2, 1, 1, 1, 1 }, - { 30720, 1782, 211, 38, 12, 1, 1, 1, 1, 1 }, - { 30848, 1679, 192, 34, 10, 1, 1, 1, 1, 1 }, - { 30976, 1575, 173, 30, 9, 1, 1, 1, 1, 1 }, - { 31104, 1469, 156, 26, 8, 1, 1, 1, 1, 1 }, - { 31232, 1364, 138, 23, 6, 1, 1, 1, 1, 1 }, - { 31360, 1257, 122, 19, 5, 1, 1, 1, 1, 1 }, - { 31488, 1149, 106, 16, 4, 1, 1, 1, 1, 1 }, - { 31616, 1038, 91, 14, 4, 1, 1, 1, 1, 1 }, - { 31744, 928, 77, 11, 3, 1, 1, 1, 1, 1 }, - { 31872, 816, 64, 9, 2, 1, 1, 1, 1, 1 }, - { 32000, 703, 51, 7, 2, 1, 1, 1, 1, 1 }, - { 32128, 589, 40, 5, 1, 1, 1, 1, 1, 1 }, - { 32256, 473, 29, 4, 1, 1, 1, 1, 1, 1 }, - { 32384, 357, 19, 2, 1, 1, 1, 1, 1, 1 }, - { 32512, 238, 11, 1, 1, 1, 1, 1, 1, 1 }, - { 32640, 117, 4, 1, 1, 1, 1, 1, 1, 1 }, - }; +// Values for tokens TWO_TOKEN through CATEGORY6_TOKEN included +// in the table here : the ONE_TOKEN probability is +// removed and the probabilities rescaled. +// +// ZERO_TOKEN and ONE_TOKEN are coded as one CDF, +// and EOB_TOKEN is coded as flags outside this coder. +const aom_cdf_prob av1_pareto8_tail_probs[COEFF_PROB_MODELS][TAIL_NODES] = { + { 128, 127, 127, 252, 497, 969, 1839, 3318, 25511 }, + { 256, 254, 251, 496, 966, 1834, 3308, 5408, 19995 }, + { 383, 378, 373, 732, 1408, 2605, 4470, 6646, 15773 }, + { 511, 502, 493, 961, 1824, 3289, 5373, 7298, 12517 }, + { 638, 625, 611, 1182, 2215, 3894, 6064, 7548, 9991 }, + { 766, 746, 726, 1396, 2582, 4428, 6578, 7529, 8017 }, + { 893, 866, 839, 1603, 2927, 4896, 6945, 7332, 6467 }, + { 1020, 984, 950, 1803, 3250, 5305, 7191, 7022, 5243 }, + { 1147, 1102, 1059, 1996, 3552, 5659, 7338, 6646, 4269 }, + { 1274, 1218, 1166, 2183, 3835, 5963, 7403, 6234, 3492 }, + { 1400, 1334, 1270, 2363, 4099, 6223, 7401, 5809, 2869 }, + { 1527, 1447, 1372, 2537, 4345, 6442, 7346, 5386, 2366 }, + { 1654, 1560, 1473, 2704, 4574, 6624, 7247, 4973, 1959 }, + { 1780, 1672, 1571, 2866, 4787, 6771, 7114, 4579, 1628 }, + { 1906, 1782, 1667, 3022, 4984, 6889, 6954, 4206, 1358 }, + { 2032, 1891, 1762, 3172, 5167, 6979, 6773, 3856, 1136 }, + { 2158, 2000, 1854, 3316, 5335, 7044, 6577, 3530, 954 }, + { 2284, 2106, 1944, 3455, 5490, 7087, 6370, 3229, 803 }, + { 2410, 2212, 2032, 3588, 5632, 7109, 6155, 2951, 679 }, + { 2535, 2317, 2119, 3717, 5761, 7113, 5936, 2695, 575 }, + { 2661, 2420, 2203, 3840, 5880, 7101, 5714, 2461, 488 }, + { 2786, 2522, 2286, 3958, 5987, 7074, 5493, 2246, 416 }, + { 2911, 2624, 2367, 4072, 6083, 7033, 5273, 2050, 355 }, + { 3037, 2724, 2446, 4180, 6170, 6981, 5055, 1871, 304 }, + { 3162, 2822, 2523, 4284, 6247, 6919, 4842, 1708, 261 }, + { 3286, 2920, 2599, 4384, 6315, 6848, 4633, 1559, 224 }, + { 3411, 3017, 2672, 4478, 6374, 6768, 4430, 1424, 194 }, + { 3536, 3112, 2745, 4569, 6426, 6681, 4232, 1300, 167 }, + { 3660, 3207, 2815, 4656, 6469, 6588, 4040, 1188, 145 }, + { 3785, 3300, 2883, 4738, 6505, 6490, 3855, 1086, 126 }, + { 3909, 3392, 2950, 4817, 6534, 6387, 3677, 993, 109 }, + { 4033, 3483, 3015, 4891, 6557, 6281, 3505, 908, 95 }, + { 4157, 3573, 3079, 4962, 6573, 6170, 3340, 831, 83 }, + { 4281, 3662, 3141, 5029, 6584, 6058, 3181, 760, 72 }, + { 4405, 3750, 3201, 5093, 6588, 5943, 3029, 696, 63 }, + { 4529, 3837, 3260, 5152, 6587, 5826, 2883, 638, 56 }, + { 4652, 3922, 3317, 5209, 6582, 5709, 2744, 584, 49 }, + { 4775, 4007, 3373, 5262, 6572, 5590, 2610, 536, 43 }, + { 4899, 4090, 3427, 5312, 6557, 5470, 2483, 492, 38 }, + { 5022, 4173, 3480, 5359, 6538, 5351, 2361, 451, 33 }, + { 5145, 4254, 3531, 5403, 6515, 5231, 2246, 414, 29 }, + { 5268, 4334, 3581, 5443, 6489, 5112, 2135, 380, 26 }, + { 5391, 4414, 3629, 5481, 6458, 4993, 2029, 350, 23 }, + { 5514, 4492, 3676, 5515, 6425, 4875, 1929, 321, 21 }, + { 5637, 4569, 3721, 5548, 6388, 4758, 1833, 296, 18 }, + { 5759, 4645, 3766, 5577, 6349, 4642, 1742, 272, 16 }, + { 5881, 4720, 3808, 5604, 6307, 4528, 1656, 250, 14 }, + { 6004, 4794, 3849, 5628, 6262, 4414, 1573, 231, 13 }, + { 6126, 4867, 3890, 5649, 6215, 4302, 1495, 213, 11 }, + { 6248, 4939, 3928, 5669, 6166, 4192, 1420, 196, 10 }, + { 6370, 5010, 3966, 5686, 6114, 4083, 1349, 181, 9 }, + { 6492, 5080, 4002, 5700, 6061, 3976, 1282, 167, 8 }, + { 6614, 5149, 4037, 5712, 6006, 3871, 1218, 154, 7 }, + { 6735, 5217, 4070, 5723, 5950, 3767, 1157, 142, 7 }, + { 6857, 5284, 4103, 5731, 5891, 3666, 1099, 131, 6 }, + { 6978, 5351, 4134, 5737, 5832, 3566, 1044, 121, 5 }, + { 7099, 5415, 4164, 5741, 5771, 3469, 992, 112, 5 }, + { 7221, 5479, 4192, 5743, 5709, 3373, 943, 104, 4 }, + { 7342, 5542, 4220, 5743, 5646, 3279, 896, 96, 4 }, + { 7462, 5604, 4246, 5742, 5583, 3187, 851, 89, 4 }, + { 7584, 5665, 4272, 5739, 5518, 3097, 808, 82, 3 }, + { 7704, 5725, 4296, 5734, 5453, 3009, 768, 76, 3 }, + { 7825, 5784, 4318, 5727, 5386, 2924, 730, 71, 3 }, + { 7945, 5843, 4341, 5719, 5320, 2840, 693, 65, 2 }, + { 8066, 5900, 4361, 5709, 5252, 2758, 659, 61, 2 }, + { 8186, 5956, 4381, 5698, 5185, 2678, 626, 56, 2 }, + { 8306, 6011, 4400, 5685, 5117, 2600, 595, 52, 2 }, + { 8426, 6066, 4418, 5671, 5049, 2523, 565, 48, 2 }, + { 8547, 6119, 4434, 5655, 4981, 2449, 537, 45, 1 }, + { 8666, 6171, 4450, 5638, 4912, 2377, 511, 42, 1 }, + { 8786, 6223, 4465, 5620, 4843, 2306, 485, 39, 1 }, + { 8906, 6274, 4478, 5600, 4775, 2237, 461, 36, 1 }, + { 9025, 6323, 4491, 5580, 4706, 2170, 438, 34, 1 }, + { 9144, 6372, 4503, 5558, 4637, 2105, 417, 31, 1 }, + { 9264, 6420, 4514, 5535, 4568, 2041, 396, 29, 1 }, + { 9383, 6467, 4524, 5511, 4500, 1979, 376, 27, 1 }, + { 9502, 6513, 4532, 5486, 4432, 1919, 358, 25, 1 }, + { 9621, 6558, 4541, 5460, 4364, 1860, 340, 23, 1 }, + { 9740, 6602, 4548, 5433, 4296, 1803, 323, 22, 1 }, + { 9859, 6645, 4554, 5405, 4229, 1748, 307, 20, 1 }, + { 9978, 6688, 4559, 5376, 4161, 1694, 292, 19, 1 }, + { 10096, 6729, 4564, 5347, 4094, 1641, 278, 18, 1 }, + { 10215, 6770, 4568, 5316, 4028, 1590, 264, 16, 1 }, + { 10333, 6809, 4571, 5285, 3962, 1541, 251, 15, 1 }, + { 10452, 6848, 4573, 5253, 3896, 1492, 239, 14, 1 }, + { 10570, 6886, 4574, 5220, 3831, 1446, 227, 13, 1 }, + { 10688, 6923, 4575, 5186, 3767, 1400, 216, 12, 1 }, + { 10806, 6959, 4575, 5152, 3702, 1356, 205, 12, 1 }, + { 10924, 6994, 4574, 5117, 3639, 1313, 195, 11, 1 }, + { 11041, 7029, 4572, 5082, 3576, 1271, 186, 10, 1 }, + { 11159, 7062, 4570, 5046, 3513, 1231, 177, 9, 1 }, + { 11277, 7095, 4566, 5009, 3451, 1192, 168, 9, 1 }, + { 11394, 7127, 4563, 4972, 3390, 1153, 160, 8, 1 }, + { 11512, 7158, 4558, 4934, 3329, 1116, 152, 8, 1 }, + { 11629, 7188, 4553, 4896, 3269, 1080, 145, 7, 1 }, + { 11746, 7217, 4547, 4857, 3210, 1045, 138, 7, 1 }, + { 11864, 7245, 4540, 4818, 3151, 1012, 131, 6, 1 }, + { 11980, 7273, 4533, 4779, 3093, 979, 124, 6, 1 }, + { 12097, 7300, 4525, 4739, 3035, 947, 118, 6, 1 }, + { 12215, 7326, 4516, 4698, 2978, 916, 113, 5, 1 }, + { 12331, 7351, 4507, 4658, 2922, 886, 107, 5, 1 }, + { 12448, 7375, 4497, 4617, 2866, 857, 102, 5, 1 }, + { 12564, 7398, 4487, 4576, 2812, 829, 97, 4, 1 }, + { 12681, 7421, 4476, 4534, 2757, 802, 92, 4, 1 }, + { 12797, 7443, 4464, 4492, 2704, 775, 88, 4, 1 }, + { 12914, 7464, 4452, 4450, 2651, 749, 84, 3, 1 }, + { 13030, 7484, 4439, 4408, 2599, 725, 79, 3, 1 }, + { 13147, 7503, 4426, 4365, 2547, 700, 76, 3, 1 }, + { 13262, 7522, 4412, 4322, 2497, 677, 72, 3, 1 }, + { 13378, 7539, 4398, 4280, 2447, 654, 68, 3, 1 }, + { 13494, 7556, 4383, 4237, 2397, 632, 65, 3, 1 }, + { 13610, 7573, 4368, 4193, 2348, 611, 62, 2, 1 }, + { 13726, 7588, 4352, 4150, 2300, 590, 59, 2, 1 }, + { 13841, 7602, 4335, 4107, 2253, 571, 56, 2, 1 }, + { 13957, 7616, 4318, 4063, 2207, 551, 53, 2, 1 }, + { 14072, 7629, 4301, 4019, 2161, 532, 51, 2, 1 }, + { 14188, 7641, 4283, 3976, 2115, 514, 48, 2, 1 }, + { 14302, 7652, 4265, 3932, 2071, 497, 46, 2, 1 }, + { 14418, 7663, 4246, 3888, 2027, 480, 44, 1, 1 }, + { 14533, 7673, 4227, 3844, 1984, 463, 42, 1, 1 }, + { 14649, 7682, 4207, 3800, 1941, 447, 40, 1, 1 }, + { 14763, 7690, 4187, 3757, 1899, 432, 38, 1, 1 }, + { 14878, 7698, 4166, 3713, 1858, 417, 36, 1, 1 }, + { 14993, 7705, 4146, 3669, 1817, 402, 34, 1, 1 }, + { 15109, 7711, 4124, 3625, 1777, 388, 32, 1, 1 }, + { 15223, 7715, 4103, 3581, 1738, 375, 31, 1, 1 }, + { 15337, 7720, 4081, 3538, 1699, 362, 29, 1, 1 }, + { 15452, 7724, 4058, 3494, 1661, 349, 28, 1, 1 }, + { 15567, 7727, 4035, 3450, 1624, 337, 26, 1, 1 }, + { 15681, 7729, 4012, 3407, 1587, 325, 25, 1, 1 }, + { 15795, 7730, 3989, 3364, 1551, 313, 24, 1, 1 }, + { 15909, 7731, 3965, 3320, 1516, 302, 23, 1, 1 }, + { 16024, 7731, 3940, 3277, 1481, 291, 22, 1, 1 }, + { 16138, 7730, 3916, 3234, 1446, 281, 21, 1, 1 }, + { 16252, 7728, 3891, 3191, 1413, 271, 20, 1, 1 }, + { 16366, 7726, 3866, 3148, 1380, 261, 19, 1, 1 }, + { 16480, 7723, 3840, 3106, 1347, 252, 18, 1, 1 }, + { 16594, 7720, 3814, 3063, 1315, 243, 17, 1, 1 }, + { 16708, 7715, 3788, 3021, 1284, 234, 16, 1, 1 }, + { 16822, 7710, 3762, 2979, 1253, 225, 15, 1, 1 }, + { 16936, 7704, 3735, 2937, 1223, 217, 14, 1, 1 }, + { 17050, 7697, 3708, 2895, 1193, 209, 14, 1, 1 }, + { 17162, 7690, 3681, 2854, 1164, 202, 13, 1, 1 }, + { 17276, 7682, 3654, 2812, 1136, 194, 12, 1, 1 }, + { 17389, 7673, 3626, 2771, 1108, 187, 12, 1, 1 }, + { 17504, 7663, 3598, 2730, 1080, 180, 11, 1, 1 }, + { 17617, 7653, 3570, 2689, 1053, 173, 11, 1, 1 }, + { 17730, 7642, 3541, 2649, 1027, 167, 10, 1, 1 }, + { 17843, 7630, 3513, 2608, 1001, 161, 10, 1, 1 }, + { 17957, 7618, 3484, 2569, 975, 154, 9, 1, 1 }, + { 18069, 7605, 3455, 2529, 950, 149, 9, 1, 1 }, + { 18183, 7591, 3426, 2489, 926, 143, 8, 1, 1 }, + { 18296, 7576, 3396, 2450, 902, 138, 8, 1, 1 }, + { 18410, 7562, 3366, 2411, 878, 132, 7, 1, 1 }, + { 18523, 7545, 3337, 2372, 855, 127, 7, 1, 1 }, + { 18636, 7529, 3306, 2333, 833, 122, 7, 1, 1 }, + { 18749, 7511, 3276, 2295, 811, 118, 6, 1, 1 }, + { 18862, 7493, 3246, 2257, 789, 113, 6, 1, 1 }, + { 18975, 7474, 3215, 2219, 768, 109, 6, 1, 1 }, + { 19088, 7455, 3185, 2182, 747, 104, 5, 1, 1 }, + { 19201, 7435, 3154, 2144, 727, 100, 5, 1, 1 }, + { 19314, 7414, 3123, 2107, 707, 96, 5, 1, 1 }, + { 19427, 7392, 3092, 2071, 687, 92, 5, 1, 1 }, + { 19541, 7370, 3060, 2034, 668, 89, 4, 1, 1 }, + { 19654, 7347, 3029, 1998, 649, 85, 4, 1, 1 }, + { 19766, 7323, 2997, 1963, 631, 82, 4, 1, 1 }, + { 19878, 7299, 2966, 1927, 613, 79, 4, 1, 1 }, + { 19991, 7274, 2934, 1892, 596, 75, 4, 1, 1 }, + { 20105, 7248, 2902, 1857, 579, 72, 3, 1, 1 }, + { 20218, 7222, 2870, 1822, 562, 69, 3, 1, 1 }, + { 20331, 7195, 2838, 1788, 545, 66, 3, 1, 1 }, + { 20443, 7167, 2806, 1754, 529, 64, 3, 1, 1 }, + { 20556, 7138, 2774, 1720, 514, 61, 3, 1, 1 }, + { 20670, 7109, 2741, 1687, 498, 58, 3, 1, 1 }, + { 20783, 7079, 2709, 1654, 483, 56, 2, 1, 1 }, + { 20895, 7049, 2676, 1621, 469, 54, 2, 1, 1 }, + { 21008, 7017, 2644, 1589, 455, 51, 2, 1, 1 }, + { 21121, 6985, 2611, 1557, 441, 49, 2, 1, 1 }, + { 21234, 6953, 2578, 1525, 427, 47, 2, 1, 1 }, + { 21347, 6919, 2545, 1494, 414, 45, 2, 1, 1 }, + { 21460, 6885, 2513, 1462, 401, 43, 2, 1, 1 }, + { 21573, 6850, 2480, 1432, 388, 41, 2, 1, 1 }, + { 21687, 6815, 2447, 1401, 375, 39, 2, 1, 1 }, + { 21801, 6778, 2414, 1371, 363, 38, 1, 1, 1 }, + { 21914, 6741, 2381, 1341, 352, 36, 1, 1, 1 }, + { 22028, 6704, 2348, 1311, 340, 34, 1, 1, 1 }, + { 22141, 6665, 2315, 1282, 329, 33, 1, 1, 1 }, + { 22255, 6626, 2282, 1253, 318, 31, 1, 1, 1 }, + { 22368, 6586, 2249, 1225, 307, 30, 1, 1, 1 }, + { 22482, 6546, 2216, 1196, 297, 28, 1, 1, 1 }, + { 22595, 6505, 2183, 1169, 286, 27, 1, 1, 1 }, + { 22709, 6463, 2149, 1141, 277, 26, 1, 1, 1 }, + { 22823, 6420, 2116, 1114, 267, 25, 1, 1, 1 }, + { 22938, 6377, 2083, 1087, 257, 23, 1, 1, 1 }, + { 23053, 6332, 2050, 1060, 248, 22, 1, 1, 1 }, + { 23167, 6287, 2017, 1034, 239, 21, 1, 1, 1 }, + { 23280, 6242, 1984, 1008, 231, 20, 1, 1, 1 }, + { 23396, 6195, 1951, 982, 222, 19, 1, 1, 1 }, + { 23510, 6148, 1918, 957, 214, 18, 1, 1, 1 }, + { 23625, 6100, 1885, 932, 206, 17, 1, 1, 1 }, + { 23741, 6051, 1852, 907, 198, 16, 1, 1, 1 }, + { 23855, 6002, 1819, 883, 190, 16, 1, 1, 1 }, + { 23971, 5951, 1786, 859, 183, 15, 1, 1, 1 }, + { 24087, 5900, 1753, 835, 176, 14, 1, 1, 1 }, + { 24203, 5848, 1720, 812, 169, 13, 1, 1, 1 }, + { 24318, 5796, 1687, 789, 162, 13, 1, 1, 1 }, + { 24435, 5742, 1655, 766, 155, 12, 1, 1, 1 }, + { 24552, 5688, 1622, 743, 149, 11, 1, 1, 1 }, + { 24669, 5632, 1589, 721, 143, 11, 1, 1, 1 }, + { 24786, 5576, 1557, 699, 137, 10, 1, 1, 1 }, + { 24903, 5519, 1524, 678, 131, 10, 1, 1, 1 }, + { 25021, 5462, 1491, 657, 125, 9, 1, 1, 1 }, + { 25139, 5403, 1459, 636, 120, 8, 1, 1, 1 }, + { 25258, 5343, 1427, 615, 114, 8, 1, 1, 1 }, + { 25376, 5283, 1394, 595, 109, 8, 1, 1, 1 }, + { 25496, 5221, 1362, 575, 104, 7, 1, 1, 1 }, + { 25614, 5159, 1330, 556, 99, 7, 1, 1, 1 }, + { 25735, 5096, 1298, 536, 94, 6, 1, 1, 1 }, + { 25856, 5031, 1265, 517, 90, 6, 1, 1, 1 }, + { 25977, 4966, 1233, 499, 85, 5, 1, 1, 1 }, + { 26098, 4899, 1202, 480, 81, 5, 1, 1, 1 }, + { 26220, 4831, 1170, 462, 77, 5, 1, 1, 1 }, + { 26343, 4763, 1138, 444, 73, 4, 1, 1, 1 }, + { 26466, 4693, 1106, 427, 69, 4, 1, 1, 1 }, + { 26589, 4622, 1075, 410, 65, 4, 1, 1, 1 }, + { 26713, 4550, 1043, 393, 62, 4, 1, 1, 1 }, + { 26840, 4476, 1012, 376, 58, 3, 1, 1, 1 }, + { 26966, 4401, 980, 360, 55, 3, 1, 1, 1 }, + { 27092, 4325, 949, 344, 52, 3, 1, 1, 1 }, + { 27220, 4248, 918, 328, 48, 3, 1, 1, 1 }, + { 27350, 4169, 886, 313, 45, 2, 1, 1, 1 }, + { 27480, 4088, 855, 298, 42, 2, 1, 1, 1 }, + { 27610, 4006, 824, 283, 40, 2, 1, 1, 1 }, + { 27743, 3922, 793, 268, 37, 2, 1, 1, 1 }, + { 27876, 3837, 762, 254, 34, 2, 1, 1, 1 }, + { 28011, 3749, 731, 240, 32, 2, 1, 1, 1 }, + { 28147, 3659, 701, 227, 30, 1, 1, 1, 1 }, + { 28286, 3568, 670, 213, 27, 1, 1, 1, 1 }, + { 28426, 3474, 639, 200, 25, 1, 1, 1, 1 }, + { 28569, 3377, 608, 187, 23, 1, 1, 1, 1 }, + { 28714, 3278, 577, 174, 21, 1, 1, 1, 1 }, + { 28860, 3176, 547, 162, 19, 1, 1, 1, 1 }, + { 29010, 3071, 516, 150, 17, 1, 1, 1, 1 }, + { 29163, 2962, 485, 138, 16, 1, 1, 1, 1 }, + { 29320, 2849, 454, 127, 14, 1, 1, 1, 1 }, + { 29483, 2731, 423, 115, 12, 1, 1, 1, 1 }, + { 29650, 2608, 391, 104, 11, 1, 1, 1, 1 }, + { 29823, 2479, 360, 93, 9, 1, 1, 1, 1 }, + { 30002, 2343, 328, 83, 8, 1, 1, 1, 1 }, + { 30192, 2198, 295, 72, 7, 1, 1, 1, 1 }, + { 30393, 2041, 262, 62, 6, 1, 1, 1, 1 }, + { 30612, 1869, 227, 52, 4, 1, 1, 1, 1 }, + { 30853, 1676, 191, 41, 3, 1, 1, 1, 1 }, + { 31131, 1448, 152, 31, 2, 1, 1, 1, 1 }, + { 31486, 1150, 107, 20, 1, 1, 1, 1, 1 }, +}; #endif // CONFIG_EC_MULTISYMBOL /* clang-format off */ @@ -3920,11 +3924,24 @@ void av1_model_to_full_probs(const aom_prob *model, aom_prob *full) { #if CONFIG_EC_MULTISYMBOL static void build_token_cdfs(const aom_prob *pdf_model, - aom_cdf_prob cdf[ENTROPY_TOKENS]) { - int i, sum = 0; + aom_cdf_prob cdf_tail[ENTROPY_TOKENS], + aom_cdf_prob cdf_head[ENTROPY_TOKENS]) { + int i, p, scale, sum = 0; + assert(pdf_model[2] != 0); - for (i = 0; i < ENTROPY_TOKENS - 2; ++i) { - cdf[i] = sum += av1_pareto8_token_probs[pdf_model[2] - 1][i]; + + // Do the head (ZERO, ONE, TWO or more) + cdf_head[ZERO_TOKEN] = sum = (pdf_model[1] << 7); + scale = (1 << 15) - cdf_head[ZERO_TOKEN]; + p = AOMMAX(1, (scale * 128 * pdf_model[2] + (1 << 14)) >> 15); + cdf_head[ONE_TOKEN] = cdf_head[ZERO_TOKEN] + p; + cdf_head[TWO_TOKEN] = (1 << 15); + + // Do the tail + sum = 0; + for (i = 0; i < ENTROPY_TOKENS - 3; ++i) { + p = av1_pareto8_tail_probs[pdf_model[2] - 1][i]; + cdf_tail[i] = sum += p; } } @@ -3937,7 +3954,8 @@ void av1_coef_pareto_cdfs(FRAME_CONTEXT *fc) { for (k = 0; k < COEF_BANDS; ++k) for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) build_token_cdfs(fc->coef_probs[t][i][j][k][l], - fc->coef_cdfs[t][i][j][k][l]); + fc->coef_tail_cdfs[t][i][j][k][l], + fc->coef_head_cdfs[t][i][j][k][l]); } #endif // CONFIG_EC_MULTISYMBOL diff --git a/av1/common/entropy.h b/av1/common/entropy.h index 08415c71a..5cc7b1bdd 100644 --- a/av1/common/entropy.h +++ b/av1/common/entropy.h @@ -183,6 +183,7 @@ static INLINE const uint8_t *get_band_translate(TX_SIZE tx_size) { #define PIVOT_NODE 2 // which node is pivot #define MODEL_NODES (ENTROPY_NODES - UNCONSTRAINED_NODES) +#define TAIL_NODES (MODEL_NODES + 1) extern const aom_tree_index av1_coef_con_tree[TREE_SIZE(ENTROPY_TOKENS)]; extern const aom_prob av1_pareto8_full[COEFF_PROB_MODELS][MODEL_NODES]; @@ -200,6 +201,8 @@ typedef aom_cdf_prob coeff_cdf_model[REF_TYPES][COEF_BANDS][COEFF_CONTEXTS] [ENTROPY_TOKENS]; extern const aom_cdf_prob av1_pareto8_token_probs[COEFF_PROB_MODELS] [ENTROPY_TOKENS - 2]; +extern const aom_cdf_prob av1_pareto8_tail_cdfs[COEFF_PROB_MODELS] + [ENTROPY_TOKENS - 3]; struct frame_contexts; void av1_coef_pareto_cdfs(struct frame_contexts *fc); #endif // CONFIG_EC_MULTISYMBOL diff --git a/av1/common/entropymode.h b/av1/common/entropymode.h index ca8425657..7a5e0e3a9 100644 --- a/av1/common/entropymode.h +++ b/av1/common/entropymode.h @@ -85,7 +85,8 @@ typedef struct frame_contexts { #endif av1_coeff_probs_model coef_probs[TX_SIZES][PLANE_TYPES]; #if CONFIG_EC_MULTISYMBOL - coeff_cdf_model coef_cdfs[TX_SIZES][PLANE_TYPES]; + coeff_cdf_model coef_tail_cdfs[TX_SIZES][PLANE_TYPES]; + coeff_cdf_model coef_head_cdfs[TX_SIZES][PLANE_TYPES]; #endif // CONFIG_EC_MULTISYMBOL aom_prob switchable_interp_prob[SWITCHABLE_FILTER_CONTEXTS] [SWITCHABLE_FILTERS - 1]; diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c index 3b1d9cc22..7bf419bdf 100644 --- a/av1/decoder/decodeframe.c +++ b/av1/decoder/decodeframe.c @@ -2395,7 +2395,7 @@ static void read_coef_probs_common(av1_coeff_probs_model *coef_probs, aom_reader *r) { int i, j, k, l, m; #if CONFIG_EC_ADAPT - const int node_limit = UNCONSTRAINED_NODES - 1; + const int node_limit = UNCONSTRAINED_NODES - 2; #else const int node_limit = UNCONSTRAINED_NODES; #endif diff --git a/av1/decoder/detokenize.c b/av1/decoder/detokenize.c index 0003c866a..4e8c36997 100644 --- a/av1/decoder/detokenize.c +++ b/av1/decoder/detokenize.c @@ -81,14 +81,20 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff, fc->coef_probs[tx_size_ctx][type][ref]; const aom_prob *prob; #if CONFIG_EC_ADAPT - aom_cdf_prob(*coef_cdfs)[COEFF_CONTEXTS][ENTROPY_TOKENS] = - xd->tile_ctx->coef_cdfs[tx_size][type][ref]; - aom_cdf_prob(*cdf)[ENTROPY_TOKENS]; + aom_cdf_prob(*coef_head_cdfs)[COEFF_CONTEXTS][ENTROPY_TOKENS] = + xd->tile_ctx->coef_head_cdfs[tx_size][type][ref]; + aom_cdf_prob(*coef_tail_cdfs)[COEFF_CONTEXTS][ENTROPY_TOKENS] = + xd->tile_ctx->coef_tail_cdfs[tx_size][type][ref]; #elif CONFIG_EC_MULTISYMBOL - aom_cdf_prob(*coef_cdfs)[COEFF_CONTEXTS][ENTROPY_TOKENS] = - fc->coef_cdfs[tx_size_ctx][type][ref]; - aom_cdf_prob(*cdf)[ENTROPY_TOKENS]; -#endif // CONFIG_EC_MULTISYMBOL + aom_cdf_prob(*coef_head_cdfs)[COEFF_CONTEXTS][ENTROPY_TOKENS] = + fc->coef_head_cdfs[tx_size_ctx][type][ref]; + aom_cdf_prob(*coef_tail_cdfs)[COEFF_CONTEXTS][ENTROPY_TOKENS] = + fc->coef_tail_cdfs[tx_size_ctx][type][ref]; +#endif +#if CONFIG_EC_MULTISYMBOL + aom_cdf_prob(*cdf_head)[ENTROPY_TOKENS]; + aom_cdf_prob(*cdf_tail)[ENTROPY_TOKENS]; +#endif unsigned int(*coef_counts)[COEFF_CONTEXTS][UNCONSTRAINED_NODES + 1] = NULL; unsigned int(*eob_branch_count)[COEFF_CONTEXTS]; uint8_t token_cache[MAX_TX_SQUARE]; @@ -165,7 +171,13 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff, dqv_val = &dq_val[band][0]; #endif // CONFIG_NEW_QUANT - while (!aom_read(r, prob[ZERO_CONTEXT_NODE], ACCT_STR)) { +#if CONFIG_EC_MULTISYMBOL + cdf_head = &coef_head_cdfs[band][ctx]; + cdf_tail = &coef_tail_cdfs[band][ctx]; + token = aom_read_symbol(r, *cdf_head, 3, ACCT_STR); + if (token > ONE_TOKEN) + token += aom_read_symbol(r, *cdf_tail, CATEGORY6_TOKEN + 1 - 2, ACCT_STR); + while (!token) { INCREMENT_COUNT(ZERO_TOKEN); dqv = dq[1]; token_cache[scan[c]] = 0; @@ -173,7 +185,12 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff, if (c >= max_eob) return c; // zero tokens at the end (no eob token) ctx = get_coef_context(nb, token_cache, c); band = *band_translate++; - prob = coef_probs[band][ctx]; + cdf_head = &coef_head_cdfs[band][ctx]; + cdf_tail = &coef_tail_cdfs[band][ctx]; + token = aom_read_symbol(r, *cdf_head, 3, ACCT_STR); + if (token > ONE_TOKEN) + token += + aom_read_symbol(r, *cdf_tail, CATEGORY6_TOKEN + 1 - 2, ACCT_STR); #if CONFIG_NEW_QUANT dqv_val = &dq_val[band][0]; #endif // CONFIG_NEW_QUANT @@ -181,11 +198,8 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff, *max_scan_line = AOMMAX(*max_scan_line, scan[c]); -#if CONFIG_EC_MULTISYMBOL - cdf = &coef_cdfs[band][ctx]; - token = ONE_TOKEN + - aom_read_symbol(r, *cdf, CATEGORY6_TOKEN - ONE_TOKEN + 1, ACCT_STR); INCREMENT_COUNT(ONE_TOKEN + (token > ONE_TOKEN)); + switch (token) { case ONE_TOKEN: case TWO_TOKEN: @@ -228,6 +242,23 @@ static int decode_coefs(MACROBLOCKD *xd, PLANE_TYPE type, tran_low_t *dqcoeff, } break; } #else // CONFIG_EC_MULTISYMBOL + + while (!aom_read(r, prob[ZERO_CONTEXT_NODE], ACCT_STR)) { + INCREMENT_COUNT(ZERO_TOKEN); + dqv = dq[1]; + token_cache[scan[c]] = 0; + ++c; + if (c >= max_eob) return c; // zero tokens at the end (no eob token) + ctx = get_coef_context(nb, token_cache, c); + band = *band_translate++; + prob = coef_probs[band][ctx]; +#if CONFIG_NEW_QUANT + dqv_val = &dq_val[band][0]; +#endif // CONFIG_NEW_QUANT + } + + *max_scan_line = AOMMAX(*max_scan_line, scan[c]); + if (!aom_read(r, prob[ONE_CONTEXT_NODE], ACCT_STR)) { INCREMENT_COUNT(ONE_TOKEN); token = ONE_TOKEN; diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index b4545fb87..5f7c42b77 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c @@ -797,11 +797,10 @@ static void pack_mb_tokens(aom_writer *w, const TOKENEXTRA **tp, aom_write_record(w, token != EOB_TOKEN, p->context_tree[0], token_stats); if (token != EOB_TOKEN) { - aom_write_record(w, token != ZERO_TOKEN, p->context_tree[1], token_stats); - - if (token != ZERO_TOKEN) { - aom_write_symbol(w, token - ONE_TOKEN, *p->token_cdf, - CATEGORY6_TOKEN - ONE_TOKEN + 1); + aom_write_symbol(w, AOMMIN(token, TWO_TOKEN), *p->head_cdf, 3); + if (token > ONE_TOKEN) { + aom_write_symbol(w, token - TWO_TOKEN, *p->tail_cdf, + CATEGORY6_TOKEN + 1 - 2); } } #else @@ -2740,7 +2739,7 @@ static void update_coef_probs_common(aom_writer *const bc, AV1_COMP *cpi, av1_coeff_probs_model *old_coef_probs = cpi->common.fc->coef_probs[tx_size]; const aom_prob upd = DIFF_UPDATE_PROB; #if CONFIG_EC_ADAPT - const int entropy_nodes_update = UNCONSTRAINED_NODES - 1; + const int entropy_nodes_update = UNCONSTRAINED_NODES - 2; #else const int entropy_nodes_update = UNCONSTRAINED_NODES; #endif diff --git a/av1/encoder/tokenize.c b/av1/encoder/tokenize.c index 9823f843b..8ed1d5f08 100644 --- a/av1/encoder/tokenize.c +++ b/av1/encoder/tokenize.c @@ -362,7 +362,8 @@ static void set_entropy_context_b(int plane, int block, int blk_row, static INLINE void add_token(TOKENEXTRA **t, const aom_prob *context_tree, #if CONFIG_EC_MULTISYMBOL - aom_cdf_prob (*token_cdf)[ENTROPY_TOKENS], + aom_cdf_prob (*tail_cdf)[ENTROPY_TOKENS], + aom_cdf_prob (*head_cdf)[ENTROPY_TOKENS], #endif // CONFIG_EC_MULTISYMBOL int32_t extra, uint8_t token, uint8_t skip_eob_node, unsigned int *counts) { @@ -370,7 +371,8 @@ static INLINE void add_token(TOKENEXTRA **t, const aom_prob *context_tree, (*t)->extra = extra; (*t)->context_tree = context_tree; #if CONFIG_EC_MULTISYMBOL - (*t)->token_cdf = token_cdf; + (*t)->tail_cdf = tail_cdf; + (*t)->head_cdf = head_cdf; #endif // CONFIG_EC_MULTISYMBOL (*t)->skip_eob_node = skip_eob_node; (*t)++; @@ -467,11 +469,15 @@ static void tokenize_b(int plane, int block, int blk_row, int blk_col, #endif // CONFIG_ENTROPY #if CONFIG_EC_ADAPT // use per-tile context - aom_cdf_prob(*const coef_cdfs)[COEFF_CONTEXTS][ENTROPY_TOKENS] = - xd->tile_ctx->coef_cdfs[tx_size][type][ref]; + aom_cdf_prob(*const coef_head_cdfs)[COEFF_CONTEXTS][ENTROPY_TOKENS] = + xd->tile_ctx->coef_head_cdfs[tx_size][type][ref]; + aom_cdf_prob(*const coef_tail_cdfs)[COEFF_CONTEXTS][ENTROPY_TOKENS] = + xd->tile_ctx->coef_tail_cdfs[tx_size][type][ref]; #elif CONFIG_EC_MULTISYMBOL - aom_cdf_prob(*const coef_cdfs)[COEFF_CONTEXTS][ENTROPY_TOKENS] = - cpi->common.fc->coef_cdfs[tx_size][type][ref]; + aom_cdf_prob(*const coef_head_cdfs)[COEFF_CONTEXTS][ENTROPY_TOKENS] = + cpi->common.fc->coef_head_cdfs[tx_size][type][ref]; + aom_cdf_prob(*const coef_tail_cdfs)[COEFF_CONTEXTS][ENTROPY_TOKENS] = + cpi->common.fc->coef_tail_cdfs[tx_size][type][ref]; #endif unsigned int(*const eob_branch)[COEFF_CONTEXTS] = td->counts->eob_branch[txsize_sqr_map[tx_size]][type][ref]; @@ -495,7 +501,7 @@ static void tokenize_b(int plane, int block, int blk_row, int blk_col, add_token(&t, coef_probs[band[c]][pt], #if CONFIG_EC_MULTISYMBOL - &coef_cdfs[band[c]][pt], + &coef_tail_cdfs[band[c]][pt], &coef_head_cdfs[band[c]][pt], #endif extra, (uint8_t)token, (uint8_t)skip_eob, counts[band[c]][pt]); @@ -508,7 +514,7 @@ static void tokenize_b(int plane, int block, int blk_row, int blk_col, assert(!skip_eob); // The last token must be non-zero. add_token(&t, coef_probs[band[c]][pt], #if CONFIG_EC_MULTISYMBOL - NULL, + NULL, NULL, #endif 0, EOB_TOKEN, 0, counts[band[c]][pt]); ++eob_branch[band[c]][pt]; diff --git a/av1/encoder/tokenize.h b/av1/encoder/tokenize.h index e44bcc77e..a2b5f9156 100644 --- a/av1/encoder/tokenize.h +++ b/av1/encoder/tokenize.h @@ -37,7 +37,8 @@ typedef struct { typedef struct { const aom_prob *context_tree; #if CONFIG_EC_MULTISYMBOL - aom_cdf_prob (*token_cdf)[ENTROPY_TOKENS]; + aom_cdf_prob (*tail_cdf)[ENTROPY_TOKENS]; + aom_cdf_prob (*head_cdf)[ENTROPY_TOKENS]; #endif EXTRABIT extra; uint8_t token; diff --git a/tools/gen_constrained_tokenset.py b/tools/gen_constrained_tokenset.py index a0f8280b6..d3cbf8d9d 100755 --- a/tools/gen_constrained_tokenset.py +++ b/tools/gen_constrained_tokenset.py @@ -96,11 +96,13 @@ def quantize_probs(p, save_first_bin, bits): def get_quantized_spareto(p, beta, bits): parray = get_spareto(p, beta) parray = parray[1:] / (1 - parray[0]) - qarray = quantize_probs(parray, True, bits) +#if CONFIG_EC_MULTISYMBOL, truncate the array again + tarray = parray[1:] / (1 - parray[0]) + qarray = quantize_probs(tarray, False, bits) return qarray.astype(np.int) -def main(bits=8): +def main(bits=15): beta = 8 for q in range(1, 256): parray = get_quantized_spareto(q / 256., beta, bits) -- GitLab